diff --git a/Documentation/PCI/endpoint/pci-vntb-howto.rst b/Documentation/PCI/endpoint/pci-vntb-howto.rst index 9a7a2f0a68498..3679f5c302549 100644 --- a/Documentation/PCI/endpoint/pci-vntb-howto.rst +++ b/Documentation/PCI/endpoint/pci-vntb-howto.rst @@ -52,14 +52,14 @@ pci-epf-vntb device, the following commands can be used:: # cd /sys/kernel/config/pci_ep/ # mkdir functions/pci_epf_vntb/func1 -The "mkdir func1" above creates the pci-epf-ntb function device that will +The "mkdir func1" above creates the pci-epf-vntb function device that will be probed by pci_epf_vntb driver. The PCI endpoint framework populates the directory with the following configurable fields:: - # ls functions/pci_epf_ntb/func1 - baseclass_code deviceid msi_interrupts pci-epf-ntb.0 + # ls functions/pci_epf_vntb/func1 + baseclass_code deviceid msi_interrupts pci-epf-vntb.0 progif_code secondary subsys_id vendorid cache_line_size interrupt_pin msix_interrupts primary revid subclass_code subsys_vendor_id @@ -111,13 +111,13 @@ A sample configuration for virtual NTB driver for virtual PCI bus:: # echo 0x080A > functions/pci_epf_vntb/func1/pci_epf_vntb.0/vntb_pid # echo 0x10 > functions/pci_epf_vntb/func1/pci_epf_vntb.0/vbus_number -Binding pci-epf-ntb Device to EP Controller +Binding pci-epf-vntb Device to EP Controller -------------------------------------------- NTB function device should be attached to PCI endpoint controllers connected to the host. - # ln -s controllers/5f010000.pcie_ep functions/pci-epf-ntb/func1/primary + # ln -s controllers/5f010000.pcie_ep functions/pci_epf_vntb/func1/primary Once the above step is completed, the PCI endpoint controllers are ready to establish a link with the host. @@ -139,7 +139,7 @@ lspci Output at Host side ------------------------- Note that the devices listed here correspond to the values populated in -"Creating pci-epf-ntb Device" section above:: +"Creating pci-epf-vntb Device" section above:: # lspci 00:00.0 PCI bridge: Freescale Semiconductor Inc Device 0000 (rev 01) @@ -152,7 +152,7 @@ lspci Output at EP Side / Virtual PCI bus ----------------------------------------- Note that the devices listed here correspond to the values populated in -"Creating pci-epf-ntb Device" section above:: +"Creating pci-epf-vntb Device" section above:: # lspci 10:00.0 Unassigned class [ffff]: Dawicontrol Computersysteme GmbH Device 1234 (rev ff) diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 0e6c67ac585a0..edcd125d6992f 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -2538,10 +2538,10 @@ Cpuset Interface Files Users can manually set it to a value that is different from "cpuset.cpus". One constraint in setting it is that the list of CPUs must be exclusive with respect to "cpuset.cpus.exclusive" - of its sibling. If "cpuset.cpus.exclusive" of a sibling cgroup - isn't set, its "cpuset.cpus" value, if set, cannot be a subset - of it to leave at least one CPU available when the exclusive - CPUs are taken away. + and "cpuset.cpus.exclusive.effective" of its siblings. Another + constraint is that it cannot be a superset of "cpuset.cpus" + of its sibling in order to leave at least one CPU available to + that sibling when the exclusive CPUs are taken away. For a parent cgroup, any one of its exclusive CPUs can only be distributed to at most one of its child cgroups. Having an diff --git a/Documentation/devicetree/bindings/media/qcom,qcs8300-camss.yaml b/Documentation/devicetree/bindings/media/qcom,qcs8300-camss.yaml index dce0a1fcb10c6..e5f170aa4d9ee 100644 --- a/Documentation/devicetree/bindings/media/qcom,qcs8300-camss.yaml +++ b/Documentation/devicetree/bindings/media/qcom,qcs8300-camss.yaml @@ -122,11 +122,11 @@ properties: vdda-phy-supply: description: - Phandle to a regulator supply to PHY core block. + Phandle to a 0.88V regulator supply to CSI PHYs. vdda-pll-supply: description: - Phandle to 1.8V regulator supply to PHY refclk pll block. + Phandle to 1.2V regulator supply to CSI PHYs pll block. ports: $ref: /schemas/graph.yaml#/properties/ports diff --git a/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml index ba757b08b9b1b..bfc4d75f50ff9 100644 --- a/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml @@ -72,7 +72,6 @@ allOf: properties: compatible: enum: - - qcom,sa8775p-edp-phy - qcom,x1e80100-dp-phy then: properties: diff --git a/Documentation/devicetree/bindings/sound/asahi-kasei,ak4458.yaml b/Documentation/devicetree/bindings/sound/asahi-kasei,ak4458.yaml index 1fdbeecc5eff9..3a3313ea0890a 100644 --- a/Documentation/devicetree/bindings/sound/asahi-kasei,ak4458.yaml +++ b/Documentation/devicetree/bindings/sound/asahi-kasei,ak4458.yaml @@ -21,10 +21,10 @@ properties: reg: maxItems: 1 - avdd-supply: + AVDD-supply: description: Analog power supply - dvdd-supply: + DVDD-supply: description: Digital power supply reset-gpios: @@ -60,7 +60,7 @@ allOf: properties: dsd-path: false -additionalProperties: false +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/sound/asahi-kasei,ak5558.yaml b/Documentation/devicetree/bindings/sound/asahi-kasei,ak5558.yaml index d3d494ae8abfe..dc8f85f266bf3 100644 --- a/Documentation/devicetree/bindings/sound/asahi-kasei,ak5558.yaml +++ b/Documentation/devicetree/bindings/sound/asahi-kasei,ak5558.yaml @@ -19,10 +19,10 @@ properties: reg: maxItems: 1 - avdd-supply: + AVDD-supply: description: A 1.8V supply that powers up the AVDD pin. - dvdd-supply: + DVDD-supply: description: A 1.2V supply that powers up the DVDD pin. reset-gpios: diff --git a/Documentation/driver-api/dpll.rst b/Documentation/driver-api/dpll.rst index be1fc643b645e..83118c728ed90 100644 --- a/Documentation/driver-api/dpll.rst +++ b/Documentation/driver-api/dpll.rst @@ -198,26 +198,28 @@ be requested with the same attribute with ``DPLL_CMD_DEVICE_SET`` command. ================================== ====================================== Device may also provide ability to adjust a signal phase on a pin. -If pin phase adjustment is supported, minimal and maximal values that pin -handle shall be provide to the user on ``DPLL_CMD_PIN_GET`` respond -with ``DPLL_A_PIN_PHASE_ADJUST_MIN`` and ``DPLL_A_PIN_PHASE_ADJUST_MAX`` +If pin phase adjustment is supported, minimal and maximal values and +granularity that pin handle shall be provided to the user on +``DPLL_CMD_PIN_GET`` respond with ``DPLL_A_PIN_PHASE_ADJUST_MIN``, +``DPLL_A_PIN_PHASE_ADJUST_MAX`` and ``DPLL_A_PIN_PHASE_ADJUST_GRAN`` attributes. Configured phase adjust value is provided with ``DPLL_A_PIN_PHASE_ADJUST`` attribute of a pin, and value change can be requested with the same attribute with ``DPLL_CMD_PIN_SET`` command. - =============================== ====================================== - ``DPLL_A_PIN_ID`` configured pin id - ``DPLL_A_PIN_PHASE_ADJUST_MIN`` attr minimum value of phase adjustment - ``DPLL_A_PIN_PHASE_ADJUST_MAX`` attr maximum value of phase adjustment - ``DPLL_A_PIN_PHASE_ADJUST`` attr configured value of phase - adjustment on parent dpll device - ``DPLL_A_PIN_PARENT_DEVICE`` nested attribute for requesting - configuration on given parent dpll - device - ``DPLL_A_PIN_PARENT_ID`` parent dpll device id - ``DPLL_A_PIN_PHASE_OFFSET`` attr measured phase difference - between a pin and parent dpll device - =============================== ====================================== + ================================ ========================================== + ``DPLL_A_PIN_ID`` configured pin id + ``DPLL_A_PIN_PHASE_ADJUST_GRAN`` attr granularity of phase adjustment value + ``DPLL_A_PIN_PHASE_ADJUST_MIN`` attr minimum value of phase adjustment + ``DPLL_A_PIN_PHASE_ADJUST_MAX`` attr maximum value of phase adjustment + ``DPLL_A_PIN_PHASE_ADJUST`` attr configured value of phase + adjustment on parent dpll device + ``DPLL_A_PIN_PARENT_DEVICE`` nested attribute for requesting + configuration on given parent dpll + device + ``DPLL_A_PIN_PARENT_ID`` parent dpll device id + ``DPLL_A_PIN_PHASE_OFFSET`` attr measured phase difference + between a pin and parent dpll device + ================================ ========================================== All phase related values are provided in pico seconds, which represents time difference between signals phase. The negative value means that @@ -384,6 +386,8 @@ according to attribute purpose. frequencies ``DPLL_A_PIN_ANY_FREQUENCY_MIN`` attr minimum value of frequency ``DPLL_A_PIN_ANY_FREQUENCY_MAX`` attr maximum value of frequency + ``DPLL_A_PIN_PHASE_ADJUST_GRAN`` attr granularity of phase + adjustment value ``DPLL_A_PIN_PHASE_ADJUST_MIN`` attr minimum value of phase adjustment ``DPLL_A_PIN_PHASE_ADJUST_MAX`` attr maximum value of phase diff --git a/Documentation/hwmon/aht10.rst b/Documentation/hwmon/aht10.rst index 213644b4ecba6..7903b6434326d 100644 --- a/Documentation/hwmon/aht10.rst +++ b/Documentation/hwmon/aht10.rst @@ -20,6 +20,14 @@ Supported chips: English: http://www.aosong.com/userfiles/files/media/Data%20Sheet%20AHT20.pdf + * Aosong DHT20 + + Prefix: 'dht20' + + Addresses scanned: None + + Datasheet: https://www.digikey.co.nz/en/htmldatasheets/production/9184855/0/0/1/101020932 + Author: Johannes Cornelis Draaijer @@ -33,7 +41,7 @@ The address of this i2c device may only be 0x38 Special Features ---------------- -AHT20 has additional CRC8 support which is sent as the last byte of the sensor +AHT20, DHT20 has additional CRC8 support which is sent as the last byte of the sensor values. Usage Notes diff --git a/Documentation/hwmon/nct6683.rst b/Documentation/hwmon/nct6683.rst index 3e549ba95a15a..45eec9dd349aa 100644 --- a/Documentation/hwmon/nct6683.rst +++ b/Documentation/hwmon/nct6683.rst @@ -65,6 +65,7 @@ AMD BC-250 NCT6686D EC firmware version 1.0 build 07/28/21 ASRock X570 NCT6683D EC firmware version 1.0 build 06/28/19 ASRock X670E NCT6686D EC firmware version 1.0 build 05/19/22 ASRock B650 Steel Legend WiFi NCT6686D EC firmware version 1.0 build 11/09/23 +ASRock Z590 Taichi NCT6686D EC firmware version 1.0 build 01/25/21 MSI B550 NCT6687D EC firmware version 1.0 build 05/07/20 MSI X670-P NCT6687D EC firmware version 0.0 build 09/27/22 MSI X870E NCT6687D EC firmware version 0.0 build 11/13/24 diff --git a/Documentation/netlink/specs/dpll.yaml b/Documentation/netlink/specs/dpll.yaml index 80728f6f9bc87..78d0724d7e12c 100644 --- a/Documentation/netlink/specs/dpll.yaml +++ b/Documentation/netlink/specs/dpll.yaml @@ -440,6 +440,12 @@ attribute-sets: doc: | Capable pin provides list of pins that can be bound to create a reference-sync pin pair. + - + name: phase-adjust-gran + type: u32 + doc: | + Granularity of phase adjustment, in picoseconds. The value of + phase adjustment must be a multiple of this granularity. - name: pin-parent-device @@ -616,6 +622,7 @@ operations: - capabilities - parent-device - parent-pin + - phase-adjust-gran - phase-adjust-min - phase-adjust-max - phase-adjust diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst index a06cb99d66dcd..7a637e87005fe 100644 --- a/Documentation/networking/ip-sysctl.rst +++ b/Documentation/networking/ip-sysctl.rst @@ -3195,12 +3195,13 @@ enhanced_dad - BOOLEAN =========== ratelimit - INTEGER - Limit the maximal rates for sending ICMPv6 messages. + Limit the maximal rates for sending ICMPv6 messages to a particular + peer. 0 to disable any limiting, - otherwise the minimal space between responses in milliseconds. + otherwise the space between responses in milliseconds. - Default: 1000 + Default: 100 ratemask - list of comma separated ranges For ICMPv6 message types matching the ranges in the ratemask, limit diff --git a/Documentation/networking/net_cachelines/inet_sock.rst b/Documentation/networking/net_cachelines/inet_sock.rst index b11bf48fa2b36..4c72a28a7012e 100644 --- a/Documentation/networking/net_cachelines/inet_sock.rst +++ b/Documentation/networking/net_cachelines/inet_sock.rst @@ -5,42 +5,43 @@ inet_sock struct fast path usage breakdown ========================================== -======================= ===================== =================== =================== ====================================================================================================== -Type Name fastpath_tx_access fastpath_rx_access comment -======================= ===================== =================== =================== ====================================================================================================== -struct sock sk read_mostly read_mostly tcp_init_buffer_space,tcp_init_transfer,tcp_finish_connect,tcp_connect,tcp_send_rcvq,tcp_send_syn_data -struct ipv6_pinfo* pinet6 -be16 inet_sport read_mostly __tcp_transmit_skb -be32 inet_daddr read_mostly ip_select_ident_segs -be32 inet_rcv_saddr -be16 inet_dport read_mostly __tcp_transmit_skb -u16 inet_num -be32 inet_saddr -s16 uc_ttl read_mostly __ip_queue_xmit/ip_select_ttl -u16 cmsg_flags -struct ip_options_rcu* inet_opt read_mostly __ip_queue_xmit -u16 inet_id read_mostly ip_select_ident_segs -u8 tos read_mostly ip_queue_xmit -u8 min_ttl -u8 mc_ttl -u8 pmtudisc -u8:1 recverr -u8:1 is_icsk -u8:1 freebind -u8:1 hdrincl -u8:1 mc_loop -u8:1 transparent -u8:1 mc_all -u8:1 nodefrag -u8:1 bind_address_no_port -u8:1 recverr_rfc4884 -u8:1 defer_connect read_mostly tcp_sendmsg_fastopen -u8 rcv_tos -u8 convert_csum -int uc_index -int mc_index -be32 mc_addr -struct ip_mc_socklist* mc_list -struct inet_cork_full cork read_mostly __tcp_transmit_skb -struct local_port_range -======================= ===================== =================== =================== ====================================================================================================== +======================== ===================== =================== =================== ====================================================================================================== +Type Name fastpath_tx_access fastpath_rx_access comment +======================== ===================== =================== =================== ====================================================================================================== +struct sock sk read_mostly read_mostly tcp_init_buffer_space,tcp_init_transfer,tcp_finish_connect,tcp_connect,tcp_send_rcvq,tcp_send_syn_data +struct ipv6_pinfo* pinet6 +struct ipv6_fl_socklist* ipv6_fl_list read_mostly tcp_v6_connect,__ip6_datagram_connect,udpv6_sendmsg,rawv6_sendmsg +be16 inet_sport read_mostly __tcp_transmit_skb +be32 inet_daddr read_mostly ip_select_ident_segs +be32 inet_rcv_saddr +be16 inet_dport read_mostly __tcp_transmit_skb +u16 inet_num +be32 inet_saddr +s16 uc_ttl read_mostly __ip_queue_xmit/ip_select_ttl +u16 cmsg_flags +struct ip_options_rcu* inet_opt read_mostly __ip_queue_xmit +u16 inet_id read_mostly ip_select_ident_segs +u8 tos read_mostly ip_queue_xmit +u8 min_ttl +u8 mc_ttl +u8 pmtudisc +u8:1 recverr +u8:1 is_icsk +u8:1 freebind +u8:1 hdrincl +u8:1 mc_loop +u8:1 transparent +u8:1 mc_all +u8:1 nodefrag +u8:1 bind_address_no_port +u8:1 recverr_rfc4884 +u8:1 defer_connect read_mostly tcp_sendmsg_fastopen +u8 rcv_tos +u8 convert_csum +int uc_index +int mc_index +be32 mc_addr +struct ip_mc_socklist* mc_list +struct inet_cork_full cork read_mostly __tcp_transmit_skb +struct local_port_range +======================== ===================== =================== =================== ====================================================================================================== diff --git a/Documentation/sound/alsa-configuration.rst b/Documentation/sound/alsa-configuration.rst index 0a4eaa7d66ddd..55b845d382368 100644 --- a/Documentation/sound/alsa-configuration.rst +++ b/Documentation/sound/alsa-configuration.rst @@ -2372,6 +2372,10 @@ quirk_flags audible volume * bit 25: ``mixer_capture_min_mute`` Similar to bit 24 but for capture streams + * bit 26: ``skip_iface_setup`` + Skip the probe-time interface setup (usb_set_interface, + init_pitch, init_sample_rate); redundant with + snd_usb_endpoint_prepare() at stream-open time This module supports multiple devices, autoprobe and hotplugging. diff --git a/Documentation/trace/events-pci.rst b/Documentation/trace/events-pci.rst new file mode 100644 index 0000000000000..03ff4ad30ddfa --- /dev/null +++ b/Documentation/trace/events-pci.rst @@ -0,0 +1,74 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=========================== +Subsystem Trace Points: PCI +=========================== + +Overview +======== +The PCI tracing system provides tracepoints to monitor critical hardware events +that can impact system performance and reliability. These events normally show +up here: + + /sys/kernel/tracing/events/pci + +Cf. include/trace/events/pci.h for the events definitions. + +Available Tracepoints +===================== + +pci_hp_event +------------ + +Monitors PCI hotplug events including card insertion/removal and link +state changes. +:: + + pci_hp_event "%s slot:%s, event:%s\n" + +**Event Types**: + +* ``LINK_UP`` - PCIe link established +* ``LINK_DOWN`` - PCIe link lost +* ``CARD_PRESENT`` - Card detected in slot +* ``CARD_NOT_PRESENT`` - Card removed from slot + +**Example Usage**:: + + # Enable the tracepoint + echo 1 > /sys/kernel/debug/tracing/events/pci/pci_hp_event/enable + + # Monitor events (the following output is generated when a device is hotplugged) + cat /sys/kernel/debug/tracing/trace_pipe + irq/51-pciehp-88 [001] ..... 1311.177459: pci_hp_event: 0000:00:02.0 slot:10, event:CARD_PRESENT + + irq/51-pciehp-88 [001] ..... 1311.177566: pci_hp_event: 0000:00:02.0 slot:10, event:LINK_UP + +pcie_link_event +--------------- + +Monitors PCIe link speed changes and provides detailed link status information. +:: + + pcie_link_event "%s type:%d, reason:%d, cur_bus_speed:%d, max_bus_speed:%d, width:%u, flit_mode:%u, status:%s\n" + +**Parameters**: + +* ``type`` - PCIe device type (4=Root Port, etc.) +* ``reason`` - Reason for link change: + + - ``0`` - Link retrain + - ``1`` - Bus enumeration + - ``2`` - Bandwidth notification enable + - ``3`` - Bandwidth notification IRQ + - ``4`` - Hotplug event + + +**Example Usage**:: + + # Enable the tracepoint + echo 1 > /sys/kernel/debug/tracing/events/pci/pcie_link_event/enable + + # Monitor events (the following output is generated when a device is hotplugged) + cat /sys/kernel/debug/tracing/trace_pipe + irq/51-pciehp-88 [001] ..... 381.545386: pcie_link_event: 0000:00:02.0 type:4, reason:4, cur_bus_speed:20, max_bus_speed:23, width:1, flit_mode:0, status:DLLLA diff --git a/Documentation/trace/index.rst b/Documentation/trace/index.rst index b4a429dc4f7ad..0a40bfabcf19b 100644 --- a/Documentation/trace/index.rst +++ b/Documentation/trace/index.rst @@ -54,6 +54,7 @@ applications. events-power events-nmi events-msr + events-pci boottime-trace histogram histogram-design diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 57061fa29e6a0..ae8b02eb776a5 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -7800,8 +7800,10 @@ Will return -EBUSY if a VCPU has already been created. Valid feature flags in args[0] are:: - #define KVM_X2APIC_API_USE_32BIT_IDS (1ULL << 0) - #define KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK (1ULL << 1) + #define KVM_X2APIC_API_USE_32BIT_IDS (1ULL << 0) + #define KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK (1ULL << 1) + #define KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST (1ULL << 2) + #define KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST (1ULL << 3) Enabling KVM_X2APIC_API_USE_32BIT_IDS changes the behavior of KVM_SET_GSI_ROUTING, KVM_SIGNAL_MSI, KVM_SET_LAPIC, and KVM_GET_LAPIC, @@ -7814,6 +7816,28 @@ as a broadcast even in x2APIC mode in order to support physical x2APIC without interrupt remapping. This is undesirable in logical mode, where 0xff represents CPUs 0-7 in cluster 0. +Setting KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST instructs KVM to enable +Suppress EOI Broadcasts. KVM will advertise support for Suppress EOI +Broadcast to the guest and suppress LAPIC EOI broadcasts when the guest +sets the Suppress EOI Broadcast bit in the SPIV register. This flag is +supported only when using a split IRQCHIP. + +Setting KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST disables support for +Suppress EOI Broadcasts entirely, i.e. instructs KVM to NOT advertise +support to the guest. + +Modern VMMs should either enable KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST +or KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST. If not, legacy quirky +behavior will be used by KVM: in split IRQCHIP mode, KVM will advertise +support for Suppress EOI Broadcasts but not actually suppress EOI +broadcasts; for in-kernel IRQCHIP mode, KVM will not advertise support for +Suppress EOI Broadcasts. + +Setting both KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST and +KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST will fail with an EINVAL error, +as will setting KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST without a split +IRCHIP. + 7.8 KVM_CAP_S390_USER_INSTR0 ---------------------------- diff --git a/Makefile b/Makefile index 09153bd3bc5d5..82972256a8426 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 VERSION = 6 PATCHLEVEL = 18 -SUBLEVEL = 12 +SUBLEVEL = 18 EXTRAVERSION = NAME = Baby Opossum Posse @@ -1440,6 +1440,15 @@ ifneq ($(wildcard $(resolve_btfids_O)),) $(Q)$(MAKE) -sC $(srctree)/tools/bpf/resolve_btfids O=$(resolve_btfids_O) clean endif +PHONY += objtool_clean + +objtool_O = $(abspath $(objtree))/tools/objtool + +objtool_clean: +ifneq ($(wildcard $(objtool_O)),) + $(Q)$(MAKE) -sC $(abs_srctree)/tools/objtool O=$(objtool_O) srctree=$(abs_srctree) clean +endif + tools/: FORCE $(Q)mkdir -p $(objtree)/tools $(Q)$(MAKE) O=$(abspath $(objtree)) subdir=tools -C $(srctree)/tools/ @@ -1603,7 +1612,7 @@ vmlinuxclean: $(Q)$(CONFIG_SHELL) $(srctree)/scripts/link-vmlinux.sh clean $(Q)$(if $(ARCH_POSTLINK), $(MAKE) -f $(ARCH_POSTLINK) clean) -clean: archclean vmlinuxclean resolve_btfids_clean +clean: archclean vmlinuxclean resolve_btfids_clean objtool_clean # mrproper - Delete all generated files, including .config # diff --git a/arch/alpha/include/asm/pgtable.h b/arch/alpha/include/asm/pgtable.h index 90e7a95391022..c9508ec37efc4 100644 --- a/arch/alpha/include/asm/pgtable.h +++ b/arch/alpha/include/asm/pgtable.h @@ -17,6 +17,7 @@ #include /* For TASK_SIZE */ #include #include +#include struct mm_struct; struct vm_area_struct; @@ -183,6 +184,9 @@ extern inline void pud_set(pud_t * pudp, pmd_t * pmdp) { pud_val(*pudp) = _PAGE_TABLE | ((((unsigned long) pmdp) - PAGE_OFFSET) << (32-PAGE_SHIFT)); } +extern void migrate_flush_tlb_page(struct vm_area_struct *vma, + unsigned long addr); + extern inline unsigned long pmd_page_vaddr(pmd_t pmd) { @@ -202,7 +206,7 @@ extern inline int pte_none(pte_t pte) { return !pte_val(pte); } extern inline int pte_present(pte_t pte) { return pte_val(pte) & _PAGE_VALID; } extern inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - pte_val(*ptep) = 0; + WRITE_ONCE(pte_val(*ptep), 0); } extern inline int pmd_none(pmd_t pmd) { return !pmd_val(pmd); } @@ -264,6 +268,33 @@ extern inline pte_t * pte_offset_kernel(pmd_t * dir, unsigned long address) extern pgd_t swapper_pg_dir[1024]; +#ifdef CONFIG_COMPACTION +#define __HAVE_ARCH_PTEP_GET_AND_CLEAR + +static inline pte_t ptep_get_and_clear(struct mm_struct *mm, + unsigned long address, + pte_t *ptep) +{ + pte_t pte = READ_ONCE(*ptep); + + pte_clear(mm, address, ptep); + return pte; +} + +#define __HAVE_ARCH_PTEP_CLEAR_FLUSH + +static inline pte_t ptep_clear_flush(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + struct mm_struct *mm = vma->vm_mm; + pte_t pte = ptep_get_and_clear(mm, addr, ptep); + + page_table_check_pte_clear(mm, pte); + migrate_flush_tlb_page(vma, addr); + return pte; +} + +#endif /* * The Alpha doesn't have any external MMU info: the kernel page * tables contain all the necessary information. diff --git a/arch/alpha/include/asm/tlbflush.h b/arch/alpha/include/asm/tlbflush.h index ba4b359d6c395..0c8529997f54e 100644 --- a/arch/alpha/include/asm/tlbflush.h +++ b/arch/alpha/include/asm/tlbflush.h @@ -58,7 +58,9 @@ flush_tlb_other(struct mm_struct *mm) unsigned long *mmc = &mm->context[smp_processor_id()]; /* Check it's not zero first to avoid cacheline ping pong when possible. */ - if (*mmc) *mmc = 0; + + if (READ_ONCE(*mmc)) + WRITE_ONCE(*mmc, 0); } #ifndef CONFIG_SMP diff --git a/arch/alpha/kernel/vmlinux.lds.S b/arch/alpha/kernel/vmlinux.lds.S index 2efa7dfc798a9..2d136c63db161 100644 --- a/arch/alpha/kernel/vmlinux.lds.S +++ b/arch/alpha/kernel/vmlinux.lds.S @@ -71,6 +71,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/alpha/mm/Makefile b/arch/alpha/mm/Makefile index 101dbd06b4ceb..2d05664058f64 100644 --- a/arch/alpha/mm/Makefile +++ b/arch/alpha/mm/Makefile @@ -3,4 +3,4 @@ # Makefile for the linux alpha-specific parts of the memory manager. # -obj-y := init.o fault.o +obj-y := init.o fault.o tlbflush.o diff --git a/arch/alpha/mm/tlbflush.c b/arch/alpha/mm/tlbflush.c new file mode 100644 index 0000000000000..ccbc317b9a348 --- /dev/null +++ b/arch/alpha/mm/tlbflush.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Alpha TLB shootdown helpers + * + * Copyright (C) 2025 Magnus Lindholm + * + * Alpha-specific TLB flush helpers that cannot be expressed purely + * as inline functions. + * + * These helpers provide combined MM context handling (ASN rollover) + * and immediate TLB invalidation for page migration and memory + * compaction paths, where lazy shootdowns are insufficient. + */ + +#include +#include +#include +#include +#include +#include + +#define asn_locked() (cpu_data[smp_processor_id()].asn_lock) + +/* + * Migration/compaction helper: combine mm context (ASN) handling with an + * immediate per-page TLB invalidate and (for exec) an instruction barrier. + * + * This mirrors the SMP combined IPI handler semantics, but runs locally on UP. + */ +#ifndef CONFIG_SMP +void migrate_flush_tlb_page(struct vm_area_struct *vma, + unsigned long addr) +{ + struct mm_struct *mm = vma->vm_mm; + int tbi_type = (vma->vm_flags & VM_EXEC) ? 3 : 2; + + /* + * First do the mm-context side: + * If we're currently running this mm, reload a fresh context ASN. + * Otherwise, mark context invalid. + * + * On UP, this is mostly about matching the SMP semantics and ensuring + * exec/i-cache tagging assumptions hold when compaction migrates pages. + */ + if (mm == current->active_mm) + flush_tlb_current(mm); + else + flush_tlb_other(mm); + + /* + * Then do the immediate translation kill for this VA. + * For exec mappings, order instruction fetch after invalidation. + */ + tbi(tbi_type, addr); +} + +#else +struct tlb_mm_and_addr { + struct mm_struct *mm; + unsigned long addr; + int tbi_type; /* 2 = DTB, 3 = ITB+DTB */ +}; + +static void ipi_flush_mm_and_page(void *x) +{ + struct tlb_mm_and_addr *d = x; + + /* Part 1: mm context side (Alpha uses ASN/context as a key mechanism). */ + if (d->mm == current->active_mm && !asn_locked()) + __load_new_mm_context(d->mm); + else + flush_tlb_other(d->mm); + + /* Part 2: immediate per-VA invalidation on this CPU. */ + tbi(d->tbi_type, d->addr); +} + +void migrate_flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) +{ + struct mm_struct *mm = vma->vm_mm; + struct tlb_mm_and_addr d = { + .mm = mm, + .addr = addr, + .tbi_type = (vma->vm_flags & VM_EXEC) ? 3 : 2, + }; + + /* + * One synchronous rendezvous: every CPU runs ipi_flush_mm_and_page(). + * This is the "combined" version of flush_tlb_mm + per-page invalidate. + */ + preempt_disable(); + on_each_cpu(ipi_flush_mm_and_page, &d, 1); + + /* + * mimic flush_tlb_mm()'s mm_users<=1 optimization. + */ + if (atomic_read(&mm->mm_users) <= 1) { + + int cpu, this_cpu; + this_cpu = smp_processor_id(); + + for (cpu = 0; cpu < NR_CPUS; cpu++) { + if (!cpu_online(cpu) || cpu == this_cpu) + continue; + if (READ_ONCE(mm->context[cpu])) + WRITE_ONCE(mm->context[cpu], 0); + } + } + preempt_enable(); +} + +#endif diff --git a/arch/arc/kernel/vmlinux.lds.S b/arch/arc/kernel/vmlinux.lds.S index 61a1b2b96e1d8..6af63084ff285 100644 --- a/arch/arc/kernel/vmlinux.lds.S +++ b/arch/arc/kernel/vmlinux.lds.S @@ -123,6 +123,7 @@ SECTIONS _end = . ; STABS_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/arm/boot/compressed/vmlinux.lds.S b/arch/arm/boot/compressed/vmlinux.lds.S index d411abd4310ea..2d916647df03c 100644 --- a/arch/arm/boot/compressed/vmlinux.lds.S +++ b/arch/arm/boot/compressed/vmlinux.lds.S @@ -21,6 +21,7 @@ SECTIONS COMMON_DISCARDS *(.ARM.exidx*) *(.ARM.extab*) + *(.modinfo) *(.note.*) *(.rel.*) *(.printk_index) diff --git a/arch/arm/boot/dts/allwinner/sun5i-a13-utoo-p66.dts b/arch/arm/boot/dts/allwinner/sun5i-a13-utoo-p66.dts index be486d28d04fa..428cab5a0e906 100644 --- a/arch/arm/boot/dts/allwinner/sun5i-a13-utoo-p66.dts +++ b/arch/arm/boot/dts/allwinner/sun5i-a13-utoo-p66.dts @@ -102,6 +102,7 @@ /* The P66 uses a different EINT then the reference design */ interrupts = <6 9 IRQ_TYPE_EDGE_FALLING>; /* EINT9 (PG9) */ /* The icn8318 binding expects wake-gpios instead of power-gpios */ + /delete-property/ power-gpios; wake-gpios = <&pio 1 3 GPIO_ACTIVE_HIGH>; /* PB3 */ touchscreen-size-x = <800>; touchscreen-size-y = <480>; diff --git a/arch/arm/boot/dts/nxp/imx/imx53-usbarmory.dts b/arch/arm/boot/dts/nxp/imx/imx53-usbarmory.dts index acc44010d5106..3ad9db4b14425 100644 --- a/arch/arm/boot/dts/nxp/imx/imx53-usbarmory.dts +++ b/arch/arm/boot/dts/nxp/imx/imx53-usbarmory.dts @@ -1,47 +1,10 @@ +// SPDX-License-Identifier: (GPL-2.0-or-later OR MIT) /* * USB armory MkI device tree file * https://inversepath.com/usbarmory * * Copyright (C) 2015, Inverse Path * Andrej Rosano - * - * This file is dual-licensed: you can use it either under the terms - * of the GPL or the X11 license, at your option. Note that this dual - * licensing only applies to this file, and not this project as a - * whole. - * - * a) This file is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation; either version 2 of the - * License, or (at your option) any later version. - * - * This file is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * Or, alternatively, - * - * b) Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. */ /dts-v1/; diff --git a/arch/arm/boot/dts/nxp/lpc/lpc32xx.dtsi b/arch/arm/boot/dts/nxp/lpc/lpc32xx.dtsi index 2236901a00313..8e9ed93da129e 100644 --- a/arch/arm/boot/dts/nxp/lpc/lpc32xx.dtsi +++ b/arch/arm/boot/dts/nxp/lpc/lpc32xx.dtsi @@ -302,6 +302,7 @@ mpwm: pwm@400e8000 { compatible = "nxp,lpc3220-motor-pwm"; reg = <0x400e8000 0x78>; + clocks = <&clk LPC32XX_CLK_MCPWM>; #pwm-cells = <3>; status = "disabled"; }; diff --git a/arch/arm/include/asm/string.h b/arch/arm/include/asm/string.h index c35250c4991bc..96fc6cf460ecb 100644 --- a/arch/arm/include/asm/string.h +++ b/arch/arm/include/asm/string.h @@ -39,13 +39,17 @@ static inline void *memset32(uint32_t *p, uint32_t v, __kernel_size_t n) } #define __HAVE_ARCH_MEMSET64 -extern void *__memset64(uint64_t *, uint32_t low, __kernel_size_t, uint32_t hi); +extern void *__memset64(uint64_t *, uint32_t first, __kernel_size_t, uint32_t second); static inline void *memset64(uint64_t *p, uint64_t v, __kernel_size_t n) { - if (IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN)) - return __memset64(p, v, n * 8, v >> 32); - else - return __memset64(p, v >> 32, n * 8, v); + union { + uint64_t val; + struct { + uint32_t first, second; + }; + } word = { .val = v }; + + return __memset64(p, word.first, n * 8, word.second); } /* diff --git a/arch/arm/kernel/vdso.c b/arch/arm/kernel/vdso.c index e38a30477f3d7..566c40f0f7c77 100644 --- a/arch/arm/kernel/vdso.c +++ b/arch/arm/kernel/vdso.c @@ -161,6 +161,7 @@ static void __init patch_vdso(void *ehdr) vdso_nullpatch_one(&einfo, "__vdso_gettimeofday"); vdso_nullpatch_one(&einfo, "__vdso_clock_gettime"); vdso_nullpatch_one(&einfo, "__vdso_clock_gettime64"); + vdso_nullpatch_one(&einfo, "__vdso_clock_getres"); } } diff --git a/arch/arm/kernel/vmlinux-xip.lds.S b/arch/arm/kernel/vmlinux-xip.lds.S index f2e8d4fac0687..5afb725998ec0 100644 --- a/arch/arm/kernel/vmlinux-xip.lds.S +++ b/arch/arm/kernel/vmlinux-xip.lds.S @@ -154,6 +154,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ARM_DETAILS ARM_ASSERTS diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S index d592a203f9c6b..c07843c3c53d3 100644 --- a/arch/arm/kernel/vmlinux.lds.S +++ b/arch/arm/kernel/vmlinux.lds.S @@ -153,6 +153,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ARM_DETAILS ARM_ASSERTS diff --git a/arch/arm/mach-omap2/control.c b/arch/arm/mach-omap2/control.c index 79860b23030de..eb6fc7c61b6e0 100644 --- a/arch/arm/mach-omap2/control.c +++ b/arch/arm/mach-omap2/control.c @@ -732,7 +732,7 @@ int __init omap2_control_base_init(void) */ int __init omap_control_init(void) { - struct device_node *np, *scm_conf; + struct device_node *np, *scm_conf, *clocks_node; const struct of_device_id *match; const struct omap_prcm_init_data *data; int ret; @@ -753,16 +753,19 @@ int __init omap_control_init(void) if (IS_ERR(syscon)) { ret = PTR_ERR(syscon); - goto of_node_put; + goto err_put_scm_conf; } - if (of_get_child_by_name(scm_conf, "clocks")) { + clocks_node = of_get_child_by_name(scm_conf, "clocks"); + if (clocks_node) { + of_node_put(clocks_node); ret = omap2_clk_provider_init(scm_conf, data->index, syscon, NULL); if (ret) - goto of_node_put; + goto err_put_scm_conf; } + of_node_put(scm_conf); } else { /* No scm_conf found, direct access */ ret = omap2_clk_provider_init(np, data->index, NULL, @@ -780,6 +783,9 @@ int __init omap_control_init(void) return 0; +err_put_scm_conf: + if (scm_conf) + of_node_put(scm_conf); of_node_put: of_node_put(np); return ret; diff --git a/arch/arm/mm/physaddr.c b/arch/arm/mm/physaddr.c index 3f263c840ebc4..1a37ebfacbba9 100644 --- a/arch/arm/mm/physaddr.c +++ b/arch/arm/mm/physaddr.c @@ -38,7 +38,7 @@ static inline bool __virt_addr_valid(unsigned long x) phys_addr_t __virt_to_phys(unsigned long x) { WARN(!__virt_addr_valid(x), - "virt_to_phys used for non-linear address: %pK (%pS)\n", + "virt_to_phys used for non-linear address: %px (%pS)\n", (void *)x, (void *)x); return __virt_to_phys_nodebug(x); diff --git a/arch/arm64/Kbuild b/arch/arm64/Kbuild index 5bfbf7d79c99b..d876bc0e54211 100644 --- a/arch/arm64/Kbuild +++ b/arch/arm64/Kbuild @@ -1,4 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-only + +# Branch profiling isn't noinstr-safe +subdir-ccflags-$(CONFIG_TRACE_BRANCH_PROFILING) += -DDISABLE_BRANCH_PROFILING + obj-y += kernel/ mm/ net/ obj-$(CONFIG_KVM) += kvm/ obj-$(CONFIG_XEN) += xen/ diff --git a/arch/arm64/boot/dts/amlogic/amlogic-c3.dtsi b/arch/arm64/boot/dts/amlogic/amlogic-c3.dtsi index 07aaaf71ea9ae..f226df3ce153b 100644 --- a/arch/arm64/boot/dts/amlogic/amlogic-c3.dtsi +++ b/arch/arm64/boot/dts/amlogic/amlogic-c3.dtsi @@ -969,6 +969,10 @@ no-sd; resets = <&reset RESET_SD_EMMC_A>; status = "disabled"; + + assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_A>; + assigned-clock-rates = <24000000>; + }; sd: mmc@8a000 { @@ -984,6 +988,9 @@ no-sdio; resets = <&reset RESET_SD_EMMC_B>; status = "disabled"; + + assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_B>; + assigned-clock-rates = <24000000>; }; nand: nand-controller@8d000 { diff --git a/arch/arm64/boot/dts/amlogic/meson-axg.dtsi b/arch/arm64/boot/dts/amlogic/meson-axg.dtsi index 04fb130ac7c6a..bbf94a1f92a10 100644 --- a/arch/arm64/boot/dts/amlogic/meson-axg.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-axg.dtsi @@ -1960,6 +1960,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_B>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_B_CLK0>; + assigned-clock-rates = <24000000>; }; sd_emmc_c: mmc@7000 { @@ -1972,6 +1975,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_C>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_C_CLK0>; + assigned-clock-rates = <24000000>; }; nfc: nand-controller@7800 { diff --git a/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi b/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi index dcc927a9da802..8d8ab775404d9 100644 --- a/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-g12-common.dtsi @@ -2431,6 +2431,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_A>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_A_CLK0>; + assigned-clock-rates = <24000000>; }; sd_emmc_b: mmc@ffe05000 { @@ -2443,6 +2446,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_B>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_B_CLK0>; + assigned-clock-rates = <24000000>; }; sd_emmc_c: mmc@ffe07000 { @@ -2455,6 +2461,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_C>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_C_CLK0>; + assigned-clock-rates = <24000000>; }; usb: usb@ffe09000 { diff --git a/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi index f69923da07feb..a9c830a570cc6 100644 --- a/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi @@ -824,6 +824,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_A>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_A_CLK0>; + assigned-clock-rates = <24000000>; }; &sd_emmc_b { @@ -832,6 +835,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_B>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_B_CLK0>; + assigned-clock-rates = <24000000>; }; &sd_emmc_c { @@ -840,6 +846,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_C>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_C_CLK0>; + assigned-clock-rates = <24000000>; }; &simplefb_hdmi { diff --git a/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi index ba535010a3c91..e202d84f06720 100644 --- a/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi @@ -894,6 +894,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_A>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_A_CLK0>; + assigned-clock-rates = <24000000>; }; &sd_emmc_b { @@ -902,6 +905,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_B>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_B_CLK0>; + assigned-clock-rates = <24000000>; }; &sd_emmc_c { @@ -910,6 +916,9 @@ <&clkc CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_C>; + + assigned-clocks = <&clkc CLKID_SD_EMMC_C_CLK0>; + assigned-clock-rates = <24000000>; }; &simplefb_hdmi { diff --git a/arch/arm64/boot/dts/amlogic/meson-s4.dtsi b/arch/arm64/boot/dts/amlogic/meson-s4.dtsi index 9d99ed2994dfa..dfc0a30a6e61b 100644 --- a/arch/arm64/boot/dts/amlogic/meson-s4.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-s4.dtsi @@ -819,13 +819,16 @@ reg = <0x0 0xfe088000 0x0 0x800>; interrupts = ; clocks = <&clkc_periphs CLKID_SDEMMC_A>, - <&xtal>, + <&clkc_periphs CLKID_SD_EMMC_A>, <&clkc_pll CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_A>; cap-sdio-irq; keep-power-in-suspend; status = "disabled"; + + assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_A>; + assigned-clock-rates = <24000000>; }; sd: mmc@fe08a000 { @@ -838,6 +841,9 @@ clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_SD_EMMC_B>; status = "disabled"; + + assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_B>; + assigned-clock-rates = <24000000>; }; emmc: mmc@fe08c000 { @@ -845,13 +851,16 @@ reg = <0x0 0xfe08c000 0x0 0x800>; interrupts = ; clocks = <&clkc_periphs CLKID_NAND>, - <&xtal>, + <&clkc_periphs CLKID_SD_EMMC_C>, <&clkc_pll CLKID_FCLK_DIV2>; clock-names = "core", "clkin0", "clkin1"; resets = <&reset RESET_NAND_EMMC>; no-sdio; no-sd; status = "disabled"; + + assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_C>; + assigned-clock-rates = <24000000>; }; }; }; diff --git a/arch/arm64/boot/dts/apple/t8112-j473.dts b/arch/arm64/boot/dts/apple/t8112-j473.dts index 06fe257f08be4..4ae1ce919dafc 100644 --- a/arch/arm64/boot/dts/apple/t8112-j473.dts +++ b/arch/arm64/boot/dts/apple/t8112-j473.dts @@ -21,6 +21,25 @@ }; }; +/* + * Keep the power-domains used for the HDMI port on. + */ +&framebuffer0 { + power-domains = <&ps_dispext_cpu0>, <&ps_dptx_ext_phy>; +}; + +/* + * The M2 Mac mini uses dispext for the HDMI output so it's not necessary to + * keep disp0 power-domains always-on. + */ +&ps_disp0_sys { + /delete-property/ apple,always-on; +}; + +&ps_disp0_fe { + /delete-property/ apple,always-on; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts index f7346b3d35fe5..a122f2ed5f531 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts @@ -704,7 +704,7 @@ fsl,pins = , , , - ; + ; }; pinctrl_gpt1: gpt1grp { diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts index 4eedd00d83b9f..ac05c05193c5b 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts @@ -860,7 +860,7 @@ fsl,pins = , , , - ; + ; }; pinctrl_hoggpio2: hoggpio2grp { diff --git a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-pico6.dts b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-pico6.dts index cce326aec1aa5..40af5656d6f15 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-pico6.dts +++ b/arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-pico6.dts @@ -91,7 +91,7 @@ &pio { bt_pins_wakeup: bt-pins-wakeup { - piins-bt-wakeup { + pins-bt-wakeup { pinmux = ; input-enable; }; diff --git a/arch/arm64/boot/dts/mediatek/mt8183.dtsi b/arch/arm64/boot/dts/mediatek/mt8183.dtsi index 960d8955d018c..27ce4b31cc99d 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8183.dtsi @@ -1812,15 +1812,23 @@ #size-cells = <0>; port@0 { + #address-cells = <1>; + #size-cells = <0>; reg = <0>; - ovl_2l1_in: endpoint { + + ovl_2l1_in: endpoint@1 { + reg = <1>; remote-endpoint = <&mmsys_ep_ext>; }; }; port@1 { + #address-cells = <1>; + #size-cells = <0>; reg = <1>; - ovl_2l1_out: endpoint { + + ovl_2l1_out: endpoint@1 { + reg = <1>; remote-endpoint = <&rdma1_in>; }; }; @@ -1872,15 +1880,23 @@ #size-cells = <0>; port@0 { + #address-cells = <1>; + #size-cells = <0>; reg = <0>; - rdma1_in: endpoint { + + rdma1_in: endpoint@1 { + reg = <1>; remote-endpoint = <&ovl_2l1_out>; }; }; port@1 { + #address-cells = <1>; + #size-cells = <0>; reg = <1>; - rdma1_out: endpoint { + + rdma1_out: endpoint@1 { + reg = <1>; remote-endpoint = <&dpi_in>; }; }; @@ -2076,15 +2092,24 @@ #size-cells = <0>; port@0 { + #address-cells = <1>; + #size-cells = <0>; reg = <0>; - dpi_in: endpoint { + + dpi_in: endpoint@1 { + reg = <1>; remote-endpoint = <&rdma1_out>; }; }; port@1 { + #address-cells = <1>; + #size-cells = <0>; reg = <1>; - dpi_out: endpoint { }; + + dpi_out: endpoint@1 { + reg = <1>; + }; }; }; }; diff --git a/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts b/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts index 5aa6afd56cbc6..dfbd1c72388c1 100644 --- a/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts +++ b/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts @@ -1809,6 +1809,8 @@ status = "okay"; vbus-supply = <&usbc_vbus>; mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; }; usb3-0 { diff --git a/arch/arm64/boot/dts/qcom/hamoa.dtsi b/arch/arm64/boot/dts/qcom/hamoa.dtsi index 730e331166e91..e0df880fe6e9b 100644 --- a/arch/arm64/boot/dts/qcom/hamoa.dtsi +++ b/arch/arm64/boot/dts/qcom/hamoa.dtsi @@ -829,8 +829,8 @@ #address-cells = <2>; #size-cells = <2>; - dma-ranges = <0 0 0 0 0x10 0>; - ranges = <0 0 0 0 0x10 0>; + dma-ranges = <0 0 0 0 0x100 0>; + ranges = <0 0 0 0 0x100 0>; gcc: clock-controller@100000 { compatible = "qcom,x1e80100-gcc"; @@ -2975,7 +2975,7 @@ reg = <0 0x00fda000 0 0x4000>; clocks = <&gcc GCC_USB3_SEC_PHY_AUX_CLK>, - <&rpmhcc RPMH_CXO_CLK>, + <&tcsr TCSR_USB4_1_CLKREF_EN>, <&gcc GCC_USB3_SEC_PHY_COM_AUX_CLK>, <&gcc GCC_USB3_SEC_PHY_PIPE_CLK>; clock-names = "aux", @@ -3046,7 +3046,7 @@ reg = <0 0x00fdf000 0 0x4000>; clocks = <&gcc GCC_USB3_TERT_PHY_AUX_CLK>, - <&rpmhcc RPMH_CXO_CLK>, + <&tcsr TCSR_USB4_2_CLKREF_EN>, <&gcc GCC_USB3_TERT_PHY_COM_AUX_CLK>, <&gcc GCC_USB3_TERT_PHY_PIPE_CLK>; clock-names = "aux", @@ -6398,9 +6398,11 @@ <0 0x0aec2000 0 0x1c8>; clocks = <&dispcc DISP_CC_MDSS_DPTX2_AUX_CLK>, - <&dispcc DISP_CC_MDSS_AHB_CLK>; + <&dispcc DISP_CC_MDSS_AHB_CLK>, + <&tcsr TCSR_EDP_CLKREF_EN>; clock-names = "aux", - "cfg_ahb"; + "cfg_ahb", + "ref"; power-domains = <&rpmhpd RPMHPD_MX>; @@ -6418,9 +6420,11 @@ <0 0x0aec5000 0 0x1c8>; clocks = <&dispcc DISP_CC_MDSS_DPTX3_AUX_CLK>, - <&dispcc DISP_CC_MDSS_AHB_CLK>; + <&dispcc DISP_CC_MDSS_AHB_CLK>, + <&tcsr TCSR_EDP_CLKREF_EN>; clock-names = "aux", - "cfg_ahb"; + "cfg_ahb", + "ref"; power-domains = <&rpmhpd RPMHPD_MX>; diff --git a/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi b/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi index 4c983b10dd925..7ace3540ef0a0 100644 --- a/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi @@ -378,7 +378,7 @@ status = "okay"; sideinteraction: touch@2c { - compatible = "ad,ad7147_captouch"; + compatible = "adi,ad7147_captouch"; reg = <0x2c>; pinctrl-names = "default", "sleep"; diff --git a/arch/arm64/boot/dts/qcom/qcm2290.dtsi b/arch/arm64/boot/dts/qcom/qcm2290.dtsi index 3b0ba590ee825..e0e400fdd2497 100644 --- a/arch/arm64/boot/dts/qcom/qcm2290.dtsi +++ b/arch/arm64/boot/dts/qcom/qcm2290.dtsi @@ -1503,8 +1503,12 @@ gpu: gpu@5900000 { compatible = "qcom,adreno-07000200", "qcom,adreno"; - reg = <0x0 0x05900000 0x0 0x40000>; - reg-names = "kgsl_3d0_reg_memory"; + reg = <0x0 0x05900000 0x0 0x40000>, + <0x0 0x0599e000 0x0 0x1000>, + <0x0 0x05961000 0x0 0x800>; + reg-names = "kgsl_3d0_reg_memory", + "cx_mem", + "cx_dbgc"; interrupts = ; diff --git a/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts b/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts index bdf2d66e40c62..44ca3e61c33d4 100644 --- a/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts +++ b/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts @@ -694,7 +694,7 @@ &uart3 { interrupts-extended = <&intc GIC_SPI 330 IRQ_TYPE_LEVEL_HIGH>, - <&tlmm 11 IRQ_TYPE_LEVEL_HIGH>; + <&tlmm 11 IRQ_TYPE_EDGE_FALLING>; pinctrl-0 = <&uart3_default>; pinctrl-1 = <&uart3_sleep>; pinctrl-names = "default", "sleep"; diff --git a/arch/arm64/boot/dts/qcom/sdm630.dtsi b/arch/arm64/boot/dts/qcom/sdm630.dtsi index 8b1a45a4e56ed..876a6871745cf 100644 --- a/arch/arm64/boot/dts/qcom/sdm630.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm630.dtsi @@ -598,8 +598,8 @@ }; gpu_speed_bin: gpu-speed-bin@41a0 { - reg = <0x41a2 0x1>; - bits = <5 7>; + reg = <0x41a2 0x2>; + bits = <5 8>; }; }; @@ -1563,6 +1563,7 @@ reg-names = "mdss_phys", "vbif_phys"; power-domains = <&mmcc MDSS_GDSC>; + resets = <&mmcc MDSS_BCR>; clocks = <&mmcc MDSS_AHB_CLK>, <&mmcc MDSS_AXI_CLK>, diff --git a/arch/arm64/boot/dts/qcom/sdm845-db845c.dts b/arch/arm64/boot/dts/qcom/sdm845-db845c.dts index 8abf3e909502f..5147d6d3cc26b 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-db845c.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-db845c.dts @@ -379,6 +379,12 @@ regulator-initial-mode = ; }; + vreg_l23a_3p3: ldo23 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3312000>; + regulator-initial-mode = ; + }; + vreg_l24a_3p075: ldo24 { regulator-min-microvolt = <3088000>; regulator-max-microvolt = <3088000>; @@ -850,7 +856,6 @@ status = "okay"; pinctrl-names = "default"; pinctrl-0 = <&qup_spi0_default>; - cs-gpios = <&tlmm 3 GPIO_ACTIVE_LOW>; can@0 { compatible = "microchip,mcp2517fd"; @@ -1156,6 +1161,7 @@ vdd-1.8-xo-supply = <&vreg_l7a_1p8>; vdd-1.3-rfa-supply = <&vreg_l17a_1p3>; vdd-3.3-ch0-supply = <&vreg_l25a_3p3>; + vdd-3.3-ch1-supply = <&vreg_l23a_3p3>; qcom,snoc-host-cap-8bit-quirk; qcom,calibration-variant = "Thundercomm_DB845C"; diff --git a/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi index 51a9a276399ac..1036305231b20 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi @@ -160,7 +160,6 @@ gpio = <&tlmm 88 0>; enable-active-high; - regulator-boot-on; }; }; @@ -257,7 +256,7 @@ regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; regulator-initial-mode = ; - regulator-always-on; + regulator-boot-on; }; vreg_l17a_1p3: ldo17 { diff --git a/arch/arm64/boot/dts/qcom/sm6115.dtsi b/arch/arm64/boot/dts/qcom/sm6115.dtsi index 91fc36b59abf9..8b8395f6a2dfc 100644 --- a/arch/arm64/boot/dts/qcom/sm6115.dtsi +++ b/arch/arm64/boot/dts/qcom/sm6115.dtsi @@ -1715,8 +1715,12 @@ gpu: gpu@5900000 { compatible = "qcom,adreno-610.0", "qcom,adreno"; - reg = <0x0 0x05900000 0x0 0x40000>; - reg-names = "kgsl_3d0_reg_memory"; + reg = <0x0 0x05900000 0x0 0x40000>, + <0x0 0x0599e000 0x0 0x1000>, + <0x0 0x05961000 0x0 0x800>; + reg-names = "kgsl_3d0_reg_memory", + "cx_mem", + "cx_dbgc"; /* There's no (real) GMU, so we have to handle quite a bunch of clocks! */ clocks = <&gpucc GPU_CC_GX_GFX3D_CLK>, diff --git a/arch/arm64/boot/dts/qcom/sm8750.dtsi b/arch/arm64/boot/dts/qcom/sm8750.dtsi index 67027cd70277c..049afe2c72043 100644 --- a/arch/arm64/boot/dts/qcom/sm8750.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8750.dtsi @@ -2077,6 +2077,8 @@ <&apps_smmu 0x481 0>; qcom,ee = <0>; + qcom,num-ees = <4>; + num-channels = <20>; qcom,controlled-remotely; }; diff --git a/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts b/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts index 08e814c03fa85..ed6fcdc337a0b 100644 --- a/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts +++ b/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts @@ -8,7 +8,6 @@ /dts-v1/; /* Switch selection settings */ -#define SW_LCD_EN 0 #define SW_GPIO8_CAN0_STB 0 #define SW_GPIO9_CAN1_STB 0 #define SW_LCD_EN 0 diff --git a/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi b/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi index 5c91002c99c48..5384a43837c1d 100644 --- a/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi +++ b/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi @@ -154,8 +154,7 @@ ctrl-pins { pinmux = , /* SD0_CLK */ , /* SD0_CMD */ - , /* SD0_CD */ - ; /* SD0_WP */ + ; /* SD0_CD */ }; }; @@ -212,6 +211,7 @@ pinctrl-names = "default", "state_uhs"; vmmc-supply = <®_3p3v>; vqmmc-supply = <&vqmmc_sdhi0>; + wp-gpios = <&pinctrl RZT2H_GPIO(22, 6) GPIO_ACTIVE_HIGH>; bus-width = <4>; sd-uhs-sdr50; sd-uhs-sdr104; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts b/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts index eaaca08a76018..a6ac89567bafe 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts +++ b/arch/arm64/boot/dts/rockchip/rk3399-pinebook-pro.dts @@ -421,10 +421,6 @@ status = "okay"; }; -&hdmi_sound { - status = "okay"; -}; - &i2c0 { clock-frequency = <400000>; i2c-scl-falling-time-ns = <4>; diff --git a/arch/arm64/boot/dts/rockchip/rk3568.dtsi b/arch/arm64/boot/dts/rockchip/rk3568.dtsi index e719a3df126c5..658097ed69714 100644 --- a/arch/arm64/boot/dts/rockchip/rk3568.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3568.dtsi @@ -185,7 +185,7 @@ <0x0 0xf2000000 0x0 0x00100000>; ranges = <0x01000000 0x0 0xf2100000 0x0 0xf2100000 0x0 0x00100000>, <0x02000000 0x0 0xf2200000 0x0 0xf2200000 0x0 0x01e00000>, - <0x03000000 0x0 0x40000000 0x3 0x40000000 0x0 0x40000000>; + <0x03000000 0x3 0x40000000 0x3 0x40000000 0x0 0x40000000>; reg-names = "dbi", "apb", "config"; resets = <&cru SRST_PCIE30X1_POWERUP>; reset-names = "pipe"; @@ -238,7 +238,7 @@ <0x0 0xf0000000 0x0 0x00100000>; ranges = <0x01000000 0x0 0xf0100000 0x0 0xf0100000 0x0 0x00100000>, <0x02000000 0x0 0xf0200000 0x0 0xf0200000 0x0 0x01e00000>, - <0x03000000 0x0 0x40000000 0x3 0x80000000 0x0 0x40000000>; + <0x03000000 0x3 0x80000000 0x3 0x80000000 0x0 0x40000000>; reg-names = "dbi", "apb", "config"; resets = <&cru SRST_PCIE30X2_POWERUP>; reset-names = "pipe"; diff --git a/arch/arm64/boot/dts/rockchip/rk356x-base.dtsi b/arch/arm64/boot/dts/rockchip/rk356x-base.dtsi index fd2214b6fad40..d654f98460ec0 100644 --- a/arch/arm64/boot/dts/rockchip/rk356x-base.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk356x-base.dtsi @@ -975,7 +975,7 @@ power-domains = <&power RK3568_PD_PIPE>; ranges = <0x01000000 0x0 0xf4100000 0x0 0xf4100000 0x0 0x00100000>, <0x02000000 0x0 0xf4200000 0x0 0xf4200000 0x0 0x01e00000>, - <0x03000000 0x0 0x40000000 0x3 0x00000000 0x0 0x40000000>; + <0x03000000 0x3 0x00000000 0x3 0x00000000 0x0 0x40000000>; resets = <&cru SRST_PCIE20_POWERUP>; reset-names = "pipe"; #address-cells = <3>; diff --git a/arch/arm64/boot/dts/rockchip/rk3576-evb1-v10.dts b/arch/arm64/boot/dts/rockchip/rk3576-evb1-v10.dts index db8fef7a4f1b9..ffe55f970f461 100644 --- a/arch/arm64/boot/dts/rockchip/rk3576-evb1-v10.dts +++ b/arch/arm64/boot/dts/rockchip/rk3576-evb1-v10.dts @@ -223,6 +223,18 @@ vin-supply = <&vcc_3v3_s3>; }; + vcc3v3_sd: regulator-vcc-3v3-sd { + compatible = "regulator-fixed"; + enable-active-high; + gpios = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_pwren>; + regulator-name = "vcc3v3_sd"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc_3v3_s0>; + }; + vcc_ufs_s0: regulator-vcc-ufs-s0 { compatible = "regulator-fixed"; regulator-name = "vcc_ufs_s0"; @@ -810,6 +822,12 @@ }; }; + sdmmc { + sdmmc_pwren: sdmmc-pwren { + rockchip,pins = <0 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + usb { usb_host_pwren: usb-host-pwren { rockchip,pins = <0 RK_PC7 RK_FUNC_GPIO &pcfg_pull_none>; @@ -851,11 +869,15 @@ bus-width = <4>; cap-mmc-highspeed; cap-sd-highspeed; + cd-gpios = <&gpio0 RK_PA7 GPIO_ACTIVE_LOW>; disable-wp; max-frequency = <200000000>; no-sdio; no-mmc; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc0_clk &sdmmc0_cmd &sdmmc0_det &sdmmc0_bus4>; sd-uhs-sdr104; + vmmc-supply = <&vcc3v3_sd>; vqmmc-supply = <&vccio_sd_s0>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3576-pinctrl.dtsi b/arch/arm64/boot/dts/rockchip/rk3576-pinctrl.dtsi index 0b0851a7e4ea9..98c9f8013158c 100644 --- a/arch/arm64/boot/dts/rockchip/rk3576-pinctrl.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3576-pinctrl.dtsi @@ -5228,6 +5228,13 @@ /* ufs_rstn */ <4 RK_PD0 1 &pcfg_pull_none>; }; + + /omit-if-no-ref/ + ufs_rstgpio: ufs-rstgpio { + rockchip,pins = + /* ufs_rstn */ + <4 RK_PD0 RK_FUNC_GPIO &pcfg_pull_down>; + }; }; ufs_testdata0 { diff --git a/arch/arm64/boot/dts/rockchip/rk3576.dtsi b/arch/arm64/boot/dts/rockchip/rk3576.dtsi index c72343e7a0456..70e67d4dccb8a 100644 --- a/arch/arm64/boot/dts/rockchip/rk3576.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3576.dtsi @@ -1826,7 +1826,7 @@ assigned-clock-parents = <&cru CLK_REF_MPHY_26M>; interrupts = ; power-domains = <&power RK3576_PD_USB>; - pinctrl-0 = <&ufs_refclk>; + pinctrl-0 = <&ufs_refclk &ufs_rstgpio>; pinctrl-names = "default"; resets = <&cru SRST_A_UFS_BIU>, <&cru SRST_A_UFS_SYS>, <&cru SRST_A_UFS>, <&cru SRST_P_UFS_GRF>; diff --git a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi index 2973f6bae1716..7e74e04057cfd 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi @@ -1955,7 +1955,7 @@ power-domains = <&power RK3588_PD_PCIE>; ranges = <0x01000000 0x0 0xf3100000 0x0 0xf3100000 0x0 0x00100000>, <0x02000000 0x0 0xf3200000 0x0 0xf3200000 0x0 0x00e00000>, - <0x03000000 0x0 0x40000000 0x9 0xc0000000 0x0 0x40000000>; + <0x03000000 0x9 0xc0000000 0x9 0xc0000000 0x0 0x40000000>; reg = <0xa 0x40c00000 0x0 0x00400000>, <0x0 0xfe180000 0x0 0x00010000>, <0x0 0xf3000000 0x0 0x00100000>; @@ -2007,7 +2007,7 @@ power-domains = <&power RK3588_PD_PCIE>; ranges = <0x01000000 0x0 0xf4100000 0x0 0xf4100000 0x0 0x00100000>, <0x02000000 0x0 0xf4200000 0x0 0xf4200000 0x0 0x00e00000>, - <0x03000000 0x0 0x40000000 0xa 0x00000000 0x0 0x40000000>; + <0x03000000 0xa 0x00000000 0xa 0x00000000 0x0 0x40000000>; reg = <0xa 0x41000000 0x0 0x00400000>, <0x0 0xfe190000 0x0 0x00010000>, <0x0 0xf4000000 0x0 0x00100000>; diff --git a/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi index 6e5a58428bbab..a2640014ee042 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi @@ -375,7 +375,7 @@ power-domains = <&power RK3588_PD_PCIE>; ranges = <0x01000000 0x0 0xf0100000 0x0 0xf0100000 0x0 0x00100000>, <0x02000000 0x0 0xf0200000 0x0 0xf0200000 0x0 0x00e00000>, - <0x03000000 0x0 0x40000000 0x9 0x00000000 0x0 0x40000000>; + <0x03000000 0x9 0x00000000 0x9 0x00000000 0x0 0x40000000>; reg = <0xa 0x40000000 0x0 0x00400000>, <0x0 0xfe150000 0x0 0x00010000>, <0x0 0xf0000000 0x0 0x00100000>; @@ -462,7 +462,7 @@ power-domains = <&power RK3588_PD_PCIE>; ranges = <0x01000000 0x0 0xf1100000 0x0 0xf1100000 0x0 0x00100000>, <0x02000000 0x0 0xf1200000 0x0 0xf1200000 0x0 0x00e00000>, - <0x03000000 0x0 0x40000000 0x9 0x40000000 0x0 0x40000000>; + <0x03000000 0x9 0x40000000 0x9 0x40000000 0x0 0x40000000>; reg = <0xa 0x40400000 0x0 0x00400000>, <0x0 0xfe160000 0x0 0x00010000>, <0x0 0xf1000000 0x0 0x00100000>; @@ -512,7 +512,7 @@ power-domains = <&power RK3588_PD_PCIE>; ranges = <0x01000000 0x0 0xf2100000 0x0 0xf2100000 0x0 0x00100000>, <0x02000000 0x0 0xf2200000 0x0 0xf2200000 0x0 0x00e00000>, - <0x03000000 0x0 0x40000000 0x9 0x80000000 0x0 0x40000000>; + <0x03000000 0x9 0x80000000 0x9 0x80000000 0x0 0x40000000>; reg = <0xa 0x40800000 0x0 0x00400000>, <0x0 0xfe170000 0x0 0x00010000>, <0x0 0xf2000000 0x0 0x00100000>; diff --git a/arch/arm64/boot/dts/ti/k3-am62p-verdin.dtsi b/arch/arm64/boot/dts/ti/k3-am62p-verdin.dtsi index 99810047614e3..b7d559c61f3f8 100644 --- a/arch/arm64/boot/dts/ti/k3-am62p-verdin.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am62p-verdin.dtsi @@ -112,7 +112,7 @@ regulator-max-microvolt = <3300000>; regulator-min-microvolt = <3300000>; regulator-name = "+V3.3_SD"; - startup-delay-us = <2000>; + startup-delay-us = <20000>; }; reg_sd1_vqmmc: regulator-sdhci1-vqmmc { diff --git a/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-main-common.dtsi b/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-main-common.dtsi index 9cc0901d58fbf..c2636e624f18b 100644 --- a/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-main-common.dtsi +++ b/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-main-common.dtsi @@ -2378,42 +2378,6 @@ assigned-clock-parents = <&k3_clks 351 4>; }; - watchdog4: watchdog@2240000 { - compatible = "ti,j7-rti-wdt"; - reg = <0x00 0x2240000 0x00 0x100>; - clocks = <&k3_clks 352 0>; - power-domains = <&k3_pds 352 TI_SCI_PD_EXCLUSIVE>; - assigned-clocks = <&k3_clks 352 0>; - assigned-clock-parents = <&k3_clks 352 4>; - }; - - watchdog5: watchdog@2250000 { - compatible = "ti,j7-rti-wdt"; - reg = <0x00 0x2250000 0x00 0x100>; - clocks = <&k3_clks 353 0>; - power-domains = <&k3_pds 353 TI_SCI_PD_EXCLUSIVE>; - assigned-clocks = <&k3_clks 353 0>; - assigned-clock-parents = <&k3_clks 353 4>; - }; - - watchdog6: watchdog@2260000 { - compatible = "ti,j7-rti-wdt"; - reg = <0x00 0x2260000 0x00 0x100>; - clocks = <&k3_clks 354 0>; - power-domains = <&k3_pds 354 TI_SCI_PD_EXCLUSIVE>; - assigned-clocks = <&k3_clks 354 0>; - assigned-clock-parents = <&k3_clks 354 4>; - }; - - watchdog7: watchdog@2270000 { - compatible = "ti,j7-rti-wdt"; - reg = <0x00 0x2270000 0x00 0x100>; - clocks = <&k3_clks 355 0>; - power-domains = <&k3_pds 355 TI_SCI_PD_EXCLUSIVE>; - assigned-clocks = <&k3_clks 355 0>; - assigned-clock-parents = <&k3_clks 355 4>; - }; - /* * The following RTI instances are coupled with MCU R5Fs, c7x and * GPU so keeping them reserved as these will be used by their diff --git a/arch/arm64/boot/dts/ti/k3-j784s4-main.dtsi b/arch/arm64/boot/dts/ti/k3-j784s4-main.dtsi index 0160fe0da9838..78fcd0c40abcf 100644 --- a/arch/arm64/boot/dts/ti/k3-j784s4-main.dtsi +++ b/arch/arm64/boot/dts/ti/k3-j784s4-main.dtsi @@ -6,17 +6,40 @@ */ &cbass_main { - c71_3: dsp@67800000 { - compatible = "ti,j721s2-c71-dsp"; - reg = <0x00 0x67800000 0x00 0x00080000>, - <0x00 0x67e00000 0x00 0x0000c000>; - reg-names = "l2sram", "l1dram"; - resets = <&k3_reset 40 1>; - firmware-name = "j784s4-c71_3-fw"; - ti,sci = <&sms>; - ti,sci-dev-id = <40>; - ti,sci-proc-ids = <0x33 0xff>; - status = "disabled"; + watchdog4: watchdog@2240000 { + compatible = "ti,j7-rti-wdt"; + reg = <0x00 0x2240000 0x00 0x100>; + clocks = <&k3_clks 352 0>; + power-domains = <&k3_pds 352 TI_SCI_PD_EXCLUSIVE>; + assigned-clocks = <&k3_clks 352 0>; + assigned-clock-parents = <&k3_clks 352 4>; + }; + + watchdog5: watchdog@2250000 { + compatible = "ti,j7-rti-wdt"; + reg = <0x00 0x2250000 0x00 0x100>; + clocks = <&k3_clks 353 0>; + power-domains = <&k3_pds 353 TI_SCI_PD_EXCLUSIVE>; + assigned-clocks = <&k3_clks 353 0>; + assigned-clock-parents = <&k3_clks 353 4>; + }; + + watchdog6: watchdog@2260000 { + compatible = "ti,j7-rti-wdt"; + reg = <0x00 0x2260000 0x00 0x100>; + clocks = <&k3_clks 354 0>; + power-domains = <&k3_pds 354 TI_SCI_PD_EXCLUSIVE>; + assigned-clocks = <&k3_clks 354 0>; + assigned-clock-parents = <&k3_clks 354 4>; + }; + + watchdog7: watchdog@2270000 { + compatible = "ti,j7-rti-wdt"; + reg = <0x00 0x2270000 0x00 0x100>; + clocks = <&k3_clks 355 0>; + power-domains = <&k3_pds 355 TI_SCI_PD_EXCLUSIVE>; + assigned-clocks = <&k3_clks 355 0>; + assigned-clock-parents = <&k3_clks 355 4>; }; pcie2_rc: pcie@2920000 { @@ -113,6 +136,19 @@ status = "disabled"; }; }; + + c71_3: dsp@67800000 { + compatible = "ti,j721s2-c71-dsp"; + reg = <0x00 0x67800000 0x00 0x00080000>, + <0x00 0x67e00000 0x00 0x0000c000>; + reg-names = "l2sram", "l1dram"; + resets = <&k3_reset 40 1>; + firmware-name = "j784s4-c71_3-fw"; + ti,sci = <&sms>; + ti,sci-dev-id = <40>; + ti,sci-proc-ids = <0x33 0xff>; + status = "disabled"; + }; }; &scm_conf { diff --git a/arch/arm64/boot/dts/xilinx/zynqmp.dtsi b/arch/arm64/boot/dts/xilinx/zynqmp.dtsi index 938b014ca9231..b55c6b2e8e0e1 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp.dtsi +++ b/arch/arm64/boot/dts/xilinx/zynqmp.dtsi @@ -192,11 +192,6 @@ }; firmware { - optee: optee { - compatible = "linaro,optee-tz"; - method = "smc"; - }; - zynqmp_firmware: zynqmp-firmware { compatible = "xlnx,zynqmp-firmware"; #power-domain-cells = <1>; diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h index 83e03abbb2ca9..8cbd1e96fd50b 100644 --- a/arch/arm64/include/asm/io.h +++ b/arch/arm64/include/asm/io.h @@ -264,19 +264,33 @@ __iowrite64_copy(void __iomem *to, const void *from, size_t count) typedef int (*ioremap_prot_hook_t)(phys_addr_t phys_addr, size_t size, pgprot_t *prot); int arm64_ioremap_prot_hook_register(const ioremap_prot_hook_t hook); +void __iomem *__ioremap_prot(phys_addr_t phys, size_t size, pgprot_t prot); -#define ioremap_prot ioremap_prot +static inline void __iomem *ioremap_prot(phys_addr_t phys, size_t size, + pgprot_t user_prot) +{ + pgprot_t prot; + ptdesc_t user_prot_val = pgprot_val(user_prot); + + if (WARN_ON_ONCE(!(user_prot_val & PTE_USER))) + return NULL; -#define _PAGE_IOREMAP PROT_DEVICE_nGnRE + prot = __pgprot_modify(PAGE_KERNEL, PTE_ATTRINDX_MASK, + user_prot_val & PTE_ATTRINDX_MASK); + return __ioremap_prot(phys, size, prot); +} +#define ioremap_prot ioremap_prot +#define ioremap(addr, size) \ + __ioremap_prot((addr), (size), __pgprot(PROT_DEVICE_nGnRE)) #define ioremap_wc(addr, size) \ - ioremap_prot((addr), (size), __pgprot(PROT_NORMAL_NC)) + __ioremap_prot((addr), (size), __pgprot(PROT_NORMAL_NC)) #define ioremap_np(addr, size) \ - ioremap_prot((addr), (size), __pgprot(PROT_DEVICE_nGnRnE)) + __ioremap_prot((addr), (size), __pgprot(PROT_DEVICE_nGnRnE)) #define ioremap_encrypted(addr, size) \ - ioremap_prot((addr), (size), PAGE_KERNEL) + __ioremap_prot((addr), (size), PAGE_KERNEL) /* * io{read,write}{16,32,64}be() macros @@ -297,7 +311,7 @@ static inline void __iomem *ioremap_cache(phys_addr_t addr, size_t size) if (pfn_is_map_memory(__phys_to_pfn(addr))) return (void __iomem *)__phys_to_virt(addr); - return ioremap_prot(addr, size, __pgprot(PROT_NORMAL)); + return __ioremap_prot(addr, size, __pgprot(PROT_NORMAL)); } /* diff --git a/arch/arm64/include/asm/pgtable-prot.h b/arch/arm64/include/asm/pgtable-prot.h index 85dceb1c66f45..a64a26aaceba4 100644 --- a/arch/arm64/include/asm/pgtable-prot.h +++ b/arch/arm64/include/asm/pgtable-prot.h @@ -164,9 +164,6 @@ static inline bool __pure lpa2_is_enabled(void) #define _PAGE_GCS (_PAGE_DEFAULT | PTE_NG | PTE_UXN | PTE_WRITE | PTE_USER) #define _PAGE_GCS_RO (_PAGE_DEFAULT | PTE_NG | PTE_UXN | PTE_USER) -#define PAGE_GCS __pgprot(_PAGE_GCS) -#define PAGE_GCS_RO __pgprot(_PAGE_GCS_RO) - #define PIE_E0 ( \ PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_GCS), PIE_GCS) | \ PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_GCS_RO), PIE_R) | \ diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 0944e296dd4a4..9016ae8de5c9e 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -175,8 +175,6 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys) __pte(__phys_to_pte_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)) #define pte_none(pte) (!pte_val(pte)) -#define __pte_clear(mm, addr, ptep) \ - __set_pte(ptep, __pte(0)) #define pte_page(pte) (pfn_to_page(pte_pfn(pte))) /* @@ -1316,6 +1314,13 @@ static inline bool pud_user_accessible_page(pud_t pud) /* * Atomic pte/pmd modifications. */ + +static inline void __pte_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + __set_pte(ptep, __pte(0)); +} + static inline int __ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) diff --git a/arch/arm64/include/asm/rwonce.h b/arch/arm64/include/asm/rwonce.h index 97d9256d33c97..ac370a4a01ee9 100644 --- a/arch/arm64/include/asm/rwonce.h +++ b/arch/arm64/include/asm/rwonce.h @@ -58,7 +58,7 @@ default: \ atomic = 0; \ } \ - atomic ? (typeof(*__x))__u.__val : (*(volatile typeof(__x))__x);\ + atomic ? (typeof(*__x))__u.__val : (*(volatile typeof(*__x) *)__x);\ }) #endif /* !BUILD_VDSO */ diff --git a/arch/arm64/kernel/acpi.c b/arch/arm64/kernel/acpi.c index f1cb2447afc9c..b285e174f4f51 100644 --- a/arch/arm64/kernel/acpi.c +++ b/arch/arm64/kernel/acpi.c @@ -377,7 +377,7 @@ void __iomem *acpi_os_ioremap(acpi_physical_address phys, acpi_size size) prot = __acpi_get_writethrough_mem_attribute(); } } - return ioremap_prot(phys, size, prot); + return __ioremap_prot(phys, size, prot); } /* diff --git a/arch/arm64/kernel/kexec_image.c b/arch/arm64/kernel/kexec_image.c index 532d72ea42ee8..b70f4df15a1ae 100644 --- a/arch/arm64/kernel/kexec_image.c +++ b/arch/arm64/kernel/kexec_image.c @@ -41,7 +41,7 @@ static void *image_load(struct kimage *image, struct arm64_image_header *h; u64 flags, value; bool be_image, be_kernel; - struct kexec_buf kbuf; + struct kexec_buf kbuf = {}; unsigned long text_offset, kernel_segment_number; struct kexec_segment *kernel_segment; int ret; diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c index 80a580e019c50..b3801f532b10b 100644 --- a/arch/arm64/kernel/proton-pack.c +++ b/arch/arm64/kernel/proton-pack.c @@ -887,6 +887,7 @@ static u8 spectre_bhb_loop_affected(void) MIDR_ALL_VERSIONS(MIDR_CORTEX_X2), MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1), + MIDR_ALL_VERSIONS(MIDR_HISI_TSV110), {}, }; static const struct midr_range spectre_bhb_k24_list[] = { diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index fd1ba43f2005a..2e9ce5a45ed2d 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -1458,6 +1458,9 @@ static int poe_get(struct task_struct *target, if (!system_supports_poe()) return -EINVAL; + if (target == current) + current->thread.por_el0 = read_sysreg_s(SYS_POR_EL0); + return membuf_write(&to, &target->thread.por_el0, sizeof(target->thread.por_el0)); } diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index ad6133b89e7a4..2964aad0362e4 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -349,6 +349,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS HEAD_SYMBOLS diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c index 834f13fb1fb7d..2d04fb56746ea 100644 --- a/arch/arm64/kvm/emulate-nested.c +++ b/arch/arm64/kvm/emulate-nested.c @@ -2428,7 +2428,7 @@ static u64 kvm_get_sysreg_res0(struct kvm *kvm, enum vcpu_sysreg sr) masks = kvm->arch.sysreg_masks; - return masks->mask[sr - __VNCR_START__].res0; + return masks->mask[sr - __SANITISED_REG_START__].res0; } static bool check_fgt_bit(struct kvm_vcpu *vcpu, enum vcpu_sysreg sr, diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 43bde061b65de..d866f6ba19b5f 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -343,6 +343,7 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc /* No restrictions for non-protected VMs. */ if (!kvm_vm_is_protected(kvm)) { hyp_vm->kvm.arch.flags = host_arch_flags; + hyp_vm->kvm.arch.flags &= ~BIT_ULL(KVM_ARCH_FLAG_ID_REGS_INITIALIZED); bitmap_copy(kvm->arch.vcpu_features, host_kvm->arch.vcpu_features, @@ -469,6 +470,35 @@ static int pkvm_vcpu_init_sve(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *h return ret; } +static int vm_copy_id_regs(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); + const struct kvm *host_kvm = hyp_vm->host_kvm; + struct kvm *kvm = &hyp_vm->kvm; + + if (!test_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &host_kvm->arch.flags)) + return -EINVAL; + + if (test_and_set_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &kvm->arch.flags)) + return 0; + + memcpy(kvm->arch.id_regs, host_kvm->arch.id_regs, sizeof(kvm->arch.id_regs)); + + return 0; +} + +static int pkvm_vcpu_init_sysregs(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + int ret = 0; + + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + kvm_init_pvm_id_regs(&hyp_vcpu->vcpu); + else + ret = vm_copy_id_regs(hyp_vcpu); + + return ret; +} + static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, struct pkvm_hyp_vm *hyp_vm, struct kvm_vcpu *host_vcpu) @@ -488,8 +518,9 @@ static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, hyp_vcpu->vcpu.arch.cflags = READ_ONCE(host_vcpu->arch.cflags); hyp_vcpu->vcpu.arch.mp_state.mp_state = KVM_MP_STATE_STOPPED; - if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) - kvm_init_pvm_id_regs(&hyp_vcpu->vcpu); + ret = pkvm_vcpu_init_sysregs(hyp_vcpu); + if (ret) + goto done; ret = pkvm_vcpu_init_traps(hyp_vcpu); if (ret) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index ec3fbe0b8d525..7b7f3c932dcd5 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1801,6 +1801,9 @@ static u64 __kvm_read_sanitised_id_reg(const struct kvm_vcpu *vcpu, ID_AA64MMFR3_EL1_SCTLRX | ID_AA64MMFR3_EL1_S1POE | ID_AA64MMFR3_EL1_S1PIE; + + if (!system_supports_poe()) + val &= ~ID_AA64MMFR3_EL1_S1POE; break; case SYS_ID_MMFR4_EL1: val &= ~ID_MMFR4_EL1_CCIDX; diff --git a/arch/arm64/lib/delay.c b/arch/arm64/lib/delay.c index cb2062e7e2340..e278e060e78a9 100644 --- a/arch/arm64/lib/delay.c +++ b/arch/arm64/lib/delay.c @@ -23,9 +23,24 @@ static inline unsigned long xloops_to_cycles(unsigned long xloops) return (xloops * loops_per_jiffy * HZ) >> 32; } +/* + * Force the use of CNTVCT_EL0 in order to have the same base as WFxT. + * This avoids some annoying issues when CNTVOFF_EL2 is not reset 0 on a + * KVM host running at EL1 until we do a vcpu_put() on the vcpu. When + * running at EL2, the effective offset is always 0. + * + * Note that userspace cannot change the offset behind our back either, + * as the vcpu mutex is held as long as KVM_RUN is in progress. + */ +static cycles_t notrace __delay_cycles(void) +{ + guard(preempt_notrace)(); + return __arch_counter_get_cntvct_stable(); +} + void __delay(unsigned long cycles) { - cycles_t start = get_cycles(); + cycles_t start = __delay_cycles(); if (alternative_has_cap_unlikely(ARM64_HAS_WFXT)) { u64 end = start + cycles; @@ -35,17 +50,17 @@ void __delay(unsigned long cycles) * early, use a WFET loop to complete the delay. */ wfit(end); - while ((get_cycles() - start) < cycles) + while ((__delay_cycles() - start) < cycles) wfet(end); } else if (arch_timer_evtstrm_available()) { const cycles_t timer_evt_period = USECS_TO_CYCLES(ARCH_TIMER_EVT_STREAM_PERIOD_US); - while ((get_cycles() - start + timer_evt_period) < cycles) + while ((__delay_cycles() - start + timer_evt_period) < cycles) wfe(); } - while ((get_cycles() - start) < cycles) + while ((__delay_cycles() - start) < cycles) cpu_relax(); } EXPORT_SYMBOL(__delay); diff --git a/arch/arm64/mm/gcs.c b/arch/arm64/mm/gcs.c index 6e93f78de79b1..04a23a497f205 100644 --- a/arch/arm64/mm/gcs.c +++ b/arch/arm64/mm/gcs.c @@ -199,8 +199,8 @@ int arch_set_shadow_stack_status(struct task_struct *task, unsigned long arg) size = gcs_size(0); gcs = alloc_gcs(0, size); - if (!gcs) - return -ENOMEM; + if (IS_ERR_VALUE(gcs)) + return gcs; task->thread.gcspr_el0 = gcs + size - sizeof(u64); task->thread.gcs_base = gcs; diff --git a/arch/arm64/mm/ioremap.c b/arch/arm64/mm/ioremap.c index 10e246f112710..1e4794a2af7d6 100644 --- a/arch/arm64/mm/ioremap.c +++ b/arch/arm64/mm/ioremap.c @@ -14,8 +14,8 @@ int arm64_ioremap_prot_hook_register(ioremap_prot_hook_t hook) return 0; } -void __iomem *ioremap_prot(phys_addr_t phys_addr, size_t size, - pgprot_t pgprot) +void __iomem *__ioremap_prot(phys_addr_t phys_addr, size_t size, + pgprot_t pgprot) { unsigned long last_addr = phys_addr + size - 1; @@ -38,7 +38,7 @@ void __iomem *ioremap_prot(phys_addr_t phys_addr, size_t size, return generic_ioremap_prot(phys_addr, size, pgprot); } -EXPORT_SYMBOL(ioremap_prot); +EXPORT_SYMBOL(__ioremap_prot); /* * Must be called after early_fixmap_init diff --git a/arch/arm64/mm/mmap.c b/arch/arm64/mm/mmap.c index 08ee177432c2f..75f343009b4b1 100644 --- a/arch/arm64/mm/mmap.c +++ b/arch/arm64/mm/mmap.c @@ -34,6 +34,8 @@ static pgprot_t protection_map[16] __ro_after_init = { [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = PAGE_SHARED_EXEC }; +static ptdesc_t gcs_page_prot __ro_after_init = _PAGE_GCS_RO; + /* * You really shouldn't be using read() or write() on /dev/mem. This might go * away in the future. @@ -73,9 +75,11 @@ static int __init adjust_protection_map(void) protection_map[VM_EXEC | VM_SHARED] = PAGE_EXECONLY; } - if (lpa2_is_enabled()) + if (lpa2_is_enabled()) { for (int i = 0; i < ARRAY_SIZE(protection_map); i++) pgprot_val(protection_map[i]) &= ~PTE_SHARED; + gcs_page_prot &= ~PTE_SHARED; + } return 0; } @@ -87,7 +91,7 @@ pgprot_t vm_get_page_prot(vm_flags_t vm_flags) /* Short circuit GCS to avoid bloating the table. */ if (system_supports_gcs() && (vm_flags & VM_SHADOW_STACK)) { - prot = _PAGE_GCS_RO; + prot = gcs_page_prot; } else { prot = pgprot_val(protection_map[vm_flags & (VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)]); diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 0dfefeedfe56c..107eb71b533a0 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2122,7 +2122,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) extable_offset = round_up(prog_size + PLT_TARGET_SIZE, extable_align); image_size = extable_offset + extable_size; ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, - sizeof(u32), &header, &image_ptr, + sizeof(u64), &header, &image_ptr, jit_fill_hole); if (!ro_header) { prog = orig_prog; @@ -2939,7 +2939,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, u64 plt_target = 0ULL; bool poking_bpf_entry; - if (!__bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf)) + if (!bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf)) /* Only poking bpf text is supported. Since kernel function * entry is set up by ftrace, we reply on ftrace to poke kernel * functions. diff --git a/arch/csky/kernel/vmlinux.lds.S b/arch/csky/kernel/vmlinux.lds.S index d718961786d24..81943981b3af4 100644 --- a/arch/csky/kernel/vmlinux.lds.S +++ b/arch/csky/kernel/vmlinux.lds.S @@ -109,6 +109,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/hexagon/kernel/vmlinux.lds.S b/arch/hexagon/kernel/vmlinux.lds.S index 1150b77fa281c..aae22283b5e00 100644 --- a/arch/hexagon/kernel/vmlinux.lds.S +++ b/arch/hexagon/kernel/vmlinux.lds.S @@ -62,6 +62,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS .hexagon.attributes 0 : { *(.hexagon.attributes) } diff --git a/arch/loongarch/include/asm/setup.h b/arch/loongarch/include/asm/setup.h index 3c2fb16b11b64..f81375e5e89c0 100644 --- a/arch/loongarch/include/asm/setup.h +++ b/arch/loongarch/include/asm/setup.h @@ -7,6 +7,7 @@ #define _LOONGARCH_SETUP_H #include +#include #include #include @@ -14,6 +15,8 @@ extern unsigned long eentry; extern unsigned long tlbrentry; +extern unsigned long pcpu_handlers[NR_CPUS]; +extern long exception_handlers[VECSIZE * 128 / sizeof(long)]; extern char init_command_line[COMMAND_LINE_SIZE]; extern void tlb_init(int cpu); extern void cpu_cache_init(void); diff --git a/arch/loongarch/include/asm/topology.h b/arch/loongarch/include/asm/topology.h index f06e7ff25bb7c..6b79d6183085a 100644 --- a/arch/loongarch/include/asm/topology.h +++ b/arch/loongarch/include/asm/topology.h @@ -12,7 +12,7 @@ extern cpumask_t cpus_on_node[]; -#define cpumask_of_node(node) (&cpus_on_node[node]) +#define cpumask_of_node(node) ((node) == NUMA_NO_NODE ? cpu_all_mask : &cpus_on_node[node]) struct pci_bus; extern int pcibus_to_node(struct pci_bus *); diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c index 20cb6f3064568..2b260d15b2e25 100644 --- a/arch/loongarch/kernel/setup.c +++ b/arch/loongarch/kernel/setup.c @@ -421,6 +421,7 @@ static void __init arch_mem_init(char **cmdline_p) PFN_UP(__pa_symbol(&__nosave_end))); memblock_dump_all(); + memblock_set_bottom_up(false); early_memtest(PFN_PHYS(ARCH_PFN_OFFSET), PFN_PHYS(max_low_pfn)); } diff --git a/arch/loongarch/kernel/unwind_orc.c b/arch/loongarch/kernel/unwind_orc.c index 0d5fa64a22252..85c2fcb76930c 100644 --- a/arch/loongarch/kernel/unwind_orc.c +++ b/arch/loongarch/kernel/unwind_orc.c @@ -358,13 +358,21 @@ static bool is_entry_func(unsigned long addr) static inline unsigned long bt_address(unsigned long ra) { - extern unsigned long eentry; +#if defined(CONFIG_NUMA) && !defined(CONFIG_PREEMPT_RT) + int cpu; + int vec_sz = sizeof(exception_handlers); - if (__kernel_text_address(ra)) - return ra; + for_each_possible_cpu(cpu) { + if (!pcpu_handlers[cpu]) + continue; - if (__module_text_address(ra)) - return ra; + if (ra >= pcpu_handlers[cpu] && + ra < pcpu_handlers[cpu] + vec_sz) { + ra = ra + eentry - pcpu_handlers[cpu]; + break; + } + } +#endif if (ra >= eentry && ra < eentry + EXCCODE_INT_END * VECSIZE) { unsigned long func; @@ -383,10 +391,13 @@ static inline unsigned long bt_address(unsigned long ra) break; } - return func + offset; + ra = func + offset; } - return ra; + if (__kernel_text_address(ra)) + return ra; + + return 0; } bool unwind_next_frame(struct unwind_state *state) @@ -508,13 +519,10 @@ bool unwind_next_frame(struct unwind_state *state) state->pc = bt_address(pc); if (!state->pc) { - pr_err("cannot find unwind pc at %p\n", (void *)pc); + pr_err("cannot find unwind pc at %px\n", (void *)pc); goto err; } - if (!__kernel_text_address(state->pc)) - goto err; - return true; err: diff --git a/arch/loongarch/kernel/unwind_prologue.c b/arch/loongarch/kernel/unwind_prologue.c index 729e775bd40dd..da07acad7973a 100644 --- a/arch/loongarch/kernel/unwind_prologue.c +++ b/arch/loongarch/kernel/unwind_prologue.c @@ -23,10 +23,6 @@ extern const int unwind_hint_lasx; extern const int unwind_hint_lbt; extern const int unwind_hint_ri; extern const int unwind_hint_watch; -extern unsigned long eentry; -#ifdef CONFIG_NUMA -extern unsigned long pcpu_handlers[NR_CPUS]; -#endif static inline bool scan_handlers(unsigned long entry_offset) { @@ -65,7 +61,7 @@ static inline bool scan_handlers(unsigned long entry_offset) static inline bool fix_exception(unsigned long pc) { -#ifdef CONFIG_NUMA +#if defined(CONFIG_NUMA) && !defined(CONFIG_PREEMPT_RT) int cpu; for_each_possible_cpu(cpu) { diff --git a/arch/loongarch/kernel/vmlinux.lds.S b/arch/loongarch/kernel/vmlinux.lds.S index 08ea921cdec16..d0e1377a041d6 100644 --- a/arch/loongarch/kernel/vmlinux.lds.S +++ b/arch/loongarch/kernel/vmlinux.lds.S @@ -147,6 +147,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS #ifdef CONFIG_EFI_STUB diff --git a/arch/loongarch/mm/kasan_init.c b/arch/loongarch/mm/kasan_init.c index 170da98ad4f55..0fc02ca064573 100644 --- a/arch/loongarch/mm/kasan_init.c +++ b/arch/loongarch/mm/kasan_init.c @@ -40,39 +40,43 @@ static pgd_t kasan_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); #define __pte_none(early, pte) (early ? pte_none(pte) : \ ((pte_val(pte) & _PFN_MASK) == (unsigned long)__pa(kasan_early_shadow_page))) -void *kasan_mem_to_shadow(const void *addr) +static void *mem_to_shadow(const void *addr) { - if (!kasan_enabled()) { + unsigned long offset = 0; + unsigned long maddr = (unsigned long)addr; + unsigned long xrange = (maddr >> XRANGE_SHIFT) & 0xffff; + + if (maddr >= FIXADDR_START) return (void *)(kasan_early_shadow_page); - } else { - unsigned long maddr = (unsigned long)addr; - unsigned long xrange = (maddr >> XRANGE_SHIFT) & 0xffff; - unsigned long offset = 0; - - if (maddr >= FIXADDR_START) - return (void *)(kasan_early_shadow_page); - - maddr &= XRANGE_SHADOW_MASK; - switch (xrange) { - case XKPRANGE_CC_SEG: - offset = XKPRANGE_CC_SHADOW_OFFSET; - break; - case XKPRANGE_UC_SEG: - offset = XKPRANGE_UC_SHADOW_OFFSET; - break; - case XKPRANGE_WC_SEG: - offset = XKPRANGE_WC_SHADOW_OFFSET; - break; - case XKVRANGE_VC_SEG: - offset = XKVRANGE_VC_SHADOW_OFFSET; - break; - default: - WARN_ON(1); - return NULL; - } - return (void *)((maddr >> KASAN_SHADOW_SCALE_SHIFT) + offset); + maddr &= XRANGE_SHADOW_MASK; + switch (xrange) { + case XKPRANGE_CC_SEG: + offset = XKPRANGE_CC_SHADOW_OFFSET; + break; + case XKPRANGE_UC_SEG: + offset = XKPRANGE_UC_SHADOW_OFFSET; + break; + case XKPRANGE_WC_SEG: + offset = XKPRANGE_WC_SHADOW_OFFSET; + break; + case XKVRANGE_VC_SEG: + offset = XKVRANGE_VC_SHADOW_OFFSET; + break; + default: + WARN_ON(1); + return NULL; } + + return (void *)((maddr >> KASAN_SHADOW_SCALE_SHIFT) + offset); +} + +void *kasan_mem_to_shadow(const void *addr) +{ + if (kasan_enabled()) + return mem_to_shadow(addr); + else + return (void *)(kasan_early_shadow_page); } const void *kasan_shadow_to_mem(const void *shadow_addr) @@ -293,11 +297,8 @@ void __init kasan_init(void) /* Maps everything to a single page of zeroes */ kasan_pgd_populate(KASAN_SHADOW_START, KASAN_SHADOW_END, NUMA_NO_NODE, true); - kasan_populate_early_shadow(kasan_mem_to_shadow((void *)VMALLOC_START), - kasan_mem_to_shadow((void *)KFENCE_AREA_END)); - - /* Enable KASAN here before kasan_mem_to_shadow(). */ - kasan_init_generic(); + kasan_populate_early_shadow(mem_to_shadow((void *)VMALLOC_START), + mem_to_shadow((void *)KFENCE_AREA_END)); /* Populate the linear mapping */ for_each_mem_range(i, &pa_start, &pa_end) { @@ -307,13 +308,13 @@ void __init kasan_init(void) if (start >= end) break; - kasan_map_populate((unsigned long)kasan_mem_to_shadow(start), - (unsigned long)kasan_mem_to_shadow(end), NUMA_NO_NODE); + kasan_map_populate((unsigned long)mem_to_shadow(start), + (unsigned long)mem_to_shadow(end), NUMA_NO_NODE); } /* Populate modules mapping */ - kasan_map_populate((unsigned long)kasan_mem_to_shadow((void *)MODULES_VADDR), - (unsigned long)kasan_mem_to_shadow((void *)MODULES_END), NUMA_NO_NODE); + kasan_map_populate((unsigned long)mem_to_shadow((void *)MODULES_VADDR), + (unsigned long)mem_to_shadow((void *)MODULES_END), NUMA_NO_NODE); /* * KAsan may reuse the contents of kasan_early_shadow_pte directly, so we * should make sure that it maps the zero page read-only. @@ -328,4 +329,5 @@ void __init kasan_init(void) /* At this point kasan is fully initialized. Enable error messages */ init_task.kasan_depth = 0; + kasan_init_generic(); } diff --git a/arch/loongarch/mm/tlb.c b/arch/loongarch/mm/tlb.c index 3b427b319db21..24add95ecb65e 100644 --- a/arch/loongarch/mm/tlb.c +++ b/arch/loongarch/mm/tlb.c @@ -202,7 +202,7 @@ void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep local_irq_restore(flags); } -static void setup_ptwalker(void) +static void __no_sanitize_address setup_ptwalker(void) { unsigned long pwctl0, pwctl1; unsigned long pgd_i = 0, pgd_w = 0; @@ -260,7 +260,6 @@ static void output_pgtable_bits_defines(void) #ifdef CONFIG_NUMA unsigned long pcpu_handlers[NR_CPUS]; #endif -extern long exception_handlers[VECSIZE * 128 / sizeof(long)]; static void setup_tlb_handler(int cpu) { diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 87ff025137873..e9d666508ae28 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -1318,7 +1318,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, /* Only poking bpf text is supported. Since kernel function entry * is set up by ftrace, we rely on ftrace to poke kernel functions. */ - if (!__bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf)) + if (!bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf)) return -ENOTSUPP; image = ip - offset; diff --git a/arch/m68k/kernel/vmlinux-nommu.lds b/arch/m68k/kernel/vmlinux-nommu.lds index 2624fc18c131f..45d7f4b0177b4 100644 --- a/arch/m68k/kernel/vmlinux-nommu.lds +++ b/arch/m68k/kernel/vmlinux-nommu.lds @@ -85,6 +85,7 @@ SECTIONS { _end = .; STABS_DEBUG + MODINFO ELF_DETAILS /* Sections to be discarded */ diff --git a/arch/m68k/kernel/vmlinux-std.lds b/arch/m68k/kernel/vmlinux-std.lds index 1ccdd04ae4624..7326586afe15f 100644 --- a/arch/m68k/kernel/vmlinux-std.lds +++ b/arch/m68k/kernel/vmlinux-std.lds @@ -58,6 +58,7 @@ SECTIONS _end = . ; STABS_DEBUG + MODINFO ELF_DETAILS /* Sections to be discarded */ diff --git a/arch/m68k/kernel/vmlinux-sun3.lds b/arch/m68k/kernel/vmlinux-sun3.lds index f13ddcc2af5c2..1b19fef201fba 100644 --- a/arch/m68k/kernel/vmlinux-sun3.lds +++ b/arch/m68k/kernel/vmlinux-sun3.lds @@ -51,6 +51,7 @@ __init_begin = .; _end = . ; STABS_DEBUG + MODINFO ELF_DETAILS /* Sections to be discarded */ diff --git a/arch/m68k/lib/memmove.c b/arch/m68k/lib/memmove.c index 6519f7f349f66..e33f00b02e4c0 100644 --- a/arch/m68k/lib/memmove.c +++ b/arch/m68k/lib/memmove.c @@ -24,6 +24,15 @@ void *memmove(void *dest, const void *src, size_t n) src = csrc; n--; } +#if defined(CONFIG_M68000) + if ((long)src & 1) { + char *cdest = dest; + const char *csrc = src; + for (; n; n--) + *cdest++ = *csrc++; + return xdest; + } +#endif if (n > 2 && (long)dest & 2) { short *sdest = dest; const short *ssrc = src; @@ -66,6 +75,15 @@ void *memmove(void *dest, const void *src, size_t n) src = csrc; n--; } +#if defined(CONFIG_M68000) + if ((long)src & 1) { + char *cdest = dest; + const char *csrc = src; + for (; n; n--) + *--cdest = *--csrc; + return xdest; + } +#endif if (n > 2 && (long)dest & 2) { short *sdest = dest; const short *ssrc = src; diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index e8683f58fd3e2..83a6b68d8a39a 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -1408,7 +1408,6 @@ config CPU_LOONGSON32 select CPU_MIPS32 select CPU_MIPSR2 select CPU_HAS_PREFETCH - select CPU_HAS_LOAD_STORE_LR select CPU_SUPPORTS_32BIT_KERNEL select CPU_SUPPORTS_HIGHMEM select CPU_SUPPORTS_CPUFREQ diff --git a/arch/mips/include/asm/mach-loongson2ef/loongson.h b/arch/mips/include/asm/mach-loongson2ef/loongson.h index 4a098fb102325..0e586787eb87a 100644 --- a/arch/mips/include/asm/mach-loongson2ef/loongson.h +++ b/arch/mips/include/asm/mach-loongson2ef/loongson.h @@ -324,4 +324,10 @@ extern unsigned long _loongson_addrwincfg_base; #endif /* ! CONFIG_CPU_SUPPORTS_ADDRWINCFG */ +#ifdef CONFIG_PCI +void loongson2ef_pcibios_init(void); +#else +static inline void loongson2ef_pcibios_init(void) { } +#endif + #endif /* __ASM_MACH_LOONGSON2EF_LOONGSON_H */ diff --git a/arch/mips/include/asm/mach-loongson64/topology.h b/arch/mips/include/asm/mach-loongson64/topology.h index 3414a1fd17835..89bb4deab98a6 100644 --- a/arch/mips/include/asm/mach-loongson64/topology.h +++ b/arch/mips/include/asm/mach-loongson64/topology.h @@ -7,7 +7,7 @@ #define cpu_to_node(cpu) (cpu_logical_map(cpu) >> 2) extern cpumask_t __node_cpumask[]; -#define cpumask_of_node(node) (&__node_cpumask[node]) +#define cpumask_of_node(node) ((node) == NUMA_NO_NODE ? cpu_all_mask : &__node_cpumask[node]) struct pci_bus; extern int pcibus_to_node(struct pci_bus *); diff --git a/arch/mips/kernel/relocate.c b/arch/mips/kernel/relocate.c index 7f1c136ad8506..59833210542ff 100644 --- a/arch/mips/kernel/relocate.c +++ b/arch/mips/kernel/relocate.c @@ -420,7 +420,20 @@ void *__init relocate_kernel(void) goto out; /* The current thread is now within the relocated image */ +#ifndef CONFIG_CC_IS_CLANG __current_thread_info = RELOCATED(&init_thread_union); +#else + /* + * LLVM may wrongly restore $gp ($28) in epilog even if it's + * intentionally modified. Work around this by using inline + * assembly to assign $gp. $gp couldn't be listed as output or + * clobber, or LLVM will still restore its original value. + * See also LLVM upstream issue + * https://github.com/llvm/llvm-project/issues/176546 + */ + asm volatile("move $28, %0" : : + "r" (RELOCATED(&init_thread_union))); +#endif /* Return the new kernel's entry point */ kernel_entry = RELOCATED(start_kernel); diff --git a/arch/mips/kernel/vmlinux.lds.S b/arch/mips/kernel/vmlinux.lds.S index 2b708fac8d2c1..579b2cc1995ae 100644 --- a/arch/mips/kernel/vmlinux.lds.S +++ b/arch/mips/kernel/vmlinux.lds.S @@ -217,6 +217,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS /* These must appear regardless of . */ diff --git a/arch/mips/loongson2ef/common/pci.c b/arch/mips/loongson2ef/common/pci.c index 7d9ea51e8c01e..0f11392104bfd 100644 --- a/arch/mips/loongson2ef/common/pci.c +++ b/arch/mips/loongson2ef/common/pci.c @@ -17,7 +17,7 @@ static struct resource loongson_pci_mem_resource = { static struct resource loongson_pci_io_resource = { .name = "pci io space", - .start = LOONGSON_PCI_IO_START, + .start = 0x00000000UL, /* See loongson2ef_pcibios_init(). */ .end = IO_SPACE_LIMIT, .flags = IORESOURCE_IO, }; @@ -73,15 +73,19 @@ static void __init setup_pcimap(void) #endif } -static int __init pcibios_init(void) +void __init loongson2ef_pcibios_init(void) { setup_pcimap(); + /* + * ISA-mode only IDE controllers have a hard dependency on ISA IO ports. + * + * Claim them by setting PCI IO space to start at 0x00000000, and set + * PCIBIOS_MIN_IO to prevent non-legacy PCI devices from touching + * reserved regions. + */ + PCIBIOS_MIN_IO = LOONGSON_PCI_IO_START; + loongson_pci_controller.io_map_base = mips_io_port_base; register_pci_controller(&loongson_pci_controller); - - - return 0; } - -arch_initcall(pcibios_init); diff --git a/arch/mips/loongson2ef/common/setup.c b/arch/mips/loongson2ef/common/setup.c index 4fd27f4f90edb..a639e35acce59 100644 --- a/arch/mips/loongson2ef/common/setup.c +++ b/arch/mips/loongson2ef/common/setup.c @@ -27,4 +27,5 @@ EXPORT_SYMBOL(__wbflush); void __init plat_mem_setup(void) { + loongson2ef_pcibios_init(); } diff --git a/arch/mips/rb532/devices.c b/arch/mips/rb532/devices.c index b7f6f782d9a13..ffa4d38ca95df 100644 --- a/arch/mips/rb532/devices.c +++ b/arch/mips/rb532/devices.c @@ -212,11 +212,12 @@ static struct platform_device rb532_wdt = { static struct plat_serial8250_port rb532_uart_res[] = { { .type = PORT_16550A, - .membase = (char *)KSEG1ADDR(REGBASE + UART0BASE), + .mapbase = REGBASE + UART0BASE, + .mapsize = 0x1000, .irq = UART0_IRQ, .regshift = 2, .iotype = UPIO_MEM, - .flags = UPF_BOOT_AUTOCONF, + .flags = UPF_BOOT_AUTOCONF | UPF_IOREMAP, }, { .flags = 0, diff --git a/arch/nios2/kernel/vmlinux.lds.S b/arch/nios2/kernel/vmlinux.lds.S index 37b9580550646..206f92445bfad 100644 --- a/arch/nios2/kernel/vmlinux.lds.S +++ b/arch/nios2/kernel/vmlinux.lds.S @@ -57,6 +57,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/openrisc/include/asm/barrier.h b/arch/openrisc/include/asm/barrier.h index 7538294721bed..8e592c9909023 100644 --- a/arch/openrisc/include/asm/barrier.h +++ b/arch/openrisc/include/asm/barrier.h @@ -4,6 +4,8 @@ #define mb() asm volatile ("l.msync" ::: "memory") +#define nop() asm volatile ("l.nop") + #include #endif /* __ASM_BARRIER_H */ diff --git a/arch/openrisc/kernel/vmlinux.lds.S b/arch/openrisc/kernel/vmlinux.lds.S index 049bff45f6126..9b29c3211774c 100644 --- a/arch/openrisc/kernel/vmlinux.lds.S +++ b/arch/openrisc/kernel/vmlinux.lds.S @@ -101,6 +101,7 @@ SECTIONS /* Throw in the debugging sections */ STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS /* Sections to be discarded -- must be last */ diff --git a/arch/parisc/boot/compressed/vmlinux.lds.S b/arch/parisc/boot/compressed/vmlinux.lds.S index ab7b439908578..87d24cc824b66 100644 --- a/arch/parisc/boot/compressed/vmlinux.lds.S +++ b/arch/parisc/boot/compressed/vmlinux.lds.S @@ -90,6 +90,7 @@ SECTIONS /* Sections to be discarded */ DISCARDS /DISCARD/ : { + *(.modinfo) #ifdef CONFIG_64BIT /* temporary hack until binutils is fixed to not emit these * for static binaries diff --git a/arch/parisc/kernel/drivers.c b/arch/parisc/kernel/drivers.c index 1f8936fc22928..427e1334f8352 100644 --- a/arch/parisc/kernel/drivers.c +++ b/arch/parisc/kernel/drivers.c @@ -435,7 +435,7 @@ static struct parisc_device * __init create_tree_node(char id, dev->dev.dma_mask = &dev->dma_mask; dev->dev.coherent_dma_mask = dev->dma_mask; if (device_register(&dev->dev)) { - kfree(dev); + put_device(&dev->dev); return NULL; } diff --git a/arch/parisc/kernel/process.c b/arch/parisc/kernel/process.c index e64ab5d2a40d6..703644e5bfc4a 100644 --- a/arch/parisc/kernel/process.c +++ b/arch/parisc/kernel/process.c @@ -85,6 +85,9 @@ void machine_restart(char *cmd) #endif /* set up a new led state on systems shipped with a LED State panel */ pdc_chassis_send_status(PDC_CHASSIS_DIRECT_SHUTDOWN); + + /* prevent interrupts during reboot */ + set_eiem(0); /* "Normal" system reset */ pdc_do_reset(); diff --git a/arch/parisc/kernel/vmlinux.lds.S b/arch/parisc/kernel/vmlinux.lds.S index b445e47903cfd..0ca93d6d72354 100644 --- a/arch/parisc/kernel/vmlinux.lds.S +++ b/arch/parisc/kernel/vmlinux.lds.S @@ -165,6 +165,7 @@ SECTIONS _end = . ; STABS_DEBUG + MODINFO ELF_DETAILS .note 0 : { *(.note) } diff --git a/arch/powerpc/include/asm/eeh.h b/arch/powerpc/include/asm/eeh.h index 5e34611de9ef4..b7ebb4ac2c710 100644 --- a/arch/powerpc/include/asm/eeh.h +++ b/arch/powerpc/include/asm/eeh.h @@ -289,6 +289,8 @@ void eeh_pe_dev_traverse(struct eeh_pe *root, void eeh_pe_restore_bars(struct eeh_pe *pe); const char *eeh_pe_loc_get(struct eeh_pe *pe); struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe); +const char *eeh_pe_loc_get_bus(struct pci_bus *bus); +struct pci_bus *eeh_pe_bus_get_nolock(struct eeh_pe *pe); void eeh_show_enabled(void); int __init eeh_init(struct eeh_ops *ops); diff --git a/arch/powerpc/include/asm/kup.h b/arch/powerpc/include/asm/kup.h index dab63b82a8d4f..f2009d7c8cfa7 100644 --- a/arch/powerpc/include/asm/kup.h +++ b/arch/powerpc/include/asm/kup.h @@ -134,7 +134,6 @@ static __always_inline void kuap_assert_locked(void) static __always_inline void allow_read_from_user(const void __user *from, unsigned long size) { - barrier_nospec(); allow_user_access(NULL, from, size, KUAP_READ); } @@ -146,7 +145,6 @@ static __always_inline void allow_write_to_user(void __user *to, unsigned long s static __always_inline void allow_read_write_user(void __user *to, const void __user *from, unsigned long size) { - barrier_nospec(); allow_user_access(to, from, size, KUAP_READ_WRITE); } diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index 4f5a46a77fa2b..3987a5c33558b 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -301,6 +301,7 @@ do { \ __typeof__(sizeof(*(ptr))) __gu_size = sizeof(*(ptr)); \ \ might_fault(); \ + barrier_nospec(); \ allow_read_from_user(__gu_addr, __gu_size); \ __get_user_size_allowed(__gu_val, __gu_addr, __gu_size, __gu_err); \ prevent_read_from_user(__gu_addr, __gu_size); \ @@ -329,6 +330,7 @@ raw_copy_in_user(void __user *to, const void __user *from, unsigned long n) { unsigned long ret; + barrier_nospec(); allow_read_write_user(to, from, n); ret = __copy_tofrom_user(to, from, n); prevent_read_write_user(to, from, n); @@ -415,6 +417,7 @@ static __must_check __always_inline bool user_access_begin(const void __user *pt might_fault(); + barrier_nospec(); allow_read_write_user((void __user *)ptr, ptr, len); return true; } @@ -431,6 +434,7 @@ user_read_access_begin(const void __user *ptr, size_t len) might_fault(); + barrier_nospec(); allow_read_from_user(ptr, len); return true; } diff --git a/arch/powerpc/kernel/eeh_driver.c b/arch/powerpc/kernel/eeh_driver.c index ef78ff77cf8f2..028f691585323 100644 --- a/arch/powerpc/kernel/eeh_driver.c +++ b/arch/powerpc/kernel/eeh_driver.c @@ -846,7 +846,7 @@ void eeh_handle_normal_event(struct eeh_pe *pe) pci_lock_rescan_remove(); - bus = eeh_pe_bus_get(pe); + bus = eeh_pe_bus_get_nolock(pe); if (!bus) { pr_err("%s: Cannot find PCI bus for PHB#%x-PE#%x\n", __func__, pe->phb->global_number, pe->addr); @@ -886,14 +886,15 @@ void eeh_handle_normal_event(struct eeh_pe *pe) /* Log the event */ if (pe->type & EEH_PE_PHB) { pr_err("EEH: Recovering PHB#%x, location: %s\n", - pe->phb->global_number, eeh_pe_loc_get(pe)); + pe->phb->global_number, eeh_pe_loc_get_bus(bus)); } else { struct eeh_pe *phb_pe = eeh_phb_pe_get(pe->phb); pr_err("EEH: Recovering PHB#%x-PE#%x\n", pe->phb->global_number, pe->addr); pr_err("EEH: PE location: %s, PHB location: %s\n", - eeh_pe_loc_get(pe), eeh_pe_loc_get(phb_pe)); + eeh_pe_loc_get_bus(bus), + eeh_pe_loc_get_bus(eeh_pe_bus_get_nolock(phb_pe))); } #ifdef CONFIG_STACKTRACE @@ -1098,7 +1099,7 @@ void eeh_handle_normal_event(struct eeh_pe *pe) eeh_pe_state_clear(pe, EEH_PE_PRI_BUS, true); eeh_pe_dev_mode_mark(pe, EEH_DEV_REMOVED); - bus = eeh_pe_bus_get(pe); + bus = eeh_pe_bus_get_nolock(pe); if (bus) pci_hp_remove_devices(bus); else @@ -1222,7 +1223,7 @@ void eeh_handle_special_event(void) (phb_pe->state & EEH_PE_RECOVERING)) continue; - bus = eeh_pe_bus_get(phb_pe); + bus = eeh_pe_bus_get_nolock(phb_pe); if (!bus) { pr_err("%s: Cannot find PCI bus for " "PHB#%x-PE#%x\n", diff --git a/arch/powerpc/kernel/eeh_pe.c b/arch/powerpc/kernel/eeh_pe.c index e740101fadf3b..040e8f69a4aa8 100644 --- a/arch/powerpc/kernel/eeh_pe.c +++ b/arch/powerpc/kernel/eeh_pe.c @@ -812,6 +812,24 @@ void eeh_pe_restore_bars(struct eeh_pe *pe) const char *eeh_pe_loc_get(struct eeh_pe *pe) { struct pci_bus *bus = eeh_pe_bus_get(pe); + return eeh_pe_loc_get_bus(bus); +} + +/** + * eeh_pe_loc_get_bus - Retrieve location code binding to the given PCI bus + * @bus: PCI bus + * + * Retrieve the location code associated with the given PCI bus. If the bus + * is a root bus, the location code is fetched from the PHB device tree node + * or root port. Otherwise, the location code is obtained from the device + * tree node of the upstream bridge of the bus. The function walks up the + * bus hierarchy if necessary, checking each node for the appropriate + * location code property ("ibm,io-base-loc-code" for root buses, + * "ibm,slot-location-code" for others). If no location code is found, + * returns "N/A". + */ +const char *eeh_pe_loc_get_bus(struct pci_bus *bus) +{ struct device_node *dn; const char *loc = NULL; @@ -838,8 +856,9 @@ const char *eeh_pe_loc_get(struct eeh_pe *pe) } /** - * eeh_pe_bus_get - Retrieve PCI bus according to the given PE + * _eeh_pe_bus_get - Retrieve PCI bus according to the given PE * @pe: EEH PE + * @do_lock: Is the caller already held the pci_lock_rescan_remove? * * Retrieve the PCI bus according to the given PE. Basically, * there're 3 types of PEs: PHB/Bus/Device. For PHB PE, the @@ -847,7 +866,7 @@ const char *eeh_pe_loc_get(struct eeh_pe *pe) * returned for BUS PE. However, we don't have associated PCI * bus for DEVICE PE. */ -struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe) +static struct pci_bus *_eeh_pe_bus_get(struct eeh_pe *pe, bool do_lock) { struct eeh_dev *edev; struct pci_dev *pdev; @@ -862,11 +881,58 @@ struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe) /* Retrieve the parent PCI bus of first (top) PCI device */ edev = list_first_entry_or_null(&pe->edevs, struct eeh_dev, entry); - pci_lock_rescan_remove(); + if (do_lock) + pci_lock_rescan_remove(); pdev = eeh_dev_to_pci_dev(edev); if (pdev) bus = pdev->bus; - pci_unlock_rescan_remove(); + if (do_lock) + pci_unlock_rescan_remove(); return bus; } + +/** + * eeh_pe_bus_get - Retrieve PCI bus associated with the given EEH PE, locking + * if needed + * @pe: Pointer to the EEH PE + * + * This function is a wrapper around _eeh_pe_bus_get(), which retrieves the PCI + * bus associated with the provided EEH PE structure. It acquires the PCI + * rescans lock to ensure safe access to shared data during the retrieval + * process. This function should be used when the caller requires the PCI bus + * while holding the rescan/remove lock, typically during operations that modify + * or inspect PCIe device state in a safe manner. + * + * RETURNS: + * A pointer to the PCI bus associated with the EEH PE, or NULL if none found. + */ + +struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe) +{ + return _eeh_pe_bus_get(pe, true); +} + +/** + * eeh_pe_bus_get_nolock - Retrieve PCI bus associated with the given EEH PE + * without locking + * @pe: Pointer to the EEH PE + * + * This function is a variant of _eeh_pe_bus_get() that retrieves the PCI bus + * associated with the specified EEH PE without acquiring the + * pci_lock_rescan_remove lock. It should only be used when the caller can + * guarantee safe access to PE structures without the need for that lock, + * typically in contexts where the lock is already held locking is otherwise + * managed. + * + * RETURNS: + * pointer to the PCI bus associated with the EEH PE, or NULL if none is found. + * + * NOTE: + * Use this function carefully to avoid race conditions and data corruption. + */ + +struct pci_bus *eeh_pe_bus_get_nolock(struct eeh_pe *pe) +{ + return _eeh_pe_bus_get(pe, false); +} diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 68edb66c2964b..0cd9c0c21af3e 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -822,6 +822,8 @@ static int parse_thread_groups(struct device_node *dn, count = of_property_count_u32_elems(dn, "ibm,thread-groups"); thread_group_array = kcalloc(count, sizeof(u32), GFP_KERNEL); + if (!thread_group_array) + return -ENOMEM; ret = of_property_read_u32_array(dn, "ibm,thread-groups", thread_group_array, count); if (ret) diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S index de6ee7d35cff0..5589af3e40844 100644 --- a/arch/powerpc/kernel/vmlinux.lds.S +++ b/arch/powerpc/kernel/vmlinux.lds.S @@ -398,6 +398,7 @@ SECTIONS _end = . ; DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 88ad5ba7b87fd..21f7f26a5e2f3 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -1122,7 +1122,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, branch_flags = poke_type == BPF_MOD_CALL ? BRANCH_SET_LINK : 0; /* We currently only support poking bpf programs */ - if (!__bpf_address_lookup(bpf_func, &size, &offset, name)) { + if (!bpf_address_lookup(bpf_func, &size, &offset, name)) { pr_err("%s (0x%lx): kernel/modules are not supported\n", __func__, bpf_func); return -EOPNOTSUPP; } diff --git a/arch/powerpc/platforms/pseries/msi.c b/arch/powerpc/platforms/pseries/msi.c index a82aaa786e9e0..edc30cda5dbcb 100644 --- a/arch/powerpc/platforms/pseries/msi.c +++ b/arch/powerpc/platforms/pseries/msi.c @@ -19,6 +19,11 @@ #include "pseries.h" +struct pseries_msi_device { + unsigned int msi_quota; + unsigned int msi_used; +}; + static int query_token, change_token; #define RTAS_QUERY_FN 0 @@ -433,8 +438,28 @@ static int pseries_msi_ops_prepare(struct irq_domain *domain, struct device *dev struct msi_domain_info *info = domain->host_data; struct pci_dev *pdev = to_pci_dev(dev); int type = (info->flags & MSI_FLAG_PCI_MSIX) ? PCI_CAP_ID_MSIX : PCI_CAP_ID_MSI; + int ret; + + struct pseries_msi_device *pseries_dev __free(kfree) + = kmalloc(sizeof(*pseries_dev), GFP_KERNEL); + if (!pseries_dev) + return -ENOMEM; + + while (1) { + ret = rtas_prepare_msi_irqs(pdev, nvec, type, arg); + if (!ret) + break; + else if (ret > 0) + nvec = ret; + else + return ret; + } - return rtas_prepare_msi_irqs(pdev, nvec, type, arg); + pseries_dev->msi_quota = nvec; + pseries_dev->msi_used = 0; + + arg->scratchpad[0].ptr = no_free_ptr(pseries_dev); + return 0; } /* @@ -443,9 +468,13 @@ static int pseries_msi_ops_prepare(struct irq_domain *domain, struct device *dev */ static void pseries_msi_ops_teardown(struct irq_domain *domain, msi_alloc_info_t *arg) { + struct pseries_msi_device *pseries_dev = arg->scratchpad[0].ptr; struct pci_dev *pdev = to_pci_dev(domain->dev); rtas_disable_msi(pdev); + + WARN_ON(pseries_dev->msi_used); + kfree(pseries_dev); } static void pseries_msi_shutdown(struct irq_data *d) @@ -546,12 +575,18 @@ static int pseries_irq_domain_alloc(struct irq_domain *domain, unsigned int virq unsigned int nr_irqs, void *arg) { struct pci_controller *phb = domain->host_data; + struct pseries_msi_device *pseries_dev; msi_alloc_info_t *info = arg; struct msi_desc *desc = info->desc; struct pci_dev *pdev = msi_desc_to_pci_dev(desc); int hwirq; int i, ret; + pseries_dev = info->scratchpad[0].ptr; + + if (pseries_dev->msi_used + nr_irqs > pseries_dev->msi_quota) + return -ENOSPC; + hwirq = rtas_query_irq_number(pci_get_pdn(pdev), desc->msi_index); if (hwirq < 0) { dev_err(&pdev->dev, "Failed to query HW IRQ: %d\n", hwirq); @@ -567,9 +602,10 @@ static int pseries_irq_domain_alloc(struct irq_domain *domain, unsigned int virq goto out; irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i, - &pseries_msi_irq_chip, domain->host_data); + &pseries_msi_irq_chip, pseries_dev); } + pseries_dev->msi_used++; return 0; out: @@ -582,9 +618,11 @@ static void pseries_irq_domain_free(struct irq_domain *domain, unsigned int virq unsigned int nr_irqs) { struct irq_data *d = irq_domain_get_irq_data(domain, virq); - struct pci_controller *phb = irq_data_get_irq_chip_data(d); + struct pseries_msi_device *pseries_dev = irq_data_get_irq_chip_data(d); + struct pci_controller *phb = domain->host_data; pr_debug("%s bridge %pOF %d #%d\n", __func__, phb->dn, virq, nr_irqs); + pseries_dev->msi_used -= nr_irqs; irq_domain_free_irqs_parent(domain, virq, nr_irqs); } diff --git a/arch/riscv/kernel/vector.c b/arch/riscv/kernel/vector.c index 901e67adf5760..34048c4c26dcd 100644 --- a/arch/riscv/kernel/vector.c +++ b/arch/riscv/kernel/vector.c @@ -109,8 +109,8 @@ bool insn_is_vector(u32 insn_buf) return false; } -static int riscv_v_thread_zalloc(struct kmem_cache *cache, - struct __riscv_v_ext_state *ctx) +static int riscv_v_thread_ctx_alloc(struct kmem_cache *cache, + struct __riscv_v_ext_state *ctx) { void *datap; @@ -120,13 +120,15 @@ static int riscv_v_thread_zalloc(struct kmem_cache *cache, ctx->datap = datap; memset(ctx, 0, offsetof(struct __riscv_v_ext_state, datap)); + ctx->vlenb = riscv_v_vsize / 32; + return 0; } void riscv_v_thread_alloc(struct task_struct *tsk) { #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE - riscv_v_thread_zalloc(riscv_v_kernel_cachep, &tsk->thread.kernel_vstate); + riscv_v_thread_ctx_alloc(riscv_v_kernel_cachep, &tsk->thread.kernel_vstate); #endif } @@ -212,12 +214,14 @@ bool riscv_v_first_use_handler(struct pt_regs *regs) * context where VS has been off. So, try to allocate the user's V * context and resume execution. */ - if (riscv_v_thread_zalloc(riscv_v_user_cachep, ¤t->thread.vstate)) { + if (riscv_v_thread_ctx_alloc(riscv_v_user_cachep, ¤t->thread.vstate)) { force_sig(SIGBUS); return true; } + riscv_v_vstate_on(regs); riscv_v_vstate_set_restore(current, regs); + return true; } diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S index 61bd5ba6680a7..997f9eb3b22b1 100644 --- a/arch/riscv/kernel/vmlinux.lds.S +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -170,6 +170,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS .riscv.attributes 0 : { *(.riscv.attributes) } diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index df22b10d91415..e60d2b823e091 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -270,6 +270,7 @@ config S390 select SPARSE_IRQ select SWIOTLB select SYSCTL_EXCEPTION_TRACE + select SYSTEM_DATA_VERIFICATION if KEXEC_SIG select THREAD_INFO_IN_TASK select TRACE_IRQFLAGS_SUPPORT select TTY @@ -296,7 +297,7 @@ config ARCH_SUPPORTS_KEXEC_FILE def_bool y config ARCH_SUPPORTS_KEXEC_SIG - def_bool MODULE_SIG_FORMAT + def_bool y config ARCH_SUPPORTS_KEXEC_PURGATORY def_bool y diff --git a/arch/s390/boot/Makefile b/arch/s390/boot/Makefile index 02f2cf0827487..1768424a1824f 100644 --- a/arch/s390/boot/Makefile +++ b/arch/s390/boot/Makefile @@ -21,6 +21,7 @@ KBUILD_AFLAGS := $(filter-out $(CC_FLAGS_MARCH),$(KBUILD_AFLAGS_DECOMPRESSOR)) KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_MARCH),$(KBUILD_CFLAGS_DECOMPRESSOR)) KBUILD_AFLAGS += $(CC_FLAGS_MARCH_MINIMUM) -D__DISABLE_EXPORTS KBUILD_CFLAGS += $(CC_FLAGS_MARCH_MINIMUM) -D__DISABLE_EXPORTS +KBUILD_CFLAGS += $(call cc-option, -Wno-default-const-init-unsafe) CFLAGS_sclp_early_core.o += -I$(srctree)/drivers/s390/char diff --git a/arch/s390/include/asm/idle.h b/arch/s390/include/asm/idle.h index 09f763b9eb40a..133059d9a949c 100644 --- a/arch/s390/include/asm/idle.h +++ b/arch/s390/include/asm/idle.h @@ -23,5 +23,6 @@ extern struct device_attribute dev_attr_idle_count; extern struct device_attribute dev_attr_idle_time_us; void psw_idle(struct s390_idle_data *data, unsigned long psw_mask); +void update_timer_idle(void); #endif /* _S390_IDLE_H */ diff --git a/arch/s390/kernel/idle.c b/arch/s390/kernel/idle.c index 39cb8d0ae3480..0f9e53f0a0686 100644 --- a/arch/s390/kernel/idle.c +++ b/arch/s390/kernel/idle.c @@ -21,11 +21,10 @@ static DEFINE_PER_CPU(struct s390_idle_data, s390_idle); -void account_idle_time_irq(void) +void update_timer_idle(void) { struct s390_idle_data *idle = this_cpu_ptr(&s390_idle); struct lowcore *lc = get_lowcore(); - unsigned long idle_time; u64 cycles_new[8]; int i; @@ -35,13 +34,19 @@ void account_idle_time_irq(void) this_cpu_add(mt_cycles[i], cycles_new[i] - idle->mt_cycles_enter[i]); } - idle_time = lc->int_clock - idle->clock_idle_enter; - lc->steal_timer += idle->clock_idle_enter - lc->last_update_clock; lc->last_update_clock = lc->int_clock; lc->system_timer += lc->last_update_timer - idle->timer_idle_enter; lc->last_update_timer = lc->sys_enter_timer; +} + +void account_idle_time_irq(void) +{ + struct s390_idle_data *idle = this_cpu_ptr(&s390_idle); + unsigned long idle_time; + + idle_time = get_lowcore()->int_clock - idle->clock_idle_enter; /* Account time spent with enabled wait psw loaded as idle time. */ WRITE_ONCE(idle->idle_time, READ_ONCE(idle->idle_time) + idle_time); diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index bdf9c7cb5685b..080e9285b3379 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -146,6 +146,10 @@ void noinstr do_io_irq(struct pt_regs *regs) struct pt_regs *old_regs = set_irq_regs(regs); bool from_idle; + from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT); + if (from_idle) + update_timer_idle(); + irq_enter_rcu(); if (user_mode(regs)) { @@ -154,7 +158,6 @@ void noinstr do_io_irq(struct pt_regs *regs) current->thread.last_break = regs->last_break; } - from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT); if (from_idle) account_idle_time_irq(); @@ -182,6 +185,10 @@ void noinstr do_ext_irq(struct pt_regs *regs) struct pt_regs *old_regs = set_irq_regs(regs); bool from_idle; + from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT); + if (from_idle) + update_timer_idle(); + irq_enter_rcu(); if (user_mode(regs)) { @@ -194,7 +201,6 @@ void noinstr do_ext_irq(struct pt_regs *regs) regs->int_parm = get_lowcore()->ext_params; regs->int_parm_long = get_lowcore()->ext_params2; - from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT); if (from_idle) account_idle_time_irq(); diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index f432869f89213..9af7228d2d792 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -842,7 +842,7 @@ static bool is_callchain_event(struct perf_event *event) u64 sample_type = event->attr.sample_type; return sample_type & (PERF_SAMPLE_CALLCHAIN | PERF_SAMPLE_REGS_USER | - PERF_SAMPLE_STACK_USER); + PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_STACK_USER); } static int cpumsf_pmu_event_init(struct perf_event *event) diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index d74d4c52ccd05..9289e9e535c70 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -212,6 +212,7 @@ SECTIONS /* Debugging sections. */ STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS /* diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c index 234a0ba305108..122d30b104401 100644 --- a/arch/s390/kernel/vtime.c +++ b/arch/s390/kernel/vtime.c @@ -225,10 +225,6 @@ static u64 vtime_delta(void) return timer - lc->last_update_timer; } -/* - * Update process times based on virtual cpu times stored by entry.S - * to the lowcore fields user_timer, system_timer & steal_clock. - */ void vtime_account_kernel(struct task_struct *tsk) { struct lowcore *lc = get_lowcore(); @@ -238,27 +234,17 @@ void vtime_account_kernel(struct task_struct *tsk) lc->guest_timer += delta; else lc->system_timer += delta; - - virt_timer_forward(delta); } EXPORT_SYMBOL_GPL(vtime_account_kernel); void vtime_account_softirq(struct task_struct *tsk) { - u64 delta = vtime_delta(); - - get_lowcore()->softirq_timer += delta; - - virt_timer_forward(delta); + get_lowcore()->softirq_timer += vtime_delta(); } void vtime_account_hardirq(struct task_struct *tsk) { - u64 delta = vtime_delta(); - - get_lowcore()->hardirq_timer += delta; - - virt_timer_forward(delta); + get_lowcore()->hardirq_timer += vtime_delta(); } /* diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index c82c577db2bcd..c541019d91356 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -232,24 +232,33 @@ int zpci_fmb_disable_device(struct zpci_dev *zdev) static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len) { u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); + int rc = -ENODEV; u64 data; - int rc; + + if (!zdev_enabled(zdev)) + goto out_err; rc = __zpci_load(&data, req, offset); - if (!rc) { - data = le64_to_cpu((__force __le64) data); - data >>= (8 - len) * 8; - *val = (u32) data; - } else - *val = 0xffffffff; + if (rc) + goto out_err; + data = le64_to_cpu((__force __le64)data); + data >>= (8 - len) * 8; + *val = (u32)data; + return 0; + +out_err: + PCI_SET_ERROR_RESPONSE(val); return rc; } static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len) { u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); + int rc = -ENODEV; u64 data = val; - int rc; + + if (!zdev_enabled(zdev)) + return rc; data <<= (8 - len) * 8; data = (__force u64) cpu_to_le64(data); diff --git a/arch/s390/purgatory/Makefile b/arch/s390/purgatory/Makefile index bd39b36e7bd68..cfb73bbcdfe85 100644 --- a/arch/s390/purgatory/Makefile +++ b/arch/s390/purgatory/Makefile @@ -22,6 +22,7 @@ KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING KBUILD_CFLAGS += -D__DISABLE_EXPORTS KBUILD_CFLAGS += $(CLANG_FLAGS) KBUILD_CFLAGS += $(call cc-option,-fno-PIE) +KBUILD_CFLAGS += $(call cc-option, -Wno-default-const-init-unsafe) KBUILD_AFLAGS := $(filter-out -DCC_USING_EXPOLINE,$(KBUILD_AFLAGS)) KBUILD_AFLAGS += -D__DISABLE_EXPORTS diff --git a/arch/sh/kernel/vmlinux.lds.S b/arch/sh/kernel/vmlinux.lds.S index 008c30289eaa6..169c63fb3c1dc 100644 --- a/arch/sh/kernel/vmlinux.lds.S +++ b/arch/sh/kernel/vmlinux.lds.S @@ -89,6 +89,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/sparc/include/uapi/asm/ioctls.h b/arch/sparc/include/uapi/asm/ioctls.h index 7fd2f5873c9e7..a8bbdf9877a41 100644 --- a/arch/sparc/include/uapi/asm/ioctls.h +++ b/arch/sparc/include/uapi/asm/ioctls.h @@ -5,10 +5,10 @@ #include /* Big T */ -#define TCGETA _IOR('T', 1, struct termio) -#define TCSETA _IOW('T', 2, struct termio) -#define TCSETAW _IOW('T', 3, struct termio) -#define TCSETAF _IOW('T', 4, struct termio) +#define TCGETA 0x40125401 /* _IOR('T', 1, struct termio) */ +#define TCSETA 0x80125402 /* _IOW('T', 2, struct termio) */ +#define TCSETAW 0x80125403 /* _IOW('T', 3, struct termio) */ +#define TCSETAF 0x80125404 /* _IOW('T', 4, struct termio) */ #define TCSBRK _IO('T', 5) #define TCXONC _IO('T', 6) #define TCFLSH _IO('T', 7) diff --git a/arch/sparc/kernel/process.c b/arch/sparc/kernel/process.c index 0442ab00518d3..7d69877511fac 100644 --- a/arch/sparc/kernel/process.c +++ b/arch/sparc/kernel/process.c @@ -17,14 +17,18 @@ asmlinkage long sparc_fork(struct pt_regs *regs) { - unsigned long orig_i1 = regs->u_regs[UREG_I1]; + unsigned long orig_i1; long ret; struct kernel_clone_args args = { .exit_signal = SIGCHLD, - /* Reuse the parent's stack for the child. */ - .stack = regs->u_regs[UREG_FP], }; + synchronize_user_stack(); + + orig_i1 = regs->u_regs[UREG_I1]; + /* Reuse the parent's stack for the child. */ + args.stack = regs->u_regs[UREG_FP]; + ret = kernel_clone(&args); /* If we get an error and potentially restart the system @@ -40,16 +44,19 @@ asmlinkage long sparc_fork(struct pt_regs *regs) asmlinkage long sparc_vfork(struct pt_regs *regs) { - unsigned long orig_i1 = regs->u_regs[UREG_I1]; + unsigned long orig_i1; long ret; - struct kernel_clone_args args = { .flags = CLONE_VFORK | CLONE_VM, .exit_signal = SIGCHLD, - /* Reuse the parent's stack for the child. */ - .stack = regs->u_regs[UREG_FP], }; + synchronize_user_stack(); + + orig_i1 = regs->u_regs[UREG_I1]; + /* Reuse the parent's stack for the child. */ + args.stack = regs->u_regs[UREG_FP]; + ret = kernel_clone(&args); /* If we get an error and potentially restart the system @@ -65,15 +72,18 @@ asmlinkage long sparc_vfork(struct pt_regs *regs) asmlinkage long sparc_clone(struct pt_regs *regs) { - unsigned long orig_i1 = regs->u_regs[UREG_I1]; - unsigned int flags = lower_32_bits(regs->u_regs[UREG_I0]); + unsigned long orig_i1; + unsigned int flags; long ret; + struct kernel_clone_args args = {0}; - struct kernel_clone_args args = { - .flags = (flags & ~CSIGNAL), - .exit_signal = (flags & CSIGNAL), - .tls = regs->u_regs[UREG_I3], - }; + synchronize_user_stack(); + + orig_i1 = regs->u_regs[UREG_I1]; + flags = lower_32_bits(regs->u_regs[UREG_I0]); + args.flags = (flags & ~CSIGNAL); + args.exit_signal = (flags & CSIGNAL); + args.tls = regs->u_regs[UREG_I3]; #ifdef CONFIG_COMPAT if (test_thread_flag(TIF_32BIT)) { diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S index f1b86eb303404..7ea510d9b42f2 100644 --- a/arch/sparc/kernel/vmlinux.lds.S +++ b/arch/sparc/kernel/vmlinux.lds.S @@ -191,6 +191,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/um/kernel/dyn.lds.S b/arch/um/kernel/dyn.lds.S index a36b7918a011a..ad3cefeff2acb 100644 --- a/arch/um/kernel/dyn.lds.S +++ b/arch/um/kernel/dyn.lds.S @@ -172,6 +172,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/um/kernel/uml.lds.S b/arch/um/kernel/uml.lds.S index a409d4b66114f..30aa24348d60c 100644 --- a/arch/um/kernel/uml.lds.S +++ b/arch/um/kernel/uml.lds.S @@ -113,6 +113,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/um/os-Linux/signal.c b/arch/um/os-Linux/signal.c index 11f07f4982700..bebeb6b4bee8b 100644 --- a/arch/um/os-Linux/signal.c +++ b/arch/um/os-Linux/signal.c @@ -35,7 +35,6 @@ void (*sig_info[NSIG])(int, struct siginfo *, struct uml_pt_regs *, void *mc) = static void sig_handler_common(int sig, struct siginfo *si, mcontext_t *mc) { struct uml_pt_regs r; - int save_errno = errno; r.is_user = 0; if (sig == SIGSEGV) { @@ -49,8 +48,6 @@ static void sig_handler_common(int sig, struct siginfo *si, mcontext_t *mc) unblock_signals_trace(); (*sig_info[sig])(sig, si, &r, mc); - - errno = save_errno; } /* @@ -201,8 +198,11 @@ static void hard_handler(int sig, siginfo_t *si, void *p) { ucontext_t *uc = p; mcontext_t *mc = &uc->uc_mcontext; + int save_errno = errno; (*handlers[sig])(sig, (struct siginfo *)si, mc); + + errno = save_errno; } void set_handler(int sig) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index a3700766a8c08..ee41af778a9dc 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -298,6 +298,7 @@ config X86 select HAVE_SYSCALL_TRACEPOINTS select HAVE_UACCESS_VALIDATION if HAVE_OBJTOOL select HAVE_UNSTABLE_SCHED_CLOCK + select HAVE_UNWIND_USER_FP if X86_64 select HAVE_USER_RETURN_NOTIFIER select HAVE_GENERIC_VDSO select VDSO_GETRANDOM if X86_64 diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 74657589264df..2013840d6318f 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -110,6 +110,7 @@ vmlinux-objs-$(CONFIG_EFI_SBAT) += $(obj)/sbat.o ifdef CONFIG_EFI_SBAT $(obj)/sbat.o: $(CONFIG_EFI_SBAT_FILE) +AFLAGS_sbat.o += -I $(srctree) endif $(obj)/vmlinux: $(vmlinux-objs-y) $(vmlinux-libs-y) FORCE diff --git a/arch/x86/boot/compressed/sev.c b/arch/x86/boot/compressed/sev.c index 6e5c32a53d034..b688303666052 100644 --- a/arch/x86/boot/compressed/sev.c +++ b/arch/x86/boot/compressed/sev.c @@ -27,17 +27,17 @@ #include "sev.h" static struct ghcb boot_ghcb_page __aligned(PAGE_SIZE); -struct ghcb *boot_ghcb; +struct ghcb *boot_ghcb __section(".data"); #undef __init #define __init #define __BOOT_COMPRESSED -u8 snp_vmpl; -u16 ghcb_version; +u8 snp_vmpl __section(".data"); +u16 ghcb_version __section(".data"); -u64 boot_svsm_caa_pa; +u64 boot_svsm_caa_pa __section(".data"); /* Include code for early handlers */ #include "../../boot/startup/sev-shared.c" @@ -187,6 +187,7 @@ bool sev_es_check_ghcb_fault(unsigned long address) MSR_AMD64_SNP_RESERVED_BIT13 | \ MSR_AMD64_SNP_RESERVED_BIT15 | \ MSR_AMD64_SNP_SECURE_AVIC | \ + MSR_AMD64_SNP_RESERVED_BITS19_22 | \ MSR_AMD64_SNP_RESERVED_MASK) #ifdef CONFIG_AMD_SECURE_AVIC diff --git a/arch/x86/boot/compressed/vmlinux.lds.S b/arch/x86/boot/compressed/vmlinux.lds.S index 587ce3e7c5048..e0b152715d9c6 100644 --- a/arch/x86/boot/compressed/vmlinux.lds.S +++ b/arch/x86/boot/compressed/vmlinux.lds.S @@ -88,7 +88,7 @@ SECTIONS /DISCARD/ : { *(.dynamic) *(.dynsym) *(.dynstr) *(.dynbss) *(.hash) *(.gnu.hash) - *(.note.*) + *(.note.*) *(.modinfo) } .got.plt (INFO) : { diff --git a/arch/x86/boot/startup/sev-shared.c b/arch/x86/boot/startup/sev-shared.c index 4e22ffd735168..e52e886ad313d 100644 --- a/arch/x86/boot/startup/sev-shared.c +++ b/arch/x86/boot/startup/sev-shared.c @@ -31,7 +31,7 @@ static u32 cpuid_std_range_max __ro_after_init; static u32 cpuid_hyp_range_max __ro_after_init; static u32 cpuid_ext_range_max __ro_after_init; -bool sev_snp_needs_sfw; +bool sev_snp_needs_sfw __section(".data"); void __noreturn sev_es_terminate(unsigned int set, unsigned int reason) diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c index 9ae3b11754e65..d20e9cc065a87 100644 --- a/arch/x86/coco/sev/core.c +++ b/arch/x86/coco/sev/core.c @@ -122,6 +122,7 @@ static const char * const sev_status_feat_names[] = { [MSR_AMD64_SNP_VMSA_REG_PROT_BIT] = "VMSARegProt", [MSR_AMD64_SNP_SMT_PROT_BIT] = "SMTProt", [MSR_AMD64_SNP_SECURE_AVIC_BIT] = "SecureAVIC", + [MSR_AMD64_SNP_IBPB_ON_ENTRY_BIT] = "IBPBOnEntry", }; /* @@ -2008,8 +2009,7 @@ void snp_msg_free(struct snp_msg_desc *mdesc) free_shared_pages(mdesc->request, sizeof(struct snp_guest_msg)); iounmap((__force void __iomem *)mdesc->secrets); - memset(mdesc, 0, sizeof(*mdesc)); - kfree(mdesc); + kfree_sensitive(mdesc); } EXPORT_SYMBOL_GPL(snp_msg_free); diff --git a/arch/x86/entry/entry_fred.c b/arch/x86/entry/entry_fred.c index f004a4dc74c2d..563e439b743f2 100644 --- a/arch/x86/entry/entry_fred.c +++ b/arch/x86/entry/entry_fred.c @@ -159,8 +159,6 @@ void __init fred_complete_exception_setup(void) static noinstr void fred_extint(struct pt_regs *regs) { unsigned int vector = regs->fred_ss.vector; - unsigned int index = array_index_nospec(vector - FIRST_SYSTEM_VECTOR, - NR_SYSTEM_VECTORS); if (WARN_ON_ONCE(vector < FIRST_EXTERNAL_VECTOR)) return; @@ -169,7 +167,8 @@ static noinstr void fred_extint(struct pt_regs *regs) irqentry_state_t state = irqentry_enter(regs); instrumentation_begin(); - sysvec_table[index](regs); + sysvec_table[array_index_nospec(vector - FIRST_SYSTEM_VECTOR, + NR_SYSTEM_VECTORS)](regs); instrumentation_end(); irqentry_exit(regs, state); } else { diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index dd9ff120ad437..64e2bf2d4a615 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -2846,46 +2846,6 @@ static unsigned long get_segment_base(unsigned int segment) return get_desc_base(desc); } -#ifdef CONFIG_UPROBES -/* - * Heuristic-based check if uprobe is installed at the function entry. - * - * Under assumption of user code being compiled with frame pointers, - * `push %rbp/%ebp` is a good indicator that we indeed are. - * - * Similarly, `endbr64` (assuming 64-bit mode) is also a common pattern. - * If we get this wrong, captured stack trace might have one extra bogus - * entry, but the rest of stack trace will still be meaningful. - */ -static bool is_uprobe_at_func_entry(struct pt_regs *regs) -{ - struct arch_uprobe *auprobe; - - if (!current->utask) - return false; - - auprobe = current->utask->auprobe; - if (!auprobe) - return false; - - /* push %rbp/%ebp */ - if (auprobe->insn[0] == 0x55) - return true; - - /* endbr64 (64-bit only) */ - if (user_64bit_mode(regs) && is_endbr((u32 *)auprobe->insn)) - return true; - - return false; -} - -#else -static bool is_uprobe_at_func_entry(struct pt_regs *regs) -{ - return false; -} -#endif /* CONFIG_UPROBES */ - #ifdef CONFIG_IA32_EMULATION #include @@ -3101,8 +3061,8 @@ void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap) cap->version = x86_pmu.version; cap->num_counters_gp = x86_pmu_num_counters(NULL); cap->num_counters_fixed = x86_pmu_num_counters_fixed(NULL); - cap->bit_width_gp = x86_pmu.cntval_bits; - cap->bit_width_fixed = x86_pmu.cntval_bits; + cap->bit_width_gp = cap->num_counters_gp ? x86_pmu.cntval_bits : 0; + cap->bit_width_fixed = cap->num_counters_fixed ? x86_pmu.cntval_bits : 0; cap->events_mask = (unsigned int)x86_pmu.events_maskl; cap->events_mask_len = x86_pmu.events_mask_len; cap->pebs_ept = x86_pmu.pebs_ept; diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 32d551f2646a7..55d8df7b0fdaa 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -7029,6 +7029,7 @@ __init int intel_pmu_init(void) case INTEL_ATOM_SILVERMONT_D: case INTEL_ATOM_SILVERMONT_MID: case INTEL_ATOM_AIRMONT: + case INTEL_ATOM_AIRMONT_NP: case INTEL_ATOM_SILVERMONT_MID2: memcpy(hw_cache_event_ids, slm_hw_cache_event_ids, sizeof(hw_cache_event_ids)); diff --git a/arch/x86/events/intel/cstate.c b/arch/x86/events/intel/cstate.c index 6f5286a99e0c3..15db12647b2f8 100644 --- a/arch/x86/events/intel/cstate.c +++ b/arch/x86/events/intel/cstate.c @@ -598,6 +598,7 @@ static const struct x86_cpu_id intel_cstates_match[] __initconst = { X86_MATCH_VFM(INTEL_ATOM_SILVERMONT, &slm_cstates), X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_D, &slm_cstates), X86_MATCH_VFM(INTEL_ATOM_AIRMONT, &slm_cstates), + X86_MATCH_VFM(INTEL_ATOM_AIRMONT_NP, &slm_cstates), X86_MATCH_VFM(INTEL_BROADWELL, &snb_cstates), X86_MATCH_VFM(INTEL_BROADWELL_D, &snb_cstates), diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index e1f370b8d065f..a338ee01bb242 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -6610,6 +6610,32 @@ static struct intel_uncore_type gnr_uncore_ubox = { .attr_update = uncore_alias_groups, }; +static struct uncore_event_desc gnr_uncore_imc_events[] = { + INTEL_UNCORE_EVENT_DESC(clockticks, "event=0x01,umask=0x00"), + INTEL_UNCORE_EVENT_DESC(cas_count_read_sch0, "event=0x05,umask=0xcf"), + INTEL_UNCORE_EVENT_DESC(cas_count_read_sch0.scale, "6.103515625e-5"), + INTEL_UNCORE_EVENT_DESC(cas_count_read_sch0.unit, "MiB"), + INTEL_UNCORE_EVENT_DESC(cas_count_read_sch1, "event=0x06,umask=0xcf"), + INTEL_UNCORE_EVENT_DESC(cas_count_read_sch1.scale, "6.103515625e-5"), + INTEL_UNCORE_EVENT_DESC(cas_count_read_sch1.unit, "MiB"), + INTEL_UNCORE_EVENT_DESC(cas_count_write_sch0, "event=0x05,umask=0xf0"), + INTEL_UNCORE_EVENT_DESC(cas_count_write_sch0.scale, "6.103515625e-5"), + INTEL_UNCORE_EVENT_DESC(cas_count_write_sch0.unit, "MiB"), + INTEL_UNCORE_EVENT_DESC(cas_count_write_sch1, "event=0x06,umask=0xf0"), + INTEL_UNCORE_EVENT_DESC(cas_count_write_sch1.scale, "6.103515625e-5"), + INTEL_UNCORE_EVENT_DESC(cas_count_write_sch1.unit, "MiB"), + { /* end: all zeroes */ }, +}; + +static struct intel_uncore_type gnr_uncore_imc = { + SPR_UNCORE_MMIO_COMMON_FORMAT(), + .name = "imc", + .fixed_ctr_bits = 48, + .fixed_ctr = SNR_IMC_MMIO_PMON_FIXED_CTR, + .fixed_ctl = SNR_IMC_MMIO_PMON_FIXED_CTL, + .event_descs = gnr_uncore_imc_events, +}; + static struct intel_uncore_type gnr_uncore_pciex8 = { SPR_UNCORE_PCI_COMMON_FORMAT(), .name = "pciex8", @@ -6657,7 +6683,7 @@ static struct intel_uncore_type *gnr_uncores[UNCORE_GNR_NUM_UNCORE_TYPES] = { NULL, &spr_uncore_pcu, &gnr_uncore_ubox, - &spr_uncore_imc, + &gnr_uncore_imc, NULL, &gnr_uncore_upi, NULL, diff --git a/arch/x86/events/msr.c b/arch/x86/events/msr.c index 7f5007a4752a1..8052596b85036 100644 --- a/arch/x86/events/msr.c +++ b/arch/x86/events/msr.c @@ -78,6 +78,7 @@ static bool test_intel(int idx, void *data) case INTEL_ATOM_SILVERMONT: case INTEL_ATOM_SILVERMONT_D: case INTEL_ATOM_AIRMONT: + case INTEL_ATOM_AIRMONT_NP: case INTEL_ATOM_GOLDMONT: case INTEL_ATOM_GOLDMONT_D: diff --git a/arch/x86/hyperv/hv_vtl.c b/arch/x86/hyperv/hv_vtl.c index 042e8712d8dea..8aafccf7a52c7 100644 --- a/arch/x86/hyperv/hv_vtl.c +++ b/arch/x86/hyperv/hv_vtl.c @@ -105,7 +105,7 @@ static void hv_vtl_ap_entry(void) static int hv_vtl_bringup_vcpu(u32 target_vp_index, int cpu, u64 eip_ignored) { - u64 status; + u64 status, rsp, rip; int ret = 0; struct hv_enable_vp_vtl *input; unsigned long irq_flags; @@ -118,9 +118,11 @@ static int hv_vtl_bringup_vcpu(u32 target_vp_index, int cpu, u64 eip_ignored) struct desc_struct *gdt; struct task_struct *idle = idle_thread_get(cpu); - u64 rsp = (unsigned long)idle->thread.sp; + if (IS_ERR(idle)) + return PTR_ERR(idle); - u64 rip = (u64)&hv_vtl_ap_entry; + rsp = (unsigned long)idle->thread.sp; + rip = (u64)&hv_vtl_ap_entry; native_store_gdt(&gdt_ptr); store_idt(&idt_ptr); diff --git a/arch/x86/include/asm/cfi.h b/arch/x86/include/asm/cfi.h index c40b9ebc1fb40..ab3fbbd947ed9 100644 --- a/arch/x86/include/asm/cfi.h +++ b/arch/x86/include/asm/cfi.h @@ -111,6 +111,12 @@ extern bhi_thunk __bhi_args_end[]; struct pt_regs; +#ifdef CONFIG_CALL_PADDING +#define CFI_OFFSET (CONFIG_FUNCTION_PADDING_CFI+5) +#else +#define CFI_OFFSET 5 +#endif + #ifdef CONFIG_CFI enum bug_trap_type handle_cfi_failure(struct pt_regs *regs); #define __bpfcall @@ -119,11 +125,9 @@ static inline int cfi_get_offset(void) { switch (cfi_mode) { case CFI_FINEIBT: - return 16; + return /* fineibt_prefix_size */ 16; case CFI_KCFI: - if (IS_ENABLED(CONFIG_CALL_PADDING)) - return 16; - return 5; + return CFI_OFFSET; default: return 0; } diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h index f227a70ac91f0..51b4cdbea061a 100644 --- a/arch/x86/include/asm/efi.h +++ b/arch/x86/include/asm/efi.h @@ -138,7 +138,7 @@ extern void __init efi_apply_memmap_quirks(void); extern int __init efi_reuse_config(u64 tables, int nr_tables); extern void efi_delete_dummy_variable(void); extern void efi_crash_gracefully_on_page_fault(unsigned long phys_addr); -extern void efi_free_boot_services(void); +extern void efi_unmap_boot_services(void); void arch_efi_call_virt_setup(void); void arch_efi_call_virt_teardown(void); diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h index b08c95872eed9..c56e1e63b8932 100644 --- a/arch/x86/include/asm/ftrace.h +++ b/arch/x86/include/asm/ftrace.h @@ -57,7 +57,7 @@ arch_ftrace_get_regs(struct ftrace_regs *fregs) } #define arch_ftrace_partial_regs(regs) do { \ - regs->flags &= ~X86_EFLAGS_FIXED; \ + regs->flags |= X86_EFLAGS_FIXED; \ regs->cs = __KERNEL_CS; \ } while (0) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index b74ae7183f3ae..0f2f9f1552a4f 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1229,6 +1229,12 @@ enum kvm_irqchip_mode { KVM_IRQCHIP_SPLIT, /* created with KVM_CAP_SPLIT_IRQCHIP */ }; +enum kvm_suppress_eoi_broadcast_mode { + KVM_SUPPRESS_EOI_BROADCAST_QUIRKED, /* Legacy behavior */ + KVM_SUPPRESS_EOI_BROADCAST_ENABLED, /* Enable Suppress EOI broadcast */ + KVM_SUPPRESS_EOI_BROADCAST_DISABLED /* Disable Suppress EOI broadcast */ +}; + struct kvm_x86_msr_filter { u8 count; bool default_allow:1; @@ -1480,6 +1486,7 @@ struct kvm_arch { bool x2apic_format; bool x2apic_broadcast_quirk_disabled; + enum kvm_suppress_eoi_broadcast_mode suppress_eoi_broadcast_mode; bool has_mapped_host_mmio; bool guest_can_read_msr_platform_info; diff --git a/arch/x86/include/asm/linkage.h b/arch/x86/include/asm/linkage.h index 9d38ae744a2e4..a7294656ad908 100644 --- a/arch/x86/include/asm/linkage.h +++ b/arch/x86/include/asm/linkage.h @@ -68,7 +68,7 @@ * Depending on -fpatchable-function-entry=N,N usage (CONFIG_CALL_PADDING) the * CFI symbol layout changes. * - * Without CALL_THUNKS: + * Without CALL_PADDING: * * .align FUNCTION_ALIGNMENT * __cfi_##name: @@ -77,7 +77,7 @@ * .long __kcfi_typeid_##name * name: * - * With CALL_THUNKS: + * With CALL_PADDING: * * .align FUNCTION_ALIGNMENT * __cfi_##name: diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 9e1720d73244f..d9e03c6d1d5c1 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -711,7 +711,10 @@ #define MSR_AMD64_SNP_SMT_PROT BIT_ULL(MSR_AMD64_SNP_SMT_PROT_BIT) #define MSR_AMD64_SNP_SECURE_AVIC_BIT 18 #define MSR_AMD64_SNP_SECURE_AVIC BIT_ULL(MSR_AMD64_SNP_SECURE_AVIC_BIT) -#define MSR_AMD64_SNP_RESV_BIT 19 +#define MSR_AMD64_SNP_RESERVED_BITS19_22 GENMASK_ULL(22, 19) +#define MSR_AMD64_SNP_IBPB_ON_ENTRY_BIT 23 +#define MSR_AMD64_SNP_IBPB_ON_ENTRY BIT_ULL(MSR_AMD64_SNP_IBPB_ON_ENTRY_BIT) +#define MSR_AMD64_SNP_RESV_BIT 24 #define MSR_AMD64_SNP_RESERVED_MASK GENMASK_ULL(63, MSR_AMD64_SNP_RESV_BIT) #define MSR_AMD64_SAVIC_CONTROL 0xc0010138 #define MSR_AMD64_SAVIC_EN_BIT 0 diff --git a/arch/x86/include/asm/unwind_user.h b/arch/x86/include/asm/unwind_user.h new file mode 100644 index 0000000000000..c4f1ff8874d67 --- /dev/null +++ b/arch/x86/include/asm/unwind_user.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_UNWIND_USER_H +#define _ASM_X86_UNWIND_USER_H + +#include +#include + +#define ARCH_INIT_USER_FP_FRAME(ws) \ + .cfa_off = 2*(ws), \ + .ra_off = -1*(ws), \ + .fp_off = -2*(ws), \ + .use_fp = true, + +#define ARCH_INIT_USER_FP_ENTRY_FRAME(ws) \ + .cfa_off = 1*(ws), \ + .ra_off = -1*(ws), \ + .fp_off = 0, \ + .use_fp = false, + +static inline int unwind_user_word_size(struct pt_regs *regs) +{ + /* We can't unwind VM86 stacks */ + if (regs->flags & X86_VM_MASK) + return 0; +#ifdef CONFIG_X86_64 + if (!user_64bit_mode(regs)) + return sizeof(int); +#endif + return sizeof(long); +} + +static inline bool unwind_user_at_function_start(struct pt_regs *regs) +{ + return is_uprobe_at_func_entry(regs); +} + +#endif /* _ASM_X86_UNWIND_USER_H */ diff --git a/arch/x86/include/asm/uprobes.h b/arch/x86/include/asm/uprobes.h index 1ee2e5115955c..362210c799987 100644 --- a/arch/x86/include/asm/uprobes.h +++ b/arch/x86/include/asm/uprobes.h @@ -62,4 +62,13 @@ struct arch_uprobe_task { unsigned int saved_tf; }; +#ifdef CONFIG_UPROBES +extern bool is_uprobe_at_func_entry(struct pt_regs *regs); +#else +static bool is_uprobe_at_func_entry(struct pt_regs *regs) +{ + return false; +} +#endif /* CONFIG_UPROBES */ + #endif /* _ASM_UPROBES_H */ diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h index d420c9c066d48..1584926cd1f4f 100644 --- a/arch/x86/include/uapi/asm/kvm.h +++ b/arch/x86/include/uapi/asm/kvm.h @@ -913,8 +913,10 @@ struct kvm_sev_snp_launch_finish { __u64 pad1[4]; }; -#define KVM_X2APIC_API_USE_32BIT_IDS (1ULL << 0) -#define KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK (1ULL << 1) +#define KVM_X2APIC_API_USE_32BIT_IDS _BITULL(0) +#define KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK _BITULL(1) +#define KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST _BITULL(2) +#define KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST _BITULL(3) struct kvm_hyperv_eventfd { __u32 conn_id; diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c index 9fa321a95eb33..d6138b2b633a3 100644 --- a/arch/x86/kernel/acpi/boot.c +++ b/arch/x86/kernel/acpi/boot.c @@ -35,6 +35,7 @@ #include #include #include +#include #include "sleep.h" /* To include x86_acpi_suspend_lowlevel */ static int __initdata acpi_force = 0; @@ -164,11 +165,14 @@ static bool __init acpi_is_processor_usable(u32 lapic_flags) if (lapic_flags & ACPI_MADT_ENABLED) return true; - if (!acpi_support_online_capable || - (lapic_flags & ACPI_MADT_ONLINE_CAPABLE)) - return true; + if (acpi_support_online_capable) + return lapic_flags & ACPI_MADT_ONLINE_CAPABLE; - return false; + /* + * QEMU expects legacy "Enabled=0" LAPIC entries to be counted as usable + * in order to support CPU hotplug in guests. + */ + return !hypervisor_is_type(X86_HYPER_NATIVE); } static int __init diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 8ee5ff547357a..bd16e9f40d51a 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -1168,7 +1168,7 @@ void __init_or_module noinline apply_seal_endbr(s32 *start, s32 *end) poison_endbr(addr); if (IS_ENABLED(CONFIG_FINEIBT)) - poison_cfi(addr - 16); + poison_cfi(addr - CFI_OFFSET); } } @@ -1375,6 +1375,8 @@ extern u8 fineibt_preamble_end[]; #define fineibt_preamble_ud 0x13 #define fineibt_preamble_hash 5 +#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE) + /* * : * 0: b8 78 56 34 12 mov $0x12345678, %eax @@ -1620,7 +1622,7 @@ static int cfi_rewrite_preamble(s32 *start, s32 *end) * have determined there are no indirect calls to it and we * don't need no CFI either. */ - if (!is_endbr(addr + 16)) + if (!is_endbr(addr + CFI_OFFSET)) continue; hash = decode_preamble_hash(addr, &arity); @@ -1628,6 +1630,15 @@ static int cfi_rewrite_preamble(s32 *start, s32 *end) addr, addr, 5, addr)) return -EINVAL; + /* + * FineIBT relies on being at func-16, so if the preamble is + * actually larger than that, place it the tail end. + * + * NOTE: this is possible with things like DEBUG_CALL_THUNKS + * and DEBUG_FORCE_FUNCTION_ALIGN_64B. + */ + addr += CFI_OFFSET - fineibt_prefix_size; + text_poke_early(addr, fineibt_preamble_start, fineibt_preamble_size); WARN_ON(*(u32 *)(addr + fineibt_preamble_hash) != 0x12345678); text_poke_early(addr + fineibt_preamble_hash, &hash, 4); @@ -1650,10 +1661,10 @@ static void cfi_rewrite_endbr(s32 *start, s32 *end) for (s = start; s < end; s++) { void *addr = (void *)s + *s; - if (!exact_endbr(addr + 16)) + if (!exact_endbr(addr + CFI_OFFSET)) continue; - poison_endbr(addr + 16); + poison_endbr(addr + CFI_OFFSET); } } @@ -1758,7 +1769,8 @@ static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline, if (FINEIBT_WARN(fineibt_preamble_size, 20) || FINEIBT_WARN(fineibt_preamble_bhi + fineibt_bhi1_size, 20) || FINEIBT_WARN(fineibt_caller_size, 14) || - FINEIBT_WARN(fineibt_paranoid_size, 20)) + FINEIBT_WARN(fineibt_paranoid_size, 20) || + WARN_ON_ONCE(CFI_OFFSET < fineibt_prefix_size)) return; if (cfi_mode == CFI_AUTO) { @@ -1871,6 +1883,11 @@ static void poison_cfi(void *addr) */ switch (cfi_mode) { case CFI_FINEIBT: + /* + * FineIBT preamble is at func-16. + */ + addr += CFI_OFFSET - fineibt_prefix_size; + /* * FineIBT prefix should start with an ENDBR. */ @@ -1909,8 +1926,6 @@ static void poison_cfi(void *addr) } } -#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE) - /* * When regs->ip points to a 0xD6 byte in the FineIBT preamble, * return true and fill out target and type. diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 5d46709c58d0b..a92750f3079af 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -951,26 +951,14 @@ static void init_amd_zen1(struct cpuinfo_x86 *c) } } -static bool cpu_has_zenbleed_microcode(void) -{ - u32 good_rev = 0; - - switch (boot_cpu_data.x86_model) { - case 0x30 ... 0x3f: good_rev = 0x0830107b; break; - case 0x60 ... 0x67: good_rev = 0x0860010c; break; - case 0x68 ... 0x6f: good_rev = 0x08608107; break; - case 0x70 ... 0x7f: good_rev = 0x08701033; break; - case 0xa0 ... 0xaf: good_rev = 0x08a00009; break; - - default: - return false; - } - - if (boot_cpu_data.microcode < good_rev) - return false; - - return true; -} +static const struct x86_cpu_id amd_zenbleed_microcode[] = { + ZEN_MODEL_STEP_UCODE(0x17, 0x31, 0x0, 0x0830107b), + ZEN_MODEL_STEP_UCODE(0x17, 0x60, 0x1, 0x0860010c), + ZEN_MODEL_STEP_UCODE(0x17, 0x68, 0x1, 0x08608107), + ZEN_MODEL_STEP_UCODE(0x17, 0x71, 0x0, 0x08701033), + ZEN_MODEL_STEP_UCODE(0x17, 0xa0, 0x0, 0x08a00009), + {} +}; static void zen2_zenbleed_check(struct cpuinfo_x86 *c) { @@ -980,7 +968,7 @@ static void zen2_zenbleed_check(struct cpuinfo_x86 *c) if (!cpu_has(c, X86_FEATURE_AVX)) return; - if (!cpu_has_zenbleed_microcode()) { + if (!x86_match_min_microcode_rev(amd_zenbleed_microcode)) { pr_notice_once("Zenbleed: please update your microcode for the most optimal fix\n"); msr_set_bit(MSR_AMD64_DE_CFG, MSR_AMD64_DE_CFG_ZEN2_FP_BACKUP_FIX_BIT); } else { diff --git a/arch/x86/kernel/cpu/topology.c b/arch/x86/kernel/cpu/topology.c index 6073a16628f9e..425404e7b7b42 100644 --- a/arch/x86/kernel/cpu/topology.c +++ b/arch/x86/kernel/cpu/topology.c @@ -27,7 +27,6 @@ #include #include -#include #include #include #include @@ -240,20 +239,6 @@ static __init void topo_register_apic(u32 apic_id, u32 acpi_id, bool present) cpuid_to_apicid[cpu] = apic_id; topo_set_cpuids(cpu, apic_id, acpi_id); } else { - u32 pkgid = topo_apicid(apic_id, TOPO_PKG_DOMAIN); - - /* - * Check for present APICs in the same package when running - * on bare metal. Allow the bogosity in a guest. - */ - if (hypervisor_is_type(X86_HYPER_NATIVE) && - topo_unit_count(pkgid, TOPO_PKG_DOMAIN, phys_cpu_present_map)) { - pr_info_once("Ignoring hot-pluggable APIC ID %x in present package.\n", - apic_id); - topo_info.nr_rejected_cpus++; - return; - } - topo_info.nr_disabled_cpus++; } diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S index 823dbdd0eb410..ae94f7b0fdf16 100644 --- a/arch/x86/kernel/ftrace_64.S +++ b/arch/x86/kernel/ftrace_64.S @@ -354,6 +354,9 @@ SYM_CODE_START(return_to_handler) UNWIND_HINT_UNDEFINED ANNOTATE_NOENDBR + /* Store original rsp for pt_regs.sp value. */ + movq %rsp, %rdi + /* Restore return_to_handler value that got eaten by previous ret instruction. */ subq $8, %rsp UNWIND_HINT_FUNC @@ -364,7 +367,7 @@ SYM_CODE_START(return_to_handler) movq %rax, RAX(%rsp) movq %rdx, RDX(%rsp) movq %rbp, RBP(%rsp) - movq %rsp, RSP(%rsp) + movq %rdi, RSP(%rsp) movq %rsp, %rdi call ftrace_return_to_handler diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c index c3244ac680d14..f3b451eb49be1 100644 --- a/arch/x86/kernel/kexec-bzimage64.c +++ b/arch/x86/kernel/kexec-bzimage64.c @@ -192,6 +192,13 @@ setup_efi_state(struct boot_params *params, unsigned long params_load_addr, struct efi_info *current_ei = &boot_params.efi_info; struct efi_info *ei = ¶ms->efi_info; + if (!params->acpi_rsdp_addr) { + if (efi.acpi20 != EFI_INVALID_TABLE_ADDR) + params->acpi_rsdp_addr = efi.acpi20; + else if (efi.acpi != EFI_INVALID_TABLE_ADDR) + params->acpi_rsdp_addr = efi.acpi; + } + if (!efi_enabled(EFI_RUNTIME_SERVICES)) return 0; diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 1b2edd07a3e17..383d4a4784f5b 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -439,9 +439,15 @@ int __init ima_free_kexec_buffer(void) int __init ima_get_kexec_buffer(void **addr, size_t *size) { + int ret; + if (!ima_kexec_buffer_size) return -ENOENT; + ret = ima_validate_range(ima_kexec_buffer_phys, ima_kexec_buffer_size); + if (ret) + return ret; + *addr = __va(ima_kexec_buffer_phys); *size = ima_kexec_buffer_size; diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c index 845aeaf36b8d2..46aed82243964 100644 --- a/arch/x86/kernel/uprobes.c +++ b/arch/x86/kernel/uprobes.c @@ -1819,3 +1819,59 @@ bool arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check ctx, else return regs->sp <= ret->stack; } + +/* + * Heuristic-based check if uprobe is installed at the function entry. + * + * Under assumption of user code being compiled with frame pointers, + * `push %rbp/%ebp` is a good indicator that we indeed are. + * + * Similarly, `endbr64` (assuming 64-bit mode) is also a common pattern. + * If we get this wrong, captured stack trace might have one extra bogus + * entry, but the rest of stack trace will still be meaningful. + */ +bool is_uprobe_at_func_entry(struct pt_regs *regs) +{ + struct arch_uprobe *auprobe; + + if (!current->utask) + return false; + + auprobe = current->utask->auprobe; + if (!auprobe) + return false; + + /* push %rbp/%ebp */ + if (auprobe->insn[0] == 0x55) + return true; + + /* endbr64 (64-bit only) */ + if (user_64bit_mode(regs) && is_endbr((u32 *)auprobe->insn)) + return true; + + return false; +} + +#ifdef CONFIG_IA32_EMULATION +unsigned long arch_uprobe_get_xol_area(void) +{ + struct thread_info *ti = current_thread_info(); + unsigned long vaddr; + + /* + * HACK: we are not in a syscall, but x86 get_unmapped_area() paths + * ignore TIF_ADDR32 and rely on in_32bit_syscall() to calculate + * vm_unmapped_area_info.high_limit. + * + * The #ifdef above doesn't cover the CONFIG_X86_X32_ABI=y case, + * but in this case in_32bit_syscall() -> in_x32_syscall() always + * (falsely) returns true because ->orig_ax == -1. + */ + if (test_thread_flag(TIF_ADDR32)) + ti->status |= TS_COMPAT; + vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, PAGE_SIZE, 0, 0); + ti->status &= ~TS_COMPAT; + + return vaddr; +} +#endif diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index d7af4a64c211b..4ed82b1fe173b 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -424,6 +424,7 @@ SECTIONS .llvm_bb_addr_map : { *(.llvm_bb_addr_map) } #endif + MODINFO ELF_DETAILS DISCARDS diff --git a/arch/x86/kvm/ioapic.c b/arch/x86/kvm/ioapic.c index 2c2783296aedb..a26fa4222f292 100644 --- a/arch/x86/kvm/ioapic.c +++ b/arch/x86/kvm/ioapic.c @@ -561,7 +561,7 @@ static void kvm_ioapic_update_eoi_one(struct kvm_vcpu *vcpu, spin_lock(&ioapic->lock); if (trigger_mode != IOAPIC_LEVEL_TRIG || - kvm_lapic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI) + kvm_lapic_suppress_eoi_broadcast(apic)) return; ASSERT(ent->fields.trig_mode == IOAPIC_LEVEL_TRIG); diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 8b6ec3304100f..a9845a1a9cd5d 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -105,6 +105,63 @@ bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector) apic_test_vector(vector, apic->regs + APIC_IRR); } +static bool kvm_lapic_advertise_suppress_eoi_broadcast(struct kvm *kvm) +{ + switch (kvm->arch.suppress_eoi_broadcast_mode) { + case KVM_SUPPRESS_EOI_BROADCAST_ENABLED: + return true; + case KVM_SUPPRESS_EOI_BROADCAST_DISABLED: + return false; + case KVM_SUPPRESS_EOI_BROADCAST_QUIRKED: + /* + * The default in-kernel I/O APIC emulates the 82093AA and does not + * implement an EOI register. Some guests (e.g. Windows with the + * Hyper-V role enabled) disable LAPIC EOI broadcast without + * checking the I/O APIC version, which can cause level-triggered + * interrupts to never be EOI'd. + * + * To avoid this, KVM doesn't advertise Suppress EOI Broadcast + * support when using the default in-kernel I/O APIC. + * + * Historically, in split IRQCHIP mode, KVM always advertised + * Suppress EOI Broadcast support but did not actually suppress + * EOIs, resulting in quirky behavior. + */ + return !ioapic_in_kernel(kvm); + default: + WARN_ON_ONCE(1); + return false; + } +} + +bool kvm_lapic_suppress_eoi_broadcast(struct kvm_lapic *apic) +{ + struct kvm *kvm = apic->vcpu->kvm; + + if (!(kvm_lapic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI)) + return false; + + switch (kvm->arch.suppress_eoi_broadcast_mode) { + case KVM_SUPPRESS_EOI_BROADCAST_ENABLED: + return true; + case KVM_SUPPRESS_EOI_BROADCAST_DISABLED: + return false; + case KVM_SUPPRESS_EOI_BROADCAST_QUIRKED: + /* + * Historically, in split IRQCHIP mode, KVM ignored the suppress + * EOI broadcast bit set by the guest and broadcasts EOIs to the + * userspace I/O APIC. For In-kernel I/O APIC, the support itself + * is not advertised, can only be enabled via KVM_SET_APIC_STATE, + * and KVM's I/O APIC doesn't emulate Directed EOIs; but if the + * feature is enabled, it is respected (with odd behavior). + */ + return ioapic_in_kernel(kvm); + default: + WARN_ON_ONCE(1); + return false; + } +} + __read_mostly DEFINE_STATIC_KEY_FALSE(kvm_has_noapic_vcpu); EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_has_noapic_vcpu); @@ -554,15 +611,9 @@ void kvm_apic_set_version(struct kvm_vcpu *vcpu) v = APIC_VERSION | ((apic->nr_lvt_entries - 1) << 16); - /* - * KVM emulates 82093AA datasheet (with in-kernel IOAPIC implementation) - * which doesn't have EOI register; Some buggy OSes (e.g. Windows with - * Hyper-V role) disable EOI broadcast in lapic not checking for IOAPIC - * version first and level-triggered interrupts never get EOIed in - * IOAPIC. - */ + if (guest_cpu_cap_has(vcpu, X86_FEATURE_X2APIC) && - !ioapic_in_kernel(vcpu->kvm)) + kvm_lapic_advertise_suppress_eoi_broadcast(vcpu->kvm)) v |= APIC_LVR_DIRECTED_EOI; kvm_lapic_set_reg(apic, APIC_LVR, v); } @@ -1517,6 +1568,15 @@ static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector) /* Request a KVM exit to inform the userspace IOAPIC. */ if (irqchip_split(apic->vcpu->kvm)) { + /* + * Don't exit to userspace if the guest has enabled Directed + * EOI, a.k.a. Suppress EOI Broadcasts, in which case the local + * APIC doesn't broadcast EOIs (the guest must EOI the target + * I/O APIC(s) directly). + */ + if (kvm_lapic_suppress_eoi_broadcast(apic)) + return; + apic->vcpu->arch.pending_ioapic_eoi = vector; kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu); return; diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index 282b9b7da98cd..e5f5a222eced0 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -231,6 +231,8 @@ static inline int kvm_lapic_latched_init(struct kvm_vcpu *vcpu) bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector); +bool kvm_lapic_suppress_eoi_broadcast(struct kvm_lapic *apic); + void kvm_wait_lapic_expire(struct kvm_vcpu *vcpu); void kvm_bitmap_or_dest_vcpus(struct kvm *kvm, struct kvm_lapic_irq *irq, diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index e56c9b37e2b5b..db35033999a8e 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -1880,10 +1880,9 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, * thus MMU might not be initialized correctly. * Set it again to fix this. */ - ret = nested_svm_load_cr3(&svm->vcpu, vcpu->arch.cr3, nested_npt_enabled(svm), false); - if (WARN_ON_ONCE(ret)) + if (ret) goto out_free; svm->nested.force_msr_bitmap_recalc = true; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index d758bff6e068b..eed104207a11a 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -2094,12 +2094,13 @@ static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload) ret = kvm_skip_emulated_instruction(vcpu); + /* KVM always performs VMLOAD/VMSAVE on VMCB01 (see __svm_vcpu_run()) */ if (vmload) { - svm_copy_vmloadsave_state(svm->vmcb, vmcb12); + svm_copy_vmloadsave_state(svm->vmcb01.ptr, vmcb12); svm->sysenter_eip_hi = 0; svm->sysenter_esp_hi = 0; } else { - svm_copy_vmloadsave_state(vmcb12, svm->vmcb); + svm_copy_vmloadsave_state(vmcb12, svm->vmcb01.ptr); } kvm_vcpu_unmap(vcpu, &map); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index c075ee23aeade..d15bd078a2d98 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -121,8 +121,10 @@ static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE); #define KVM_CAP_PMU_VALID_MASK KVM_PMU_CAP_DISABLE -#define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \ - KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK) +#define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \ + KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK | \ + KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST | \ + KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST) static void update_cr8_intercept(struct kvm_vcpu *vcpu); static void process_nmi(struct kvm_vcpu *vcpu); @@ -4136,47 +4138,47 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) break; case MSR_KVM_WALL_CLOCK_NEW: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; vcpu->kvm->arch.wall_clock = data; kvm_write_wall_clock(vcpu->kvm, data, 0); break; case MSR_KVM_WALL_CLOCK: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; vcpu->kvm->arch.wall_clock = data; kvm_write_wall_clock(vcpu->kvm, data, 0); break; case MSR_KVM_SYSTEM_TIME_NEW: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; kvm_write_system_time(vcpu, data, false, msr_info->host_initiated); break; case MSR_KVM_SYSTEM_TIME: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; kvm_write_system_time(vcpu, data, true, msr_info->host_initiated); break; case MSR_KVM_ASYNC_PF_EN: if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; if (kvm_pv_enable_async_pf(vcpu, data)) return 1; break; case MSR_KVM_ASYNC_PF_INT: if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; if (kvm_pv_enable_async_pf_int(vcpu, data)) return 1; break; case MSR_KVM_ASYNC_PF_ACK: if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; if (data & 0x1) { vcpu->arch.apf.pageready_pending = false; kvm_check_async_pf_completion(vcpu); @@ -4184,7 +4186,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) break; case MSR_KVM_STEAL_TIME: if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; if (unlikely(!sched_info_on())) return 1; @@ -4202,7 +4204,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) break; case MSR_KVM_PV_EOI_EN: if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; if (kvm_lapic_set_pv_eoi(vcpu, data, sizeof(u8))) return 1; @@ -4210,7 +4212,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) case MSR_KVM_POLL_CONTROL: if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; /* only enable bit supported */ if (data & (-1ULL << 1)) @@ -4511,61 +4513,61 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) break; case MSR_KVM_WALL_CLOCK: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->kvm->arch.wall_clock; break; case MSR_KVM_WALL_CLOCK_NEW: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->kvm->arch.wall_clock; break; case MSR_KVM_SYSTEM_TIME: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->arch.time; break; case MSR_KVM_SYSTEM_TIME_NEW: if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->arch.time; break; case MSR_KVM_ASYNC_PF_EN: if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->arch.apf.msr_en_val; break; case MSR_KVM_ASYNC_PF_INT: if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->arch.apf.msr_int_val; break; case MSR_KVM_ASYNC_PF_ACK: if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = 0; break; case MSR_KVM_STEAL_TIME: if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->arch.st.msr_val; break; case MSR_KVM_PV_EOI_EN: if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->arch.pv_eoi.msr_val; break; case MSR_KVM_POLL_CONTROL: if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL)) - return 1; + return KVM_MSR_RET_UNSUPPORTED; msr_info->data = vcpu->arch.msr_kvm_poll_control; break; @@ -4966,6 +4968,8 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) break; case KVM_CAP_X2APIC_API: r = KVM_X2APIC_API_VALID_FLAGS; + if (kvm && !irqchip_split(kvm)) + r &= ~KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST; break; case KVM_CAP_NESTED_STATE: r = kvm_x86_ops.nested_ops->get_state ? @@ -6783,11 +6787,24 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, if (cap->args[0] & ~KVM_X2APIC_API_VALID_FLAGS) break; + if ((cap->args[0] & KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST) && + (cap->args[0] & KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST)) + break; + + if ((cap->args[0] & KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST) && + !irqchip_split(kvm)) + break; + if (cap->args[0] & KVM_X2APIC_API_USE_32BIT_IDS) kvm->arch.x2apic_format = true; if (cap->args[0] & KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK) kvm->arch.x2apic_broadcast_quirk_disabled = true; + if (cap->args[0] & KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST) + kvm->arch.suppress_eoi_broadcast_mode = KVM_SUPPRESS_EOI_BROADCAST_ENABLED; + if (cap->args[0] & KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST) + kvm->arch.suppress_eoi_broadcast_mode = KVM_SUPPRESS_EOI_BROADCAST_DISABLED; + r = 0; break; case KVM_CAP_X86_DISABLE_EXITS: @@ -11598,8 +11615,7 @@ static inline int vcpu_block(struct kvm_vcpu *vcpu) if (is_guest_mode(vcpu)) { int r = kvm_check_nested_events(vcpu); - WARN_ON_ONCE(r == -EBUSY); - if (r < 0) + if (r < 0 && r != -EBUSY) return 0; } @@ -12147,9 +12163,11 @@ static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2) return; if (is_pae_paging(vcpu)) { + kvm_vcpu_srcu_read_lock(vcpu); for (i = 0 ; i < 4 ; i++) sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i); sregs2->flags |= KVM_SREGS2_FLAGS_PDPTRS_VALID; + kvm_vcpu_srcu_read_unlock(vcpu); } } diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index de5083cb1d374..788671a32d8ee 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -438,17 +438,8 @@ static void emit_kcfi(u8 **pprog, u32 hash) EMIT1_off32(0xb8, hash); /* movl $hash, %eax */ #ifdef CONFIG_CALL_PADDING - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); - EMIT1(0x90); + for (int i = 0; i < CONFIG_FUNCTION_PADDING_CFI; i++) + EMIT1(0x90); #endif EMIT_ENDBR(); diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c index 463b784499a8f..791c52c8393f4 100644 --- a/arch/x86/platform/efi/efi.c +++ b/arch/x86/platform/efi/efi.c @@ -837,7 +837,7 @@ static void __init __efi_enter_virtual_mode(void) } efi_check_for_embedded_firmwares(); - efi_free_boot_services(); + efi_unmap_boot_services(); if (!efi_is_mixed()) efi_native_runtime_setup(); diff --git a/arch/x86/platform/efi/quirks.c b/arch/x86/platform/efi/quirks.c index 553f330198f2f..35caa5746115d 100644 --- a/arch/x86/platform/efi/quirks.c +++ b/arch/x86/platform/efi/quirks.c @@ -341,7 +341,7 @@ void __init efi_reserve_boot_services(void) /* * Because the following memblock_reserve() is paired - * with memblock_free_late() for this region in + * with free_reserved_area() for this region in * efi_free_boot_services(), we must be extremely * careful not to reserve, and subsequently free, * critical regions of memory (like the kernel image) or @@ -404,17 +404,33 @@ static void __init efi_unmap_pages(efi_memory_desc_t *md) pr_err("Failed to unmap VA mapping for 0x%llx\n", va); } -void __init efi_free_boot_services(void) +struct efi_freeable_range { + u64 start; + u64 end; +}; + +static struct efi_freeable_range *ranges_to_free; + +void __init efi_unmap_boot_services(void) { struct efi_memory_map_data data = { 0 }; efi_memory_desc_t *md; int num_entries = 0; + int idx = 0; + size_t sz; void *new, *new_md; /* Keep all regions for /sys/kernel/debug/efi */ if (efi_enabled(EFI_DBG)) return; + sz = sizeof(*ranges_to_free) * efi.memmap.nr_map + 1; + ranges_to_free = kzalloc(sz, GFP_KERNEL); + if (!ranges_to_free) { + pr_err("Failed to allocate storage for freeable EFI regions\n"); + return; + } + for_each_efi_memory_desc(md) { unsigned long long start = md->phys_addr; unsigned long long size = md->num_pages << EFI_PAGE_SHIFT; @@ -471,7 +487,15 @@ void __init efi_free_boot_services(void) start = SZ_1M; } - memblock_free_late(start, size); + /* + * With CONFIG_DEFERRED_STRUCT_PAGE_INIT parts of the memory + * map are still not initialized and we can't reliably free + * memory here. + * Queue the ranges to free at a later point. + */ + ranges_to_free[idx].start = start; + ranges_to_free[idx].end = start + size; + idx++; } if (!num_entries) @@ -512,6 +536,31 @@ void __init efi_free_boot_services(void) } } +static int __init efi_free_boot_services(void) +{ + struct efi_freeable_range *range = ranges_to_free; + unsigned long freed = 0; + + if (!ranges_to_free) + return 0; + + while (range->start) { + void *start = phys_to_virt(range->start); + void *end = phys_to_virt(range->end); + + free_reserved_area(start, end, -1, NULL); + freed += (end - start); + range++; + } + kfree(ranges_to_free); + + if (freed) + pr_info("Freeing EFI boot services memory: %ldK\n", freed / SZ_1K); + + return 0; +} +arch_initcall(efi_free_boot_services); + /* * A number of config table entries get remapped to virtual addresses * after entering EFI virtual mode. However, the kexec kernel requires diff --git a/arch/x86/platform/pvh/head.S b/arch/x86/platform/pvh/head.S index 344030c1a81d4..53ee2d53fcf8e 100644 --- a/arch/x86/platform/pvh/head.S +++ b/arch/x86/platform/pvh/head.S @@ -91,10 +91,12 @@ SYM_CODE_START(pvh_start_xen) leal rva(early_stack_end)(%ebp), %esp +#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) /* Enable PAE mode. */ mov %cr4, %eax orl $X86_CR4_PAE, %eax mov %eax, %cr4 +#endif #ifdef CONFIG_X86_64 /* Enable Long mode. */ diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index 53282dc7d5ac5..23b91bf9b6630 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c @@ -470,7 +470,7 @@ int __init arch_xen_unpopulated_init(struct resource **res) * driver to know how much of the physmap is unpopulated and * set an accurate initial memory target. */ - xen_released_pages += xen_extra_mem[i].n_pfns; + xen_unpopulated_pages += xen_extra_mem[i].n_pfns; /* Zero so region is not also added to the balloon driver. */ xen_extra_mem[i].n_pfns = 0; } diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index 6e54b1d3d8bc2..9e9d081e86bb2 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -697,7 +697,7 @@ static void bfq_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) unsigned int limit, act_idx; /* Sync reads have full depth available */ - if (op_is_sync(opf) && !op_is_write(opf)) + if (blk_mq_is_sync_read(opf)) limit = data->q->nr_requests; else limit = bfqd->async_depths[!!bfqd->wr_busy_queues][op_is_sync(opf)]; diff --git a/block/blk-merge.c b/block/blk-merge.c index 37864c5d287ef..03b61923cf109 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -158,8 +158,9 @@ static struct bio *bio_submit_split(struct bio *bio, int split_sectors) return bio; } -struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, - unsigned *nsegs) +static struct bio *__bio_split_discard(struct bio *bio, + const struct queue_limits *lim, unsigned *nsegs, + unsigned int max_sectors) { unsigned int max_discard_sectors, granularity; sector_t tmp; @@ -169,8 +170,7 @@ struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, granularity = max(lim->discard_granularity >> 9, 1U); - max_discard_sectors = - min(lim->max_discard_sectors, bio_allowed_max_sectors(lim)); + max_discard_sectors = min(max_sectors, bio_allowed_max_sectors(lim)); max_discard_sectors -= max_discard_sectors % granularity; if (unlikely(!max_discard_sectors)) return bio; @@ -194,6 +194,19 @@ struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, return bio_submit_split(bio, split_sectors); } +struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, + unsigned *nsegs) +{ + unsigned int max_sectors; + + if (bio_op(bio) == REQ_OP_SECURE_ERASE) + max_sectors = lim->max_secure_erase_sectors; + else + max_sectors = lim->max_discard_sectors; + + return __bio_split_discard(bio, lim, nsegs, max_sectors); +} + static inline unsigned int blk_boundary_sectors(const struct queue_limits *lim, bool is_atomic) { diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c index 4896525b1c054..553d93b88e194 100644 --- a/block/blk-mq-debugfs.c +++ b/block/blk-mq-debugfs.c @@ -686,8 +686,10 @@ void blk_mq_debugfs_register_hctxs(struct request_queue *q) struct blk_mq_hw_ctx *hctx; unsigned long i; + mutex_lock(&q->debugfs_mutex); queue_for_each_hw_ctx(q, hctx, i) blk_mq_debugfs_register_hctx(q, hctx); + mutex_unlock(&q->debugfs_mutex); } void blk_mq_debugfs_unregister_hctxs(struct request_queue *q) diff --git a/block/blk-mq-sched.h b/block/blk-mq-sched.h index 02c40a72e9598..5678e15bd33c4 100644 --- a/block/blk-mq-sched.h +++ b/block/blk-mq-sched.h @@ -137,4 +137,9 @@ static inline void blk_mq_set_min_shallow_depth(struct request_queue *q, depth); } +static inline bool blk_mq_is_sync_read(blk_opf_t opf) +{ + return op_is_sync(opf) && !op_is_write(opf); +} + #endif diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index e0a70d26972b3..af12526d866a9 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -78,8 +78,14 @@ queue_requests_store(struct gendisk *disk, const char *page, size_t count) /* * Serialize updating nr_requests with concurrent queue_requests_store() * and switching elevator. + * + * Use trylock to avoid circular lock dependency with kernfs active + * reference during concurrent disk deletion: + * update_nr_hwq_lock -> kn->active (via del_gendisk -> kobject_del) + * kn->active -> update_nr_hwq_lock (via this sysfs write path) */ - down_write(&set->update_nr_hwq_lock); + if (!down_write_trylock(&set->update_nr_hwq_lock)) + return -EBUSY; if (nr == q->nr_requests) goto unlock; diff --git a/block/blk.h b/block/blk.h index 37b9b6a95c11c..06dfb5b670179 100644 --- a/block/blk.h +++ b/block/blk.h @@ -208,10 +208,14 @@ static inline unsigned int blk_queue_get_max_sectors(struct request *rq) struct request_queue *q = rq->q; enum req_op op = req_op(rq); - if (unlikely(op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE)) + if (unlikely(op == REQ_OP_DISCARD)) return min(q->limits.max_discard_sectors, UINT_MAX >> SECTOR_SHIFT); + if (unlikely(op == REQ_OP_SECURE_ERASE)) + return min(q->limits.max_secure_erase_sectors, + UINT_MAX >> SECTOR_SHIFT); + if (unlikely(op == REQ_OP_WRITE_ZEROES)) return q->limits.max_write_zeroes_sectors; diff --git a/block/elevator.c b/block/elevator.c index a2f8b2251dc6e..7a97998cd8bd7 100644 --- a/block/elevator.c +++ b/block/elevator.c @@ -806,7 +806,16 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *buf, elv_iosched_load_module(ctx.name); ctx.type = elevator_find_get(ctx.name); - down_read(&set->update_nr_hwq_lock); + /* + * Use trylock to avoid circular lock dependency with kernfs active + * reference during concurrent disk deletion: + * update_nr_hwq_lock -> kn->active (via del_gendisk -> kobject_del) + * kn->active -> update_nr_hwq_lock (via this sysfs write path) + */ + if (!down_read_trylock(&set->update_nr_hwq_lock)) { + ret = -EBUSY; + goto out; + } if (!blk_queue_no_elv_switch(q)) { ret = elevator_change(q, &ctx); if (!ret) @@ -816,6 +825,7 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *buf, } up_read(&set->update_nr_hwq_lock); +out: if (ctx.type) elevator_put(ctx.type); return ret; diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c index 18efd6ef2a2b9..e3eaeea62e24d 100644 --- a/block/kyber-iosched.c +++ b/block/kyber-iosched.c @@ -544,7 +544,7 @@ static void kyber_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) * We use the scheduler tags as per-hardware queue queueing tokens. * Async requests can be limited at this stage. */ - if (!op_is_sync(opf)) { + if (!blk_mq_is_sync_read(opf)) { struct kyber_queue_data *kqd = data->q->elevator->elevator_data; data->shallow_depth = kqd->async_depth; diff --git a/block/mq-deadline.c b/block/mq-deadline.c index 3e3719093aec7..29d00221fbea6 100644 --- a/block/mq-deadline.c +++ b/block/mq-deadline.c @@ -495,7 +495,7 @@ static void dd_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) struct deadline_data *dd = data->q->elevator->elevator_data; /* Do not throttle synchronous reads. */ - if (op_is_sync(opf) && !op_is_write(opf)) + if (blk_mq_is_sync_read(opf)) return; /* diff --git a/drivers/accel/amdxdna/aie2_ctx.c b/drivers/accel/amdxdna/aie2_ctx.c index 75246c481fa50..4610f491f0881 100644 --- a/drivers/accel/amdxdna/aie2_ctx.c +++ b/drivers/accel/amdxdna/aie2_ctx.c @@ -294,7 +294,7 @@ aie2_sched_cmdlist_resp_handler(void *handle, void __iomem *data, size_t size) ret = -EINVAL; goto out; } - amdxdna_cmd_set_state(cmd_abo, fail_cmd_status); + amdxdna_cmd_set_state(cmd_abo, ERT_CMD_STATE_ERROR); if (amdxdna_cmd_get_op(cmd_abo) == ERT_CMD_CHAIN) { struct amdxdna_cmd_chain *cc = amdxdna_cmd_get_payload(cmd_abo, NULL); @@ -317,6 +317,9 @@ aie2_sched_job_run(struct drm_sched_job *sched_job) struct dma_fence *fence; int ret; + if (hwctx->status != HWCTX_STAT_READY) + return NULL; + if (!mmget_not_zero(job->mm)) return ERR_PTR(-ESRCH); @@ -684,7 +687,10 @@ void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx) aie2_hwctx_wait_for_idle(hwctx); /* Request fw to destroy hwctx and cancel the rest pending requests */ + drm_sched_stop(&hwctx->priv->sched, NULL); aie2_release_resource(hwctx); + hwctx->status = HWCTX_STAT_STOP; + drm_sched_start(&hwctx->priv->sched, 0); mutex_unlock(&xdna->dev_lock); drm_sched_entity_destroy(&hwctx->priv->entity); diff --git a/drivers/accel/amdxdna/amdxdna_ctx.c b/drivers/accel/amdxdna/amdxdna_ctx.c index 856fb25086f12..cfee89681ff3c 100644 --- a/drivers/accel/amdxdna/amdxdna_ctx.c +++ b/drivers/accel/amdxdna/amdxdna_ctx.c @@ -104,7 +104,10 @@ void *amdxdna_cmd_get_payload(struct amdxdna_gem_obj *abo, u32 *size) if (size) { count = FIELD_GET(AMDXDNA_CMD_COUNT, cmd->header); - if (unlikely(count <= num_masks)) { + if (unlikely(count <= num_masks || + count * sizeof(u32) + + offsetof(struct amdxdna_cmd, data[0]) > + abo->mem.size)) { *size = 0; return NULL; } diff --git a/drivers/accel/amdxdna/amdxdna_gem.c b/drivers/accel/amdxdna/amdxdna_gem.c index 7f91863c3f24c..a1c9cc4a2b9d8 100644 --- a/drivers/accel/amdxdna/amdxdna_gem.c +++ b/drivers/accel/amdxdna/amdxdna_gem.c @@ -20,8 +20,6 @@ #include "amdxdna_pci_drv.h" #include "amdxdna_ubuf.h" -#define XDNA_MAX_CMD_BO_SIZE SZ_32K - MODULE_IMPORT_NS("DMA_BUF"); static int @@ -745,12 +743,6 @@ amdxdna_drm_create_cmd_bo(struct drm_device *dev, { struct amdxdna_dev *xdna = to_xdna_dev(dev); struct amdxdna_gem_obj *abo; - int ret; - - if (args->size > XDNA_MAX_CMD_BO_SIZE) { - XDNA_ERR(xdna, "Command bo size 0x%llx too large", args->size); - return ERR_PTR(-EINVAL); - } if (args->size < sizeof(struct amdxdna_cmd)) { XDNA_DBG(xdna, "Command BO size 0x%llx too small", args->size); @@ -764,17 +756,7 @@ amdxdna_drm_create_cmd_bo(struct drm_device *dev, abo->type = AMDXDNA_BO_CMD; abo->client = filp->driver_priv; - ret = amdxdna_gem_obj_vmap(abo, &abo->mem.kva); - if (ret) { - XDNA_ERR(xdna, "Vmap cmd bo failed, ret %d", ret); - goto release_obj; - } - return abo; - -release_obj: - drm_gem_object_put(to_gobj(abo)); - return ERR_PTR(ret); } int amdxdna_drm_create_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) @@ -871,6 +853,7 @@ struct amdxdna_gem_obj *amdxdna_gem_get_obj(struct amdxdna_client *client, struct amdxdna_dev *xdna = client->xdna; struct amdxdna_gem_obj *abo; struct drm_gem_object *gobj; + int ret; gobj = drm_gem_object_lookup(client->filp, bo_hdl); if (!gobj) { @@ -879,9 +862,26 @@ struct amdxdna_gem_obj *amdxdna_gem_get_obj(struct amdxdna_client *client, } abo = to_xdna_obj(gobj); - if (bo_type == AMDXDNA_BO_INVALID || abo->type == bo_type) + if (bo_type != AMDXDNA_BO_INVALID && abo->type != bo_type) + goto put_obj; + + if (bo_type != AMDXDNA_BO_CMD || abo->mem.kva) return abo; + if (abo->mem.size > SZ_32K) { + XDNA_ERR(xdna, "Cmd bo is too big %ld", abo->mem.size); + goto put_obj; + } + + ret = amdxdna_gem_obj_vmap(abo, &abo->mem.kva); + if (ret) { + XDNA_ERR(xdna, "Vmap cmd bo failed, ret %d", ret); + goto put_obj; + } + + return abo; + +put_obj: drm_gem_object_put(gobj); return NULL; } diff --git a/drivers/accel/amdxdna/amdxdna_mailbox.c b/drivers/accel/amdxdna/amdxdna_mailbox.c index 6634a4d5717ff..2bacb89cd80c9 100644 --- a/drivers/accel/amdxdna/amdxdna_mailbox.c +++ b/drivers/accel/amdxdna/amdxdna_mailbox.c @@ -112,22 +112,6 @@ static u32 mailbox_reg_read(struct mailbox_channel *mb_chann, u32 mbox_reg) return readl(ringbuf_addr); } -static int mailbox_reg_read_non_zero(struct mailbox_channel *mb_chann, u32 mbox_reg, u32 *val) -{ - struct xdna_mailbox_res *mb_res = &mb_chann->mb->res; - void __iomem *ringbuf_addr = mb_res->mbox_base + mbox_reg; - int ret, value; - - /* Poll till value is not zero */ - ret = readx_poll_timeout(readl, ringbuf_addr, value, - value, 1 /* us */, 100); - if (ret < 0) - return ret; - - *val = value; - return 0; -} - static inline void mailbox_set_headptr(struct mailbox_channel *mb_chann, u32 headptr_val) { @@ -206,26 +190,34 @@ mailbox_send_msg(struct mailbox_channel *mb_chann, struct mailbox_msg *mb_msg) u32 head, tail; u32 start_addr; u32 tmp_tail; + int ret; head = mailbox_get_headptr(mb_chann, CHAN_RES_X2I); tail = mb_chann->x2i_tail; - ringbuf_size = mailbox_get_ringbuf_size(mb_chann, CHAN_RES_X2I); + ringbuf_size = mailbox_get_ringbuf_size(mb_chann, CHAN_RES_X2I) - sizeof(u32); start_addr = mb_chann->res[CHAN_RES_X2I].rb_start_addr; tmp_tail = tail + mb_msg->pkg_size; - if (tail < head && tmp_tail >= head) - goto no_space; - if (tail >= head && (tmp_tail > ringbuf_size - sizeof(u32) && - mb_msg->pkg_size >= head)) - goto no_space; - - if (tail >= head && tmp_tail > ringbuf_size - sizeof(u32)) { +check_again: + if (tail >= head && tmp_tail > ringbuf_size) { write_addr = mb_chann->mb->res.ringbuf_base + start_addr + tail; writel(TOMBSTONE, write_addr); /* tombstone is set. Write from the start of the ringbuf */ tail = 0; + tmp_tail = tail + mb_msg->pkg_size; + } + + if (tail < head && tmp_tail >= head) { + ret = read_poll_timeout(mailbox_get_headptr, head, + tmp_tail < head || tail >= head, + 1, 100, false, mb_chann, CHAN_RES_X2I); + if (ret) + return ret; + + if (tail >= head) + goto check_again; } write_addr = mb_chann->mb->res.ringbuf_base + start_addr + tail; @@ -237,9 +229,6 @@ mailbox_send_msg(struct mailbox_channel *mb_chann, struct mailbox_msg *mb_msg) mb_msg->pkg.header.id); return 0; - -no_space: - return -ENOSPC; } static int @@ -283,8 +272,7 @@ static int mailbox_get_msg(struct mailbox_channel *mb_chann) u32 start_addr; int ret; - if (mailbox_reg_read_non_zero(mb_chann, mb_chann->res[CHAN_RES_I2X].mb_tail_ptr_reg, &tail)) - return -EINVAL; + tail = mailbox_get_tailptr(mb_chann, CHAN_RES_I2X); head = mb_chann->i2x_head; ringbuf_size = mailbox_get_ringbuf_size(mb_chann, CHAN_RES_I2X); start_addr = mb_chann->res[CHAN_RES_I2X].rb_start_addr; diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.c b/drivers/accel/amdxdna/amdxdna_pci_drv.c index 569cd703729d3..88643e28af84b 100644 --- a/drivers/accel/amdxdna/amdxdna_pci_drv.c +++ b/drivers/accel/amdxdna/amdxdna_pci_drv.c @@ -88,6 +88,8 @@ static int amdxdna_drm_open(struct drm_device *ddev, struct drm_file *filp) ret = -ENODEV; goto unbind_sva; } + client->mm = current->mm; + mmgrab(client->mm); init_srcu_struct(&client->hwctx_srcu); xa_init_flags(&client->hwctx_xa, XA_FLAGS_ALLOC); mutex_init(&client->mm_lock); @@ -127,6 +129,7 @@ static void amdxdna_drm_close(struct drm_device *ddev, struct drm_file *filp) drm_gem_object_put(to_gobj(client->dev_heap)); iommu_sva_unbind_device(client->sva); + mmdrop(client->mm); XDNA_DBG(xdna, "pid %d closed", client->pid); kfree(client); @@ -292,7 +295,7 @@ static int amdxdna_probe(struct pci_dev *pdev, const struct pci_device_id *id) fs_reclaim_release(GFP_KERNEL); } - xdna->notifier_wq = alloc_ordered_workqueue("notifier_wq", 0); + xdna->notifier_wq = alloc_ordered_workqueue("notifier_wq", WQ_MEM_RECLAIM); if (!xdna->notifier_wq) return -ENOMEM; diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.h b/drivers/accel/amdxdna/amdxdna_pci_drv.h index 72d6696d49da8..64009ca249827 100644 --- a/drivers/accel/amdxdna/amdxdna_pci_drv.h +++ b/drivers/accel/amdxdna/amdxdna_pci_drv.h @@ -128,6 +128,7 @@ struct amdxdna_client { struct iommu_sva *sva; int pasid; + struct mm_struct *mm; }; #define amdxdna_for_each_hwctx(client, hwctx_id, entry) \ diff --git a/drivers/accel/amdxdna/amdxdna_ubuf.c b/drivers/accel/amdxdna/amdxdna_ubuf.c index 077b2261cf2a0..62a478f6b45fb 100644 --- a/drivers/accel/amdxdna/amdxdna_ubuf.c +++ b/drivers/accel/amdxdna/amdxdna_ubuf.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -34,15 +35,21 @@ static struct sg_table *amdxdna_ubuf_map(struct dma_buf_attachment *attach, ret = sg_alloc_table_from_pages(sg, ubuf->pages, ubuf->nr_pages, 0, ubuf->nr_pages << PAGE_SHIFT, GFP_KERNEL); if (ret) - return ERR_PTR(ret); + goto err_free_sg; if (ubuf->flags & AMDXDNA_UBUF_FLAG_MAP_DMA) { ret = dma_map_sgtable(attach->dev, sg, direction, 0); if (ret) - return ERR_PTR(ret); + goto err_free_table; } return sg; + +err_free_table: + sg_free_table(sg); +err_free_sg: + kfree(sg); + return ERR_PTR(ret); } static void amdxdna_ubuf_unmap(struct dma_buf_attachment *attach, @@ -170,7 +177,10 @@ struct dma_buf *amdxdna_get_ubuf(struct drm_device *dev, goto free_ent; } - exp_info.size += va_ent[i].len; + if (check_add_overflow(exp_info.size, va_ent[i].len, &exp_info.size)) { + ret = -EINVAL; + goto free_ent; + } } ubuf->nr_pages = exp_info.size >> PAGE_SHIFT; diff --git a/drivers/accel/qaic/mhi_controller.c b/drivers/accel/qaic/mhi_controller.c index 13a14c6c61689..4d787f77ce419 100644 --- a/drivers/accel/qaic/mhi_controller.c +++ b/drivers/accel/qaic/mhi_controller.c @@ -39,7 +39,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -55,7 +54,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -71,7 +69,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -87,7 +84,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -103,7 +99,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -119,7 +114,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -135,7 +129,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -151,7 +144,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -167,7 +159,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -183,7 +174,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -199,7 +189,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -215,7 +204,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -231,7 +219,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -247,7 +234,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -263,7 +249,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -279,7 +264,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -295,7 +279,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -311,7 +294,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -327,7 +309,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -343,7 +324,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -359,7 +339,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -375,7 +354,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -391,7 +369,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -407,7 +384,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -423,7 +399,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -439,7 +414,6 @@ static const struct mhi_channel_config aic100_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = true, .wake_capable = false, }, }; @@ -458,7 +432,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -474,7 +447,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -490,7 +462,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -506,7 +477,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -522,7 +492,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -538,7 +507,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -554,7 +522,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -570,7 +537,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -586,7 +552,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -602,7 +567,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -618,7 +582,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -634,7 +597,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -650,7 +612,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -666,7 +627,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -682,7 +642,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -698,7 +657,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -714,7 +672,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, .wake_capable = false, }, { @@ -730,7 +687,6 @@ static const struct mhi_channel_config aic200_channels[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = true, .wake_capable = false, }, }; diff --git a/drivers/accel/rocket/rocket_core.c b/drivers/accel/rocket/rocket_core.c index abe7719c1db46..b3b2fa9ba645a 100644 --- a/drivers/accel/rocket/rocket_core.c +++ b/drivers/accel/rocket/rocket_core.c @@ -59,8 +59,11 @@ int rocket_core_init(struct rocket_core *core) core->iommu_group = iommu_group_get(dev); err = rocket_job_init(core); - if (err) + if (err) { + iommu_group_put(core->iommu_group); + core->iommu_group = NULL; return err; + } pm_runtime_use_autosuspend(dev); @@ -76,7 +79,7 @@ int rocket_core_init(struct rocket_core *core) err = pm_runtime_resume_and_get(dev); if (err) { - rocket_job_fini(core); + rocket_core_fini(core); return err; } diff --git a/drivers/accel/rocket/rocket_drv.c b/drivers/accel/rocket/rocket_drv.c index 5c0b63f0a8f00..f6ef4c7aeef11 100644 --- a/drivers/accel/rocket/rocket_drv.c +++ b/drivers/accel/rocket/rocket_drv.c @@ -13,6 +13,7 @@ #include #include +#include "rocket_device.h" #include "rocket_drv.h" #include "rocket_gem.h" #include "rocket_job.h" @@ -158,6 +159,8 @@ static const struct drm_driver rocket_drm_driver = { static int rocket_probe(struct platform_device *pdev) { + int ret; + if (rdev == NULL) { /* First core probing, initialize DRM device. */ rdev = rocket_device_init(drm_dev, &rocket_drm_driver); @@ -177,7 +180,17 @@ static int rocket_probe(struct platform_device *pdev) rdev->num_cores++; - return rocket_core_init(&rdev->cores[core]); + ret = rocket_core_init(&rdev->cores[core]); + if (ret) { + rdev->num_cores--; + + if (rdev->num_cores == 0) { + rocket_device_fini(rdev); + rdev = NULL; + } + } + + return ret; } static void rocket_remove(struct platform_device *pdev) diff --git a/drivers/acpi/acpi_processor.c b/drivers/acpi/acpi_processor.c index 7ec1dc04fd11b..85096ce7b658b 100644 --- a/drivers/acpi/acpi_processor.c +++ b/drivers/acpi/acpi_processor.c @@ -50,6 +50,7 @@ static int acpi_processor_errata_piix4(struct pci_dev *dev) { u8 value1 = 0; u8 value2 = 0; + struct pci_dev *ide_dev = NULL, *isa_dev = NULL; if (!dev) @@ -107,12 +108,12 @@ static int acpi_processor_errata_piix4(struct pci_dev *dev) * each IDE controller's DMA status to make sure we catch all * DMA activity. */ - dev = pci_get_subsys(PCI_VENDOR_ID_INTEL, + ide_dev = pci_get_subsys(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82371AB, PCI_ANY_ID, PCI_ANY_ID, NULL); - if (dev) { - errata.piix4.bmisx = pci_resource_start(dev, 4); - pci_dev_put(dev); + if (ide_dev) { + errata.piix4.bmisx = pci_resource_start(ide_dev, 4); + pci_dev_put(ide_dev); } /* @@ -124,24 +125,25 @@ static int acpi_processor_errata_piix4(struct pci_dev *dev) * disable C3 support if this is enabled, as some legacy * devices won't operate well if fast DMA is disabled. */ - dev = pci_get_subsys(PCI_VENDOR_ID_INTEL, + isa_dev = pci_get_subsys(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82371AB_0, PCI_ANY_ID, PCI_ANY_ID, NULL); - if (dev) { - pci_read_config_byte(dev, 0x76, &value1); - pci_read_config_byte(dev, 0x77, &value2); + if (isa_dev) { + pci_read_config_byte(isa_dev, 0x76, &value1); + pci_read_config_byte(isa_dev, 0x77, &value2); if ((value1 & 0x80) || (value2 & 0x80)) errata.piix4.fdma = 1; - pci_dev_put(dev); + pci_dev_put(isa_dev); } break; } - if (errata.piix4.bmisx) - dev_dbg(&dev->dev, "Bus master activity detection (BM-IDE) erratum enabled\n"); - if (errata.piix4.fdma) - dev_dbg(&dev->dev, "Type-F DMA livelock erratum (C3 disabled)\n"); + if (ide_dev) + dev_dbg(&ide_dev->dev, "Bus master activity detection (BM-IDE) erratum enabled\n"); + + if (isa_dev) + dev_dbg(&isa_dev->dev, "Type-F DMA livelock erratum (C3 disabled)\n"); return 0; } diff --git a/drivers/acpi/acpica/evregion.c b/drivers/acpi/acpica/evregion.c index fa3475da7ea9b..b6198f73c81df 100644 --- a/drivers/acpi/acpica/evregion.c +++ b/drivers/acpi/acpica/evregion.c @@ -163,7 +163,9 @@ acpi_ev_address_space_dispatch(union acpi_operand_object *region_obj, return_ACPI_STATUS(AE_NOT_EXIST); } - if (region_obj->region.space_id == ACPI_ADR_SPACE_PLATFORM_COMM) { + if (field_obj + && region_obj->region.space_id == + ACPI_ADR_SPACE_PLATFORM_COMM) { struct acpi_pcc_info *ctx = handler_desc->address_space.context; diff --git a/drivers/acpi/acpica/exoparg3.c b/drivers/acpi/acpica/exoparg3.c index bf08110ed6d25..c8c8c4e49563e 100644 --- a/drivers/acpi/acpica/exoparg3.c +++ b/drivers/acpi/acpica/exoparg3.c @@ -10,6 +10,7 @@ #include #include "accommon.h" #include "acinterp.h" +#include #include "acparser.h" #include "amlcode.h" @@ -51,8 +52,7 @@ ACPI_MODULE_NAME("exoparg3") acpi_status acpi_ex_opcode_3A_0T_0R(struct acpi_walk_state *walk_state) { union acpi_operand_object **operand = &walk_state->operands[0]; - struct acpi_signal_fatal_info *fatal; - acpi_status status = AE_OK; + struct acpi_signal_fatal_info fatal; ACPI_FUNCTION_TRACE_STR(ex_opcode_3A_0T_0R, acpi_ps_get_opcode_name(walk_state->opcode)); @@ -60,28 +60,23 @@ acpi_status acpi_ex_opcode_3A_0T_0R(struct acpi_walk_state *walk_state) switch (walk_state->opcode) { case AML_FATAL_OP: /* Fatal (fatal_type fatal_code fatal_arg) */ - ACPI_DEBUG_PRINT((ACPI_DB_INFO, - "FatalOp: Type %X Code %X Arg %X " - "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n", - (u32)operand[0]->integer.value, - (u32)operand[1]->integer.value, - (u32)operand[2]->integer.value)); - - fatal = ACPI_ALLOCATE(sizeof(struct acpi_signal_fatal_info)); - if (fatal) { - fatal->type = (u32) operand[0]->integer.value; - fatal->code = (u32) operand[1]->integer.value; - fatal->argument = (u32) operand[2]->integer.value; - } + fatal.type = (u32)operand[0]->integer.value; + fatal.code = (u32)operand[1]->integer.value; + fatal.argument = (u32)operand[2]->integer.value; - /* Always signal the OS! */ + ACPI_BIOS_ERROR((AE_INFO, + "Fatal ACPI BIOS error (Type 0x%X Code 0x%X Arg 0x%X)\n", + fatal.type, fatal.code, fatal.argument)); - status = acpi_os_signal(ACPI_SIGNAL_FATAL, fatal); + /* Always signal the OS! */ - /* Might return while OS is shutting down, just continue */ + acpi_os_signal(ACPI_SIGNAL_FATAL, &fatal); - ACPI_FREE(fatal); - goto cleanup; + /* + * Might return while OS is shutting down, so abort the AML execution + * by returning an error. + */ + return_ACPI_STATUS(AE_ERROR); case AML_EXTERNAL_OP: /* @@ -93,21 +88,16 @@ acpi_status acpi_ex_opcode_3A_0T_0R(struct acpi_walk_state *walk_state) * wrong if an external opcode ever gets here. */ ACPI_ERROR((AE_INFO, "Executed External Op")); - status = AE_OK; - goto cleanup; + + return_ACPI_STATUS(AE_OK); default: ACPI_ERROR((AE_INFO, "Unknown AML opcode 0x%X", walk_state->opcode)); - status = AE_AML_BAD_OPCODE; - goto cleanup; + return_ACPI_STATUS(AE_AML_BAD_OPCODE); } - -cleanup: - - return_ACPI_STATUS(status); } /******************************************************************************* diff --git a/drivers/acpi/apei/Makefile b/drivers/acpi/apei/Makefile index 2c474e6477e12..1a0b85923cd42 100644 --- a/drivers/acpi/apei/Makefile +++ b/drivers/acpi/apei/Makefile @@ -1,6 +1,11 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_ACPI_APEI) += apei.o obj-$(CONFIG_ACPI_APEI_GHES) += ghes.o +# clang versions prior to 18 may blow out the stack with KASAN +ifeq ($(CONFIG_COMPILE_TEST)_$(CONFIG_CC_IS_CLANG)_$(call clang-min-version, 180000),y_y_) +KASAN_SANITIZE_ghes.o := n +endif +obj-$(CONFIG_ACPI_APEI_PCIEAER) += ghes_helpers.o obj-$(CONFIG_ACPI_APEI_EINJ) += einj.o einj-y := einj-core.o einj-$(CONFIG_ACPI_APEI_EINJ_CXL) += einj-cxl.o diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index 56107aa002744..fc96a5e234f06 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -293,6 +294,7 @@ static struct ghes *ghes_new(struct acpi_hest_generic *generic) error_block_length = GHES_ESTATUS_MAX_SIZE; } ghes->estatus = kmalloc(error_block_length, GFP_KERNEL); + ghes->estatus_length = error_block_length; if (!ghes->estatus) { rc = -ENOMEM; goto err_unmap_status_addr; @@ -364,13 +366,15 @@ static int __ghes_check_estatus(struct ghes *ghes, struct acpi_hest_generic_status *estatus) { u32 len = cper_estatus_len(estatus); + u32 max_len = min(ghes->generic->error_block_length, + ghes->estatus_length); if (len < sizeof(*estatus)) { pr_warn_ratelimited(FW_WARN GHES_PFX "Truncated error status block!\n"); return -EIO; } - if (len > ghes->generic->error_block_length) { + if (!len || len > max_len) { pr_warn_ratelimited(FW_WARN GHES_PFX "Invalid error status block length!\n"); return -EIO; } @@ -557,21 +561,45 @@ static bool ghes_handle_arm_hw_error(struct acpi_hest_generic_data *gdata, { struct cper_sec_proc_arm *err = acpi_hest_get_payload(gdata); int flags = sync ? MF_ACTION_REQUIRED : 0; + int length = gdata->error_data_length; char error_type[120]; bool queued = false; int sec_sev, i; char *p; sec_sev = ghes_severity(gdata->error_severity); - log_arm_hw_error(err, sec_sev); + if (length >= sizeof(*err)) { + log_arm_hw_error(err, sec_sev); + } else { + pr_warn(FW_BUG "arm error length: %d\n", length); + pr_warn(FW_BUG "length is too small\n"); + pr_warn(FW_BUG "firmware-generated error record is incorrect\n"); + return false; + } + if (sev != GHES_SEV_RECOVERABLE || sec_sev != GHES_SEV_RECOVERABLE) return false; p = (char *)(err + 1); + length -= sizeof(err); + for (i = 0; i < err->err_info_num; i++) { - struct cper_arm_err_info *err_info = (struct cper_arm_err_info *)p; - bool is_cache = err_info->type & CPER_ARM_CACHE_ERROR; - bool has_pa = (err_info->validation_bits & CPER_ARM_INFO_VALID_PHYSICAL_ADDR); + struct cper_arm_err_info *err_info; + bool is_cache, has_pa; + + /* Ensure we have enough data for the error info header */ + if (length < sizeof(*err_info)) + break; + + err_info = (struct cper_arm_err_info *)p; + + /* Validate the claimed length before using it */ + length -= err_info->length; + if (length < 0) + break; + + is_cache = err_info->type & CPER_ARM_CACHE_ERROR; + has_pa = (err_info->validation_bits & CPER_ARM_INFO_VALID_PHYSICAL_ADDR); /* * The field (err_info->error_info & BIT(26)) is fixed to set to @@ -718,24 +746,8 @@ static void cxl_cper_post_prot_err(struct cxl_cper_sec_prot_err *prot_err, struct cxl_cper_prot_err_work_data wd; u8 *dvsec_start, *cap_start; - if (!(prot_err->valid_bits & PROT_ERR_VALID_AGENT_ADDRESS)) { - pr_err_ratelimited("CXL CPER invalid agent type\n"); + if (cxl_cper_sec_prot_err_valid(prot_err)) return; - } - - if (!(prot_err->valid_bits & PROT_ERR_VALID_ERROR_LOG)) { - pr_err_ratelimited("CXL CPER invalid protocol error log\n"); - return; - } - - if (prot_err->err_len != sizeof(struct cxl_ras_capability_regs)) { - pr_err_ratelimited("CXL CPER invalid RAS Cap size (%u)\n", - prot_err->err_len); - return; - } - - if (!(prot_err->valid_bits & PROT_ERR_VALID_SERIAL_NUMBER)) - pr_warn(FW_WARN "CXL CPER no device serial number\n"); guard(spinlock_irqsave)(&cxl_cper_prot_err_work_lock); diff --git a/drivers/acpi/apei/ghes_helpers.c b/drivers/acpi/apei/ghes_helpers.c new file mode 100644 index 0000000000000..f3d162139a974 --- /dev/null +++ b/drivers/acpi/apei/ghes_helpers.c @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright(c) 2025 Intel Corporation. All rights reserved + +#include +#include + +int cxl_cper_sec_prot_err_valid(struct cxl_cper_sec_prot_err *prot_err) +{ + if (!(prot_err->valid_bits & PROT_ERR_VALID_AGENT_ADDRESS)) { + pr_err_ratelimited("CXL CPER invalid agent type\n"); + return -EINVAL; + } + + if (!(prot_err->valid_bits & PROT_ERR_VALID_ERROR_LOG)) { + pr_err_ratelimited("CXL CPER invalid protocol error log\n"); + return -EINVAL; + } + + if (prot_err->err_len != sizeof(struct cxl_ras_capability_regs)) { + pr_err_ratelimited("CXL CPER invalid RAS Cap size (%u)\n", + prot_err->err_len); + return -EINVAL; + } + + if ((prot_err->agent_type == RCD || prot_err->agent_type == DEVICE || + prot_err->agent_type == LD || prot_err->agent_type == FMLD) && + !(prot_err->valid_bits & PROT_ERR_VALID_SERIAL_NUMBER)) + pr_warn_ratelimited(FW_WARN + "CXL CPER no device serial number\n"); + + return 0; +} +EXPORT_SYMBOL_GPL(cxl_cper_sec_prot_err_valid); diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c index 67b76492c839c..8196c17b5a970 100644 --- a/drivers/acpi/battery.c +++ b/drivers/acpi/battery.c @@ -212,7 +212,14 @@ static int acpi_battery_get_property(struct power_supply *psy, if (battery->state & ACPI_BATTERY_STATE_DISCHARGING) val->intval = acpi_battery_handle_discharging(battery); else if (battery->state & ACPI_BATTERY_STATE_CHARGING) - val->intval = POWER_SUPPLY_STATUS_CHARGING; + /* Validate the status by checking the current. */ + if (battery->rate_now != ACPI_BATTERY_VALUE_UNKNOWN && + battery->rate_now == 0) { + /* On charge but no current (0W/0mA). */ + val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; + } else { + val->intval = POWER_SUPPLY_STATUS_CHARGING; + } else if (battery->state & ACPI_BATTERY_STATE_CHARGE_LIMITING) val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; else if (acpi_battery_is_charged(battery)) diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index e66e20d1f31b7..b59b0100d03c5 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -362,7 +362,7 @@ static int send_pcc_cmd(int pcc_ss_id, u16 cmd) end: if (cmd == CMD_WRITE) { if (unlikely(ret)) { - for_each_possible_cpu(i) { + for_each_online_cpu(i) { struct cpc_desc *desc = per_cpu(cpc_desc_ptr, i); if (!desc) @@ -524,7 +524,7 @@ int acpi_get_psd_map(unsigned int cpu, struct cppc_cpudata *cpu_data) else if (pdomain->coord_type == DOMAIN_COORD_TYPE_SW_ANY) cpu_data->shared_type = CPUFREQ_SHARED_TYPE_ANY; - for_each_possible_cpu(i) { + for_each_online_cpu(i) { if (i == cpu) continue; diff --git a/drivers/acpi/power.c b/drivers/acpi/power.c index 361a7721a6a87..7da5ae5594a72 100644 --- a/drivers/acpi/power.c +++ b/drivers/acpi/power.c @@ -1113,6 +1113,19 @@ static const struct dmi_system_id dmi_leave_unused_power_resources_on[] = { DMI_MATCH(DMI_PRODUCT_NAME, "SATELLITE Click Mini L9W-B"), }, }, + { + /* + * THUNDEROBOT ZERO laptop: Due to its SSDT table bug, power + * resource 'PXP' will be shut down on initialization, making + * the NVMe #2 and the NVIDIA dGPU both unavailable (they're + * both controlled by 'PXP'). + */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "THUNDEROBOT"), + DMI_MATCH(DMI_PRODUCT_NAME, "ZERO"), + } + + }, {} }; diff --git a/drivers/acpi/resource.c b/drivers/acpi/resource.c index d16906f46484d..bc8050d8a6f51 100644 --- a/drivers/acpi/resource.c +++ b/drivers/acpi/resource.c @@ -532,6 +532,12 @@ static const struct dmi_system_id irq1_level_low_skip_override[] = { DMI_MATCH(DMI_BOARD_NAME, "16T90SP"), }, }, + { + /* JWIPC JVC9100 */ + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "JVC9100"), + }, + }, { } }; @@ -706,6 +712,8 @@ struct irq_override_cmp { static const struct irq_override_cmp override_table[] = { { irq1_level_low_skip_override, 1, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 0, false }, + { irq1_level_low_skip_override, 10, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 1, false }, + { irq1_level_low_skip_override, 11, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 1, false }, { irq1_edge_low_force_override, 1, ACPI_EDGE_SENSITIVE, ACPI_ACTIVE_LOW, 1, true }, }; diff --git a/drivers/acpi/x86/s2idle.c b/drivers/acpi/x86/s2idle.c index dd0b40b9bbe8b..377a268867c21 100644 --- a/drivers/acpi/x86/s2idle.c +++ b/drivers/acpi/x86/s2idle.c @@ -45,6 +45,7 @@ static const struct acpi_device_id lps0_device_ids[] = { #define ACPI_LPS0_EXIT 6 #define ACPI_LPS0_MS_ENTRY 7 #define ACPI_LPS0_MS_EXIT 8 +#define ACPI_MS_TURN_ON_DISPLAY 9 /* AMD */ #define ACPI_LPS0_DSM_UUID_AMD "e3f32452-febc-43ce-9039-932122d37721" @@ -373,6 +374,8 @@ static const char *acpi_sleep_dsm_state_to_str(unsigned int state) return "lps0 ms entry"; case ACPI_LPS0_MS_EXIT: return "lps0 ms exit"; + case ACPI_MS_TURN_ON_DISPLAY: + return "lps0 ms turn on display"; } } else { switch (state) { @@ -619,6 +622,9 @@ void acpi_s2idle_restore_early(void) if (lps0_dsm_func_mask_microsoft > 0) { acpi_sleep_run_lps0_dsm(ACPI_LPS0_EXIT, lps0_dsm_func_mask_microsoft, lps0_dsm_guid_microsoft); + /* Intent to turn on display */ + acpi_sleep_run_lps0_dsm(ACPI_MS_TURN_ON_DISPLAY, + lps0_dsm_func_mask_microsoft, lps0_dsm_guid_microsoft); /* Modern Standby exit */ acpi_sleep_run_lps0_dsm(ACPI_LPS0_MS_EXIT, lps0_dsm_func_mask_microsoft, lps0_dsm_guid_microsoft); diff --git a/drivers/acpi/x86/utils.c b/drivers/acpi/x86/utils.c index 4ee30c2897a2b..418951639f511 100644 --- a/drivers/acpi/x86/utils.c +++ b/drivers/acpi/x86/utils.c @@ -81,6 +81,18 @@ static const struct override_status_id override_status_ids[] = { DMI_MATCH(DMI_PRODUCT_NAME, "Mipad2"), }), + /* + * Lenovo Yoga Book uses PWM2 for touch keyboard backlight control. + * It needs to be enabled only for the Android device version (YB1-X90* + * aka YETI-11); the Windows version (YB1-X91*) uses ACPI control + * methods. + */ + PRESENT_ENTRY_HID("80862289", "2", INTEL_ATOM_AIRMONT, { + DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Intel Corporation"), + DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "CHERRYVIEW D1 PLATFORM"), + DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "YETI-11"), + }), + /* * The INT0002 device is necessary to clear wakeup interrupt sources * on Cherry Trail devices, without it we get nobody cared IRQ msgs. diff --git a/drivers/android/binder.c b/drivers/android/binder.c index 8e2989fb56a71..a6b3bfe0d9b81 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -4523,7 +4523,7 @@ static int binder_thread_write(struct binder_proc *proc, } } binder_debug(BINDER_DEBUG_DEAD_BINDER, - "%d:%d BC_DEAD_BINDER_DONE %016llx found %pK\n", + "%d:%d BC_DEAD_BINDER_DONE %016llx found %p\n", proc->pid, thread->pid, (u64)cookie, death); if (death == NULL) { diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c index 979c96b74cad3..d5ed64543bbf4 100644 --- a/drivers/android/binder_alloc.c +++ b/drivers/android/binder_alloc.c @@ -81,7 +81,7 @@ static void binder_insert_free_buffer(struct binder_alloc *alloc, new_buffer_size = binder_alloc_buffer_size(alloc, new_buffer); binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, - "%d: add free buffer, size %zd, at %pK\n", + "%d: add free buffer, size %zd, at %p\n", alloc->pid, new_buffer_size, new_buffer); while (*p) { @@ -572,7 +572,7 @@ static struct binder_buffer *binder_alloc_new_buf_locked( } binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, - "%d: binder_alloc_buf size %zd got buffer %pK size %zd\n", + "%d: binder_alloc_buf size %zd got buffer %p size %zd\n", alloc->pid, size, buffer, buffer_size); /* @@ -748,7 +748,7 @@ static void binder_free_buf_locked(struct binder_alloc *alloc, ALIGN(buffer->extra_buffers_size, sizeof(void *)); binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, - "%d: binder_free_buf %pK size %zd buffer_size %zd\n", + "%d: binder_free_buf %p size %zd buffer_size %zd\n", alloc->pid, buffer, size, buffer_size); BUG_ON(buffer->free); diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index 699919e4579e1..1a57560ecc90f 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -2358,6 +2358,24 @@ static bool ata_dev_check_adapter(struct ata_device *dev, return false; } +bool ata_adapter_is_online(struct ata_port *ap) +{ + struct device *dev; + + if (!ap || !ap->host) + return false; + + dev = ap->host->dev; + if (!dev) + return false; + + if (dev_is_pci(dev) && + pci_channel_offline(to_pci_dev(dev))) + return false; + + return true; +} + static int ata_dev_config_ncq(struct ata_device *dev, char *desc, size_t desc_sz) { @@ -5067,6 +5085,12 @@ void ata_qc_issue(struct ata_queued_cmd *qc) qc->flags |= ATA_QCFLAG_ACTIVE; ap->qc_active |= 1ULL << qc->tag; + /* Make sure the device is still accessible. */ + if (!ata_adapter_is_online(ap)) { + qc->err_mask |= AC_ERR_HOST_BUS; + goto sys_err; + } + /* * We guarantee to LLDs that they will have at least one * non-zero sg if the command is a data command. @@ -5552,6 +5576,7 @@ struct ata_port *ata_port_alloc(struct ata_host *host) mutex_init(&ap->scsi_scan_mutex); INIT_DELAYED_WORK(&ap->hotplug_task, ata_scsi_hotplug); INIT_DELAYED_WORK(&ap->scsi_rescan_task, ata_scsi_dev_rescan); + INIT_WORK(&ap->deferred_qc_work, ata_scsi_deferred_qc_work); INIT_LIST_HEAD(&ap->eh_done_q); init_waitqueue_head(&ap->eh_wait_q); init_completion(&ap->park_req_pending); @@ -6172,9 +6197,11 @@ static void ata_port_detach(struct ata_port *ap) /* wait till EH commits suicide */ ata_port_wait_eh(ap); - /* it better be dead now */ + /* It better be dead now and not have any remaining deferred qc. */ WARN_ON(!(ap->pflags & ATA_PFLAG_UNLOADED)); + WARN_ON(ap->deferred_qc); + cancel_work_sync(&ap->deferred_qc_work); cancel_delayed_work_sync(&ap->hotplug_task); cancel_delayed_work_sync(&ap->scsi_rescan_task); diff --git a/drivers/ata/libata-eh.c b/drivers/ata/libata-eh.c index 2586e77ebf45d..23be85418b3b1 100644 --- a/drivers/ata/libata-eh.c +++ b/drivers/ata/libata-eh.c @@ -640,12 +640,29 @@ void ata_scsi_cmd_error_handler(struct Scsi_Host *host, struct ata_port *ap, set_host_byte(scmd, DID_OK); ata_qc_for_each_raw(ap, qc, i) { - if (qc->flags & ATA_QCFLAG_ACTIVE && - qc->scsicmd == scmd) + if (qc->scsicmd != scmd) + continue; + if ((qc->flags & ATA_QCFLAG_ACTIVE) || + qc == ap->deferred_qc) break; } - if (i < ATA_MAX_QUEUE) { + if (i < ATA_MAX_QUEUE && qc == ap->deferred_qc) { + /* + * This is a deferred command that timed out while + * waiting for the command queue to drain. Since the qc + * is not active yet (deferred_qc is still set, so the + * deferred qc work has not issued the command yet), + * simply signal the timeout by finishing the SCSI + * command and clear the deferred qc to prevent the + * deferred qc work from issuing this qc. + */ + WARN_ON_ONCE(qc->flags & ATA_QCFLAG_ACTIVE); + ap->deferred_qc = NULL; + cancel_work(&ap->deferred_qc_work); + set_host_byte(scmd, DID_TIME_OUT); + scsi_eh_finish_cmd(scmd, &ap->eh_done_q); + } else if (i < ATA_MAX_QUEUE) { /* the scmd has an associated qc */ if (!(qc->flags & ATA_QCFLAG_EH)) { /* which hasn't failed yet, timeout */ @@ -736,7 +753,8 @@ void ata_scsi_port_error_handler(struct Scsi_Host *host, struct ata_port *ap) spin_unlock_irqrestore(ap->lock, flags); /* invoke EH, skip if unloading or suspended */ - if (!(ap->pflags & (ATA_PFLAG_UNLOADING | ATA_PFLAG_SUSPENDED))) + if (!(ap->pflags & (ATA_PFLAG_UNLOADING | ATA_PFLAG_SUSPENDED)) && + ata_adapter_is_online(ap)) ap->ops->error_handler(ap); else { /* if unloading, commence suicide */ @@ -917,6 +935,12 @@ static void ata_eh_set_pending(struct ata_port *ap, bool fastdrain) ap->pflags |= ATA_PFLAG_EH_PENDING; + /* + * If we have a deferred qc, requeue it so that it is retried once EH + * completes. + */ + ata_scsi_requeue_deferred_qc(ap); + if (!fastdrain) return; diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c index 434774e71fe61..a70d98405a797 100644 --- a/drivers/ata/libata-scsi.c +++ b/drivers/ata/libata-scsi.c @@ -1657,8 +1657,78 @@ static void ata_qc_done(struct ata_queued_cmd *qc) done(cmd); } +void ata_scsi_deferred_qc_work(struct work_struct *work) +{ + struct ata_port *ap = + container_of(work, struct ata_port, deferred_qc_work); + struct ata_queued_cmd *qc; + unsigned long flags; + + spin_lock_irqsave(ap->lock, flags); + + /* + * If we still have a deferred qc and we are not in EH, issue it. In + * such case, we should not need any more deferring the qc, so warn if + * qc_defer() says otherwise. + */ + qc = ap->deferred_qc; + if (qc && !ata_port_eh_scheduled(ap)) { + WARN_ON_ONCE(ap->ops->qc_defer(qc)); + ap->deferred_qc = NULL; + ata_qc_issue(qc); + } + + spin_unlock_irqrestore(ap->lock, flags); +} + +void ata_scsi_requeue_deferred_qc(struct ata_port *ap) +{ + struct ata_queued_cmd *qc = ap->deferred_qc; + struct scsi_cmnd *scmd; + + lockdep_assert_held(ap->lock); + + /* + * If we have a deferred qc when a reset occurs or NCQ commands fail, + * do not try to be smart about what to do with this deferred command + * and simply retry it by completing it with DID_SOFT_ERROR. + */ + if (!qc) + return; + + scmd = qc->scsicmd; + ap->deferred_qc = NULL; + cancel_work(&ap->deferred_qc_work); + ata_qc_free(qc); + scmd->result = (DID_SOFT_ERROR << 16); + scsi_done(scmd); +} + +static void ata_scsi_schedule_deferred_qc(struct ata_port *ap) +{ + struct ata_queued_cmd *qc = ap->deferred_qc; + + lockdep_assert_held(ap->lock); + + /* + * If we have a deferred qc, then qc_defer() is defined and we can use + * this callback to determine if this qc is good to go, unless EH has + * been scheduled. + */ + if (!qc) + return; + + if (ata_port_eh_scheduled(ap)) { + ata_scsi_requeue_deferred_qc(ap); + return; + } + if (!ap->ops->qc_defer(qc)) + queue_work(system_highpri_wq, &ap->deferred_qc_work); +} + static void ata_scsi_qc_complete(struct ata_queued_cmd *qc) { + struct ata_port *ap = qc->ap; struct scsi_cmnd *cmd = qc->scsicmd; u8 *cdb = cmd->cmnd; bool have_sense = qc->flags & ATA_QCFLAG_SENSE_VALID; @@ -1688,6 +1758,66 @@ static void ata_scsi_qc_complete(struct ata_queued_cmd *qc) } ata_qc_done(qc); + + ata_scsi_schedule_deferred_qc(ap); +} + +static int ata_scsi_qc_issue(struct ata_port *ap, struct ata_queued_cmd *qc) +{ + int ret; + + if (!ap->ops->qc_defer) + goto issue; + + /* + * If we already have a deferred qc, then rely on the SCSI layer to + * requeue and defer all incoming commands until the deferred qc is + * processed, once all on-going commands complete. + */ + if (ap->deferred_qc) { + ata_qc_free(qc); + return SCSI_MLQUEUE_DEVICE_BUSY; + } + + /* Check if the command needs to be deferred. */ + ret = ap->ops->qc_defer(qc); + switch (ret) { + case 0: + break; + case ATA_DEFER_LINK: + ret = SCSI_MLQUEUE_DEVICE_BUSY; + break; + case ATA_DEFER_PORT: + ret = SCSI_MLQUEUE_HOST_BUSY; + break; + default: + WARN_ON_ONCE(1); + ret = SCSI_MLQUEUE_HOST_BUSY; + break; + } + + if (ret) { + /* + * We must defer this qc: if this is not an NCQ command, keep + * this qc as a deferred one and report to the SCSI layer that + * we issued it so that it is not requeued. The deferred qc will + * be issued with the port deferred_qc_work once all on-going + * commands complete. + */ + if (!ata_is_ncq(qc->tf.protocol)) { + ap->deferred_qc = qc; + return 0; + } + + /* Force a requeue of the command to defer its execution. */ + ata_qc_free(qc); + return ret; + } + +issue: + ata_qc_issue(qc); + + return 0; } /** @@ -1713,66 +1843,49 @@ static void ata_scsi_qc_complete(struct ata_queued_cmd *qc) * spin_lock_irqsave(host lock) * * RETURNS: - * 0 on success, SCSI_ML_QUEUE_DEVICE_BUSY if the command - * needs to be deferred. + * 0 on success, SCSI_ML_QUEUE_DEVICE_BUSY or SCSI_MLQUEUE_HOST_BUSY if the + * command needs to be deferred. */ static int ata_scsi_translate(struct ata_device *dev, struct scsi_cmnd *cmd, ata_xlat_func_t xlat_func) { struct ata_port *ap = dev->link->ap; struct ata_queued_cmd *qc; - int rc; + lockdep_assert_held(ap->lock); + + /* + * ata_scsi_qc_new() calls scsi_done(cmd) in case of failure. So we + * have nothing further to do when allocating a qc fails. + */ qc = ata_scsi_qc_new(dev, cmd); if (!qc) - goto err_mem; + return 0; /* data is present; dma-map it */ if (cmd->sc_data_direction == DMA_FROM_DEVICE || cmd->sc_data_direction == DMA_TO_DEVICE) { if (unlikely(scsi_bufflen(cmd) < 1)) { ata_dev_warn(dev, "WARNING: zero len r/w req\n"); - goto err_did; + cmd->result = (DID_ERROR << 16); + goto done; } ata_sg_init(qc, scsi_sglist(cmd), scsi_sg_count(cmd)); - qc->dma_dir = cmd->sc_data_direction; } qc->complete_fn = ata_scsi_qc_complete; if (xlat_func(qc)) - goto early_finish; + goto done; - if (ap->ops->qc_defer) { - if ((rc = ap->ops->qc_defer(qc))) - goto defer; - } - - /* select device, send command to hardware */ - ata_qc_issue(qc); - - return 0; + return ata_scsi_qc_issue(ap, qc); -early_finish: +done: ata_qc_free(qc); scsi_done(cmd); return 0; - -err_did: - ata_qc_free(qc); - cmd->result = (DID_ERROR << 16); - scsi_done(cmd); -err_mem: - return 0; - -defer: - ata_qc_free(qc); - if (rc == ATA_DEFER_LINK) - return SCSI_MLQUEUE_DEVICE_BUSY; - else - return SCSI_MLQUEUE_HOST_BUSY; } /** @@ -2981,6 +3094,9 @@ ata_scsi_find_dev(struct ata_port *ap, const struct scsi_device *scsidev) { struct ata_device *dev = __ata_scsi_find_dev(ap, scsidev); + if (!ata_adapter_is_online(ap)) + return NULL; + if (unlikely(!dev || !ata_dev_enabled(dev))) return NULL; diff --git a/drivers/ata/libata.h b/drivers/ata/libata.h index e5b977a8d3e1e..0002994ddfc9b 100644 --- a/drivers/ata/libata.h +++ b/drivers/ata/libata.h @@ -94,6 +94,7 @@ extern int atapi_check_dma(struct ata_queued_cmd *qc); extern void swap_buf_le16(u16 *buf, unsigned int buf_words); extern bool ata_phys_link_online(struct ata_link *link); extern bool ata_phys_link_offline(struct ata_link *link); +bool ata_adapter_is_online(struct ata_port *ap); extern void ata_dev_init(struct ata_device *dev); extern void ata_link_init(struct ata_port *ap, struct ata_link *link, int pmp); extern int sata_link_init_spd(struct ata_link *link); @@ -161,6 +162,8 @@ void ata_scsi_sdev_config(struct scsi_device *sdev); int ata_scsi_dev_config(struct scsi_device *sdev, struct queue_limits *lim, struct ata_device *dev); int __ata_scsi_queuecmd(struct scsi_cmnd *scmd, struct ata_device *dev); +void ata_scsi_deferred_qc_work(struct work_struct *work); +void ata_scsi_requeue_deferred_qc(struct ata_port *ap); /* libata-eh.c */ extern unsigned int ata_internal_cmd_timeout(struct ata_device *dev, u8 cmd); diff --git a/drivers/ata/pata_ftide010.c b/drivers/ata/pata_ftide010.c index c3a8384c3e04d..c41da296eb389 100644 --- a/drivers/ata/pata_ftide010.c +++ b/drivers/ata/pata_ftide010.c @@ -122,10 +122,10 @@ static const u8 mwdma_50_active_time[3] = {6, 2, 2}; static const u8 mwdma_50_recovery_time[3] = {6, 2, 1}; static const u8 mwdma_66_active_time[3] = {8, 3, 3}; static const u8 mwdma_66_recovery_time[3] = {8, 2, 1}; -static const u8 udma_50_setup_time[6] = {3, 3, 2, 2, 1, 1}; +static const u8 udma_50_setup_time[6] = {3, 3, 2, 2, 1, 9}; static const u8 udma_50_hold_time[6] = {3, 1, 1, 1, 1, 1}; -static const u8 udma_66_setup_time[7] = {4, 4, 3, 2, }; -static const u8 udma_66_hold_time[7] = {}; +static const u8 udma_66_setup_time[7] = {4, 4, 3, 2, 1, 9, 9}; +static const u8 udma_66_hold_time[7] = {4, 2, 1, 1, 1, 1, 1}; /* * We set 66 MHz for all MWDMA modes diff --git a/drivers/atm/fore200e.c b/drivers/atm/fore200e.c index f62e385714403..fec081db36dc4 100644 --- a/drivers/atm/fore200e.c +++ b/drivers/atm/fore200e.c @@ -373,6 +373,10 @@ fore200e_shutdown(struct fore200e* fore200e) fallthrough; case FORE200E_STATE_IRQ: free_irq(fore200e->irq, fore200e->atm_dev); +#ifdef FORE200E_USE_TASKLET + tasklet_kill(&fore200e->tx_tasklet); + tasklet_kill(&fore200e->rx_tasklet); +#endif fallthrough; case FORE200E_STATE_ALLOC_BUF: diff --git a/drivers/auxdisplay/arm-charlcd.c b/drivers/auxdisplay/arm-charlcd.c index a7eae99a48f77..4e22882f57c9c 100644 --- a/drivers/auxdisplay/arm-charlcd.c +++ b/drivers/auxdisplay/arm-charlcd.c @@ -323,7 +323,7 @@ static int __init charlcd_probe(struct platform_device *pdev) out_no_irq: iounmap(lcd->virtbase); out_no_memregion: - release_mem_region(lcd->phybase, SZ_4K); + release_mem_region(lcd->phybase, lcd->physize); out_no_resource: kfree(lcd); return ret; diff --git a/drivers/base/faux.c b/drivers/base/faux.c index 21dd02124231a..23d7258172325 100644 --- a/drivers/base/faux.c +++ b/drivers/base/faux.c @@ -29,9 +29,7 @@ struct faux_object { }; #define to_faux_object(dev) container_of_const(dev, struct faux_object, faux_dev.dev) -static struct device faux_bus_root = { - .init_name = "faux", -}; +static struct device *faux_bus_root; static int faux_match(struct device *dev, const struct device_driver *drv) { @@ -152,7 +150,7 @@ struct faux_device *faux_device_create_with_groups(const char *name, if (parent) dev->parent = parent; else - dev->parent = &faux_bus_root; + dev->parent = faux_bus_root; dev->bus = &faux_bus_type; dev_set_name(dev, "%s", name); device_set_pm_not_required(dev); @@ -236,9 +234,15 @@ int __init faux_bus_init(void) { int ret; - ret = device_register(&faux_bus_root); + faux_bus_root = kzalloc(sizeof(*faux_bus_root), GFP_KERNEL); + if (!faux_bus_root) + return -ENOMEM; + + dev_set_name(faux_bus_root, "faux"); + + ret = device_register(faux_bus_root); if (ret) { - put_device(&faux_bus_root); + put_device(faux_bus_root); return ret; } @@ -256,6 +260,6 @@ int __init faux_bus_init(void) bus_unregister(&faux_bus_type); error_bus: - device_unregister(&faux_bus_root); + device_unregister(faux_bus_root); return ret; } diff --git a/drivers/base/power/wakeirq.c b/drivers/base/power/wakeirq.c index 8aa28c08b2891..c0809d18fc540 100644 --- a/drivers/base/power/wakeirq.c +++ b/drivers/base/power/wakeirq.c @@ -83,13 +83,16 @@ EXPORT_SYMBOL_GPL(dev_pm_set_wake_irq); */ void dev_pm_clear_wake_irq(struct device *dev) { - struct wake_irq *wirq = dev->power.wakeirq; + struct wake_irq *wirq; unsigned long flags; - if (!wirq) + spin_lock_irqsave(&dev->power.lock, flags); + wirq = dev->power.wakeirq; + if (!wirq) { + spin_unlock_irqrestore(&dev->power.lock, flags); return; + } - spin_lock_irqsave(&dev->power.lock, flags); device_wakeup_detach_irq(dev); dev->power.wakeirq = NULL; spin_unlock_irqrestore(&dev->power.lock, flags); diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c index d1283ff1080bb..2f630df16bfe1 100644 --- a/drivers/base/power/wakeup.c +++ b/drivers/base/power/wakeup.c @@ -276,9 +276,7 @@ EXPORT_SYMBOL_GPL(wakeup_sources_read_unlock); */ struct wakeup_source *wakeup_sources_walk_start(void) { - struct list_head *ws_head = &wakeup_sources; - - return list_entry_rcu(ws_head->next, struct wakeup_source, entry); + return list_first_or_null_rcu(&wakeup_sources, struct wakeup_source, entry); } EXPORT_SYMBOL_GPL(wakeup_sources_walk_start); diff --git a/drivers/block/drbd/drbd_actlog.c b/drivers/block/drbd/drbd_actlog.c index 742b2908ff686..b3dbf6c76e98f 100644 --- a/drivers/block/drbd/drbd_actlog.c +++ b/drivers/block/drbd/drbd_actlog.c @@ -483,38 +483,20 @@ void drbd_al_begin_io(struct drbd_device *device, struct drbd_interval *i) int drbd_al_begin_io_nonblock(struct drbd_device *device, struct drbd_interval *i) { - struct lru_cache *al = device->act_log; /* for bios crossing activity log extent boundaries, * we may need to activate two extents in one go */ unsigned first = i->sector >> (AL_EXTENT_SHIFT-9); unsigned last = i->size == 0 ? first : (i->sector + (i->size >> 9) - 1) >> (AL_EXTENT_SHIFT-9); - unsigned nr_al_extents; - unsigned available_update_slots; unsigned enr; - D_ASSERT(device, first <= last); - - nr_al_extents = 1 + last - first; /* worst case: all touched extends are cold. */ - available_update_slots = min(al->nr_elements - al->used, - al->max_pending_changes - al->pending_changes); - - /* We want all necessary updates for a given request within the same transaction - * We could first check how many updates are *actually* needed, - * and use that instead of the worst-case nr_al_extents */ - if (available_update_slots < nr_al_extents) { - /* Too many activity log extents are currently "hot". - * - * If we have accumulated pending changes already, - * we made progress. - * - * If we cannot get even a single pending change through, - * stop the fast path until we made some progress, - * or requests to "cold" extents could be starved. */ - if (!al->pending_changes) - __set_bit(__LC_STARVING, &device->act_log->flags); - return -ENOBUFS; + if (i->partially_in_al_next_enr) { + D_ASSERT(device, first < i->partially_in_al_next_enr); + D_ASSERT(device, last >= i->partially_in_al_next_enr); + first = i->partially_in_al_next_enr; } + D_ASSERT(device, first <= last); + /* Is resync active in this area? */ for (enr = first; enr <= last; enr++) { struct lc_element *tmp; @@ -529,14 +511,21 @@ int drbd_al_begin_io_nonblock(struct drbd_device *device, struct drbd_interval * } } - /* Checkout the refcounts. - * Given that we checked for available elements and update slots above, - * this has to be successful. */ + /* Try to checkout the refcounts. */ for (enr = first; enr <= last; enr++) { struct lc_element *al_ext; al_ext = lc_get_cumulative(device->act_log, enr); - if (!al_ext) - drbd_info(device, "LOGIC BUG for enr=%u\n", enr); + + if (!al_ext) { + /* Did not work. We may have exhausted the possible + * changes per transaction. Or raced with someone + * "locking" it against changes. + * Remember where to continue from. + */ + if (enr > first) + i->partially_in_al_next_enr = enr; + return -ENOBUFS; + } } return 0; } @@ -556,7 +545,11 @@ void drbd_al_complete_io(struct drbd_device *device, struct drbd_interval *i) for (enr = first; enr <= last; enr++) { extent = lc_find(device->act_log, enr); - if (!extent) { + /* Yes, this masks a bug elsewhere. However, during normal + * operation this is harmless, so no need to crash the kernel + * by the BUG_ON(refcount == 0) in lc_put(). + */ + if (!extent || extent->refcnt == 0) { drbd_err(device, "al_complete_io() called on inactive extent %u\n", enr); continue; } diff --git a/drivers/block/drbd/drbd_interval.h b/drivers/block/drbd/drbd_interval.h index 366489b72fe97..5d3213b81eede 100644 --- a/drivers/block/drbd/drbd_interval.h +++ b/drivers/block/drbd/drbd_interval.h @@ -8,12 +8,15 @@ struct drbd_interval { struct rb_node rb; sector_t sector; /* start sector of the interval */ - unsigned int size; /* size in bytes */ sector_t end; /* highest interval end in subtree */ + unsigned int size; /* size in bytes */ unsigned int local:1 /* local or remote request? */; unsigned int waiting:1; /* someone is waiting for completion */ unsigned int completed:1; /* this has been completed already; * ignore for conflict detection */ + + /* to resume a partially successful drbd_al_begin_io_nonblock(); */ + unsigned int partially_in_al_next_enr; }; static inline void drbd_clear_interval(struct drbd_interval *i) diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c index c73376886e7a5..1f6ac9202b66a 100644 --- a/drivers/block/drbd/drbd_main.c +++ b/drivers/block/drbd/drbd_main.c @@ -2659,9 +2659,6 @@ enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsig * connect. */ .max_hw_sectors = DRBD_MAX_BIO_SIZE_SAFE >> 8, - .features = BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | - BLK_FEAT_ROTATIONAL | - BLK_FEAT_STABLE_WRITES, }; device = minor_to_device(minor); diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c index 91f3b8afb63ce..b502038be0a92 100644 --- a/drivers/block/drbd/drbd_nl.c +++ b/drivers/block/drbd/drbd_nl.c @@ -1296,6 +1296,8 @@ void drbd_reconsider_queue_parameters(struct drbd_device *device, lim.max_segments = drbd_backing_dev_max_segments(device); } else { lim.max_segments = BLK_MAX_SEGMENTS; + lim.features = BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | + BLK_FEAT_ROTATIONAL | BLK_FEAT_STABLE_WRITES; } lim.max_hw_sectors = new >> SECTOR_SHIFT; @@ -1318,8 +1320,24 @@ void drbd_reconsider_queue_parameters(struct drbd_device *device, lim.max_hw_discard_sectors = 0; } - if (bdev) + if (bdev) { blk_stack_limits(&lim, &b->limits, 0); + /* + * blk_set_stacking_limits() cleared the features, and + * blk_stack_limits() may or may not have inherited + * BLK_FEAT_STABLE_WRITES from the backing device. + * + * DRBD always requires stable writes because: + * 1. The same bio data is read for both local disk I/O and + * network transmission. If the page changes mid-flight, + * the local and remote copies could diverge. + * 2. When data integrity is enabled, DRBD calculates a + * checksum before sending the data. If the page changes + * between checksum calculation and transmission, the + * receiver will detect a checksum mismatch. + */ + lim.features |= BLK_FEAT_STABLE_WRITES; + } /* * If we can handle "zeroes" efficiently on the protocol, we want to do diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c index d15826f6ee81d..70f75ef079457 100644 --- a/drivers/block/drbd/drbd_req.c +++ b/drivers/block/drbd/drbd_req.c @@ -621,7 +621,8 @@ int __req_mod(struct drbd_request *req, enum drbd_req_event what, break; case READ_COMPLETED_WITH_ERROR: - drbd_set_out_of_sync(peer_device, req->i.sector, req->i.size); + drbd_set_out_of_sync(first_peer_device(device), + req->i.sector, req->i.size); drbd_report_io_error(device, req); __drbd_chk_io_error(device, DRBD_READ_ERROR); fallthrough; diff --git a/drivers/block/rnbd/rnbd-srv.c b/drivers/block/rnbd/rnbd-srv.c index 2df8941a6b146..7eeb321d61402 100644 --- a/drivers/block/rnbd/rnbd-srv.c +++ b/drivers/block/rnbd/rnbd-srv.c @@ -145,18 +145,30 @@ static int process_rdma(struct rnbd_srv_session *srv_sess, priv->sess_dev = sess_dev; priv->id = id; - bio = bio_alloc(file_bdev(sess_dev->bdev_file), 1, + bio = bio_alloc(file_bdev(sess_dev->bdev_file), !!datalen, rnbd_to_bio_flags(le32_to_cpu(msg->rw)), GFP_KERNEL); - bio_add_virt_nofail(bio, data, datalen); - - bio->bi_opf = rnbd_to_bio_flags(le32_to_cpu(msg->rw)); - if (bio_has_data(bio) && - bio->bi_iter.bi_size != le32_to_cpu(msg->bi_size)) { - rnbd_srv_err_rl(sess_dev, "Datalen mismatch: bio bi_size (%u), bi_size (%u)\n", - bio->bi_iter.bi_size, msg->bi_size); - err = -EINVAL; - goto bio_put; + if (unlikely(!bio)) { + err = -ENOMEM; + goto put_sess_dev; } + + if (!datalen) { + /* + * For special requests like DISCARD and WRITE_ZEROES, the datalen is zero. + */ + bio->bi_iter.bi_size = le32_to_cpu(msg->bi_size); + } else { + bio_add_virt_nofail(bio, data, datalen); + bio->bi_opf = rnbd_to_bio_flags(le32_to_cpu(msg->rw)); + if (bio->bi_iter.bi_size != le32_to_cpu(msg->bi_size)) { + rnbd_srv_err_rl(sess_dev, + "Datalen mismatch: bio bi_size (%u), bi_size (%u)\n", + bio->bi_iter.bi_size, msg->bi_size); + err = -EINVAL; + goto bio_put; + } + } + bio->bi_end_io = rnbd_dev_bi_end_io; bio->bi_private = priv; bio->bi_iter.bi_sector = le64_to_cpu(msg->sector); @@ -170,6 +182,7 @@ static int process_rdma(struct rnbd_srv_session *srv_sess, bio_put: bio_put(bio); +put_sess_dev: rnbd_put_sess_dev(sess_dev); err: kfree(priv); @@ -538,6 +551,8 @@ static void rnbd_srv_fill_msg_open_rsp(struct rnbd_msg_open_rsp *rsp, { struct block_device *bdev = file_bdev(sess_dev->bdev_file); + memset(rsp, 0, sizeof(*rsp)); + rsp->hdr.type = cpu_to_le16(RNBD_MSG_OPEN_RSP); rsp->device_id = cpu_to_le32(sess_dev->device_id); rsp->nsectors = cpu_to_le64(bdev_nr_sectors(bdev)); @@ -644,6 +659,7 @@ static void process_msg_sess_info(struct rnbd_srv_session *srv_sess, trace_process_msg_sess_info(srv_sess, sess_info_msg); + memset(rsp, 0, sizeof(*rsp)); rsp->hdr.type = cpu_to_le16(RNBD_MSG_SESS_INFO_RSP); rsp->ver = srv_sess->ver; } diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 4b6d7b785d7b3..965460d4fc76e 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -2526,11 +2526,11 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, io->res = result; req = ublk_fill_io_cmd(io, cmd); ret = ublk_config_io_buf(ub, io, cmd, addr, &buf_idx); + if (buf_idx != UBLK_INVALID_BUF_IDX) + io_buffer_unregister_bvec(cmd, buf_idx, issue_flags); compl = ublk_need_complete_req(ub, io); /* can't touch 'ublk_io' any more */ - if (buf_idx != UBLK_INVALID_BUF_IDX) - io_buffer_unregister_bvec(cmd, buf_idx, issue_flags); if (req_op(req) == REQ_OP_ZONE_APPEND) req->__sector = addr; if (compl) @@ -3841,10 +3841,10 @@ static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd, if (issue_flags & IO_URING_F_NONBLOCK) return -EAGAIN; - ublk_ctrl_cmd_dump(cmd); - if (!(issue_flags & IO_URING_F_SQE128)) - goto out; + return -EINVAL; + + ublk_ctrl_cmd_dump(cmd); ret = ublk_check_cmd_op(cmd_op); if (ret) diff --git a/drivers/block/zloop.c b/drivers/block/zloop.c index 39a425db670c8..26364e43aeb34 100644 --- a/drivers/block/zloop.c +++ b/drivers/block/zloop.c @@ -499,6 +499,21 @@ static void zloop_rw(struct zloop_cmd *cmd) zloop_put_cmd(cmd); } +/* + * Sync the entire FS containing the zone files instead of walking all files. + */ +static int zloop_flush(struct zloop_device *zlo) +{ + struct super_block *sb = file_inode(zlo->data_dir)->i_sb; + int ret; + + down_read(&sb->s_umount); + ret = sync_filesystem(sb); + up_read(&sb->s_umount); + + return ret; +} + static void zloop_handle_cmd(struct zloop_cmd *cmd) { struct request *rq = blk_mq_rq_from_pdu(cmd); @@ -515,11 +530,7 @@ static void zloop_handle_cmd(struct zloop_cmd *cmd) zloop_rw(cmd); return; case REQ_OP_FLUSH: - /* - * Sync the entire FS containing the zone files instead of - * walking all files - */ - cmd->ret = sync_filesystem(file_inode(zlo->data_dir)->i_sb); + cmd->ret = zloop_flush(zlo); break; case REQ_OP_ZONE_RESET: cmd->ret = zloop_reset_zone(zlo, rq_zone_no(rq)); @@ -892,7 +903,8 @@ static int zloop_ctl_add(struct zloop_options *opts) .max_hw_sectors = SZ_1M >> SECTOR_SHIFT, .max_hw_zone_append_sectors = SZ_1M >> SECTOR_SHIFT, .chunk_sectors = opts->zone_size, - .features = BLK_FEAT_ZONED, + .features = BLK_FEAT_ZONED | BLK_FEAT_WRITE_CACHE, + }; unsigned int nr_zones, i, j; struct zloop_device *zlo; @@ -1064,7 +1076,12 @@ static int zloop_ctl_remove(struct zloop_options *opts) int ret; if (!(opts->mask & ZLOOP_OPT_ID)) { - pr_err("No ID specified\n"); + pr_err("No ID specified for remove\n"); + return -EINVAL; + } + + if (opts->mask & ~ZLOOP_OPT_ID) { + pr_err("Invalid option specified for remove\n"); return -EINVAL; } diff --git a/drivers/bluetooth/btintel_pcie.c b/drivers/bluetooth/btintel_pcie.c index f280bcc61bbfb..c68a8de3025b0 100644 --- a/drivers/bluetooth/btintel_pcie.c +++ b/drivers/bluetooth/btintel_pcie.c @@ -1430,11 +1430,6 @@ static void btintel_pcie_msix_rx_handle(struct btintel_pcie_data *data) } } -static irqreturn_t btintel_pcie_msix_isr(int irq, void *data) -{ - return IRQ_WAKE_THREAD; -} - static inline bool btintel_pcie_is_rxq_empty(struct btintel_pcie_data *data) { return data->ia.cr_hia[BTINTEL_PCIE_RXQ_NUM] == data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM]; @@ -1536,9 +1531,9 @@ static int btintel_pcie_setup_irq(struct btintel_pcie_data *data) err = devm_request_threaded_irq(&data->pdev->dev, msix_entry->vector, - btintel_pcie_msix_isr, + NULL, btintel_pcie_irq_msix_handler, - IRQF_SHARED, + IRQF_ONESHOT | IRQF_SHARED, KBUILD_MODNAME, msix_entry); if (err) { diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c index a953fa9af85c6..7c7955afa8e8a 100644 --- a/drivers/bluetooth/btusb.c +++ b/drivers/bluetooth/btusb.c @@ -521,6 +521,8 @@ static const struct usb_device_id quirks_table[] = { { USB_DEVICE(0x0bda, 0xb850), .driver_info = BTUSB_REALTEK }, { USB_DEVICE(0x13d3, 0x3600), .driver_info = BTUSB_REALTEK }, { USB_DEVICE(0x13d3, 0x3601), .driver_info = BTUSB_REALTEK }, + { USB_DEVICE(0x0489, 0xe112), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, /* Realtek 8851BU Bluetooth devices */ { USB_DEVICE(0x3625, 0x010b), .driver_info = BTUSB_REALTEK | @@ -559,6 +561,8 @@ static const struct usb_device_id quirks_table[] = { BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3592), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x13d3, 0x3612), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x0489, 0xe122), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, @@ -635,6 +639,8 @@ static const struct usb_device_id quirks_table[] = { BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3622), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0489, 0xe158), .driver_info = BTUSB_MEDIATEK | + BTUSB_WIDEBAND_SPEECH }, /* Additional MediaTek MT7921 Bluetooth devices */ { USB_DEVICE(0x0489, 0xe0c8), .driver_info = BTUSB_MEDIATEK | @@ -771,6 +777,7 @@ static const struct usb_device_id quirks_table[] = { /* Additional Realtek 8723BU Bluetooth devices */ { USB_DEVICE(0x7392, 0xa611), .driver_info = BTUSB_REALTEK }, + { USB_DEVICE(0x2c0a, 0x8761), .driver_info = BTUSB_REALTEK }, /* Additional Realtek 8723DE Bluetooth devices */ { USB_DEVICE(0x0bda, 0xb009), .driver_info = BTUSB_REALTEK }, diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c index a97baf2cbcdd5..eb7b6c0ba9e7c 100644 --- a/drivers/bus/fsl-mc/fsl-mc-bus.c +++ b/drivers/bus/fsl-mc/fsl-mc-bus.c @@ -909,11 +909,7 @@ int fsl_mc_device_add(struct fsl_mc_obj_desc *obj_desc, return 0; error_cleanup_dev: - kfree(mc_dev->regions); - if (mc_bus) - kfree(mc_bus); - else - kfree(mc_dev); + put_device(&mc_dev->dev); return error; } diff --git a/drivers/bus/mhi/host/pci_generic.c b/drivers/bus/mhi/host/pci_generic.c index 773d569c7a512..ed5f407ac2939 100644 --- a/drivers/bus/mhi/host/pci_generic.c +++ b/drivers/bus/mhi/host/pci_generic.c @@ -94,22 +94,6 @@ struct mhi_pci_dev_info { .doorbell_mode_switch = false, \ } -#define MHI_CHANNEL_CONFIG_DL_AUTOQUEUE(ch_num, ch_name, el_count, ev_ring) \ - { \ - .num = ch_num, \ - .name = ch_name, \ - .num_elements = el_count, \ - .event_ring = ev_ring, \ - .dir = DMA_FROM_DEVICE, \ - .ee_mask = BIT(MHI_EE_AMSS), \ - .pollcfg = 0, \ - .doorbell = MHI_DB_BRST_DISABLE, \ - .lpm_notify = false, \ - .offload_channel = false, \ - .doorbell_mode_switch = false, \ - .auto_queue = true, \ - } - #define MHI_EVENT_CONFIG_CTRL(ev_ring, el_count) \ { \ .num_elements = el_count, \ @@ -329,7 +313,7 @@ static const struct mhi_channel_config modem_qcom_v1_mhi_channels[] = { MHI_CHANNEL_CONFIG_UL(14, "QMI", 4, 0), MHI_CHANNEL_CONFIG_DL(15, "QMI", 4, 0), MHI_CHANNEL_CONFIG_UL(20, "IPCR", 8, 0), - MHI_CHANNEL_CONFIG_DL_AUTOQUEUE(21, "IPCR", 8, 0), + MHI_CHANNEL_CONFIG_DL(21, "IPCR", 8, 0), MHI_CHANNEL_CONFIG_UL_FP(34, "FIREHOSE", 32, 0), MHI_CHANNEL_CONFIG_DL_FP(35, "FIREHOSE", 32, 0), MHI_CHANNEL_CONFIG_UL(46, "IP_SW0", 64, 2), @@ -761,7 +745,7 @@ static const struct mhi_channel_config mhi_telit_fn980_hw_v1_channels[] = { MHI_CHANNEL_CONFIG_UL(14, "QMI", 32, 0), MHI_CHANNEL_CONFIG_DL(15, "QMI", 32, 0), MHI_CHANNEL_CONFIG_UL(20, "IPCR", 16, 0), - MHI_CHANNEL_CONFIG_DL_AUTOQUEUE(21, "IPCR", 16, 0), + MHI_CHANNEL_CONFIG_DL(21, "IPCR", 16, 0), MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0", 128, 1), MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0", 128, 2), }; diff --git a/drivers/bus/omap-ocp2scp.c b/drivers/bus/omap-ocp2scp.c index e4dfda7b3b102..eee5ad191ea9c 100644 --- a/drivers/bus/omap-ocp2scp.c +++ b/drivers/bus/omap-ocp2scp.c @@ -17,15 +17,6 @@ #define OCP2SCP_TIMING 0x18 #define SYNC2_MASK 0xf -static int ocp2scp_remove_devices(struct device *dev, void *c) -{ - struct platform_device *pdev = to_platform_device(dev); - - platform_device_unregister(pdev); - - return 0; -} - static int omap_ocp2scp_probe(struct platform_device *pdev) { int ret; @@ -79,7 +70,7 @@ static int omap_ocp2scp_probe(struct platform_device *pdev) pm_runtime_disable(&pdev->dev); err0: - device_for_each_child(&pdev->dev, NULL, ocp2scp_remove_devices); + of_platform_depopulate(&pdev->dev); return ret; } @@ -87,7 +78,7 @@ static int omap_ocp2scp_probe(struct platform_device *pdev) static void omap_ocp2scp_remove(struct platform_device *pdev) { pm_runtime_disable(&pdev->dev); - device_for_each_child(&pdev->dev, NULL, ocp2scp_remove_devices); + of_platform_depopulate(&pdev->dev); } #ifdef CONFIG_OF diff --git a/drivers/char/hw_random/airoha-trng.c b/drivers/char/hw_random/airoha-trng.c index 1dbfa9505c214..9a648f6d9fd40 100644 --- a/drivers/char/hw_random/airoha-trng.c +++ b/drivers/char/hw_random/airoha-trng.c @@ -212,6 +212,7 @@ static int airoha_trng_probe(struct platform_device *pdev) trng->rng.init = airoha_trng_init; trng->rng.cleanup = airoha_trng_cleanup; trng->rng.read = airoha_trng_read; + trng->rng.quality = 900; ret = devm_hwrng_register(dev, &trng->rng); if (ret) { diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 018316f546215..036de7294bbda 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -20,23 +20,25 @@ #include #include #include +#include #include #include #include #include #include +#include #define RNG_MODULE_NAME "hw_random" #define RNG_BUFFER_SIZE (SMP_CACHE_BYTES < 32 ? 32 : SMP_CACHE_BYTES) -static struct hwrng *current_rng; +static struct hwrng __rcu *current_rng; /* the current rng has been explicitly chosen by user via sysfs */ static int cur_rng_set_by_user; static struct task_struct *hwrng_fill; /* list of registered rngs */ static LIST_HEAD(rng_list); -/* Protects rng_list and current_rng */ +/* Protects rng_list, hwrng_fill and updating on current_rng */ static DEFINE_MUTEX(rng_mutex); /* Protects rng read functions, data_avail, rng_buffer and rng_fillbuf */ static DEFINE_MUTEX(reading_mutex); @@ -64,18 +66,39 @@ static size_t rng_buffer_size(void) return RNG_BUFFER_SIZE; } -static inline void cleanup_rng(struct kref *kref) +static void cleanup_rng_work(struct work_struct *work) { - struct hwrng *rng = container_of(kref, struct hwrng, ref); + struct hwrng *rng = container_of(work, struct hwrng, cleanup_work); + + /* + * Hold rng_mutex here so we serialize in case they set_current_rng + * on rng again immediately. + */ + mutex_lock(&rng_mutex); + + /* Skip if rng has been reinitialized. */ + if (kref_read(&rng->ref)) { + mutex_unlock(&rng_mutex); + return; + } if (rng->cleanup) rng->cleanup(rng); complete(&rng->cleanup_done); + mutex_unlock(&rng_mutex); +} + +static inline void cleanup_rng(struct kref *kref) +{ + struct hwrng *rng = container_of(kref, struct hwrng, ref); + + schedule_work(&rng->cleanup_work); } static int set_current_rng(struct hwrng *rng) { + struct hwrng *old_rng; int err; BUG_ON(!mutex_is_locked(&rng_mutex)); @@ -84,8 +107,14 @@ static int set_current_rng(struct hwrng *rng) if (err) return err; - drop_current_rng(); - current_rng = rng; + old_rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + rcu_assign_pointer(current_rng, rng); + + if (old_rng) { + synchronize_rcu(); + kref_put(&old_rng->ref, cleanup_rng); + } /* if necessary, start hwrng thread */ if (!hwrng_fill) { @@ -101,47 +130,56 @@ static int set_current_rng(struct hwrng *rng) static void drop_current_rng(void) { - BUG_ON(!mutex_is_locked(&rng_mutex)); - if (!current_rng) + struct hwrng *rng; + + rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + if (!rng) return; + RCU_INIT_POINTER(current_rng, NULL); + synchronize_rcu(); + + if (hwrng_fill) { + kthread_stop(hwrng_fill); + hwrng_fill = NULL; + } + /* decrease last reference for triggering the cleanup */ - kref_put(¤t_rng->ref, cleanup_rng); - current_rng = NULL; + kref_put(&rng->ref, cleanup_rng); } -/* Returns ERR_PTR(), NULL or refcounted hwrng */ +/* Returns NULL or refcounted hwrng */ static struct hwrng *get_current_rng_nolock(void) { - if (current_rng) - kref_get(¤t_rng->ref); + struct hwrng *rng; + + rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + if (rng) + kref_get(&rng->ref); - return current_rng; + return rng; } static struct hwrng *get_current_rng(void) { struct hwrng *rng; - if (mutex_lock_interruptible(&rng_mutex)) - return ERR_PTR(-ERESTARTSYS); + rcu_read_lock(); + rng = rcu_dereference(current_rng); + if (rng) + kref_get(&rng->ref); - rng = get_current_rng_nolock(); + rcu_read_unlock(); - mutex_unlock(&rng_mutex); return rng; } static void put_rng(struct hwrng *rng) { - /* - * Hold rng_mutex here so we serialize in case they set_current_rng - * on rng again immediately. - */ - mutex_lock(&rng_mutex); if (rng) kref_put(&rng->ref, cleanup_rng); - mutex_unlock(&rng_mutex); } static int hwrng_init(struct hwrng *rng) @@ -213,10 +251,6 @@ static ssize_t rng_dev_read(struct file *filp, char __user *buf, while (size) { rng = get_current_rng(); - if (IS_ERR(rng)) { - err = PTR_ERR(rng); - goto out; - } if (!rng) { err = -ENODEV; goto out; @@ -303,7 +337,7 @@ static struct miscdevice rng_miscdev = { static int enable_best_rng(void) { - struct hwrng *rng, *new_rng = NULL; + struct hwrng *rng, *cur_rng, *new_rng = NULL; int ret = -ENODEV; BUG_ON(!mutex_is_locked(&rng_mutex)); @@ -321,7 +355,9 @@ static int enable_best_rng(void) new_rng = rng; } - ret = ((new_rng == current_rng) ? 0 : set_current_rng(new_rng)); + cur_rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + ret = ((new_rng == cur_rng) ? 0 : set_current_rng(new_rng)); if (!ret) cur_rng_set_by_user = 0; @@ -341,6 +377,9 @@ static ssize_t rng_current_store(struct device *dev, if (sysfs_streq(buf, "")) { err = enable_best_rng(); + } else if (sysfs_streq(buf, "none")) { + cur_rng_set_by_user = 1; + drop_current_rng(); } else { list_for_each_entry(rng, &rng_list, list) { if (sysfs_streq(rng->name, buf)) { @@ -368,8 +407,6 @@ static ssize_t rng_current_show(struct device *dev, struct hwrng *rng; rng = get_current_rng(); - if (IS_ERR(rng)) - return PTR_ERR(rng); ret = sysfs_emit(buf, "%s\n", rng ? rng->name : "none"); put_rng(rng); @@ -392,7 +429,7 @@ static ssize_t rng_available_show(struct device *dev, strlcat(buf, rng->name, PAGE_SIZE); strlcat(buf, " ", PAGE_SIZE); } - strlcat(buf, "\n", PAGE_SIZE); + strlcat(buf, "none\n", PAGE_SIZE); mutex_unlock(&rng_mutex); return strlen(buf); @@ -413,8 +450,6 @@ static ssize_t rng_quality_show(struct device *dev, struct hwrng *rng; rng = get_current_rng(); - if (IS_ERR(rng)) - return PTR_ERR(rng); if (!rng) /* no need to put_rng */ return -ENODEV; @@ -429,6 +464,7 @@ static ssize_t rng_quality_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len) { + struct hwrng *rng; u16 quality; int ret = -EINVAL; @@ -445,12 +481,13 @@ static ssize_t rng_quality_store(struct device *dev, goto out; } - if (!current_rng) { + rng = rcu_dereference_protected(current_rng, lockdep_is_held(&rng_mutex)); + if (!rng) { ret = -ENODEV; goto out; } - current_rng->quality = quality; + rng->quality = quality; current_quality = quality; /* obsolete */ /* the best available RNG may have changed */ @@ -486,8 +523,20 @@ static int hwrng_fillfn(void *unused) struct hwrng *rng; rng = get_current_rng(); - if (IS_ERR(rng) || !rng) + if (!rng) { + /* + * Keep the task_struct alive until kthread_stop() + * is called to avoid UAF in drop_current_rng(). + */ + while (!kthread_should_stop()) { + set_current_state(TASK_INTERRUPTIBLE); + if (!kthread_should_stop()) + schedule(); + } + set_current_state(TASK_RUNNING); break; + } + mutex_lock(&reading_mutex); rc = rng_get_data(rng, rng_fillbuf, rng_buffer_size(), 1); @@ -515,14 +564,13 @@ static int hwrng_fillfn(void *unused) add_hwgenerator_randomness((void *)rng_fillbuf, rc, entropy >> 10, true); } - hwrng_fill = NULL; return 0; } int hwrng_register(struct hwrng *rng) { int err = -EINVAL; - struct hwrng *tmp; + struct hwrng *cur_rng, *tmp; if (!rng->name || (!rng->data_read && !rng->read)) goto out; @@ -537,6 +585,7 @@ int hwrng_register(struct hwrng *rng) } list_add_tail(&rng->list, &rng_list); + INIT_WORK(&rng->cleanup_work, cleanup_rng_work); init_completion(&rng->cleanup_done); complete(&rng->cleanup_done); init_completion(&rng->dying); @@ -544,16 +593,19 @@ int hwrng_register(struct hwrng *rng) /* Adjust quality field to always have a proper value */ rng->quality = min_t(u16, min_t(u16, default_quality, 1024), rng->quality ?: 1024); - if (!current_rng || - (!cur_rng_set_by_user && rng->quality > current_rng->quality)) { - /* - * Set new rng as current as the new rng source - * provides better entropy quality and was not - * chosen by userspace. - */ - err = set_current_rng(rng); - if (err) - goto out_unlock; + if (!cur_rng_set_by_user) { + cur_rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + if (!cur_rng || rng->quality > cur_rng->quality) { + /* + * Set new rng as current as the new rng source + * provides better entropy quality and was not + * chosen by userspace. + */ + err = set_current_rng(rng); + if (err) + goto out_unlock; + } } mutex_unlock(&rng_mutex); return 0; @@ -566,14 +618,17 @@ EXPORT_SYMBOL_GPL(hwrng_register); void hwrng_unregister(struct hwrng *rng) { - struct hwrng *new_rng; + struct hwrng *cur_rng; int err; mutex_lock(&rng_mutex); list_del(&rng->list); complete_all(&rng->dying); - if (current_rng == rng) { + + cur_rng = rcu_dereference_protected(current_rng, + lockdep_is_held(&rng_mutex)); + if (cur_rng == rng) { err = enable_best_rng(); if (err) { drop_current_rng(); @@ -581,17 +636,7 @@ void hwrng_unregister(struct hwrng *rng) } } - new_rng = get_current_rng_nolock(); - if (list_empty(&rng_list)) { - mutex_unlock(&rng_mutex); - if (hwrng_fill) - kthread_stop(hwrng_fill); - } else - mutex_unlock(&rng_mutex); - - if (new_rng) - put_rng(new_rng); - + mutex_unlock(&rng_mutex); wait_for_completion(&rng->cleanup_done); } EXPORT_SYMBOL_GPL(hwrng_unregister); @@ -679,7 +724,7 @@ static int __init hwrng_modinit(void) static void __exit hwrng_modexit(void) { mutex_lock(&rng_mutex); - BUG_ON(current_rng); + WARN_ON(rcu_access_pointer(current_rng)); kfree(rng_buffer); kfree(rng_fillbuf); mutex_unlock(&rng_mutex); diff --git a/drivers/char/ipmi/ipmi_ipmb.c b/drivers/char/ipmi/ipmi_ipmb.c index 3a51e58b24875..28818952a7a4b 100644 --- a/drivers/char/ipmi/ipmi_ipmb.c +++ b/drivers/char/ipmi/ipmi_ipmb.c @@ -202,11 +202,16 @@ static int ipmi_ipmb_slave_cb(struct i2c_client *client, break; case I2C_SLAVE_READ_REQUESTED: + *val = 0xff; + ipmi_ipmb_check_msg_done(iidev); + break; + case I2C_SLAVE_STOP: ipmi_ipmb_check_msg_done(iidev); break; case I2C_SLAVE_READ_PROCESSED: + *val = 0xff; break; } diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c index 0a886399f9daf..5ed8e95589fb7 100644 --- a/drivers/char/ipmi/ipmi_msghandler.c +++ b/drivers/char/ipmi/ipmi_msghandler.c @@ -4848,8 +4848,15 @@ static void smi_work(struct work_struct *t) if (newmsg->recv_msg) deliver_err_response(intf, newmsg->recv_msg, cc); - else - ipmi_free_smi_msg(newmsg); + if (!run_to_completion) + spin_lock_irqsave(&intf->xmit_msgs_lock, + flags); + intf->curr_msg = NULL; + if (!run_to_completion) + spin_unlock_irqrestore(&intf->xmit_msgs_lock, + flags); + ipmi_free_smi_msg(newmsg); + newmsg = NULL; goto restart; } } diff --git a/drivers/char/misc_minor_kunit.c b/drivers/char/misc_minor_kunit.c index 6fc8b05169c57..e930c78e1ef97 100644 --- a/drivers/char/misc_minor_kunit.c +++ b/drivers/char/misc_minor_kunit.c @@ -166,7 +166,7 @@ static void __init miscdev_test_can_open(struct kunit *test, struct miscdevice * KUNIT_FAIL(test, "failed to create node\n"); filp = filp_open(devname, O_RDONLY, 0); - if (IS_ERR_OR_NULL(filp)) + if (IS_ERR(filp)) KUNIT_FAIL(test, "failed to open misc device: %ld\n", PTR_ERR(filp)); else fput(filp); diff --git a/drivers/char/random.c b/drivers/char/random.c index 4ba5f0c4c8b24..f344f67c83f4e 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -96,8 +96,7 @@ static ATOMIC_NOTIFIER_HEAD(random_ready_notifier); /* Control how we warn userspace. */ static struct ratelimit_state urandom_warning = RATELIMIT_STATE_INIT_FLAGS("urandom_warning", HZ, 3, RATELIMIT_MSG_ON_RELEASE); -static int ratelimit_disable __read_mostly = - IS_ENABLED(CONFIG_WARN_ALL_UNSEEDED_RANDOM); +static int ratelimit_disable __read_mostly = 0; module_param_named(ratelimit_disable, ratelimit_disable, int, 0644); MODULE_PARM_DESC(ratelimit_disable, "Disable random ratelimit suppression"); @@ -168,12 +167,6 @@ int __cold execute_with_initialized_rng(struct notifier_block *nb) return ret; } -#define warn_unseeded_randomness() \ - if (IS_ENABLED(CONFIG_WARN_ALL_UNSEEDED_RANDOM) && !crng_ready()) \ - printk_deferred(KERN_NOTICE "random: %s called from %pS with crng_init=%d\n", \ - __func__, (void *)_RET_IP_, crng_init) - - /********************************************************************* * * Fast key erasure RNG, the "crng". @@ -434,7 +427,6 @@ static void _get_random_bytes(void *buf, size_t len) */ void get_random_bytes(void *buf, size_t len) { - warn_unseeded_randomness(); _get_random_bytes(buf, len); } EXPORT_SYMBOL(get_random_bytes); @@ -523,8 +515,6 @@ type get_random_ ##type(void) \ struct batch_ ##type *batch; \ unsigned long next_gen; \ \ - warn_unseeded_randomness(); \ - \ if (!crng_ready()) { \ _get_random_bytes(&ret, sizeof(ret)); \ return ret; \ diff --git a/drivers/char/tpm/st33zp24/st33zp24.c b/drivers/char/tpm/st33zp24/st33zp24.c index 2ed7815e4899b..e2b7451ea7ccd 100644 --- a/drivers/char/tpm/st33zp24/st33zp24.c +++ b/drivers/char/tpm/st33zp24/st33zp24.c @@ -328,8 +328,10 @@ static int st33zp24_send(struct tpm_chip *chip, unsigned char *buf, for (i = 0; i < len - 1;) { burstcnt = get_burstcount(chip); - if (burstcnt < 0) - return burstcnt; + if (burstcnt < 0) { + ret = burstcnt; + goto out_err; + } size = min_t(int, len - i - 1, burstcnt); ret = tpm_dev->ops->send(tpm_dev->phy_id, TPM_DATA_FIFO, buf + i, size); diff --git a/drivers/char/tpm/tpm_i2c_infineon.c b/drivers/char/tpm/tpm_i2c_infineon.c index bdf1f329a6794..8b7d32de0b2ef 100644 --- a/drivers/char/tpm/tpm_i2c_infineon.c +++ b/drivers/char/tpm/tpm_i2c_infineon.c @@ -544,8 +544,10 @@ static int tpm_tis_i2c_send(struct tpm_chip *chip, u8 *buf, size_t bufsiz, burstcnt = get_burstcount(chip); /* burstcnt < 0 = TPM is busy */ - if (burstcnt < 0) - return burstcnt; + if (burstcnt < 0) { + rc = burstcnt; + goto out_err; + } if (burstcnt > (len - 1 - count)) burstcnt = len - 1 - count; diff --git a/drivers/char/tpm/tpm_tis_i2c_cr50.c b/drivers/char/tpm/tpm_tis_i2c_cr50.c index fc6891a0b6936..b48cacacc0664 100644 --- a/drivers/char/tpm/tpm_tis_i2c_cr50.c +++ b/drivers/char/tpm/tpm_tis_i2c_cr50.c @@ -749,8 +749,7 @@ static int tpm_cr50_i2c_probe(struct i2c_client *client) if (client->irq > 0) { rc = devm_request_irq(dev, client->irq, tpm_cr50_i2c_int_handler, - IRQF_TRIGGER_FALLING | IRQF_ONESHOT | - IRQF_NO_AUTOEN, + IRQF_TRIGGER_FALLING | IRQF_NO_AUTOEN, dev->driver->name, chip); if (rc < 0) { dev_err(dev, "Failed to probe IRQ %d\n", client->irq); diff --git a/drivers/char/tpm/tpm_tis_spi_cr50.c b/drivers/char/tpm/tpm_tis_spi_cr50.c index f4937280e9406..32920b4cecfb4 100644 --- a/drivers/char/tpm/tpm_tis_spi_cr50.c +++ b/drivers/char/tpm/tpm_tis_spi_cr50.c @@ -287,7 +287,7 @@ int cr50_spi_probe(struct spi_device *spi) if (spi->irq > 0) { ret = devm_request_irq(&spi->dev, spi->irq, cr50_spi_irq_handler, - IRQF_TRIGGER_RISING | IRQF_ONESHOT, + IRQF_TRIGGER_RISING, "cr50_spi", cr50_phy); if (ret < 0) { if (ret == -EPROBE_DEFER) diff --git a/drivers/clk/actions/owl-composite.c b/drivers/clk/actions/owl-composite.c index 00b74f8bc4375..9540444307d6c 100644 --- a/drivers/clk/actions/owl-composite.c +++ b/drivers/clk/actions/owl-composite.c @@ -57,15 +57,10 @@ static int owl_comp_div_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { struct owl_composite *comp = hw_to_owl_comp(hw); - long rate; - - rate = owl_divider_helper_round_rate(&comp->common, &comp->rate.div_hw, - req->rate, &req->best_parent_rate); - if (rate < 0) - return rate; + struct owl_divider_hw *div = &comp->rate.div_hw; - req->rate = rate; - return 0; + return divider_determine_rate(&comp->common.hw, req, div->table, + div->width, div->div_flags); } static unsigned long owl_comp_div_recalc_rate(struct clk_hw *hw, diff --git a/drivers/clk/actions/owl-divider.c b/drivers/clk/actions/owl-divider.c index 118f1393c6780..316ace80e87e3 100644 --- a/drivers/clk/actions/owl-divider.c +++ b/drivers/clk/actions/owl-divider.c @@ -13,26 +13,13 @@ #include "owl-divider.h" -long owl_divider_helper_round_rate(struct owl_clk_common *common, - const struct owl_divider_hw *div_hw, - unsigned long rate, - unsigned long *parent_rate) -{ - return divider_round_rate(&common->hw, rate, parent_rate, - div_hw->table, div_hw->width, - div_hw->div_flags); -} - static int owl_divider_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { struct owl_divider *div = hw_to_owl_divider(hw); - req->rate = owl_divider_helper_round_rate(&div->common, &div->div_hw, - req->rate, - &req->best_parent_rate); - - return 0; + return divider_determine_rate(hw, req, div->div_hw.table, + div->div_hw.width, div->div_hw.div_flags); } unsigned long owl_divider_helper_recalc_rate(struct owl_clk_common *common, diff --git a/drivers/clk/actions/owl-divider.h b/drivers/clk/actions/owl-divider.h index 083be6d80954d..2ba957740c381 100644 --- a/drivers/clk/actions/owl-divider.h +++ b/drivers/clk/actions/owl-divider.h @@ -56,11 +56,6 @@ static inline struct owl_divider *hw_to_owl_divider(const struct clk_hw *hw) return container_of(common, struct owl_divider, common); } -long owl_divider_helper_round_rate(struct owl_clk_common *common, - const struct owl_divider_hw *div_hw, - unsigned long rate, - unsigned long *parent_rate); - unsigned long owl_divider_helper_recalc_rate(struct owl_clk_common *common, const struct owl_divider_hw *div_hw, unsigned long parent_rate); diff --git a/drivers/clk/clk-apple-nco.c b/drivers/clk/clk-apple-nco.c index d3ced4a0f029e..434c067968bbc 100644 --- a/drivers/clk/clk-apple-nco.c +++ b/drivers/clk/clk-apple-nco.c @@ -320,6 +320,7 @@ static int applnco_probe(struct platform_device *pdev) } static const struct of_device_id applnco_ids[] = { + { .compatible = "apple,t8103-nco" }, { .compatible = "apple,nco" }, { } }; diff --git a/drivers/clk/clk-bm1880.c b/drivers/clk/clk-bm1880.c index dac190bc6e19a..d2617fe16d2e4 100644 --- a/drivers/clk/clk-bm1880.c +++ b/drivers/clk/clk-bm1880.c @@ -629,10 +629,7 @@ static int bm1880_clk_div_determine_rate(struct clk_hw *hw, return 0; } - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - div->table, div->width, div->flags); - - return 0; + return divider_determine_rate(hw, req, div->table, div->width, div->flags); } static int bm1880_clk_div_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/clk-loongson1.c b/drivers/clk/clk-loongson1.c index f9f060d08a5fa..1674181a1107d 100644 --- a/drivers/clk/clk-loongson1.c +++ b/drivers/clk/clk-loongson1.c @@ -99,10 +99,7 @@ static int ls1x_divider_determine_rate(struct clk_hw *hw, struct ls1x_clk *ls1x_clk = to_ls1x_clk(hw); const struct ls1x_clk_div_data *d = ls1x_clk->data; - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - d->table, d->width, d->flags); - - return 0; + return divider_determine_rate(hw, req, d->table, d->width, d->flags); } static int ls1x_divider_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/clk-milbeaut.c b/drivers/clk/clk-milbeaut.c index b4f9b7143eaa6..bb94d02a76cf1 100644 --- a/drivers/clk/clk-milbeaut.c +++ b/drivers/clk/clk-milbeaut.c @@ -407,10 +407,7 @@ static int m10v_clk_divider_determine_rate(struct clk_hw *hw, return 0; } - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - divider->table, divider->width, divider->flags); - - return 0; + return divider_determine_rate(hw, req, divider->table, divider->width, divider->flags); } static int m10v_clk_divider_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/clk-renesas-pcie.c b/drivers/clk/clk-renesas-pcie.c index 4c3a5e4eb77ac..f94a9c4d0b670 100644 --- a/drivers/clk/clk-renesas-pcie.c +++ b/drivers/clk/clk-renesas-pcie.c @@ -64,7 +64,7 @@ struct rs9_driver_data { struct i2c_client *client; struct regmap *regmap; const struct rs9_chip_info *chip_info; - struct clk_hw *clk_dif[4]; + struct clk_hw *clk_dif[8]; u8 pll_amplitude; u8 pll_ssc; u8 clk_dif_sr; diff --git a/drivers/clk/clk-versaclock3.c b/drivers/clk/clk-versaclock3.c index 1849863dbd673..27b6cf70f3ae1 100644 --- a/drivers/clk/clk-versaclock3.c +++ b/drivers/clk/clk-versaclock3.c @@ -523,11 +523,8 @@ static int vc3_div_determine_rate(struct clk_hw *hw, return 0; } - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - div_data->table, - div_data->width, div_data->flags); - - return 0; + return divider_determine_rate(hw, req, div_data->table, div_data->width, + div_data->flags); } static int vc3_div_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/hisilicon/clkdivider-hi6220.c b/drivers/clk/hisilicon/clkdivider-hi6220.c index 6bae18a84cb6c..fd7ceb92d6515 100644 --- a/drivers/clk/hisilicon/clkdivider-hi6220.c +++ b/drivers/clk/hisilicon/clkdivider-hi6220.c @@ -60,10 +60,8 @@ static int hi6220_clkdiv_determine_rate(struct clk_hw *hw, { struct hi6220_clk_divider *dclk = to_hi6220_clk_divider(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, dclk->table, - dclk->width, CLK_DIVIDER_ROUND_CLOSEST); - - return 0; + return divider_determine_rate(hw, req, dclk->table, dclk->width, + CLK_DIVIDER_ROUND_CLOSEST); } static int hi6220_clkdiv_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/mediatek/clk-mt7981-eth.c b/drivers/clk/mediatek/clk-mt7981-eth.c index 906aec9ddff54..0655ebb6c561f 100644 --- a/drivers/clk/mediatek/clk-mt7981-eth.c +++ b/drivers/clk/mediatek/clk-mt7981-eth.c @@ -31,7 +31,7 @@ static const struct mtk_gate_regs sgmii0_cg_regs = { .ops = &mtk_clk_gate_ops_no_setclr_inv, \ } -static const struct mtk_gate sgmii0_clks[] __initconst = { +static const struct mtk_gate sgmii0_clks[] = { GATE_SGMII0(CLK_SGM0_TX_EN, "sgm0_tx_en", "usb_tx250m", 2), GATE_SGMII0(CLK_SGM0_RX_EN, "sgm0_rx_en", "usb_eq_rx250m", 3), GATE_SGMII0(CLK_SGM0_CK0_EN, "sgm0_ck0_en", "usb_ln0", 4), @@ -53,7 +53,7 @@ static const struct mtk_gate_regs sgmii1_cg_regs = { .ops = &mtk_clk_gate_ops_no_setclr_inv, \ } -static const struct mtk_gate sgmii1_clks[] __initconst = { +static const struct mtk_gate sgmii1_clks[] = { GATE_SGMII1(CLK_SGM1_TX_EN, "sgm1_tx_en", "usb_tx250m", 2), GATE_SGMII1(CLK_SGM1_RX_EN, "sgm1_rx_en", "usb_eq_rx250m", 3), GATE_SGMII1(CLK_SGM1_CK1_EN, "sgm1_ck1_en", "usb_ln0", 4), @@ -75,7 +75,7 @@ static const struct mtk_gate_regs eth_cg_regs = { .ops = &mtk_clk_gate_ops_no_setclr_inv, \ } -static const struct mtk_gate eth_clks[] __initconst = { +static const struct mtk_gate eth_clks[] = { GATE_ETH(CLK_ETH_FE_EN, "eth_fe_en", "netsys_2x", 6), GATE_ETH(CLK_ETH_GP2_EN, "eth_gp2_en", "sgm_325m", 7), GATE_ETH(CLK_ETH_GP1_EN, "eth_gp1_en", "sgm_325m", 8), diff --git a/drivers/clk/mediatek/clk-mt8196-mfg.c b/drivers/clk/mediatek/clk-mt8196-mfg.c index ae1eb9de79ae2..f40795b47ff1f 100644 --- a/drivers/clk/mediatek/clk-mt8196-mfg.c +++ b/drivers/clk/mediatek/clk-mt8196-mfg.c @@ -58,24 +58,25 @@ .pcw_shift = _pcw_shift, \ .pcwbits = _pcwbits, \ .pcwibits = MT8196_INTEGER_BITS, \ + .parent_name = "mfg_eb", \ } static const struct mtk_pll_data mfg_ao_plls[] = { - PLL(CLK_MFG_AO_MFGPLL, "mfgpll", MFGPLL_CON0, MFGPLL_CON0, 0, 0, 0, - BIT(0), MFGPLL_CON1, 24, 0, 0, 0, + PLL(CLK_MFG_AO_MFGPLL, "mfgpll", MFGPLL_CON0, MFGPLL_CON0, 0, 0, + PLL_PARENT_EN, BIT(0), MFGPLL_CON1, 24, 0, 0, 0, MFGPLL_CON1, 0, 22), }; static const struct mtk_pll_data mfgsc0_ao_plls[] = { PLL(CLK_MFGSC0_AO_MFGPLL_SC0, "mfgpll-sc0", MFGPLL_SC0_CON0, - MFGPLL_SC0_CON0, 0, 0, 0, BIT(0), MFGPLL_SC0_CON1, 24, 0, 0, 0, - MFGPLL_SC0_CON1, 0, 22), + MFGPLL_SC0_CON0, 0, 0, PLL_PARENT_EN, BIT(0), MFGPLL_SC0_CON1, 24, + 0, 0, 0, MFGPLL_SC0_CON1, 0, 22), }; static const struct mtk_pll_data mfgsc1_ao_plls[] = { PLL(CLK_MFGSC1_AO_MFGPLL_SC1, "mfgpll-sc1", MFGPLL_SC1_CON0, - MFGPLL_SC1_CON0, 0, 0, 0, BIT(0), MFGPLL_SC1_CON1, 24, 0, 0, 0, - MFGPLL_SC1_CON1, 0, 22), + MFGPLL_SC1_CON0, 0, 0, PLL_PARENT_EN, BIT(0), MFGPLL_SC1_CON1, 24, + 0, 0, 0, MFGPLL_SC1_CON1, 0, 22), }; static const struct of_device_id of_match_clk_mt8196_mfg[] = { diff --git a/drivers/clk/mediatek/clk-mt8516.c b/drivers/clk/mediatek/clk-mt8516.c index 21eb052b0a539..342a59019fea9 100644 --- a/drivers/clk/mediatek/clk-mt8516.c +++ b/drivers/clk/mediatek/clk-mt8516.c @@ -544,7 +544,7 @@ static const struct mtk_gate_regs top5_cg_regs = { #define GATE_TOP5(_id, _name, _parent, _shift) \ GATE_MTK(_id, _name, _parent, &top5_cg_regs, _shift, &mtk_clk_gate_ops_setclr) -static const struct mtk_gate top_clks[] __initconst = { +static const struct mtk_gate top_clks[] = { /* TOP1 */ GATE_TOP1(CLK_TOP_THEM, "them", "ahb_infra_sel", 1), GATE_TOP1(CLK_TOP_APDMA, "apdma", "ahb_infra_sel", 2), diff --git a/drivers/clk/mediatek/clk-mtk.c b/drivers/clk/mediatek/clk-mtk.c index 19cd27941747a..deafe55a96cb1 100644 --- a/drivers/clk/mediatek/clk-mtk.c +++ b/drivers/clk/mediatek/clk-mtk.c @@ -497,14 +497,16 @@ static int __mtk_clk_simple_probe(struct platform_device *pdev, if (mcd->need_runtime_pm) { - devm_pm_runtime_enable(&pdev->dev); + r = devm_pm_runtime_enable(&pdev->dev); + if (r) + goto unmap_io; /* * Do a pm_runtime_resume_and_get() to workaround a possible * deadlock between clk_register() and the genpd framework. */ r = pm_runtime_resume_and_get(&pdev->dev); if (r) - return r; + goto unmap_io; } /* Calculate how many clk_hw_onecell_data entries to allocate */ @@ -618,11 +620,11 @@ static int __mtk_clk_simple_probe(struct platform_device *pdev, free_data: mtk_free_clk_data(clk_data); free_base: - if (mcd->shared_io && base) - iounmap(base); - if (mcd->need_runtime_pm) pm_runtime_put(&pdev->dev); +unmap_io: + if (mcd->shared_io && base) + iounmap(base); return r; } diff --git a/drivers/clk/mediatek/clk-pll.c b/drivers/clk/mediatek/clk-pll.c index cd2b6ce551c6b..de3eb02670554 100644 --- a/drivers/clk/mediatek/clk-pll.c +++ b/drivers/clk/mediatek/clk-pll.c @@ -358,6 +358,9 @@ struct clk_hw *mtk_clk_register_pll_ops(struct mtk_clk_pll *pll, init.name = data->name; init.flags = (data->flags & PLL_AO) ? CLK_IS_CRITICAL : 0; + if (data->flags & PLL_PARENT_EN) + init.flags |= CLK_OPS_PARENT_ENABLE; + init.ops = pll_ops; if (data->parent_name) init.parent_names = &data->parent_name; diff --git a/drivers/clk/mediatek/clk-pll.h b/drivers/clk/mediatek/clk-pll.h index d71c150ce83e4..de5a8fb7cbcfe 100644 --- a/drivers/clk/mediatek/clk-pll.h +++ b/drivers/clk/mediatek/clk-pll.h @@ -21,6 +21,7 @@ struct mtk_pll_div_table { #define HAVE_RST_BAR BIT(0) #define PLL_AO BIT(1) +#define PLL_PARENT_EN BIT(2) #define POSTDIV_MASK GENMASK(2, 0) struct mtk_pll_data { diff --git a/drivers/clk/meson/g12a.c b/drivers/clk/meson/g12a.c index 185b6348251db..d0d4c7b6dc827 100644 --- a/drivers/clk/meson/g12a.c +++ b/drivers/clk/meson/g12a.c @@ -777,12 +777,23 @@ static struct clk_regmap g12a_hdmi_pll_dco = { }, }; +/* + * G12/SM1 hdmi OD dividers are POWER_OF_TWO dividers but limited to /4. + * A divider value of 3 should map to /8 but instead map /4 so ignore it. + */ +static const struct clk_div_table g12a_hdmi_pll_od_div_table[] = { + { .val = 0, .div = 1 }, + { .val = 1, .div = 2 }, + { .val = 2, .div = 4 }, + { /* sentinel */ } +}; + static struct clk_regmap g12a_hdmi_pll_od = { .data = &(struct clk_regmap_div_data){ .offset = HHI_HDMI_PLL_CNTL0, .shift = 16, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = g12a_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll_od", @@ -800,7 +811,7 @@ static struct clk_regmap g12a_hdmi_pll_od2 = { .offset = HHI_HDMI_PLL_CNTL0, .shift = 18, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = g12a_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll_od2", @@ -818,7 +829,7 @@ static struct clk_regmap g12a_hdmi_pll = { .offset = HHI_HDMI_PLL_CNTL0, .shift = 20, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = g12a_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll", diff --git a/drivers/clk/meson/gxbb.c b/drivers/clk/meson/gxbb.c index 5a229c4ffae10..ec9a3414875ac 100644 --- a/drivers/clk/meson/gxbb.c +++ b/drivers/clk/meson/gxbb.c @@ -349,12 +349,23 @@ static struct clk_regmap gxbb_hdmi_pll = { }, }; +/* + * GXL hdmi OD dividers are POWER_OF_TWO dividers but limited to /4. + * A divider value of 3 should map to /8 but instead map /4 so ignore it. + */ +static const struct clk_div_table gxl_hdmi_pll_od_div_table[] = { + { .val = 0, .div = 1 }, + { .val = 1, .div = 2 }, + { .val = 2, .div = 4 }, + { /* sentinel */ } +}; + static struct clk_regmap gxl_hdmi_pll_od = { .data = &(struct clk_regmap_div_data){ .offset = HHI_HDMI_PLL_CNTL + 8, .shift = 21, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = gxl_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll_od", @@ -372,7 +383,7 @@ static struct clk_regmap gxl_hdmi_pll_od2 = { .offset = HHI_HDMI_PLL_CNTL + 8, .shift = 23, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = gxl_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll_od2", @@ -390,7 +401,7 @@ static struct clk_regmap gxl_hdmi_pll = { .offset = HHI_HDMI_PLL_CNTL + 8, .shift = 19, .width = 2, - .flags = CLK_DIVIDER_POWER_OF_TWO, + .table = gxl_hdmi_pll_od_div_table, }, .hw.init = &(struct clk_init_data){ .name = "hdmi_pll", diff --git a/drivers/clk/meson/s4-peripherals.c b/drivers/clk/meson/s4-peripherals.c index 6d69b132d1e1f..bab4f5700de47 100644 --- a/drivers/clk/meson/s4-peripherals.c +++ b/drivers/clk/meson/s4-peripherals.c @@ -1106,7 +1106,6 @@ static struct clk_regmap s4_cts_enci_sel = { .ops = &clk_regmap_mux_ops, .parent_hws = s4_cts_parents, .num_parents = ARRAY_SIZE(s4_cts_parents), - .flags = CLK_SET_RATE_PARENT, }, }; @@ -1122,7 +1121,6 @@ static struct clk_regmap s4_cts_encp_sel = { .ops = &clk_regmap_mux_ops, .parent_hws = s4_cts_parents, .num_parents = ARRAY_SIZE(s4_cts_parents), - .flags = CLK_SET_RATE_PARENT, }, }; @@ -1138,7 +1136,6 @@ static struct clk_regmap s4_cts_vdac_sel = { .ops = &clk_regmap_mux_ops, .parent_hws = s4_cts_parents, .num_parents = ARRAY_SIZE(s4_cts_parents), - .flags = CLK_SET_RATE_PARENT, }, }; @@ -1169,7 +1166,6 @@ static struct clk_regmap s4_hdmi_tx_sel = { .ops = &clk_regmap_mux_ops, .parent_hws = s4_hdmi_tx_parents, .num_parents = ARRAY_SIZE(s4_hdmi_tx_parents), - .flags = CLK_SET_RATE_PARENT, }, }; diff --git a/drivers/clk/microchip/clk-core.c b/drivers/clk/microchip/clk-core.c index b34348d491f3e..82f62731fc0ed 100644 --- a/drivers/clk/microchip/clk-core.c +++ b/drivers/clk/microchip/clk-core.c @@ -283,14 +283,13 @@ static u8 roclk_get_parent(struct clk_hw *hw) v = (readl(refo->ctrl_reg) >> REFO_SEL_SHIFT) & REFO_SEL_MASK; - if (!refo->parent_map) - return v; - - for (i = 0; i < clk_hw_get_num_parents(hw); i++) - if (refo->parent_map[i] == v) - return i; + if (refo->parent_map) { + for (i = 0; i < clk_hw_get_num_parents(hw); i++) + if (refo->parent_map[i] == v) + return i; + } - return -EINVAL; + return v; } static unsigned long roclk_calc_rate(unsigned long parent_rate, @@ -780,15 +779,6 @@ static unsigned long sclk_get_rate(struct clk_hw *hw, unsigned long parent_rate) return parent_rate / div; } -static int sclk_determine_rate(struct clk_hw *hw, - struct clk_rate_request *req) -{ - req->rate = calc_best_divided_rate(req->rate, req->best_parent_rate, - SLEW_SYSDIV, 1); - - return 0; -} - static int sclk_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate) { @@ -826,13 +816,13 @@ static u8 sclk_get_parent(struct clk_hw *hw) v = (readl(sclk->mux_reg) >> OSC_CUR_SHIFT) & OSC_CUR_MASK; - if (!sclk->parent_map) - return v; + if (sclk->parent_map) { + for (i = 0; i < clk_hw_get_num_parents(hw); i++) + if (sclk->parent_map[i] == v) + return i; + } - for (i = 0; i < clk_hw_get_num_parents(hw); i++) - if (sclk->parent_map[i] == v) - return i; - return -EINVAL; + return v; } static int sclk_set_parent(struct clk_hw *hw, u8 index) @@ -912,7 +902,6 @@ static int sclk_init(struct clk_hw *hw) const struct clk_ops pic32_sclk_ops = { .get_parent = sclk_get_parent, .set_parent = sclk_set_parent, - .determine_rate = sclk_determine_rate, .set_rate = sclk_set_rate, .recalc_rate = sclk_get_rate, .init = sclk_init, diff --git a/drivers/clk/nuvoton/clk-ma35d1-divider.c b/drivers/clk/nuvoton/clk-ma35d1-divider.c index e39f53d5bf457..e992e7c303419 100644 --- a/drivers/clk/nuvoton/clk-ma35d1-divider.c +++ b/drivers/clk/nuvoton/clk-ma35d1-divider.c @@ -44,11 +44,8 @@ static int ma35d1_clkdiv_determine_rate(struct clk_hw *hw, { struct ma35d1_adc_clk_div *dclk = to_ma35d1_adc_clk_div(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - dclk->table, dclk->width, - CLK_DIVIDER_ROUND_CLOSEST); - - return 0; + return divider_determine_rate(hw, req, dclk->table, dclk->width, + CLK_DIVIDER_ROUND_CLOSEST); } static int ma35d1_clkdiv_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate) diff --git a/drivers/clk/nxp/clk-lpc32xx.c b/drivers/clk/nxp/clk-lpc32xx.c index 23f980cf6a2b5..ae2fa5341a2e4 100644 --- a/drivers/clk/nxp/clk-lpc32xx.c +++ b/drivers/clk/nxp/clk-lpc32xx.c @@ -975,10 +975,8 @@ static int clk_divider_determine_rate(struct clk_hw *hw, return 0; } - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - divider->table, divider->width, divider->flags); - - return 0; + return divider_determine_rate(hw, req, divider->table, divider->width, + divider->flags); } static int clk_divider_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/qcom/clk-alpha-pll.c b/drivers/clk/qcom/clk-alpha-pll.c index 6aeba40358c11..a84e8bee65346 100644 --- a/drivers/clk/qcom/clk-alpha-pll.c +++ b/drivers/clk/qcom/clk-alpha-pll.c @@ -1257,11 +1257,8 @@ static int clk_alpha_pll_postdiv_determine_rate(struct clk_hw *hw, else table = clk_alpha_div_table; - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - table, pll->width, - CLK_DIVIDER_POWER_OF_TWO); - - return 0; + return divider_determine_rate(hw, req, table, pll->width, + CLK_DIVIDER_POWER_OF_TWO); } static int clk_alpha_pll_postdiv_ro_determine_rate(struct clk_hw *hw, @@ -1617,11 +1614,8 @@ static int clk_trion_pll_postdiv_determine_rate(struct clk_hw *hw, { struct clk_alpha_pll_postdiv *pll = to_clk_alpha_pll_postdiv(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - pll->post_div_table, - pll->width, CLK_DIVIDER_ROUND_CLOSEST); - - return 0; + return divider_determine_rate(hw, req, pll->post_div_table, pll->width, + CLK_DIVIDER_ROUND_CLOSEST); }; static int @@ -1657,11 +1651,8 @@ static int clk_alpha_pll_postdiv_fabia_determine_rate(struct clk_hw *hw, { struct clk_alpha_pll_postdiv *pll = to_clk_alpha_pll_postdiv(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - pll->post_div_table, - pll->width, CLK_DIVIDER_ROUND_CLOSEST); - - return 0; + return divider_determine_rate(hw, req, pll->post_div_table, pll->width, + CLK_DIVIDER_ROUND_CLOSEST); } static int clk_alpha_pll_postdiv_fabia_set_rate(struct clk_hw *hw, diff --git a/drivers/clk/qcom/clk-rcg2.c b/drivers/clk/qcom/clk-rcg2.c index e18cb8807d735..d0a5847f91114 100644 --- a/drivers/clk/qcom/clk-rcg2.c +++ b/drivers/clk/qcom/clk-rcg2.c @@ -755,7 +755,7 @@ static int clk_rcg2_get_duty_cycle(struct clk_hw *hw, struct clk_duty *duty) static int clk_rcg2_set_duty_cycle(struct clk_hw *hw, struct clk_duty *duty) { struct clk_rcg2 *rcg = to_clk_rcg2(hw); - u32 notn_m, n, m, d, not2d, mask, duty_per, cfg; + u32 notn_m, n, m, d, not2d, mask, cfg; int ret; /* Duty-cycle cannot be modified for non-MND RCGs */ @@ -774,10 +774,8 @@ static int clk_rcg2_set_duty_cycle(struct clk_hw *hw, struct clk_duty *duty) n = (~(notn_m) + m) & mask; - duty_per = (duty->num * 100) / duty->den; - /* Calculate 2d value */ - d = DIV_ROUND_CLOSEST(n * duty_per * 2, 100); + d = DIV_ROUND_CLOSEST(n * duty->num * 2, duty->den); /* * Check bit widths of 2d. If D is too big reduce duty cycle. @@ -1266,6 +1264,7 @@ static int clk_gfx3d_determine_rate(struct clk_hw *hw, if (req->max_rate < parent_req.max_rate) parent_req.max_rate = req->max_rate; + parent_req.best_parent_hw = req->best_parent_hw; ret = __clk_determine_rate(req->best_parent_hw, &parent_req); if (ret) return ret; diff --git a/drivers/clk/qcom/clk-regmap-divider.c b/drivers/clk/qcom/clk-regmap-divider.c index 4f5395f0ab6d0..672e82caf2050 100644 --- a/drivers/clk/qcom/clk-regmap-divider.c +++ b/drivers/clk/qcom/clk-regmap-divider.c @@ -26,24 +26,16 @@ static int div_ro_determine_rate(struct clk_hw *hw, val >>= divider->shift; val &= BIT(divider->width) - 1; - req->rate = divider_ro_round_rate(hw, req->rate, - &req->best_parent_rate, NULL, - divider->width, - CLK_DIVIDER_ROUND_CLOSEST, val); - - return 0; + return divider_ro_determine_rate(hw, req, NULL, divider->width, + CLK_DIVIDER_ROUND_CLOSEST, val); } static int div_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { struct clk_regmap_div *divider = to_clk_regmap_div(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - NULL, - divider->width, - CLK_DIVIDER_ROUND_CLOSEST); - - return 0; + return divider_determine_rate(hw, req, NULL, divider->width, + CLK_DIVIDER_ROUND_CLOSEST); } static int div_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/clk/qcom/common.c b/drivers/clk/qcom/common.c index 1215918867741..eec369d2173b5 100644 --- a/drivers/clk/qcom/common.c +++ b/drivers/clk/qcom/common.c @@ -454,7 +454,7 @@ int qcom_cc_probe_by_index(struct platform_device *pdev, int index, base = devm_platform_ioremap_resource(pdev, index); if (IS_ERR(base)) - return -ENOMEM; + return PTR_ERR(base); regmap = devm_regmap_init_mmio(&pdev->dev, base, desc->config); if (IS_ERR(regmap)) diff --git a/drivers/clk/qcom/dispcc-sdm845.c b/drivers/clk/qcom/dispcc-sdm845.c index 2f9e9665d7e93..78e43f6d75026 100644 --- a/drivers/clk/qcom/dispcc-sdm845.c +++ b/drivers/clk/qcom/dispcc-sdm845.c @@ -280,7 +280,7 @@ static struct clk_rcg2 disp_cc_mdss_pclk0_clk_src = { .name = "disp_cc_mdss_pclk0_clk_src", .parent_data = disp_cc_parent_data_4, .num_parents = ARRAY_SIZE(disp_cc_parent_data_4), - .flags = CLK_SET_RATE_PARENT, + .flags = CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, .ops = &clk_pixel_ops, }, }; @@ -295,7 +295,7 @@ static struct clk_rcg2 disp_cc_mdss_pclk1_clk_src = { .name = "disp_cc_mdss_pclk1_clk_src", .parent_data = disp_cc_parent_data_4, .num_parents = ARRAY_SIZE(disp_cc_parent_data_4), - .flags = CLK_SET_RATE_PARENT, + .flags = CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, .ops = &clk_pixel_ops, }, }; diff --git a/drivers/clk/qcom/dispcc-sm7150.c b/drivers/clk/qcom/dispcc-sm7150.c index ddc7230b8aea7..923f0f38e804c 100644 --- a/drivers/clk/qcom/dispcc-sm7150.c +++ b/drivers/clk/qcom/dispcc-sm7150.c @@ -370,7 +370,7 @@ static struct clk_rcg2 dispcc_mdss_pclk1_clk_src = { .name = "dispcc_mdss_pclk1_clk_src", .parent_data = dispcc_parent_data_4, .num_parents = ARRAY_SIZE(dispcc_parent_data_4), - .flags = CLK_SET_RATE_PARENT, + .flags = CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, .ops = &clk_pixel_ops, }, }; diff --git a/drivers/clk/qcom/gcc-glymur.c b/drivers/clk/qcom/gcc-glymur.c index d938e7dc5b66e..17e860307fa10 100644 --- a/drivers/clk/qcom/gcc-glymur.c +++ b/drivers/clk/qcom/gcc-glymur.c @@ -2317,7 +2317,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_17, .num_parents = ARRAY_SIZE(gcc_parent_data_17), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -2339,7 +2339,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_3, .num_parents = ARRAY_SIZE(gcc_parent_data_3), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-ipq5018.c b/drivers/clk/qcom/gcc-ipq5018.c index dcda2be8c1a51..64792cda06202 100644 --- a/drivers/clk/qcom/gcc-ipq5018.c +++ b/drivers/clk/qcom/gcc-ipq5018.c @@ -1340,6 +1340,7 @@ static struct clk_branch gcc_sleep_clk_src = { .name = "gcc_sleep_clk_src", .parent_data = gcc_sleep_clk_data, .num_parents = ARRAY_SIZE(gcc_sleep_clk_data), + .flags = CLK_IS_CRITICAL, .ops = &clk_branch2_ops, }, }, diff --git a/drivers/clk/qcom/gcc-milos.c b/drivers/clk/qcom/gcc-milos.c index c9d61b05bafa1..81fa09ec55d7f 100644 --- a/drivers/clk/qcom/gcc-milos.c +++ b/drivers/clk/qcom/gcc-milos.c @@ -917,7 +917,7 @@ static struct clk_rcg2 gcc_sdcc1_apps_clk_src = { .name = "gcc_sdcc1_apps_clk_src", .parent_data = gcc_parent_data_9, .num_parents = ARRAY_SIZE(gcc_parent_data_9), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -938,7 +938,7 @@ static struct clk_rcg2 gcc_sdcc1_ice_core_clk_src = { .name = "gcc_sdcc1_ice_core_clk_src", .parent_data = gcc_parent_data_10, .num_parents = ARRAY_SIZE(gcc_parent_data_10), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -962,7 +962,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .name = "gcc_sdcc2_apps_clk_src", .parent_data = gcc_parent_data_11, .num_parents = ARRAY_SIZE(gcc_parent_data_11), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-msm8917.c b/drivers/clk/qcom/gcc-msm8917.c index 0a1aa623cd49a..9d1c5a9953e2c 100644 --- a/drivers/clk/qcom/gcc-msm8917.c +++ b/drivers/clk/qcom/gcc-msm8917.c @@ -3409,7 +3409,6 @@ static struct gdsc cpp_gdsc = { .pd = { .name = "cpp_gdsc", }, - .flags = ALWAYS_ON, .pwrsts = PWRSTS_OFF_ON, }; diff --git a/drivers/clk/qcom/gcc-msm8953.c b/drivers/clk/qcom/gcc-msm8953.c index 8f29ecc74c50b..8fe1d3e421440 100644 --- a/drivers/clk/qcom/gcc-msm8953.c +++ b/drivers/clk/qcom/gcc-msm8953.c @@ -3946,7 +3946,6 @@ static struct gdsc cpp_gdsc = { .pd = { .name = "cpp_gdsc", }, - .flags = ALWAYS_ON, .pwrsts = PWRSTS_OFF_ON, }; diff --git a/drivers/clk/qcom/gcc-qdu1000.c b/drivers/clk/qcom/gcc-qdu1000.c index dbe9e9437939a..915bb9b4ff813 100644 --- a/drivers/clk/qcom/gcc-qdu1000.c +++ b/drivers/clk/qcom/gcc-qdu1000.c @@ -904,7 +904,7 @@ static struct clk_rcg2 gcc_sdcc5_apps_clk_src = { .name = "gcc_sdcc5_apps_clk_src", .parent_data = gcc_parent_data_8, .num_parents = ARRAY_SIZE(gcc_parent_data_8), - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -923,7 +923,7 @@ static struct clk_rcg2 gcc_sdcc5_ice_core_clk_src = { .name = "gcc_sdcc5_ice_core_clk_src", .parent_data = gcc_parent_data_2, .num_parents = ARRAY_SIZE(gcc_parent_data_2), - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sdx75.c b/drivers/clk/qcom/gcc-sdx75.c index 453a6bf8e8786..1f3cd58483a2d 100644 --- a/drivers/clk/qcom/gcc-sdx75.c +++ b/drivers/clk/qcom/gcc-sdx75.c @@ -1033,7 +1033,7 @@ static struct clk_rcg2 gcc_sdcc1_apps_clk_src = { .name = "gcc_sdcc1_apps_clk_src", .parent_data = gcc_parent_data_17, .num_parents = ARRAY_SIZE(gcc_parent_data_17), - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1057,7 +1057,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .name = "gcc_sdcc2_apps_clk_src", .parent_data = gcc_parent_data_18, .num_parents = ARRAY_SIZE(gcc_parent_data_18), - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sm4450.c b/drivers/clk/qcom/gcc-sm4450.c index e2d9e4691c5b7..023d840e9f4ef 100644 --- a/drivers/clk/qcom/gcc-sm4450.c +++ b/drivers/clk/qcom/gcc-sm4450.c @@ -769,7 +769,7 @@ static struct clk_rcg2 gcc_sdcc1_apps_clk_src = { .parent_data = gcc_parent_data_4, .num_parents = ARRAY_SIZE(gcc_parent_data_4), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -791,7 +791,7 @@ static struct clk_rcg2 gcc_sdcc1_ice_core_clk_src = { .parent_data = gcc_parent_data_4, .num_parents = ARRAY_SIZE(gcc_parent_data_4), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -815,7 +815,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_6, .num_parents = ARRAY_SIZE(gcc_parent_data_6), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sm8450.c b/drivers/clk/qcom/gcc-sm8450.c index 65d7d52bce034..b18bb34889ab2 100644 --- a/drivers/clk/qcom/gcc-sm8450.c +++ b/drivers/clk/qcom/gcc-sm8450.c @@ -1034,7 +1034,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_7, .num_parents = ARRAY_SIZE(gcc_parent_data_7), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1057,7 +1057,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_0, .num_parents = ARRAY_SIZE(gcc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sm8550.c b/drivers/clk/qcom/gcc-sm8550.c index 862a9bf73bcb5..36a5b7de5b55d 100644 --- a/drivers/clk/qcom/gcc-sm8550.c +++ b/drivers/clk/qcom/gcc-sm8550.c @@ -1025,7 +1025,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_9, .num_parents = ARRAY_SIZE(gcc_parent_data_9), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1048,7 +1048,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_0, .num_parents = ARRAY_SIZE(gcc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sm8650.c b/drivers/clk/qcom/gcc-sm8650.c index 24f98062b9dd5..2dd6444ce0365 100644 --- a/drivers/clk/qcom/gcc-sm8650.c +++ b/drivers/clk/qcom/gcc-sm8650.c @@ -1257,7 +1257,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_11, .num_parents = ARRAY_SIZE(gcc_parent_data_11), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1279,7 +1279,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_0, .num_parents = ARRAY_SIZE(gcc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sm8750.c b/drivers/clk/qcom/gcc-sm8750.c index def86b71a3da5..db81569dd4b17 100644 --- a/drivers/clk/qcom/gcc-sm8750.c +++ b/drivers/clk/qcom/gcc-sm8750.c @@ -1030,7 +1030,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_8, .num_parents = ARRAY_SIZE(gcc_parent_data_8), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1052,7 +1052,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_0, .num_parents = ARRAY_SIZE(gcc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-x1e80100.c b/drivers/clk/qcom/gcc-x1e80100.c index f4580b65e42bd..47a143507d419 100644 --- a/drivers/clk/qcom/gcc-x1e80100.c +++ b/drivers/clk/qcom/gcc-x1e80100.c @@ -1516,7 +1516,7 @@ static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { .parent_data = gcc_parent_data_9, .num_parents = ARRAY_SIZE(gcc_parent_data_9), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; @@ -1538,7 +1538,7 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { .parent_data = gcc_parent_data_0, .num_parents = ARRAY_SIZE(gcc_parent_data_0), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_floor_ops, + .ops = &clk_rcg2_shared_floor_ops, }, }; diff --git a/drivers/clk/renesas/rzg2l-cpg.c b/drivers/clk/renesas/rzg2l-cpg.c index 07909e80bae24..edad47ca33ec5 100644 --- a/drivers/clk/renesas/rzg2l-cpg.c +++ b/drivers/clk/renesas/rzg2l-cpg.c @@ -122,8 +122,8 @@ struct div_hw_data { struct rzg2l_pll5_param { u32 pl5_fracin; + u16 pl5_intin; u8 pl5_refdiv; - u8 pl5_intin; u8 pl5_postdiv1; u8 pl5_postdiv2; u8 pl5_spread; @@ -572,8 +572,8 @@ rzg2l_cpg_get_foutpostdiv_rate(struct rzg2l_pll5_param *params, foutvco_rate = div_u64(mul_u32_u32(EXTAL_FREQ_IN_MEGA_HZ * MEGA, (params->pl5_intin << 24) + params->pl5_fracin), params->pl5_refdiv) >> 24; - foutpostdiv_rate = DIV_ROUND_CLOSEST_ULL(foutvco_rate, - params->pl5_postdiv1 * params->pl5_postdiv2); + foutpostdiv_rate = DIV_ROUND_CLOSEST(foutvco_rate, + params->pl5_postdiv1 * params->pl5_postdiv2); return foutpostdiv_rate; } @@ -1647,6 +1647,7 @@ static int __rzg2l_cpg_assert(struct reset_controller_dev *rcdev, u32 mask = BIT(info->resets[id].bit); s8 monbit = info->resets[id].monbit; u32 value = mask << 16; + u32 mon; int ret; dev_dbg(rcdev->dev, "%s id:%ld offset:0x%x\n", @@ -1667,10 +1668,10 @@ static int __rzg2l_cpg_assert(struct reset_controller_dev *rcdev, return 0; } - ret = readl_poll_timeout_atomic(priv->base + reg, value, - assert == !!(value & mask), 10, 200); - if (ret && !assert) { - value = mask << 16; + ret = readl_poll_timeout_atomic(priv->base + reg, mon, + assert == !!(mon & mask), 10, 200); + if (ret) { + value ^= mask; writel(value, priv->base + CLK_RST_R(info->resets[id].off)); } diff --git a/drivers/clk/rockchip/clk.c b/drivers/clk/rockchip/clk.c index 19caf26c991bd..2d30b1e24f013 100644 --- a/drivers/clk/rockchip/clk.c +++ b/drivers/clk/rockchip/clk.c @@ -693,7 +693,7 @@ void rockchip_clk_register_late_branches(struct device *dev, break; } - if (!pdev) + if (IS_ERR_OR_NULL(pdev)) dev_err(dev, "failed to register device for clock %s\n", list->name); } } diff --git a/drivers/clk/sophgo/clk-sg2042-clkgen.c b/drivers/clk/sophgo/clk-sg2042-clkgen.c index 683661b71787c..9725ac4e050a4 100644 --- a/drivers/clk/sophgo/clk-sg2042-clkgen.c +++ b/drivers/clk/sophgo/clk-sg2042-clkgen.c @@ -180,7 +180,6 @@ static int sg2042_clk_divider_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { struct sg2042_divider_clock *divider = to_sg2042_clk_divider(hw); - unsigned long ret_rate; u32 bestdiv; /* if read only, just return current value */ @@ -191,17 +190,13 @@ static int sg2042_clk_divider_determine_rate(struct clk_hw *hw, bestdiv = readl(divider->reg) >> divider->shift; bestdiv &= clk_div_mask(divider->width); } - ret_rate = DIV_ROUND_UP_ULL((u64)req->best_parent_rate, bestdiv); - } else { - ret_rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, NULL, - divider->width, divider->div_flags); - } + req->rate = DIV_ROUND_UP_ULL((u64)req->best_parent_rate, bestdiv); - pr_debug("--> %s: divider_round_rate: val = %ld\n", - clk_hw_get_name(hw), ret_rate); - req->rate = ret_rate; + return 0; + } - return 0; + return divider_determine_rate(hw, req, NULL, divider->width, + divider->div_flags); } static int sg2042_clk_divider_set_rate(struct clk_hw *hw, diff --git a/drivers/clk/spacemit/Makefile b/drivers/clk/spacemit/Makefile index 5ec6da61db98e..ad2bf315109b8 100644 --- a/drivers/clk/spacemit/Makefile +++ b/drivers/clk/spacemit/Makefile @@ -1,5 +1,10 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_SPACEMIT_K1_CCU) = spacemit-ccu-k1.o -spacemit-ccu-k1-y = ccu_pll.o ccu_mix.o ccu_ddn.o +obj-$(CONFIG_SPACEMIT_CCU) += spacemit-ccu.o +spacemit-ccu-y += ccu_common.o +spacemit-ccu-y += ccu_pll.o +spacemit-ccu-y += ccu_mix.o +spacemit-ccu-y += ccu_ddn.o + +obj-$(CONFIG_SPACEMIT_K1_CCU) += spacemit-ccu-k1.o spacemit-ccu-k1-y += ccu-k1.o diff --git a/drivers/clk/spacemit/ccu-k1.c b/drivers/clk/spacemit/ccu-k1.c index 4761bc1e3b6e6..01d9485b615d3 100644 --- a/drivers/clk/spacemit/ccu-k1.c +++ b/drivers/clk/spacemit/ccu-k1.c @@ -1204,6 +1204,7 @@ static struct platform_driver k1_ccu_driver = { }; module_platform_driver(k1_ccu_driver); +MODULE_IMPORT_NS("CLK_SPACEMIT"); MODULE_DESCRIPTION("SpacemiT K1 CCU driver"); MODULE_AUTHOR("Haylen Chu "); MODULE_LICENSE("GPL"); diff --git a/drivers/clk/spacemit/ccu_common.c b/drivers/clk/spacemit/ccu_common.c new file mode 100644 index 0000000000000..4412c4104dabb --- /dev/null +++ b/drivers/clk/spacemit/ccu_common.c @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include + +MODULE_DESCRIPTION("SpacemiT CCU common clock driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/spacemit/ccu_ddn.c b/drivers/clk/spacemit/ccu_ddn.c index 5b16e273bee5b..b5540e0781ffa 100644 --- a/drivers/clk/spacemit/ccu_ddn.c +++ b/drivers/clk/spacemit/ccu_ddn.c @@ -84,3 +84,4 @@ const struct clk_ops spacemit_ccu_ddn_ops = { .determine_rate = ccu_ddn_determine_rate, .set_rate = ccu_ddn_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_ddn_ops, "CLK_SPACEMIT"); diff --git a/drivers/clk/spacemit/ccu_mix.c b/drivers/clk/spacemit/ccu_mix.c index 7b79908753723..67f8b12b4f5b7 100644 --- a/drivers/clk/spacemit/ccu_mix.c +++ b/drivers/clk/spacemit/ccu_mix.c @@ -198,24 +198,28 @@ const struct clk_ops spacemit_ccu_gate_ops = { .enable = ccu_gate_enable, .is_enabled = ccu_gate_is_enabled, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_gate_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_factor_ops = { .determine_rate = ccu_factor_determine_rate, .recalc_rate = ccu_factor_recalc_rate, .set_rate = ccu_factor_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_factor_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_mux_ops = { .determine_rate = ccu_mix_determine_rate, .get_parent = ccu_mux_get_parent, .set_parent = ccu_mux_set_parent, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_mux_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_div_ops = { .determine_rate = ccu_mix_determine_rate, .recalc_rate = ccu_div_recalc_rate, .set_rate = ccu_mix_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_div_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_factor_gate_ops = { .disable = ccu_gate_disable, @@ -226,6 +230,7 @@ const struct clk_ops spacemit_ccu_factor_gate_ops = { .recalc_rate = ccu_factor_recalc_rate, .set_rate = ccu_factor_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_factor_gate_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_mux_gate_ops = { .disable = ccu_gate_disable, @@ -236,6 +241,7 @@ const struct clk_ops spacemit_ccu_mux_gate_ops = { .get_parent = ccu_mux_get_parent, .set_parent = ccu_mux_set_parent, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_mux_gate_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_div_gate_ops = { .disable = ccu_gate_disable, @@ -246,6 +252,7 @@ const struct clk_ops spacemit_ccu_div_gate_ops = { .recalc_rate = ccu_div_recalc_rate, .set_rate = ccu_mix_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_div_gate_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_mux_div_gate_ops = { .disable = ccu_gate_disable, @@ -259,6 +266,7 @@ const struct clk_ops spacemit_ccu_mux_div_gate_ops = { .recalc_rate = ccu_div_recalc_rate, .set_rate = ccu_mix_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_mux_div_gate_ops, "CLK_SPACEMIT"); const struct clk_ops spacemit_ccu_mux_div_ops = { .get_parent = ccu_mux_get_parent, @@ -268,3 +276,4 @@ const struct clk_ops spacemit_ccu_mux_div_ops = { .recalc_rate = ccu_div_recalc_rate, .set_rate = ccu_mix_set_rate, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_mux_div_ops, "CLK_SPACEMIT"); diff --git a/drivers/clk/spacemit/ccu_pll.c b/drivers/clk/spacemit/ccu_pll.c index d92f0dae65a49..76d0244873d87 100644 --- a/drivers/clk/spacemit/ccu_pll.c +++ b/drivers/clk/spacemit/ccu_pll.c @@ -157,3 +157,4 @@ const struct clk_ops spacemit_ccu_pll_ops = { .determine_rate = ccu_pll_determine_rate, .is_enabled = ccu_pll_is_enabled, }; +EXPORT_SYMBOL_NS_GPL(spacemit_ccu_pll_ops, "CLK_SPACEMIT"); diff --git a/drivers/clk/sprd/div.c b/drivers/clk/sprd/div.c index 0134238819680..cd57163a7204c 100644 --- a/drivers/clk/sprd/div.c +++ b/drivers/clk/sprd/div.c @@ -14,11 +14,7 @@ static int sprd_div_determine_rate(struct clk_hw *hw, { struct sprd_div *cd = hw_to_sprd_div(hw); - req->rate = divider_round_rate(&cd->common.hw, req->rate, - &req->best_parent_rate, - NULL, cd->div.width, 0); - - return 0; + return divider_determine_rate(&cd->common.hw, req, NULL, cd->div.width, 0); } unsigned long sprd_div_helper_recalc_rate(struct sprd_clk_common *common, diff --git a/drivers/clk/stm32/clk-stm32-core.c b/drivers/clk/stm32/clk-stm32-core.c index 72825b9c36a4d..e921c25a929c3 100644 --- a/drivers/clk/stm32/clk-stm32-core.c +++ b/drivers/clk/stm32/clk-stm32-core.c @@ -369,22 +369,14 @@ static int clk_stm32_divider_determine_rate(struct clk_hw *hw, val = readl(div->base + divider->offset) >> divider->shift; val &= clk_div_mask(divider->width); - req->rate = divider_ro_round_rate(hw, req->rate, - &req->best_parent_rate, - divider->table, - divider->width, - divider->flags, val); - - return 0; + return divider_ro_determine_rate(hw, req, + divider->table, + divider->width, + divider->flags, val); } - req->rate = divider_round_rate_parent(hw, clk_hw_get_parent(hw), - req->rate, - &req->best_parent_rate, - divider->table, - divider->width, divider->flags); - - return 0; + return divider_determine_rate(hw, req, divider->table, divider->width, + divider->flags); } static unsigned long clk_stm32_divider_recalc_rate(struct clk_hw *hw, @@ -441,7 +433,6 @@ static int clk_stm32_composite_determine_rate(struct clk_hw *hw, { struct clk_stm32_composite *composite = to_clk_stm32_composite(hw); const struct stm32_div_cfg *divider; - long rate; if (composite->div_id == NO_STM32_DIV) return 0; @@ -455,24 +446,13 @@ static int clk_stm32_composite_determine_rate(struct clk_hw *hw, val = readl(composite->base + divider->offset) >> divider->shift; val &= clk_div_mask(divider->width); - rate = divider_ro_round_rate(hw, req->rate, &req->best_parent_rate, - divider->table, divider->width, divider->flags, - val); - if (rate < 0) - return rate; - - req->rate = rate; - return 0; + return divider_ro_determine_rate(hw, req, divider->table, + divider->width, divider->flags, + val); } - rate = divider_round_rate_parent(hw, clk_hw_get_parent(hw), - req->rate, &req->best_parent_rate, - divider->table, divider->width, divider->flags); - if (rate < 0) - return rate; - - req->rate = rate; - return 0; + return divider_determine_rate(hw, req, divider->table, divider->width, + divider->flags); } static u8 clk_stm32_composite_get_parent(struct clk_hw *hw) diff --git a/drivers/clk/tegra/clk-tegra124-emc.c b/drivers/clk/tegra/clk-tegra124-emc.c index 2a6db04342815..5f1af6dfe7154 100644 --- a/drivers/clk/tegra/clk-tegra124-emc.c +++ b/drivers/clk/tegra/clk-tegra124-emc.c @@ -197,8 +197,8 @@ static struct tegra_emc *emc_ensure_emc_driver(struct tegra_clk_emc *tegra) tegra->emc_node = NULL; tegra->emc = platform_get_drvdata(pdev); + put_device(&pdev->dev); if (!tegra->emc) { - put_device(&pdev->dev); pr_err("%s: cannot find EMC driver\n", __func__); return NULL; } @@ -538,8 +538,10 @@ struct clk *tegra124_clk_register_emc(void __iomem *base, struct device_node *np tegra->hw.init = &init; clk = clk_register(NULL, &tegra->hw); - if (IS_ERR(clk)) + if (IS_ERR(clk)) { + kfree(tegra); return clk; + } tegra->prev_parent = clk_hw_get_parent_by_index( &tegra->hw, emc_get_parent(&tegra->hw))->clk; diff --git a/drivers/clk/thead/clk-th1520-ap.c b/drivers/clk/thead/clk-th1520-ap.c index 71ad03a998e8e..d870f0c665f8a 100644 --- a/drivers/clk/thead/clk-th1520-ap.c +++ b/drivers/clk/thead/clk-th1520-ap.c @@ -8,11 +8,14 @@ #include #include #include +#include #include #include #include #include +#define TH1520_PLL_STS 0x80 + #define TH1520_PLL_POSTDIV2 GENMASK(26, 24) #define TH1520_PLL_POSTDIV1 GENMASK(22, 20) #define TH1520_PLL_FBDIV GENMASK(19, 8) @@ -23,6 +26,13 @@ #define TH1520_PLL_FRAC GENMASK(23, 0) #define TH1520_PLL_FRAC_BITS 24 +/* + * All PLLs in TH1520 take 21250ns at maximum to lock, let's take its double + * for safety. + */ +#define TH1520_PLL_LOCK_TIMEOUT_US 44 +#define TH1520_PLL_STABLE_DELAY_US 30 + struct ccu_internal { u8 shift; u8 width; @@ -64,6 +74,7 @@ struct ccu_div { struct ccu_pll { struct ccu_common common; + u32 lock_sts_mask; }; #define TH_CCU_ARG(_shift, _width) \ @@ -299,9 +310,21 @@ static void ccu_pll_disable(struct clk_hw *hw) static int ccu_pll_enable(struct clk_hw *hw) { struct ccu_pll *pll = hw_to_ccu_pll(hw); + u32 reg; + int ret; - return regmap_clear_bits(pll->common.map, pll->common.cfg1, - TH1520_PLL_VCO_RST); + regmap_clear_bits(pll->common.map, pll->common.cfg1, + TH1520_PLL_VCO_RST); + + ret = regmap_read_poll_timeout_atomic(pll->common.map, TH1520_PLL_STS, + reg, reg & pll->lock_sts_mask, + 5, TH1520_PLL_LOCK_TIMEOUT_US); + if (ret) + return ret; + + udelay(TH1520_PLL_STABLE_DELAY_US); + + return 0; } static int ccu_pll_is_enabled(struct clk_hw *hw) @@ -389,6 +412,7 @@ static struct ccu_pll cpu_pll0_clk = { &clk_pll_ops, CLK_IS_CRITICAL), }, + .lock_sts_mask = BIT(1), }; static struct ccu_pll cpu_pll1_clk = { @@ -401,6 +425,7 @@ static struct ccu_pll cpu_pll1_clk = { &clk_pll_ops, CLK_IS_CRITICAL), }, + .lock_sts_mask = BIT(4), }; static struct ccu_pll gmac_pll_clk = { @@ -413,6 +438,7 @@ static struct ccu_pll gmac_pll_clk = { &clk_pll_ops, CLK_IS_CRITICAL), }, + .lock_sts_mask = BIT(3), }; static const struct clk_hw *gmac_pll_clk_parent[] = { @@ -433,6 +459,7 @@ static struct ccu_pll video_pll_clk = { &clk_pll_ops, CLK_IS_CRITICAL), }, + .lock_sts_mask = BIT(7), }; static const struct clk_hw *video_pll_clk_parent[] = { @@ -453,6 +480,7 @@ static struct ccu_pll dpu0_pll_clk = { &clk_pll_ops, 0), }, + .lock_sts_mask = BIT(8), }; static const struct clk_hw *dpu0_pll_clk_parent[] = { @@ -469,6 +497,7 @@ static struct ccu_pll dpu1_pll_clk = { &clk_pll_ops, 0), }, + .lock_sts_mask = BIT(9), }; static const struct clk_hw *dpu1_pll_clk_parent[] = { @@ -485,6 +514,7 @@ static struct ccu_pll tee_pll_clk = { &clk_pll_ops, CLK_IS_CRITICAL), }, + .lock_sts_mask = BIT(10), }; static const struct clk_parent_data c910_i0_parents[] = { diff --git a/drivers/clk/x86/clk-cgu.c b/drivers/clk/x86/clk-cgu.c index d099667355f8d..92ee05d75af2b 100644 --- a/drivers/clk/x86/clk-cgu.c +++ b/drivers/clk/x86/clk-cgu.c @@ -137,10 +137,8 @@ static int lgm_clk_divider_determine_rate(struct clk_hw *hw, { struct lgm_clk_divider *divider = to_lgm_clk_divider(hw); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, divider->table, - divider->width, divider->flags); - - return 0; + return divider_determine_rate(hw, req, divider->table, divider->width, + divider->flags); } static int diff --git a/drivers/clk/zynqmp/divider.c b/drivers/clk/zynqmp/divider.c index c824eeacd8ebd..984e577ea6711 100644 --- a/drivers/clk/zynqmp/divider.c +++ b/drivers/clk/zynqmp/divider.c @@ -111,10 +111,9 @@ static unsigned long zynqmp_clk_divider_recalc_rate(struct clk_hw *hw, } /** - * zynqmp_clk_divider_round_rate() - Round rate of divider clock + * zynqmp_clk_divider_determine_rate() - Determine rate of divider clock * @hw: handle between common and hardware-specific interfaces - * @rate: rate of clock to be set - * @prate: rate of parent clock + * @req: rate of clock to be set * * Return: 0 on success else error+reason */ @@ -151,8 +150,9 @@ static int zynqmp_clk_divider_determine_rate(struct clk_hw *hw, width = fls(divider->max_div); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - NULL, width, divider->flags); + ret = divider_determine_rate(hw, req, NULL, width, divider->flags); + if (ret != 0) + return ret; if (divider->is_frac && (clk_hw_get_flags(hw) & CLK_SET_RATE_PARENT) && (req->rate % req->best_parent_rate)) diff --git a/drivers/clk/zynqmp/pll.c b/drivers/clk/zynqmp/pll.c index 630a3936c97c3..6bc2c3934f564 100644 --- a/drivers/clk/zynqmp/pll.c +++ b/drivers/clk/zynqmp/pll.c @@ -91,10 +91,9 @@ static inline void zynqmp_pll_set_mode(struct clk_hw *hw, bool on) } /** - * zynqmp_pll_round_rate() - Round a clock frequency + * zynqmp_pll_determine_rate() - Round a clock frequency * @hw: Handle between common and hardware-specific interfaces - * @rate: Desired clock frequency - * @prate: Clock frequency of parent clock + * @req: Desired clock frequency * * Return: Frequency closest to @rate the hardware can generate */ diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index ffcd23668763f..1119fbd52e1c9 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -254,6 +254,7 @@ config KEYSTONE_TIMER config INTEGRATOR_AP_TIMER bool "Integrator-AP timer driver" if COMPILE_TEST + depends on OF select CLKSRC_MMIO help Enables support for the Integrator-AP timer. diff --git a/drivers/clocksource/sh_tmu.c b/drivers/clocksource/sh_tmu.c index beffff81c00f3..3fc6ed9b56300 100644 --- a/drivers/clocksource/sh_tmu.c +++ b/drivers/clocksource/sh_tmu.c @@ -143,16 +143,6 @@ static void sh_tmu_start_stop_ch(struct sh_tmu_channel *ch, int start) static int __sh_tmu_enable(struct sh_tmu_channel *ch) { - int ret; - - /* enable clock */ - ret = clk_enable(ch->tmu->clk); - if (ret) { - dev_err(&ch->tmu->pdev->dev, "ch%u: cannot enable clock\n", - ch->index); - return ret; - } - /* make sure channel is disabled */ sh_tmu_start_stop_ch(ch, 0); @@ -174,7 +164,6 @@ static int sh_tmu_enable(struct sh_tmu_channel *ch) if (ch->enable_count++ > 0) return 0; - pm_runtime_get_sync(&ch->tmu->pdev->dev); dev_pm_syscore_device(&ch->tmu->pdev->dev, true); return __sh_tmu_enable(ch); @@ -187,9 +176,6 @@ static void __sh_tmu_disable(struct sh_tmu_channel *ch) /* disable interrupts in TMU block */ sh_tmu_write(ch, TCR, TCR_TPSC_CLK4); - - /* stop clock */ - clk_disable(ch->tmu->clk); } static void sh_tmu_disable(struct sh_tmu_channel *ch) @@ -203,7 +189,6 @@ static void sh_tmu_disable(struct sh_tmu_channel *ch) __sh_tmu_disable(ch); dev_pm_syscore_device(&ch->tmu->pdev->dev, false); - pm_runtime_put(&ch->tmu->pdev->dev); } static void sh_tmu_set_next(struct sh_tmu_channel *ch, unsigned long delta, @@ -552,7 +537,6 @@ static int sh_tmu_setup(struct sh_tmu_device *tmu, struct platform_device *pdev) goto err_clk_unprepare; tmu->rate = clk_get_rate(tmu->clk) / 4; - clk_disable(tmu->clk); /* Map the memory resource. */ ret = sh_tmu_map_memory(tmu); @@ -626,8 +610,6 @@ static int sh_tmu_probe(struct platform_device *pdev) out: if (tmu->has_clockevent || tmu->has_clocksource) pm_runtime_irq_safe(&pdev->dev); - else - pm_runtime_idle(&pdev->dev); return 0; } diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c index dc11b62399ad5..47dd76acd31e5 100644 --- a/drivers/cpufreq/cpufreq-dt-platdev.c +++ b/drivers/cpufreq/cpufreq-dt-platdev.c @@ -169,8 +169,11 @@ static const struct of_device_id blocklist[] __initconst = { { .compatible = "qcom,sdm845", }, { .compatible = "qcom,sdx75", }, { .compatible = "qcom,sm6115", }, + { .compatible = "qcom,sm6125", }, + { .compatible = "qcom,sm6150", }, { .compatible = "qcom,sm6350", }, { .compatible = "qcom,sm6375", }, + { .compatible = "qcom,sm7125", }, { .compatible = "qcom,sm7225", }, { .compatible = "qcom,sm7325", }, { .compatible = "qcom,sm8150", }, diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 492a10f1bdbfa..00b87f8ee70bc 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1152,7 +1152,7 @@ static void hybrid_init_cpu_capacity_scaling(bool refresh) * the capacity of SMT threads is not deterministic even approximately, * do not do that when SMT is in use. */ - if (hwp_is_hybrid && !sched_smt_active() && arch_enable_hybrid_capacity_scale()) { + if (hwp_is_hybrid && !cpu_smt_possible() && arch_enable_hybrid_capacity_scale()) { hybrid_refresh_cpu_capacity_scaling(); /* * Disabling ITMT causes sched domains to be rebuilt to disable asym @@ -1467,13 +1467,13 @@ static void __intel_pstate_update_max_freq(struct cpufreq_policy *policy, refresh_frequency_limits(policy); } -static bool intel_pstate_update_max_freq(struct cpudata *cpudata) +static bool intel_pstate_update_max_freq(int cpu) { - struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); if (!policy) return false; - __intel_pstate_update_max_freq(policy, cpudata); + __intel_pstate_update_max_freq(policy, all_cpu_data[cpu]); return true; } @@ -1492,7 +1492,7 @@ static void intel_pstate_update_limits_for_all(void) int cpu; for_each_possible_cpu(cpu) - intel_pstate_update_max_freq(all_cpu_data[cpu]); + intel_pstate_update_max_freq(cpu); mutex_lock(&hybrid_capacity_lock); @@ -1932,7 +1932,7 @@ static void intel_pstate_notify_work(struct work_struct *work) struct cpudata *cpudata = container_of(to_delayed_work(work), struct cpudata, hwp_notify_work); - if (intel_pstate_update_max_freq(cpudata)) { + if (intel_pstate_update_max_freq(cpudata->cpu)) { /* * The driver will not be unregistered while this function is * running, so update the capacity without acquiring the driver diff --git a/drivers/cpufreq/scmi-cpufreq.c b/drivers/cpufreq/scmi-cpufreq.c index d2a110079f5fd..c450cf9c881d4 100644 --- a/drivers/cpufreq/scmi-cpufreq.c +++ b/drivers/cpufreq/scmi-cpufreq.c @@ -101,6 +101,7 @@ static int scmi_cpu_domain_id(struct device *cpu_dev) return -EINVAL; } + of_node_put(domain_id.np); return domain_id.args[0]; } diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c index 56132e843c991..8950796a493de 100644 --- a/drivers/cpuidle/cpuidle.c +++ b/drivers/cpuidle/cpuidle.c @@ -357,6 +357,16 @@ noinstr int cpuidle_enter_state(struct cpuidle_device *dev, int cpuidle_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, bool *stop_tick) { + /* + * If there is only a single idle state (or none), there is nothing + * meaningful for the governor to choose. Skip the governor and + * always use state 0 with the tick running. + */ + if (drv->state_count <= 1) { + *stop_tick = false; + return 0; + } + return cpuidle_curr_governor->select(drv, dev, stop_tick); } diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c index 64d6f7a1c7766..ca863ba03d454 100644 --- a/drivers/cpuidle/governors/menu.c +++ b/drivers/cpuidle/governors/menu.c @@ -239,7 +239,7 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, /* Find the shortest expected idle interval. */ predicted_ns = get_typical_interval(data) * NSEC_PER_USEC; - if (predicted_ns > RESIDENCY_THRESHOLD_NS) { + if (predicted_ns > RESIDENCY_THRESHOLD_NS || tick_nohz_tick_stopped()) { unsigned int timer_us; /* Determine the time till the closest timer. */ @@ -259,6 +259,16 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, RESOLUTION * DECAY * NSEC_PER_USEC); /* Use the lowest expected idle interval to pick the idle state. */ predicted_ns = min((u64)timer_us * NSEC_PER_USEC, predicted_ns); + /* + * If the tick is already stopped, the cost of possible short + * idle duration misprediction is much higher, because the CPU + * may be stuck in a shallow idle state for a long time as a + * result of it. In that case, say we might mispredict and use + * the known time till the closest timer event for the idle + * state selection. + */ + if (tick_nohz_tick_stopped() && predicted_ns < TICK_NSEC) + predicted_ns = data->next_timer_ns; } else { /* * Because the next timer event is not going to be determined @@ -284,16 +294,6 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, return 0; } - /* - * If the tick is already stopped, the cost of possible short idle - * duration misprediction is much higher, because the CPU may be stuck - * in a shallow idle state for a long time as a result of it. In that - * case, say we might mispredict and use the known time till the closest - * timer event for the idle state selection. - */ - if (tick_nohz_tick_stopped() && predicted_ns < TICK_NSEC) - predicted_ns = data->next_timer_ns; - /* * Find the idle state with the lowest power while satisfying * our constraints. diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index 107ccb2ade420..c6117c23eb25b 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -4814,7 +4814,8 @@ static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv) { struct device *dev = priv->dev; struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); - int err; + struct dpaa2_caam_priv_per_cpu *ppriv; + int i, err; if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) { err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle); @@ -4822,6 +4823,12 @@ static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv) dev_err(dev, "dpseci_reset() failed\n"); } + for_each_cpu(i, priv->clean_mask) { + ppriv = per_cpu_ptr(priv->ppriv, i); + free_netdev(ppriv->net_dev); + } + free_cpumask_var(priv->clean_mask); + dpaa2_dpseci_congestion_free(priv); dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); } @@ -5007,16 +5014,15 @@ static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) struct device *dev = &ls_dev->dev; struct dpaa2_caam_priv *priv; struct dpaa2_caam_priv_per_cpu *ppriv; - cpumask_var_t clean_mask; int err, cpu; u8 i; err = -ENOMEM; - if (!zalloc_cpumask_var(&clean_mask, GFP_KERNEL)) - goto err_cpumask; - priv = dev_get_drvdata(dev); + if (!zalloc_cpumask_var(&priv->clean_mask, GFP_KERNEL)) + goto err_cpumask; + priv->dev = dev; priv->dpsec_id = ls_dev->obj_desc.id; @@ -5118,7 +5124,7 @@ static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) err = -ENOMEM; goto err_alloc_netdev; } - cpumask_set_cpu(cpu, clean_mask); + cpumask_set_cpu(cpu, priv->clean_mask); ppriv->net_dev->dev = *dev; netif_napi_add_tx_weight(ppriv->net_dev, &ppriv->napi, @@ -5126,18 +5132,16 @@ static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) DPAA2_CAAM_NAPI_WEIGHT); } - err = 0; - goto free_cpumask; + return 0; err_alloc_netdev: - free_dpaa2_pcpu_netdev(priv, clean_mask); + free_dpaa2_pcpu_netdev(priv, priv->clean_mask); err_get_rx_queue: dpaa2_dpseci_congestion_free(priv); err_get_vers: dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); err_open: -free_cpumask: - free_cpumask_var(clean_mask); + free_cpumask_var(priv->clean_mask); err_cpumask: return err; } @@ -5182,7 +5186,6 @@ static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv) ppriv = per_cpu_ptr(priv->ppriv, i); napi_disable(&ppriv->napi); netif_napi_del(&ppriv->napi); - free_netdev(ppriv->net_dev); } return 0; diff --git a/drivers/crypto/caam/caamalg_qi2.h b/drivers/crypto/caam/caamalg_qi2.h index 61d1219a202fc..8e65b4b28c7ba 100644 --- a/drivers/crypto/caam/caamalg_qi2.h +++ b/drivers/crypto/caam/caamalg_qi2.h @@ -42,6 +42,7 @@ * @mc_io: pointer to MC portal's I/O object * @domain: IOMMU domain * @ppriv: per CPU pointers to privata data + * @clean_mask: CPU mask of CPUs that have allocated netdevs */ struct dpaa2_caam_priv { int dpsec_id; @@ -65,6 +66,7 @@ struct dpaa2_caam_priv { struct dpaa2_caam_priv_per_cpu __percpu *ppriv; struct dentry *dfs_root; + cpumask_var_t clean_mask; }; /** diff --git a/drivers/crypto/cavium/cpt/cptvf_main.c b/drivers/crypto/cavium/cpt/cptvf_main.c index c246920e6f540..bccd680c7f7ee 100644 --- a/drivers/crypto/cavium/cpt/cptvf_main.c +++ b/drivers/crypto/cavium/cpt/cptvf_main.c @@ -180,7 +180,8 @@ static void free_command_queues(struct cpt_vf *cptvf, hlist_for_each_entry_safe(chunk, node, &cqinfo->queue[i].chead, nextchunk) { - dma_free_coherent(&pdev->dev, chunk->size, + dma_free_coherent(&pdev->dev, + chunk->size + CPT_NEXT_CHUNK_PTR_SIZE, chunk->head, chunk->dma_addr); chunk->head = NULL; diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c index d78865d9d5f09..d0412e5847625 100644 --- a/drivers/crypto/ccp/ccp-ops.c +++ b/drivers/crypto/ccp/ccp-ops.c @@ -642,7 +642,7 @@ ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) struct ccp_data dst; struct ccp_data aad; struct ccp_op op; - } *wa __cleanup(kfree) = kzalloc(sizeof *wa, GFP_KERNEL); + } *wa __free(kfree) = kzalloc(sizeof(*wa), GFP_KERNEL); unsigned int dm_offset; unsigned int authsize; unsigned int jobid; diff --git a/drivers/crypto/ccp/psp-dev.c b/drivers/crypto/ccp/psp-dev.c index 9e21da0e298ad..5c7f7e02a7d8a 100644 --- a/drivers/crypto/ccp/psp-dev.c +++ b/drivers/crypto/ccp/psp-dev.c @@ -351,6 +351,17 @@ struct psp_device *psp_get_master_device(void) return sp ? sp->psp_data : NULL; } +int psp_restore(struct sp_device *sp) +{ + struct psp_device *psp = sp->psp_data; + int ret = 0; + + if (psp->tee_data) + ret = tee_restore(psp); + + return ret; +} + void psp_pci_init(void) { psp_master = psp_get_master_device(); diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 0d13d47c164bb..5fdba0fe4acce 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -119,13 +119,6 @@ static size_t sev_es_tmr_size = SEV_TMR_SIZE; #define NV_LENGTH (32 * 1024) static void *sev_init_ex_buffer; -/* - * SEV_DATA_RANGE_LIST: - * Array containing range of pages that firmware transitions to HV-fixed - * page state. - */ -static struct sev_data_range_list *snp_range_list; - static void __sev_firmware_shutdown(struct sev_device *sev, bool panic); static int snp_shutdown_on_panic(struct notifier_block *nb, @@ -1365,6 +1358,7 @@ static int snp_filter_reserved_mem_regions(struct resource *rs, void *arg) static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid) { + struct sev_data_range_list *snp_range_list __free(kfree) = NULL; struct psp_device *psp = psp_master; struct sev_data_snp_init_ex data; struct sev_device *sev; @@ -2351,11 +2345,10 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable) static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) { struct sev_device *sev = psp_master->sev_data; - bool shutdown_required = false; struct sev_data_snp_addr buf; struct page *status_page; - int ret, error; void *data; + int ret; if (!argp->data) return -EINVAL; @@ -2366,31 +2359,35 @@ static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) data = page_address(status_page); - if (!sev->snp_initialized) { - ret = snp_move_to_init_state(argp, &shutdown_required); - if (ret) - goto cleanup; - } - /* - * Firmware expects status page to be in firmware-owned state, otherwise - * it will report firmware error code INVALID_PAGE_STATE (0x1A). + * SNP_PLATFORM_STATUS can be executed in any SNP state. But if executed + * when SNP has been initialized, the status page must be firmware-owned. */ - if (rmp_mark_pages_firmware(__pa(data), 1, true)) { - ret = -EFAULT; - goto cleanup; + if (sev->snp_initialized) { + /* + * Firmware expects the status page to be in Firmware state, + * otherwise it will report an error INVALID_PAGE_STATE. + */ + if (rmp_mark_pages_firmware(__pa(data), 1, true)) { + ret = -EFAULT; + goto cleanup; + } } buf.address = __psp_pa(data); ret = __sev_do_cmd_locked(SEV_CMD_SNP_PLATFORM_STATUS, &buf, &argp->error); - /* - * Status page will be transitioned to Reclaim state upon success, or - * left in Firmware state in failure. Use snp_reclaim_pages() to - * transition either case back to Hypervisor-owned state. - */ - if (snp_reclaim_pages(__pa(data), 1, true)) - return -EFAULT; + if (sev->snp_initialized) { + /* + * The status page will be in Reclaim state on success, or left + * in Firmware state on failure. Use snp_reclaim_pages() to + * transition either case back to Hypervisor-owned state. + */ + if (snp_reclaim_pages(__pa(data), 1, true)) { + snp_leak_pages(__page_to_pfn(status_page), 1); + return -EFAULT; + } + } if (ret) goto cleanup; @@ -2400,9 +2397,6 @@ static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) ret = -EFAULT; cleanup: - if (shutdown_required) - __sev_snp_shutdown_locked(&error, false); - __free_pages(status_page, 0); return ret; } @@ -2753,11 +2747,6 @@ static void __sev_firmware_shutdown(struct sev_device *sev, bool panic) sev_init_ex_buffer = NULL; } - if (snp_range_list) { - kfree(snp_range_list); - snp_range_list = NULL; - } - __sev_snp_shutdown_locked(&error, panic); } diff --git a/drivers/crypto/ccp/sp-dev.c b/drivers/crypto/ccp/sp-dev.c index 3467f6db4f505..f204aa5df96e2 100644 --- a/drivers/crypto/ccp/sp-dev.c +++ b/drivers/crypto/ccp/sp-dev.c @@ -230,6 +230,18 @@ int sp_resume(struct sp_device *sp) return 0; } +int sp_restore(struct sp_device *sp) +{ + if (sp->psp_data) { + int ret = psp_restore(sp); + + if (ret) + return ret; + } + + return sp_resume(sp); +} + struct sp_device *sp_get_psp_master_device(void) { struct sp_device *i, *ret = NULL; diff --git a/drivers/crypto/ccp/sp-dev.h b/drivers/crypto/ccp/sp-dev.h index 6f9d7063257d7..c8a611ef275b5 100644 --- a/drivers/crypto/ccp/sp-dev.h +++ b/drivers/crypto/ccp/sp-dev.h @@ -141,6 +141,7 @@ void sp_destroy(struct sp_device *sp); int sp_suspend(struct sp_device *sp); int sp_resume(struct sp_device *sp); +int sp_restore(struct sp_device *sp); int sp_request_ccp_irq(struct sp_device *sp, irq_handler_t handler, const char *name, void *data); void sp_free_ccp_irq(struct sp_device *sp, void *data); @@ -174,6 +175,7 @@ int psp_dev_init(struct sp_device *sp); void psp_pci_init(void); void psp_dev_destroy(struct sp_device *sp); void psp_pci_exit(void); +int psp_restore(struct sp_device *sp); #else /* !CONFIG_CRYPTO_DEV_SP_PSP */ @@ -181,6 +183,7 @@ static inline int psp_dev_init(struct sp_device *sp) { return 0; } static inline void psp_pci_init(void) { } static inline void psp_dev_destroy(struct sp_device *sp) { } static inline void psp_pci_exit(void) { } +static inline int psp_restore(struct sp_device *sp) { return 0; } #endif /* CONFIG_CRYPTO_DEV_SP_PSP */ diff --git a/drivers/crypto/ccp/sp-pci.c b/drivers/crypto/ccp/sp-pci.c index 8891ceee1d7d0..6ac805d99ccb3 100644 --- a/drivers/crypto/ccp/sp-pci.c +++ b/drivers/crypto/ccp/sp-pci.c @@ -353,6 +353,13 @@ static int __maybe_unused sp_pci_resume(struct device *dev) return sp_resume(sp); } +static int __maybe_unused sp_pci_restore(struct device *dev) +{ + struct sp_device *sp = dev_get_drvdata(dev); + + return sp_restore(sp); +} + #ifdef CONFIG_CRYPTO_DEV_SP_PSP static const struct sev_vdata sevv1 = { .cmdresp_reg = 0x10580, /* C2PMSG_32 */ @@ -563,7 +570,14 @@ static const struct pci_device_id sp_pci_table[] = { }; MODULE_DEVICE_TABLE(pci, sp_pci_table); -static SIMPLE_DEV_PM_OPS(sp_pci_pm_ops, sp_pci_suspend, sp_pci_resume); +static const struct dev_pm_ops sp_pci_pm_ops = { + .suspend = pm_sleep_ptr(sp_pci_suspend), + .resume = pm_sleep_ptr(sp_pci_resume), + .freeze = pm_sleep_ptr(sp_pci_suspend), + .thaw = pm_sleep_ptr(sp_pci_resume), + .poweroff = pm_sleep_ptr(sp_pci_suspend), + .restore_early = pm_sleep_ptr(sp_pci_restore), +}; static struct pci_driver sp_pci_driver = { .name = "ccp", diff --git a/drivers/crypto/ccp/tee-dev.c b/drivers/crypto/ccp/tee-dev.c index 5e1d80724678d..92ffa412622a2 100644 --- a/drivers/crypto/ccp/tee-dev.c +++ b/drivers/crypto/ccp/tee-dev.c @@ -86,10 +86,34 @@ static inline void tee_free_cmd_buffer(struct tee_init_ring_cmd *cmd) kfree(cmd); } +static bool tee_send_destroy_cmd(struct psp_tee_device *tee) +{ + unsigned int reg; + int ret; + + ret = psp_mailbox_command(tee->psp, PSP_CMD_TEE_RING_DESTROY, NULL, + TEE_DEFAULT_CMD_TIMEOUT, ®); + if (ret) { + dev_err(tee->dev, "tee: ring destroy command timed out, disabling TEE support\n"); + psp_dead = true; + return false; + } + + if (FIELD_GET(PSP_CMDRESP_STS, reg)) { + dev_err(tee->dev, "tee: ring destroy command failed (%#010lx)\n", + FIELD_GET(PSP_CMDRESP_STS, reg)); + psp_dead = true; + return false; + } + + return true; +} + static int tee_init_ring(struct psp_tee_device *tee) { int ring_size = MAX_RING_BUFFER_ENTRIES * sizeof(struct tee_ring_cmd); struct tee_init_ring_cmd *cmd; + bool retry = false; unsigned int reg; int ret; @@ -112,6 +136,7 @@ static int tee_init_ring(struct psp_tee_device *tee) /* Send command buffer details to Trusted OS by writing to * CPU-PSP message registers */ +retry_init: ret = psp_mailbox_command(tee->psp, PSP_CMD_TEE_RING_INIT, cmd, TEE_DEFAULT_CMD_TIMEOUT, ®); if (ret) { @@ -122,9 +147,22 @@ static int tee_init_ring(struct psp_tee_device *tee) } if (FIELD_GET(PSP_CMDRESP_STS, reg)) { + /* + * During the hibernate resume sequence driver may have gotten loaded + * but the ring not properly destroyed. If the ring doesn't work, try + * to destroy and re-init once. + */ + if (!retry && FIELD_GET(PSP_CMDRESP_STS, reg) == PSP_TEE_STS_RING_BUSY) { + dev_info(tee->dev, "tee: ring init command failed with busy status, retrying\n"); + if (tee_send_destroy_cmd(tee)) { + retry = true; + goto retry_init; + } + } dev_err(tee->dev, "tee: ring init command failed (%#010lx)\n", FIELD_GET(PSP_CMDRESP_STS, reg)); tee_free_ring(tee); + psp_dead = true; ret = -EIO; } @@ -136,24 +174,13 @@ static int tee_init_ring(struct psp_tee_device *tee) static void tee_destroy_ring(struct psp_tee_device *tee) { - unsigned int reg; - int ret; - if (!tee->rb_mgr.ring_start) return; if (psp_dead) goto free_ring; - ret = psp_mailbox_command(tee->psp, PSP_CMD_TEE_RING_DESTROY, NULL, - TEE_DEFAULT_CMD_TIMEOUT, ®); - if (ret) { - dev_err(tee->dev, "tee: ring destroy command timed out, disabling TEE support\n"); - psp_dead = true; - } else if (FIELD_GET(PSP_CMDRESP_STS, reg)) { - dev_err(tee->dev, "tee: ring destroy command failed (%#010lx)\n", - FIELD_GET(PSP_CMDRESP_STS, reg)); - } + tee_send_destroy_cmd(tee); free_ring: tee_free_ring(tee); @@ -365,3 +392,8 @@ int psp_check_tee_status(void) return 0; } EXPORT_SYMBOL(psp_check_tee_status); + +int tee_restore(struct psp_device *psp) +{ + return tee_init_ring(psp->tee_data); +} diff --git a/drivers/crypto/ccp/tee-dev.h b/drivers/crypto/ccp/tee-dev.h index ea9a2b7c05f57..c23416cb7bb37 100644 --- a/drivers/crypto/ccp/tee-dev.h +++ b/drivers/crypto/ccp/tee-dev.h @@ -111,5 +111,6 @@ struct tee_ring_cmd { int tee_dev_init(struct psp_device *psp); void tee_dev_destroy(struct psp_device *psp); +int tee_restore(struct psp_device *psp); #endif /* __TEE_DEV_H__ */ diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig index 4835bdebdbb38..a0cb1a8186ac9 100644 --- a/drivers/crypto/hisilicon/Kconfig +++ b/drivers/crypto/hisilicon/Kconfig @@ -57,6 +57,7 @@ config CRYPTO_DEV_HISI_ZIP depends on UACCE || UACCE=n depends on ACPI select CRYPTO_DEV_HISI_QM + select CRYPTO_DEFLATE help Support for HiSilicon ZIP Driver diff --git a/drivers/crypto/hisilicon/hpre/hpre.h b/drivers/crypto/hisilicon/hpre/hpre.h index 0f3ddbadbcf99..021dbd9a1d48f 100644 --- a/drivers/crypto/hisilicon/hpre/hpre.h +++ b/drivers/crypto/hisilicon/hpre/hpre.h @@ -94,9 +94,8 @@ struct hpre_sqe { __le64 key; __le64 in; __le64 out; - __le16 tag; - __le16 resv2; -#define _HPRE_SQE_ALIGN_EXT 7 + __le64 tag; +#define _HPRE_SQE_ALIGN_EXT 6 __le32 rsvd1[_HPRE_SQE_ALIGN_EXT]; }; diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index 21ccf879f70c5..839c1f6771436 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -93,6 +93,7 @@ struct hpre_dh_ctx { char *g; /* m */ dma_addr_t dma_g; + struct crypto_kpp *soft_tfm; }; struct hpre_ecdh_ctx { @@ -103,17 +104,15 @@ struct hpre_ecdh_ctx { /* low address: x->y */ unsigned char *g; dma_addr_t dma_g; + struct crypto_kpp *soft_tfm; }; struct hpre_ctx { struct hisi_qp *qp; struct device *dev; - struct hpre_asym_request **req_list; struct hpre *hpre; - spinlock_t req_lock; unsigned int key_sz; bool crt_g2_mode; - struct idr req_idr; union { struct hpre_rsa_ctx rsa; struct hpre_dh_ctx dh; @@ -123,6 +122,7 @@ struct hpre_ctx { unsigned int curve_id; /* for high performance core */ u8 enable_hpcore; + bool fallback; }; struct hpre_asym_request { @@ -136,7 +136,6 @@ struct hpre_asym_request { struct kpp_request *ecdh; } areq; int err; - int req_id; hpre_cb cb; struct timespec64 req_time; }; @@ -151,79 +150,13 @@ static inline unsigned int hpre_align_pd(void) return (hpre_align_sz() - 1) & ~(crypto_tfm_ctx_alignment() - 1); } -static int hpre_alloc_req_id(struct hpre_ctx *ctx) +static void hpre_dfx_add_req_time(struct hpre_asym_request *hpre_req) { - unsigned long flags; - int id; - - spin_lock_irqsave(&ctx->req_lock, flags); - id = idr_alloc(&ctx->req_idr, NULL, 0, ctx->qp->sq_depth, GFP_ATOMIC); - spin_unlock_irqrestore(&ctx->req_lock, flags); - - return id; -} - -static void hpre_free_req_id(struct hpre_ctx *ctx, int req_id) -{ - unsigned long flags; - - spin_lock_irqsave(&ctx->req_lock, flags); - idr_remove(&ctx->req_idr, req_id); - spin_unlock_irqrestore(&ctx->req_lock, flags); -} - -static int hpre_add_req_to_ctx(struct hpre_asym_request *hpre_req) -{ - struct hpre_ctx *ctx; - struct hpre_dfx *dfx; - int id; - - ctx = hpre_req->ctx; - id = hpre_alloc_req_id(ctx); - if (unlikely(id < 0)) - return -EINVAL; - - ctx->req_list[id] = hpre_req; - hpre_req->req_id = id; + struct hpre_ctx *ctx = hpre_req->ctx; + struct hpre_dfx *dfx = ctx->hpre->debug.dfx; - dfx = ctx->hpre->debug.dfx; if (atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value)) ktime_get_ts64(&hpre_req->req_time); - - return id; -} - -static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req) -{ - struct hpre_ctx *ctx = hpre_req->ctx; - int id = hpre_req->req_id; - - if (hpre_req->req_id >= 0) { - hpre_req->req_id = HPRE_INVLD_REQ_ID; - ctx->req_list[id] = NULL; - hpre_free_req_id(ctx, id); - } -} - -static struct hisi_qp *hpre_get_qp_and_start(u8 type) -{ - struct hisi_qp *qp; - int ret; - - qp = hpre_create_qp(type); - if (!qp) { - pr_err("Can not create hpre qp!\n"); - return ERR_PTR(-ENODEV); - } - - ret = hisi_qm_start_qp(qp, 0); - if (ret < 0) { - hisi_qm_free_qps(&qp, 1); - pci_err(qp->qm->pdev, "Can not start qp!\n"); - return ERR_PTR(-EINVAL); - } - - return qp; } static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req, @@ -340,26 +273,19 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx, static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe, void **kreq) { - struct hpre_asym_request *req; unsigned int err, done, alg; - int id; #define HPRE_NO_HW_ERR 0 #define HPRE_HW_TASK_DONE 3 #define HREE_HW_ERR_MASK GENMASK(10, 0) #define HREE_SQE_DONE_MASK GENMASK(1, 0) #define HREE_ALG_TYPE_MASK GENMASK(4, 0) - id = (int)le16_to_cpu(sqe->tag); - req = ctx->req_list[id]; - hpre_rm_req_from_ctx(req); - *kreq = req; + *kreq = (void *)le64_to_cpu(sqe->tag); err = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_ALG_BITS) & HREE_HW_ERR_MASK; - done = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_DONE_SHIFT) & HREE_SQE_DONE_MASK; - if (likely(err == HPRE_NO_HW_ERR && done == HPRE_HW_TASK_DONE)) return 0; @@ -370,36 +296,10 @@ static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe, return -EINVAL; } -static int hpre_ctx_set(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen) -{ - struct hpre *hpre; - - if (!ctx || !qp || qlen < 0) - return -EINVAL; - - spin_lock_init(&ctx->req_lock); - ctx->qp = qp; - ctx->dev = &qp->qm->pdev->dev; - - hpre = container_of(ctx->qp->qm, struct hpre, qm); - ctx->hpre = hpre; - ctx->req_list = kcalloc(qlen, sizeof(void *), GFP_KERNEL); - if (!ctx->req_list) - return -ENOMEM; - ctx->key_sz = 0; - ctx->crt_g2_mode = false; - idr_init(&ctx->req_idr); - - return 0; -} - static void hpre_ctx_clear(struct hpre_ctx *ctx, bool is_clear_all) { - if (is_clear_all) { - idr_destroy(&ctx->req_idr); - kfree(ctx->req_list); + if (is_clear_all) hisi_qm_free_qps(&ctx->qp, 1); - } ctx->crt_g2_mode = false; ctx->key_sz = 0; @@ -467,49 +367,44 @@ static void hpre_rsa_cb(struct hpre_ctx *ctx, void *resp) static void hpre_alg_cb(struct hisi_qp *qp, void *resp) { - struct hpre_ctx *ctx = qp->qp_ctx; - struct hpre_dfx *dfx = ctx->hpre->debug.dfx; + struct hpre_asym_request *h_req; struct hpre_sqe *sqe = resp; - struct hpre_asym_request *req = ctx->req_list[le16_to_cpu(sqe->tag)]; - if (unlikely(!req)) { - atomic64_inc(&dfx[HPRE_INVALID_REQ_CNT].value); + h_req = (struct hpre_asym_request *)le64_to_cpu(sqe->tag); + if (unlikely(!h_req)) { + pr_err("Failed to get request, and qp_id is %u\n", qp->qp_id); return; } - req->cb(ctx, resp); -} - -static void hpre_stop_qp_and_put(struct hisi_qp *qp) -{ - hisi_qm_stop_qp(qp); - hisi_qm_free_qps(&qp, 1); + h_req->cb(h_req->ctx, resp); } static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type) { struct hisi_qp *qp; - int ret; + struct hpre *hpre; - qp = hpre_get_qp_and_start(type); - if (IS_ERR(qp)) - return PTR_ERR(qp); + qp = hpre_create_qp(type); + if (!qp) { + ctx->qp = NULL; + return -ENODEV; + } - qp->qp_ctx = ctx; qp->req_cb = hpre_alg_cb; + ctx->qp = qp; + ctx->dev = &qp->qm->pdev->dev; + hpre = container_of(ctx->qp->qm, struct hpre, qm); + ctx->hpre = hpre; + ctx->key_sz = 0; + ctx->crt_g2_mode = false; - ret = hpre_ctx_set(ctx, qp, qp->sq_depth); - if (ret) - hpre_stop_qp_and_put(qp); - - return ret; + return 0; } static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa) { struct hpre_asym_request *h_req; struct hpre_sqe *msg; - int req_id; void *tmp; if (is_rsa) { @@ -549,11 +444,8 @@ static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa) msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; h_req->ctx = ctx; - req_id = hpre_add_req_to_ctx(h_req); - if (req_id < 0) - return -EBUSY; - - msg->tag = cpu_to_le16((u16)req_id); + hpre_dfx_add_req_time(h_req); + msg->tag = cpu_to_le64((uintptr_t)h_req); return 0; } @@ -566,9 +458,7 @@ static int hpre_send(struct hpre_ctx *ctx, struct hpre_sqe *msg) do { atomic64_inc(&dfx[HPRE_SEND_CNT].value); - spin_lock_bh(&ctx->req_lock); ret = hisi_qp_send(ctx->qp, msg); - spin_unlock_bh(&ctx->req_lock); if (ret != -EBUSY) break; atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value); @@ -619,12 +509,53 @@ static int hpre_dh_compute_value(struct kpp_request *req) return -EINPROGRESS; clear_all: - hpre_rm_req_from_ctx(hpre_req); hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); return ret; } +static struct kpp_request *hpre_dh_prepare_fb_req(struct kpp_request *req) +{ + struct kpp_request *fb_req = kpp_request_ctx(req); + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + kpp_request_set_tfm(fb_req, ctx->dh.soft_tfm); + kpp_request_set_callback(fb_req, req->base.flags, req->base.complete, req->base.data); + kpp_request_set_input(fb_req, req->src, req->src_len); + kpp_request_set_output(fb_req, req->dst, req->dst_len); + + return fb_req; +} + +static int hpre_dh_generate_public_key(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + struct kpp_request *fb_req; + + if (ctx->fallback) { + fb_req = hpre_dh_prepare_fb_req(req); + return crypto_kpp_generate_public_key(fb_req); + } + + return hpre_dh_compute_value(req); +} + +static int hpre_dh_compute_shared_secret(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + struct kpp_request *fb_req; + + if (ctx->fallback) { + fb_req = hpre_dh_prepare_fb_req(req); + return crypto_kpp_compute_shared_secret(fb_req); + } + + return hpre_dh_compute_value(req); +} + static int hpre_is_dh_params_length_valid(unsigned int key_sz) { #define _HPRE_DH_GRP1 768 @@ -651,13 +582,6 @@ static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params) struct device *dev = ctx->dev; unsigned int sz; - if (params->p_size > HPRE_DH_MAX_P_SZ) - return -EINVAL; - - if (hpre_is_dh_params_length_valid(params->p_size << - HPRE_BITS_2_BYTES_SHIFT)) - return -EINVAL; - sz = ctx->key_sz = params->p_size; ctx->dh.xa_p = dma_alloc_coherent(dev, sz << 1, &ctx->dh.dma_xa_p, GFP_KERNEL); @@ -690,8 +614,8 @@ static void hpre_dh_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) struct device *dev = ctx->dev; unsigned int sz = ctx->key_sz; - if (is_clear_all) - hisi_qm_stop_qp(ctx->qp); + if (!ctx->qp) + return; if (ctx->dh.g) { dma_free_coherent(dev, sz, ctx->dh.g, ctx->dh.dma_g); @@ -718,6 +642,13 @@ static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf, if (crypto_dh_decode_key(buf, len, ¶ms) < 0) return -EINVAL; + if (!ctx->qp) + goto set_soft_secret; + + if (hpre_is_dh_params_length_valid(params.p_size << + HPRE_BITS_2_BYTES_SHIFT)) + goto set_soft_secret; + /* Free old secret if any */ hpre_dh_clear_ctx(ctx, false); @@ -728,27 +659,55 @@ static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf, memcpy(ctx->dh.xa_p + (ctx->key_sz - params.key_size), params.key, params.key_size); + ctx->fallback = false; return 0; err_clear_ctx: hpre_dh_clear_ctx(ctx, false); return ret; +set_soft_secret: + ctx->fallback = true; + return crypto_kpp_set_secret(ctx->dh.soft_tfm, buf, len); } static unsigned int hpre_dh_max_size(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + if (ctx->fallback) + return crypto_kpp_maxsize(ctx->dh.soft_tfm); + return ctx->key_sz; } static int hpre_dh_init_tfm(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + const char *alg = kpp_alg_name(tfm); + unsigned int reqsize; + int ret; + + ctx->dh.soft_tfm = crypto_alloc_kpp(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->dh.soft_tfm)) { + pr_err("Failed to alloc dh tfm!\n"); + return PTR_ERR(ctx->dh.soft_tfm); + } - kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); + crypto_kpp_set_flags(ctx->dh.soft_tfm, crypto_kpp_get_flags(tfm)); - return hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); + reqsize = max(sizeof(struct hpre_asym_request) + hpre_align_pd(), + sizeof(struct kpp_request) + crypto_kpp_reqsize(ctx->dh.soft_tfm)); + kpp_set_reqsize(tfm, reqsize); + + ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); + if (ret && ret != -ENODEV) { + crypto_free_kpp(ctx->dh.soft_tfm); + return ret; + } else if (ret == -ENODEV) { + ctx->fallback = true; + } + + return 0; } static void hpre_dh_exit_tfm(struct crypto_kpp *tfm) @@ -756,6 +715,7 @@ static void hpre_dh_exit_tfm(struct crypto_kpp *tfm) struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); hpre_dh_clear_ctx(ctx, true); + crypto_free_kpp(ctx->dh.soft_tfm); } static void hpre_rsa_drop_leading_zeros(const char **ptr, size_t *len) @@ -795,9 +755,8 @@ static int hpre_rsa_enc(struct akcipher_request *req) struct hpre_sqe *msg = &hpre_req->req; int ret; - /* For 512 and 1536 bits key size, use soft tfm instead */ - if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || - ctx->key_sz == HPRE_RSA_1536BITS_KSZ) { + /* For unsupported key size and unavailable devices, use soft tfm instead */ + if (ctx->fallback) { akcipher_request_set_tfm(req, ctx->rsa.soft_tfm); ret = crypto_akcipher_encrypt(req); akcipher_request_set_tfm(req, tfm); @@ -828,7 +787,6 @@ static int hpre_rsa_enc(struct akcipher_request *req) return -EINPROGRESS; clear_all: - hpre_rm_req_from_ctx(hpre_req); hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); return ret; @@ -843,9 +801,8 @@ static int hpre_rsa_dec(struct akcipher_request *req) struct hpre_sqe *msg = &hpre_req->req; int ret; - /* For 512 and 1536 bits key size, use soft tfm instead */ - if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || - ctx->key_sz == HPRE_RSA_1536BITS_KSZ) { + /* For unsupported key size and unavailable devices, use soft tfm instead */ + if (ctx->fallback) { akcipher_request_set_tfm(req, ctx->rsa.soft_tfm); ret = crypto_akcipher_decrypt(req); akcipher_request_set_tfm(req, tfm); @@ -883,7 +840,6 @@ static int hpre_rsa_dec(struct akcipher_request *req) return -EINPROGRESS; clear_all: - hpre_rm_req_from_ctx(hpre_req); hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); return ret; @@ -899,8 +855,10 @@ static int hpre_rsa_set_n(struct hpre_ctx *ctx, const char *value, ctx->key_sz = vlen; /* if invalid key size provided, we use software tfm */ - if (!hpre_rsa_key_size_is_support(ctx->key_sz)) + if (!hpre_rsa_key_size_is_support(ctx->key_sz)) { + ctx->fallback = true; return 0; + } ctx->rsa.pubkey = dma_alloc_coherent(ctx->dev, vlen << 1, &ctx->rsa.dma_pubkey, @@ -1035,8 +993,8 @@ static void hpre_rsa_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) unsigned int half_key_sz = ctx->key_sz >> 1; struct device *dev = ctx->dev; - if (is_clear_all) - hisi_qm_stop_qp(ctx->qp); + if (!ctx->qp) + return; if (ctx->rsa.pubkey) { dma_free_coherent(dev, ctx->key_sz << 1, @@ -1117,6 +1075,7 @@ static int hpre_rsa_setkey(struct hpre_ctx *ctx, const void *key, goto free; } + ctx->fallback = false; return 0; free: @@ -1134,6 +1093,9 @@ static int hpre_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key, if (ret) return ret; + if (!ctx->qp) + return 0; + return hpre_rsa_setkey(ctx, key, keylen, false); } @@ -1147,6 +1109,9 @@ static int hpre_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key, if (ret) return ret; + if (!ctx->qp) + return 0; + return hpre_rsa_setkey(ctx, key, keylen, true); } @@ -1154,9 +1119,8 @@ static unsigned int hpre_rsa_max_size(struct crypto_akcipher *tfm) { struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); - /* For 512 and 1536 bits key size, use soft tfm instead */ - if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || - ctx->key_sz == HPRE_RSA_1536BITS_KSZ) + /* For unsupported key size and unavailable devices, use soft tfm instead */ + if (ctx->fallback) return crypto_akcipher_maxsize(ctx->rsa.soft_tfm); return ctx->key_sz; @@ -1177,10 +1141,14 @@ static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm) hpre_align_pd()); ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); - if (ret) + if (ret && ret != -ENODEV) { crypto_free_akcipher(ctx->rsa.soft_tfm); + return ret; + } else if (ret == -ENODEV) { + ctx->fallback = true; + } - return ret; + return 0; } static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm) @@ -1207,9 +1175,6 @@ static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) unsigned int sz = ctx->key_sz; unsigned int shift = sz << 1; - if (is_clear_all) - hisi_qm_stop_qp(ctx->qp); - if (ctx->ecdh.p) { /* ecdh: p->a->k->b */ memzero_explicit(ctx->ecdh.p + shift, sz); @@ -1346,7 +1311,7 @@ static int hpre_ecdh_set_param(struct hpre_ctx *ctx, struct ecdh *params) return 0; } -static bool hpre_key_is_zero(char *key, unsigned short key_sz) +static bool hpre_key_is_zero(const char *key, unsigned short key_sz) { int i; @@ -1387,6 +1352,9 @@ static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, struct ecdh params; int ret; + if (ctx->fallback) + return crypto_kpp_set_secret(ctx->ecdh.soft_tfm, buf, len); + if (crypto_ecdh_decode_key(buf, len, ¶ms) < 0) { dev_err(dev, "failed to decode ecdh key!\n"); return -EINVAL; @@ -1488,7 +1456,6 @@ static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx, { struct hpre_asym_request *h_req; struct hpre_sqe *msg; - int req_id; void *tmp; if (req->dst_len < ctx->key_sz << 1) { @@ -1510,11 +1477,8 @@ static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx, msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; h_req->ctx = ctx; - req_id = hpre_add_req_to_ctx(h_req); - if (req_id < 0) - return -EBUSY; - - msg->tag = cpu_to_le16((u16)req_id); + hpre_dfx_add_req_time(h_req); + msg->tag = cpu_to_le64((uintptr_t)h_req); return 0; } @@ -1612,28 +1576,86 @@ static int hpre_ecdh_compute_value(struct kpp_request *req) return -EINPROGRESS; clear_all: - hpre_rm_req_from_ctx(hpre_req); hpre_ecdh_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); return ret; } +static int hpre_ecdh_generate_public_key(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + int ret; + + if (ctx->fallback) { + kpp_request_set_tfm(req, ctx->ecdh.soft_tfm); + ret = crypto_kpp_generate_public_key(req); + kpp_request_set_tfm(req, tfm); + return ret; + } + + return hpre_ecdh_compute_value(req); +} + +static int hpre_ecdh_compute_shared_secret(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + int ret; + + if (ctx->fallback) { + kpp_request_set_tfm(req, ctx->ecdh.soft_tfm); + ret = crypto_kpp_compute_shared_secret(req); + kpp_request_set_tfm(req, tfm); + return ret; + } + + return hpre_ecdh_compute_value(req); +} + static unsigned int hpre_ecdh_max_size(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + if (ctx->fallback) + return crypto_kpp_maxsize(ctx->ecdh.soft_tfm); + /* max size is the pub_key_size, include x and y */ return ctx->key_sz << 1; } +static int hpre_ecdh_init_tfm(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + const char *alg = kpp_alg_name(tfm); + int ret; + + ret = hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); + if (!ret) { + kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); + return 0; + } else if (ret && ret != -ENODEV) { + return ret; + } + + ctx->ecdh.soft_tfm = crypto_alloc_kpp(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->ecdh.soft_tfm)) { + pr_err("Failed to alloc %s tfm!\n", alg); + return PTR_ERR(ctx->ecdh.soft_tfm); + } + + crypto_kpp_set_flags(ctx->ecdh.soft_tfm, crypto_kpp_get_flags(tfm)); + ctx->fallback = true; + + return 0; +} + static int hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); ctx->curve_id = ECC_CURVE_NIST_P192; - kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); - - return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); + return hpre_ecdh_init_tfm(tfm); } static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) @@ -1643,9 +1665,7 @@ static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) ctx->curve_id = ECC_CURVE_NIST_P256; ctx->enable_hpcore = 1; - kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); - - return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); + return hpre_ecdh_init_tfm(tfm); } static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm) @@ -1654,15 +1674,18 @@ static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm) ctx->curve_id = ECC_CURVE_NIST_P384; - kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); - - return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); + return hpre_ecdh_init_tfm(tfm); } static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + if (ctx->fallback) { + crypto_free_kpp(ctx->ecdh.soft_tfm); + return; + } + hpre_ecc_clear_ctx(ctx, true); } @@ -1680,13 +1703,14 @@ static struct akcipher_alg rsa = { .cra_name = "rsa", .cra_driver_name = "hpre-rsa", .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, }, }; static struct kpp_alg dh = { .set_secret = hpre_dh_set_secret, - .generate_public_key = hpre_dh_compute_value, - .compute_shared_secret = hpre_dh_compute_value, + .generate_public_key = hpre_dh_generate_public_key, + .compute_shared_secret = hpre_dh_compute_shared_secret, .max_size = hpre_dh_max_size, .init = hpre_dh_init_tfm, .exit = hpre_dh_exit_tfm, @@ -1696,14 +1720,15 @@ static struct kpp_alg dh = { .cra_name = "dh", .cra_driver_name = "hpre-dh", .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, }, }; static struct kpp_alg ecdh_curves[] = { { .set_secret = hpre_ecdh_set_secret, - .generate_public_key = hpre_ecdh_compute_value, - .compute_shared_secret = hpre_ecdh_compute_value, + .generate_public_key = hpre_ecdh_generate_public_key, + .compute_shared_secret = hpre_ecdh_compute_shared_secret, .max_size = hpre_ecdh_max_size, .init = hpre_ecdh_nist_p192_init_tfm, .exit = hpre_ecdh_exit_tfm, @@ -1713,11 +1738,12 @@ static struct kpp_alg ecdh_curves[] = { .cra_name = "ecdh-nist-p192", .cra_driver_name = "hpre-ecdh-nist-p192", .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, }, }, { .set_secret = hpre_ecdh_set_secret, - .generate_public_key = hpre_ecdh_compute_value, - .compute_shared_secret = hpre_ecdh_compute_value, + .generate_public_key = hpre_ecdh_generate_public_key, + .compute_shared_secret = hpre_ecdh_compute_shared_secret, .max_size = hpre_ecdh_max_size, .init = hpre_ecdh_nist_p256_init_tfm, .exit = hpre_ecdh_exit_tfm, @@ -1727,11 +1753,12 @@ static struct kpp_alg ecdh_curves[] = { .cra_name = "ecdh-nist-p256", .cra_driver_name = "hpre-ecdh-nist-p256", .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, }, }, { .set_secret = hpre_ecdh_set_secret, - .generate_public_key = hpre_ecdh_compute_value, - .compute_shared_secret = hpre_ecdh_compute_value, + .generate_public_key = hpre_ecdh_generate_public_key, + .compute_shared_secret = hpre_ecdh_compute_shared_secret, .max_size = hpre_ecdh_max_size, .init = hpre_ecdh_nist_p384_init_tfm, .exit = hpre_ecdh_exit_tfm, @@ -1741,6 +1768,7 @@ static struct kpp_alg ecdh_curves[] = { .cra_name = "ecdh-nist-p384", .cra_driver_name = "hpre-ecdh-nist-p384", .cra_module = THIS_MODULE, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, }, } }; diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index b94fecd765eeb..884d5d0afaf41 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -465,7 +465,7 @@ struct hisi_qp *hpre_create_qp(u8 type) * type: 0 - RSA/DH. algorithm supported in V2, * 1 - ECC algorithm in V3. */ - ret = hisi_qm_alloc_qps_node(&hpre_devices, 1, type, node, &qp); + ret = hisi_qm_alloc_qps_node(&hpre_devices, 1, &type, node, &qp); if (!ret) return qp; diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 0968304c0cb51..b92ee2fcb18aa 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -609,9 +609,13 @@ static void qm_mb_write(struct hisi_qm *qm, const void *src) } #if IS_ENABLED(CONFIG_ARM64) + /* + * The dmb oshst instruction ensures that the data in the + * mailbox is written before it is sent to the hardware. + */ asm volatile("ldp %0, %1, %3\n" - "stp %0, %1, %2\n" "dmb oshst\n" + "stp %0, %1, %2\n" : "=&r" (tmp0), "=&r" (tmp1), "+Q" (*((char __iomem *)fun_base)) @@ -2210,6 +2214,7 @@ static void qp_stop_fail_cb(struct hisi_qp *qp) for (i = 0; i < qp_used; i++) { pos = (i + cur_head) % sq_depth; qp->req_cb(qp, qp->sqe + (u32)(qm->sqe_size * pos)); + qm_cq_head_update(qp); atomic_dec(&qp->qp_status.used); } } @@ -2359,25 +2364,33 @@ EXPORT_SYMBOL_GPL(hisi_qm_stop_qp); int hisi_qp_send(struct hisi_qp *qp, const void *msg) { struct hisi_qp_status *qp_status = &qp->qp_status; - u16 sq_tail = qp_status->sq_tail; - u16 sq_tail_next = (sq_tail + 1) % qp->sq_depth; - void *sqe = qm_get_avail_sqe(qp); + u16 sq_tail, sq_tail_next; + void *sqe; + spin_lock_bh(&qp->qp_lock); if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP || atomic_read(&qp->qm->status.flags) == QM_STOP || qp->is_resetting)) { + spin_unlock_bh(&qp->qp_lock); dev_info_ratelimited(&qp->qm->pdev->dev, "QP is stopped or resetting\n"); return -EAGAIN; } - if (!sqe) + sqe = qm_get_avail_sqe(qp); + if (!sqe) { + spin_unlock_bh(&qp->qp_lock); return -EBUSY; + } + sq_tail = qp_status->sq_tail; + sq_tail_next = (sq_tail + 1) % qp->sq_depth; memcpy(sqe, msg, qp->qm->sqe_size); + qp->msg[sq_tail] = msg; qm_db(qp->qm, qp->qp_id, QM_DOORBELL_CMD_SQ, sq_tail_next, 0); atomic_inc(&qp->qp_status.used); qp_status->sq_tail = sq_tail_next; + spin_unlock_bh(&qp->qp_lock); return 0; } @@ -2907,12 +2920,13 @@ EXPORT_SYMBOL_GPL(hisi_qm_wait_task_finish); static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num) { struct device *dev = &qm->pdev->dev; - struct qm_dma *qdma; + struct hisi_qp *qp; int i; for (i = num - 1; i >= 0; i--) { - qdma = &qm->qp_array[i].qdma; - dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma); + qp = &qm->qp_array[i]; + dma_free_coherent(dev, qp->qdma.size, qp->qdma.va, qp->qdma.dma); + kfree(qp->msg); kfree(qm->poll_data[i].qp_finish_id); } @@ -2934,10 +2948,14 @@ static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id, return -ENOMEM; qp = &qm->qp_array[id]; + qp->msg = kmalloc_array(sq_depth, sizeof(void *), GFP_KERNEL); + if (!qp->msg) + goto err_free_qp_finish_id; + qp->qdma.va = dma_alloc_coherent(dev, dma_size, &qp->qdma.dma, GFP_KERNEL); if (!qp->qdma.va) - goto err_free_qp_finish_id; + goto err_free_qp_msg; qp->sqe = qp->qdma.va; qp->sqe_dma = qp->qdma.dma; @@ -2949,8 +2967,14 @@ static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id, qp->qm = qm; qp->qp_id = id; + spin_lock_init(&qp->qp_lock); + spin_lock_init(&qp->backlog.lock); + INIT_LIST_HEAD(&qp->backlog.list); + return 0; +err_free_qp_msg: + kfree(qp->msg); err_free_qp_finish_id: kfree(qm->poll_data[id].qp_finish_id); return ret; @@ -3496,6 +3520,14 @@ void hisi_qm_dev_err_uninit(struct hisi_qm *qm) } EXPORT_SYMBOL_GPL(hisi_qm_dev_err_uninit); +static void qm_release_qp_nolock(struct hisi_qp *qp) +{ + struct hisi_qm *qm = qp->qm; + + qm->qp_in_used--; + idr_remove(&qm->qp_idr, qp->qp_id); +} + /** * hisi_qm_free_qps() - free multiple queue pairs. * @qps: The queue pairs need to be freed. @@ -3508,8 +3540,15 @@ void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num) if (!qps || qp_num <= 0) return; - for (i = qp_num - 1; i >= 0; i--) - hisi_qm_release_qp(qps[i]); + down_write(&qps[0]->qm->qps_lock); + + for (i = qp_num - 1; i >= 0; i--) { + qm_stop_qp_nolock(qps[i]); + qm_release_qp_nolock(qps[i]); + } + + up_write(&qps[0]->qm->qps_lock); + qm_pm_put_sync(qps[0]->qm); } EXPORT_SYMBOL_GPL(hisi_qm_free_qps); @@ -3523,6 +3562,43 @@ static void free_list(struct list_head *head) } } +static int qm_get_and_start_qp(struct hisi_qm *qm, int qp_num, struct hisi_qp **qps, u8 *alg_type) +{ + int i, ret; + + ret = qm_pm_get_sync(qm); + if (ret) + return ret; + + down_write(&qm->qps_lock); + for (i = 0; i < qp_num; i++) { + qps[i] = qm_create_qp_nolock(qm, alg_type[i]); + if (IS_ERR(qps[i])) { + ret = -ENODEV; + goto stop_and_free; + } + + ret = qm_start_qp_nolock(qps[i], 0); + if (ret) { + qm_release_qp_nolock(qps[i]); + goto stop_and_free; + } + } + up_write(&qm->qps_lock); + + return 0; + +stop_and_free: + for (i--; i >= 0; i--) { + qm_stop_qp_nolock(qps[i]); + qm_release_qp_nolock(qps[i]); + } + up_write(&qm->qps_lock); + qm_pm_put_sync(qm); + + return ret; +} + static int hisi_qm_sort_devices(int node, struct list_head *head, struct hisi_qm_list *qm_list) { @@ -3571,12 +3647,11 @@ static int hisi_qm_sort_devices(int node, struct list_head *head, * not meet the requirements will return error. */ int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, - u8 alg_type, int node, struct hisi_qp **qps) + u8 *alg_type, int node, struct hisi_qp **qps) { struct hisi_qm_resource *tmp; int ret = -ENODEV; LIST_HEAD(head); - int i; if (!qps || !qm_list || qp_num <= 0) return -EINVAL; @@ -3588,24 +3663,15 @@ int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, } list_for_each_entry(tmp, &head, list) { - for (i = 0; i < qp_num; i++) { - qps[i] = hisi_qm_create_qp(tmp->qm, alg_type); - if (IS_ERR(qps[i])) { - hisi_qm_free_qps(qps, i); - break; - } - } - - if (i == qp_num) { - ret = 0; + ret = qm_get_and_start_qp(tmp->qm, qp_num, qps, alg_type); + if (!ret) break; - } } mutex_unlock(&qm_list->lock); if (ret) - pr_info("Failed to create qps, node[%d], alg[%u], qp[%d]!\n", - node, alg_type, qp_num); + pr_info("Failed to create qps, node[%d], qp[%d]!\n", + node, qp_num); err: free_list(&head); diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index 81d0beda93b2b..0710977861f32 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -82,11 +82,6 @@ struct sec_aead_req { __u8 out_mac_buf[SEC_MAX_MAC_LEN]; }; -struct sec_instance_backlog { - struct list_head list; - spinlock_t lock; -}; - /* SEC request of Crypto */ struct sec_req { union { @@ -112,7 +107,6 @@ struct sec_req { bool use_pbuf; struct list_head list; - struct sec_instance_backlog *backlog; struct sec_request_buf buf; }; @@ -172,7 +166,6 @@ struct sec_qp_ctx { spinlock_t id_lock; struct hisi_acc_sgl_pool *c_in_pool; struct hisi_acc_sgl_pool *c_out_pool; - struct sec_instance_backlog backlog; u16 send_head; }; diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 31590d01139a3..c462b58d30343 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -54,7 +54,6 @@ #define SEC_AUTH_CIPHER_V3 0x40 #define SEC_FLAG_OFFSET 7 #define SEC_FLAG_MASK 0x0780 -#define SEC_TYPE_MASK 0x0F #define SEC_DONE_MASK 0x0001 #define SEC_ICV_MASK 0x000E @@ -148,7 +147,7 @@ static void sec_free_req_id(struct sec_req *req) spin_unlock_bh(&qp_ctx->id_lock); } -static u8 pre_parse_finished_bd(struct bd_status *status, void *resp) +static void pre_parse_finished_bd(struct bd_status *status, void *resp) { struct sec_sqe *bd = resp; @@ -158,11 +157,9 @@ static u8 pre_parse_finished_bd(struct bd_status *status, void *resp) SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; status->tag = le16_to_cpu(bd->type2.tag); status->err_type = bd->type2.error_type; - - return bd->type_cipher_auth & SEC_TYPE_MASK; } -static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp) +static void pre_parse_finished_bd3(struct bd_status *status, void *resp) { struct sec_sqe3 *bd3 = resp; @@ -172,8 +169,6 @@ static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp) SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; status->tag = le64_to_cpu(bd3->tag); status->err_type = bd3->error_type; - - return le32_to_cpu(bd3->bd_param) & SEC_TYPE_MASK; } static int sec_cb_status_check(struct sec_req *req, @@ -244,7 +239,7 @@ static void sec_alg_send_backlog_soft(struct sec_ctx *ctx, struct sec_qp_ctx *qp struct sec_req *req, *tmp; int ret; - list_for_each_entry_safe(req, tmp, &qp_ctx->backlog.list, list) { + list_for_each_entry_safe(req, tmp, &qp_ctx->qp->backlog.list, list) { list_del(&req->list); ctx->req_op->buf_unmap(ctx, req); if (req->req_id >= 0) @@ -265,11 +260,12 @@ static void sec_alg_send_backlog_soft(struct sec_ctx *ctx, struct sec_qp_ctx *qp static void sec_alg_send_backlog(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) { + struct hisi_qp *qp = qp_ctx->qp; struct sec_req *req, *tmp; int ret; - spin_lock_bh(&qp_ctx->backlog.lock); - list_for_each_entry_safe(req, tmp, &qp_ctx->backlog.list, list) { + spin_lock_bh(&qp->backlog.lock); + list_for_each_entry_safe(req, tmp, &qp->backlog.list, list) { ret = qp_send_message(req); switch (ret) { case -EINPROGRESS: @@ -287,42 +283,46 @@ static void sec_alg_send_backlog(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) } unlock: - spin_unlock_bh(&qp_ctx->backlog.lock); + spin_unlock_bh(&qp->backlog.lock); } static void sec_req_cb(struct hisi_qp *qp, void *resp) { - struct sec_qp_ctx *qp_ctx = qp->qp_ctx; - struct sec_dfx *dfx = &qp_ctx->ctx->sec->debug.dfx; - u8 type_supported = qp_ctx->ctx->type_supported; + const struct sec_sqe *sqe = qp->msg[qp->qp_status.cq_head]; + struct sec_req *req = container_of(sqe, struct sec_req, sec_sqe); + struct sec_ctx *ctx = req->ctx; + struct sec_dfx *dfx = &ctx->sec->debug.dfx; struct bd_status status; - struct sec_ctx *ctx; - struct sec_req *req; int err; - u8 type; - if (type_supported == SEC_BD_TYPE2) { - type = pre_parse_finished_bd(&status, resp); - req = qp_ctx->req_list[status.tag]; - } else { - type = pre_parse_finished_bd3(&status, resp); - req = (void *)(uintptr_t)status.tag; - } + pre_parse_finished_bd(&status, resp); - if (unlikely(type != type_supported)) { - atomic64_inc(&dfx->err_bd_cnt); - pr_err("err bd type [%u]\n", type); - return; - } + req->err_type = status.err_type; + err = sec_cb_status_check(req, &status); + if (err) + atomic64_inc(&dfx->done_flag_cnt); - if (unlikely(!req)) { - atomic64_inc(&dfx->invalid_req_cnt); - atomic_inc(&qp->qp_status.used); - return; - } + atomic64_inc(&dfx->recv_cnt); + ctx->req_op->buf_unmap(ctx, req); + ctx->req_op->callback(ctx, req, err); +} + +static void sec_req_cb3(struct hisi_qp *qp, void *resp) +{ + struct bd_status status; + struct sec_ctx *ctx; + struct sec_dfx *dfx; + struct sec_req *req; + int err; + + pre_parse_finished_bd3(&status, resp); + + req = (void *)(uintptr_t)status.tag; req->err_type = status.err_type; ctx = req->ctx; + dfx = &ctx->sec->debug.dfx; + err = sec_cb_status_check(req, &status); if (err) atomic64_inc(&dfx->done_flag_cnt); @@ -330,7 +330,6 @@ static void sec_req_cb(struct hisi_qp *qp, void *resp) atomic64_inc(&dfx->recv_cnt); ctx->req_op->buf_unmap(ctx, req); - ctx->req_op->callback(ctx, req, err); } @@ -348,8 +347,10 @@ static int sec_alg_send_message_retry(struct sec_req *req) static int sec_alg_try_enqueue(struct sec_req *req) { + struct hisi_qp *qp = req->qp_ctx->qp; + /* Check if any request is already backlogged */ - if (!list_empty(&req->backlog->list)) + if (!list_empty(&qp->backlog.list)) return -EBUSY; /* Try to enqueue to HW ring */ @@ -359,17 +360,18 @@ static int sec_alg_try_enqueue(struct sec_req *req) static int sec_alg_send_message_maybacklog(struct sec_req *req) { + struct hisi_qp *qp = req->qp_ctx->qp; int ret; ret = sec_alg_try_enqueue(req); if (ret != -EBUSY) return ret; - spin_lock_bh(&req->backlog->lock); + spin_lock_bh(&qp->backlog.lock); ret = sec_alg_try_enqueue(req); if (ret == -EBUSY) - list_add_tail(&req->list, &req->backlog->list); - spin_unlock_bh(&req->backlog->lock); + list_add_tail(&req->list, &qp->backlog.list); + spin_unlock_bh(&qp->backlog.lock); return ret; } @@ -624,32 +626,25 @@ static int sec_create_qp_ctx(struct sec_ctx *ctx, int qp_ctx_id) qp_ctx = &ctx->qp_ctx[qp_ctx_id]; qp = ctx->qps[qp_ctx_id]; - qp->req_type = 0; - qp->qp_ctx = qp_ctx; qp_ctx->qp = qp; qp_ctx->ctx = ctx; - qp->req_cb = sec_req_cb; + if (ctx->type_supported == SEC_BD_TYPE3) + qp->req_cb = sec_req_cb3; + else + qp->req_cb = sec_req_cb; spin_lock_init(&qp_ctx->req_lock); idr_init(&qp_ctx->req_idr); - spin_lock_init(&qp_ctx->backlog.lock); spin_lock_init(&qp_ctx->id_lock); - INIT_LIST_HEAD(&qp_ctx->backlog.list); qp_ctx->send_head = 0; ret = sec_alloc_qp_ctx_resource(ctx, qp_ctx); if (ret) goto err_destroy_idr; - ret = hisi_qm_start_qp(qp, 0); - if (ret < 0) - goto err_resource_free; - return 0; -err_resource_free: - sec_free_qp_ctx_resource(ctx, qp_ctx); err_destroy_idr: idr_destroy(&qp_ctx->req_idr); return ret; @@ -658,7 +653,6 @@ static int sec_create_qp_ctx(struct sec_ctx *ctx, int qp_ctx_id) static void sec_release_qp_ctx(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) { - hisi_qm_stop_qp(qp_ctx->qp); sec_free_qp_ctx_resource(ctx, qp_ctx); idr_destroy(&qp_ctx->req_idr); } @@ -669,10 +663,8 @@ static int sec_ctx_base_init(struct sec_ctx *ctx) int i, ret; ctx->qps = sec_create_qps(); - if (!ctx->qps) { - pr_err("Can not create sec qps!\n"); + if (!ctx->qps) return -ENODEV; - } sec = container_of(ctx->qps[0]->qm, struct sec_dev, qm); ctx->sec = sec; @@ -708,6 +700,9 @@ static void sec_ctx_base_uninit(struct sec_ctx *ctx) { int i; + if (!ctx->qps) + return; + for (i = 0; i < ctx->sec->ctx_q_num; i++) sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); @@ -719,6 +714,9 @@ static int sec_cipher_init(struct sec_ctx *ctx) { struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + if (!ctx->qps) + return 0; + c_ctx->c_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE, &c_ctx->c_key_dma, GFP_KERNEL); if (!c_ctx->c_key) @@ -731,6 +729,9 @@ static void sec_cipher_uninit(struct sec_ctx *ctx) { struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + if (!ctx->qps) + return; + memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE); dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE, c_ctx->c_key, c_ctx->c_key_dma); @@ -752,6 +753,9 @@ static void sec_auth_uninit(struct sec_ctx *ctx) { struct sec_auth_ctx *a_ctx = &ctx->a_ctx; + if (!ctx->qps) + return; + memzero_explicit(a_ctx->a_key, SEC_MAX_AKEY_SIZE); dma_free_coherent(ctx->dev, SEC_MAX_AKEY_SIZE, a_ctx->a_key, a_ctx->a_key_dma); @@ -789,7 +793,7 @@ static int sec_skcipher_init(struct crypto_skcipher *tfm) } ret = sec_ctx_base_init(ctx); - if (ret) + if (ret && ret != -ENODEV) return ret; ret = sec_cipher_init(ctx); @@ -898,6 +902,9 @@ static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, struct device *dev = ctx->dev; int ret; + if (!ctx->qps) + goto set_soft_key; + if (c_mode == SEC_CMODE_XTS) { ret = xts_verify_key(tfm, key, keylen); if (ret) { @@ -928,13 +935,14 @@ static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, } memcpy(c_ctx->c_key, key, keylen); - if (c_ctx->fbtfm) { - ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen); - if (ret) { - dev_err(dev, "failed to set fallback skcipher key!\n"); - return ret; - } + +set_soft_key: + ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen); + if (ret) { + dev_err(dev, "failed to set fallback skcipher key!\n"); + return ret; } + return 0; } @@ -1398,6 +1406,9 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, struct crypto_authenc_keys keys; int ret; + if (!ctx->qps) + return sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); + ctx->a_ctx.a_alg = a_alg; ctx->c_ctx.c_alg = c_alg; c_ctx->c_mode = c_mode; @@ -1952,7 +1963,6 @@ static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req) } while (req->req_id < 0 && ++i < ctx->sec->ctx_q_num); req->qp_ctx = qp_ctx; - req->backlog = &qp_ctx->backlog; return 0; } @@ -2055,6 +2065,9 @@ static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm) if (ret) return ret; + if (!ctx->qps) + return 0; + if (ctx->sec->qm.ver < QM_HW_V3) { ctx->type_supported = SEC_BD_TYPE2; ctx->req_op = &sec_skcipher_req_ops; @@ -2063,7 +2076,7 @@ static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm) ctx->req_op = &sec_skcipher_req_ops_v3; } - return ret; + return 0; } static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm) @@ -2131,7 +2144,7 @@ static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name) int ret; ret = sec_aead_init(tfm); - if (ret) { + if (ret && ret != -ENODEV) { pr_err("hisi_sec2: aead init error!\n"); return ret; } @@ -2173,7 +2186,7 @@ static int sec_aead_xcm_ctx_init(struct crypto_aead *tfm) int ret; ret = sec_aead_init(tfm); - if (ret) { + if (ret && ret != -ENODEV) { dev_err(ctx->dev, "hisi_sec2: aead xcm init error!\n"); return ret; } @@ -2318,6 +2331,9 @@ static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) bool need_fallback = false; int ret; + if (!ctx->qps) + goto soft_crypto; + if (!sk_req->cryptlen) { if (ctx->c_ctx.c_mode == SEC_CMODE_XTS) return -EINVAL; @@ -2335,9 +2351,12 @@ static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) return -EINVAL; if (unlikely(ctx->c_ctx.fallback || need_fallback)) - return sec_skcipher_soft_crypto(ctx, sk_req, encrypt); + goto soft_crypto; return ctx->req_op->process(ctx, req); + +soft_crypto: + return sec_skcipher_soft_crypto(ctx, sk_req, encrypt); } static int sec_skcipher_encrypt(struct skcipher_request *sk_req) @@ -2545,6 +2564,9 @@ static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) bool need_fallback = false; int ret; + if (!ctx->qps) + goto soft_crypto; + req->flag = a_req->base.flags; req->aead_req.aead_req = a_req; req->c_req.encrypt = encrypt; @@ -2555,11 +2577,14 @@ static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) ret = sec_aead_param_check(ctx, req, &need_fallback); if (unlikely(ret)) { if (need_fallback) - return sec_aead_soft_crypto(ctx, a_req, encrypt); + goto soft_crypto; return -EINVAL; } return ctx->req_op->process(ctx, req); + +soft_crypto: + return sec_aead_soft_crypto(ctx, a_req, encrypt); } static int sec_aead_encrypt(struct aead_request *a_req) diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 5eb2d68207426..7dd125f5f511f 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -417,18 +417,29 @@ struct hisi_qp **sec_create_qps(void) int node = cpu_to_node(raw_smp_processor_id()); u32 ctx_num = ctx_q_num; struct hisi_qp **qps; + u8 *type; int ret; qps = kcalloc(ctx_num, sizeof(struct hisi_qp *), GFP_KERNEL); if (!qps) return NULL; - ret = hisi_qm_alloc_qps_node(&sec_devices, ctx_num, 0, node, qps); - if (!ret) - return qps; + /* The type of SEC is all 0, so just allocated by kcalloc */ + type = kcalloc(ctx_num, sizeof(u8), GFP_KERNEL); + if (!type) { + kfree(qps); + return NULL; + } - kfree(qps); - return NULL; + ret = hisi_qm_alloc_qps_node(&sec_devices, ctx_num, type, node, qps); + if (ret) { + kfree(type); + kfree(qps); + return NULL; + } + + kfree(type); + return qps; } u64 sec_get_alg_bitmap(struct hisi_qm *qm, u32 high, u32 low) diff --git a/drivers/crypto/hisilicon/sgl.c b/drivers/crypto/hisilicon/sgl.c index 7a9ef2a9972a2..848ad7b101d93 100644 --- a/drivers/crypto/hisilicon/sgl.c +++ b/drivers/crypto/hisilicon/sgl.c @@ -265,7 +265,7 @@ hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev, struct scatterlist *sgl, return curr_hw_sgl; err_unmap: - dma_unmap_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL); + dma_unmap_sg(dev, sgl, sg_n, dir); return ERR_PTR(ret); } diff --git a/drivers/crypto/hisilicon/trng/trng.c b/drivers/crypto/hisilicon/trng/trng.c index ac74df4a94712..5ca0b90859a81 100644 --- a/drivers/crypto/hisilicon/trng/trng.c +++ b/drivers/crypto/hisilicon/trng/trng.c @@ -40,6 +40,7 @@ #define SEED_SHIFT_24 24 #define SEED_SHIFT_16 16 #define SEED_SHIFT_8 8 +#define SW_MAX_RANDOM_BYTES 65520 struct hisi_trng_list { struct mutex lock; @@ -53,8 +54,10 @@ struct hisi_trng { struct list_head list; struct hwrng rng; u32 ver; - bool is_used; - struct mutex mutex; + u32 ctx_num; + /* The bytes of the random number generated since the last seeding. */ + u32 random_bytes; + struct mutex lock; }; struct hisi_trng_ctx { @@ -63,10 +66,14 @@ struct hisi_trng_ctx { static atomic_t trng_active_devs; static struct hisi_trng_list trng_devices; +static int hisi_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait); -static void hisi_trng_set_seed(struct hisi_trng *trng, const u8 *seed) +static int hisi_trng_set_seed(struct hisi_trng *trng, const u8 *seed) { u32 val, seed_reg, i; + int ret; + + writel(0x0, trng->base + SW_DRBG_BLOCKS); for (i = 0; i < SW_DRBG_SEED_SIZE; i += SW_DRBG_SEED_SIZE / SW_DRBG_SEED_REGS_NUM) { @@ -78,6 +85,20 @@ static void hisi_trng_set_seed(struct hisi_trng *trng, const u8 *seed) seed_reg = (i >> SW_DRBG_NUM_SHIFT) % SW_DRBG_SEED_REGS_NUM; writel(val, trng->base + SW_DRBG_SEED(seed_reg)); } + + writel(SW_DRBG_BLOCKS_NUM | (0x1 << SW_DRBG_ENABLE_SHIFT), + trng->base + SW_DRBG_BLOCKS); + writel(0x1, trng->base + SW_DRBG_INIT); + ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, + val, val & BIT(0), SLEEP_US, TIMEOUT_US); + if (ret) { + pr_err("failed to init trng(%d)\n", ret); + return -EIO; + } + + trng->random_bytes = 0; + + return 0; } static int hisi_trng_seed(struct crypto_rng *tfm, const u8 *seed, @@ -85,8 +106,7 @@ static int hisi_trng_seed(struct crypto_rng *tfm, const u8 *seed, { struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); struct hisi_trng *trng = ctx->trng; - u32 val = 0; - int ret = 0; + int ret; if (slen < SW_DRBG_SEED_SIZE) { pr_err("slen(%u) is not matched with trng(%d)\n", slen, @@ -94,43 +114,45 @@ static int hisi_trng_seed(struct crypto_rng *tfm, const u8 *seed, return -EINVAL; } - writel(0x0, trng->base + SW_DRBG_BLOCKS); - hisi_trng_set_seed(trng, seed); + mutex_lock(&trng->lock); + ret = hisi_trng_set_seed(trng, seed); + mutex_unlock(&trng->lock); - writel(SW_DRBG_BLOCKS_NUM | (0x1 << SW_DRBG_ENABLE_SHIFT), - trng->base + SW_DRBG_BLOCKS); - writel(0x1, trng->base + SW_DRBG_INIT); + return ret; +} - ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, - val, val & BIT(0), SLEEP_US, TIMEOUT_US); - if (ret) - pr_err("fail to init trng(%d)\n", ret); +static int hisi_trng_reseed(struct hisi_trng *trng) +{ + u8 seed[SW_DRBG_SEED_SIZE]; + int size; - return ret; + if (!trng->random_bytes) + return 0; + + size = hisi_trng_read(&trng->rng, seed, SW_DRBG_SEED_SIZE, false); + if (size != SW_DRBG_SEED_SIZE) + return -EIO; + + return hisi_trng_set_seed(trng, seed); } -static int hisi_trng_generate(struct crypto_rng *tfm, const u8 *src, - unsigned int slen, u8 *dstn, unsigned int dlen) +static int hisi_trng_get_bytes(struct hisi_trng *trng, u8 *dstn, unsigned int dlen) { - struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); - struct hisi_trng *trng = ctx->trng; u32 data[SW_DRBG_DATA_NUM]; u32 currsize = 0; u32 val = 0; int ret; u32 i; - if (dlen > SW_DRBG_BLOCKS_NUM * SW_DRBG_BYTES || dlen == 0) { - pr_err("dlen(%u) exceeds limit(%d)!\n", dlen, - SW_DRBG_BLOCKS_NUM * SW_DRBG_BYTES); - return -EINVAL; - } + ret = hisi_trng_reseed(trng); + if (ret) + return ret; do { ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, - val, val & BIT(1), SLEEP_US, TIMEOUT_US); + val, val & BIT(1), SLEEP_US, TIMEOUT_US); if (ret) { - pr_err("fail to generate random number(%d)!\n", ret); + pr_err("failed to generate random number(%d)!\n", ret); break; } @@ -145,30 +167,57 @@ static int hisi_trng_generate(struct crypto_rng *tfm, const u8 *src, currsize = dlen; } + trng->random_bytes += SW_DRBG_BYTES; writel(0x1, trng->base + SW_DRBG_GEN); } while (currsize < dlen); return ret; } +static int hisi_trng_generate(struct crypto_rng *tfm, const u8 *src, + unsigned int slen, u8 *dstn, unsigned int dlen) +{ + struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); + struct hisi_trng *trng = ctx->trng; + unsigned int currsize = 0; + unsigned int block_size; + int ret; + + if (!dstn || !dlen) { + pr_err("output is error, dlen %u!\n", dlen); + return -EINVAL; + } + + do { + block_size = min_t(unsigned int, dlen - currsize, SW_MAX_RANDOM_BYTES); + mutex_lock(&trng->lock); + ret = hisi_trng_get_bytes(trng, dstn + currsize, block_size); + mutex_unlock(&trng->lock); + if (ret) + return ret; + currsize += block_size; + } while (currsize < dlen); + + return 0; +} + static int hisi_trng_init(struct crypto_tfm *tfm) { struct hisi_trng_ctx *ctx = crypto_tfm_ctx(tfm); struct hisi_trng *trng; - int ret = -EBUSY; + u32 ctx_num = ~0; mutex_lock(&trng_devices.lock); list_for_each_entry(trng, &trng_devices.list, list) { - if (!trng->is_used) { - trng->is_used = true; + if (trng->ctx_num < ctx_num) { + ctx_num = trng->ctx_num; ctx->trng = trng; - ret = 0; - break; } } + ctx->trng->ctx_num++; mutex_unlock(&trng_devices.lock); - return ret; + return 0; } static void hisi_trng_exit(struct crypto_tfm *tfm) @@ -176,7 +225,7 @@ static void hisi_trng_exit(struct crypto_tfm *tfm) struct hisi_trng_ctx *ctx = crypto_tfm_ctx(tfm); mutex_lock(&trng_devices.lock); - ctx->trng->is_used = false; + ctx->trng->ctx_num--; mutex_unlock(&trng_devices.lock); } @@ -238,7 +287,7 @@ static int hisi_trng_del_from_list(struct hisi_trng *trng) int ret = -EBUSY; mutex_lock(&trng_devices.lock); - if (!trng->is_used) { + if (!trng->ctx_num) { list_del(&trng->list); ret = 0; } @@ -262,7 +311,9 @@ static int hisi_trng_probe(struct platform_device *pdev) if (IS_ERR(trng->base)) return PTR_ERR(trng->base); - trng->is_used = false; + trng->ctx_num = 0; + trng->random_bytes = SW_MAX_RANDOM_BYTES; + mutex_init(&trng->lock); trng->ver = readl(trng->base + HISI_TRNG_VERSION); if (!trng_devices.is_init) { INIT_LIST_HEAD(&trng_devices.list); diff --git a/drivers/crypto/hisilicon/zip/zip.h b/drivers/crypto/hisilicon/zip/zip.h index 9fb2a9c01132b..b83f228281ab1 100644 --- a/drivers/crypto/hisilicon/zip/zip.h +++ b/drivers/crypto/hisilicon/zip/zip.h @@ -99,7 +99,7 @@ enum zip_cap_table_type { ZIP_CORE5_BITMAP, }; -int zip_create_qps(struct hisi_qp **qps, int qp_num, int node); +int zip_create_qps(struct hisi_qp **qps, int qp_num, int node, u8 *alg_type); int hisi_zip_register_to_crypto(struct hisi_qm *qm); void hisi_zip_unregister_from_crypto(struct hisi_qm *qm); bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg); diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c index b97513981a3b7..e140d4f8afe0e 100644 --- a/drivers/crypto/hisilicon/zip/zip_crypto.c +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c @@ -39,6 +39,7 @@ enum { HZIP_CTX_Q_NUM }; +#define GET_REQ_FROM_SQE(sqe) ((u64)(sqe)->dw26 | (u64)(sqe)->dw27 << 32) #define COMP_NAME_TO_TYPE(alg_name) \ (!strcmp((alg_name), "deflate") ? HZIP_ALG_TYPE_DEFLATE : 0) @@ -48,6 +49,7 @@ struct hisi_zip_req { struct hisi_acc_hw_sgl *hw_dst; dma_addr_t dma_src; dma_addr_t dma_dst; + struct hisi_zip_qp_ctx *qp_ctx; u16 req_id; }; @@ -64,6 +66,7 @@ struct hisi_zip_qp_ctx { struct hisi_acc_sgl_pool *sgl_pool; struct hisi_zip *zip_dev; struct hisi_zip_ctx *ctx; + u8 req_type; }; struct hisi_zip_sqe_ops { @@ -74,7 +77,6 @@ struct hisi_zip_sqe_ops { void (*fill_req_type)(struct hisi_zip_sqe *sqe, u8 req_type); void (*fill_tag)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); void (*fill_sqe_type)(struct hisi_zip_sqe *sqe, u8 sqe_type); - u32 (*get_tag)(struct hisi_zip_sqe *sqe); u32 (*get_status)(struct hisi_zip_sqe *sqe); u32 (*get_dstlen)(struct hisi_zip_sqe *sqe); }; @@ -82,6 +84,7 @@ struct hisi_zip_sqe_ops { struct hisi_zip_ctx { struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM]; const struct hisi_zip_sqe_ops *ops; + bool fallback; }; static int sgl_sge_nr_set(const char *val, const struct kernel_param *kp) @@ -108,6 +111,24 @@ static u16 sgl_sge_nr = HZIP_SGL_SGE_NR; module_param_cb(sgl_sge_nr, &sgl_sge_nr_ops, &sgl_sge_nr, 0444); MODULE_PARM_DESC(sgl_sge_nr, "Number of sge in sgl(1-255)"); +static int hisi_zip_fallback_do_work(struct acomp_req *acomp_req, bool is_decompress) +{ + ACOMP_FBREQ_ON_STACK(fbreq, acomp_req); + int ret; + + if (!is_decompress) + ret = crypto_acomp_compress(fbreq); + else + ret = crypto_acomp_decompress(fbreq); + if (ret) { + pr_err("failed to do fallback work, ret=%d\n", ret); + return ret; + } + + acomp_req->dlen = fbreq->dlen; + return ret; +} + static struct hisi_zip_req *hisi_zip_create_req(struct hisi_zip_qp_ctx *qp_ctx, struct acomp_req *req) { @@ -131,6 +152,7 @@ static struct hisi_zip_req *hisi_zip_create_req(struct hisi_zip_qp_ctx *qp_ctx, req_cache = q + req_id; req_cache->req_id = req_id; req_cache->req = req; + req_cache->qp_ctx = qp_ctx; return req_cache; } @@ -181,7 +203,8 @@ static void hisi_zip_fill_req_type(struct hisi_zip_sqe *sqe, u8 req_type) static void hisi_zip_fill_tag(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) { - sqe->dw26 = req->req_id; + sqe->dw26 = lower_32_bits((u64)req); + sqe->dw27 = upper_32_bits((u64)req); } static void hisi_zip_fill_sqe_type(struct hisi_zip_sqe *sqe, u8 sqe_type) @@ -213,7 +236,6 @@ static int hisi_zip_do_work(struct hisi_zip_qp_ctx *qp_ctx, { struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool; struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; - struct hisi_zip_req_q *req_q = &qp_ctx->req_q; struct acomp_req *a_req = req->req; struct hisi_qp *qp = qp_ctx->qp; struct device *dev = &qp->qm->pdev->dev; @@ -237,18 +259,16 @@ static int hisi_zip_do_work(struct hisi_zip_qp_ctx *qp_ctx, &req->dma_dst, DMA_FROM_DEVICE); if (IS_ERR(req->hw_dst)) { ret = PTR_ERR(req->hw_dst); - dev_err(dev, "failed to map the dst buffer to hw slg (%d)!\n", + dev_err(dev, "failed to map the dst buffer to hw sgl (%d)!\n", ret); goto err_unmap_input; } - hisi_zip_fill_sqe(qp_ctx->ctx, &zip_sqe, qp->req_type, req); + hisi_zip_fill_sqe(qp_ctx->ctx, &zip_sqe, qp_ctx->req_type, req); /* send command to start a task */ atomic64_inc(&dfx->send_cnt); - spin_lock_bh(&req_q->req_lock); ret = hisi_qp_send(qp, &zip_sqe); - spin_unlock_bh(&req_q->req_lock); if (unlikely(ret < 0)) { atomic64_inc(&dfx->send_busy_cnt); ret = -EAGAIN; @@ -265,11 +285,6 @@ static int hisi_zip_do_work(struct hisi_zip_qp_ctx *qp_ctx, return ret; } -static u32 hisi_zip_get_tag(struct hisi_zip_sqe *sqe) -{ - return sqe->dw26; -} - static u32 hisi_zip_get_status(struct hisi_zip_sqe *sqe) { return sqe->dw3 & HZIP_BD_STATUS_M; @@ -282,14 +297,12 @@ static u32 hisi_zip_get_dstlen(struct hisi_zip_sqe *sqe) static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) { - struct hisi_zip_qp_ctx *qp_ctx = qp->qp_ctx; + struct hisi_zip_sqe *sqe = data; + struct hisi_zip_req *req = (struct hisi_zip_req *)GET_REQ_FROM_SQE(sqe); + struct hisi_zip_qp_ctx *qp_ctx = req->qp_ctx; const struct hisi_zip_sqe_ops *ops = qp_ctx->ctx->ops; struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; - struct hisi_zip_req_q *req_q = &qp_ctx->req_q; struct device *dev = &qp->qm->pdev->dev; - struct hisi_zip_sqe *sqe = data; - u32 tag = ops->get_tag(sqe); - struct hisi_zip_req *req = req_q->q + tag; struct acomp_req *acomp_req = req->req; int err = 0; u32 status; @@ -319,10 +332,15 @@ static int hisi_zip_acompress(struct acomp_req *acomp_req) { struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP]; - struct device *dev = &qp_ctx->qp->qm->pdev->dev; struct hisi_zip_req *req; + struct device *dev; int ret; + if (ctx->fallback) + return hisi_zip_fallback_do_work(acomp_req, 0); + + dev = &qp_ctx->qp->qm->pdev->dev; + req = hisi_zip_create_req(qp_ctx, acomp_req); if (IS_ERR(req)) return PTR_ERR(req); @@ -340,10 +358,15 @@ static int hisi_zip_adecompress(struct acomp_req *acomp_req) { struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP]; - struct device *dev = &qp_ctx->qp->qm->pdev->dev; struct hisi_zip_req *req; + struct device *dev; int ret; + if (ctx->fallback) + return hisi_zip_fallback_do_work(acomp_req, 1); + + dev = &qp_ctx->qp->qm->pdev->dev; + req = hisi_zip_create_req(qp_ctx, acomp_req); if (IS_ERR(req)) return PTR_ERR(req); @@ -358,33 +381,6 @@ static int hisi_zip_adecompress(struct acomp_req *acomp_req) return ret; } -static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *qp_ctx, - int alg_type, int req_type) -{ - struct device *dev = &qp->qm->pdev->dev; - int ret; - - qp->req_type = req_type; - qp->alg_type = alg_type; - qp->qp_ctx = qp_ctx; - - ret = hisi_qm_start_qp(qp, 0); - if (ret < 0) { - dev_err(dev, "failed to start qp (%d)!\n", ret); - return ret; - } - - qp_ctx->qp = qp; - - return 0; -} - -static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *qp_ctx) -{ - hisi_qm_stop_qp(qp_ctx->qp); - hisi_qm_free_qps(&qp_ctx->qp, 1); -} - static const struct hisi_zip_sqe_ops hisi_zip_ops = { .sqe_type = 0x3, .fill_addr = hisi_zip_fill_addr, @@ -393,7 +389,6 @@ static const struct hisi_zip_sqe_ops hisi_zip_ops = { .fill_req_type = hisi_zip_fill_req_type, .fill_tag = hisi_zip_fill_tag, .fill_sqe_type = hisi_zip_fill_sqe_type, - .get_tag = hisi_zip_get_tag, .get_status = hisi_zip_get_status, .get_dstlen = hisi_zip_get_dstlen, }; @@ -402,10 +397,15 @@ static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int { struct hisi_qp *qps[HZIP_CTX_Q_NUM] = { NULL }; struct hisi_zip_qp_ctx *qp_ctx; + u8 alg_type[HZIP_CTX_Q_NUM]; struct hisi_zip *hisi_zip; - int ret, i, j; + int ret, i; - ret = zip_create_qps(qps, HZIP_CTX_Q_NUM, node); + /* alg_type = 0 for compress, 1 for decompress in hw sqe */ + for (i = 0; i < HZIP_CTX_Q_NUM; i++) + alg_type[i] = i; + + ret = zip_create_qps(qps, HZIP_CTX_Q_NUM, node, alg_type); if (ret) { pr_err("failed to create zip qps (%d)!\n", ret); return -ENODEV; @@ -414,19 +414,11 @@ static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int hisi_zip = container_of(qps[0]->qm, struct hisi_zip, qm); for (i = 0; i < HZIP_CTX_Q_NUM; i++) { - /* alg_type = 0 for compress, 1 for decompress in hw sqe */ qp_ctx = &hisi_zip_ctx->qp_ctx[i]; qp_ctx->ctx = hisi_zip_ctx; - ret = hisi_zip_start_qp(qps[i], qp_ctx, i, req_type); - if (ret) { - for (j = i - 1; j >= 0; j--) - hisi_qm_stop_qp(hisi_zip_ctx->qp_ctx[j].qp); - - hisi_qm_free_qps(qps, HZIP_CTX_Q_NUM); - return ret; - } - qp_ctx->zip_dev = hisi_zip; + qp_ctx->req_type = req_type; + qp_ctx->qp = qps[i]; } hisi_zip_ctx->ops = &hisi_zip_ops; @@ -436,10 +428,13 @@ static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx) { + struct hisi_qp *qps[HZIP_CTX_Q_NUM] = { NULL }; int i; for (i = 0; i < HZIP_CTX_Q_NUM; i++) - hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[i]); + qps[i] = hisi_zip_ctx->qp_ctx[i].qp; + + hisi_qm_free_qps(qps, HZIP_CTX_Q_NUM); } static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) @@ -549,7 +544,7 @@ static int hisi_zip_acomp_init(struct crypto_acomp *tfm) ret = hisi_zip_ctx_init(ctx, COMP_NAME_TO_TYPE(alg_name), tfm->base.node); if (ret) { pr_err("failed to init ctx (%d)!\n", ret); - return ret; + goto switch_to_soft; } dev = &ctx->qp_ctx[0].qp->qm->pdev->dev; @@ -574,17 +569,20 @@ static int hisi_zip_acomp_init(struct crypto_acomp *tfm) hisi_zip_release_req_q(ctx); err_ctx_exit: hisi_zip_ctx_exit(ctx); - return ret; +switch_to_soft: + ctx->fallback = true; + return 0; } static void hisi_zip_acomp_exit(struct crypto_acomp *tfm) { struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base); - hisi_zip_set_acomp_cb(ctx, NULL); - hisi_zip_release_sgl_pool(ctx); - hisi_zip_release_req_q(ctx); - hisi_zip_ctx_exit(ctx); + if (!ctx->fallback) { + hisi_zip_release_sgl_pool(ctx); + hisi_zip_release_req_q(ctx); + hisi_zip_ctx_exit(ctx); + } } static struct acomp_alg hisi_zip_acomp_deflate = { @@ -595,7 +593,8 @@ static struct acomp_alg hisi_zip_acomp_deflate = { .base = { .cra_name = "deflate", .cra_driver_name = "hisi-deflate-acomp", - .cra_flags = CRYPTO_ALG_ASYNC, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, .cra_module = THIS_MODULE, .cra_priority = HZIP_ALG_PRIORITY, .cra_ctxsize = sizeof(struct hisi_zip_ctx), diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 4fcbe6bada066..85b26ef175485 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -446,12 +446,12 @@ static const struct pci_device_id hisi_zip_dev_ids[] = { }; MODULE_DEVICE_TABLE(pci, hisi_zip_dev_ids); -int zip_create_qps(struct hisi_qp **qps, int qp_num, int node) +int zip_create_qps(struct hisi_qp **qps, int qp_num, int node, u8 *alg_type) { if (node == NUMA_NO_NODE) node = cpu_to_node(raw_smp_processor_id()); - return hisi_qm_alloc_qps_node(&zip_devices, qp_num, 0, node, qps); + return hisi_qm_alloc_qps_node(&zip_devices, qp_num, alg_type, node, qps); } bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg) diff --git a/drivers/crypto/inside-secure/eip93/eip93-main.c b/drivers/crypto/inside-secure/eip93/eip93-main.c index 0b38a567da0e0..b7fd9795062d4 100644 --- a/drivers/crypto/inside-secure/eip93/eip93-main.c +++ b/drivers/crypto/inside-secure/eip93/eip93-main.c @@ -77,11 +77,44 @@ inline void eip93_irq_clear(struct eip93_device *eip93, u32 mask) __raw_writel(mask, eip93->base + EIP93_REG_INT_CLR); } -static void eip93_unregister_algs(unsigned int i) +static int eip93_algo_is_supported(u32 alg_flags, u32 supported_algo_flags) +{ + if ((IS_DES(alg_flags) || IS_3DES(alg_flags)) && + !(supported_algo_flags & EIP93_PE_OPTION_TDES)) + return 0; + + if (IS_AES(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_AES)) + return 0; + + if (IS_HASH_MD5(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_MD5)) + return 0; + + if (IS_HASH_SHA1(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_SHA_1)) + return 0; + + if (IS_HASH_SHA224(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_SHA_224)) + return 0; + + if (IS_HASH_SHA256(alg_flags) && + !(supported_algo_flags & EIP93_PE_OPTION_SHA_256)) + return 0; + + return 1; +} + +static void eip93_unregister_algs(u32 supported_algo_flags, unsigned int i) { unsigned int j; for (j = 0; j < i; j++) { + if (!eip93_algo_is_supported(eip93_algs[j]->flags, + supported_algo_flags)) + continue; + switch (eip93_algs[j]->type) { case EIP93_ALG_TYPE_SKCIPHER: crypto_unregister_skcipher(&eip93_algs[j]->alg.skcipher); @@ -90,7 +123,7 @@ static void eip93_unregister_algs(unsigned int i) crypto_unregister_aead(&eip93_algs[j]->alg.aead); break; case EIP93_ALG_TYPE_HASH: - crypto_unregister_ahash(&eip93_algs[i]->alg.ahash); + crypto_unregister_ahash(&eip93_algs[j]->alg.ahash); break; } } @@ -106,49 +139,27 @@ static int eip93_register_algs(struct eip93_device *eip93, u32 supported_algo_fl eip93_algs[i]->eip93 = eip93; - if ((IS_DES(alg_flags) || IS_3DES(alg_flags)) && - !(supported_algo_flags & EIP93_PE_OPTION_TDES)) + if (!eip93_algo_is_supported(alg_flags, supported_algo_flags)) continue; - if (IS_AES(alg_flags)) { - if (!(supported_algo_flags & EIP93_PE_OPTION_AES)) - continue; + if (IS_AES(alg_flags) && !IS_HMAC(alg_flags)) { + if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY128) + eip93_algs[i]->alg.skcipher.max_keysize = + AES_KEYSIZE_128; - if (!IS_HMAC(alg_flags)) { - if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY128) - eip93_algs[i]->alg.skcipher.max_keysize = - AES_KEYSIZE_128; + if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY192) + eip93_algs[i]->alg.skcipher.max_keysize = + AES_KEYSIZE_192; - if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY192) - eip93_algs[i]->alg.skcipher.max_keysize = - AES_KEYSIZE_192; + if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY256) + eip93_algs[i]->alg.skcipher.max_keysize = + AES_KEYSIZE_256; - if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY256) - eip93_algs[i]->alg.skcipher.max_keysize = - AES_KEYSIZE_256; - - if (IS_RFC3686(alg_flags)) - eip93_algs[i]->alg.skcipher.max_keysize += - CTR_RFC3686_NONCE_SIZE; - } + if (IS_RFC3686(alg_flags)) + eip93_algs[i]->alg.skcipher.max_keysize += + CTR_RFC3686_NONCE_SIZE; } - if (IS_HASH_MD5(alg_flags) && - !(supported_algo_flags & EIP93_PE_OPTION_MD5)) - continue; - - if (IS_HASH_SHA1(alg_flags) && - !(supported_algo_flags & EIP93_PE_OPTION_SHA_1)) - continue; - - if (IS_HASH_SHA224(alg_flags) && - !(supported_algo_flags & EIP93_PE_OPTION_SHA_224)) - continue; - - if (IS_HASH_SHA256(alg_flags) && - !(supported_algo_flags & EIP93_PE_OPTION_SHA_256)) - continue; - switch (eip93_algs[i]->type) { case EIP93_ALG_TYPE_SKCIPHER: ret = crypto_register_skcipher(&eip93_algs[i]->alg.skcipher); @@ -167,7 +178,7 @@ static int eip93_register_algs(struct eip93_device *eip93, u32 supported_algo_fl return 0; fail: - eip93_unregister_algs(i); + eip93_unregister_algs(supported_algo_flags, i); return ret; } @@ -469,8 +480,11 @@ static int eip93_crypto_probe(struct platform_device *pdev) static void eip93_crypto_remove(struct platform_device *pdev) { struct eip93_device *eip93 = platform_get_drvdata(pdev); + u32 algo_flags; + + algo_flags = readl(eip93->base + EIP93_REG_PE_OPTION_1); - eip93_unregister_algs(ARRAY_SIZE(eip93_algs)); + eip93_unregister_algs(algo_flags, ARRAY_SIZE(eip93_algs)); eip93_cleanup(eip93); } diff --git a/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c index b9b5e744a3f16..af8dbc7517cf8 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c +++ b/drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c @@ -148,6 +148,16 @@ static struct pfvf_message handle_blkmsg_req(struct adf_accel_vf_info *vf_info, blk_byte = FIELD_GET(ADF_VF2PF_SMALL_BLOCK_BYTE_MASK, req.data); byte_max = ADF_VF2PF_SMALL_BLOCK_BYTE_MAX; break; + default: + dev_err(&GET_DEV(vf_info->accel_dev), + "Invalid BlockMsg type 0x%.4x received from VF%u\n", + req.type, vf_info->vf_nr); + resp.type = ADF_PF2VF_MSGTYPE_BLKMSG_RESP; + resp.data = FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_TYPE_MASK, + ADF_PF2VF_BLKMSG_RESP_TYPE_ERROR) | + FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_DATA_MASK, + ADF_PF2VF_UNSPECIFIED_ERROR); + return resp; } /* Is this a request for CRC or data? */ diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c index 88a41d1ca5f64..6c0bfb3ea1c9f 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c @@ -168,7 +168,8 @@ static void free_command_queues(struct otx_cptvf *cptvf, chunk = list_first_entry(&cqinfo->queue[i].chead, struct otx_cpt_cmd_chunk, nextchunk); - dma_free_coherent(&pdev->dev, chunk->size, + dma_free_coherent(&pdev->dev, + chunk->size + OTX_CPT_NEXT_CHUNK_PTR_SIZE, chunk->head, chunk->dma_addr); chunk->head = NULL; diff --git a/drivers/crypto/starfive/jh7110-aes.c b/drivers/crypto/starfive/jh7110-aes.c index 426b24889af85..01195664cc7cd 100644 --- a/drivers/crypto/starfive/jh7110-aes.c +++ b/drivers/crypto/starfive/jh7110-aes.c @@ -669,8 +669,10 @@ static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq return -ENOMEM; if (sg_copy_to_buffer(req->src, sg_nents_for_len(req->src, cryp->assoclen), - rctx->adata, cryp->assoclen) != cryp->assoclen) + rctx->adata, cryp->assoclen) != cryp->assoclen) { + kfree(rctx->adata); return -EINVAL; + } } if (cryp->total_in) @@ -681,8 +683,11 @@ static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq ctx->rctx = rctx; ret = starfive_aes_hw_init(ctx); - if (ret) + if (ret) { + if (cryp->assoclen) + kfree(rctx->adata); return ret; + } if (!cryp->assoclen) goto write_text; diff --git a/drivers/cxl/core/edac.c b/drivers/cxl/core/edac.c index 79994ca9bc9f3..81160260e26b7 100644 --- a/drivers/cxl/core/edac.c +++ b/drivers/cxl/core/edac.c @@ -1988,6 +1988,40 @@ static int cxl_memdev_soft_ppr_init(struct cxl_memdev *cxlmd, return 0; } +static void err_rec_free(void *_cxlmd) +{ + struct cxl_memdev *cxlmd = _cxlmd; + struct cxl_mem_err_rec *array_rec = cxlmd->err_rec_array; + struct cxl_event_gen_media *rec_gen_media; + struct cxl_event_dram *rec_dram; + unsigned long index; + + cxlmd->err_rec_array = NULL; + xa_for_each(&array_rec->rec_dram, index, rec_dram) + kfree(rec_dram); + xa_destroy(&array_rec->rec_dram); + + xa_for_each(&array_rec->rec_gen_media, index, rec_gen_media) + kfree(rec_gen_media); + xa_destroy(&array_rec->rec_gen_media); + kfree(array_rec); +} + +static int devm_cxl_memdev_setup_err_rec(struct cxl_memdev *cxlmd) +{ + struct cxl_mem_err_rec *array_rec = + kzalloc(sizeof(*array_rec), GFP_KERNEL); + + if (!array_rec) + return -ENOMEM; + + xa_init(&array_rec->rec_gen_media); + xa_init(&array_rec->rec_dram); + cxlmd->err_rec_array = array_rec; + + return devm_add_action_or_reset(&cxlmd->dev, err_rec_free, cxlmd); +} + int devm_cxl_memdev_edac_register(struct cxl_memdev *cxlmd) { struct edac_dev_feature ras_features[CXL_NR_EDAC_DEV_FEATURES]; @@ -2038,15 +2072,9 @@ int devm_cxl_memdev_edac_register(struct cxl_memdev *cxlmd) } if (repair_inst) { - struct cxl_mem_err_rec *array_rec = - devm_kzalloc(&cxlmd->dev, sizeof(*array_rec), - GFP_KERNEL); - if (!array_rec) - return -ENOMEM; - - xa_init(&array_rec->rec_gen_media); - xa_init(&array_rec->rec_dram); - cxlmd->err_rec_array = array_rec; + rc = devm_cxl_memdev_setup_err_rec(cxlmd); + if (rc) + return rc; } } @@ -2088,22 +2116,4 @@ int devm_cxl_region_edac_register(struct cxl_region *cxlr) } EXPORT_SYMBOL_NS_GPL(devm_cxl_region_edac_register, "CXL"); -void devm_cxl_memdev_edac_release(struct cxl_memdev *cxlmd) -{ - struct cxl_mem_err_rec *array_rec = cxlmd->err_rec_array; - struct cxl_event_gen_media *rec_gen_media; - struct cxl_event_dram *rec_dram; - unsigned long index; - - if (!IS_ENABLED(CONFIG_CXL_EDAC_MEM_REPAIR) || !array_rec) - return; - - xa_for_each(&array_rec->rec_dram, index, rec_dram) - kfree(rec_dram); - xa_destroy(&array_rec->rec_dram); - xa_for_each(&array_rec->rec_gen_media, index, rec_gen_media) - kfree(rec_gen_media); - xa_destroy(&array_rec->rec_gen_media); -} -EXPORT_SYMBOL_NS_GPL(devm_cxl_memdev_edac_release, "CXL"); diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c index 20dd638108062..13dafac7c6d56 100644 --- a/drivers/cxl/core/hdm.c +++ b/drivers/cxl/core/hdm.c @@ -844,14 +844,13 @@ static int cxl_decoder_commit(struct cxl_decoder *cxld) scoped_guard(rwsem_read, &cxl_rwsem.dpa) setup_hw_decoder(cxld, hdm); - port->commit_end++; rc = cxld_await_commit(hdm, cxld->id); if (rc) { dev_dbg(&port->dev, "%s: error %d committing decoder\n", dev_name(&cxld->dev), rc); - cxld->reset(cxld); return rc; } + port->commit_end++; cxld->flags |= CXL_DECODER_F_ENABLE; return 0; diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c index e370d733e4400..4dff7f44d908e 100644 --- a/drivers/cxl/core/memdev.c +++ b/drivers/cxl/core/memdev.c @@ -27,7 +27,6 @@ static void cxl_memdev_release(struct device *dev) struct cxl_memdev *cxlmd = to_cxl_memdev(dev); ida_free(&cxl_memdev_ida, cxlmd->id); - devm_cxl_memdev_edac_release(cxlmd); kfree(cxlmd); } diff --git a/drivers/cxl/core/pmem.c b/drivers/cxl/core/pmem.c index 8853415c106a9..e3a8b8d813333 100644 --- a/drivers/cxl/core/pmem.c +++ b/drivers/cxl/core/pmem.c @@ -115,15 +115,17 @@ static void unregister_nvb(void *_cxl_nvb) device_unregister(&cxl_nvb->dev); } -/** - * devm_cxl_add_nvdimm_bridge() - add the root of a LIBNVDIMM topology - * @host: platform firmware root device - * @port: CXL port at the root of a CXL topology - * - * Return: bridge device that can host cxl_nvdimm objects - */ -struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host, - struct cxl_port *port) +static bool cxl_nvdimm_bridge_failed_attach(struct cxl_nvdimm_bridge *cxl_nvb) +{ + struct device *dev = &cxl_nvb->dev; + + guard(device)(dev); + /* If the device has no driver, then it failed to attach. */ + return dev->driver == NULL; +} + +struct cxl_nvdimm_bridge *__devm_cxl_add_nvdimm_bridge(struct device *host, + struct cxl_port *port) { struct cxl_nvdimm_bridge *cxl_nvb; struct device *dev; @@ -145,6 +147,11 @@ struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host, if (rc) goto err; + if (cxl_nvdimm_bridge_failed_attach(cxl_nvb)) { + unregister_nvb(cxl_nvb); + return ERR_PTR(-ENODEV); + } + rc = devm_add_action_or_reset(host, unregister_nvb, cxl_nvb); if (rc) return ERR_PTR(rc); @@ -155,7 +162,7 @@ struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host, put_device(dev); return ERR_PTR(rc); } -EXPORT_SYMBOL_NS_GPL(devm_cxl_add_nvdimm_bridge, "CXL"); +EXPORT_SYMBOL_FOR_MODULES(__devm_cxl_add_nvdimm_bridge, "cxl_pmem"); static void cxl_nvdimm_release(struct device *dev) { @@ -254,6 +261,21 @@ int devm_cxl_add_nvdimm(struct cxl_port *parent_port, if (!cxl_nvb) return -ENODEV; + /* + * Take the uport_dev lock to guard against race of nvdimm_bus object. + * cxl_acpi_probe() registers the nvdimm_bus and is done under the + * root port uport_dev lock. + * + * Take the cxl_nvb device lock to ensure that cxl_nvb driver is in a + * consistent state. And the driver registers nvdimm_bus. + */ + guard(device)(cxl_nvb->port->uport_dev); + guard(device)(&cxl_nvb->dev); + if (!cxl_nvb->nvdimm_bus) { + rc = -ENODEV; + goto err_alloc; + } + cxl_nvd = cxl_nvdimm_alloc(cxl_nvb, cxlmd); if (IS_ERR(cxl_nvd)) { rc = PTR_ERR(cxl_nvd); diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 804e4a48540f6..85131872d7f6e 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -823,16 +823,18 @@ DEFINE_DEBUGFS_ATTRIBUTE(cxl_einj_inject_fops, NULL, cxl_einj_inject, static void cxl_debugfs_create_dport_dir(struct cxl_dport *dport) { + struct cxl_port *parent = parent_port_of(dport->port); struct dentry *dir; if (!einj_cxl_is_initialized()) return; /* - * dport_dev needs to be a PCIe port for CXL 2.0+ ports because - * EINJ expects a dport SBDF to be specified for 2.0 error injection. + * Protocol error injection is only available for CXL 2.0+ root ports + * and CXL 1.1 downstream ports */ - if (!dport->rch && !dev_is_pci(dport->dport_dev)) + if (!dport->rch && + !(dev_is_pci(dport->dport_dev) && parent && is_cxl_root(parent))) return; dir = cxl_debugfs_create_dir(dev_name(dport->dport_dev)); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 231ddccf89773..3a794278cc7fe 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -555,11 +555,16 @@ struct cxl_nvdimm_bridge { #define CXL_DEV_ID_LEN 19 +enum { + CXL_NVD_F_INVALIDATED = 0, +}; + struct cxl_nvdimm { struct device dev; struct cxl_memdev *cxlmd; u8 dev_id[CXL_DEV_ID_LEN]; /* for nvdimm, string of 'serial' */ u64 dirty_shutdowns; + unsigned long flags; }; struct cxl_pmem_region_mapping { @@ -866,6 +871,8 @@ void cxl_driver_unregister(struct cxl_driver *cxl_drv); struct cxl_nvdimm_bridge *to_cxl_nvdimm_bridge(struct device *dev); struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host, struct cxl_port *port); +struct cxl_nvdimm_bridge *__devm_cxl_add_nvdimm_bridge(struct device *host, + struct cxl_port *port); struct cxl_nvdimm *to_cxl_nvdimm(struct device *dev); bool is_cxl_nvdimm(struct device *dev); int devm_cxl_add_nvdimm(struct cxl_port *parent_port, struct cxl_memdev *cxlmd); diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index 434031a0c1f74..c12ab4fc95123 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -63,7 +63,7 @@ struct cxl_memdev { int depth; u8 scrub_cycle; int scrub_region_id; - void *err_rec_array; + struct cxl_mem_err_rec *err_rec_array; }; static inline struct cxl_memdev *to_cxl_memdev(struct device *dev) @@ -877,7 +877,6 @@ int devm_cxl_memdev_edac_register(struct cxl_memdev *cxlmd); int devm_cxl_region_edac_register(struct cxl_region *cxlr); int cxl_store_rec_gen_media(struct cxl_memdev *cxlmd, union cxl_event *evt); int cxl_store_rec_dram(struct cxl_memdev *cxlmd, union cxl_event *evt); -void devm_cxl_memdev_edac_release(struct cxl_memdev *cxlmd); #else static inline int devm_cxl_memdev_edac_register(struct cxl_memdev *cxlmd) { return 0; } @@ -889,8 +888,6 @@ static inline int cxl_store_rec_gen_media(struct cxl_memdev *cxlmd, static inline int cxl_store_rec_dram(struct cxl_memdev *cxlmd, union cxl_event *evt) { return 0; } -static inline void devm_cxl_memdev_edac_release(struct cxl_memdev *cxlmd) -{ return; } #endif #ifdef CONFIG_CXL_SUSPEND diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c index e197883690efc..c00b84b960761 100644 --- a/drivers/cxl/pmem.c +++ b/drivers/cxl/pmem.c @@ -13,6 +13,20 @@ static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX); +/** + * devm_cxl_add_nvdimm_bridge() - add the root of a LIBNVDIMM topology + * @host: platform firmware root device + * @port: CXL port at the root of a CXL topology + * + * Return: bridge device that can host cxl_nvdimm objects + */ +struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host, + struct cxl_port *port) +{ + return __devm_cxl_add_nvdimm_bridge(host, port); +} +EXPORT_SYMBOL_NS_GPL(devm_cxl_add_nvdimm_bridge, "CXL"); + static void clear_exclusive(void *mds) { clear_exclusive_cxl_commands(mds, exclusive_cmds); @@ -129,6 +143,9 @@ static int cxl_nvdimm_probe(struct device *dev) struct nvdimm *nvdimm; int rc; + if (test_bit(CXL_NVD_F_INVALIDATED, &cxl_nvd->flags)) + return -EBUSY; + set_exclusive_cxl_commands(mds, exclusive_cmds); rc = devm_add_action_or_reset(dev, clear_exclusive, mds); if (rc) @@ -309,8 +326,10 @@ static int detach_nvdimm(struct device *dev, void *data) scoped_guard(device, dev) { if (dev->driver) { cxl_nvd = to_cxl_nvdimm(dev); - if (cxl_nvd->cxlmd && cxl_nvd->cxlmd->cxl_nvb == data) + if (cxl_nvd->cxlmd && cxl_nvd->cxlmd->cxl_nvb == data) { release = true; + set_bit(CXL_NVD_F_INVALIDATED, &cxl_nvd->flags); + } } } if (release) @@ -353,6 +372,7 @@ static struct cxl_driver cxl_nvdimm_bridge_driver = { .probe = cxl_nvdimm_bridge_probe, .id = CXL_DEVICE_NVDIMM_BRIDGE, .drv = { + .probe_type = PROBE_FORCE_SYNCHRONOUS, .suppress_bind_attrs = true, }, }; diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c index 5b06b0dc67ee1..0f25f6d8ae71f 100644 --- a/drivers/dma/dma-axi-dmac.c +++ b/drivers/dma/dma-axi-dmac.c @@ -233,11 +233,9 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) unsigned int flags = 0; unsigned int val; - if (!chan->hw_sg) { - val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); - if (val) /* Queue is full, wait for the next SOT IRQ */ - return; - } + val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); + if (val) /* Queue is full, wait for the next SOT IRQ */ + return; desc = chan->next_desc; @@ -247,6 +245,7 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) return; list_move_tail(&vdesc->node, &chan->active_descs); desc = to_axi_dmac_desc(vdesc); + chan->next_desc = desc; } sg = &desc->sg[desc->num_submitted]; @@ -265,8 +264,6 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) else chan->next_desc = NULL; flags |= AXI_DMAC_FLAG_LAST; - } else { - chan->next_desc = desc; } sg->hw->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID); diff --git a/drivers/dma/fsl-edma-main.c b/drivers/dma/fsl-edma-main.c index 97583c7d51a2e..093185768ad8e 100644 --- a/drivers/dma/fsl-edma-main.c +++ b/drivers/dma/fsl-edma-main.c @@ -915,7 +915,6 @@ static void fsl_edma_remove(struct platform_device *pdev) of_dma_controller_free(np); dma_async_device_unregister(&fsl_edma->dma_dev); fsl_edma_cleanup_vchan(&fsl_edma->dma_dev); - fsl_disable_clocks(fsl_edma, fsl_edma->drvdata->dmamuxs); } static int fsl_edma_suspend_late(struct device *dev) diff --git a/drivers/dma/mediatek/mtk-uart-apdma.c b/drivers/dma/mediatek/mtk-uart-apdma.c index 08e15177427b9..96c18c815f1df 100644 --- a/drivers/dma/mediatek/mtk-uart-apdma.c +++ b/drivers/dma/mediatek/mtk-uart-apdma.c @@ -41,7 +41,7 @@ #define VFF_STOP_CLR_B 0 #define VFF_EN_CLR_B 0 #define VFF_INT_EN_CLR_B 0 -#define VFF_4G_SUPPORT_CLR_B 0 +#define VFF_ADDR2_CLR_B 0 /* * interrupt trigger level for tx @@ -72,7 +72,7 @@ /* TX: the buffer size SW can write. RX: the buffer size HW can write. */ #define VFF_LEFT_SIZE 0x40 #define VFF_DEBUG_STATUS 0x50 -#define VFF_4G_SUPPORT 0x54 +#define VFF_ADDR2 0x54 struct mtk_uart_apdmadev { struct dma_device ddev; @@ -149,7 +149,7 @@ static void mtk_uart_apdma_start_tx(struct mtk_chan *c) mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_TX_INT_CLR_B); if (mtkd->support_33bits) - mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_EN_B); + mtk_uart_apdma_write(c, VFF_ADDR2, upper_32_bits(d->addr)); } mtk_uart_apdma_write(c, VFF_EN, VFF_EN_B); @@ -192,7 +192,7 @@ static void mtk_uart_apdma_start_rx(struct mtk_chan *c) mtk_uart_apdma_write(c, VFF_INT_FLAG, VFF_RX_INT_CLR_B); if (mtkd->support_33bits) - mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_EN_B); + mtk_uart_apdma_write(c, VFF_ADDR2, upper_32_bits(d->addr)); } mtk_uart_apdma_write(c, VFF_INT_EN, VFF_RX_INT_EN_B); @@ -298,7 +298,7 @@ static int mtk_uart_apdma_alloc_chan_resources(struct dma_chan *chan) } if (mtkd->support_33bits) - mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_SUPPORT_CLR_B); + mtk_uart_apdma_write(c, VFF_ADDR2, VFF_ADDR2_CLR_B); err_pm: pm_runtime_put_noidle(mtkd->ddev.dev); diff --git a/drivers/dma/stm32/stm32-dma3.c b/drivers/dma/stm32/stm32-dma3.c index 50e7106c5cb73..9500164c8f688 100644 --- a/drivers/dma/stm32/stm32-dma3.c +++ b/drivers/dma/stm32/stm32-dma3.c @@ -1914,12 +1914,7 @@ static struct platform_driver stm32_dma3_driver = { }, }; -static int __init stm32_dma3_init(void) -{ - return platform_driver_register(&stm32_dma3_driver); -} - -subsys_initcall(stm32_dma3_init); +module_platform_driver(stm32_dma3_driver); MODULE_DESCRIPTION("STM32 DMA3 controller driver"); MODULE_AUTHOR("Amelie Delaunay "); diff --git a/drivers/dma/stm32/stm32-mdma.c b/drivers/dma/stm32/stm32-mdma.c index 080c1c725216c..b87d41b234df1 100644 --- a/drivers/dma/stm32/stm32-mdma.c +++ b/drivers/dma/stm32/stm32-mdma.c @@ -731,7 +731,7 @@ static int stm32_mdma_setup_xfer(struct stm32_mdma_chan *chan, struct stm32_mdma_chan_config *chan_config = &chan->chan_config; struct scatterlist *sg; dma_addr_t src_addr, dst_addr; - u32 m2m_hw_period, ccr, ctcr, ctbr; + u32 m2m_hw_period = 0, ccr = 0, ctcr, ctbr; int i, ret = 0; if (chan_config->m2m_hw) diff --git a/drivers/dma/sun6i-dma.c b/drivers/dma/sun6i-dma.c index 2215ff877bf7d..f9d876deb1f05 100644 --- a/drivers/dma/sun6i-dma.c +++ b/drivers/dma/sun6i-dma.c @@ -583,6 +583,22 @@ static irqreturn_t sun6i_dma_interrupt(int irq, void *dev_id) return ret; } +static u32 find_burst_size(const u32 burst_lengths, u32 maxburst) +{ + if (!maxburst) + return 1; + + if (BIT(maxburst) & burst_lengths) + return maxburst; + + /* Hardware only does power-of-two bursts. */ + for (u32 burst = rounddown_pow_of_two(maxburst); burst > 0; burst /= 2) + if (BIT(burst) & burst_lengths) + return burst; + + return 1; +} + static int set_config(struct sun6i_dma_dev *sdev, struct dma_slave_config *sconfig, enum dma_transfer_direction direction, @@ -616,15 +632,13 @@ static int set_config(struct sun6i_dma_dev *sdev, return -EINVAL; if (!(BIT(dst_addr_width) & sdev->slave.dst_addr_widths)) return -EINVAL; - if (!(BIT(src_maxburst) & sdev->cfg->src_burst_lengths)) - return -EINVAL; - if (!(BIT(dst_maxburst) & sdev->cfg->dst_burst_lengths)) - return -EINVAL; src_width = convert_buswidth(src_addr_width); dst_width = convert_buswidth(dst_addr_width); - dst_burst = convert_burst(dst_maxburst); - src_burst = convert_burst(src_maxburst); + src_burst = find_burst_size(sdev->cfg->src_burst_lengths, src_maxburst); + dst_burst = find_burst_size(sdev->cfg->dst_burst_lengths, dst_maxburst); + dst_burst = convert_burst(dst_burst); + src_burst = convert_burst(src_burst); *p_cfg = DMA_CHAN_CFG_SRC_WIDTH(src_width) | DMA_CHAN_CFG_DST_WIDTH(dst_width); diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c index a4153bcb6dcfe..64944f601ee5a 100644 --- a/drivers/dpll/dpll_netlink.c +++ b/drivers/dpll/dpll_netlink.c @@ -637,6 +637,10 @@ dpll_cmd_pin_get_one(struct sk_buff *msg, struct dpll_pin *pin, ret = dpll_msg_add_pin_freq(msg, pin, ref, extack); if (ret) return ret; + if (prop->phase_gran && + nla_put_u32(msg, DPLL_A_PIN_PHASE_ADJUST_GRAN, + prop->phase_gran)) + return -EMSGSIZE; if (nla_put_s32(msg, DPLL_A_PIN_PHASE_ADJUST_MIN, prop->phase_range.min)) return -EMSGSIZE; @@ -1261,7 +1265,13 @@ dpll_pin_phase_adj_set(struct dpll_pin *pin, struct nlattr *phase_adj_attr, if (phase_adj > pin->prop.phase_range.max || phase_adj < pin->prop.phase_range.min) { NL_SET_ERR_MSG_ATTR(extack, phase_adj_attr, - "phase adjust value not supported"); + "phase adjust value of out range"); + return -EINVAL; + } + if (pin->prop.phase_gran && phase_adj % (s32)pin->prop.phase_gran) { + NL_SET_ERR_MSG_ATTR_FMT(extack, phase_adj_attr, + "phase adjust value not multiple of %u", + pin->prop.phase_gran); return -EINVAL; } diff --git a/drivers/dpll/zl3073x/Makefile b/drivers/dpll/zl3073x/Makefile index 84e22aae57e5f..bd324c7fe7101 100644 --- a/drivers/dpll/zl3073x/Makefile +++ b/drivers/dpll/zl3073x/Makefile @@ -1,7 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_ZL3073X) += zl3073x.o -zl3073x-objs := core.o devlink.o dpll.o flash.o fw.o prop.o +zl3073x-objs := core.o devlink.o dpll.o flash.o fw.o \ + out.o prop.o ref.o synth.o obj-$(CONFIG_ZL3073X_I2C) += zl3073x_i2c.o zl3073x_i2c-objs := i2c.o diff --git a/drivers/dpll/zl3073x/core.c b/drivers/dpll/zl3073x/core.c index e42e527813cf8..b9b7c751b7602 100644 --- a/drivers/dpll/zl3073x/core.c +++ b/drivers/dpll/zl3073x/core.c @@ -129,47 +129,6 @@ const struct regmap_config zl3073x_regmap_config = { }; EXPORT_SYMBOL_NS_GPL(zl3073x_regmap_config, "ZL3073X"); -/** - * zl3073x_ref_freq_factorize - factorize given frequency - * @freq: input frequency - * @base: base frequency - * @mult: multiplier - * - * Checks if the given frequency can be factorized using one of the - * supported base frequencies. If so the base frequency and multiplier - * are stored into appropriate parameters if they are not NULL. - * - * Return: 0 on success, -EINVAL if the frequency cannot be factorized - */ -int -zl3073x_ref_freq_factorize(u32 freq, u16 *base, u16 *mult) -{ - static const u16 base_freqs[] = { - 1, 2, 4, 5, 8, 10, 16, 20, 25, 32, 40, 50, 64, 80, 100, 125, - 128, 160, 200, 250, 256, 320, 400, 500, 625, 640, 800, 1000, - 1250, 1280, 1600, 2000, 2500, 3125, 3200, 4000, 5000, 6250, - 6400, 8000, 10000, 12500, 15625, 16000, 20000, 25000, 31250, - 32000, 40000, 50000, 62500, - }; - u32 div; - int i; - - for (i = 0; i < ARRAY_SIZE(base_freqs); i++) { - div = freq / base_freqs[i]; - - if (div <= U16_MAX && (freq % base_freqs[i]) == 0) { - if (base) - *base = base_freqs[i]; - if (mult) - *mult = div; - - return 0; - } - } - - return -EINVAL; -} - static bool zl3073x_check_reg(struct zl3073x_dev *zldev, unsigned int reg, size_t size) { @@ -593,190 +552,6 @@ int zl3073x_write_hwreg_seq(struct zl3073x_dev *zldev, return rc; } -/** - * zl3073x_ref_state_fetch - get input reference state - * @zldev: pointer to zl3073x_dev structure - * @index: input reference index to fetch state for - * - * Function fetches information for the given input reference that are - * invariant and stores them for later use. - * - * Return: 0 on success, <0 on error - */ -static int -zl3073x_ref_state_fetch(struct zl3073x_dev *zldev, u8 index) -{ - struct zl3073x_ref *input = &zldev->ref[index]; - u8 ref_config; - int rc; - - /* If the input is differential then the configuration for N-pin - * reference is ignored and P-pin config is used for both. - */ - if (zl3073x_is_n_pin(index) && - zl3073x_ref_is_diff(zldev, index - 1)) { - input->enabled = zl3073x_ref_is_enabled(zldev, index - 1); - input->diff = true; - - return 0; - } - - guard(mutex)(&zldev->multiop_lock); - - /* Read reference configuration */ - rc = zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_RD, - ZL_REG_REF_MB_MASK, BIT(index)); - if (rc) - return rc; - - /* Read ref_config register */ - rc = zl3073x_read_u8(zldev, ZL_REG_REF_CONFIG, &ref_config); - if (rc) - return rc; - - input->enabled = FIELD_GET(ZL_REF_CONFIG_ENABLE, ref_config); - input->diff = FIELD_GET(ZL_REF_CONFIG_DIFF_EN, ref_config); - - dev_dbg(zldev->dev, "REF%u is %s and configured as %s\n", index, - str_enabled_disabled(input->enabled), - input->diff ? "differential" : "single-ended"); - - return rc; -} - -/** - * zl3073x_out_state_fetch - get output state - * @zldev: pointer to zl3073x_dev structure - * @index: output index to fetch state for - * - * Function fetches information for the given output (not output pin) - * that are invariant and stores them for later use. - * - * Return: 0 on success, <0 on error - */ -static int -zl3073x_out_state_fetch(struct zl3073x_dev *zldev, u8 index) -{ - struct zl3073x_out *out = &zldev->out[index]; - u8 output_ctrl, output_mode; - int rc; - - /* Read output configuration */ - rc = zl3073x_read_u8(zldev, ZL_REG_OUTPUT_CTRL(index), &output_ctrl); - if (rc) - return rc; - - /* Store info about output enablement and synthesizer the output - * is connected to. - */ - out->enabled = FIELD_GET(ZL_OUTPUT_CTRL_EN, output_ctrl); - out->synth = FIELD_GET(ZL_OUTPUT_CTRL_SYNTH_SEL, output_ctrl); - - dev_dbg(zldev->dev, "OUT%u is %s and connected to SYNTH%u\n", index, - str_enabled_disabled(out->enabled), out->synth); - - guard(mutex)(&zldev->multiop_lock); - - /* Read output configuration */ - rc = zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_RD, - ZL_REG_OUTPUT_MB_MASK, BIT(index)); - if (rc) - return rc; - - /* Read output_mode */ - rc = zl3073x_read_u8(zldev, ZL_REG_OUTPUT_MODE, &output_mode); - if (rc) - return rc; - - /* Extract and store output signal format */ - out->signal_format = FIELD_GET(ZL_OUTPUT_MODE_SIGNAL_FORMAT, - output_mode); - - dev_dbg(zldev->dev, "OUT%u has signal format 0x%02x\n", index, - out->signal_format); - - return rc; -} - -/** - * zl3073x_synth_state_fetch - get synth state - * @zldev: pointer to zl3073x_dev structure - * @index: synth index to fetch state for - * - * Function fetches information for the given synthesizer that are - * invariant and stores them for later use. - * - * Return: 0 on success, <0 on error - */ -static int -zl3073x_synth_state_fetch(struct zl3073x_dev *zldev, u8 index) -{ - struct zl3073x_synth *synth = &zldev->synth[index]; - u16 base, m, n; - u8 synth_ctrl; - u32 mult; - int rc; - - /* Read synth control register */ - rc = zl3073x_read_u8(zldev, ZL_REG_SYNTH_CTRL(index), &synth_ctrl); - if (rc) - return rc; - - /* Store info about synth enablement and DPLL channel the synth is - * driven by. - */ - synth->enabled = FIELD_GET(ZL_SYNTH_CTRL_EN, synth_ctrl); - synth->dpll = FIELD_GET(ZL_SYNTH_CTRL_DPLL_SEL, synth_ctrl); - - dev_dbg(zldev->dev, "SYNTH%u is %s and driven by DPLL%u\n", index, - str_enabled_disabled(synth->enabled), synth->dpll); - - guard(mutex)(&zldev->multiop_lock); - - /* Read synth configuration */ - rc = zl3073x_mb_op(zldev, ZL_REG_SYNTH_MB_SEM, ZL_SYNTH_MB_SEM_RD, - ZL_REG_SYNTH_MB_MASK, BIT(index)); - if (rc) - return rc; - - /* The output frequency is determined by the following formula: - * base * multiplier * numerator / denominator - * - * Read registers with these values - */ - rc = zl3073x_read_u16(zldev, ZL_REG_SYNTH_FREQ_BASE, &base); - if (rc) - return rc; - - rc = zl3073x_read_u32(zldev, ZL_REG_SYNTH_FREQ_MULT, &mult); - if (rc) - return rc; - - rc = zl3073x_read_u16(zldev, ZL_REG_SYNTH_FREQ_M, &m); - if (rc) - return rc; - - rc = zl3073x_read_u16(zldev, ZL_REG_SYNTH_FREQ_N, &n); - if (rc) - return rc; - - /* Check denominator for zero to avoid div by 0 */ - if (!n) { - dev_err(zldev->dev, - "Zero divisor for SYNTH%u retrieved from device\n", - index); - return -EINVAL; - } - - /* Compute and store synth frequency */ - zldev->synth[index].freq = div_u64(mul_u32_u32(base * m, mult), n); - - dev_dbg(zldev->dev, "SYNTH%u frequency: %u Hz\n", index, - zldev->synth[index].freq); - - return rc; -} - static int zl3073x_dev_state_fetch(struct zl3073x_dev *zldev) { @@ -816,6 +591,21 @@ zl3073x_dev_state_fetch(struct zl3073x_dev *zldev) return rc; } +static void +zl3073x_dev_ref_status_update(struct zl3073x_dev *zldev) +{ + int i, rc; + + for (i = 0; i < ZL3073X_NUM_REFS; i++) { + rc = zl3073x_read_u8(zldev, ZL_REG_REF_MON_STATUS(i), + &zldev->ref[i].mon_status); + if (rc) + dev_warn(zldev->dev, + "Failed to get REF%u status: %pe\n", i, + ERR_PTR(rc)); + } +} + /** * zl3073x_ref_phase_offsets_update - update reference phase offsets * @zldev: pointer to zl3073x_dev structure @@ -935,6 +725,9 @@ zl3073x_dev_periodic_work(struct kthread_work *work) struct zl3073x_dpll *zldpll; int rc; + /* Update input references status */ + zl3073x_dev_ref_status_update(zldev); + /* Update DPLL-to-connected-ref phase offsets registers */ rc = zl3073x_ref_phase_offsets_update(zldev, -1); if (rc) @@ -1185,11 +978,7 @@ zl3073x_devm_dpll_init(struct zl3073x_dev *zldev, u8 num_dplls) } /* Add devres action to release DPLL related resources */ - rc = devm_add_action_or_reset(zldev->dev, zl3073x_dev_dpll_fini, zldev); - if (rc) - goto error; - - return 0; + return devm_add_action_or_reset(zldev->dev, zl3073x_dev_dpll_fini, zldev); error: zl3073x_dev_dpll_fini(zldev); @@ -1230,6 +1019,7 @@ int zl3073x_dev_probe(struct zl3073x_dev *zldev, "Unknown or non-match chip ID: 0x%0x\n", id); } + zldev->chip_id = id; /* Read revision, firmware version and custom config version */ rc = zl3073x_read_u16(zldev, ZL_REG_REVISION, &revision); diff --git a/drivers/dpll/zl3073x/core.h b/drivers/dpll/zl3073x/core.h index 1dca4ddcf2350..08f1b49cccca4 100644 --- a/drivers/dpll/zl3073x/core.h +++ b/drivers/dpll/zl3073x/core.h @@ -9,7 +9,10 @@ #include #include +#include "out.h" +#include "ref.h" #include "regs.h" +#include "synth.h" struct device; struct regmap; @@ -27,47 +30,12 @@ struct zl3073x_dpll; #define ZL3073X_NUM_PINS (ZL3073X_NUM_INPUT_PINS + \ ZL3073X_NUM_OUTPUT_PINS) -/** - * struct zl3073x_ref - input reference invariant info - * @enabled: input reference is enabled or disabled - * @diff: true if input reference is differential - * @ffo: current fractional frequency offset - */ -struct zl3073x_ref { - bool enabled; - bool diff; - s64 ffo; -}; - -/** - * struct zl3073x_out - output invariant info - * @enabled: out is enabled or disabled - * @synth: synthesizer the out is connected to - * @signal_format: out signal format - */ -struct zl3073x_out { - bool enabled; - u8 synth; - u8 signal_format; -}; - -/** - * struct zl3073x_synth - synthesizer invariant info - * @freq: synthesizer frequency - * @dpll: ID of DPLL the synthesizer is driven by - * @enabled: synth is enabled or disabled - */ -struct zl3073x_synth { - u32 freq; - u8 dpll; - bool enabled; -}; - /** * struct zl3073x_dev - zl3073x device * @dev: pointer to device * @regmap: regmap to access device registers * @multiop_lock: to serialize multiple register operations + * @chip_id: chip ID read from hardware * @ref: array of input references' invariants * @out: array of outs' invariants * @synth: array of synths' invariants @@ -81,6 +49,7 @@ struct zl3073x_dev { struct device *dev; struct regmap *regmap; struct mutex multiop_lock; + u16 chip_id; /* Invariants */ struct zl3073x_ref ref[ZL3073X_NUM_REFS]; @@ -175,9 +144,34 @@ int zl3073x_write_hwreg_seq(struct zl3073x_dev *zldev, * Misc operations *****************/ -int zl3073x_ref_freq_factorize(u32 freq, u16 *base, u16 *mult); int zl3073x_ref_phase_offsets_update(struct zl3073x_dev *zldev, int channel); +/** + * zl3073x_dev_is_ref_phase_comp_32bit - check ref phase comp register size + * @zldev: pointer to zl3073x device + * + * Some chip IDs have a 32-bit wide ref_phase_offset_comp register instead + * of the default 48-bit. + * + * Return: true if the register is 32-bit, false if 48-bit + */ +static inline bool +zl3073x_dev_is_ref_phase_comp_32bit(struct zl3073x_dev *zldev) +{ + switch (zldev->chip_id) { + case 0x0E30: + case 0x0E93: + case 0x0E94: + case 0x0E95: + case 0x0E96: + case 0x0E97: + case 0x1F60: + return true; + default: + return false; + } +} + static inline bool zl3073x_is_n_pin(u8 id) { @@ -217,172 +211,218 @@ zl3073x_output_pin_out_get(u8 id) } /** - * zl3073x_ref_ffo_get - get current fractional frequency offset + * zl3073x_dev_ref_ffo_get - get current fractional frequency offset * @zldev: pointer to zl3073x device * @index: input reference index * * Return: the latest measured fractional frequency offset */ static inline s64 -zl3073x_ref_ffo_get(struct zl3073x_dev *zldev, u8 index) +zl3073x_dev_ref_ffo_get(struct zl3073x_dev *zldev, u8 index) +{ + const struct zl3073x_ref *ref = zl3073x_ref_state_get(zldev, index); + + return zl3073x_ref_ffo_get(ref); +} + +/** + * zl3073x_dev_ref_freq_get - get input reference frequency + * @zldev: pointer to zl3073x device + * @index: input reference index + * + * Return: frequency of given input reference + */ +static inline u32 +zl3073x_dev_ref_freq_get(struct zl3073x_dev *zldev, u8 index) { - return zldev->ref[index].ffo; + const struct zl3073x_ref *ref = zl3073x_ref_state_get(zldev, index); + + return zl3073x_ref_freq_get(ref); } /** - * zl3073x_ref_is_diff - check if the given input reference is differential + * zl3073x_dev_ref_is_diff - check if the given input reference is differential * @zldev: pointer to zl3073x device * @index: input reference index * * Return: true if reference is differential, false if reference is single-ended */ static inline bool -zl3073x_ref_is_diff(struct zl3073x_dev *zldev, u8 index) +zl3073x_dev_ref_is_diff(struct zl3073x_dev *zldev, u8 index) { - return zldev->ref[index].diff; + const struct zl3073x_ref *ref = zl3073x_ref_state_get(zldev, index); + + return zl3073x_ref_is_diff(ref); } /** - * zl3073x_ref_is_enabled - check if the given input reference is enabled + * zl3073x_dev_ref_is_enabled - check if the given input reference is enabled * @zldev: pointer to zl3073x device * @index: input reference index * * Return: true if input refernce is enabled, false otherwise */ static inline bool -zl3073x_ref_is_enabled(struct zl3073x_dev *zldev, u8 index) +zl3073x_dev_ref_is_enabled(struct zl3073x_dev *zldev, u8 index) { - return zldev->ref[index].enabled; + const struct zl3073x_ref *ref = zl3073x_ref_state_get(zldev, index); + + return zl3073x_ref_is_enabled(ref); +} + +/* + * zl3073x_dev_ref_is_status_ok - check the given input reference status + * @zldev: pointer to zl3073x device + * @index: input reference index + * + * Return: true if the status is ok, false otherwise + */ +static inline bool +zl3073x_dev_ref_is_status_ok(struct zl3073x_dev *zldev, u8 index) +{ + const struct zl3073x_ref *ref = zl3073x_ref_state_get(zldev, index); + + return zl3073x_ref_is_status_ok(ref); } /** - * zl3073x_synth_dpll_get - get DPLL ID the synth is driven by + * zl3073x_dev_synth_dpll_get - get DPLL ID the synth is driven by * @zldev: pointer to zl3073x device * @index: synth index * * Return: ID of DPLL the given synthetizer is driven by */ static inline u8 -zl3073x_synth_dpll_get(struct zl3073x_dev *zldev, u8 index) +zl3073x_dev_synth_dpll_get(struct zl3073x_dev *zldev, u8 index) { - return zldev->synth[index].dpll; + const struct zl3073x_synth *synth; + + synth = zl3073x_synth_state_get(zldev, index); + return zl3073x_synth_dpll_get(synth); } /** - * zl3073x_synth_freq_get - get synth current freq + * zl3073x_dev_synth_freq_get - get synth current freq * @zldev: pointer to zl3073x device * @index: synth index * * Return: frequency of given synthetizer */ static inline u32 -zl3073x_synth_freq_get(struct zl3073x_dev *zldev, u8 index) +zl3073x_dev_synth_freq_get(struct zl3073x_dev *zldev, u8 index) { - return zldev->synth[index].freq; + const struct zl3073x_synth *synth; + + synth = zl3073x_synth_state_get(zldev, index); + return zl3073x_synth_freq_get(synth); } /** - * zl3073x_synth_is_enabled - check if the given synth is enabled + * zl3073x_dev_synth_is_enabled - check if the given synth is enabled * @zldev: pointer to zl3073x device * @index: synth index * * Return: true if synth is enabled, false otherwise */ static inline bool -zl3073x_synth_is_enabled(struct zl3073x_dev *zldev, u8 index) +zl3073x_dev_synth_is_enabled(struct zl3073x_dev *zldev, u8 index) { - return zldev->synth[index].enabled; + const struct zl3073x_synth *synth; + + synth = zl3073x_synth_state_get(zldev, index); + return zl3073x_synth_is_enabled(synth); } /** - * zl3073x_out_synth_get - get synth connected to given output + * zl3073x_dev_out_synth_get - get synth connected to given output * @zldev: pointer to zl3073x device * @index: output index * * Return: index of synth connected to given output. */ static inline u8 -zl3073x_out_synth_get(struct zl3073x_dev *zldev, u8 index) +zl3073x_dev_out_synth_get(struct zl3073x_dev *zldev, u8 index) { - return zldev->out[index].synth; + const struct zl3073x_out *out = zl3073x_out_state_get(zldev, index); + + return zl3073x_out_synth_get(out); } /** - * zl3073x_out_is_enabled - check if the given output is enabled + * zl3073x_dev_out_is_enabled - check if the given output is enabled * @zldev: pointer to zl3073x device * @index: output index * * Return: true if the output is enabled, false otherwise */ static inline bool -zl3073x_out_is_enabled(struct zl3073x_dev *zldev, u8 index) +zl3073x_dev_out_is_enabled(struct zl3073x_dev *zldev, u8 index) { - u8 synth; + const struct zl3073x_out *out = zl3073x_out_state_get(zldev, index); + const struct zl3073x_synth *synth; + u8 synth_id; /* Output is enabled only if associated synth is enabled */ - synth = zl3073x_out_synth_get(zldev, index); - if (zl3073x_synth_is_enabled(zldev, synth)) - return zldev->out[index].enabled; + synth_id = zl3073x_out_synth_get(out); + synth = zl3073x_synth_state_get(zldev, synth_id); - return false; + return zl3073x_synth_is_enabled(synth) && zl3073x_out_is_enabled(out); } /** - * zl3073x_out_signal_format_get - get output signal format + * zl3073x_dev_out_signal_format_get - get output signal format * @zldev: pointer to zl3073x device * @index: output index * * Return: signal format of given output */ static inline u8 -zl3073x_out_signal_format_get(struct zl3073x_dev *zldev, u8 index) +zl3073x_dev_out_signal_format_get(struct zl3073x_dev *zldev, u8 index) { - return zldev->out[index].signal_format; + const struct zl3073x_out *out = zl3073x_out_state_get(zldev, index); + + return zl3073x_out_signal_format_get(out); } /** - * zl3073x_out_dpll_get - get DPLL ID the output is driven by + * zl3073x_dev_out_dpll_get - get DPLL ID the output is driven by * @zldev: pointer to zl3073x device * @index: output index * * Return: ID of DPLL the given output is driven by */ static inline -u8 zl3073x_out_dpll_get(struct zl3073x_dev *zldev, u8 index) +u8 zl3073x_dev_out_dpll_get(struct zl3073x_dev *zldev, u8 index) { - u8 synth; + const struct zl3073x_out *out = zl3073x_out_state_get(zldev, index); + const struct zl3073x_synth *synth; + u8 synth_id; /* Get synthesizer connected to given output */ - synth = zl3073x_out_synth_get(zldev, index); + synth_id = zl3073x_out_synth_get(out); + synth = zl3073x_synth_state_get(zldev, synth_id); /* Return DPLL that drives the synth */ - return zl3073x_synth_dpll_get(zldev, synth); + return zl3073x_synth_dpll_get(synth); } /** - * zl3073x_out_is_diff - check if the given output is differential + * zl3073x_dev_out_is_diff - check if the given output is differential * @zldev: pointer to zl3073x device * @index: output index * * Return: true if output is differential, false if output is single-ended */ static inline bool -zl3073x_out_is_diff(struct zl3073x_dev *zldev, u8 index) +zl3073x_dev_out_is_diff(struct zl3073x_dev *zldev, u8 index) { - switch (zl3073x_out_signal_format_get(zldev, index)) { - case ZL_OUTPUT_MODE_SIGNAL_FORMAT_LVDS: - case ZL_OUTPUT_MODE_SIGNAL_FORMAT_DIFF: - case ZL_OUTPUT_MODE_SIGNAL_FORMAT_LOWVCM: - return true; - default: - break; - } + const struct zl3073x_out *out = zl3073x_out_state_get(zldev, index); - return false; + return zl3073x_out_is_diff(out); } /** - * zl3073x_output_pin_is_enabled - check if the given output pin is enabled + * zl3073x_dev_output_pin_is_enabled - check if the given output pin is enabled * @zldev: pointer to zl3073x device * @id: output pin id * @@ -392,16 +432,21 @@ zl3073x_out_is_diff(struct zl3073x_dev *zldev, u8 index) * Return: true if output pin is enabled, false if output pin is disabled */ static inline bool -zl3073x_output_pin_is_enabled(struct zl3073x_dev *zldev, u8 id) +zl3073x_dev_output_pin_is_enabled(struct zl3073x_dev *zldev, u8 id) { - u8 output = zl3073x_output_pin_out_get(id); + u8 out_id = zl3073x_output_pin_out_get(id); + const struct zl3073x_out *out; + + out = zl3073x_out_state_get(zldev, out_id); - /* Check if the whole output is enabled */ - if (!zl3073x_out_is_enabled(zldev, output)) + /* Check if the output is enabled - call _dev_ helper that + * additionally checks for attached synth enablement. + */ + if (!zl3073x_dev_out_is_enabled(zldev, out_id)) return false; /* Check signal format */ - switch (zl3073x_out_signal_format_get(zldev, output)) { + switch (zl3073x_out_signal_format_get(out)) { case ZL_OUTPUT_MODE_SIGNAL_FORMAT_DISABLED: /* Both output pins are disabled by signal format */ return false; diff --git a/drivers/dpll/zl3073x/dpll.c b/drivers/dpll/zl3073x/dpll.c index f93f9a4583243..d7194418d1568 100644 --- a/drivers/dpll/zl3073x/dpll.c +++ b/drivers/dpll/zl3073x/dpll.c @@ -35,6 +35,7 @@ * @prio: pin priority <0, 14> * @selectable: pin is selectable in automatic mode * @esync_control: embedded sync is controllable + * @phase_gran: phase adjustment granularity * @pin_state: last saved pin state * @phase_offset: last saved pin phase offset * @freq_offset: last saved fractional frequency offset @@ -49,6 +50,7 @@ struct zl3073x_dpll_pin { u8 prio; bool selectable; bool esync_control; + s32 phase_gran; enum dpll_pin_state pin_state; s64 phase_offset; s64 freq_offset; @@ -98,60 +100,6 @@ zl3073x_dpll_pin_direction_get(const struct dpll_pin *dpll_pin, void *pin_priv, return 0; } -/** - * zl3073x_dpll_input_ref_frequency_get - get input reference frequency - * @zldpll: pointer to zl3073x_dpll - * @ref_id: reference id - * @frequency: pointer to variable to store frequency - * - * Reads frequency of given input reference. - * - * Return: 0 on success, <0 on error - */ -static int -zl3073x_dpll_input_ref_frequency_get(struct zl3073x_dpll *zldpll, u8 ref_id, - u32 *frequency) -{ - struct zl3073x_dev *zldev = zldpll->dev; - u16 base, mult, num, denom; - int rc; - - guard(mutex)(&zldev->multiop_lock); - - /* Read reference configuration */ - rc = zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_RD, - ZL_REG_REF_MB_MASK, BIT(ref_id)); - if (rc) - return rc; - - /* Read registers to compute resulting frequency */ - rc = zl3073x_read_u16(zldev, ZL_REG_REF_FREQ_BASE, &base); - if (rc) - return rc; - rc = zl3073x_read_u16(zldev, ZL_REG_REF_FREQ_MULT, &mult); - if (rc) - return rc; - rc = zl3073x_read_u16(zldev, ZL_REG_REF_RATIO_M, &num); - if (rc) - return rc; - rc = zl3073x_read_u16(zldev, ZL_REG_REF_RATIO_N, &denom); - if (rc) - return rc; - - /* Sanity check that HW has not returned zero denominator */ - if (!denom) { - dev_err(zldev->dev, - "Zero divisor for ref %u frequency got from device\n", - ref_id); - return -EINVAL; - } - - /* Compute the frequency */ - *frequency = mul_u64_u32_div(base * mult, num, denom); - - return rc; -} - static int zl3073x_dpll_input_pin_esync_get(const struct dpll_pin *dpll_pin, void *pin_priv, @@ -163,39 +111,15 @@ zl3073x_dpll_input_pin_esync_get(const struct dpll_pin *dpll_pin, struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; - u8 ref, ref_sync_ctrl, sync_mode; - u32 esync_div, ref_freq; - int rc; - - /* Get reference frequency */ - ref = zl3073x_input_pin_ref_get(pin->id); - rc = zl3073x_dpll_input_ref_frequency_get(zldpll, pin->id, &ref_freq); - if (rc) - return rc; - - guard(mutex)(&zldev->multiop_lock); - - /* Read reference configuration into mailbox */ - rc = zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_RD, - ZL_REG_REF_MB_MASK, BIT(ref)); - if (rc) - return rc; - - /* Get ref sync mode */ - rc = zl3073x_read_u8(zldev, ZL_REG_REF_SYNC_CTRL, &ref_sync_ctrl); - if (rc) - return rc; - - /* Get esync divisor */ - rc = zl3073x_read_u32(zldev, ZL_REG_REF_ESYNC_DIV, &esync_div); - if (rc) - return rc; + const struct zl3073x_ref *ref; + u8 ref_id; - sync_mode = FIELD_GET(ZL_REF_SYNC_CTRL_MODE, ref_sync_ctrl); + ref_id = zl3073x_input_pin_ref_get(pin->id); + ref = zl3073x_ref_state_get(zldev, ref_id); - switch (sync_mode) { + switch (FIELD_GET(ZL_REF_SYNC_CTRL_MODE, ref->sync_ctrl)) { case ZL_REF_SYNC_CTRL_MODE_50_50_ESYNC_25_75: - esync->freq = (esync_div == ZL_REF_ESYNC_DIV_1HZ) ? 1 : 0; + esync->freq = ref->esync_n_div == ZL_REF_ESYNC_DIV_1HZ ? 1 : 0; esync->pulse = 25; break; default: @@ -207,7 +131,7 @@ zl3073x_dpll_input_pin_esync_get(const struct dpll_pin *dpll_pin, /* If the pin supports esync control expose its range but only * if the current reference frequency is > 1 Hz. */ - if (pin->esync_control && ref_freq > 1) { + if (pin->esync_control && zl3073x_ref_freq_get(ref) > 1) { esync->range = esync_freq_ranges; esync->range_num = ARRAY_SIZE(esync_freq_ranges); } else { @@ -215,7 +139,7 @@ zl3073x_dpll_input_pin_esync_get(const struct dpll_pin *dpll_pin, esync->range_num = 0; } - return rc; + return 0; } static int @@ -228,22 +152,11 @@ zl3073x_dpll_input_pin_esync_set(const struct dpll_pin *dpll_pin, struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; - u8 ref, ref_sync_ctrl, sync_mode; - int rc; - - guard(mutex)(&zldev->multiop_lock); + struct zl3073x_ref ref; + u8 ref_id, sync_mode; - /* Read reference configuration into mailbox */ - ref = zl3073x_input_pin_ref_get(pin->id); - rc = zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_RD, - ZL_REG_REF_MB_MASK, BIT(ref)); - if (rc) - return rc; - - /* Get ref sync mode */ - rc = zl3073x_read_u8(zldev, ZL_REG_REF_SYNC_CTRL, &ref_sync_ctrl); - if (rc) - return rc; + ref_id = zl3073x_input_pin_ref_get(pin->id); + ref = *zl3073x_ref_state_get(zldev, ref_id); /* Use freq == 0 to disable esync */ if (!freq) @@ -251,25 +164,16 @@ zl3073x_dpll_input_pin_esync_set(const struct dpll_pin *dpll_pin, else sync_mode = ZL_REF_SYNC_CTRL_MODE_50_50_ESYNC_25_75; - ref_sync_ctrl &= ~ZL_REF_SYNC_CTRL_MODE; - ref_sync_ctrl |= FIELD_PREP(ZL_REF_SYNC_CTRL_MODE, sync_mode); - - /* Update ref sync control register */ - rc = zl3073x_write_u8(zldev, ZL_REG_REF_SYNC_CTRL, ref_sync_ctrl); - if (rc) - return rc; + ref.sync_ctrl &= ~ZL_REF_SYNC_CTRL_MODE; + ref.sync_ctrl |= FIELD_PREP(ZL_REF_SYNC_CTRL_MODE, sync_mode); if (freq) { - /* 1 Hz is only supported frequnecy currently */ - rc = zl3073x_write_u32(zldev, ZL_REG_REF_ESYNC_DIV, - ZL_REF_ESYNC_DIV_1HZ); - if (rc) - return rc; + /* 1 Hz is only supported frequency now */ + ref.esync_n_div = ZL_REF_ESYNC_DIV_1HZ; } - /* Commit reference configuration */ - return zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_WR, - ZL_REG_REF_MB_MASK, BIT(ref)); + /* Update reference configuration */ + return zl3073x_ref_state_set(zldev, ref_id, &ref); } static int @@ -293,17 +197,12 @@ zl3073x_dpll_input_pin_frequency_get(const struct dpll_pin *dpll_pin, { struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dpll_pin *pin = pin_priv; - u32 ref_freq; - u8 ref; - int rc; + u8 ref_id; - /* Read and return ref frequency */ - ref = zl3073x_input_pin_ref_get(pin->id); - rc = zl3073x_dpll_input_ref_frequency_get(zldpll, ref, &ref_freq); - if (!rc) - *frequency = ref_freq; + ref_id = zl3073x_input_pin_ref_get(pin->id); + *frequency = zl3073x_dev_ref_freq_get(zldpll->dev, ref_id); - return rc; + return 0; } static int @@ -316,39 +215,18 @@ zl3073x_dpll_input_pin_frequency_set(const struct dpll_pin *dpll_pin, struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; - u16 base, mult; - u8 ref; - int rc; + struct zl3073x_ref ref; + u8 ref_id; - /* Get base frequency and multiplier for the requested frequency */ - rc = zl3073x_ref_freq_factorize(frequency, &base, &mult); - if (rc) - return rc; + /* Get reference state */ + ref_id = zl3073x_input_pin_ref_get(pin->id); + ref = *zl3073x_ref_state_get(zldev, ref_id); - guard(mutex)(&zldev->multiop_lock); + /* Update frequency */ + zl3073x_ref_freq_set(&ref, frequency); - /* Load reference configuration */ - ref = zl3073x_input_pin_ref_get(pin->id); - rc = zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_RD, - ZL_REG_REF_MB_MASK, BIT(ref)); - - /* Update base frequency, multiplier, numerator & denominator */ - rc = zl3073x_write_u16(zldev, ZL_REG_REF_FREQ_BASE, base); - if (rc) - return rc; - rc = zl3073x_write_u16(zldev, ZL_REG_REF_FREQ_MULT, mult); - if (rc) - return rc; - rc = zl3073x_write_u16(zldev, ZL_REG_REF_RATIO_M, 1); - if (rc) - return rc; - rc = zl3073x_write_u16(zldev, ZL_REG_REF_RATIO_N, 1); - if (rc) - return rc; - - /* Commit reference configuration */ - return zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_WR, - ZL_REG_REF_MB_MASK, BIT(ref)); + /* Commit reference state */ + return zl3073x_ref_state_set(zldev, ref_id, &ref); } /** @@ -495,19 +373,10 @@ zl3073x_dpll_connected_ref_get(struct zl3073x_dpll *zldpll, u8 *ref) if (rc) return rc; - if (ZL3073X_DPLL_REF_IS_VALID(*ref)) { - u8 ref_status; - - /* Read the reference monitor status */ - rc = zl3073x_read_u8(zldev, ZL_REG_REF_MON_STATUS(*ref), - &ref_status); - if (rc) - return rc; - - /* If the monitor indicates an error nothing is connected */ - if (ref_status != ZL_REF_MON_STATUS_OK) - *ref = ZL3073X_DPLL_REF_NONE; - } + /* If the monitor indicates an error nothing is connected */ + if (ZL3073X_DPLL_REF_IS_VALID(*ref) && + !zl3073x_dev_ref_is_status_ok(zldev, *ref)) + *ref = ZL3073X_DPLL_REF_NONE; return 0; } @@ -522,34 +391,25 @@ zl3073x_dpll_input_pin_phase_offset_get(const struct dpll_pin *dpll_pin, struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; - u8 conn_ref, ref, ref_status; + const struct zl3073x_ref *ref; + u8 conn_id, ref_id; s64 ref_phase; int rc; /* Get currently connected reference */ - rc = zl3073x_dpll_connected_ref_get(zldpll, &conn_ref); + rc = zl3073x_dpll_connected_ref_get(zldpll, &conn_id); if (rc) return rc; /* Report phase offset only for currently connected pin if the phase - * monitor feature is disabled. + * monitor feature is disabled and only if the input pin signal is + * present. */ - ref = zl3073x_input_pin_ref_get(pin->id); - if (!zldpll->phase_monitor && ref != conn_ref) { + ref_id = zl3073x_input_pin_ref_get(pin->id); + ref = zl3073x_ref_state_get(zldev, ref_id); + if ((!zldpll->phase_monitor && ref_id != conn_id) || + !zl3073x_ref_is_status_ok(ref)) { *phase_offset = 0; - - return 0; - } - - /* Get this pin monitor status */ - rc = zl3073x_read_u8(zldev, ZL_REG_REF_MON_STATUS(ref), &ref_status); - if (rc) - return rc; - - /* Report phase offset only if the input pin signal is present */ - if (ref_status != ZL_REF_MON_STATUS_OK) { - *phase_offset = 0; - return 0; } @@ -559,20 +419,12 @@ zl3073x_dpll_input_pin_phase_offset_get(const struct dpll_pin *dpll_pin, * the phase offset is modded to the period of the signal * the dpll is locked to. */ - if (ZL3073X_DPLL_REF_IS_VALID(conn_ref) && conn_ref != ref) { + if (ZL3073X_DPLL_REF_IS_VALID(conn_id) && conn_id != ref_id) { u32 conn_freq, ref_freq; - /* Get frequency of connected ref */ - rc = zl3073x_dpll_input_ref_frequency_get(zldpll, conn_ref, - &conn_freq); - if (rc) - return rc; - - /* Get frequency of given ref */ - rc = zl3073x_dpll_input_ref_frequency_get(zldpll, ref, - &ref_freq); - if (rc) - return rc; + /* Get frequency of connected and given ref */ + conn_freq = zl3073x_dev_ref_freq_get(zldev, conn_id); + ref_freq = zl3073x_ref_freq_get(ref); if (conn_freq > ref_freq) { s64 conn_period, div_factor; @@ -599,33 +451,26 @@ zl3073x_dpll_input_pin_phase_adjust_get(const struct dpll_pin *dpll_pin, struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; + const struct zl3073x_ref *ref; s64 phase_comp; - u8 ref; - int rc; - - guard(mutex)(&zldev->multiop_lock); + u8 ref_id; /* Read reference configuration */ - ref = zl3073x_input_pin_ref_get(pin->id); - rc = zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_RD, - ZL_REG_REF_MB_MASK, BIT(ref)); - if (rc) - return rc; + ref_id = zl3073x_input_pin_ref_get(pin->id); + ref = zl3073x_ref_state_get(zldev, ref_id); - /* Read current phase offset compensation */ - rc = zl3073x_read_u48(zldev, ZL_REG_REF_PHASE_OFFSET_COMP, &phase_comp); - if (rc) - return rc; - - /* Perform sign extension for 48bit signed value */ - phase_comp = sign_extend64(phase_comp, 47); + /* Perform sign extension based on register width */ + if (zl3073x_dev_is_ref_phase_comp_32bit(zldev)) + phase_comp = sign_extend64(ref->phase_comp, 31); + else + phase_comp = sign_extend64(ref->phase_comp, 47); /* Reverse two's complement negation applied during set and convert * to 32bit signed int */ *phase_adjust = (s32)-phase_comp; - return rc; + return 0; } static int @@ -639,32 +484,20 @@ zl3073x_dpll_input_pin_phase_adjust_set(const struct dpll_pin *dpll_pin, struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; - s64 phase_comp; - u8 ref; - int rc; + struct zl3073x_ref ref; + u8 ref_id; + + /* Read reference configuration */ + ref_id = zl3073x_input_pin_ref_get(pin->id); + ref = *zl3073x_ref_state_get(zldev, ref_id); /* The value in the register is stored as two's complement negation * of requested value. */ - phase_comp = -phase_adjust; + ref.phase_comp = -phase_adjust; - guard(mutex)(&zldev->multiop_lock); - - /* Read reference configuration */ - ref = zl3073x_input_pin_ref_get(pin->id); - rc = zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_RD, - ZL_REG_REF_MB_MASK, BIT(ref)); - if (rc) - return rc; - - /* Write the requested value into the compensation register */ - rc = zl3073x_write_u48(zldev, ZL_REG_REF_PHASE_OFFSET_COMP, phase_comp); - if (rc) - return rc; - - /* Commit reference configuration */ - return zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_WR, - ZL_REG_REF_MB_MASK, BIT(ref)); + /* Update reference configuration */ + return zl3073x_ref_state_set(zldev, ref_id, &ref); } /** @@ -775,7 +608,7 @@ zl3073x_dpll_ref_state_get(struct zl3073x_dpll_pin *pin, { struct zl3073x_dpll *zldpll = pin->dpll; struct zl3073x_dev *zldev = zldpll->dev; - u8 ref, ref_conn, status; + u8 ref, ref_conn; int rc; ref = zl3073x_input_pin_ref_get(pin->id); @@ -795,20 +628,9 @@ zl3073x_dpll_ref_state_get(struct zl3073x_dpll_pin *pin, * pin as selectable. */ if (zldpll->refsel_mode == ZL_DPLL_MODE_REFSEL_MODE_AUTO && - pin->selectable) { - /* Read reference monitor status */ - rc = zl3073x_read_u8(zldev, ZL_REG_REF_MON_STATUS(ref), - &status); - if (rc) - return rc; - - /* If the monitor indicates errors report the reference - * as disconnected - */ - if (status == ZL_REF_MON_STATUS_OK) { - *state = DPLL_PIN_STATE_SELECTABLE; - return 0; - } + zl3073x_dev_ref_is_status_ok(zldev, ref) && pin->selectable) { + *state = DPLL_PIN_STATE_SELECTABLE; + return 0; } /* Otherwise report the pin as disconnected */ @@ -951,21 +773,19 @@ zl3073x_dpll_output_pin_esync_get(const struct dpll_pin *dpll_pin, struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; - struct device *dev = zldev->dev; - u32 esync_period, esync_width; - u8 clock_type, synth; - u8 out, output_mode; - u32 output_div; + const struct zl3073x_synth *synth; + const struct zl3073x_out *out; + u8 clock_type, out_id; u32 synth_freq; - int rc; - out = zl3073x_output_pin_out_get(pin->id); + out_id = zl3073x_output_pin_out_get(pin->id); + out = zl3073x_out_state_get(zldev, out_id); /* If N-division is enabled, esync is not supported. The register used * for N-division is also used for the esync divider so both cannot * be used. */ - switch (zl3073x_out_signal_format_get(zldev, out)) { + switch (zl3073x_out_signal_format_get(out)) { case ZL_OUTPUT_MODE_SIGNAL_FORMAT_2_NDIV: case ZL_OUTPUT_MODE_SIGNAL_FORMAT_2_NDIV_INV: return -EOPNOTSUPP; @@ -973,38 +793,11 @@ zl3073x_dpll_output_pin_esync_get(const struct dpll_pin *dpll_pin, break; } - guard(mutex)(&zldev->multiop_lock); - - /* Read output configuration into mailbox */ - rc = zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_RD, - ZL_REG_OUTPUT_MB_MASK, BIT(out)); - if (rc) - return rc; - - /* Read output mode */ - rc = zl3073x_read_u8(zldev, ZL_REG_OUTPUT_MODE, &output_mode); - if (rc) - return rc; - - /* Read output divisor */ - rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_DIV, &output_div); - if (rc) - return rc; - - /* Check output divisor for zero */ - if (!output_div) { - dev_err(dev, "Zero divisor for OUTPUT%u got from device\n", - out); - return -EINVAL; - } - - /* Get synth attached to output pin */ - synth = zl3073x_out_synth_get(zldev, out); + /* Get attached synth frequency */ + synth = zl3073x_synth_state_get(zldev, zl3073x_out_synth_get(out)); + synth_freq = zl3073x_synth_freq_get(synth); - /* Get synth frequency */ - synth_freq = zl3073x_synth_freq_get(zldev, synth); - - clock_type = FIELD_GET(ZL_OUTPUT_MODE_CLOCK_TYPE, output_mode); + clock_type = FIELD_GET(ZL_OUTPUT_MODE_CLOCK_TYPE, out->mode); if (clock_type != ZL_OUTPUT_MODE_CLOCK_TYPE_ESYNC) { /* No need to read esync data if it is not enabled */ esync->freq = 0; @@ -1013,38 +806,21 @@ zl3073x_dpll_output_pin_esync_get(const struct dpll_pin *dpll_pin, goto finish; } - /* Read esync period */ - rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_ESYNC_PERIOD, &esync_period); - if (rc) - return rc; - - /* Check esync divisor for zero */ - if (!esync_period) { - dev_err(dev, "Zero esync divisor for OUTPUT%u got from device\n", - out); - return -EINVAL; - } - - /* Get esync pulse width in units of half synth cycles */ - rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_ESYNC_WIDTH, &esync_width); - if (rc) - return rc; - /* Compute esync frequency */ - esync->freq = synth_freq / output_div / esync_period; + esync->freq = synth_freq / out->div / out->esync_n_period; /* By comparing the esync_pulse_width to the half of the pulse width * the esync pulse percentage can be determined. * Note that half pulse width is in units of half synth cycles, which * is why it reduces down to be output_div. */ - esync->pulse = (50 * esync_width) / output_div; + esync->pulse = (50 * out->esync_n_width) / out->div; finish: /* Set supported esync ranges if the pin supports esync control and * if the output frequency is > 1 Hz. */ - if (pin->esync_control && (synth_freq / output_div) > 1) { + if (pin->esync_control && (synth_freq / out->div) > 1) { esync->range = esync_freq_ranges; esync->range_num = ARRAY_SIZE(esync_freq_ranges); } else { @@ -1062,21 +838,22 @@ zl3073x_dpll_output_pin_esync_set(const struct dpll_pin *dpll_pin, void *dpll_priv, u64 freq, struct netlink_ext_ack *extack) { - u32 esync_period, esync_width, output_div; struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; - u8 clock_type, out, output_mode, synth; + const struct zl3073x_synth *synth; + struct zl3073x_out out; + u8 clock_type, out_id; u32 synth_freq; - int rc; - out = zl3073x_output_pin_out_get(pin->id); + out_id = zl3073x_output_pin_out_get(pin->id); + out = *zl3073x_out_state_get(zldev, out_id); /* If N-division is enabled, esync is not supported. The register used * for N-division is also used for the esync divider so both cannot * be used. */ - switch (zl3073x_out_signal_format_get(zldev, out)) { + switch (zl3073x_out_signal_format_get(&out)) { case ZL_OUTPUT_MODE_SIGNAL_FORMAT_2_NDIV: case ZL_OUTPUT_MODE_SIGNAL_FORMAT_2_NDIV_INV: return -EOPNOTSUPP; @@ -1084,19 +861,6 @@ zl3073x_dpll_output_pin_esync_set(const struct dpll_pin *dpll_pin, break; } - guard(mutex)(&zldev->multiop_lock); - - /* Read output configuration into mailbox */ - rc = zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_RD, - ZL_REG_OUTPUT_MB_MASK, BIT(out)); - if (rc) - return rc; - - /* Read output mode */ - rc = zl3073x_read_u8(zldev, ZL_REG_OUTPUT_MODE, &output_mode); - if (rc) - return rc; - /* Select clock type */ if (freq) clock_type = ZL_OUTPUT_MODE_CLOCK_TYPE_ESYNC; @@ -1104,38 +868,19 @@ zl3073x_dpll_output_pin_esync_set(const struct dpll_pin *dpll_pin, clock_type = ZL_OUTPUT_MODE_CLOCK_TYPE_NORMAL; /* Update clock type in output mode */ - output_mode &= ~ZL_OUTPUT_MODE_CLOCK_TYPE; - output_mode |= FIELD_PREP(ZL_OUTPUT_MODE_CLOCK_TYPE, clock_type); - rc = zl3073x_write_u8(zldev, ZL_REG_OUTPUT_MODE, output_mode); - if (rc) - return rc; + out.mode &= ~ZL_OUTPUT_MODE_CLOCK_TYPE; + out.mode |= FIELD_PREP(ZL_OUTPUT_MODE_CLOCK_TYPE, clock_type); /* If esync is being disabled just write mailbox and finish */ if (!freq) goto write_mailbox; - /* Get synth attached to output pin */ - synth = zl3073x_out_synth_get(zldev, out); - - /* Get synth frequency */ - synth_freq = zl3073x_synth_freq_get(zldev, synth); - - rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_DIV, &output_div); - if (rc) - return rc; - - /* Check output divisor for zero */ - if (!output_div) { - dev_err(zldev->dev, - "Zero divisor for OUTPUT%u got from device\n", out); - return -EINVAL; - } + /* Get attached synth frequency */ + synth = zl3073x_synth_state_get(zldev, zl3073x_out_synth_get(&out)); + synth_freq = zl3073x_synth_freq_get(synth); /* Compute and update esync period */ - esync_period = synth_freq / (u32)freq / output_div; - rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_ESYNC_PERIOD, esync_period); - if (rc) - return rc; + out.esync_n_period = synth_freq / (u32)freq / out.div; /* Half of the period in units of 1/2 synth cycle can be represented by * the output_div. To get the supported esync pulse width of 25% of the @@ -1143,15 +888,11 @@ zl3073x_dpll_output_pin_esync_set(const struct dpll_pin *dpll_pin, * assumes that output_div is even, otherwise some resolution will be * lost. */ - esync_width = output_div / 2; - rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_ESYNC_WIDTH, esync_width); - if (rc) - return rc; + out.esync_n_width = out.div / 2; write_mailbox: /* Commit output configuration */ - return zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_WR, - ZL_REG_OUTPUT_MB_MASK, BIT(out)); + return zl3073x_out_state_set(zldev, out_id, &out); } static int @@ -1164,83 +905,46 @@ zl3073x_dpll_output_pin_frequency_get(const struct dpll_pin *dpll_pin, struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; - struct device *dev = zldev->dev; - u8 out, signal_format, synth; - u32 output_div, synth_freq; - int rc; - - out = zl3073x_output_pin_out_get(pin->id); - synth = zl3073x_out_synth_get(zldev, out); - synth_freq = zl3073x_synth_freq_get(zldev, synth); - - guard(mutex)(&zldev->multiop_lock); - - /* Read output configuration into mailbox */ - rc = zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_RD, - ZL_REG_OUTPUT_MB_MASK, BIT(out)); - if (rc) - return rc; - - rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_DIV, &output_div); - if (rc) - return rc; + const struct zl3073x_synth *synth; + const struct zl3073x_out *out; + u32 synth_freq; + u8 out_id; - /* Check output divisor for zero */ - if (!output_div) { - dev_err(dev, "Zero divisor for output %u got from device\n", - out); - return -EINVAL; - } + out_id = zl3073x_output_pin_out_get(pin->id); + out = zl3073x_out_state_get(zldev, out_id); - /* Read used signal format for the given output */ - signal_format = zl3073x_out_signal_format_get(zldev, out); + /* Get attached synth frequency */ + synth = zl3073x_synth_state_get(zldev, zl3073x_out_synth_get(out)); + synth_freq = zl3073x_synth_freq_get(synth); - switch (signal_format) { + switch (zl3073x_out_signal_format_get(out)) { case ZL_OUTPUT_MODE_SIGNAL_FORMAT_2_NDIV: case ZL_OUTPUT_MODE_SIGNAL_FORMAT_2_NDIV_INV: /* In case of divided format we have to distiguish between * given output pin type. + * + * For P-pin the resulting frequency is computed as simple + * division of synth frequency and output divisor. + * + * For N-pin we have to divide additionally by divisor stored + * in esync_n_period output mailbox register that is used as + * N-pin divisor for these modes. */ - if (zl3073x_dpll_is_p_pin(pin)) { - /* For P-pin the resulting frequency is computed as - * simple division of synth frequency and output - * divisor. - */ - *frequency = synth_freq / output_div; - } else { - /* For N-pin we have to divide additionally by - * divisor stored in esync_period output mailbox - * register that is used as N-pin divisor for these - * modes. - */ - u32 ndiv; - - rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_ESYNC_PERIOD, - &ndiv); - if (rc) - return rc; + *frequency = synth_freq / out->div; - /* Check N-pin divisor for zero */ - if (!ndiv) { - dev_err(dev, - "Zero N-pin divisor for output %u got from device\n", - out); - return -EINVAL; - } + if (!zl3073x_dpll_is_p_pin(pin)) + *frequency = (u32)*frequency / out->esync_n_period; - /* Compute final divisor for N-pin */ - *frequency = synth_freq / output_div / ndiv; - } break; default: /* In other modes the resulting frequency is computed as * division of synth frequency and output divisor. */ - *frequency = synth_freq / output_div; + *frequency = synth_freq / out->div; break; } - return rc; + return 0; } static int @@ -1253,28 +957,21 @@ zl3073x_dpll_output_pin_frequency_set(const struct dpll_pin *dpll_pin, struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; - struct device *dev = zldev->dev; - u32 output_n_freq, output_p_freq; - u8 out, signal_format, synth; - u32 cur_div, new_div, ndiv; - u32 synth_freq; - int rc; + const struct zl3073x_synth *synth; + u8 out_id, signal_format; + u32 new_div, synth_freq; + struct zl3073x_out out; - out = zl3073x_output_pin_out_get(pin->id); - synth = zl3073x_out_synth_get(zldev, out); - synth_freq = zl3073x_synth_freq_get(zldev, synth); + out_id = zl3073x_output_pin_out_get(pin->id); + out = *zl3073x_out_state_get(zldev, out_id); + + /* Get attached synth frequency and compute new divisor */ + synth = zl3073x_synth_state_get(zldev, zl3073x_out_synth_get(&out)); + synth_freq = zl3073x_synth_freq_get(synth); new_div = synth_freq / (u32)frequency; /* Get used signal format for the given output */ - signal_format = zl3073x_out_signal_format_get(zldev, out); - - guard(mutex)(&zldev->multiop_lock); - - /* Load output configuration */ - rc = zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_RD, - ZL_REG_OUTPUT_MB_MASK, BIT(out)); - if (rc) - return rc; + signal_format = zl3073x_out_signal_format_get(&out); /* Check signal format */ if (signal_format != ZL_OUTPUT_MODE_SIGNAL_FORMAT_2_NDIV && @@ -1282,99 +979,50 @@ zl3073x_dpll_output_pin_frequency_set(const struct dpll_pin *dpll_pin, /* For non N-divided signal formats the frequency is computed * as division of synth frequency and output divisor. */ - rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_DIV, new_div); - if (rc) - return rc; + out.div = new_div; /* For 50/50 duty cycle the divisor is equal to width */ - rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_WIDTH, new_div); - if (rc) - return rc; + out.width = new_div; /* Commit output configuration */ - return zl3073x_mb_op(zldev, - ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_WR, - ZL_REG_OUTPUT_MB_MASK, BIT(out)); + return zl3073x_out_state_set(zldev, out_id, &out); } - /* For N-divided signal format get current divisor */ - rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_DIV, &cur_div); - if (rc) - return rc; - - /* Check output divisor for zero */ - if (!cur_div) { - dev_err(dev, "Zero divisor for output %u got from device\n", - out); - return -EINVAL; - } - - /* Get N-pin divisor (shares the same register with esync */ - rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_ESYNC_PERIOD, &ndiv); - if (rc) - return rc; - - /* Check N-pin divisor for zero */ - if (!ndiv) { - dev_err(dev, - "Zero N-pin divisor for output %u got from device\n", - out); - return -EINVAL; - } - - /* Compute current output frequency for P-pin */ - output_p_freq = synth_freq / cur_div; - - /* Compute current N-pin frequency */ - output_n_freq = output_p_freq / ndiv; - if (zl3073x_dpll_is_p_pin(pin)) { /* We are going to change output frequency for P-pin but * if the requested frequency is less than current N-pin * frequency then indicate a failure as we are not able * to compute N-pin divisor to keep its frequency unchanged. + * + * Update divisor for N-pin to keep N-pin frequency. */ - if (frequency <= output_n_freq) + out.esync_n_period = (out.esync_n_period * out.div) / new_div; + if (!out.esync_n_period) return -EINVAL; /* Update the output divisor */ - rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_DIV, new_div); - if (rc) - return rc; + out.div = new_div; /* For 50/50 duty cycle the divisor is equal to width */ - rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_WIDTH, new_div); - if (rc) - return rc; - - /* Compute new divisor for N-pin */ - ndiv = (u32)frequency / output_n_freq; + out.width = out.div; } else { /* We are going to change frequency of N-pin but if * the requested freq is greater or equal than freq of P-pin * in the output pair we cannot compute divisor for the N-pin. * In this case indicate a failure. + * + * Update divisor for N-pin */ - if (output_p_freq <= frequency) + out.esync_n_period = div64_u64(synth_freq, frequency * out.div); + if (!out.esync_n_period) return -EINVAL; - - /* Compute new divisor for N-pin */ - ndiv = output_p_freq / (u32)frequency; } - /* Update divisor for the N-pin */ - rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_ESYNC_PERIOD, ndiv); - if (rc) - return rc; - /* For 50/50 duty cycle the divisor is equal to width */ - rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_ESYNC_WIDTH, ndiv); - if (rc) - return rc; + out.esync_n_width = out.esync_n_period; /* Commit output configuration */ - return zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_WR, - ZL_REG_OUTPUT_MB_MASK, BIT(out)); + return zl3073x_out_state_set(zldev, out_id, &out); } static int @@ -1388,42 +1036,16 @@ zl3073x_dpll_output_pin_phase_adjust_get(const struct dpll_pin *dpll_pin, struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; - u32 synth_freq; - s32 phase_comp; - u8 out, synth; - int rc; + const struct zl3073x_out *out; + u8 out_id; - out = zl3073x_output_pin_out_get(pin->id); - synth = zl3073x_out_synth_get(zldev, out); - synth_freq = zl3073x_synth_freq_get(zldev, synth); + out_id = zl3073x_output_pin_out_get(pin->id); + out = zl3073x_out_state_get(zldev, out_id); - /* Check synth freq for zero */ - if (!synth_freq) { - dev_err(zldev->dev, "Got zero synth frequency for output %u\n", - out); - return -EINVAL; - } - - guard(mutex)(&zldev->multiop_lock); + /* The value in the register is expressed in half synth clock cycles. */ + *phase_adjust = out->phase_comp * pin->phase_gran; - /* Read output configuration */ - rc = zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_RD, - ZL_REG_OUTPUT_MB_MASK, BIT(out)); - if (rc) - return rc; - - /* Read current output phase compensation */ - rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_PHASE_COMP, &phase_comp); - if (rc) - return rc; - - /* Value in register is expressed in half synth clock cycles */ - phase_comp *= (int)div_u64(PSEC_PER_SEC, 2 * synth_freq); - - /* Reverse two's complement negation applied during 'set' */ - *phase_adjust = -phase_comp; - - return rc; + return 0; } static int @@ -1437,52 +1059,17 @@ zl3073x_dpll_output_pin_phase_adjust_set(const struct dpll_pin *dpll_pin, struct zl3073x_dpll *zldpll = dpll_priv; struct zl3073x_dev *zldev = zldpll->dev; struct zl3073x_dpll_pin *pin = pin_priv; - int half_synth_cycle; - u32 synth_freq; - u8 out, synth; - int rc; + struct zl3073x_out out; + u8 out_id; - /* Get attached synth */ - out = zl3073x_output_pin_out_get(pin->id); - synth = zl3073x_out_synth_get(zldev, out); + out_id = zl3073x_output_pin_out_get(pin->id); + out = *zl3073x_out_state_get(zldev, out_id); - /* Get synth's frequency */ - synth_freq = zl3073x_synth_freq_get(zldev, synth); - - /* Value in register is expressed in half synth clock cycles so - * the given phase adjustment a multiple of half synth clock. - */ - half_synth_cycle = (int)div_u64(PSEC_PER_SEC, 2 * synth_freq); - - if ((phase_adjust % half_synth_cycle) != 0) { - NL_SET_ERR_MSG_FMT(extack, - "Phase adjustment value has to be multiple of %d", - half_synth_cycle); - return -EINVAL; - } - phase_adjust /= half_synth_cycle; - - /* The value in the register is stored as two's complement negation - * of requested value. - */ - phase_adjust = -phase_adjust; - - guard(mutex)(&zldev->multiop_lock); - - /* Read output configuration */ - rc = zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_RD, - ZL_REG_OUTPUT_MB_MASK, BIT(out)); - if (rc) - return rc; - - /* Write the requested value into the compensation register */ - rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_PHASE_COMP, phase_adjust); - if (rc) - return rc; + /* The value in the register is expressed in half synth clock cycles. */ + out.phase_comp = phase_adjust / pin->phase_gran; /* Update output configuration from mailbox */ - return zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_WR, - ZL_REG_OUTPUT_MB_MASK, BIT(out)); + return zl3073x_out_state_set(zldev, out_id, &out); } static int @@ -1758,9 +1345,10 @@ zl3073x_dpll_pin_register(struct zl3073x_dpll_pin *pin, u32 index) if (IS_ERR(props)) return PTR_ERR(props); - /* Save package label & esync capability */ + /* Save package label, esync capability and phase adjust granularity */ strscpy(pin->label, props->package_label); pin->esync_control = props->esync_control; + pin->phase_gran = props->dpll_props.phase_gran; if (zl3073x_dpll_is_input_pin(pin)) { rc = zl3073x_dpll_ref_prio_get(pin, &pin->prio); @@ -1878,33 +1466,32 @@ zl3073x_dpll_pin_is_registrable(struct zl3073x_dpll *zldpll, const char *name; if (dir == DPLL_PIN_DIRECTION_INPUT) { - u8 ref = zl3073x_input_pin_ref_get(index); - - name = "REF"; + u8 ref_id = zl3073x_input_pin_ref_get(index); + const struct zl3073x_ref *ref; /* Skip the pin if the DPLL is running in NCO mode */ if (zldpll->refsel_mode == ZL_DPLL_MODE_REFSEL_MODE_NCO) return false; - is_diff = zl3073x_ref_is_diff(zldev, ref); - is_enabled = zl3073x_ref_is_enabled(zldev, ref); + name = "REF"; + ref = zl3073x_ref_state_get(zldev, ref_id); + is_diff = zl3073x_ref_is_diff(ref); + is_enabled = zl3073x_ref_is_enabled(ref); } else { /* Output P&N pair shares single HW output */ u8 out = zl3073x_output_pin_out_get(index); - name = "OUT"; - /* Skip the pin if it is connected to different DPLL channel */ - if (zl3073x_out_dpll_get(zldev, out) != zldpll->id) { + if (zl3073x_dev_out_dpll_get(zldev, out) != zldpll->id) { dev_dbg(zldev->dev, - "%s%u is driven by different DPLL\n", name, - out); + "OUT%u is driven by different DPLL\n", out); return false; } - is_diff = zl3073x_out_is_diff(zldev, out); - is_enabled = zl3073x_output_pin_is_enabled(zldev, index); + name = "OUT"; + is_diff = zl3073x_dev_out_is_diff(zldev, out); + is_enabled = zl3073x_dev_output_pin_is_enabled(zldev, index); } /* Skip N-pin if the corresponding input/output is differential */ @@ -2061,42 +1648,26 @@ zl3073x_dpll_pin_phase_offset_check(struct zl3073x_dpll_pin *pin) struct zl3073x_dev *zldev = zldpll->dev; unsigned int reg; s64 phase_offset; - u8 ref; + u8 ref_id; int rc; - ref = zl3073x_input_pin_ref_get(pin->id); + /* No phase offset if the ref monitor reports signal errors */ + ref_id = zl3073x_input_pin_ref_get(pin->id); + if (!zl3073x_dev_ref_is_status_ok(zldev, ref_id)) + return false; /* Select register to read phase offset value depending on pin and * phase monitor state: * 1) For connected pin use dpll_phase_err_data register * 2) For other pins use appropriate ref_phase register if the phase - * monitor feature is enabled and reference monitor does not - * report signal errors for given input pin + * monitor feature is enabled. */ - if (pin->pin_state == DPLL_PIN_STATE_CONNECTED) { + if (pin->pin_state == DPLL_PIN_STATE_CONNECTED) reg = ZL_REG_DPLL_PHASE_ERR_DATA(zldpll->id); - } else if (zldpll->phase_monitor) { - u8 status; - - /* Get reference monitor status */ - rc = zl3073x_read_u8(zldev, ZL_REG_REF_MON_STATUS(ref), - &status); - if (rc) { - dev_err(zldev->dev, - "Failed to read %s refmon status: %pe\n", - pin->label, ERR_PTR(rc)); - - return false; - } - - if (status != ZL_REF_MON_STATUS_OK) - return false; - - reg = ZL_REG_REF_PHASE(ref); - } else { - /* The pin is not connected or phase monitor disabled */ + else if (zldpll->phase_monitor) + reg = ZL_REG_REF_PHASE(ref_id); + else return false; - } /* Read measured phase offset value */ rc = zl3073x_read_u48(zldev, reg, &phase_offset); @@ -2135,32 +1706,22 @@ zl3073x_dpll_pin_ffo_check(struct zl3073x_dpll_pin *pin) { struct zl3073x_dpll *zldpll = pin->dpll; struct zl3073x_dev *zldev = zldpll->dev; - u8 ref, status; - s64 ffo; - int rc; + const struct zl3073x_ref *ref; + u8 ref_id; /* Get reference monitor status */ - ref = zl3073x_input_pin_ref_get(pin->id); - rc = zl3073x_read_u8(zldev, ZL_REG_REF_MON_STATUS(ref), &status); - if (rc) { - dev_err(zldev->dev, "Failed to read %s refmon status: %pe\n", - pin->label, ERR_PTR(rc)); - - return false; - } + ref_id = zl3073x_input_pin_ref_get(pin->id); + ref = zl3073x_ref_state_get(zldev, ref_id); /* Do not report ffo changes if the reference monitor report errors */ - if (status != ZL_REF_MON_STATUS_OK) + if (!zl3073x_ref_is_status_ok(ref)) return false; - /* Get the latest measured ref's ffo */ - ffo = zl3073x_ref_ffo_get(zldev, ref); - /* Compare with previous value */ - if (pin->freq_offset != ffo) { + if (pin->freq_offset != ref->ffo) { dev_dbg(zldev->dev, "%s freq offset changed: %lld -> %lld\n", - pin->label, pin->freq_offset, ffo); - pin->freq_offset = ffo; + pin->label, pin->freq_offset, ref->ffo); + pin->freq_offset = ref->ffo; return true; } diff --git a/drivers/dpll/zl3073x/out.c b/drivers/dpll/zl3073x/out.c new file mode 100644 index 0000000000000..86829a0c1c022 --- /dev/null +++ b/drivers/dpll/zl3073x/out.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include + +#include "core.h" +#include "out.h" + +/** + * zl3073x_out_state_fetch - fetch output state from hardware + * @zldev: pointer to zl3073x_dev structure + * @index: output index to fetch state for + * + * Function fetches state of the given output from hardware and stores it + * for later use. + * + * Return: 0 on success, <0 on error + */ +int zl3073x_out_state_fetch(struct zl3073x_dev *zldev, u8 index) +{ + struct zl3073x_out *out = &zldev->out[index]; + int rc; + + /* Read output configuration */ + rc = zl3073x_read_u8(zldev, ZL_REG_OUTPUT_CTRL(index), &out->ctrl); + if (rc) + return rc; + + dev_dbg(zldev->dev, "OUT%u is %s and connected to SYNTH%u\n", index, + str_enabled_disabled(zl3073x_out_is_enabled(out)), + zl3073x_out_synth_get(out)); + + guard(mutex)(&zldev->multiop_lock); + + /* Read output configuration */ + rc = zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_RD, + ZL_REG_OUTPUT_MB_MASK, BIT(index)); + if (rc) + return rc; + + /* Read output mode */ + rc = zl3073x_read_u8(zldev, ZL_REG_OUTPUT_MODE, &out->mode); + if (rc) + return rc; + + dev_dbg(zldev->dev, "OUT%u has signal format 0x%02x\n", index, + zl3073x_out_signal_format_get(out)); + + /* Read output divisor */ + rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_DIV, &out->div); + if (rc) + return rc; + + if (!out->div) { + dev_err(zldev->dev, "Zero divisor for OUT%u got from device\n", + index); + return -EINVAL; + } + + dev_dbg(zldev->dev, "OUT%u divisor: %u\n", index, out->div); + + /* Read output width */ + rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_WIDTH, &out->width); + if (rc) + return rc; + + rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_ESYNC_PERIOD, + &out->esync_n_period); + if (rc) + return rc; + + if (!out->esync_n_period) { + dev_err(zldev->dev, + "Zero esync divisor for OUT%u got from device\n", + index); + return -EINVAL; + } + + rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_ESYNC_WIDTH, + &out->esync_n_width); + if (rc) + return rc; + + rc = zl3073x_read_u32(zldev, ZL_REG_OUTPUT_PHASE_COMP, + &out->phase_comp); + if (rc) + return rc; + + return rc; +} + +/** + * zl3073x_out_state_get - get current output state + * @zldev: pointer to zl3073x_dev structure + * @index: output index to get state for + * + * Return: pointer to given output state + */ +const struct zl3073x_out *zl3073x_out_state_get(struct zl3073x_dev *zldev, + u8 index) +{ + return &zldev->out[index]; +} + +int zl3073x_out_state_set(struct zl3073x_dev *zldev, u8 index, + const struct zl3073x_out *out) +{ + struct zl3073x_out *dout = &zldev->out[index]; + int rc; + + guard(mutex)(&zldev->multiop_lock); + + /* Read output configuration into mailbox */ + rc = zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_RD, + ZL_REG_OUTPUT_MB_MASK, BIT(index)); + if (rc) + return rc; + + /* Update mailbox with changed values */ + if (dout->div != out->div) + rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_DIV, out->div); + if (!rc && dout->width != out->width) + rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_WIDTH, out->width); + if (!rc && dout->esync_n_period != out->esync_n_period) + rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_ESYNC_PERIOD, + out->esync_n_period); + if (!rc && dout->esync_n_width != out->esync_n_width) + rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_ESYNC_WIDTH, + out->esync_n_width); + if (!rc && dout->mode != out->mode) + rc = zl3073x_write_u8(zldev, ZL_REG_OUTPUT_MODE, out->mode); + if (!rc && dout->phase_comp != out->phase_comp) + rc = zl3073x_write_u32(zldev, ZL_REG_OUTPUT_PHASE_COMP, + out->phase_comp); + if (rc) + return rc; + + /* Commit output configuration */ + rc = zl3073x_mb_op(zldev, ZL_REG_OUTPUT_MB_SEM, ZL_OUTPUT_MB_SEM_WR, + ZL_REG_OUTPUT_MB_MASK, BIT(index)); + if (rc) + return rc; + + /* After successful commit store new state */ + dout->div = out->div; + dout->width = out->width; + dout->esync_n_period = out->esync_n_period; + dout->esync_n_width = out->esync_n_width; + dout->mode = out->mode; + dout->phase_comp = out->phase_comp; + + return 0; +} diff --git a/drivers/dpll/zl3073x/out.h b/drivers/dpll/zl3073x/out.h new file mode 100644 index 0000000000000..e8ea7a0e0f071 --- /dev/null +++ b/drivers/dpll/zl3073x/out.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _ZL3073X_OUT_H +#define _ZL3073X_OUT_H + +#include +#include + +#include "regs.h" + +struct zl3073x_dev; + +/** + * struct zl3073x_out - output state + * @div: output divisor + * @width: output pulse width + * @esync_n_period: embedded sync or n-pin period (for n-div formats) + * @esync_n_width: embedded sync or n-pin pulse width + * @phase_comp: phase compensation + * @ctrl: output control + * @mode: output mode + */ +struct zl3073x_out { + u32 div; + u32 width; + u32 esync_n_period; + u32 esync_n_width; + s32 phase_comp; + u8 ctrl; + u8 mode; +}; + +int zl3073x_out_state_fetch(struct zl3073x_dev *zldev, u8 index); +const struct zl3073x_out *zl3073x_out_state_get(struct zl3073x_dev *zldev, + u8 index); + +int zl3073x_out_state_set(struct zl3073x_dev *zldev, u8 index, + const struct zl3073x_out *out); + +/** + * zl3073x_out_signal_format_get - get output signal format + * @out: pointer to out state + * + * Return: signal format of given output + */ +static inline u8 zl3073x_out_signal_format_get(const struct zl3073x_out *out) +{ + return FIELD_GET(ZL_OUTPUT_MODE_SIGNAL_FORMAT, out->mode); +} + +/** + * zl3073x_out_is_diff - check if the given output is differential + * @out: pointer to out state + * + * Return: true if output is differential, false if output is single-ended + */ +static inline bool zl3073x_out_is_diff(const struct zl3073x_out *out) +{ + switch (zl3073x_out_signal_format_get(out)) { + case ZL_OUTPUT_MODE_SIGNAL_FORMAT_LVDS: + case ZL_OUTPUT_MODE_SIGNAL_FORMAT_DIFF: + case ZL_OUTPUT_MODE_SIGNAL_FORMAT_LOWVCM: + return true; + default: + break; + } + + return false; +} + +/** + * zl3073x_out_is_enabled - check if the given output is enabled + * @out: pointer to out state + * + * Return: true if output is enabled, false if output is disabled + */ +static inline bool zl3073x_out_is_enabled(const struct zl3073x_out *out) +{ + return !!FIELD_GET(ZL_OUTPUT_CTRL_EN, out->ctrl); +} + +/** + * zl3073x_out_synth_get - get synth connected to given output + * @out: pointer to out state + * + * Return: index of synth connected to given output. + */ +static inline u8 zl3073x_out_synth_get(const struct zl3073x_out *out) +{ + return FIELD_GET(ZL_OUTPUT_CTRL_SYNTH_SEL, out->ctrl); +} + +#endif /* _ZL3073X_OUT_H */ diff --git a/drivers/dpll/zl3073x/prop.c b/drivers/dpll/zl3073x/prop.c index 4cf7e8aefcb37..4ed153087570b 100644 --- a/drivers/dpll/zl3073x/prop.c +++ b/drivers/dpll/zl3073x/prop.c @@ -46,10 +46,10 @@ zl3073x_pin_check_freq(struct zl3073x_dev *zldev, enum dpll_pin_direction dir, /* Get output pin synthesizer */ out = zl3073x_output_pin_out_get(id); - synth = zl3073x_out_synth_get(zldev, out); + synth = zl3073x_dev_out_synth_get(zldev, out); /* Get synth frequency */ - synth_freq = zl3073x_synth_freq_get(zldev, synth); + synth_freq = zl3073x_dev_synth_freq_get(zldev, synth); /* Check the frequency divides synth frequency */ if (synth_freq % (u32)freq) @@ -93,13 +93,13 @@ zl3073x_prop_pin_package_label_set(struct zl3073x_dev *zldev, prefix = "REF"; ref = zl3073x_input_pin_ref_get(id); - is_diff = zl3073x_ref_is_diff(zldev, ref); + is_diff = zl3073x_dev_ref_is_diff(zldev, ref); } else { u8 out; prefix = "OUT"; out = zl3073x_output_pin_out_get(id); - is_diff = zl3073x_out_is_diff(zldev, out); + is_diff = zl3073x_dev_out_is_diff(zldev, out); } if (!is_diff) @@ -208,7 +208,18 @@ struct zl3073x_pin_props *zl3073x_pin_props_get(struct zl3073x_dev *zldev, DPLL_PIN_CAPABILITIES_PRIORITY_CAN_CHANGE | DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE; } else { + u8 out, synth; + u32 f; + props->dpll_props.type = DPLL_PIN_TYPE_GNSS; + + /* The output pin phase adjustment granularity equals half of + * the synth frequency count. + */ + out = zl3073x_output_pin_out_get(index); + synth = zl3073x_dev_out_synth_get(zldev, out); + f = 2 * zl3073x_dev_synth_freq_get(zldev, synth); + props->dpll_props.phase_gran = f ? div_u64(PSEC_PER_SEC, f) : 1; } props->dpll_props.phase_range.min = S32_MIN; diff --git a/drivers/dpll/zl3073x/ref.c b/drivers/dpll/zl3073x/ref.c new file mode 100644 index 0000000000000..6b65e61039999 --- /dev/null +++ b/drivers/dpll/zl3073x/ref.c @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include + +#include "core.h" +#include "ref.h" + +/** + * zl3073x_ref_freq_factorize - factorize given frequency + * @freq: input frequency + * @base: base frequency + * @mult: multiplier + * + * Checks if the given frequency can be factorized using one of the + * supported base frequencies. If so the base frequency and multiplier + * are stored into appropriate parameters if they are not NULL. + * + * Return: 0 on success, -EINVAL if the frequency cannot be factorized + */ +int +zl3073x_ref_freq_factorize(u32 freq, u16 *base, u16 *mult) +{ + static const u16 base_freqs[] = { + 1, 2, 4, 5, 8, 10, 16, 20, 25, 32, 40, 50, 64, 80, 100, 125, + 128, 160, 200, 250, 256, 320, 400, 500, 625, 640, 800, 1000, + 1250, 1280, 1600, 2000, 2500, 3125, 3200, 4000, 5000, 6250, + 6400, 8000, 10000, 12500, 15625, 16000, 20000, 25000, 31250, + 32000, 40000, 50000, 62500, + }; + u32 div; + int i; + + for (i = 0; i < ARRAY_SIZE(base_freqs); i++) { + div = freq / base_freqs[i]; + + if (div <= U16_MAX && (freq % base_freqs[i]) == 0) { + if (base) + *base = base_freqs[i]; + if (mult) + *mult = div; + + return 0; + } + } + + return -EINVAL; +} + +/** + * zl3073x_ref_state_fetch - fetch input reference state from hardware + * @zldev: pointer to zl3073x_dev structure + * @index: input reference index to fetch state for + * + * Function fetches state for the given input reference from hardware and + * stores it for later use. + * + * Return: 0 on success, <0 on error + */ +int zl3073x_ref_state_fetch(struct zl3073x_dev *zldev, u8 index) +{ + struct zl3073x_ref *ref = &zldev->ref[index]; + int rc; + + /* For differential type inputs the N-pin reference shares + * part of the configuration with the P-pin counterpart. + */ + if (zl3073x_is_n_pin(index) && zl3073x_ref_is_diff(ref - 1)) { + struct zl3073x_ref *p_ref = ref - 1; /* P-pin counterpart*/ + + /* Copy the shared items from the P-pin */ + ref->config = p_ref->config; + ref->esync_n_div = p_ref->esync_n_div; + ref->freq_base = p_ref->freq_base; + ref->freq_mult = p_ref->freq_mult; + ref->freq_ratio_m = p_ref->freq_ratio_m; + ref->freq_ratio_n = p_ref->freq_ratio_n; + ref->phase_comp = p_ref->phase_comp; + ref->sync_ctrl = p_ref->sync_ctrl; + + return 0; /* Finish - no non-shared items for now */ + } + + guard(mutex)(&zldev->multiop_lock); + + /* Read reference configuration */ + rc = zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_RD, + ZL_REG_REF_MB_MASK, BIT(index)); + if (rc) + return rc; + + /* Read ref_config register */ + rc = zl3073x_read_u8(zldev, ZL_REG_REF_CONFIG, &ref->config); + if (rc) + return rc; + + /* Read frequency related registers */ + rc = zl3073x_read_u16(zldev, ZL_REG_REF_FREQ_BASE, &ref->freq_base); + if (rc) + return rc; + rc = zl3073x_read_u16(zldev, ZL_REG_REF_FREQ_MULT, &ref->freq_mult); + if (rc) + return rc; + rc = zl3073x_read_u16(zldev, ZL_REG_REF_RATIO_M, &ref->freq_ratio_m); + if (rc) + return rc; + rc = zl3073x_read_u16(zldev, ZL_REG_REF_RATIO_N, &ref->freq_ratio_n); + if (rc) + return rc; + + /* Read eSync and N-div rated registers */ + rc = zl3073x_read_u32(zldev, ZL_REG_REF_ESYNC_DIV, &ref->esync_n_div); + if (rc) + return rc; + rc = zl3073x_read_u8(zldev, ZL_REG_REF_SYNC_CTRL, &ref->sync_ctrl); + if (rc) + return rc; + + /* Read phase compensation register */ + if (zl3073x_dev_is_ref_phase_comp_32bit(zldev)) { + u32 val; + + rc = zl3073x_read_u32(zldev, ZL_REG_REF_PHASE_OFFSET_COMP_32, + &val); + ref->phase_comp = val; + } else { + rc = zl3073x_read_u48(zldev, ZL_REG_REF_PHASE_OFFSET_COMP, + &ref->phase_comp); + } + if (rc) + return rc; + + dev_dbg(zldev->dev, "REF%u is %s and configured as %s\n", index, + str_enabled_disabled(zl3073x_ref_is_enabled(ref)), + zl3073x_ref_is_diff(ref) ? "differential" : "single-ended"); + + return rc; +} + +/** + * zl3073x_ref_state_get - get current input reference state + * @zldev: pointer to zl3073x_dev structure + * @index: input reference index to get state for + * + * Return: pointer to given input reference state + */ +const struct zl3073x_ref * +zl3073x_ref_state_get(struct zl3073x_dev *zldev, u8 index) +{ + return &zldev->ref[index]; +} + +int zl3073x_ref_state_set(struct zl3073x_dev *zldev, u8 index, + const struct zl3073x_ref *ref) +{ + struct zl3073x_ref *dref = &zldev->ref[index]; + int rc; + + guard(mutex)(&zldev->multiop_lock); + + /* Read reference configuration into mailbox */ + rc = zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_RD, + ZL_REG_REF_MB_MASK, BIT(index)); + if (rc) + return rc; + + /* Update mailbox with changed values */ + if (dref->freq_base != ref->freq_base) + rc = zl3073x_write_u16(zldev, ZL_REG_REF_FREQ_BASE, + ref->freq_base); + if (!rc && dref->freq_mult != ref->freq_mult) + rc = zl3073x_write_u16(zldev, ZL_REG_REF_FREQ_MULT, + ref->freq_mult); + if (!rc && dref->freq_ratio_m != ref->freq_ratio_m) + rc = zl3073x_write_u16(zldev, ZL_REG_REF_RATIO_M, + ref->freq_ratio_m); + if (!rc && dref->freq_ratio_n != ref->freq_ratio_n) + rc = zl3073x_write_u16(zldev, ZL_REG_REF_RATIO_N, + ref->freq_ratio_n); + if (!rc && dref->esync_n_div != ref->esync_n_div) + rc = zl3073x_write_u32(zldev, ZL_REG_REF_ESYNC_DIV, + ref->esync_n_div); + if (!rc && dref->sync_ctrl != ref->sync_ctrl) + rc = zl3073x_write_u8(zldev, ZL_REG_REF_SYNC_CTRL, + ref->sync_ctrl); + if (!rc && dref->phase_comp != ref->phase_comp) { + if (zl3073x_dev_is_ref_phase_comp_32bit(zldev)) + rc = zl3073x_write_u32(zldev, + ZL_REG_REF_PHASE_OFFSET_COMP_32, + ref->phase_comp); + else + rc = zl3073x_write_u48(zldev, + ZL_REG_REF_PHASE_OFFSET_COMP, + ref->phase_comp); + } + if (rc) + return rc; + + /* Commit reference configuration */ + rc = zl3073x_mb_op(zldev, ZL_REG_REF_MB_SEM, ZL_REF_MB_SEM_WR, + ZL_REG_REF_MB_MASK, BIT(index)); + if (rc) + return rc; + + /* After successful commit store new state */ + dref->freq_base = ref->freq_base; + dref->freq_mult = ref->freq_mult; + dref->freq_ratio_m = ref->freq_ratio_m; + dref->freq_ratio_n = ref->freq_ratio_n; + dref->esync_n_div = ref->esync_n_div; + dref->sync_ctrl = ref->sync_ctrl; + dref->phase_comp = ref->phase_comp; + + return 0; +} diff --git a/drivers/dpll/zl3073x/ref.h b/drivers/dpll/zl3073x/ref.h new file mode 100644 index 0000000000000..efc7f59cd9f9c --- /dev/null +++ b/drivers/dpll/zl3073x/ref.h @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _ZL3073X_REF_H +#define _ZL3073X_REF_H + +#include +#include +#include + +#include "regs.h" + +struct zl3073x_dev; + +/** + * struct zl3073x_ref - input reference state + * @ffo: current fractional frequency offset + * @phase_comp: phase compensation + * @esync_n_div: divisor for embedded sync or n-divided signal formats + * @freq_base: frequency base + * @freq_mult: frequnecy multiplier + * @freq_ratio_m: FEC mode multiplier + * @freq_ratio_n: FEC mode divisor + * @config: reference config + * @sync_ctrl: reference sync control + * @mon_status: reference monitor status + */ +struct zl3073x_ref { + s64 ffo; + u64 phase_comp; + u32 esync_n_div; + u16 freq_base; + u16 freq_mult; + u16 freq_ratio_m; + u16 freq_ratio_n; + u8 config; + u8 sync_ctrl; + u8 mon_status; +}; + +int zl3073x_ref_state_fetch(struct zl3073x_dev *zldev, u8 index); + +const struct zl3073x_ref *zl3073x_ref_state_get(struct zl3073x_dev *zldev, + u8 index); + +int zl3073x_ref_state_set(struct zl3073x_dev *zldev, u8 index, + const struct zl3073x_ref *ref); + +int zl3073x_ref_freq_factorize(u32 freq, u16 *base, u16 *mult); + +/** + * zl3073x_ref_ffo_get - get current fractional frequency offset + * @ref: pointer to ref state + * + * Return: the latest measured fractional frequency offset + */ +static inline s64 +zl3073x_ref_ffo_get(const struct zl3073x_ref *ref) +{ + return ref->ffo; +} + +/** + * zl3073x_ref_freq_get - get given input reference frequency + * @ref: pointer to ref state + * + * Return: frequency of the given input reference + */ +static inline u32 +zl3073x_ref_freq_get(const struct zl3073x_ref *ref) +{ + return mul_u64_u32_div(ref->freq_base * ref->freq_mult, + ref->freq_ratio_m, ref->freq_ratio_n); +} + +/** + * zl3073x_ref_freq_set - set given input reference frequency + * @ref: pointer to ref state + * @freq: frequency to be set + * + * Return: 0 on success, <0 when frequency cannot be factorized + */ +static inline int +zl3073x_ref_freq_set(struct zl3073x_ref *ref, u32 freq) +{ + u16 base, mult; + int rc; + + rc = zl3073x_ref_freq_factorize(freq, &base, &mult); + if (rc) + return rc; + + ref->freq_base = base; + ref->freq_mult = mult; + + return 0; +} + +/** + * zl3073x_ref_is_diff - check if the given input reference is differential + * @ref: pointer to ref state + * + * Return: true if reference is differential, false if reference is single-ended + */ +static inline bool +zl3073x_ref_is_diff(const struct zl3073x_ref *ref) +{ + return !!FIELD_GET(ZL_REF_CONFIG_DIFF_EN, ref->config); +} + +/** + * zl3073x_ref_is_enabled - check if the given input reference is enabled + * @ref: pointer to ref state + * + * Return: true if input refernce is enabled, false otherwise + */ +static inline bool +zl3073x_ref_is_enabled(const struct zl3073x_ref *ref) +{ + return !!FIELD_GET(ZL_REF_CONFIG_ENABLE, ref->config); +} + +/** + * zl3073x_ref_is_status_ok - check the given input reference status + * @ref: pointer to ref state + * + * Return: true if the status is ok, false otherwise + */ +static inline bool +zl3073x_ref_is_status_ok(const struct zl3073x_ref *ref) +{ + return ref->mon_status == ZL_REF_MON_STATUS_OK; +} + +#endif /* _ZL3073X_REF_H */ diff --git a/drivers/dpll/zl3073x/regs.h b/drivers/dpll/zl3073x/regs.h index d837bee72b178..5573d7188406b 100644 --- a/drivers/dpll/zl3073x/regs.h +++ b/drivers/dpll/zl3073x/regs.h @@ -194,6 +194,7 @@ #define ZL_REF_CONFIG_DIFF_EN BIT(2) #define ZL_REG_REF_PHASE_OFFSET_COMP ZL_REG(10, 0x28, 6) +#define ZL_REG_REF_PHASE_OFFSET_COMP_32 ZL_REG(10, 0x28, 4) #define ZL_REG_REF_SYNC_CTRL ZL_REG(10, 0x2e, 1) #define ZL_REF_SYNC_CTRL_MODE GENMASK(2, 0) diff --git a/drivers/dpll/zl3073x/synth.c b/drivers/dpll/zl3073x/synth.c new file mode 100644 index 0000000000000..da839572dab26 --- /dev/null +++ b/drivers/dpll/zl3073x/synth.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include + +#include "core.h" +#include "synth.h" + +/** + * zl3073x_synth_state_fetch - fetch synth state from hardware + * @zldev: pointer to zl3073x_dev structure + * @index: synth index to fetch state for + * + * Function fetches state of the given synthesizer from the hardware and + * stores it for later use. + * + * Return: 0 on success, <0 on error + */ +int zl3073x_synth_state_fetch(struct zl3073x_dev *zldev, u8 index) +{ + struct zl3073x_synth *synth = &zldev->synth[index]; + int rc; + + /* Read synth control register */ + rc = zl3073x_read_u8(zldev, ZL_REG_SYNTH_CTRL(index), &synth->ctrl); + if (rc) + return rc; + + guard(mutex)(&zldev->multiop_lock); + + /* Read synth configuration */ + rc = zl3073x_mb_op(zldev, ZL_REG_SYNTH_MB_SEM, ZL_SYNTH_MB_SEM_RD, + ZL_REG_SYNTH_MB_MASK, BIT(index)); + if (rc) + return rc; + + /* The output frequency is determined by the following formula: + * base * multiplier * numerator / denominator + * + * Read registers with these values + */ + rc = zl3073x_read_u16(zldev, ZL_REG_SYNTH_FREQ_BASE, &synth->freq_base); + if (rc) + return rc; + + rc = zl3073x_read_u32(zldev, ZL_REG_SYNTH_FREQ_MULT, &synth->freq_mult); + if (rc) + return rc; + + rc = zl3073x_read_u16(zldev, ZL_REG_SYNTH_FREQ_M, &synth->freq_m); + if (rc) + return rc; + + rc = zl3073x_read_u16(zldev, ZL_REG_SYNTH_FREQ_N, &synth->freq_n); + if (rc) + return rc; + + /* Check denominator for zero to avoid div by 0 */ + if (!synth->freq_n) { + dev_err(zldev->dev, + "Zero divisor for SYNTH%u retrieved from device\n", + index); + return -EINVAL; + } + + dev_dbg(zldev->dev, "SYNTH%u frequency: %u Hz\n", index, + zl3073x_synth_freq_get(synth)); + + return rc; +} + +/** + * zl3073x_synth_state_get - get current synth state + * @zldev: pointer to zl3073x_dev structure + * @index: synth index to get state for + * + * Return: pointer to given synth state + */ +const struct zl3073x_synth *zl3073x_synth_state_get(struct zl3073x_dev *zldev, + u8 index) +{ + return &zldev->synth[index]; +} diff --git a/drivers/dpll/zl3073x/synth.h b/drivers/dpll/zl3073x/synth.h new file mode 100644 index 0000000000000..6c55eb8a888c2 --- /dev/null +++ b/drivers/dpll/zl3073x/synth.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _ZL3073X_SYNTH_H +#define _ZL3073X_SYNTH_H + +#include +#include +#include + +#include "regs.h" + +struct zl3073x_dev; + +/** + * struct zl3073x_synth - synthesizer state + * @freq_mult: frequency multiplier + * @freq_base: frequency base + * @freq_m: frequency numerator + * @freq_n: frequency denominator + * @ctrl: synth control + */ +struct zl3073x_synth { + u32 freq_mult; + u16 freq_base; + u16 freq_m; + u16 freq_n; + u8 ctrl; +}; + +int zl3073x_synth_state_fetch(struct zl3073x_dev *zldev, u8 synth_id); + +const struct zl3073x_synth *zl3073x_synth_state_get(struct zl3073x_dev *zldev, + u8 synth_id); + +int zl3073x_synth_state_set(struct zl3073x_dev *zldev, u8 synth_id, + const struct zl3073x_synth *synth); + +/** + * zl3073x_synth_dpll_get - get DPLL ID the synth is driven by + * @synth: pointer to synth state + * + * Return: ID of DPLL the given synthetizer is driven by + */ +static inline u8 zl3073x_synth_dpll_get(const struct zl3073x_synth *synth) +{ + return FIELD_GET(ZL_SYNTH_CTRL_DPLL_SEL, synth->ctrl); +} + +/** + * zl3073x_synth_freq_get - get synth current freq + * @synth: pointer to synth state + * + * Return: frequency of given synthetizer + */ +static inline u32 zl3073x_synth_freq_get(const struct zl3073x_synth *synth) +{ + return mul_u64_u32_div(synth->freq_base * synth->freq_m, + synth->freq_mult, synth->freq_n); +} + +/** + * zl3073x_synth_is_enabled - check if the given synth is enabled + * @synth: pointer to synth state + * + * Return: true if synth is enabled, false otherwise + */ +static inline bool zl3073x_synth_is_enabled(const struct zl3073x_synth *synth) +{ + return FIELD_GET(ZL_SYNTH_CTRL_EN, synth->ctrl); +} + +#endif /* _ZL3073X_SYNTH_H */ diff --git a/drivers/edac/altera_edac.c b/drivers/edac/altera_edac.c index 0c5b94e64ea15..4edd2088c2db6 100644 --- a/drivers/edac/altera_edac.c +++ b/drivers/edac/altera_edac.c @@ -1563,8 +1563,7 @@ static int altr_portb_setup(struct altr_edac_device_dev *device) goto err_release_group_1; } rc = devm_request_irq(&altdev->ddev, altdev->sb_irq, - prv->ecc_irq_handler, - IRQF_ONESHOT | IRQF_TRIGGER_HIGH, + prv->ecc_irq_handler, IRQF_TRIGGER_HIGH, ecc_name, altdev); if (rc) { edac_printk(KERN_ERR, EDAC_DEVICE, "PortB SBERR IRQ error\n"); @@ -1587,8 +1586,7 @@ static int altr_portb_setup(struct altr_edac_device_dev *device) goto err_release_group_1; } rc = devm_request_irq(&altdev->ddev, altdev->db_irq, - prv->ecc_irq_handler, - IRQF_ONESHOT | IRQF_TRIGGER_HIGH, + prv->ecc_irq_handler, IRQF_TRIGGER_HIGH, ecc_name, altdev); if (rc) { edac_printk(KERN_ERR, EDAC_DEVICE, "PortB DBERR IRQ error\n"); @@ -1970,8 +1968,7 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac, goto err_release_group1; } rc = devm_request_irq(edac->dev, altdev->sb_irq, prv->ecc_irq_handler, - IRQF_ONESHOT | IRQF_TRIGGER_HIGH, - ecc_name, altdev); + IRQF_TRIGGER_HIGH, ecc_name, altdev); if (rc) { edac_printk(KERN_ERR, EDAC_DEVICE, "No SBERR IRQ resource\n"); goto err_release_group1; @@ -1993,7 +1990,7 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac, goto err_release_group1; } rc = devm_request_irq(edac->dev, altdev->db_irq, prv->ecc_irq_handler, - IRQF_ONESHOT | IRQF_TRIGGER_HIGH, + IRQF_TRIGGER_HIGH, ecc_name, altdev); if (rc) { edac_printk(KERN_ERR, EDAC_DEVICE, "No DBERR IRQ resource\n"); diff --git a/drivers/edac/i5000_edac.c b/drivers/edac/i5000_edac.c index 4a1bebc1ff14d..471b8540d18b0 100644 --- a/drivers/edac/i5000_edac.c +++ b/drivers/edac/i5000_edac.c @@ -1111,6 +1111,7 @@ static void calculate_dimm_size(struct i5000_pvt *pvt) n = snprintf(p, space, " "); p += n; + space -= n; for (branch = 0; branch < MAX_BRANCHES; branch++) { n = snprintf(p, space, " branch %d | ", branch); p += n; diff --git a/drivers/edac/i5400_edac.c b/drivers/edac/i5400_edac.c index b5cf25905b059..fb49a1d1df112 100644 --- a/drivers/edac/i5400_edac.c +++ b/drivers/edac/i5400_edac.c @@ -1026,13 +1026,13 @@ static void calculate_dimm_size(struct i5400_pvt *pvt) space -= n; } - space -= n; edac_dbg(2, "%s\n", mem_buffer); p = mem_buffer; space = PAGE_SIZE; n = snprintf(p, space, " "); p += n; + space -= n; for (branch = 0; branch < MAX_BRANCHES; branch++) { n = snprintf(p, space, " branch %d | ", branch); p += n; diff --git a/drivers/edac/igen6_edac.c b/drivers/edac/igen6_edac.c index 2fc59f9eed691..463bbcd484c3c 100644 --- a/drivers/edac/igen6_edac.c +++ b/drivers/edac/igen6_edac.c @@ -246,6 +246,8 @@ static struct work_struct ecclog_work; /* Compute did IDs for Amston Lake with IBECC */ #define DID_ASL_SKU1 0x464a +#define DID_ASL_SKU2 0x4646 +#define DID_ASL_SKU3 0x4652 /* Compute die IDs for Raptor Lake-P with IBECC */ #define DID_RPL_P_SKU1 0xa706 @@ -274,6 +276,16 @@ static struct work_struct ecclog_work; #define DID_PTL_H_SKU1 0xb000 #define DID_PTL_H_SKU2 0xb001 #define DID_PTL_H_SKU3 0xb002 +#define DID_PTL_H_SKU4 0xb003 +#define DID_PTL_H_SKU5 0xb004 +#define DID_PTL_H_SKU6 0xb005 +#define DID_PTL_H_SKU7 0xb008 +#define DID_PTL_H_SKU8 0xb011 +#define DID_PTL_H_SKU9 0xb014 +#define DID_PTL_H_SKU10 0xb015 +#define DID_PTL_H_SKU11 0xb028 +#define DID_PTL_H_SKU12 0xb029 +#define DID_PTL_H_SKU13 0xb02a /* Compute die IDs for Wildcat Lake with IBECC */ #define DID_WCL_SKU1 0xfd00 @@ -618,6 +630,8 @@ static struct pci_device_id igen6_pci_tbl[] = { { PCI_VDEVICE(INTEL, DID_ADL_N_SKU12), (kernel_ulong_t)&adl_n_cfg }, { PCI_VDEVICE(INTEL, DID_AZB_SKU1), (kernel_ulong_t)&adl_n_cfg }, { PCI_VDEVICE(INTEL, DID_ASL_SKU1), (kernel_ulong_t)&adl_n_cfg }, + { PCI_VDEVICE(INTEL, DID_ASL_SKU2), (kernel_ulong_t)&adl_n_cfg }, + { PCI_VDEVICE(INTEL, DID_ASL_SKU3), (kernel_ulong_t)&adl_n_cfg }, { PCI_VDEVICE(INTEL, DID_RPL_P_SKU1), (kernel_ulong_t)&rpl_p_cfg }, { PCI_VDEVICE(INTEL, DID_RPL_P_SKU2), (kernel_ulong_t)&rpl_p_cfg }, { PCI_VDEVICE(INTEL, DID_RPL_P_SKU3), (kernel_ulong_t)&rpl_p_cfg }, @@ -636,6 +650,16 @@ static struct pci_device_id igen6_pci_tbl[] = { { PCI_VDEVICE(INTEL, DID_PTL_H_SKU1), (kernel_ulong_t)&mtl_p_cfg }, { PCI_VDEVICE(INTEL, DID_PTL_H_SKU2), (kernel_ulong_t)&mtl_p_cfg }, { PCI_VDEVICE(INTEL, DID_PTL_H_SKU3), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU4), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU5), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU6), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU7), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU8), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU9), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU10), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU11), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU12), (kernel_ulong_t)&mtl_p_cfg }, + { PCI_VDEVICE(INTEL, DID_PTL_H_SKU13), (kernel_ulong_t)&mtl_p_cfg }, { PCI_VDEVICE(INTEL, DID_WCL_SKU1), (kernel_ulong_t)&wcl_cfg }, { }, }; diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c index c72ee47565856..11a702e7f641c 100644 --- a/drivers/firmware/arm_ffa/driver.c +++ b/drivers/firmware/arm_ffa/driver.c @@ -981,10 +981,27 @@ static void __do_sched_recv_cb(u16 part_id, u16 vcpu, bool is_per_vcpu) } } +/* + * Map logical ID index to the u16 index within the packed ID list. + * + * For native responses (FF-A width == kernel word size), IDs are + * tightly packed: idx -> idx. + * + * For 32-bit responses on a 64-bit kernel, each 64-bit register + * contributes 4 x u16 values but only the lower 2 are defined; the + * upper 2 are garbage. This mapping skips those upper halves: + * 0,1,2,3,4,5,... -> 0,1,4,5,8,9,... + */ +static int list_idx_to_u16_idx(int idx, bool is_native_resp) +{ + return is_native_resp ? idx : idx + 2 * (idx >> 1); +} + static void ffa_notification_info_get(void) { - int idx, list, max_ids, lists_cnt, ids_processed, ids_count[MAX_IDS_64]; - bool is_64b_resp; + int ids_processed, ids_count[MAX_IDS_64]; + int idx, list, max_ids, lists_cnt; + bool is_64b_resp, is_native_resp; ffa_value_t ret; u64 id_list; @@ -1001,6 +1018,7 @@ static void ffa_notification_info_get(void) } is_64b_resp = (ret.a0 == FFA_FN64_SUCCESS); + is_native_resp = (ret.a0 == FFA_FN_NATIVE(SUCCESS)); ids_processed = 0; lists_cnt = FIELD_GET(NOTIFICATION_INFO_GET_ID_COUNT, ret.a2); @@ -1017,12 +1035,16 @@ static void ffa_notification_info_get(void) /* Process IDs */ for (list = 0; list < lists_cnt; list++) { + int u16_idx; u16 vcpu_id, part_id, *packed_id_list = (u16 *)&ret.a3; if (ids_processed >= max_ids - 1) break; - part_id = packed_id_list[ids_processed++]; + u16_idx = list_idx_to_u16_idx(ids_processed, + is_native_resp); + part_id = packed_id_list[u16_idx]; + ids_processed++; if (ids_count[list] == 1) { /* Global Notification */ __do_sched_recv_cb(part_id, 0, false); @@ -1034,7 +1056,10 @@ static void ffa_notification_info_get(void) if (ids_processed >= max_ids - 1) break; - vcpu_id = packed_id_list[ids_processed++]; + u16_idx = list_idx_to_u16_idx(ids_processed, + is_native_resp); + vcpu_id = packed_id_list[u16_idx]; + ids_processed++; __do_sched_recv_cb(part_id, vcpu_id, true); } @@ -2068,6 +2093,7 @@ static int __init ffa_init(void) pr_err("failed to setup partitions\n"); ffa_notifications_cleanup(); + ffa_rxtx_unmap(drv_info->vm_id); free_pages: if (drv_info->tx_buffer) free_pages_exact(drv_info->tx_buffer, rxtx_bufsz); diff --git a/drivers/firmware/efi/cper-arm.c b/drivers/firmware/efi/cper-arm.c index 76542a53e2027..b21cb1232d820 100644 --- a/drivers/firmware/efi/cper-arm.c +++ b/drivers/firmware/efi/cper-arm.c @@ -226,7 +226,8 @@ static void cper_print_arm_err_info(const char *pfx, u32 type, } void cper_print_proc_arm(const char *pfx, - const struct cper_sec_proc_arm *proc) + const struct cper_sec_proc_arm *proc, + u32 length) { int i, len, max_ctx_type; struct cper_arm_err_info *err_info; @@ -238,9 +239,12 @@ void cper_print_proc_arm(const char *pfx, len = proc->section_length - (sizeof(*proc) + proc->err_info_num * (sizeof(*err_info))); - if (len < 0) { - printk("%ssection length: %d\n", pfx, proc->section_length); - printk("%ssection length is too small\n", pfx); + + if (len < 0 || proc->section_length > length) { + printk("%ssection length: %d, CPER size: %d\n", + pfx, proc->section_length, length); + printk("%ssection length is too %s\n", pfx, + (len < 0) ? "small" : "big"); printk("%sfirmware-generated error record is incorrect\n", pfx); printk("%sERR_INFO_NUM is %d\n", pfx, proc->err_info_num); return; diff --git a/drivers/firmware/efi/cper.c b/drivers/firmware/efi/cper.c index 322c6bdefb611..d4dc69693ece4 100644 --- a/drivers/firmware/efi/cper.c +++ b/drivers/firmware/efi/cper.c @@ -560,6 +560,11 @@ static void cper_print_fw_err(const char *pfx, } else { offset = sizeof(*fw_err); } + if (offset > length) { + printk("%s""error section length is too small: offset=%d, length=%d\n", + pfx, offset, length); + return; + } buf += offset; length -= offset; @@ -659,7 +664,8 @@ cper_estatus_print_section(const char *pfx, struct acpi_hest_generic_data *gdata printk("%ssection_type: ARM processor error\n", newpfx); if (gdata->error_data_length >= sizeof(*arm_err)) - cper_print_proc_arm(newpfx, arm_err); + cper_print_proc_arm(newpfx, arm_err, + gdata->error_data_length); else goto err_section_too_small; #endif diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c index fc407d891348f..c3cf5541ed682 100644 --- a/drivers/firmware/efi/efi.c +++ b/drivers/firmware/efi/efi.c @@ -691,13 +691,13 @@ static __init int match_config_table(const efi_guid_t *guid, static __init void reserve_unaccepted(struct efi_unaccepted_memory *unaccepted) { - phys_addr_t start, size; + phys_addr_t start, end; start = PAGE_ALIGN_DOWN(efi.unaccepted); - size = PAGE_ALIGN(sizeof(*unaccepted) + unaccepted->size); + end = PAGE_ALIGN(efi.unaccepted + sizeof(*unaccepted) + unaccepted->size); - memblock_add(start, size); - memblock_reserve(start, size); + memblock_add(start, end - start); + memblock_reserve(start, end - start); } int __init efi_config_parse_tables(const efi_config_table_t *config_tables, diff --git a/drivers/firmware/efi/mokvar-table.c b/drivers/firmware/efi/mokvar-table.c index aedbbd627706a..741674a0a70c5 100644 --- a/drivers/firmware/efi/mokvar-table.c +++ b/drivers/firmware/efi/mokvar-table.c @@ -85,7 +85,7 @@ static struct kobject *mokvar_kobj; * as an alternative to ordinary EFI variables, due to platform-dependent * limitations. The memory occupied by this table is marked as reserved. * - * This routine must be called before efi_free_boot_services() in order + * This routine must be called before efi_unmap_boot_services() in order * to guarantee that it can mark the table as reserved. * * Implicit inputs: diff --git a/drivers/fpga/dfl.c b/drivers/fpga/dfl.c index 7022657243c0a..449c3a082e232 100644 --- a/drivers/fpga/dfl.c +++ b/drivers/fpga/dfl.c @@ -2018,7 +2018,7 @@ static void __exit dfl_fpga_exit(void) bus_unregister(&dfl_bus_type); } -module_init(dfl_fpga_init); +subsys_initcall(dfl_fpga_init); module_exit(dfl_fpga_exit); MODULE_DESCRIPTION("FPGA Device Feature List (DFL) Support"); diff --git a/drivers/fpga/of-fpga-region.c b/drivers/fpga/of-fpga-region.c index 43db4bb77138a..caa091224dc54 100644 --- a/drivers/fpga/of-fpga-region.c +++ b/drivers/fpga/of-fpga-region.c @@ -83,7 +83,7 @@ static struct fpga_manager *of_fpga_region_get_mgr(struct device_node *np) * done with the bridges. * * Return: 0 for success (even if there are no bridges specified) - * or -EBUSY if any of the bridges are in use. + * or an error code if any of the bridges are not available. */ static int of_fpga_region_get_bridges(struct fpga_region *region) { @@ -130,10 +130,10 @@ static int of_fpga_region_get_bridges(struct fpga_region *region) ®ion->bridge_list); of_node_put(br); - /* If any of the bridges are in use, give up */ - if (ret == -EBUSY) { + /* If any of the bridges are not available, give up */ + if (ret) { fpga_bridges_put(®ion->bridge_list); - return -EBUSY; + return ret; } } diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index e053524c5e35f..4d644dcecad93 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -1194,11 +1194,11 @@ config GPIO_PCA953X 8 bits: max7310, max7315, pca6107, pca9534, pca9538, pca9554, pca9556, pca9557, pca9574, tca6408, tca9554, xra1202, - pcal6408, pcal9554b, tca9538 + pcal6408, pcal9554b, tca9538, tcal6408 16 bits: max7312, max7313, pca9535, pca9539, pca9555, pca9575, tca6416, pca6416, pcal6416, pcal9535, pcal9555a, max7318, - tca9539 + tca9539, tcal6416 18 bits: tca6418 diff --git a/drivers/gpio/gpio-amd-fch.c b/drivers/gpio/gpio-amd-fch.c index e6c6c3ec7656e..9f329938202bf 100644 --- a/drivers/gpio/gpio-amd-fch.c +++ b/drivers/gpio/gpio-amd-fch.c @@ -8,6 +8,7 @@ * */ +#include #include #include #include @@ -120,15 +121,15 @@ static int amd_fch_gpio_get(struct gpio_chip *gc, unsigned int offset) { unsigned long flags; - int ret; + u32 val; struct amd_fch_gpio_priv *priv = gpiochip_get_data(gc); void __iomem *ptr = amd_fch_gpio_addr(priv, offset); spin_lock_irqsave(&priv->lock, flags); - ret = (readl_relaxed(ptr) & AMD_FCH_GPIO_FLAG_READ); + val = readl_relaxed(ptr); spin_unlock_irqrestore(&priv->lock, flags); - return ret; + return FIELD_GET(AMD_FCH_GPIO_FLAG_READ, val); } static int amd_fch_gpio_request(struct gpio_chip *chip, diff --git a/drivers/gpio/gpio-aspeed-sgpio.c b/drivers/gpio/gpio-aspeed-sgpio.c index 7622f9e9f54af..318cd0e397416 100644 --- a/drivers/gpio/gpio-aspeed-sgpio.c +++ b/drivers/gpio/gpio-aspeed-sgpio.c @@ -516,7 +516,7 @@ static const struct of_device_id aspeed_sgpio_of_table[] = { MODULE_DEVICE_TABLE(of, aspeed_sgpio_of_table); -static int __init aspeed_sgpio_probe(struct platform_device *pdev) +static int aspeed_sgpio_probe(struct platform_device *pdev) { u32 nr_gpios, sgpio_freq, sgpio_clk_div, gpio_cnt_regval, pin_mask; const struct aspeed_sgpio_pdata *pdata; @@ -611,11 +611,12 @@ static int __init aspeed_sgpio_probe(struct platform_device *pdev) } static struct platform_driver aspeed_sgpio_driver = { + .probe = aspeed_sgpio_probe, .driver = { .name = KBUILD_MODNAME, .of_match_table = aspeed_sgpio_of_table, }, }; -module_platform_driver_probe(aspeed_sgpio_driver, aspeed_sgpio_probe); +module_platform_driver(aspeed_sgpio_driver); MODULE_DESCRIPTION("Aspeed Serial GPIO Driver"); diff --git a/drivers/gpio/gpio-nomadik.c b/drivers/gpio/gpio-nomadik.c index 97c5cd33279d5..e22b713166d71 100644 --- a/drivers/gpio/gpio-nomadik.c +++ b/drivers/gpio/gpio-nomadik.c @@ -430,6 +430,9 @@ void nmk_gpio_dbg_show_one(struct seq_file *s, struct pinctrl_dev *pctldev, #ifdef CONFIG_PINCTRL_NOMADIK if (mode == NMK_GPIO_ALT_C && pctldev) { desc = gpio_device_get_desc(chip->gpiodev, offset); + if (IS_ERR(desc)) + return; + mode = nmk_prcm_gpiocr_get_mode(pctldev, desc_to_gpio(desc)); } #endif diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c index 34cadce9b3101..f7daeadaea57c 100644 --- a/drivers/gpio/gpio-pca953x.c +++ b/drivers/gpio/gpio-pca953x.c @@ -126,6 +126,9 @@ static const struct i2c_device_id pca953x_id[] = { { "tca9539", 16 | PCA953X_TYPE | PCA_INT, }, { "tca9554", 8 | PCA953X_TYPE | PCA_INT, }, { "xra1202", 8 | PCA953X_TYPE }, + + { "tcal6408", 8 | PCA953X_TYPE | PCA_LATCH_INT, }, + { "tcal6416", 16 | PCA953X_TYPE | PCA_LATCH_INT, }, { } }; MODULE_DEVICE_TABLE(i2c, pca953x_id); @@ -1466,6 +1469,9 @@ static const struct of_device_id pca953x_dt_ids[] = { { .compatible = "ti,tca9538", .data = OF_953X( 8, PCA_INT), }, { .compatible = "ti,tca9539", .data = OF_953X(16, PCA_INT), }, + { .compatible = "ti,tcal6408", .data = OF_953X( 8, PCA_LATCH_INT), }, + { .compatible = "ti,tcal6416", .data = OF_953X(16, PCA_LATCH_INT), }, + { .compatible = "onnn,cat9554", .data = OF_953X( 8, PCA_INT), }, { .compatible = "onnn,pca9654", .data = OF_953X( 8, PCA_INT), }, { .compatible = "onnn,pca9655", .data = OF_953X(16, PCA_INT), }, diff --git a/drivers/gpio/gpio-sprd.c b/drivers/gpio/gpio-sprd.c index 413bcd0a42405..2cc8abe705cdb 100644 --- a/drivers/gpio/gpio-sprd.c +++ b/drivers/gpio/gpio-sprd.c @@ -35,7 +35,7 @@ struct sprd_gpio { struct gpio_chip chip; void __iomem *base; - spinlock_t lock; + raw_spinlock_t lock; int irq; }; @@ -54,7 +54,7 @@ static void sprd_gpio_update(struct gpio_chip *chip, unsigned int offset, unsigned long flags; u32 tmp; - spin_lock_irqsave(&sprd_gpio->lock, flags); + raw_spin_lock_irqsave(&sprd_gpio->lock, flags); tmp = readl_relaxed(base + reg); if (val) @@ -63,7 +63,7 @@ static void sprd_gpio_update(struct gpio_chip *chip, unsigned int offset, tmp &= ~BIT(SPRD_GPIO_BIT(offset)); writel_relaxed(tmp, base + reg); - spin_unlock_irqrestore(&sprd_gpio->lock, flags); + raw_spin_unlock_irqrestore(&sprd_gpio->lock, flags); } static int sprd_gpio_read(struct gpio_chip *chip, unsigned int offset, u16 reg) @@ -236,7 +236,7 @@ static int sprd_gpio_probe(struct platform_device *pdev) if (IS_ERR(sprd_gpio->base)) return PTR_ERR(sprd_gpio->base); - spin_lock_init(&sprd_gpio->lock); + raw_spin_lock_init(&sprd_gpio->lock); sprd_gpio->chip.label = dev_name(&pdev->dev); sprd_gpio->chip.ngpio = SPRD_GPIO_NR; diff --git a/drivers/gpio/gpiolib-acpi-core.c b/drivers/gpio/gpiolib-acpi-core.c index e64e21fd6bbaa..8110690ea69d0 100644 --- a/drivers/gpio/gpiolib-acpi-core.c +++ b/drivers/gpio/gpiolib-acpi-core.c @@ -1359,6 +1359,7 @@ static int acpi_gpio_package_count(const union acpi_object *obj) while (element < end) { switch (element->type) { case ACPI_TYPE_LOCAL_REFERENCE: + case ACPI_TYPE_STRING: element += 3; fallthrough; case ACPI_TYPE_INTEGER: diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c index 7d5fc1ea2aa54..e044690ad412b 100644 --- a/drivers/gpio/gpiolib-sysfs.c +++ b/drivers/gpio/gpiolib-sysfs.c @@ -919,63 +919,68 @@ int gpiod_export_link(struct device *dev, const char *name, } EXPORT_SYMBOL_GPL(gpiod_export_link); -/** - * gpiod_unexport - reverse effect of gpiod_export() - * @desc: GPIO to make unavailable - * - * This is implicit on gpiod_free(). - */ -void gpiod_unexport(struct gpio_desc *desc) +static void gpiod_unexport_unlocked(struct gpio_desc *desc) { struct gpiod_data *tmp, *desc_data = NULL; struct gpiodev_data *gdev_data; struct gpio_device *gdev; - if (!desc) { - pr_warn("%s: invalid GPIO\n", __func__); + if (!test_bit(GPIOD_FLAG_EXPORT, &desc->flags)) return; - } - scoped_guard(mutex, &sysfs_lock) { - if (!test_bit(GPIOD_FLAG_EXPORT, &desc->flags)) - return; - - gdev = gpiod_to_gpio_device(desc); - gdev_data = gdev_get_data(gdev); - if (!gdev_data) - return; + gdev = gpiod_to_gpio_device(desc); + gdev_data = gdev_get_data(gdev); + if (!gdev_data) + return; - list_for_each_entry(tmp, &gdev_data->exported_lines, list) { - if (gpiod_is_equal(desc, tmp->desc)) { - desc_data = tmp; - break; - } + list_for_each_entry(tmp, &gdev_data->exported_lines, list) { + if (gpiod_is_equal(desc, tmp->desc)) { + desc_data = tmp; + break; } + } - if (!desc_data) - return; + if (!desc_data) + return; - list_del(&desc_data->list); - clear_bit(GPIOD_FLAG_EXPORT, &desc->flags); + list_del(&desc_data->list); + clear_bit(GPIOD_FLAG_EXPORT, &desc->flags); #if IS_ENABLED(CONFIG_GPIO_SYSFS_LEGACY) - sysfs_put(desc_data->value_kn); - device_unregister(desc_data->dev); - - /* - * Release irq after deregistration to prevent race with - * edge_store. - */ - if (desc_data->irq_flags) - gpio_sysfs_free_irq(desc_data); + sysfs_put(desc_data->value_kn); + device_unregister(desc_data->dev); + + /* + * Release irq after deregistration to prevent race with + * edge_store. + */ + if (desc_data->irq_flags) + gpio_sysfs_free_irq(desc_data); #endif /* CONFIG_GPIO_SYSFS_LEGACY */ - sysfs_remove_groups(desc_data->parent, - desc_data->chip_attr_groups); - } + sysfs_remove_groups(desc_data->parent, + desc_data->chip_attr_groups); mutex_destroy(&desc_data->mutex); kfree(desc_data); } + +/** + * gpiod_unexport - reverse effect of gpiod_export() + * @desc: GPIO to make unavailable + * + * This is implicit on gpiod_free(). + */ +void gpiod_unexport(struct gpio_desc *desc) +{ + if (!desc) { + pr_warn("%s: invalid GPIO\n", __func__); + return; + } + + guard(mutex)(&sysfs_lock); + + gpiod_unexport_unlocked(desc); +} EXPORT_SYMBOL_GPL(gpiod_unexport); int gpiochip_sysfs_register(struct gpio_device *gdev) @@ -1054,29 +1059,28 @@ void gpiochip_sysfs_unregister(struct gpio_device *gdev) struct gpio_desc *desc; struct gpio_chip *chip; - scoped_guard(mutex, &sysfs_lock) { - data = gdev_get_data(gdev); - if (!data) - return; + guard(mutex)(&sysfs_lock); -#if IS_ENABLED(CONFIG_GPIO_SYSFS_LEGACY) - device_unregister(data->cdev_base); -#endif /* CONFIG_GPIO_SYSFS_LEGACY */ - device_unregister(data->cdev_id); - kfree(data); - } + data = gdev_get_data(gdev); + if (!data) + return; guard(srcu)(&gdev->srcu); - chip = srcu_dereference(gdev->chip, &gdev->srcu); if (!chip) return; /* unregister gpiod class devices owned by sysfs */ for_each_gpio_desc_with_flag(chip, desc, GPIOD_FLAG_SYSFS) { - gpiod_unexport(desc); + gpiod_unexport_unlocked(desc); gpiod_free(desc); } + +#if IS_ENABLED(CONFIG_GPIO_SYSFS_LEGACY) + device_unregister(data->cdev_base); +#endif /* CONFIG_GPIO_SYSFS_LEGACY */ + device_unregister(data->cdev_id); + kfree(data); } /* diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 6f5b4a0e0a343..803b6fc360a07 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -274,6 +274,8 @@ extern int amdgpu_rebar; extern int amdgpu_wbrf; extern int amdgpu_user_queue; +extern uint amdgpu_hdmi_hpd_debounce_delay_ms; + #define AMDGPU_VM_MAX_NUM_CTX 4096 #define AMDGPU_SG_THRESHOLD (256*1024*1024) #define AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS 3000 diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.c index 9b31804491500..3f9b094e93a29 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.c @@ -641,6 +641,7 @@ static void aca_error_fini(struct aca_error *aerr) aca_bank_error_remove(aerr, bank_error); out_unlock: + mutex_unlock(&aerr->lock); mutex_destroy(&aerr->lock); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c index 6c62e27b98002..67db986eda3f6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c @@ -1136,8 +1136,10 @@ static int amdgpu_acpi_enumerate_xcc(void) if (!dev_info) ret = amdgpu_acpi_dev_init(&dev_info, xcc_info, sbdf); - if (ret == -ENOMEM) + if (ret == -ENOMEM) { + kfree(xcc_info); return ret; + } if (!dev_info) { kfree(xcc_info); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 53b33a636971a..dbcd55611a37d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -3280,7 +3280,8 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev) if (r) goto init_failed; - if (adev->mman.buffer_funcs_ring->sched.ready) + if (adev->mman.buffer_funcs_ring && + adev->mman.buffer_funcs_ring->sched.ready) amdgpu_ttm_set_buffer_funcs_status(adev, true); /* Don't init kfd if whole hive need to be reset during init */ @@ -3629,9 +3630,6 @@ static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev) } } - amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); - amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); - amdgpu_amdkfd_suspend(adev, true); amdgpu_userq_suspend(adev); @@ -4960,6 +4958,9 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) amdgpu_virt_fini_data_exchange(adev); } + amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); + amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); + /* disable all interrupts */ amdgpu_irq_disable_all(adev); if (adev->mode_info.mode_config_initialized) { @@ -4990,7 +4991,7 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) * before ip_fini_early to prevent kfd locking refcount issues by calling * amdgpu_amdkfd_suspend() */ - if (drm_dev_is_unplugged(adev_to_drm(adev))) + if (pci_dev_is_disconnected(adev->pdev)) amdgpu_amdkfd_device_fini_sw(adev); amdgpu_device_ip_fini_early(adev); @@ -5002,7 +5003,7 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) amdgpu_gart_dummy_page_fini(adev); - if (drm_dev_is_unplugged(adev_to_drm(adev))) + if (pci_dev_is_disconnected(adev->pdev)) amdgpu_device_unmap_mmio(adev); } @@ -7061,6 +7062,15 @@ pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev) dev_info(adev->dev, "PCI error: slot reset callback!!\n"); memset(&reset_context, 0, sizeof(reset_context)); + INIT_LIST_HEAD(&device_list); + hive = amdgpu_get_xgmi_hive(adev); + if (hive) { + mutex_lock(&hive->hive_lock); + list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) + list_add_tail(&tmp_adev->reset_list, &device_list); + } else { + list_add_tail(&adev->reset_list, &device_list); + } if (adev->pcie_reset_ctx.swus) link_dev = adev->pcie_reset_ctx.swus; @@ -7101,19 +7111,13 @@ pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev) reset_context.reset_req_dev = adev; set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags); - INIT_LIST_HEAD(&device_list); - hive = amdgpu_get_xgmi_hive(adev); if (hive) { - mutex_lock(&hive->hive_lock); reset_context.hive = hive; - list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { + list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) tmp_adev->pcie_reset_ctx.in_link_reset = true; - list_add_tail(&tmp_adev->reset_list, &device_list); - } } else { set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags); - list_add_tail(&adev->reset_list, &device_list); } r = amdgpu_device_asic_reset(adev, &device_list, &reset_context); @@ -7353,6 +7357,9 @@ void amdgpu_device_halt(struct amdgpu_device *adev) amdgpu_xcp_dev_unplug(adev); drm_dev_unplug(ddev); + amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); + amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); + amdgpu_irq_disable_all(adev); amdgpu_fence_driver_hw_fini(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c index dd7b2b796427c..54a045a0bda96 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c @@ -2995,6 +2995,7 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev) case IP_VERSION(6, 0, 0): case IP_VERSION(6, 0, 1): case IP_VERSION(6, 1, 0): + case IP_VERSION(6, 1, 1): adev->hdp.funcs = &hdp_v6_0_funcs; break; case IP_VERSION(7, 0, 0): diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c index ed3bef1edfe44..f20d4bc58904d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c @@ -489,8 +489,15 @@ amdgpu_dma_buf_move_notify(struct dma_buf_attachment *attach) r = dma_resv_reserve_fences(resv, 2); if (!r) r = amdgpu_vm_clear_freed(adev, vm, NULL); + + /* Don't pass 'ticket' to amdgpu_vm_handle_moved: we want the clear=true + * path to be used otherwise we might update the PT of another process + * while it's using the BO. + * With clear=true, amdgpu_vm_bo_update will sync to command submission + * from the same VM. + */ if (!r) - r = amdgpu_vm_handle_moved(adev, vm, ticket); + r = amdgpu_vm_handle_moved(adev, vm, NULL); if (r && r != -EBUSY) DRM_ERROR("Failed to invalidate VM page tables (%d))\n", diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index ec9516d6ae976..335f7e2f4ce55 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -246,6 +246,7 @@ int amdgpu_damage_clips = -1; /* auto */ int amdgpu_umsch_mm_fwlog; int amdgpu_rebar = -1; /* auto */ int amdgpu_user_queue = -1; +uint amdgpu_hdmi_hpd_debounce_delay_ms; DECLARE_DYNDBG_CLASSMAP(drm_debug_classes, DD_CLASS_TYPE_DISJOINT_BITS, 0, "DRM_UT_CORE", @@ -1128,6 +1129,16 @@ module_param_named(rebar, amdgpu_rebar, int, 0444); MODULE_PARM_DESC(user_queue, "Enable user queues (-1 = auto (default), 0 = disable, 1 = enable, 2 = enable UQs and disable KQs)"); module_param_named(user_queue, amdgpu_user_queue, int, 0444); +/* + * DOC: hdmi_hpd_debounce_delay_ms (uint) + * HDMI HPD disconnect debounce delay in milliseconds. + * + * Used to filter short disconnect->reconnect HPD toggles some HDMI sinks + * generate while entering/leaving power save. Set to 0 to disable by default. + */ +MODULE_PARM_DESC(hdmi_hpd_debounce_delay_ms, "HDMI HPD disconnect debounce delay in milliseconds (0 to disable (by default), 1500 is common)"); +module_param_named(hdmi_hpd_debounce_delay_ms, amdgpu_hdmi_hpd_debounce_delay_ms, uint, 0644); + /* These devices are not supported by amdgpu. * They are supported by the mach64, r128, radeon drivers */ @@ -3155,7 +3166,6 @@ static int __init amdgpu_init(void) if (r) goto error_fence; - DRM_INFO("amdgpu kernel modesetting enabled.\n"); amdgpu_register_atpx_handler(); amdgpu_acpi_detect(); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index b7ebae289beab..1631a0431ea8e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -112,47 +112,6 @@ amdgpu_gem_update_timeline_node(struct drm_file *filp, return 0; } -static void -amdgpu_gem_update_bo_mapping(struct drm_file *filp, - struct amdgpu_bo_va *bo_va, - uint32_t operation, - uint64_t point, - struct dma_fence *fence, - struct drm_syncobj *syncobj, - struct dma_fence_chain *chain) -{ - struct amdgpu_bo *bo = bo_va ? bo_va->base.bo : NULL; - struct amdgpu_fpriv *fpriv = filp->driver_priv; - struct amdgpu_vm *vm = &fpriv->vm; - struct dma_fence *last_update; - - if (!syncobj) - return; - - /* Find the last update fence */ - switch (operation) { - case AMDGPU_VA_OP_MAP: - case AMDGPU_VA_OP_REPLACE: - if (bo && (bo->tbo.base.resv == vm->root.bo->tbo.base.resv)) - last_update = vm->last_update; - else - last_update = bo_va->last_pt_update; - break; - case AMDGPU_VA_OP_UNMAP: - case AMDGPU_VA_OP_CLEAR: - last_update = fence; - break; - default: - return; - } - - /* Add fence to timeline */ - if (!point) - drm_syncobj_replace_fence(syncobj, last_update); - else - drm_syncobj_add_point(syncobj, chain, last_update, point); -} - static vm_fault_t amdgpu_gem_fault(struct vm_fault *vmf) { struct ttm_buffer_object *bo = vmf->vma->vm_private_data; @@ -761,16 +720,27 @@ amdgpu_gem_va_update_vm(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va, uint32_t operation) { - struct dma_fence *fence = dma_fence_get_stub(); - int r; + struct dma_fence *fence; + int r = 0; + + /* Always start from the VM's existing last update fence. */ + fence = dma_fence_get(vm->last_update); if (!amdgpu_vm_ready(vm)) return fence; + /* + * First clean up any freed mappings in the VM. + * + * amdgpu_vm_clear_freed() may replace @fence with a new fence if it + * schedules GPU work. If nothing needs clearing, @fence can remain as + * the original vm->last_update. + */ r = amdgpu_vm_clear_freed(adev, vm, &fence); if (r) goto error; + /* For MAP/REPLACE we also need to update the BO mappings. */ if (operation == AMDGPU_VA_OP_MAP || operation == AMDGPU_VA_OP_REPLACE) { r = amdgpu_vm_bo_update(adev, bo_va, false); @@ -778,7 +748,46 @@ amdgpu_gem_va_update_vm(struct amdgpu_device *adev, goto error; } + /* Always update PDEs after we touched the mappings. */ r = amdgpu_vm_update_pdes(adev, vm, false); + if (r) + goto error; + + /* + * Decide which fence best represents the last update: + * + * MAP/REPLACE: + * - For always-valid mappings, use vm->last_update. + * - Otherwise, export bo_va->last_pt_update. + * + * UNMAP/CLEAR: + * Keep the fence returned by amdgpu_vm_clear_freed(). If no work was + * needed, it can remain as vm->last_pt_update. + * + * The VM and BO update fences are always initialized to a valid value. + * vm->last_update and bo_va->last_pt_update always start as valid fences. + * and are never expected to be NULL. + */ + switch (operation) { + case AMDGPU_VA_OP_MAP: + case AMDGPU_VA_OP_REPLACE: + /* + * For MAP/REPLACE, return the page table update fence for the + * mapping we just modified. bo_va is expected to be valid here. + */ + dma_fence_put(fence); + + if (amdgpu_vm_is_bo_always_valid(vm, bo_va->base.bo)) + fence = dma_fence_get(vm->last_update); + else + fence = dma_fence_get(bo_va->last_pt_update); + break; + case AMDGPU_VA_OP_UNMAP: + case AMDGPU_VA_OP_CLEAR: + default: + /* keep @fence as returned by amdgpu_vm_clear_freed() */ + break; + } error: if (r && r != -ERESTARTSYS) @@ -810,6 +819,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, uint64_t vm_size; int r = 0; + /* Validate virtual address range against reserved regions. */ if (args->va_address < AMDGPU_VA_RESERVED_BOTTOM) { dev_dbg(dev->dev, "va_address 0x%llx is in reserved area 0x%llx\n", @@ -843,6 +853,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, return -EINVAL; } + /* Validate operation type. */ switch (args->operation) { case AMDGPU_VA_OP_MAP: case AMDGPU_VA_OP_UNMAP: @@ -866,6 +877,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, abo = NULL; } + /* Add input syncobj fences (if any) for synchronization. */ r = amdgpu_gem_add_input_fence(filp, args->input_fence_syncobj_handles, args->num_syncobj_handles); @@ -888,6 +900,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, goto error; } + /* Resolve the BO-VA mapping for this VM/BO combination. */ if (abo) { bo_va = amdgpu_vm_bo_find(&fpriv->vm, abo); if (!bo_va) { @@ -900,6 +913,11 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, bo_va = NULL; } + /* + * Prepare the timeline syncobj node if the user requested a VM + * timeline update. This only allocates/looks up the syncobj and + * chain node; the actual fence is attached later. + */ r = amdgpu_gem_update_timeline_node(filp, args->vm_timeline_syncobj_out, args->vm_timeline_point, @@ -931,18 +949,30 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, default: break; } + + /* + * Once the VA operation is done, update the VM and obtain the fence + * that represents the last relevant update for this mapping. This + * fence can then be exported to the user-visible VM timeline. + */ if (!r && !(args->flags & AMDGPU_VM_DELAY_UPDATE) && !adev->debug_vm) { fence = amdgpu_gem_va_update_vm(adev, &fpriv->vm, bo_va, args->operation); - if (timeline_syncobj) - amdgpu_gem_update_bo_mapping(filp, bo_va, - args->operation, - args->vm_timeline_point, - fence, timeline_syncobj, - timeline_chain); - else - dma_fence_put(fence); + if (timeline_syncobj && fence) { + if (!args->vm_timeline_point) { + /* Replace the existing fence when no point is given. */ + drm_syncobj_replace_fence(timeline_syncobj, + fence); + } else { + /* Attach the last-update fence at a specific point. */ + drm_syncobj_add_point(timeline_syncobj, + timeline_chain, + fence, + args->vm_timeline_point); + } + } + dma_fence_put(fence); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c index aef1ba1bdca9e..1e3b8a506d1b9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c @@ -1013,6 +1013,16 @@ void amdgpu_gmc_get_vbios_allocations(struct amdgpu_device *adev) case CHIP_RENOIR: adev->mman.keep_stolen_vga_memory = true; break; + case CHIP_POLARIS10: + case CHIP_POLARIS11: + case CHIP_POLARIS12: + /* MacBookPros with switchable graphics put VRAM at 0 when + * the iGPU is enabled which results in cursor issues if + * the cursor ends up at 0. Reserve vram at 0 in that case. + */ + if (adev->gmc.vram_start == 0) + adev->mman.keep_stolen_vga_memory = true; + break; default: adev->mman.keep_stolen_vga_memory = false; break; @@ -1381,7 +1391,7 @@ int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev, if (!*exp_ranges) *exp_ranges = range_cnt; err: - kfree(ranges); + kvfree(ranges); return ret; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c index d020a890a0ea4..ffc720d189007 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c @@ -130,11 +130,9 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job) } /* attempt a per ring reset */ - if (unlikely(adev->debug_disable_gpu_ring_reset)) { - dev_err(adev->dev, "Ring reset disabled by debug mask\n"); - } else if (amdgpu_gpu_recovery && - amdgpu_ring_is_reset_type_supported(ring, AMDGPU_RESET_TYPE_PER_QUEUE) && - ring->funcs->reset) { + if (amdgpu_gpu_recovery && + amdgpu_ring_is_reset_type_supported(ring, AMDGPU_RESET_TYPE_PER_QUEUE) && + ring->funcs->reset) { dev_err(adev->dev, "Starting %s ring reset\n", s_job->sched->name); r = amdgpu_ring_reset(ring, job->vmid, &job->hw_fence); @@ -149,7 +147,8 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job) dev_err(adev->dev, "Ring %s reset failed\n", ring->sched.name); } - dma_fence_set_error(&s_job->s_fence->finished, -ETIME); + if (dma_fence_get_status(&s_job->s_fence->finished) == 0) + dma_fence_set_error(&s_job->s_fence->finished, -ETIME); if (amdgpu_device_should_recover_gpu(ring->adev)) { struct amdgpu_reset_context reset_context; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index aa7987d0806c6..5f7aa840b2151 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -726,7 +726,7 @@ psp_cmd_submit_buf(struct psp_context *psp, ras_intr = amdgpu_ras_intr_triggered(); if (ras_intr) break; - usleep_range(10, 100); + usleep_range(60, 100); amdgpu_device_invalidate_hdp(psp->adev, NULL); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c index 6e8aad91bcd30..0d3c18f04ac36 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c @@ -332,13 +332,13 @@ static ssize_t ta_if_invoke_debugfs_write(struct file *fp, const char *buf, size if (!context || !context->initialized) { dev_err(adev->dev, "TA is not initialized\n"); ret = -EINVAL; - goto err_free_shared_buf; + goto free_shared_buf; } if (!psp->ta_funcs || !psp->ta_funcs->fn_ta_invoke) { dev_err(adev->dev, "Unsupported function to invoke TA\n"); ret = -EOPNOTSUPP; - goto err_free_shared_buf; + goto free_shared_buf; } context->session_id = ta_id; @@ -346,7 +346,7 @@ static ssize_t ta_if_invoke_debugfs_write(struct file *fp, const char *buf, size mutex_lock(&psp->ras_context.mutex); ret = prep_ta_mem_context(&context->mem_context, shared_buf, shared_buf_len); if (ret) - goto err_free_shared_buf; + goto unlock; ret = psp_fn_ta_invoke(psp, cmd_id); if (ret || context->resp_status) { @@ -354,15 +354,17 @@ static ssize_t ta_if_invoke_debugfs_write(struct file *fp, const char *buf, size ret, context->resp_status); if (!ret) { ret = -EINVAL; - goto err_free_shared_buf; + goto unlock; } } if (copy_to_user((char *)&buf[copy_pos], context->mem_context.shared_buf, shared_buf_len)) ret = -EFAULT; -err_free_shared_buf: +unlock: mutex_unlock(&psp->ras_context.mutex); + +free_shared_buf: kfree(shared_buf); return ret; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c index e0ee211508607..e9ed1a3b135cc 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c @@ -2892,6 +2892,11 @@ static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev, struct ras_err_handler_data *data = con->eh_data; for (j = 0; j < count; j++) { + if (!data->space_left && + amdgpu_ras_realloc_eh_data_space(adev, data, 256)) { + return -ENOMEM; + } + if (amdgpu_ras_check_bad_page_unlock(con, bps[j].retired_page << AMDGPU_GPU_PAGE_SHIFT)) { data->count++; @@ -2899,11 +2904,6 @@ static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev, continue; } - if (!data->space_left && - amdgpu_ras_realloc_eh_data_space(adev, data, 256)) { - return -ENOMEM; - } - amdgpu_ras_reserve_page(adev, bps[j].retired_page); memcpy(&data->bps[data->count], &(bps[j]), @@ -3056,8 +3056,6 @@ int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, /* deal with retire_unit records a time */ ret = __amdgpu_ras_convert_rec_array_from_rom(adev, &bps[i], &err_data, nps); - if (ret) - con->bad_page_num -= adev->umc.retire_unit; i += (adev->umc.retire_unit - 1); } else { break; @@ -3070,8 +3068,6 @@ int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, for (; i < pages; i++) { ret = __amdgpu_ras_convert_rec_from_rom(adev, &bps[i], &err_data, nps); - if (ret) - con->bad_page_num -= adev->umc.retire_unit; } con->eh_data->count_saved = con->eh_data->count; @@ -4137,7 +4133,7 @@ int amdgpu_ras_init(struct amdgpu_device *adev) * to handle fatal error */ r = amdgpu_nbio_ras_sw_init(adev); if (r) - return r; + goto release_con; if (adev->nbio.ras && adev->nbio.ras->init_ras_controller_interrupt) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c index 3eb3fb55ccb05..dafa46a9656ca 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c @@ -1475,10 +1475,12 @@ int amdgpu_ras_eeprom_check(struct amdgpu_ras_eeprom_control *control) } res = __verify_ras_table_checksum(control); - if (res) + if (res) { dev_err(adev->dev, "RAS table incorrect checksum or error:%d\n", res); + return -EINVAL; + } /* Warn if we are at 90% of the threshold or above */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c index 5ec5c3ff22bb0..304564ec2f59a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c @@ -460,9 +460,6 @@ bool amdgpu_ring_soft_recovery(struct amdgpu_ring *ring, unsigned int vmid, ktime_t deadline; bool ret; - if (unlikely(ring->adev->debug_disable_soft_recovery)) - return false; - deadline = ktime_add_us(ktime_get(), 10000); if (amdgpu_sriov_vf(ring->adev) || !ring->funcs->soft_recovery || !fence) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c index 8b8a04138711c..321310ba2c08e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c @@ -558,6 +558,9 @@ int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id, struct amdgpu_ring *gfx_ring = &sdma_instance->ring; struct amdgpu_ring *page_ring = &sdma_instance->page; + if (amdgpu_sriov_vf(adev)) + return -EOPNOTSUPP; + mutex_lock(&sdma_instance->engine_reset_mutex); if (!caller_handles_kernel_queues) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c index 8c41951feb437..eef65833a1c99 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c @@ -656,6 +656,17 @@ static int amdgpu_userq_input_args_validate(struct drm_device *dev, drm_file_err(filp, "invalidate userq queue va or size\n"); return -EINVAL; } + + if (!is_power_of_2(args->in.queue_size)) { + drm_file_err(filp, "Queue size must be a power of 2\n"); + return -EINVAL; + } + + if (args->in.queue_size < AMDGPU_GPU_PAGE_SIZE) { + drm_file_err(filp, "Queue size smaller than AMDGPU_GPU_PAGE_SIZE\n"); + return -EINVAL; + } + if (!args->in.wptr_va || !args->in.rptr_va) { drm_file_err(filp, "invalidate userq queue rptr or wptr\n"); return -EINVAL; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c index 5c181ac75d548..ead1538974454 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c @@ -829,7 +829,7 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data, dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv, DMA_RESV_USAGE_READ, fence) { - if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) { + if (num_fences >= wait_info->num_fences) { r = -EINVAL; goto free_fences; } @@ -846,7 +846,7 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data, dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv, DMA_RESV_USAGE_WRITE, fence) { - if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) { + if (num_fences >= wait_info->num_fences) { r = -EINVAL; goto free_fences; } @@ -870,7 +870,7 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data, goto free_fences; dma_fence_unwrap_for_each(f, &iter, fence) { - if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) { + if (num_fences >= wait_info->num_fences) { r = -EINVAL; goto free_fences; } @@ -894,7 +894,7 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data, if (r) goto free_fences; - if (WARN_ON_ONCE(num_fences >= wait_info->num_fences)) { + if (num_fences >= wait_info->num_fences) { r = -EINVAL; goto free_fences; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c index f96beb96c75cc..38183b4f03dfd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c @@ -972,6 +972,7 @@ bool amdgpu_virt_fw_load_skip_check(struct amdgpu_device *adev, uint32_t ucode_i || ucode_id == AMDGPU_UCODE_ID_SDMA5 || ucode_id == AMDGPU_UCODE_ID_SDMA6 || ucode_id == AMDGPU_UCODE_ID_SDMA7 + || ucode_id == AMDGPU_UCODE_ID_SDMA_RS64 || ucode_id == AMDGPU_UCODE_ID_RLC_G || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index 726b2bdfbba33..003bcece715eb 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -4956,7 +4956,8 @@ static int gfx_v10_0_sw_init(struct amdgpu_ip_block *ip_block) amdgpu_get_soft_full_reset_mask(&adev->gfx.gfx_ring[0]); adev->gfx.compute_supported_reset = amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]); - if (!amdgpu_sriov_vf(adev)) { + if (!amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) { adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; } diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index c936772c03725..1dd9fd486eecf 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -1821,13 +1821,15 @@ static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block) case IP_VERSION(11, 0, 3): if ((adev->gfx.me_fw_version >= 2280) && (adev->gfx.mec_fw_version >= 2410) && - !amdgpu_sriov_vf(adev)) { + !amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) { adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; } break; default: - if (!amdgpu_sriov_vf(adev)) { + if (!amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) { adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; } diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c index f80e9e356e252..50e39b9d9df6f 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c @@ -1547,7 +1547,8 @@ static int gfx_v12_0_sw_init(struct amdgpu_ip_block *ip_block) case IP_VERSION(12, 0, 1): if ((adev->gfx.me_fw_version >= 2660) && (adev->gfx.mec_fw_version >= 2920) && - !amdgpu_sriov_vf(adev)) { + !amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) { adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; } diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index dd19a97436db9..7d0a2d239b78d 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -2409,7 +2409,7 @@ static int gfx_v9_0_sw_init(struct amdgpu_ip_block *ip_block) amdgpu_get_soft_full_reset_mask(&adev->gfx.gfx_ring[0]); adev->gfx.compute_supported_reset = amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]); - if (!amdgpu_sriov_vf(adev)) + if (!amdgpu_sriov_vf(adev) && !adev->debug_disable_gpu_ring_reset) adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE, 0); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c index c90cbe053ef37..a4ebb6c5af557 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c @@ -1149,14 +1149,16 @@ static int gfx_v9_4_3_sw_init(struct amdgpu_ip_block *ip_block) case IP_VERSION(9, 4, 3): case IP_VERSION(9, 4, 4): if ((adev->gfx.mec_fw_version >= 155) && - !amdgpu_sriov_vf(adev)) { + !amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) { adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_PIPE; } break; case IP_VERSION(9, 5, 0): if ((adev->gfx.mec_fw_version >= 21) && - !amdgpu_sriov_vf(adev)) { + !amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) { adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_PIPE; } diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c index 36b1ca73c2ed3..a1443990d5c60 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c @@ -2361,11 +2361,15 @@ static void sdma_v4_4_2_update_reset_mask(struct amdgpu_device *adev) switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { case IP_VERSION(9, 4, 3): case IP_VERSION(9, 4, 4): - if ((adev->gfx.mec_fw_version >= 0xb0) && amdgpu_dpm_reset_sdma_is_supported(adev)) + if ((adev->gfx.mec_fw_version >= 0xb0) && + amdgpu_dpm_reset_sdma_is_supported(adev) && + !adev->debug_disable_gpu_ring_reset) adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; break; case IP_VERSION(9, 5, 0): - if ((adev->gfx.mec_fw_version >= 0xf) && amdgpu_dpm_reset_sdma_is_supported(adev)) + if ((adev->gfx.mec_fw_version >= 0xf) && + amdgpu_dpm_reset_sdma_is_supported(adev) && + !adev->debug_disable_gpu_ring_reset) adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; break; default: diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c index 7dc67a22a7a01..45e2933214a80 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c @@ -1424,17 +1424,9 @@ static int sdma_v5_0_sw_init(struct amdgpu_ip_block *ip_block) adev->sdma.supported_reset = amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring); - switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) { - case IP_VERSION(5, 0, 0): - case IP_VERSION(5, 0, 2): - case IP_VERSION(5, 0, 5): - if ((adev->sdma.instance[0].fw_version >= 35) && - !amdgpu_sriov_vf(adev)) - adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; - break; - default: - break; - } + if (!amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) + adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; /* Allocate memory for SDMA IP Dump buffer */ ptr = kcalloc(adev->sdma.num_instances * reg_count, sizeof(uint32_t), GFP_KERNEL); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c index 3bd44c24f692d..5b982cc91af39 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c @@ -1342,23 +1342,9 @@ static int sdma_v5_2_sw_init(struct amdgpu_ip_block *ip_block) adev->sdma.supported_reset = amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring); - switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) { - case IP_VERSION(5, 2, 0): - case IP_VERSION(5, 2, 2): - case IP_VERSION(5, 2, 3): - case IP_VERSION(5, 2, 4): - if ((adev->sdma.instance[0].fw_version >= 76) && - !amdgpu_sriov_vf(adev)) - adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; - break; - case IP_VERSION(5, 2, 5): - if ((adev->sdma.instance[0].fw_version >= 34) && - !amdgpu_sriov_vf(adev)) - adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; - break; - default: - break; - } + if (!amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) + adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; /* Allocate memory for SDMA IP Dump buffer */ ptr = kcalloc(adev->sdma.num_instances * reg_count, sizeof(uint32_t), GFP_KERNEL); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c index 3c6568d501994..6809c6d4be5b1 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c @@ -1351,17 +1351,9 @@ static int sdma_v6_0_sw_init(struct amdgpu_ip_block *ip_block) adev->sdma.supported_reset = amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring); - switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) { - case IP_VERSION(6, 0, 0): - case IP_VERSION(6, 0, 2): - case IP_VERSION(6, 0, 3): - if ((adev->sdma.instance[0].fw_version >= 21) && - !amdgpu_sriov_vf(adev)) - adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; - break; - default: - break; - } + if (!amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) + adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; if (amdgpu_sdma_ras_sw_init(adev)) { dev_err(adev->dev, "Failed to initialize sdma ras block!\n"); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c index 326ecc8d37d21..2b81344dcd668 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c @@ -1337,7 +1337,8 @@ static int sdma_v7_0_sw_init(struct amdgpu_ip_block *ip_block) adev->sdma.supported_reset = amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring); - if (!amdgpu_sriov_vf(adev)) + if (!amdgpu_sriov_vf(adev) && + !adev->debug_disable_gpu_ring_reset) adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; r = amdgpu_sdma_sysfs_reset_mask_init(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c index 8897dcc9c1a0a..e35fae9cdaf66 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c @@ -1964,7 +1964,8 @@ static int vcn_v2_0_start_sriov(struct amdgpu_device *adev) struct mmsch_v2_0_cmd_end end = { {0} }; struct mmsch_v2_0_init_header *header; uint32_t *init_table = adev->virt.mm_table.cpu_addr; - uint8_t i = 0; + + /* This path only programs VCN instance 0. */ header = (struct mmsch_v2_0_init_header *)init_table; direct_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_WRITE; @@ -1983,93 +1984,93 @@ static int vcn_v2_0_start_sriov(struct amdgpu_device *adev) size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.inst[0].fw->size + 4); MMSCH_V2_0_INSERT_DIRECT_RD_MOD_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_STATUS), + SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS), 0xFFFFFFFF, 0x00000004); /* mc resume*/ if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi); offset = 0; } else { MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), lower_32_bits(adev->vcn.inst->gpu_addr)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), upper_32_bits(adev->vcn.inst->gpu_addr)); offset = size; } MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET0), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0), 0); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE0), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_SIZE0), size); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), lower_32_bits(adev->vcn.inst->gpu_addr + offset)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), upper_32_bits(adev->vcn.inst->gpu_addr + offset)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET1), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1), 0); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE1), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), lower_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), upper_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET2), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2), 0); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE2), + SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE); for (r = 0; r < adev->vcn.inst[0].num_enc_rings; ++r) { ring = &adev->vcn.inst->ring_enc[r]; ring->wptr = 0; MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_RB_BASE_LO), + SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_BASE_LO), lower_32_bits(ring->gpu_addr)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_RB_BASE_HI), + SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_BASE_HI), upper_32_bits(ring->gpu_addr)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_RB_SIZE), + SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_SIZE), ring->ring_size / 4); } ring = &adev->vcn.inst->ring_dec; ring->wptr = 0; MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW), lower_32_bits(ring->gpu_addr)); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, + SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH), upper_32_bits(ring->gpu_addr)); /* force RBC into idle state */ @@ -2080,7 +2081,7 @@ static int vcn_v2_0_start_sriov(struct amdgpu_device *adev) tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1); tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1); MMSCH_V2_0_INSERT_DIRECT_WT( - SOC15_REG_OFFSET(UVD, i, mmUVD_RBC_RB_CNTL), tmp); + SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_RB_CNTL), tmp); /* add end packet */ tmp = sizeof(struct mmsch_v2_0_cmd_end); diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c index cebee453871c1..006a154511971 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c @@ -521,7 +521,9 @@ static int vcn_v2_5_hw_fini(struct amdgpu_ip_block *ip_block) RREG32_SOC15(VCN, i, mmUVD_STATUS))) vinst->set_pg_state(vinst, AMD_PG_STATE_GATE); - if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) + /* VF doesn't enable interrupt operations for RAS */ + if (!amdgpu_sriov_vf(adev) && + amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) amdgpu_irq_put(adev, &vinst->ras_poison_irq, 0); } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c index ba99e0f258aee..986cb297de8f8 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -401,27 +401,25 @@ static int kfd_dbg_get_dev_watch_id(struct kfd_process_device *pdd, int *watch_i return -ENOMEM; } -static void kfd_dbg_clear_dev_watch_id(struct kfd_process_device *pdd, int watch_id) +static void kfd_dbg_clear_dev_watch_id(struct kfd_process_device *pdd, u32 watch_id) { spin_lock(&pdd->dev->watch_points_lock); /* process owns device watch point so safe to clear */ - if ((pdd->alloc_watch_ids >> watch_id) & 0x1) { - pdd->alloc_watch_ids &= ~(0x1 << watch_id); - pdd->dev->alloc_watch_ids &= ~(0x1 << watch_id); + if (pdd->alloc_watch_ids & BIT(watch_id)) { + pdd->alloc_watch_ids &= ~BIT(watch_id); + pdd->dev->alloc_watch_ids &= ~BIT(watch_id); } spin_unlock(&pdd->dev->watch_points_lock); } -static bool kfd_dbg_owns_dev_watch_id(struct kfd_process_device *pdd, int watch_id) +static bool kfd_dbg_owns_dev_watch_id(struct kfd_process_device *pdd, u32 watch_id) { bool owns_watch_id = false; spin_lock(&pdd->dev->watch_points_lock); - owns_watch_id = watch_id < MAX_WATCH_ADDRESSES && - ((pdd->alloc_watch_ids >> watch_id) & 0x1); - + owns_watch_id = pdd->alloc_watch_ids & BIT(watch_id); spin_unlock(&pdd->dev->watch_points_lock); return owns_watch_id; @@ -432,6 +430,9 @@ int kfd_dbg_trap_clear_dev_address_watch(struct kfd_process_device *pdd, { int r; + if (watch_id >= MAX_WATCH_ADDRESSES) + return -EINVAL; + if (!kfd_dbg_owns_dev_watch_id(pdd, watch_id)) return -EINVAL; @@ -469,6 +470,9 @@ int kfd_dbg_trap_set_dev_address_watch(struct kfd_process_device *pdd, if (r) return r; + if (*watch_id >= MAX_WATCH_ADDRESSES) + return -EINVAL; + if (!pdd->dev->kfd->shared_resources.enable_mes) { r = debug_lock_and_unmap(pdd->dev->dqm); if (r) { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c index 82905f3e54ddd..261db87e86fe6 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c @@ -331,6 +331,12 @@ static int kfd_event_page_set(struct kfd_process *p, void *kernel_address, if (p->signal_page) return -EBUSY; + if (size < KFD_SIGNAL_EVENT_LIMIT * 8) { + pr_err("Event page size %llu is too small, need at least %lu bytes\n", + size, (unsigned long)(KFD_SIGNAL_EVENT_LIMIT * 8)); + return -EINVAL; + } + page = kzalloc(sizeof(*page), GFP_KERNEL); if (!page) return -ENOMEM; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c index 59a5a3fea65df..ea8377071c390 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c @@ -62,7 +62,7 @@ svm_migrate_gart_map(struct amdgpu_ring *ring, u64 npages, *gart_addr = adev->gmc.gart_start; num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8); - num_bytes = npages * 8; + num_bytes = npages * 8 * AMDGPU_GPU_PAGES_IN_CPU_PAGE; r = amdgpu_job_alloc_with_ib(adev, &adev->mman.high_pr, AMDGPU_FENCE_OWNER_UNDEFINED, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c index ddfe30c13e9d6..8ed513a77d38f 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -1992,7 +1992,7 @@ static int signal_eviction_fence(struct kfd_process *p) ef = dma_fence_get_rcu_safe(&p->ef); rcu_read_unlock(); if (!ef) - return -EINVAL; + return true; ret = dma_fence_signal(ef); dma_fence_put(ef); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c index 80c4fa2b0975d..2822c90bd7be4 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c @@ -275,8 +275,8 @@ int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_prope /* EOP buffer is not required for all ASICs */ if (properties->eop_ring_buffer_address) { - if (properties->eop_ring_buffer_size != topo_dev->node_props.eop_buffer_size) { - pr_debug("queue eop bo size 0x%x not equal to node eop buf size 0x%x\n", + if (properties->eop_ring_buffer_size < topo_dev->node_props.eop_buffer_size) { + pr_debug("queue eop bo size 0x%x is less than node eop buf size 0x%x\n", properties->eop_ring_buffer_size, topo_dev->node_props.eop_buffer_size); err = -EINVAL; @@ -284,7 +284,7 @@ int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_prope } err = kfd_queue_buffer_get(vm, (void *)properties->eop_ring_buffer_address, &properties->eop_buf_bo, - properties->eop_ring_buffer_size); + ALIGN(properties->eop_ring_buffer_size, PAGE_SIZE)); if (err) goto out_err_unreserve; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index 49dd0a81114e4..6daa70ace261f 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -33,6 +33,7 @@ #include "amdgpu_hmm.h" #include "amdgpu.h" #include "amdgpu_xgmi.h" +#include "amdgpu_reset.h" #include "kfd_priv.h" #include "kfd_svm.h" #include "kfd_migrate.h" @@ -2343,6 +2344,9 @@ static void svm_range_drain_retry_fault(struct svm_range_list *svms) pr_debug("drain retry fault gpu %d svms %p\n", i, svms); + if (!down_read_trylock(&pdd->dev->adev->reset_domain->sem)) + continue; + amdgpu_ih_wait_on_checkpoint_process_ts(pdd->dev->adev, pdd->dev->adev->irq.retry_cam_enabled ? &pdd->dev->adev->irq.ih : @@ -2352,6 +2356,7 @@ static void svm_range_drain_retry_fault(struct svm_range_list *svms) amdgpu_ih_wait_on_checkpoint_process_ts(pdd->dev->adev, &pdd->dev->adev->irq.ih_soft); + up_read(&pdd->dev->adev->reset_domain->sem); pr_debug("drain retry fault gpu %d svms 0x%p done\n", i, svms); } @@ -2535,7 +2540,7 @@ svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange, adev = pdd->dev->adev; /* Check and drain ih1 ring if cam not available */ - if (adev->irq.ih1.ring_size) { + if (!adev->irq.retry_cam_enabled && adev->irq.ih1.ring_size) { ih = &adev->irq.ih1; checkpoint_wptr = amdgpu_ih_get_wptr(adev, ih); if (ih->rptr != checkpoint_wptr) { diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 6252afd1d087f..7db2d1a3784bd 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -3407,7 +3407,17 @@ static int dm_resume(struct amdgpu_ip_block *ip_block) struct dc_commit_streams_params commit_params = {}; if (dm->dc->caps.ips_support) { + if (!amdgpu_in_reset(adev)) + mutex_lock(&dm->dc_lock); + + /* Need to set POWER_STATE_D0 first or it will not execute + * idle_power_optimizations command to DMUB. + */ + dc_dmub_srv_set_power_state(dm->dc->ctx->dmub_srv, DC_ACPI_CM_POWER_STATE_D0); dc_dmub_srv_apply_idle_power_optimizations(dm->dc, false); + + if (!amdgpu_in_reset(adev)) + mutex_unlock(&dm->dc_lock); } if (amdgpu_in_reset(adev)) { @@ -7548,10 +7558,12 @@ static void amdgpu_dm_connector_destroy(struct drm_connector *connector) drm_dp_mst_topology_mgr_destroy(&aconnector->mst_mgr); /* Cancel and flush any pending HDMI HPD debounce work */ - cancel_delayed_work_sync(&aconnector->hdmi_hpd_debounce_work); - if (aconnector->hdmi_prev_sink) { - dc_sink_release(aconnector->hdmi_prev_sink); - aconnector->hdmi_prev_sink = NULL; + if (aconnector->hdmi_hpd_debounce_delay_ms) { + cancel_delayed_work_sync(&aconnector->hdmi_hpd_debounce_work); + if (aconnector->hdmi_prev_sink) { + dc_sink_release(aconnector->hdmi_prev_sink); + aconnector->hdmi_prev_sink = NULL; + } } if (aconnector->bl_idx != -1) { @@ -8715,9 +8727,18 @@ void amdgpu_dm_connector_init_helper(struct amdgpu_display_manager *dm, mutex_init(&aconnector->hpd_lock); mutex_init(&aconnector->handle_mst_msg_ready); - aconnector->hdmi_hpd_debounce_delay_ms = AMDGPU_DM_HDMI_HPD_DEBOUNCE_MS; - INIT_DELAYED_WORK(&aconnector->hdmi_hpd_debounce_work, hdmi_hpd_debounce_work); - aconnector->hdmi_prev_sink = NULL; + /* + * If HDMI HPD debounce delay is set, use the minimum between selected + * value and AMDGPU_DM_MAX_HDMI_HPD_DEBOUNCE_MS + */ + if (amdgpu_hdmi_hpd_debounce_delay_ms) { + aconnector->hdmi_hpd_debounce_delay_ms = min(amdgpu_hdmi_hpd_debounce_delay_ms, + AMDGPU_DM_MAX_HDMI_HPD_DEBOUNCE_MS); + INIT_DELAYED_WORK(&aconnector->hdmi_hpd_debounce_work, hdmi_hpd_debounce_work); + aconnector->hdmi_prev_sink = NULL; + } else { + aconnector->hdmi_hpd_debounce_delay_ms = 0; + } /* * configure support HPD hot plug connector_>polled default value is 0 @@ -10400,10 +10421,10 @@ static void dm_set_writeback(struct amdgpu_display_manager *dm, wb_info->dwb_params.capture_rate = dwb_capture_rate_0; - wb_info->dwb_params.scaler_taps.h_taps = 4; - wb_info->dwb_params.scaler_taps.v_taps = 4; - wb_info->dwb_params.scaler_taps.h_taps_c = 2; - wb_info->dwb_params.scaler_taps.v_taps_c = 2; + wb_info->dwb_params.scaler_taps.h_taps = 1; + wb_info->dwb_params.scaler_taps.v_taps = 1; + wb_info->dwb_params.scaler_taps.h_taps_c = 1; + wb_info->dwb_params.scaler_taps.v_taps_c = 1; wb_info->dwb_params.subsample_position = DWB_INTERSTITIAL_SUBSAMPLING; wb_info->mcif_buf_params.luma_pitch = afb->base.pitches[0]; @@ -10703,7 +10724,7 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state) continue; } for (j = 0; j < status->plane_count; j++) - dummy_updates[j].surface = status->plane_states[0]; + dummy_updates[j].surface = status->plane_states[j]; sort(dummy_updates, status->plane_count, sizeof(*dummy_updates), dm_plane_layer_index_cmp, NULL); @@ -11419,6 +11440,8 @@ static bool should_reset_plane(struct drm_atomic_state *state, struct drm_crtc_state *old_crtc_state, *new_crtc_state; struct dm_crtc_state *old_dm_crtc_state, *new_dm_crtc_state; struct amdgpu_device *adev = drm_to_adev(plane->dev); + struct drm_connector_state *new_con_state; + struct drm_connector *connector; int i; /* @@ -11429,6 +11452,15 @@ static bool should_reset_plane(struct drm_atomic_state *state, state->allow_modeset) return true; + /* Check for writeback commit */ + for_each_new_connector_in_state(state, connector, new_con_state, i) { + if (connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK) + continue; + + if (new_con_state->writeback_job) + return true; + } + if (amdgpu_in_reset(adev) && state->allow_modeset) return true; @@ -11605,7 +11637,7 @@ static int dm_check_cursor_fb(struct amdgpu_crtc *new_acrtc, * check tiling flags when the FB doesn't have a modifier. */ if (!(fb->flags & DRM_MODE_FB_MODIFIERS)) { - if (adev->family >= AMDGPU_FAMILY_GC_12_0_0) { + if (adev->family == AMDGPU_FAMILY_GC_12_0_0) { linear = AMDGPU_TILING_GET(afb->tiling_flags, GFX12_SWIZZLE_MODE) == 0; } else if (adev->family >= AMDGPU_FAMILY_AI) { linear = AMDGPU_TILING_GET(afb->tiling_flags, SWIZZLE_MODE) == 0; @@ -12027,10 +12059,9 @@ static int dm_crtc_get_cursor_mode(struct amdgpu_device *adev, /* Overlay cursor not supported on HW before DCN * DCN401 does not have the cursor-on-scaled-plane or cursor-on-yuv-plane restrictions - * as previous DCN generations, so enable native mode on DCN401 in addition to DCE + * as previous DCN generations, so enable native mode on DCN401 */ - if (amdgpu_ip_version(adev, DCE_HWIP, 0) == 0 || - amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 0, 1)) { + if (amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 0, 1)) { *cursor_mode = DM_CURSOR_NATIVE_MODE; return 0; } @@ -12350,6 +12381,12 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev, * need to be added for DC to not disable a plane by mistake */ if (dm_new_crtc_state->cursor_mode == DM_CURSOR_OVERLAY_MODE) { + if (amdgpu_ip_version(adev, DCE_HWIP, 0) == 0) { + drm_dbg(dev, "Overlay cursor not supported on DCE\n"); + ret = -EINVAL; + goto fail; + } + ret = drm_atomic_add_affected_planes(state, crtc); if (ret) goto fail; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h index 8ca7389575989..adcd7ea696719 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h @@ -59,7 +59,10 @@ #define AMDGPU_HDR_MULT_DEFAULT (0x100000000LL) -#define AMDGPU_DM_HDMI_HPD_DEBOUNCE_MS 1500 +/* + * Maximum HDMI HPD debounce delay in milliseconds + */ +#define AMDGPU_DM_MAX_HDMI_HPD_DEBOUNCE_MS 5000 /* #include "include/amdgpu_dal_power_if.h" #include "amdgpu_dm_irq.h" diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c index e027798ece032..2ecebf9a00fa4 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c @@ -277,7 +277,7 @@ static int amdgpu_dm_plane_validate_dcc(struct amdgpu_device *adev, if (!dcc->enable) return 0; - if (adev->family < AMDGPU_FAMILY_GC_12_0_0 && + if (adev->family != AMDGPU_FAMILY_GC_12_0_0 && format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN) return -EINVAL; @@ -900,7 +900,7 @@ int amdgpu_dm_plane_fill_plane_buffer_attributes(struct amdgpu_device *adev, upper_32_bits(chroma_addr); } - if (adev->family >= AMDGPU_FAMILY_GC_12_0_0) { + if (adev->family == AMDGPU_FAMILY_GC_12_0_0) { ret = amdgpu_dm_plane_fill_gfx12_plane_attributes_from_modifiers(adev, afb, format, rotation, plane_size, tiling_info, dcc, @@ -1059,10 +1059,15 @@ static void amdgpu_dm_plane_get_min_max_dc_plane_scaling(struct drm_device *dev, *min_downscale = plane_cap->max_downscale_factor.nv12; break; + /* All 64 bpp formats have the same fp16 scaling limits */ case DRM_FORMAT_XRGB16161616F: case DRM_FORMAT_ARGB16161616F: case DRM_FORMAT_XBGR16161616F: case DRM_FORMAT_ABGR16161616F: + case DRM_FORMAT_XRGB16161616: + case DRM_FORMAT_ARGB16161616: + case DRM_FORMAT_XBGR16161616: + case DRM_FORMAT_ABGR16161616: *max_upscale = plane_cap->max_upscale_factor.fp16; *min_downscale = plane_cap->max_downscale_factor.fp16; break; diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c index b315ed91e010b..c49268db85f68 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c @@ -138,7 +138,7 @@ static void dcn315_update_clocks(struct clk_mgr *clk_mgr_base, if (dc->work_arounds.skip_clock_update) return; - clk_mgr_base->clks.zstate_support = new_clocks->zstate_support; + display_count = dcn315_get_active_display_cnt_wa(dc, context); /* * if it is safe to lower, but we are already in the lower state, we don't have to do anything * also if safe to lower is false, we just go in the higher state @@ -151,7 +151,6 @@ static void dcn315_update_clocks(struct clk_mgr *clk_mgr_base, } /* check that we're not already in lower */ if (clk_mgr_base->clks.pwr_state != DCN_PWR_STATE_LOW_POWER) { - display_count = dcn315_get_active_display_cnt_wa(dc, context); /* if we can go lower, go lower */ if (display_count == 0) { union display_idle_optimization_u idle_info = { 0 }; diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c index 1eb04772f5da2..817a0253d10e5 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c @@ -768,32 +768,32 @@ static struct wm_table ddr5_wm_table = { .wm_inst = WM_A, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.72, - .sr_exit_time_us = 28.0, - .sr_enter_plus_exit_time_us = 30.0, + .sr_exit_time_us = 31.0, + .sr_enter_plus_exit_time_us = 33.0, .valid = true, }, { .wm_inst = WM_B, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.72, - .sr_exit_time_us = 28.0, - .sr_enter_plus_exit_time_us = 30.0, + .sr_exit_time_us = 31.0, + .sr_enter_plus_exit_time_us = 33.0, .valid = true, }, { .wm_inst = WM_C, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.72, - .sr_exit_time_us = 28.0, - .sr_enter_plus_exit_time_us = 30.0, + .sr_exit_time_us = 31.0, + .sr_enter_plus_exit_time_us = 33.0, .valid = true, }, { .wm_inst = WM_D, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.72, - .sr_exit_time_us = 28.0, - .sr_enter_plus_exit_time_us = 30.0, + .sr_exit_time_us = 31.0, + .sr_enter_plus_exit_time_us = 33.0, .valid = true, }, } diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c index 0a46e834357a1..0886bef32a5de 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c @@ -169,7 +169,7 @@ struct dc_stream_state *dc_create_stream_for_sink( if (sink == NULL) return NULL; - stream = kzalloc(sizeof(struct dc_stream_state), GFP_KERNEL); + stream = kzalloc(sizeof(struct dc_stream_state), GFP_ATOMIC); if (stream == NULL) goto alloc_fail; diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c index a4f14b16564c2..227aa8672d17b 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c @@ -105,9 +105,12 @@ void cm_helper_program_gamcor_xfer_func( #define NUMBER_REGIONS 32 #define NUMBER_SW_SEGMENTS 16 -bool cm3_helper_translate_curve_to_hw_format( - const struct dc_transfer_func *output_tf, - struct pwl_params *lut_params, bool fixpoint) +#define DC_LOGGER \ + ctx->logger + +bool cm3_helper_translate_curve_to_hw_format(struct dc_context *ctx, + const struct dc_transfer_func *output_tf, + struct pwl_params *lut_params, bool fixpoint) { struct curve_points3 *corner_points; struct pwl_result_data *rgb_resulted; @@ -251,6 +254,10 @@ bool cm3_helper_translate_curve_to_hw_format( if (fixpoint == true) { i = 1; while (i != hw_points + 2) { + uint32_t red_clamp; + uint32_t green_clamp; + uint32_t blue_clamp; + if (i >= hw_points) { if (dc_fixpt_lt(rgb_plus_1->red, rgb->red)) rgb_plus_1->red = dc_fixpt_add(rgb->red, @@ -263,9 +270,20 @@ bool cm3_helper_translate_curve_to_hw_format( rgb_minus_1->delta_blue); } - rgb->delta_red_reg = dc_fixpt_clamp_u0d10(rgb->delta_red); - rgb->delta_green_reg = dc_fixpt_clamp_u0d10(rgb->delta_green); - rgb->delta_blue_reg = dc_fixpt_clamp_u0d10(rgb->delta_blue); + rgb->delta_red = dc_fixpt_sub(rgb_plus_1->red, rgb->red); + rgb->delta_green = dc_fixpt_sub(rgb_plus_1->green, rgb->green); + rgb->delta_blue = dc_fixpt_sub(rgb_plus_1->blue, rgb->blue); + + red_clamp = dc_fixpt_clamp_u0d14(rgb->delta_red); + green_clamp = dc_fixpt_clamp_u0d14(rgb->delta_green); + blue_clamp = dc_fixpt_clamp_u0d14(rgb->delta_blue); + + if (red_clamp >> 10 || green_clamp >> 10 || blue_clamp >> 10) + DC_LOG_ERROR("Losing delta precision while programming shaper LUT."); + + rgb->delta_red_reg = red_clamp & 0x3ff; + rgb->delta_green_reg = green_clamp & 0x3ff; + rgb->delta_blue_reg = blue_clamp & 0x3ff; rgb->red_reg = dc_fixpt_clamp_u0d14(rgb->red); rgb->green_reg = dc_fixpt_clamp_u0d14(rgb->green); rgb->blue_reg = dc_fixpt_clamp_u0d14(rgb->blue); diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_link_encoder.c index 06907e8a4eda1..ddc736af776c9 100644 --- a/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_link_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_link_encoder.c @@ -188,9 +188,18 @@ void dcn32_link_encoder_get_max_link_cap(struct link_encoder *enc, if (!query_dp_alt_from_dmub(enc, &cmd)) return; - if (cmd.query_dp_alt.data.is_usb && - cmd.query_dp_alt.data.is_dp4 == 0) - link_settings->lane_count = MIN(LANE_COUNT_TWO, link_settings->lane_count); + /* + * USB-C DisplayPort Alt Mode lane count limitation logic: + * When USB and DP share the same USB-C connector, hardware must allocate + * some lanes for USB data, limiting DP to maximum 2 lanes instead of 4. + * This ensures USB functionality remains available while DP is active. + */ + if (cmd.query_dp_alt.data.is_dp_alt_disable == 0 && + cmd.query_dp_alt.data.is_usb && + cmd.query_dp_alt.data.is_dp4 == 0) { + link_settings->lane_count = + MIN(LANE_COUNT_TWO, link_settings->lane_count); + } } diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c index 817a370e80a77..8a177d5ae213e 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c @@ -164,8 +164,8 @@ struct _vcs_dpi_soc_bounding_box_st dcn3_5_soc = { }, }, .num_states = 5, - .sr_exit_time_us = 28.0, - .sr_enter_plus_exit_time_us = 30.0, + .sr_exit_time_us = 31.0, + .sr_enter_plus_exit_time_us = 33.0, .sr_exit_z8_time_us = 250.0, .sr_enter_plus_exit_z8_time_us = 350.0, .fclk_change_latency_us = 24.0, diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c index 4f569cd8a5d61..272ebdd8b9ebd 100644 --- a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c +++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c @@ -360,10 +360,10 @@ void dpp3_cnv_setup ( tbl_entry.color_space = input_color_space; - if (color_space >= COLOR_SPACE_YCBCR601) - select = INPUT_CSC_SELECT_ICSC; - else + if (dpp3_should_bypass_post_csc_for_colorspace(color_space)) select = INPUT_CSC_SELECT_BYPASS; + else + select = INPUT_CSC_SELECT_ICSC; dpp3_program_post_csc(dpp_base, color_space, select, &tbl_entry); @@ -1521,3 +1521,18 @@ bool dpp3_construct( return true; } +bool dpp3_should_bypass_post_csc_for_colorspace(enum dc_color_space dc_color_space) +{ + switch (dc_color_space) { + case COLOR_SPACE_UNKNOWN: + case COLOR_SPACE_SRGB: + case COLOR_SPACE_XR_RGB: + case COLOR_SPACE_SRGB_LIMITED: + case COLOR_SPACE_MSREF_SCRGB: + case COLOR_SPACE_2020_RGB_FULLRANGE: + case COLOR_SPACE_2020_RGB_LIMITEDRANGE: + return true; + default: + return false; + } +} diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.h b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.h index f236824126e94..2a76105fa9b1c 100644 --- a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.h +++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.h @@ -642,4 +642,8 @@ void dpp3_program_cm_dealpha( void dpp3_cm_get_gamut_remap(struct dpp *dpp_base, struct dpp_grph_csc_adjustment *adjust); + +bool dpp3_should_bypass_post_csc_for_colorspace( + enum dc_color_space dc_color_space); + #endif /* __DC_HWSS_DCN30_H__ */ diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.c index 36187f890d5d0..b62bbadb0d440 100644 --- a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.c +++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.c @@ -206,10 +206,10 @@ void dpp401_dpp_setup( tbl_entry.color_space = input_color_space; - if (color_space >= COLOR_SPACE_YCBCR601) - select = INPUT_CSC_SELECT_ICSC; - else + if (dpp3_should_bypass_post_csc_for_colorspace(color_space)) select = INPUT_CSC_SELECT_BYPASS; + else + select = INPUT_CSC_SELECT_ICSC; dpp3_program_post_csc(dpp_base, color_space, select, &tbl_entry); diff --git a/drivers/gpu/drm/amd/display/dc/dwb/dcn30/dcn30_cm_common.h b/drivers/gpu/drm/amd/display/dc/dwb/dcn30/dcn30_cm_common.h index b86347c9b0389..95f9318a54efc 100644 --- a/drivers/gpu/drm/amd/display/dc/dwb/dcn30/dcn30_cm_common.h +++ b/drivers/gpu/drm/amd/display/dc/dwb/dcn30/dcn30_cm_common.h @@ -59,7 +59,7 @@ void cm_helper_program_gamcor_xfer_func( const struct pwl_params *params, const struct dcn3_xfer_func_reg *reg); -bool cm3_helper_translate_curve_to_hw_format( +bool cm3_helper_translate_curve_to_hw_format(struct dc_context *ctx, const struct dc_transfer_func *output_tf, struct pwl_params *lut_params, bool fixpoint); diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c index 39be5a58f837a..8f86177de48dc 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c @@ -59,6 +59,7 @@ #include "dc_state_priv.h" #include "dpcd_defs.h" #include "dsc.h" +#include "dc_dp_types.h" /* include DCE11 register header files */ #include "dce/dce_11_0_d.h" #include "dce/dce_11_0_sh_mask.h" @@ -1729,20 +1730,25 @@ static void power_down_encoders(struct dc *dc) int i; for (i = 0; i < dc->link_count; i++) { - enum signal_type signal = dc->links[i]->connector_signal; - - dc->link_srv->blank_dp_stream(dc->links[i], false); + struct dc_link *link = dc->links[i]; + struct link_encoder *link_enc = link->link_enc; + enum signal_type signal = link->connector_signal; + dc->link_srv->blank_dp_stream(link, false); if (signal != SIGNAL_TYPE_EDP) signal = SIGNAL_TYPE_NONE; - if (dc->links[i]->ep_type == DISPLAY_ENDPOINT_PHY) - dc->links[i]->link_enc->funcs->disable_output( - dc->links[i]->link_enc, signal); + if (link->ep_type == DISPLAY_ENDPOINT_PHY) + link_enc->funcs->disable_output(link_enc, signal); + + if (link->fec_state == dc_link_fec_enabled) { + link_enc->funcs->fec_set_enable(link_enc, false); + link_enc->funcs->fec_set_ready(link_enc, false); + link->fec_state = dc_link_fec_not_ready; + } - dc->links[i]->link_status.link_active = false; - memset(&dc->links[i]->cur_link_settings, 0, - sizeof(dc->links[i]->cur_link_settings)); + link->link_status.link_active = false; + memset(&link->cur_link_settings, 0, sizeof(link->cur_link_settings)); } } @@ -1790,6 +1796,9 @@ static void disable_vga_and_power_gate_all_controllers( struct timing_generator *tg; struct dc_context *ctx = dc->ctx; + if (dc->caps.ips_support) + return; + for (i = 0; i < dc->res_pool->timing_generator_count; i++) { tg = dc->res_pool->timing_generators[i]; @@ -1866,13 +1875,16 @@ static void clean_up_dsc_blocks(struct dc *dc) /* disable DSC in OPTC */ if (i < dc->res_pool->timing_generator_count) { tg = dc->res_pool->timing_generators[i]; - tg->funcs->set_dsc_config(tg, OPTC_DSC_DISABLED, 0, 0); + if (tg->funcs->set_dsc_config) + tg->funcs->set_dsc_config(tg, OPTC_DSC_DISABLED, 0, 0); } /* disable DSC in stream encoder */ if (i < dc->res_pool->stream_enc_count) { se = dc->res_pool->stream_enc[i]; - se->funcs->dp_set_dsc_config(se, OPTC_DSC_DISABLED, 0, 0); - se->funcs->dp_set_dsc_pps_info_packet(se, false, NULL, true); + if (se->funcs->dp_set_dsc_config) + se->funcs->dp_set_dsc_config(se, OPTC_DSC_DISABLED, 0, 0); + if (se->funcs->dp_set_dsc_pps_info_packet) + se->funcs->dp_set_dsc_pps_info_packet(se, false, NULL, true); } /* disable DSC block */ if (dccg->funcs->set_ref_dscclk) @@ -1921,8 +1933,8 @@ void dce110_enable_accelerated_mode(struct dc *dc, struct dc_state *context) get_edp_streams(context, edp_streams, &edp_stream_num); - /* Check fastboot support, disable on DCE 6-8 because of blank screens */ - if (edp_num && edp_stream_num && dc->ctx->dce_version < DCE_VERSION_10_0) { + /* Check fastboot support, disable on DCE 6-8-10 because of blank screens */ + if (edp_num && edp_stream_num && dc->ctx->dce_version > DCE_VERSION_10_0) { for (i = 0; i < edp_num; i++) { edp_link = edp_links[i]; if (edp_link != edp_streams[0]->link) diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c index 56c1ab6c73308..a4025a09a38a3 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c @@ -3055,9 +3055,17 @@ void dcn20_enable_stream(struct pipe_ctx *pipe_ctx) dccg->funcs->enable_symclk32_se(dccg, dp_hpo_inst, phyd32clk); } } else { - if (dccg->funcs->enable_symclk_se) - dccg->funcs->enable_symclk_se(dccg, stream_enc->stream_enc_inst, + if (dccg->funcs->enable_symclk_se && link_enc) { + if (link->ep_type == DISPLAY_ENDPOINT_USB4_DPIA + && link->cur_link_settings.link_rate == LINK_RATE_UNKNOWN + && !link->link_status.link_active) { + if (dccg->funcs->disable_symclk_se) + dccg->funcs->disable_symclk_se(dccg, stream_enc->stream_enc_inst, link_enc->transmitter - TRANSMITTER_UNIPHY_A); + } else + dccg->funcs->enable_symclk_se(dccg, stream_enc->stream_enc_inst, + link_enc->transmitter - TRANSMITTER_UNIPHY_A); + } } if (dc->res_pool->dccg->funcs->set_pixel_rate_div) diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c index e47ed5571dfdd..731645a2ab9aa 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c @@ -238,7 +238,7 @@ bool dcn30_set_blend_lut( if (plane_state->blend_tf.type == TF_TYPE_HWPWL) blend_lut = &plane_state->blend_tf.pwl; else if (plane_state->blend_tf.type == TF_TYPE_DISTRIBUTED_POINTS) { - result = cm3_helper_translate_curve_to_hw_format( + result = cm3_helper_translate_curve_to_hw_format(plane_state->ctx, &plane_state->blend_tf, &dpp_base->regamma_params, false); if (!result) return result; @@ -333,8 +333,9 @@ bool dcn30_set_input_transfer_func(struct dc *dc, if (plane_state->in_transfer_func.type == TF_TYPE_HWPWL) params = &plane_state->in_transfer_func.pwl; else if (plane_state->in_transfer_func.type == TF_TYPE_DISTRIBUTED_POINTS && - cm3_helper_translate_curve_to_hw_format(&plane_state->in_transfer_func, - &dpp_base->degamma_params, false)) + cm3_helper_translate_curve_to_hw_format(plane_state->ctx, + &plane_state->in_transfer_func, + &dpp_base->degamma_params, false)) params = &dpp_base->degamma_params; result = dpp_base->funcs->dpp_program_gamcor_lut(dpp_base, params); @@ -405,7 +406,7 @@ bool dcn30_set_output_transfer_func(struct dc *dc, params = &stream->out_transfer_func.pwl; else if (pipe_ctx->stream->out_transfer_func.type == TF_TYPE_DISTRIBUTED_POINTS && - cm3_helper_translate_curve_to_hw_format( + cm3_helper_translate_curve_to_hw_format(stream->ctx, &stream->out_transfer_func, &mpc->blender_params, false)) params = &mpc->blender_params; diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c index b822f2dffff0e..e9bd43a72ce58 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c @@ -546,8 +546,22 @@ static void dcn31_reset_back_end_for_pipe( if (pipe_ctx->stream_res.tg->funcs->set_odm_bypass) pipe_ctx->stream_res.tg->funcs->set_odm_bypass( pipe_ctx->stream_res.tg, &pipe_ctx->stream->timing); + /* + * TODO - convert symclk_ref_cnts for otg to a bit map to solve + * the case where the same symclk is shared across multiple otg + * instances + */ if (dc_is_hdmi_tmds_signal(pipe_ctx->stream->signal)) - pipe_ctx->stream->link->phy_state.symclk_ref_cnts.otg = 0; + link->phy_state.symclk_ref_cnts.otg = 0; + + if (pipe_ctx->top_pipe == NULL) { + if (link->phy_state.symclk_state == SYMCLK_ON_TX_OFF) { + const struct link_hwss *link_hwss = get_link_hwss(link, &pipe_ctx->link_res); + + link_hwss->disable_link_output(link, &pipe_ctx->link_res, pipe_ctx->stream->signal); + link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF; + } + } set_drr_and_clear_adjust_pending(pipe_ctx, pipe_ctx->stream, NULL); diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c index f39292952702f..c6fde355ac823 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c @@ -486,8 +486,9 @@ bool dcn32_set_mcm_luts( if (plane_state->blend_tf.type == TF_TYPE_HWPWL) lut_params = &plane_state->blend_tf.pwl; else if (plane_state->blend_tf.type == TF_TYPE_DISTRIBUTED_POINTS) { - result = cm3_helper_translate_curve_to_hw_format(&plane_state->blend_tf, - &dpp_base->regamma_params, false); + result = cm3_helper_translate_curve_to_hw_format(plane_state->ctx, + &plane_state->blend_tf, + &dpp_base->regamma_params, false); if (!result) return result; @@ -501,9 +502,9 @@ bool dcn32_set_mcm_luts( lut_params = &plane_state->in_shaper_func.pwl; else if (plane_state->in_shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) { // TODO: dpp_base replace - ASSERT(false); - cm3_helper_translate_curve_to_hw_format(&plane_state->in_shaper_func, - &dpp_base->shaper_params, true); + cm3_helper_translate_curve_to_hw_format(plane_state->ctx, + &plane_state->in_shaper_func, + &dpp_base->shaper_params, true); lut_params = &dpp_base->shaper_params; } @@ -543,8 +544,9 @@ bool dcn32_set_input_transfer_func(struct dc *dc, if (plane_state->in_transfer_func.type == TF_TYPE_HWPWL) params = &plane_state->in_transfer_func.pwl; else if (plane_state->in_transfer_func.type == TF_TYPE_DISTRIBUTED_POINTS && - cm3_helper_translate_curve_to_hw_format(&plane_state->in_transfer_func, - &dpp_base->degamma_params, false)) + cm3_helper_translate_curve_to_hw_format(plane_state->ctx, + &plane_state->in_transfer_func, + &dpp_base->degamma_params, false)) params = &dpp_base->degamma_params; dpp_base->funcs->dpp_program_gamcor_lut(dpp_base, params); @@ -575,7 +577,7 @@ bool dcn32_set_output_transfer_func(struct dc *dc, params = &stream->out_transfer_func.pwl; else if (pipe_ctx->stream->out_transfer_func.type == TF_TYPE_DISTRIBUTED_POINTS && - cm3_helper_translate_curve_to_hw_format( + cm3_helper_translate_curve_to_hw_format(stream->ctx, &stream->out_transfer_func, &mpc->blender_params, false)) params = &mpc->blender_params; diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c index 68e48a2492c9e..d4a0961f6b516 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c @@ -284,6 +284,8 @@ void dcn401_init_hw(struct dc *dc) for (i = 0; i < dc->link_count; i++) { struct dc_link *link = dc->links[i]; + if (link->ep_type != DISPLAY_ENDPOINT_PHY) + continue; if (link->link_enc->funcs->is_dig_enabled && link->link_enc->funcs->is_dig_enabled(link->link_enc) && hws->funcs.power_down) { @@ -427,7 +429,7 @@ void dcn401_populate_mcm_luts(struct dc *dc, if (mcm_luts.lut1d_func->type == TF_TYPE_HWPWL) m_lut_params.pwl = &mcm_luts.lut1d_func->pwl; else if (mcm_luts.lut1d_func->type == TF_TYPE_DISTRIBUTED_POINTS) { - rval = cm3_helper_translate_curve_to_hw_format( + rval = cm3_helper_translate_curve_to_hw_format(mpc->ctx, mcm_luts.lut1d_func, &dpp_base->regamma_params, false); m_lut_params.pwl = rval ? &dpp_base->regamma_params : NULL; @@ -447,7 +449,7 @@ void dcn401_populate_mcm_luts(struct dc *dc, m_lut_params.pwl = &mcm_luts.shaper->pwl; else if (mcm_luts.shaper->type == TF_TYPE_DISTRIBUTED_POINTS) { ASSERT(false); - rval = cm3_helper_translate_curve_to_hw_format( + rval = cm3_helper_translate_curve_to_hw_format(mpc->ctx, mcm_luts.shaper, &dpp_base->regamma_params, true); m_lut_params.pwl = rval ? &dpp_base->regamma_params : NULL; @@ -624,8 +626,9 @@ bool dcn401_set_mcm_luts(struct pipe_ctx *pipe_ctx, if (plane_state->blend_tf.type == TF_TYPE_HWPWL) lut_params = &plane_state->blend_tf.pwl; else if (plane_state->blend_tf.type == TF_TYPE_DISTRIBUTED_POINTS) { - rval = cm3_helper_translate_curve_to_hw_format(&plane_state->blend_tf, - &dpp_base->regamma_params, false); + rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx, + &plane_state->blend_tf, + &dpp_base->regamma_params, false); lut_params = rval ? &dpp_base->regamma_params : NULL; } result = mpc->funcs->program_1dlut(mpc, lut_params, mpcc_id); @@ -636,8 +639,9 @@ bool dcn401_set_mcm_luts(struct pipe_ctx *pipe_ctx, lut_params = &plane_state->in_shaper_func.pwl; else if (plane_state->in_shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) { // TODO: dpp_base replace - rval = cm3_helper_translate_curve_to_hw_format(&plane_state->in_shaper_func, - &dpp_base->shaper_params, true); + rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx, + &plane_state->in_shaper_func, + &dpp_base->shaper_params, true); lut_params = rval ? &dpp_base->shaper_params : NULL; } result &= mpc->funcs->program_shaper(mpc, lut_params, mpcc_id); @@ -671,7 +675,7 @@ bool dcn401_set_output_transfer_func(struct dc *dc, params = &stream->out_transfer_func.pwl; else if (pipe_ctx->stream->out_transfer_func.type == TF_TYPE_DISTRIBUTED_POINTS && - cm3_helper_translate_curve_to_hw_format( + cm3_helper_translate_curve_to_hw_format(stream->ctx, &stream->out_transfer_func, &mpc->blender_params, false)) params = &mpc->blender_params; @@ -912,10 +916,10 @@ static void dcn401_enable_stream_calc( pipe_ctx->stream->link->cur_link_settings.lane_count; uint32_t active_total_with_borders; - if (dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) + if (dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) { *dp_hpo_inst = pipe_ctx->stream_res.hpo_dp_stream_enc->inst; - - *phyd32clk = get_phyd32clk_src(pipe_ctx->stream->link); + *phyd32clk = get_phyd32clk_src(pipe_ctx->stream->link); + } if (dc_is_tmds_signal(pipe_ctx->stream->signal)) dcn401_calculate_dccg_tmds_div_value(pipe_ctx, tmds_div); @@ -1626,7 +1630,8 @@ void dcn401_unblank_stream(struct pipe_ctx *pipe_ctx, void dcn401_hardware_release(struct dc *dc) { if (!dc->debug.disable_force_pstate_allow_on_hw_release) { - dc_dmub_srv_fams2_update_config(dc, dc->current_state, false); + if (dc->ctx->dmub_srv && dc->debug.fams2_config.bits.enable) + dc_dmub_srv_fams2_update_config(dc, dc->current_state, false); /* If pstate unsupported, or still supported * by firmware, force it supported by dcn @@ -1646,7 +1651,9 @@ void dcn401_hardware_release(struct dc *dc) dc->clk_mgr->clks.p_state_change_support = false; dc->clk_mgr->funcs->update_clocks(dc->clk_mgr, dc->current_state, true); } - dc_dmub_srv_fams2_update_config(dc, dc->current_state, false); + + if (dc->ctx->dmub_srv && dc->debug.fams2_config.bits.enable) + dc_dmub_srv_fams2_update_config(dc, dc->current_state, false); } } diff --git a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c index b66fbcb0040d6..a084c698ed6ef 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c @@ -1928,7 +1928,7 @@ static void disable_link_dp(struct dc_link *link, link->dc->hwss.edp_power_control(link, false); } - if (signal == SIGNAL_TYPE_DISPLAY_PORT_MST) + if (signal == SIGNAL_TYPE_DISPLAY_PORT_MST && link->sink_count == 0) /* set the sink to SST mode after disabling the link */ enable_mst_on_sink(link, false); @@ -2079,7 +2079,12 @@ static enum dc_status enable_link_dp(struct dc_state *state, pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT && link->dc->debug.set_mst_en_for_sst) { enable_mst_on_sink(link, true); + } else if (link->dpcd_caps.is_mst_capable && + pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT) { + /* disable mst on sink */ + enable_mst_on_sink(link, false); } + if (pipe_ctx->stream->signal == SIGNAL_TYPE_EDP) { /*in case it is not on*/ if (!link->dc->config.edp_no_power_sequencing) @@ -2358,9 +2363,9 @@ void link_set_dpms_off(struct pipe_ctx *pipe_ctx) if (pipe_ctx->stream->sink) { if (pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_VIRTUAL && pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_NONE) { - DC_LOG_DC("%s pipe_ctx dispname=%s signal=%x link=%d\n", __func__, + DC_LOG_DC("%s pipe_ctx dispname=%s signal=%x link=%d sink_count=%d\n", __func__, pipe_ctx->stream->sink->edid_caps.display_name, - pipe_ctx->stream->signal, link->link_index); + pipe_ctx->stream->signal, link->link_index, link->sink_count); } } @@ -2474,10 +2479,11 @@ void link_set_dpms_on( if (pipe_ctx->stream->sink) { if (pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_VIRTUAL && pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_NONE) { - DC_LOG_DC("%s pipe_ctx dispname=%s signal=%x link=%d\n", __func__, + DC_LOG_DC("%s pipe_ctx dispname=%s signal=%x link=%d sink_count=%d\n", __func__, pipe_ctx->stream->sink->edid_caps.display_name, pipe_ctx->stream->signal, - link->link_index); + link->link_index, + link->sink_count); } } diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.c index ce174ce5579c0..6a7c4a59ff4c7 100644 --- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.c +++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_fixed_vs_pe_retimer.c @@ -271,7 +271,7 @@ enum link_training_result dp_perform_fixed_vs_pe_training_sequence( rate = get_dpcd_link_rate(<_settings->link_settings); // Only perform toggle if FIXED_VS LTTPR reports no IEEE OUI - if (memcmp("\x0,\x0,\x0", &link->dpcd_caps.lttpr_caps.lttpr_ieee_oui[0], 3) == 0) { + if (memcmp("\x00\x00\x00", &link->dpcd_caps.lttpr_caps.lttpr_ieee_oui[0], 3) == 0) { /* Vendor specific: Toggle link rate */ toggle_rate = (rate == 0x6) ? 0xA : 0x6; diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn32/dcn32_mpc.c b/drivers/gpu/drm/amd/display/dc/mpc/dcn32/dcn32_mpc.c index 6f0e017a8ae29..a9d2aa0b5390f 100644 --- a/drivers/gpu/drm/amd/display/dc/mpc/dcn32/dcn32_mpc.c +++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn32/dcn32_mpc.c @@ -724,8 +724,7 @@ bool mpc32_program_shaper( return false; } - if (mpc->ctx->dc->debug.enable_mem_low_power.bits.mpc) - mpc32_power_on_shaper_3dlut(mpc, mpcc_id, true); + mpc32_power_on_shaper_3dlut(mpc, mpcc_id, true); current_mode = mpc32_get_shaper_current(mpc, mpcc_id); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c index 82cc78c291d82..12c2a0d9fb2a3 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c @@ -1226,12 +1226,12 @@ static struct stream_encoder *dcn315_stream_encoder_create( /*PHYB is wired off in HW, allow front end to remapping, otherwise needs more changes*/ /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn31_vpg_create(ctx, vpg_inst); afmt = dcn31_afmt_create(ctx, afmt_inst); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c index 636110e48d01b..3c77c14c5a5ed 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c @@ -1220,12 +1220,12 @@ static struct stream_encoder *dcn316_stream_encoder_create( int afmt_inst; /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn31_vpg_create(ctx, vpg_inst); afmt = dcn31_afmt_create(ctx, afmt_inst); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c index 3965a7f1b64b7..9cace432ce364 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c @@ -1208,12 +1208,12 @@ static struct stream_encoder *dcn32_stream_encoder_create( int afmt_inst; /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn32_vpg_create(ctx, vpg_inst); afmt = dcn32_afmt_create(ctx, afmt_inst); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c index ad214986f7ac7..26fd5c03c0147 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c @@ -1189,12 +1189,12 @@ static struct stream_encoder *dcn321_stream_encoder_create( int afmt_inst; /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn321_vpg_create(ctx, vpg_inst); afmt = dcn321_afmt_create(ctx, afmt_inst); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c index 06bec7dcc7556..e8d74ceb9dc29 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c @@ -1271,12 +1271,12 @@ static struct stream_encoder *dcn35_stream_encoder_create( int afmt_inst; /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn31_vpg_create(ctx, vpg_inst); afmt = dcn31_afmt_create(ctx, afmt_inst); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c index 7974e306126e0..532e5d9bc4337 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c @@ -1251,12 +1251,12 @@ static struct stream_encoder *dcn35_stream_encoder_create( int afmt_inst; /* Mapping of VPG, AFMT, DME register blocks to DIO block instance */ - if (eng_id <= ENGINE_ID_DIGF) { - vpg_inst = eng_id; - afmt_inst = eng_id; - } else + if (eng_id < 0 || eng_id >= ARRAY_SIZE(stream_enc_regs)) return NULL; + vpg_inst = eng_id; + afmt_inst = eng_id; + enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL); vpg = dcn31_vpg_create(ctx, vpg_inst); afmt = dcn31_afmt_create(ctx, afmt_inst); diff --git a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c index a1da3e5812ce3..9342f0b8bab2a 100644 --- a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c +++ b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c @@ -3469,6 +3469,11 @@ static void si_apply_state_adjust_rules(struct amdgpu_device *adev, max_sclk = 60000; max_mclk = 80000; } + if ((adev->pdev->device == 0x666f) && + (adev->pdev->revision == 0x00)) { + max_sclk = 80000; + max_mclk = 95000; + } } else if (adev->asic_type == CHIP_OLAND) { if ((adev->pdev->revision == 0xC7) || (adev->pdev->revision == 0x80) || diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c index 3787db014501e..ae8d7b017968d 100644 --- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c +++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c @@ -78,8 +78,6 @@ drm_plane_state_to_atmel_hlcdc_plane_state(struct drm_plane_state *s) return container_of(s, struct atmel_hlcdc_plane_state, base); } -#define SUBPIXEL_MASK 0xffff - static uint32_t rgb_formats[] = { DRM_FORMAT_C8, DRM_FORMAT_XRGB4444, @@ -744,24 +742,15 @@ static int atmel_hlcdc_plane_atomic_check(struct drm_plane *p, if (ret || !s->visible) return ret; - hstate->src_x = s->src.x1; - hstate->src_y = s->src.y1; - hstate->src_w = drm_rect_width(&s->src); - hstate->src_h = drm_rect_height(&s->src); + hstate->src_x = s->src.x1 >> 16; + hstate->src_y = s->src.y1 >> 16; + hstate->src_w = drm_rect_width(&s->src) >> 16; + hstate->src_h = drm_rect_height(&s->src) >> 16; hstate->crtc_x = s->dst.x1; hstate->crtc_y = s->dst.y1; hstate->crtc_w = drm_rect_width(&s->dst); hstate->crtc_h = drm_rect_height(&s->dst); - if ((hstate->src_x | hstate->src_y | hstate->src_w | hstate->src_h) & - SUBPIXEL_MASK) - return -EINVAL; - - hstate->src_x >>= 16; - hstate->src_y >>= 16; - hstate->src_w >>= 16; - hstate->src_h >>= 16; - hstate->nplanes = fb->format->num_planes; if (hstate->nplanes > ATMEL_HLCDC_LAYER_MAX_PLANES) return -EINVAL; @@ -1185,8 +1174,7 @@ atmel_hlcdc_plane_atomic_duplicate_state(struct drm_plane *p) return NULL; } - if (copy->base.fb) - drm_framebuffer_get(copy->base.fb); + __drm_atomic_helper_plane_duplicate_state(p, ©->base); return ©->base; } @@ -1204,8 +1192,7 @@ static void atmel_hlcdc_plane_atomic_destroy_state(struct drm_plane *p, state->dscrs[i]->self); } - if (s->fb) - drm_framebuffer_put(s->fb); + __drm_atomic_helper_plane_destroy_state(s); kfree(state); } diff --git a/drivers/gpu/drm/bridge/analogix/anx7625.c b/drivers/gpu/drm/bridge/analogix/anx7625.c index 0bb8be08a77cc..4e49e4f28d552 100644 --- a/drivers/gpu/drm/bridge/analogix/anx7625.c +++ b/drivers/gpu/drm/bridge/analogix/anx7625.c @@ -1801,7 +1801,7 @@ static const struct drm_edid *anx7625_edid_read(struct anx7625_data *ctx) return NULL; } - ctx->cached_drm_edid = drm_edid_alloc(edid_buf, edid_num * EDID_LENGTH); + ctx->cached_drm_edid = drm_edid_alloc(edid_buf, edid_num * ONE_BLOCK_SIZE); kfree(edid_buf); out: diff --git a/drivers/gpu/drm/display/drm_dp_mst_topology.c b/drivers/gpu/drm/display/drm_dp_mst_topology.c index 64e5c176d5cce..be749dcad3b58 100644 --- a/drivers/gpu/drm/display/drm_dp_mst_topology.c +++ b/drivers/gpu/drm/display/drm_dp_mst_topology.c @@ -4572,7 +4572,8 @@ int drm_dp_atomic_release_time_slots(struct drm_atomic_state *state, if (!payload->delete) { payload->pbn = 0; payload->delete = true; - topology_state->payload_mask &= ~BIT(payload->vcpi - 1); + if (payload->vcpi > 0) + topology_state->payload_mask &= ~BIT(payload->vcpi - 1); } return 0; diff --git a/drivers/gpu/drm/drm_buddy.c b/drivers/gpu/drm/drm_buddy.c index f2c92902e4a30..640d93070bb7c 100644 --- a/drivers/gpu/drm/drm_buddy.c +++ b/drivers/gpu/drm/drm_buddy.c @@ -419,6 +419,7 @@ void drm_buddy_fini(struct drm_buddy *mm) for_each_free_tree(i) kfree(mm->free_trees[i]); + kfree(mm->free_trees); kfree(mm->roots); } EXPORT_SYMBOL(drm_buddy_fini); @@ -1154,6 +1155,15 @@ int drm_buddy_alloc_blocks(struct drm_buddy *mm, order = fls(pages) - 1; min_order = ilog2(min_block_size) - ilog2(mm->chunk_size); + if (order > mm->max_order || size > mm->size) { + if ((flags & DRM_BUDDY_CONTIGUOUS_ALLOCATION) && + !(flags & DRM_BUDDY_RANGE_ALLOCATION)) + return __alloc_contig_try_harder(mm, original_size, + original_min_size, blocks); + + return -EINVAL; + } + do { order = min(order, (unsigned int)fls(pages) - 1); BUG_ON(order > mm->max_order); diff --git a/drivers/gpu/drm/drm_client_modeset.c b/drivers/gpu/drm/drm_client_modeset.c index 9c2c3b0c8c470..eaf71c9ecf136 100644 --- a/drivers/gpu/drm/drm_client_modeset.c +++ b/drivers/gpu/drm/drm_client_modeset.c @@ -930,7 +930,8 @@ int drm_client_modeset_probe(struct drm_client_dev *client, unsigned int width, mutex_unlock(&client->modeset_mutex); out: kfree(crtcs); - modes_destroy(dev, modes, connector_count); + if (modes) + modes_destroy(dev, modes, connector_count); kfree(modes); kfree(offsets); kfree(enabled); diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index 365b5737ca2c6..5c6da91bbba17 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -15,6 +15,8 @@ #include #endif +#include + #include #include #include @@ -861,6 +863,67 @@ struct drm_gem_object *drm_gem_shmem_prime_import_no_map(struct drm_device *dev, } EXPORT_SYMBOL_GPL(drm_gem_shmem_prime_import_no_map); +/* + * Kunit helpers + */ + +#if IS_ENABLED(CONFIG_KUNIT) +int drm_gem_shmem_vmap(struct drm_gem_shmem_object *shmem, struct iosys_map *map) +{ + struct drm_gem_object *obj = &shmem->base; + int ret; + + ret = dma_resv_lock_interruptible(obj->resv, NULL); + if (ret) + return ret; + ret = drm_gem_shmem_vmap_locked(shmem, map); + dma_resv_unlock(obj->resv); + + return ret; +} +EXPORT_SYMBOL_IF_KUNIT(drm_gem_shmem_vmap); + +void drm_gem_shmem_vunmap(struct drm_gem_shmem_object *shmem, struct iosys_map *map) +{ + struct drm_gem_object *obj = &shmem->base; + + dma_resv_lock_interruptible(obj->resv, NULL); + drm_gem_shmem_vunmap_locked(shmem, map); + dma_resv_unlock(obj->resv); +} +EXPORT_SYMBOL_IF_KUNIT(drm_gem_shmem_vunmap); + +int drm_gem_shmem_madvise(struct drm_gem_shmem_object *shmem, int madv) +{ + struct drm_gem_object *obj = &shmem->base; + int ret; + + ret = dma_resv_lock_interruptible(obj->resv, NULL); + if (ret) + return ret; + ret = drm_gem_shmem_madvise_locked(shmem, madv); + dma_resv_unlock(obj->resv); + + return ret; +} +EXPORT_SYMBOL_IF_KUNIT(drm_gem_shmem_madvise); + +int drm_gem_shmem_purge(struct drm_gem_shmem_object *shmem) +{ + struct drm_gem_object *obj = &shmem->base; + int ret; + + ret = dma_resv_lock_interruptible(obj->resv, NULL); + if (ret) + return ret; + drm_gem_shmem_purge_locked(shmem); + dma_resv_unlock(obj->resv); + + return 0; +} +EXPORT_SYMBOL_IF_KUNIT(drm_gem_shmem_purge); +#endif + MODULE_DESCRIPTION("DRM SHMEM memory-management helpers"); MODULE_IMPORT_NS("DMA_BUF"); MODULE_LICENSE("GPL"); diff --git a/drivers/gpu/drm/drm_property.c b/drivers/gpu/drm/drm_property.c index 596272149a359..3c88b5fbdf28c 100644 --- a/drivers/gpu/drm/drm_property.c +++ b/drivers/gpu/drm/drm_property.c @@ -562,7 +562,7 @@ drm_property_create_blob(struct drm_device *dev, size_t length, if (!length || length > INT_MAX - sizeof(struct drm_property_blob)) return ERR_PTR(-EINVAL); - blob = kvzalloc(sizeof(struct drm_property_blob)+length, GFP_KERNEL); + blob = kvzalloc(sizeof(struct drm_property_blob) + length, GFP_KERNEL_ACCOUNT); if (!blob) return ERR_PTR(-ENOMEM); diff --git a/drivers/gpu/drm/drm_syncobj.c b/drivers/gpu/drm/drm_syncobj.c index e1b0fa4000cdd..7eb2cdbc574a0 100644 --- a/drivers/gpu/drm/drm_syncobj.c +++ b/drivers/gpu/drm/drm_syncobj.c @@ -900,7 +900,7 @@ drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data, return drm_syncobj_export_sync_file(file_private, args->handle, point, &args->fd); - if (args->point) + if (point) return -EINVAL; return drm_syncobj_handle_to_fd(file_private, args->handle, @@ -934,7 +934,7 @@ drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data, args->handle, point); - if (args->point) + if (point) return -EINVAL; return drm_syncobj_fd_to_handle(file_private, args->fd, diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.h b/drivers/gpu/drm/exynos/exynos_drm_drv.h index 23646e55f142c..06c29ff2aac0e 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_drv.h +++ b/drivers/gpu/drm/exynos/exynos_drm_drv.h @@ -199,6 +199,7 @@ struct drm_exynos_file_private { struct exynos_drm_private { struct device *g2d_dev; struct device *dma_dev; + struct device *vidi_dev; void *mapping; /* for atomic commit */ diff --git a/drivers/gpu/drm/exynos/exynos_drm_vidi.c b/drivers/gpu/drm/exynos/exynos_drm_vidi.c index b80410a3e4aad..37733f2ac0e7a 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_vidi.c +++ b/drivers/gpu/drm/exynos/exynos_drm_vidi.c @@ -231,9 +231,14 @@ ATTRIBUTE_GROUPS(vidi); int vidi_connection_ioctl(struct drm_device *drm_dev, void *data, struct drm_file *file_priv) { - struct vidi_context *ctx = dev_get_drvdata(drm_dev->dev); + struct exynos_drm_private *priv = drm_dev->dev_private; + struct device *dev = priv ? priv->vidi_dev : NULL; + struct vidi_context *ctx = dev ? dev_get_drvdata(dev) : NULL; struct drm_exynos_vidi_connection *vidi = data; + if (!ctx) + return -ENODEV; + if (!vidi) { DRM_DEV_DEBUG_KMS(ctx->dev, "user data for vidi is null.\n"); @@ -257,13 +262,27 @@ int vidi_connection_ioctl(struct drm_device *drm_dev, void *data, if (vidi->connection) { const struct drm_edid *drm_edid; - const struct edid *raw_edid; + const void __user *edid_userptr = u64_to_user_ptr(vidi->edid); + void *edid_buf; + struct edid hdr; size_t size; - raw_edid = (const struct edid *)(unsigned long)vidi->edid; - size = (raw_edid->extensions + 1) * EDID_LENGTH; + if (copy_from_user(&hdr, edid_userptr, sizeof(hdr))) + return -EFAULT; + + size = (hdr.extensions + 1) * EDID_LENGTH; + + edid_buf = kmalloc(size, GFP_KERNEL); + if (!edid_buf) + return -ENOMEM; + + if (copy_from_user(edid_buf, edid_userptr, size)) { + kfree(edid_buf); + return -EFAULT; + } - drm_edid = drm_edid_alloc(raw_edid, size); + drm_edid = drm_edid_alloc(edid_buf, size); + kfree(edid_buf); if (!drm_edid) return -ENOMEM; @@ -393,6 +412,7 @@ static int vidi_bind(struct device *dev, struct device *master, void *data) { struct vidi_context *ctx = dev_get_drvdata(dev); struct drm_device *drm_dev = data; + struct exynos_drm_private *priv = drm_dev->dev_private; struct drm_encoder *encoder = &ctx->encoder; struct exynos_drm_plane *exynos_plane; struct exynos_drm_plane_config plane_config = { 0 }; @@ -400,6 +420,8 @@ static int vidi_bind(struct device *dev, struct device *master, void *data) int ret; ctx->drm_dev = drm_dev; + if (priv) + priv->vidi_dev = dev; plane_config.pixel_formats = formats; plane_config.num_pixel_formats = ARRAY_SIZE(formats); @@ -445,8 +467,12 @@ static int vidi_bind(struct device *dev, struct device *master, void *data) static void vidi_unbind(struct device *dev, struct device *master, void *data) { struct vidi_context *ctx = dev_get_drvdata(dev); + struct drm_device *drm_dev = data; + struct exynos_drm_private *priv = drm_dev->dev_private; timer_delete_sync(&ctx->timer); + if (priv) + priv->vidi_dev = NULL; } static const struct component_ops vidi_component_ops = { diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h index 4add05c7f161a..f9ee7ebfec55c 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h +++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h @@ -40,6 +40,10 @@ struct hibmc_dp_dev { struct mutex lock; /* protects concurrent RW in hibmc_dp_reg_write_field() */ struct hibmc_dp_link link; u8 dpcd[DP_RECEIVER_CAP_SIZE]; + u8 downstream_ports[DP_MAX_DOWNSTREAM_PORTS]; + struct drm_dp_desc desc; + bool is_branch; + int hpd_status; void __iomem *serdes_base; }; diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_config.h b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_config.h index 08f9e1caf7fcb..efb30a7584758 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_config.h +++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_config.h @@ -17,5 +17,7 @@ #define HIBMC_DP_LINK_RATE_CAL 27 #define HIBMC_DP_SYNC_DELAY(lanes) ((lanes) == 0x2 ? 86 : 46) #define HIBMC_DP_INT_ENABLE 0xc +/* HIBMC_DP_LINK_RATE_CAL * 10000 * 80% = 216000 */ +#define DP_MODE_VALI_CAL 216000 #endif diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.c b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.c index 8f0daec7d1749..d5bd3c45649b2 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.c +++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.c @@ -2,6 +2,7 @@ // Copyright (c) 2024 Hisilicon Limited. #include +#include #include #include "dp_config.h" #include "dp_comm.h" @@ -176,13 +177,18 @@ int hibmc_dp_hw_init(struct hibmc_dp *dp) dp_dev->link.cap.lanes = 0x2; dp_dev->link.cap.link_rate = DP_LINK_BW_8_1; - /* hdcp data */ - writel(HIBMC_DP_HDCP, dp_dev->base + HIBMC_DP_HDCP_CFG); /* int init */ writel(0, dp_dev->base + HIBMC_DP_INTR_ENABLE); writel(HIBMC_DP_INT_RST, dp_dev->base + HIBMC_DP_INTR_ORIGINAL_STATUS); + /* clr colorbar */ + writel(0, dp_dev->base + HIBMC_DP_COLOR_BAR_CTRL); /* rst */ + writel(0, dp_dev->base + HIBMC_DP_DPTX_RST_CTRL); + usleep_range(30, 50); + /* de-rst */ writel(HIBMC_DP_DPTX_RST, dp_dev->base + HIBMC_DP_DPTX_RST_CTRL); + /* hdcp data */ + writel(HIBMC_DP_HDCP, dp_dev->base + HIBMC_DP_HDCP_CFG); /* clock enable */ writel(HIBMC_DP_CLK_EN, dp_dev->base + HIBMC_DP_DPTX_CLK_CTRL); @@ -263,6 +269,16 @@ void hibmc_dp_reset_link(struct hibmc_dp *dp) dp->dp_dev->link.status.channel_equalized = false; } +u8 hibmc_dp_get_link_rate(struct hibmc_dp *dp) +{ + return dp->dp_dev->link.cap.link_rate; +} + +u8 hibmc_dp_get_lanes(struct hibmc_dp *dp) +{ + return dp->dp_dev->link.cap.lanes; +} + static const struct hibmc_dp_color_raw g_rgb_raw[] = { {CBAR_COLOR_BAR, 0x000, 0x000, 0x000}, {CBAR_WHITE, 0xfff, 0xfff, 0xfff}, @@ -305,3 +321,21 @@ void hibmc_dp_set_cbar(struct hibmc_dp *dp, const struct hibmc_dp_cbar_cfg *cfg) hibmc_dp_reg_write_field(dp_dev, HIBMC_DP_COLOR_BAR_CTRL, BIT(0), cfg->enable); writel(HIBMC_DP_SYNC_EN_MASK, dp_dev->base + HIBMC_DP_TIMING_SYNC_CTRL); } + +bool hibmc_dp_check_hpd_status(struct hibmc_dp *dp, int exp_status) +{ + u32 status; + int ret; + + ret = readl_poll_timeout(dp->dp_dev->base + HIBMC_DP_HPD_STATUS, status, + FIELD_GET(HIBMC_DP_HPD_CUR_STATE, status) == exp_status, + 1000, 100000); /* DP spec says 100ms */ + if (ret) { + drm_dbg_dp(dp->drm_dev, "wait hpd status timeout"); + return false; + } + + dp->dp_dev->hpd_status = exp_status; + + return true; +} diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h index 665f5b166dfb5..31316fe1ea8db 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h +++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h @@ -14,6 +14,11 @@ struct hibmc_dp_dev; +enum hibmc_hpd_status { + HIBMC_HPD_OUT, + HIBMC_HPD_IN, +}; + enum hibmc_dp_cbar_pattern { CBAR_COLOR_BAR, CBAR_WHITE, @@ -60,5 +65,8 @@ void hibmc_dp_reset_link(struct hibmc_dp *dp); void hibmc_dp_hpd_cfg(struct hibmc_dp *dp); void hibmc_dp_enable_int(struct hibmc_dp *dp); void hibmc_dp_disable_int(struct hibmc_dp *dp); +bool hibmc_dp_check_hpd_status(struct hibmc_dp *dp, int exp_status); +u8 hibmc_dp_get_link_rate(struct hibmc_dp *dp); +u8 hibmc_dp_get_lanes(struct hibmc_dp *dp); #endif diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_reg.h b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_reg.h index 394b1e933c3ae..64306abcd9866 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_reg.h +++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_reg.h @@ -24,6 +24,9 @@ #define HIBMC_DP_CFG_AUX_READY_DATA_BYTE GENMASK(16, 12) #define HIBMC_DP_CFG_AUX GENMASK(24, 17) +#define HIBMC_DP_HPD_STATUS 0x98 +#define HIBMC_DP_HPD_CUR_STATE GENMASK(7, 4) + #define HIBMC_DP_PHYIF_CTRL0 0xa0 #define HIBMC_DP_CFG_SCRAMBLE_EN BIT(0) #define HIBMC_DP_CFG_PAT_SEL GENMASK(7, 4) diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c index d06832e62e966..616821e3c933b 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c +++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c @@ -12,6 +12,8 @@ #include "hibmc_drm_drv.h" #include "dp/dp_hw.h" +#include "dp/dp_comm.h" +#include "dp/dp_config.h" #define DP_MASKED_SINK_HPD_PLUG_INT BIT(2) @@ -31,17 +33,76 @@ static int hibmc_dp_connector_get_modes(struct drm_connector *connector) return count; } +static bool hibmc_dp_get_dpcd(struct hibmc_dp_dev *dp_dev) +{ + int ret; + + ret = drm_dp_read_dpcd_caps(dp_dev->aux, dp_dev->dpcd); + if (ret) + return false; + + dp_dev->is_branch = drm_dp_is_branch(dp_dev->dpcd); + + ret = drm_dp_read_desc(dp_dev->aux, &dp_dev->desc, dp_dev->is_branch); + if (ret) + return false; + + ret = drm_dp_read_downstream_info(dp_dev->aux, dp_dev->dpcd, dp_dev->downstream_ports); + if (ret) + return false; + + return true; +} + static int hibmc_dp_detect(struct drm_connector *connector, struct drm_modeset_acquire_ctx *ctx, bool force) { - mdelay(200); + struct hibmc_dp *dp = to_hibmc_dp(connector); + struct hibmc_dp_dev *dp_dev = dp->dp_dev; + int ret; + + if (dp->irq_status) { + if (dp_dev->hpd_status != HIBMC_HPD_IN) + return connector_status_disconnected; + } + + if (!hibmc_dp_get_dpcd(dp_dev)) + return connector_status_disconnected; + + if (!dp_dev->is_branch) + return connector_status_connected; + + if (drm_dp_read_sink_count_cap(connector, dp_dev->dpcd, &dp_dev->desc) && + dp_dev->downstream_ports[0] & DP_DS_PORT_HPD) { + ret = drm_dp_read_sink_count(dp_dev->aux); + if (ret > 0) + return connector_status_connected; + } + + return connector_status_disconnected; +} + +static int hibmc_dp_mode_valid(struct drm_connector *connector, + const struct drm_display_mode *mode, + struct drm_modeset_acquire_ctx *ctx, + enum drm_mode_status *status) +{ + struct hibmc_dp *dp = to_hibmc_dp(connector); + u64 cur_val, max_val; - return drm_connector_helper_detect_from_ddc(connector, ctx, force); + /* check DP link BW */ + cur_val = (u64)mode->clock * HIBMC_DP_BPP; + max_val = (u64)hibmc_dp_get_link_rate(dp) * DP_MODE_VALI_CAL * hibmc_dp_get_lanes(dp); + + *status = cur_val > max_val ? MODE_CLOCK_HIGH : MODE_OK; + + return 0; } static const struct drm_connector_helper_funcs hibmc_dp_conn_helper_funcs = { .get_modes = hibmc_dp_connector_get_modes, .detect_ctx = hibmc_dp_detect, + .mode_valid_ctx = hibmc_dp_mode_valid, }; static int hibmc_dp_late_register(struct drm_connector *connector) @@ -115,7 +176,7 @@ irqreturn_t hibmc_dp_hpd_isr(int irq, void *arg) { struct drm_device *dev = (struct drm_device *)arg; struct hibmc_drm_private *priv = to_hibmc_drm_private(dev); - int idx; + int idx, exp_status; if (!drm_dev_enter(dev, &idx)) return -ENODEV; @@ -123,12 +184,14 @@ irqreturn_t hibmc_dp_hpd_isr(int irq, void *arg) if (priv->dp.irq_status & DP_MASKED_SINK_HPD_PLUG_INT) { drm_dbg_dp(&priv->dev, "HPD IN isr occur!\n"); hibmc_dp_hpd_cfg(&priv->dp); + exp_status = HIBMC_HPD_IN; } else { drm_dbg_dp(&priv->dev, "HPD OUT isr occur!\n"); hibmc_dp_reset_link(&priv->dp); + exp_status = HIBMC_HPD_OUT; } - if (dev->registered) + if (hibmc_dp_check_hpd_status(&priv->dp, exp_status)) drm_connector_helper_hpd_irq_event(&priv->dp.connector); drm_dev_exit(idx); diff --git a/drivers/gpu/drm/i915/display/intel_acpi.c b/drivers/gpu/drm/i915/display/intel_acpi.c index 1addd62882413..1e8b9d1756988 100644 --- a/drivers/gpu/drm/i915/display/intel_acpi.c +++ b/drivers/gpu/drm/i915/display/intel_acpi.c @@ -96,6 +96,7 @@ static void intel_dsm_platform_mux_info(acpi_handle dhandle) if (!pkg->package.count) { DRM_DEBUG_DRIVER("no connection in _DSM\n"); + ACPI_FREE(pkg); return; } diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c index 2eab591a8ef56..be3d54729a440 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.c +++ b/drivers/gpu/drm/i915/display/intel_dp.c @@ -2523,16 +2523,30 @@ intel_dp_compute_config_link_bpp_limits(struct intel_dp *intel_dp, return true; } -static void -intel_dp_dsc_compute_pipe_bpp_limits(struct intel_dp *intel_dp, +static bool +intel_dp_dsc_compute_pipe_bpp_limits(struct intel_connector *connector, struct link_config_limits *limits) { - struct intel_display *display = to_intel_display(intel_dp); + struct intel_display *display = to_intel_display(connector); + const struct link_config_limits orig_limits = *limits; int dsc_min_bpc = intel_dp_dsc_min_src_input_bpc(); int dsc_max_bpc = intel_dp_dsc_max_src_input_bpc(display); - limits->pipe.max_bpp = clamp(limits->pipe.max_bpp, dsc_min_bpc * 3, dsc_max_bpc * 3); - limits->pipe.min_bpp = clamp(limits->pipe.min_bpp, dsc_min_bpc * 3, dsc_max_bpc * 3); + limits->pipe.min_bpp = max(limits->pipe.min_bpp, dsc_min_bpc * 3); + limits->pipe.max_bpp = min(limits->pipe.max_bpp, dsc_max_bpc * 3); + + if (limits->pipe.min_bpp <= 0 || + limits->pipe.min_bpp > limits->pipe.max_bpp) { + drm_dbg_kms(display->drm, + "[CONNECTOR:%d:%s] Invalid DSC src/sink input BPP (src:%d-%d pipe:%d-%d)\n", + connector->base.base.id, connector->base.name, + dsc_min_bpc * 3, dsc_max_bpc * 3, + orig_limits.pipe.min_bpp, orig_limits.pipe.max_bpp); + + return false; + } + + return true; } bool @@ -2543,6 +2557,7 @@ intel_dp_compute_config_limits(struct intel_dp *intel_dp, bool dsc, struct link_config_limits *limits) { + struct intel_display *display = to_intel_display(intel_dp); bool is_mst = intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST); struct intel_connector *connector = to_intel_connector(conn_state->connector); @@ -2555,8 +2570,7 @@ intel_dp_compute_config_limits(struct intel_dp *intel_dp, limits->min_lane_count = intel_dp_min_lane_count(intel_dp); limits->max_lane_count = intel_dp_max_lane_count(intel_dp); - limits->pipe.min_bpp = intel_dp_in_hdr_mode(conn_state) ? 30 : - intel_dp_min_bpp(crtc_state->output_format); + limits->pipe.min_bpp = intel_dp_min_bpp(crtc_state->output_format); if (is_mst) { /* * FIXME: If all the streams can't fit into the link with their @@ -2572,8 +2586,21 @@ intel_dp_compute_config_limits(struct intel_dp *intel_dp, respect_downstream_limits); } - if (dsc) - intel_dp_dsc_compute_pipe_bpp_limits(intel_dp, limits); + if (!dsc && intel_dp_in_hdr_mode(conn_state)) { + if (intel_dp_supports_dsc(intel_dp, connector, crtc_state) && + limits->pipe.max_bpp >= 30) + limits->pipe.min_bpp = max(limits->pipe.min_bpp, 30); + else + drm_dbg_kms(display->drm, + "[CONNECTOR:%d:%s] Can't force 30 bpp for HDR (pipe bpp: %d-%d DSC-support: %s)\n", + connector->base.base.id, connector->base.name, + limits->pipe.min_bpp, limits->pipe.max_bpp, + str_yes_no(intel_dp_supports_dsc(intel_dp, connector, + crtc_state))); + } + + if (dsc && !intel_dp_dsc_compute_pipe_bpp_limits(connector, limits)) + return false; if (is_mst || intel_dp->use_max_params) { /* @@ -2702,10 +2729,11 @@ intel_dp_compute_link_config(struct intel_encoder *encoder, } drm_dbg_kms(display->drm, - "DP lane count %d clock %d bpp input %d compressed " FXP_Q4_FMT " link rate required %d available %d\n", + "DP lane count %d clock %d bpp input %d compressed " FXP_Q4_FMT " HDR %s link rate required %d available %d\n", pipe_config->lane_count, pipe_config->port_clock, pipe_config->pipe_bpp, FXP_Q4_ARGS(pipe_config->dsc.compressed_bpp_x16), + str_yes_no(intel_dp_in_hdr_mode(conn_state)), intel_dp_config_required_rate(pipe_config), intel_dp_max_link_data_rate(intel_dp, pipe_config->port_clock, diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c index 6d9c95e5c0255..38d1df919d1ab 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.c +++ b/drivers/gpu/drm/i915/display/intel_psr.c @@ -813,7 +813,12 @@ static void intel_psr_enable_sink(struct intel_dp *intel_dp, void intel_psr_panel_replay_enable_sink(struct intel_dp *intel_dp) { - if (CAN_PANEL_REPLAY(intel_dp)) + /* + * NOTE: We might want to trigger mode set when + * disabling/enabling Panel Replay via debugfs interface to + * ensure this bit is cleared/set accordingly. + */ + if (CAN_PANEL_REPLAY(intel_dp) && panel_replay_global_enabled(intel_dp)) drm_dp_dpcd_writeb(&intel_dp->aux, PANEL_REPLAY_CONFIG, DP_PANEL_REPLAY_ENABLE); } diff --git a/drivers/gpu/drm/i915/display/intel_quirks.c b/drivers/gpu/drm/i915/display/intel_quirks.c index d2e16b79d6be1..1abbdd426e587 100644 --- a/drivers/gpu/drm/i915/display/intel_quirks.c +++ b/drivers/gpu/drm/i915/display/intel_quirks.c @@ -239,7 +239,7 @@ static struct intel_quirk intel_quirks[] = { { 0x0f31, 0x103c, 0x220f, quirk_invert_brightness }, /* Dell XPS 13 7390 2-in-1 */ - { 0x8a12, 0x1028, 0x08b0, quirk_edp_limit_rate_hbr2 }, + { 0x8a52, 0x1028, 0x08b0, quirk_edp_limit_rate_hbr2 }, }; static const struct intel_dpcd_quirk intel_dpcd_quirks[] = { diff --git a/drivers/gpu/drm/i915/intel_wakeref.c b/drivers/gpu/drm/i915/intel_wakeref.c index 7fa194de5d35b..5bd75a35d0a39 100644 --- a/drivers/gpu/drm/i915/intel_wakeref.c +++ b/drivers/gpu/drm/i915/intel_wakeref.c @@ -78,7 +78,7 @@ void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags) /* Assume we are not in process context and so cannot sleep. */ if (flags & INTEL_WAKEREF_PUT_ASYNC || !mutex_trylock(&wf->mutex)) { mod_delayed_work(wf->i915->unordered_wq, &wf->work, - FIELD_GET(INTEL_WAKEREF_PUT_DELAY, flags)); + FIELD_GET(INTEL_WAKEREF_PUT_DELAY_MASK, flags)); return; } diff --git a/drivers/gpu/drm/i915/intel_wakeref.h b/drivers/gpu/drm/i915/intel_wakeref.h index a2894a56e18fc..81308bac34bab 100644 --- a/drivers/gpu/drm/i915/intel_wakeref.h +++ b/drivers/gpu/drm/i915/intel_wakeref.h @@ -128,17 +128,16 @@ intel_wakeref_get_if_active(struct intel_wakeref *wf) return atomic_inc_not_zero(&wf->count); } -enum { - INTEL_WAKEREF_PUT_ASYNC_BIT = 0, - __INTEL_WAKEREF_PUT_LAST_BIT__ -}; - static inline void intel_wakeref_might_get(struct intel_wakeref *wf) { might_lock(&wf->mutex); } +/* flags for __intel_wakeref_put() and __intel_wakeref_put_last */ +#define INTEL_WAKEREF_PUT_ASYNC BIT(0) +#define INTEL_WAKEREF_PUT_DELAY_MASK GENMASK(BITS_PER_LONG - 1, 1) + /** * __intel_wakeref_put: Release the wakeref * @wf: the wakeref @@ -154,9 +153,6 @@ intel_wakeref_might_get(struct intel_wakeref *wf) */ static inline void __intel_wakeref_put(struct intel_wakeref *wf, unsigned long flags) -#define INTEL_WAKEREF_PUT_ASYNC BIT(INTEL_WAKEREF_PUT_ASYNC_BIT) -#define INTEL_WAKEREF_PUT_DELAY \ - GENMASK(BITS_PER_LONG - 1, __INTEL_WAKEREF_PUT_LAST_BIT__) { INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0); if (unlikely(!atomic_add_unless(&wf->count, -1, 1))) @@ -181,7 +177,7 @@ intel_wakeref_put_delay(struct intel_wakeref *wf, unsigned long delay) { __intel_wakeref_put(wf, INTEL_WAKEREF_PUT_ASYNC | - FIELD_PREP(INTEL_WAKEREF_PUT_DELAY, delay)); + FIELD_PREP(INTEL_WAKEREF_PUT_DELAY_MASK, delay)); } static inline void diff --git a/drivers/gpu/drm/imx/ipuv3/parallel-display.c b/drivers/gpu/drm/imx/ipuv3/parallel-display.c index 7fc6af7033078..d5f2ee41c03fe 100644 --- a/drivers/gpu/drm/imx/ipuv3/parallel-display.c +++ b/drivers/gpu/drm/imx/ipuv3/parallel-display.c @@ -256,7 +256,9 @@ static int imx_pd_probe(struct platform_device *pdev) platform_set_drvdata(pdev, imxpd); - devm_drm_bridge_add(dev, &imxpd->bridge); + ret = devm_drm_bridge_add(dev, &imxpd->bridge); + if (ret) + return ret; return component_add(dev, &imx_pd_ops); } diff --git a/drivers/gpu/drm/logicvc/logicvc_drm.c b/drivers/gpu/drm/logicvc/logicvc_drm.c index 204b0fee55d0b..bbebf4fc7f51a 100644 --- a/drivers/gpu/drm/logicvc/logicvc_drm.c +++ b/drivers/gpu/drm/logicvc/logicvc_drm.c @@ -92,7 +92,6 @@ static int logicvc_drm_config_parse(struct logicvc_drm *logicvc) struct device *dev = drm_dev->dev; struct device_node *of_node = dev->of_node; struct logicvc_drm_config *config = &logicvc->config; - struct device_node *layers_node; int ret; logicvc_of_property_parse_bool(of_node, LOGICVC_OF_PROPERTY_DITHERING, @@ -128,7 +127,8 @@ static int logicvc_drm_config_parse(struct logicvc_drm *logicvc) if (ret) return ret; - layers_node = of_get_child_by_name(of_node, "layers"); + struct device_node *layers_node __free(device_node) = + of_get_child_by_name(of_node, "layers"); if (!layers_node) { drm_err(drm_dev, "Missing non-optional layers node\n"); return -EINVAL; diff --git a/drivers/gpu/drm/msm/adreno/a2xx_gpu.c b/drivers/gpu/drm/msm/adreno/a2xx_gpu.c index 963c0f669ee50..e67ed58aa3d8f 100644 --- a/drivers/gpu/drm/msm/adreno/a2xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a2xx_gpu.c @@ -77,7 +77,10 @@ static bool a2xx_me_init(struct msm_gpu *gpu) /* Vertex and Pixel Shader Start Addresses in instructions * (3 DWORDS per instruction) */ - OUT_RING(ring, 0x80000180); + if (adreno_is_a225(adreno_gpu)) + OUT_RING(ring, 0x80000300); + else + OUT_RING(ring, 0x80000180); /* Maximum Contexts */ OUT_RING(ring, 0x00000001); /* Write Confirm Interval and The CP will wait the diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c index 258edaa18fc02..777eab5ad844e 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c @@ -784,24 +784,24 @@ static void _dpu_encoder_update_vsync_source(struct dpu_encoder_virt *dpu_enc, return; } + vsync_cfg.vsync_source = disp_info->vsync_source; + vsync_cfg.frame_rate = drm_mode_vrefresh(&dpu_enc->base.crtc->state->adjusted_mode); + if (hw_mdptop->ops.setup_vsync_source) { for (i = 0; i < dpu_enc->num_phys_encs; i++) vsync_cfg.ppnumber[i] = dpu_enc->hw_pp[i]->idx; vsync_cfg.pp_count = dpu_enc->num_phys_encs; - vsync_cfg.frame_rate = drm_mode_vrefresh(&dpu_enc->base.crtc->state->adjusted_mode); - - vsync_cfg.vsync_source = disp_info->vsync_source; hw_mdptop->ops.setup_vsync_source(hw_mdptop, &vsync_cfg); + } - for (i = 0; i < dpu_enc->num_phys_encs; i++) { - phys_enc = dpu_enc->phys_encs[i]; + for (i = 0; i < dpu_enc->num_phys_encs; i++) { + phys_enc = dpu_enc->phys_encs[i]; - if (phys_enc->has_intf_te && phys_enc->hw_intf->ops.vsync_sel) - phys_enc->hw_intf->ops.vsync_sel(phys_enc->hw_intf, - vsync_cfg.vsync_source); - } + if (phys_enc->has_intf_te && phys_enc->hw_intf->ops.vsync_sel) + phys_enc->hw_intf->ops.vsync_sel(phys_enc->hw_intf, + &vsync_cfg); } } diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c index 0ec6d67c7c70b..93db1484f6069 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c @@ -681,10 +681,11 @@ static int dpu_encoder_phys_cmd_wait_for_commit_done( if (!dpu_encoder_phys_cmd_is_master(phys_enc)) return 0; - if (phys_enc->hw_ctl->ops.is_started(phys_enc->hw_ctl)) - return dpu_encoder_phys_cmd_wait_for_tx_complete(phys_enc); + if (phys_enc->irq[INTR_IDX_CTL_START] && + !phys_enc->hw_ctl->ops.is_started(phys_enc->hw_ctl)) + return _dpu_encoder_phys_cmd_wait_for_ctl_start(phys_enc); - return _dpu_encoder_phys_cmd_wait_for_ctl_start(phys_enc); + return dpu_encoder_phys_cmd_wait_for_tx_complete(phys_enc); } static void dpu_encoder_phys_cmd_handle_post_kickoff( diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c index a80ac82a96255..7e620f5909849 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.c @@ -67,6 +67,10 @@ #define INTF_MISR_CTRL 0x180 #define INTF_MISR_SIGNATURE 0x184 +#define INTF_WD_TIMER_0_CTL 0x230 +#define INTF_WD_TIMER_0_CTL2 0x234 +#define INTF_WD_TIMER_0_LOAD_VALUE 0x238 + #define INTF_MUX 0x25C #define INTF_STATUS 0x26C #define INTF_AVR_CONTROL 0x270 @@ -475,7 +479,20 @@ static int dpu_hw_intf_get_vsync_info(struct dpu_hw_intf *intf, } static void dpu_hw_intf_vsync_sel(struct dpu_hw_intf *intf, - enum dpu_vsync_source vsync_source) + struct dpu_vsync_source_cfg *cfg) +{ + struct dpu_hw_blk_reg_map *c; + + if (!intf) + return; + + c = &intf->hw; + + DPU_REG_WRITE(c, INTF_TEAR_MDP_VSYNC_SEL, (cfg->vsync_source & 0xf)); +} + +static void dpu_hw_intf_vsync_sel_v8(struct dpu_hw_intf *intf, + struct dpu_vsync_source_cfg *cfg) { struct dpu_hw_blk_reg_map *c; @@ -484,7 +501,30 @@ static void dpu_hw_intf_vsync_sel(struct dpu_hw_intf *intf, c = &intf->hw; - DPU_REG_WRITE(c, INTF_TEAR_MDP_VSYNC_SEL, (vsync_source & 0xf)); + if (cfg->vsync_source >= DPU_VSYNC_SOURCE_WD_TIMER_4 && + cfg->vsync_source <= DPU_VSYNC_SOURCE_WD_TIMER_1) { + pr_warn_once("DPU 8.x supports only GPIOs and timer0 as TE sources\n"); + return; + } + + if (cfg->vsync_source == DPU_VSYNC_SOURCE_WD_TIMER_0) { + u32 reg; + + DPU_REG_WRITE(c, INTF_WD_TIMER_0_LOAD_VALUE, + CALCULATE_WD_LOAD_VALUE(cfg->frame_rate)); + + DPU_REG_WRITE(c, INTF_WD_TIMER_0_CTL, BIT(0)); /* clear timer */ + + reg = BIT(8); /* enable heartbeat timer */ + reg |= BIT(0); /* enable WD timer */ + reg |= BIT(1); /* select default 16 clock ticks */ + DPU_REG_WRITE(c, INTF_WD_TIMER_0_CTL2, reg); + + /* make sure that timers are enabled/disabled for vsync state */ + wmb(); + } + + dpu_hw_intf_vsync_sel(intf, cfg); } static void dpu_hw_intf_disable_autorefresh(struct dpu_hw_intf *intf, @@ -598,7 +638,10 @@ struct dpu_hw_intf *dpu_hw_intf_init(struct drm_device *dev, c->ops.enable_tearcheck = dpu_hw_intf_enable_te; c->ops.disable_tearcheck = dpu_hw_intf_disable_te; c->ops.connect_external_te = dpu_hw_intf_connect_external_te; - c->ops.vsync_sel = dpu_hw_intf_vsync_sel; + if (mdss_rev->core_major_ver >= 8) + c->ops.vsync_sel = dpu_hw_intf_vsync_sel_v8; + else + c->ops.vsync_sel = dpu_hw_intf_vsync_sel; c->ops.disable_autorefresh = dpu_hw_intf_disable_autorefresh; } diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.h index f31067a9aaf1d..e84ab849d71a9 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_intf.h @@ -12,6 +12,7 @@ #include "dpu_hw_util.h" struct dpu_hw_intf; +struct dpu_vsync_source_cfg; /* intf timing settings */ struct dpu_hw_intf_timing_params { @@ -107,7 +108,7 @@ struct dpu_hw_intf_ops { int (*connect_external_te)(struct dpu_hw_intf *intf, bool enable_external_te); - void (*vsync_sel)(struct dpu_hw_intf *intf, enum dpu_vsync_source vsync_source); + void (*vsync_sel)(struct dpu_hw_intf *intf, struct dpu_vsync_source_cfg *cfg); /** * Disable autorefresh if enabled diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c index 6f1fc790ad6d8..6ff4902fce08e 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c @@ -72,6 +72,8 @@ #define SSPP_EXCL_REC_XY_REC1 0x188 #define SSPP_EXCL_REC_SIZE 0x1B4 #define SSPP_EXCL_REC_XY 0x1B8 +#define SSPP_UBWC_STATIC_CTRL_REC1 0x1c0 +#define SSPP_UBWC_ERROR_STATUS_REC1 0x1c8 #define SSPP_CLK_CTRL 0x330 /* SSPP_SRC_OP_MODE & OP_MODE_REC1 */ @@ -215,7 +217,7 @@ static void dpu_hw_sspp_setup_format(struct dpu_sw_pipe *pipe, u32 chroma_samp, unpack, src_format; u32 opmode = 0; u32 fast_clear = 0; - u32 op_mode_off, unpack_pat_off, format_off; + u32 op_mode_off, unpack_pat_off, format_off, ubwc_ctrl_off, ubwc_error_off; if (!ctx || !fmt) return; @@ -225,10 +227,21 @@ static void dpu_hw_sspp_setup_format(struct dpu_sw_pipe *pipe, op_mode_off = SSPP_SRC_OP_MODE; unpack_pat_off = SSPP_SRC_UNPACK_PATTERN; format_off = SSPP_SRC_FORMAT; + ubwc_ctrl_off = SSPP_UBWC_STATIC_CTRL; + ubwc_error_off = SSPP_UBWC_ERROR_STATUS; } else { op_mode_off = SSPP_SRC_OP_MODE_REC1; unpack_pat_off = SSPP_SRC_UNPACK_PATTERN_REC1; format_off = SSPP_SRC_FORMAT_REC1; + + /* reg wasn't present before DPU 8.0 */ + if (ctx->mdss_ver->core_major_ver >= 8) { + ubwc_ctrl_off = SSPP_UBWC_STATIC_CTRL_REC1; + ubwc_error_off = SSPP_UBWC_ERROR_STATUS_REC1; + } else { + ubwc_ctrl_off = SSPP_UBWC_STATIC_CTRL; + ubwc_error_off = SSPP_UBWC_ERROR_STATUS; + } } c = &ctx->hw; @@ -270,33 +283,35 @@ static void dpu_hw_sspp_setup_format(struct dpu_sw_pipe *pipe, ((fmt->bpp - 1) << 9); if (fmt->fetch_mode != MDP_FETCH_LINEAR) { + u32 hbb = ctx->ubwc->highest_bank_bit - 13; + if (MSM_FORMAT_IS_UBWC(fmt)) opmode |= MDSS_MDP_OP_BWC_EN; src_format |= (fmt->fetch_mode & 3) << 30; /*FRAME_FORMAT */ DPU_REG_WRITE(c, SSPP_FETCH_CONFIG, DPU_FETCH_CONFIG_RESET_VALUE | - ctx->ubwc->highest_bank_bit << 18); + hbb << 18); switch (ctx->ubwc->ubwc_enc_version) { case UBWC_1_0: fast_clear = fmt->alpha_enable ? BIT(31) : 0; - DPU_REG_WRITE(c, SSPP_UBWC_STATIC_CTRL, + DPU_REG_WRITE(c, ubwc_ctrl_off, fast_clear | (ctx->ubwc->ubwc_swizzle & 0x1) | BIT(8) | - (ctx->ubwc->highest_bank_bit << 4)); + (hbb << 4)); break; case UBWC_2_0: fast_clear = fmt->alpha_enable ? BIT(31) : 0; - DPU_REG_WRITE(c, SSPP_UBWC_STATIC_CTRL, + DPU_REG_WRITE(c, ubwc_ctrl_off, fast_clear | (ctx->ubwc->ubwc_swizzle) | - (ctx->ubwc->highest_bank_bit << 4)); + (hbb << 4)); break; case UBWC_3_0: - DPU_REG_WRITE(c, SSPP_UBWC_STATIC_CTRL, + DPU_REG_WRITE(c, ubwc_ctrl_off, BIT(30) | (ctx->ubwc->ubwc_swizzle) | - (ctx->ubwc->highest_bank_bit << 4)); + (hbb << 4)); break; case UBWC_4_0: - DPU_REG_WRITE(c, SSPP_UBWC_STATIC_CTRL, + DPU_REG_WRITE(c, ubwc_ctrl_off, MSM_FORMAT_IS_YUV(fmt) ? 0 : BIT(30)); break; } @@ -325,7 +340,7 @@ static void dpu_hw_sspp_setup_format(struct dpu_sw_pipe *pipe, DPU_REG_WRITE(c, op_mode_off, opmode); /* clear previous UBWC error */ - DPU_REG_WRITE(c, SSPP_UBWC_ERROR_STATUS, BIT(31)); + DPU_REG_WRITE(c, ubwc_error_off, BIT(31)); } static void dpu_hw_sspp_setup_pe_config(struct dpu_hw_sspp *ctx, diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.c index 96dc10589bee6..1ebd75d4f9be8 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_top.c @@ -22,13 +22,6 @@ #define TRAFFIC_SHAPER_WR_CLIENT(num) (0x060 + (num * 4)) #define TRAFFIC_SHAPER_FIXPOINT_FACTOR 4 -#define MDP_TICK_COUNT 16 -#define XO_CLK_RATE 19200 -#define MS_TICKS_IN_SEC 1000 - -#define CALCULATE_WD_LOAD_VALUE(fps) \ - ((uint32_t)((MS_TICKS_IN_SEC * XO_CLK_RATE)/(MDP_TICK_COUNT * fps))) - static void dpu_hw_setup_split_pipe(struct dpu_hw_mdp *mdp, struct split_pipe_cfg *cfg) { diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h index 67b08e99335dc..6fe65bc3bff4e 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_util.h @@ -21,6 +21,13 @@ #define TO_S15D16(_x_)((_x_) << 7) +#define MDP_TICK_COUNT 16 +#define XO_CLK_RATE 19200 +#define MS_TICKS_IN_SEC 1000 + +#define CALCULATE_WD_LOAD_VALUE(fps) \ + ((uint32_t)((MS_TICKS_IN_SEC * XO_CLK_RATE)/(MDP_TICK_COUNT * fps))) + extern const struct dpu_csc_cfg dpu_csc_YUV2RGB_601L; extern const struct dpu_csc_cfg dpu_csc10_YUV2RGB_601L; extern const struct dpu_csc_cfg dpu_csc10_rgb2yuv_601l; diff --git a/drivers/gpu/drm/msm/disp/mdp_format.c b/drivers/gpu/drm/msm/disp/mdp_format.c index 426782d50cb49..eebedb1a2636e 100644 --- a/drivers/gpu/drm/msm/disp/mdp_format.c +++ b/drivers/gpu/drm/msm/disp/mdp_format.c @@ -479,25 +479,25 @@ static const struct msm_format mdp_formats[] = { 0, BPC8, BPC8, BPC8, C2_R_Cr, C0_G_Y, C1_B_Cb, C0_G_Y, false, CHROMA_H2V1, 4, 2, MSM_FORMAT_FLAG_YUV, - MDP_FETCH_LINEAR, 2), + MDP_FETCH_LINEAR, 1), INTERLEAVED_YUV_FMT(UYVY, 0, BPC8, BPC8, BPC8, C1_B_Cb, C0_G_Y, C2_R_Cr, C0_G_Y, false, CHROMA_H2V1, 4, 2, MSM_FORMAT_FLAG_YUV, - MDP_FETCH_LINEAR, 2), + MDP_FETCH_LINEAR, 1), INTERLEAVED_YUV_FMT(YUYV, 0, BPC8, BPC8, BPC8, C0_G_Y, C1_B_Cb, C0_G_Y, C2_R_Cr, false, CHROMA_H2V1, 4, 2, MSM_FORMAT_FLAG_YUV, - MDP_FETCH_LINEAR, 2), + MDP_FETCH_LINEAR, 1), INTERLEAVED_YUV_FMT(YVYU, 0, BPC8, BPC8, BPC8, C0_G_Y, C2_R_Cr, C0_G_Y, C1_B_Cb, false, CHROMA_H2V1, 4, 2, MSM_FORMAT_FLAG_YUV, - MDP_FETCH_LINEAR, 2), + MDP_FETCH_LINEAR, 1), /* 3 plane YUV */ PLANAR_YUV_FMT(YUV420, diff --git a/drivers/gpu/drm/msm/dp/dp_ctrl.c b/drivers/gpu/drm/msm/dp/dp_ctrl.c index cbcc7c2f0ffc4..94411870a5e0a 100644 --- a/drivers/gpu/drm/msm/dp/dp_ctrl.c +++ b/drivers/gpu/drm/msm/dp/dp_ctrl.c @@ -2395,20 +2395,32 @@ static void msm_dp_ctrl_config_msa(struct msm_dp_ctrl_private *ctrl, bool is_ycbcr_420) { u32 pixel_m, pixel_n; - u32 mvid, nvid, pixel_div = 0, dispcc_input_rate; + u32 mvid, nvid, pixel_div, dispcc_input_rate; u32 const nvid_fixed = DP_LINK_CONSTANT_N_VALUE; u32 const link_rate_hbr2 = 540000; u32 const link_rate_hbr3 = 810000; unsigned long den, num; - if (rate == link_rate_hbr3) + switch (rate) { + case link_rate_hbr3: pixel_div = 6; - else if (rate == 162000 || rate == 270000) - pixel_div = 2; - else if (rate == link_rate_hbr2) + break; + case link_rate_hbr2: pixel_div = 4; - else + break; + case 162000: + case 270000: + pixel_div = 2; + break; + default: + /* + * This cannot be reached but the compiler is not able to know + * that statically so return early to avoid a possibly invalid + * division. + */ DRM_ERROR("Invalid pixel mux divider\n"); + return; + } dispcc_input_rate = (rate * 10) / pixel_div; diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c index fdefcbd9c2848..a156c7e7cea83 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c @@ -628,12 +628,7 @@ static int dsi_pll_14nm_postdiv_determine_rate(struct clk_hw *hw, DBG("DSI%d PLL parent rate=%lu", pll_14nm->phy->id, req->rate); - req->rate = divider_round_rate(hw, req->rate, &req->best_parent_rate, - NULL, - postdiv->width, - postdiv->flags); - - return 0; + return divider_determine_rate(hw, req, NULL, postdiv->width, postdiv->flags); } static int dsi_pll_14nm_postdiv_set_rate(struct clk_hw *hw, unsigned long rate, diff --git a/drivers/gpu/drm/msm/msm_mdss.c b/drivers/gpu/drm/msm/msm_mdss.c index db2f0064eb72e..944a4e08a68e1 100644 --- a/drivers/gpu/drm/msm/msm_mdss.c +++ b/drivers/gpu/drm/msm/msm_mdss.c @@ -229,7 +229,7 @@ static void msm_mdss_setup_ubwc_dec_50(struct msm_mdss *msm_mdss) { const struct qcom_ubwc_cfg_data *data = msm_mdss->mdss_data; u32 value = MDSS_UBWC_STATIC_UBWC_SWIZZLE(data->ubwc_swizzle) | - MDSS_UBWC_STATIC_HIGHEST_BANK_BIT(data->highest_bank_bit); + MDSS_UBWC_STATIC_HIGHEST_BANK_BIT(data->highest_bank_bit - 13); if (data->ubwc_bank_spread) value |= MDSS_UBWC_STATIC_UBWC_BANK_SPREAD; diff --git a/drivers/gpu/drm/panel/panel-edp.c b/drivers/gpu/drm/panel/panel-edp.c index 62435e3cd9f4d..d1e68c464e308 100644 --- a/drivers/gpu/drm/panel/panel-edp.c +++ b/drivers/gpu/drm/panel/panel-edp.c @@ -1730,6 +1730,12 @@ static const struct panel_delay delay_200_500_p2e100 = { .prepare_to_enable = 100, }; +static const struct panel_delay delay_200_500_p2e200 = { + .hpd_absent = 200, + .unprepare = 500, + .prepare_to_enable = 200, +}; + static const struct panel_delay delay_200_500_e50 = { .hpd_absent = 200, .unprepare = 500, @@ -1880,6 +1886,7 @@ static const struct panel_delay delay_80_500_e50_d50 = { */ static const struct edp_panel_entry edp_panels[] = { EDP_PANEL_ENTRY('A', 'U', 'O', 0x04a4, &delay_200_500_e50, "B122UAN01.0"), + EDP_PANEL_ENTRY('A', 'U', 'O', 0x0ba4, &delay_200_500_e50, "B140QAX01.H"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x105c, &delay_200_500_e50, "B116XTN01.0"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x1062, &delay_200_500_e50, "B120XAN01.0"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x125c, &delay_200_500_e50, "Unknown"), @@ -1903,6 +1910,7 @@ static const struct edp_panel_entry edp_panels[] = { EDP_PANEL_ENTRY('A', 'U', 'O', 0x615c, &delay_200_500_e50, "B116XAN06.1"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x635c, &delay_200_500_e50, "B116XAN06.3"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x639c, &delay_200_500_e50, "B140HAK02.7"), + EDP_PANEL_ENTRY('A', 'U', 'O', 0x643d, &delay_200_500_e50, "B140HAN06.4"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x723c, &delay_200_500_e50, "B140XTN07.2"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x73aa, &delay_200_500_e50, "B116XTN02.3"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x8594, &delay_200_500_e50, "B133UAN01.0"), @@ -1972,6 +1980,7 @@ static const struct edp_panel_entry edp_panels[] = { EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b56, &delay_200_500_e80, "NT140FHM-N47"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b66, &delay_200_500_e80, "NE140WUM-N6G"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0c20, &delay_200_500_e80, "NT140FHM-N47"), + EDP_PANEL_ENTRY('B', 'O', 'E', 0x0c26, &delay_200_500_p2e200, "NV140WUM-T08"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0c93, &delay_200_500_e200, "Unknown"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0cb6, &delay_200_500_e200, "NT116WHM-N44"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0cf6, &delay_200_500_e200, "NV140WUM-N64"), diff --git a/drivers/gpu/drm/panel/panel-jdi-lpm102a188a.c b/drivers/gpu/drm/panel/panel-jdi-lpm102a188a.c index 23462065d726b..ea975170fafff 100644 --- a/drivers/gpu/drm/panel/panel-jdi-lpm102a188a.c +++ b/drivers/gpu/drm/panel/panel-jdi-lpm102a188a.c @@ -434,8 +434,10 @@ static void jdi_panel_dsi_remove(struct mipi_dsi_device *dsi) int err; /* only detach from host for the DSI-LINK2 interface */ - if (!jdi) + if (!jdi) { mipi_dsi_detach(dsi); + return; + } err = jdi_panel_disable(&jdi->base); if (err < 0) diff --git a/drivers/gpu/drm/panel/panel-lg-sw43408.c b/drivers/gpu/drm/panel/panel-lg-sw43408.c index 46a56ea92ad9f..6e307fba658f7 100644 --- a/drivers/gpu/drm/panel/panel-lg-sw43408.c +++ b/drivers/gpu/drm/panel/panel-lg-sw43408.c @@ -294,10 +294,6 @@ static void sw43408_remove(struct mipi_dsi_device *dsi) struct sw43408_panel *ctx = mipi_dsi_get_drvdata(dsi); int ret; - ret = sw43408_unprepare(&ctx->base); - if (ret < 0) - dev_err(&dsi->dev, "failed to unprepare panel: %d\n", ret); - ret = mipi_dsi_detach(dsi); if (ret < 0) dev_err(&dsi->dev, "failed to detach from DSI host: %d\n", ret); diff --git a/drivers/gpu/drm/panthor/panthor_gpu.c b/drivers/gpu/drm/panthor/panthor_gpu.c index db69449a5be09..77d0f4ced1206 100644 --- a/drivers/gpu/drm/panthor/panthor_gpu.c +++ b/drivers/gpu/drm/panthor/panthor_gpu.c @@ -49,7 +49,7 @@ struct panthor_gpu { static void panthor_gpu_coherency_set(struct panthor_device *ptdev) { gpu_write(ptdev, GPU_COHERENCY_PROTOCOL, - ptdev->coherent ? GPU_COHERENCY_PROT_BIT(ACE_LITE) : GPU_COHERENCY_NONE); + ptdev->coherent ? GPU_COHERENCY_ACE_LITE : GPU_COHERENCY_NONE); } static void panthor_gpu_irq_handler(struct panthor_device *ptdev, u32 status) @@ -259,38 +259,42 @@ int panthor_gpu_l2_power_on(struct panthor_device *ptdev) int panthor_gpu_flush_caches(struct panthor_device *ptdev, u32 l2, u32 lsc, u32 other) { - bool timedout = false; unsigned long flags; + int ret = 0; /* Serialize cache flush operations. */ guard(mutex)(&ptdev->gpu->cache_flush_lock); spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); - if (!drm_WARN_ON(&ptdev->base, - ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED)) { + if (!(ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED)) { ptdev->gpu->pending_reqs |= GPU_IRQ_CLEAN_CACHES_COMPLETED; gpu_write(ptdev, GPU_CMD, GPU_FLUSH_CACHES(l2, lsc, other)); + } else { + ret = -EIO; } spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); + if (ret) + return ret; + if (!wait_event_timeout(ptdev->gpu->reqs_acked, !(ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED), msecs_to_jiffies(100))) { spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); if ((ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED) != 0 && !(gpu_read(ptdev, GPU_INT_RAWSTAT) & GPU_IRQ_CLEAN_CACHES_COMPLETED)) - timedout = true; + ret = -ETIMEDOUT; else ptdev->gpu->pending_reqs &= ~GPU_IRQ_CLEAN_CACHES_COMPLETED; spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); } - if (timedout) { + if (ret) { + panthor_device_schedule_reset(ptdev); drm_err(&ptdev->base, "Flush caches timeout"); - return -ETIMEDOUT; } - return 0; + return ret; } /** @@ -330,6 +334,7 @@ int panthor_gpu_soft_reset(struct panthor_device *ptdev) return -ETIMEDOUT; } + ptdev->gpu->pending_reqs = 0; return 0; } diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c index 15961629872e1..5647a24741095 100644 --- a/drivers/gpu/drm/panthor/panthor_mmu.c +++ b/drivers/gpu/drm/panthor/panthor_mmu.c @@ -523,27 +523,29 @@ static int wait_ready(struct panthor_device *ptdev, u32 as_nr) return ret; } -static int write_cmd(struct panthor_device *ptdev, u32 as_nr, u32 cmd) +static int as_send_cmd_and_wait(struct panthor_device *ptdev, u32 as_nr, u32 cmd) { int status; /* write AS_COMMAND when MMU is ready to accept another command */ status = wait_ready(ptdev, as_nr); - if (!status) + if (!status) { gpu_write(ptdev, AS_COMMAND(as_nr), cmd); + status = wait_ready(ptdev, as_nr); + } return status; } -static void lock_region(struct panthor_device *ptdev, u32 as_nr, - u64 region_start, u64 size) +static int lock_region(struct panthor_device *ptdev, u32 as_nr, + u64 region_start, u64 size) { u8 region_width; u64 region; u64 region_end = region_start + size; if (!size) - return; + return 0; /* * The locked region is a naturally aligned power of 2 block encoded as @@ -566,7 +568,7 @@ static void lock_region(struct panthor_device *ptdev, u32 as_nr, /* Lock the region that needs to be updated */ gpu_write64(ptdev, AS_LOCKADDR(as_nr), region); - write_cmd(ptdev, as_nr, AS_COMMAND_LOCK); + return as_send_cmd_and_wait(ptdev, as_nr, AS_COMMAND_LOCK); } static int mmu_hw_do_operation_locked(struct panthor_device *ptdev, int as_nr, @@ -599,9 +601,7 @@ static int mmu_hw_do_operation_locked(struct panthor_device *ptdev, int as_nr, * power it up */ - lock_region(ptdev, as_nr, iova, size); - - ret = wait_ready(ptdev, as_nr); + ret = lock_region(ptdev, as_nr, iova, size); if (ret) return ret; @@ -614,10 +614,7 @@ static int mmu_hw_do_operation_locked(struct panthor_device *ptdev, int as_nr, * at the end of the GPU_CONTROL cache flush command, unlike * AS_COMMAND_FLUSH_MEM or AS_COMMAND_FLUSH_PT. */ - write_cmd(ptdev, as_nr, AS_COMMAND_UNLOCK); - - /* Wait for the unlock command to complete */ - return wait_ready(ptdev, as_nr); + return as_send_cmd_and_wait(ptdev, as_nr, AS_COMMAND_UNLOCK); } static int mmu_hw_do_operation(struct panthor_vm *vm, @@ -646,7 +643,7 @@ static int panthor_mmu_as_enable(struct panthor_device *ptdev, u32 as_nr, gpu_write64(ptdev, AS_MEMATTR(as_nr), memattr); gpu_write64(ptdev, AS_TRANSCFG(as_nr), transcfg); - return write_cmd(ptdev, as_nr, AS_COMMAND_UPDATE); + return as_send_cmd_and_wait(ptdev, as_nr, AS_COMMAND_UPDATE); } static int panthor_mmu_as_disable(struct panthor_device *ptdev, u32 as_nr) @@ -661,7 +658,7 @@ static int panthor_mmu_as_disable(struct panthor_device *ptdev, u32 as_nr) gpu_write64(ptdev, AS_MEMATTR(as_nr), 0); gpu_write64(ptdev, AS_TRANSCFG(as_nr), AS_TRANSCFG_ADRMODE_UNMAPPED); - return write_cmd(ptdev, as_nr, AS_COMMAND_UPDATE); + return as_send_cmd_and_wait(ptdev, as_nr, AS_COMMAND_UPDATE); } static u32 panthor_mmu_fault_mask(struct panthor_device *ptdev, u32 value) @@ -1561,6 +1558,10 @@ static void panthor_vm_destroy(struct panthor_vm *vm) vm->destroyed = true; + /* Tell scheduler to stop all GPU work related to this VM */ + if (refcount_read(&vm->as.active_cnt) > 0) + panthor_sched_prepare_for_vm_destruction(vm->ptdev); + mutex_lock(&vm->heaps.lock); panthor_heap_pool_destroy(vm->heaps.pool); vm->heaps.pool = NULL; diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c index 881a07ffbabc8..6930053009d5c 100644 --- a/drivers/gpu/drm/panthor/panthor_sched.c +++ b/drivers/gpu/drm/panthor/panthor_sched.c @@ -22,6 +22,7 @@ #include #include #include +#include #include "panthor_devfreq.h" #include "panthor_device.h" @@ -926,6 +927,9 @@ static void group_release_work(struct work_struct *work) release_work); u32 i; + /* dma-fences may still be accessing group->queues under rcu lock. */ + synchronize_rcu(); + for (i = 0; i < group->queue_count; i++) group_free_queue(group, group->queues[i]); @@ -1902,10 +1906,10 @@ struct panthor_sched_tick_ctx { struct list_head groups[PANTHOR_CSG_PRIORITY_COUNT]; u32 idle_group_count; u32 group_count; - enum panthor_csg_priority min_priority; struct panthor_vm *vms[MAX_CS_PER_CSG]; u32 as_count; bool immediate_tick; + bool stop_tick; u32 csg_upd_failed_mask; }; @@ -1970,17 +1974,21 @@ tick_ctx_pick_groups_from_list(const struct panthor_scheduler *sched, if (!owned_by_tick_ctx) group_get(group); - list_move_tail(&group->run_node, &ctx->groups[group->priority]); ctx->group_count++; + + /* If we have more than one active group with the same priority, + * we need to keep ticking to rotate the CSG priority. + */ if (group_is_idle(group)) ctx->idle_group_count++; + else if (!list_empty(&ctx->groups[group->priority])) + ctx->stop_tick = false; + + list_move_tail(&group->run_node, &ctx->groups[group->priority]); if (i == ctx->as_count) ctx->vms[ctx->as_count++] = group->vm; - if (ctx->min_priority > group->priority) - ctx->min_priority = group->priority; - if (tick_ctx_is_full(sched, ctx)) return; } @@ -1989,31 +1997,22 @@ tick_ctx_pick_groups_from_list(const struct panthor_scheduler *sched, static void tick_ctx_insert_old_group(struct panthor_scheduler *sched, struct panthor_sched_tick_ctx *ctx, - struct panthor_group *group, - bool full_tick) + struct panthor_group *group) { struct panthor_csg_slot *csg_slot = &sched->csg_slots[group->csg_id]; struct panthor_group *other_group; - if (!full_tick) { - list_add_tail(&group->run_node, &ctx->old_groups[group->priority]); - return; - } - - /* Rotate to make sure groups with lower CSG slot - * priorities have a chance to get a higher CSG slot - * priority next time they get picked. This priority - * has an impact on resource request ordering, so it's - * important to make sure we don't let one group starve - * all other groups with the same group priority. - */ + /* Class groups in descending priority order so we can easily rotate. */ list_for_each_entry(other_group, &ctx->old_groups[csg_slot->group->priority], run_node) { struct panthor_csg_slot *other_csg_slot = &sched->csg_slots[other_group->csg_id]; - if (other_csg_slot->priority > csg_slot->priority) { - list_add_tail(&csg_slot->group->run_node, &other_group->run_node); + /* Our group has a higher prio than the one we're testing against, + * place it just before. + */ + if (csg_slot->priority > other_csg_slot->priority) { + list_add_tail(&group->run_node, &other_group->run_node); return; } } @@ -2023,8 +2022,7 @@ tick_ctx_insert_old_group(struct panthor_scheduler *sched, static void tick_ctx_init(struct panthor_scheduler *sched, - struct panthor_sched_tick_ctx *ctx, - bool full_tick) + struct panthor_sched_tick_ctx *ctx) { struct panthor_device *ptdev = sched->ptdev; struct panthor_csg_slots_upd_ctx upd_ctx; @@ -2034,7 +2032,7 @@ tick_ctx_init(struct panthor_scheduler *sched, memset(ctx, 0, sizeof(*ctx)); csgs_upd_ctx_init(&upd_ctx); - ctx->min_priority = PANTHOR_CSG_PRIORITY_COUNT; + ctx->stop_tick = true; for (i = 0; i < ARRAY_SIZE(ctx->groups); i++) { INIT_LIST_HEAD(&ctx->groups[i]); INIT_LIST_HEAD(&ctx->old_groups[i]); @@ -2062,7 +2060,7 @@ tick_ctx_init(struct panthor_scheduler *sched, group->fatal_queues |= GENMASK(group->queue_count - 1, 0); } - tick_ctx_insert_old_group(sched, ctx, group, full_tick); + tick_ctx_insert_old_group(sched, ctx, group); csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, i, csg_iface->output->ack ^ CSG_STATUS_UPDATE, CSG_STATUS_UPDATE); @@ -2346,32 +2344,18 @@ static u64 tick_ctx_update_resched_target(struct panthor_scheduler *sched, const struct panthor_sched_tick_ctx *ctx) { - /* We had space left, no need to reschedule until some external event happens. */ - if (!tick_ctx_is_full(sched, ctx)) - goto no_tick; - - /* If idle groups were scheduled, no need to wake up until some external - * event happens (group unblocked, new job submitted, ...). - */ - if (ctx->idle_group_count) - goto no_tick; + u64 resched_target; - if (drm_WARN_ON(&sched->ptdev->base, ctx->min_priority >= PANTHOR_CSG_PRIORITY_COUNT)) + if (ctx->stop_tick) goto no_tick; - /* If there are groups of the same priority waiting, we need to - * keep the scheduler ticking, otherwise, we'll just wait for - * new groups with higher priority to be queued. - */ - if (!list_empty(&sched->groups.runnable[ctx->min_priority])) { - u64 resched_target = sched->last_tick + sched->tick_period; + resched_target = sched->last_tick + sched->tick_period; - if (time_before64(sched->resched_target, sched->last_tick) || - time_before64(resched_target, sched->resched_target)) - sched->resched_target = resched_target; + if (time_before64(sched->resched_target, sched->last_tick) || + time_before64(resched_target, sched->resched_target)) + sched->resched_target = resched_target; - return sched->resched_target - sched->last_tick; - } + return sched->resched_target - sched->last_tick; no_tick: sched->resched_target = U64_MAX; @@ -2384,9 +2368,11 @@ static void tick_work(struct work_struct *work) tick_work.work); struct panthor_device *ptdev = sched->ptdev; struct panthor_sched_tick_ctx ctx; + u64 resched_target = sched->resched_target; u64 remaining_jiffies = 0, resched_delay; u64 now = get_jiffies_64(); int prio, ret, cookie; + bool full_tick; if (!drm_dev_enter(&ptdev->base, &cookie)) return; @@ -2395,18 +2381,24 @@ static void tick_work(struct work_struct *work) if (drm_WARN_ON(&ptdev->base, ret)) goto out_dev_exit; - if (time_before64(now, sched->resched_target)) - remaining_jiffies = sched->resched_target - now; + /* If the tick is stopped, calculate when the next tick would be */ + if (resched_target == U64_MAX) + resched_target = sched->last_tick + sched->tick_period; + + if (time_before64(now, resched_target)) + remaining_jiffies = resched_target - now; + + full_tick = remaining_jiffies == 0; mutex_lock(&sched->lock); if (panthor_device_reset_is_pending(sched->ptdev)) goto out_unlock; - tick_ctx_init(sched, &ctx, remaining_jiffies != 0); + tick_ctx_init(sched, &ctx); if (ctx.csg_upd_failed_mask) goto out_cleanup_ctx; - if (remaining_jiffies) { + if (!full_tick) { /* Scheduling forced in the middle of a tick. Only RT groups * can preempt non-RT ones. Currently running RT groups can't be * preempted. @@ -2428,9 +2420,29 @@ static void tick_work(struct work_struct *work) for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0 && !tick_ctx_is_full(sched, &ctx); prio--) { + struct panthor_group *old_highest_prio_group = + list_first_entry_or_null(&ctx.old_groups[prio], + struct panthor_group, run_node); + + /* Pull out the group with the highest prio for rotation. */ + if (old_highest_prio_group) + list_del(&old_highest_prio_group->run_node); + + /* Re-insert old active groups so they get a chance to run with higher prio. */ + tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], true, true); + + /* Fill the remaining slots with runnable groups. */ tick_ctx_pick_groups_from_list(sched, &ctx, &sched->groups.runnable[prio], true, false); - tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], true, true); + + /* Re-insert the old group with the highest prio, and give it a chance to be + * scheduled again (but with a lower prio) if there's room left. + */ + if (old_highest_prio_group) { + list_add_tail(&old_highest_prio_group->run_node, &ctx.old_groups[prio]); + tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], + true, true); + } } /* If we have free CSG slots left, pick idle groups */ @@ -2555,14 +2567,33 @@ static void sync_upd_work(struct work_struct *work) sched_queue_delayed_work(sched, tick, 0); } +static void sched_resume_tick(struct panthor_device *ptdev) +{ + struct panthor_scheduler *sched = ptdev->scheduler; + u64 delay_jiffies, now; + + drm_WARN_ON(&ptdev->base, sched->resched_target != U64_MAX); + + /* Scheduler tick was off, recalculate the resched_target based on the + * last tick event, and queue the scheduler work. + */ + now = get_jiffies_64(); + sched->resched_target = sched->last_tick + sched->tick_period; + if (sched->used_csg_slot_count == sched->csg_slot_count && + time_before64(now, sched->resched_target)) + delay_jiffies = min_t(unsigned long, sched->resched_target - now, ULONG_MAX); + else + delay_jiffies = 0; + + sched_queue_delayed_work(sched, tick, delay_jiffies); +} + static void group_schedule_locked(struct panthor_group *group, u32 queue_mask) { struct panthor_device *ptdev = group->ptdev; struct panthor_scheduler *sched = ptdev->scheduler; struct list_head *queue = &sched->groups.runnable[group->priority]; - u64 delay_jiffies = 0; bool was_idle; - u64 now; if (!group_can_run(group)) return; @@ -2607,13 +2638,7 @@ static void group_schedule_locked(struct panthor_group *group, u32 queue_mask) /* Scheduler tick was off, recalculate the resched_target based on the * last tick event, and queue the scheduler work. */ - now = get_jiffies_64(); - sched->resched_target = sched->last_tick + sched->tick_period; - if (sched->used_csg_slot_count == sched->csg_slot_count && - time_before64(now, sched->resched_target)) - delay_jiffies = min_t(unsigned long, sched->resched_target - now, ULONG_MAX); - - sched_queue_delayed_work(sched, tick, delay_jiffies); + sched_resume_tick(ptdev); } static void queue_stop(struct panthor_queue *queue, @@ -2686,6 +2711,20 @@ void panthor_sched_report_mmu_fault(struct panthor_device *ptdev) panthor_sched_immediate_tick(ptdev); } +void panthor_sched_prepare_for_vm_destruction(struct panthor_device *ptdev) +{ + /* FW can write out internal state, like the heap context, during CSG + * suspend. It is therefore important that the scheduler has fully + * evicted any pending and related groups before VM destruction can + * safely continue. Failure to do so can lead to GPU page faults. + * A controlled termination of a Panthor instance involves destroying + * the group(s) before the VM. This means any relevant group eviction + * has already been initiated by this point, and we just need to + * ensure that any pending tick_work() has been completed. + */ + flush_work(&ptdev->scheduler->tick_work.work); +} + void panthor_sched_resume(struct panthor_device *ptdev) { /* Force a tick to re-evaluate after a resume. */ @@ -3214,6 +3253,18 @@ queue_run_job(struct drm_sched_job *sched_job) group_schedule_locked(group, BIT(job->queue_idx)); } else { + u32 queue_mask = BIT(job->queue_idx); + bool resume_tick = group_is_idle(group) && + (group->idle_queues & queue_mask) && + !(group->blocked_queues & queue_mask) && + sched->resched_target == U64_MAX; + + /* We just added something to the queue, so it's no longer idle. */ + group->idle_queues &= ~queue_mask; + + if (resume_tick) + sched_resume_tick(ptdev); + gpu_write(ptdev, CSF_DOORBELL(queue->doorbell_id), 1); if (!sched->pm.has_ref && !(group->blocked_queues & BIT(job->queue_idx))) { diff --git a/drivers/gpu/drm/panthor/panthor_sched.h b/drivers/gpu/drm/panthor/panthor_sched.h index 742b0b4ff3a3c..6a560ab0a5b31 100644 --- a/drivers/gpu/drm/panthor/panthor_sched.h +++ b/drivers/gpu/drm/panthor/panthor_sched.h @@ -49,6 +49,7 @@ void panthor_sched_suspend(struct panthor_device *ptdev); void panthor_sched_resume(struct panthor_device *ptdev); void panthor_sched_report_mmu_fault(struct panthor_device *ptdev); +void panthor_sched_prepare_for_vm_destruction(struct panthor_device *ptdev); void panthor_sched_report_fw_events(struct panthor_device *ptdev, u32 events); void panthor_fdinfo_gather_group_samples(struct panthor_file *pfile); diff --git a/drivers/gpu/drm/radeon/si_dpm.c b/drivers/gpu/drm/radeon/si_dpm.c index 9deb91970d4df..f12227145ef08 100644 --- a/drivers/gpu/drm/radeon/si_dpm.c +++ b/drivers/gpu/drm/radeon/si_dpm.c @@ -2925,6 +2925,11 @@ static void si_apply_state_adjust_rules(struct radeon_device *rdev, max_sclk = 60000; max_mclk = 80000; } + if ((rdev->pdev->device == 0x666f) && + (rdev->pdev->revision == 0x00)) { + max_sclk = 80000; + max_mclk = 95000; + } } else if (rdev->family == CHIP_OLAND) { if ((rdev->pdev->revision == 0xC7) || (rdev->pdev->revision == 0x80) || diff --git a/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c b/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c index 3b52dfc0ea1e0..b164e3a62cc2f 100644 --- a/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c +++ b/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c @@ -646,6 +646,13 @@ static void rzg2l_mipi_dsi_atomic_disable(struct drm_bridge *bridge, rzg2l_mipi_dsi_stop_video(dsi); rzg2l_mipi_dsi_stop_hs_clock(dsi); +} + +static void rzg2l_mipi_dsi_atomic_post_disable(struct drm_bridge *bridge, + struct drm_atomic_state *state) +{ + struct rzg2l_mipi_dsi *dsi = bridge_to_rzg2l_mipi_dsi(bridge); + rzg2l_mipi_dsi_stop(dsi); } @@ -681,6 +688,7 @@ static const struct drm_bridge_funcs rzg2l_mipi_dsi_bridge_ops = { .atomic_pre_enable = rzg2l_mipi_dsi_atomic_pre_enable, .atomic_enable = rzg2l_mipi_dsi_atomic_enable, .atomic_disable = rzg2l_mipi_dsi_atomic_disable, + .atomic_post_disable = rzg2l_mipi_dsi_atomic_post_disable, .mode_valid = rzg2l_mipi_dsi_bridge_mode_valid, }; diff --git a/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c b/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c index 9ac45e7bc987a..409f1a1e82a06 100644 --- a/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c +++ b/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c @@ -267,12 +267,7 @@ static irqreturn_t dw_hdmi_qp_rk3576_hardirq(int irq, void *dev_id) static irqreturn_t dw_hdmi_qp_rk3576_irq(int irq, void *dev_id) { struct rockchip_hdmi_qp *hdmi = dev_id; - u32 intr_stat, val; - - regmap_read(hdmi->regmap, RK3576_IOC_HDMI_HPD_STATUS, &intr_stat); - - if (!intr_stat) - return IRQ_NONE; + u32 val; val = FIELD_PREP_WM16(RK3576_HDMI_HPD_INT_CLR, 1); regmap_write(hdmi->regmap, RK3576_IOC_MISC_CON0, val); diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index c39f0245e3a97..3f138776d35fb 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -361,6 +361,7 @@ static void drm_sched_run_free_queue(struct drm_gpu_scheduler *sched) /** * drm_sched_job_done - complete a job * @s_job: pointer to the job which is done + * @result: 0 on success, -ERRNO on error * * Finish the job's fence and resubmit the work items. */ diff --git a/drivers/gpu/drm/solomon/ssd130x.c b/drivers/gpu/drm/solomon/ssd130x.c index eec43d1a55951..18d2294c526de 100644 --- a/drivers/gpu/drm/solomon/ssd130x.c +++ b/drivers/gpu/drm/solomon/ssd130x.c @@ -736,6 +736,7 @@ static int ssd130x_update_rect(struct ssd130x_device *ssd130x, unsigned int height = drm_rect_height(rect); unsigned int line_length = DIV_ROUND_UP(width, 8); unsigned int page_height = SSD130X_PAGE_HEIGHT; + u8 page_start = ssd130x->page_offset + y / page_height; unsigned int pages = DIV_ROUND_UP(height, page_height); struct drm_device *drm = &ssd130x->drm; u32 array_idx = 0; @@ -773,14 +774,11 @@ static int ssd130x_update_rect(struct ssd130x_device *ssd130x, */ if (!ssd130x->page_address_mode) { - u8 page_start; - /* Set address range for horizontal addressing mode */ ret = ssd130x_set_col_range(ssd130x, ssd130x->col_offset + x, width); if (ret < 0) return ret; - page_start = ssd130x->page_offset + y / page_height; ret = ssd130x_set_page_range(ssd130x, page_start, pages); if (ret < 0) return ret; @@ -812,7 +810,7 @@ static int ssd130x_update_rect(struct ssd130x_device *ssd130x, */ if (ssd130x->page_address_mode) { ret = ssd130x_set_page_pos(ssd130x, - ssd130x->page_offset + i, + page_start + i, ssd130x->col_offset + x); if (ret < 0) return ret; diff --git a/drivers/gpu/drm/tegra/dsi.c b/drivers/gpu/drm/tegra/dsi.c index ddfb2858acbf1..5aa78b902adac 100644 --- a/drivers/gpu/drm/tegra/dsi.c +++ b/drivers/gpu/drm/tegra/dsi.c @@ -1540,11 +1540,9 @@ static int tegra_dsi_ganged_probe(struct tegra_dsi *dsi) return -EPROBE_DEFER; dsi->slave = platform_get_drvdata(gangster); - - if (!dsi->slave) { - put_device(&gangster->dev); + put_device(&gangster->dev); + if (!dsi->slave) return -EPROBE_DEFER; - } dsi->slave->master = dsi; } diff --git a/drivers/gpu/drm/tegra/hdmi.c b/drivers/gpu/drm/tegra/hdmi.c index 8cd2969e7d4bf..c4820f5e76581 100644 --- a/drivers/gpu/drm/tegra/hdmi.c +++ b/drivers/gpu/drm/tegra/hdmi.c @@ -658,7 +658,7 @@ static void tegra_hdmi_write_infopack(struct tegra_hdmi *hdmi, const void *data, { const u8 *ptr = data; unsigned long offset; - size_t i, j; + size_t i; u32 value; switch (ptr[0]) { @@ -691,7 +691,7 @@ static void tegra_hdmi_write_infopack(struct tegra_hdmi *hdmi, const void *data, * - subpack_low: bytes 0 - 3 * - subpack_high: bytes 4 - 6 (with byte 7 padded to 0x00) */ - for (i = 3, j = 0; i < size; i += 7, j += 8) { + for (i = 3; i < size; i += 7) { size_t rem = size - i, num = min_t(size_t, rem, 4); value = tegra_hdmi_subpack(&ptr[i], num); diff --git a/drivers/gpu/drm/tegra/sor.c b/drivers/gpu/drm/tegra/sor.c index 21f3dfdcc5c95..bc7dd562cf6b6 100644 --- a/drivers/gpu/drm/tegra/sor.c +++ b/drivers/gpu/drm/tegra/sor.c @@ -1864,7 +1864,7 @@ static void tegra_sor_hdmi_write_infopack(struct tegra_sor *sor, { const u8 *ptr = data; unsigned long offset; - size_t i, j; + size_t i; u32 value; switch (ptr[0]) { @@ -1897,7 +1897,7 @@ static void tegra_sor_hdmi_write_infopack(struct tegra_sor *sor, * - subpack_low: bytes 0 - 3 * - subpack_high: bytes 4 - 6 (with byte 7 padded to 0x00) */ - for (i = 3, j = 0; i < size; i += 7, j += 8) { + for (i = 3; i < size; i += 7) { size_t rem = size - i, num = min_t(size_t, rem, 4); value = tegra_sor_hdmi_subpack(&ptr[i], num); diff --git a/drivers/gpu/drm/tests/drm_gem_shmem_test.c b/drivers/gpu/drm/tests/drm_gem_shmem_test.c index 68f2c31623547..4b459f21acfd9 100644 --- a/drivers/gpu/drm/tests/drm_gem_shmem_test.c +++ b/drivers/gpu/drm/tests/drm_gem_shmem_test.c @@ -19,6 +19,8 @@ #include #include +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); + #define TEST_SIZE SZ_1M #define TEST_BYTE 0xae @@ -34,6 +36,9 @@ KUNIT_DEFINE_ACTION_WRAPPER(sg_free_table_wrapper, sg_free_table, KUNIT_DEFINE_ACTION_WRAPPER(drm_gem_shmem_free_wrapper, drm_gem_shmem_free, struct drm_gem_shmem_object *); +KUNIT_DEFINE_ACTION_WRAPPER(drm_gem_shmem_unpin_wrapper, drm_gem_shmem_unpin, + struct drm_gem_shmem_object *); + /* * Test creating a shmem GEM object backed by shmem buffer. The test * case succeeds if the GEM object is successfully allocated with the @@ -173,7 +178,7 @@ static void drm_gem_shmem_test_vmap(struct kunit *test) ret = kunit_add_action_or_reset(test, drm_gem_shmem_free_wrapper, shmem); KUNIT_ASSERT_EQ(test, ret, 0); - ret = drm_gem_shmem_vmap_locked(shmem, &map); + ret = drm_gem_shmem_vmap(shmem, &map); KUNIT_ASSERT_EQ(test, ret, 0); KUNIT_ASSERT_NOT_NULL(test, shmem->vaddr); KUNIT_ASSERT_FALSE(test, iosys_map_is_null(&map)); @@ -183,7 +188,7 @@ static void drm_gem_shmem_test_vmap(struct kunit *test) for (i = 0; i < TEST_SIZE; i++) KUNIT_EXPECT_EQ(test, iosys_map_rd(&map, i, u8), TEST_BYTE); - drm_gem_shmem_vunmap_locked(shmem, &map); + drm_gem_shmem_vunmap(shmem, &map); KUNIT_EXPECT_NULL(test, shmem->vaddr); KUNIT_EXPECT_EQ(test, refcount_read(&shmem->vmap_use_count), 0); } @@ -194,7 +199,7 @@ static void drm_gem_shmem_test_vmap(struct kunit *test) * scatter/gather table large enough to accommodate the backing memory * is successfully exported. */ -static void drm_gem_shmem_test_get_pages_sgt(struct kunit *test) +static void drm_gem_shmem_test_get_sg_table(struct kunit *test) { struct drm_device *drm_dev = test->priv; struct drm_gem_shmem_object *shmem; @@ -212,6 +217,9 @@ static void drm_gem_shmem_test_get_pages_sgt(struct kunit *test) ret = drm_gem_shmem_pin(shmem); KUNIT_ASSERT_EQ(test, ret, 0); + ret = kunit_add_action_or_reset(test, drm_gem_shmem_unpin_wrapper, shmem); + KUNIT_ASSERT_EQ(test, ret, 0); + sgt = drm_gem_shmem_get_sg_table(shmem); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sgt); KUNIT_EXPECT_NULL(test, shmem->sgt); @@ -236,7 +244,7 @@ static void drm_gem_shmem_test_get_pages_sgt(struct kunit *test) * backing pages are pinned and a scatter/gather table large enough to * accommodate the backing memory is successfully exported. */ -static void drm_gem_shmem_test_get_sg_table(struct kunit *test) +static void drm_gem_shmem_test_get_pages_sgt(struct kunit *test) { struct drm_device *drm_dev = test->priv; struct drm_gem_shmem_object *shmem; @@ -284,17 +292,17 @@ static void drm_gem_shmem_test_madvise(struct kunit *test) ret = kunit_add_action_or_reset(test, drm_gem_shmem_free_wrapper, shmem); KUNIT_ASSERT_EQ(test, ret, 0); - ret = drm_gem_shmem_madvise_locked(shmem, 1); + ret = drm_gem_shmem_madvise(shmem, 1); KUNIT_EXPECT_TRUE(test, ret); KUNIT_ASSERT_EQ(test, shmem->madv, 1); /* Set madv to a negative value */ - ret = drm_gem_shmem_madvise_locked(shmem, -1); + ret = drm_gem_shmem_madvise(shmem, -1); KUNIT_EXPECT_FALSE(test, ret); KUNIT_ASSERT_EQ(test, shmem->madv, -1); /* Check that madv cannot be set back to a positive value */ - ret = drm_gem_shmem_madvise_locked(shmem, 0); + ret = drm_gem_shmem_madvise(shmem, 0); KUNIT_EXPECT_FALSE(test, ret); KUNIT_ASSERT_EQ(test, shmem->madv, -1); } @@ -322,7 +330,7 @@ static void drm_gem_shmem_test_purge(struct kunit *test) ret = drm_gem_shmem_is_purgeable(shmem); KUNIT_EXPECT_FALSE(test, ret); - ret = drm_gem_shmem_madvise_locked(shmem, 1); + ret = drm_gem_shmem_madvise(shmem, 1); KUNIT_EXPECT_TRUE(test, ret); /* The scatter/gather table will be freed by drm_gem_shmem_free */ @@ -332,7 +340,9 @@ static void drm_gem_shmem_test_purge(struct kunit *test) ret = drm_gem_shmem_is_purgeable(shmem); KUNIT_EXPECT_TRUE(test, ret); - drm_gem_shmem_purge_locked(shmem); + ret = drm_gem_shmem_purge(shmem); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_NULL(test, shmem->pages); KUNIT_EXPECT_NULL(test, shmem->sgt); KUNIT_EXPECT_EQ(test, shmem->madv, -1); @@ -366,8 +376,8 @@ static struct kunit_case drm_gem_shmem_test_cases[] = { KUNIT_CASE(drm_gem_shmem_test_obj_create_private), KUNIT_CASE(drm_gem_shmem_test_pin_pages), KUNIT_CASE(drm_gem_shmem_test_vmap), - KUNIT_CASE(drm_gem_shmem_test_get_pages_sgt), KUNIT_CASE(drm_gem_shmem_test_get_sg_table), + KUNIT_CASE(drm_gem_shmem_test_get_pages_sgt), KUNIT_CASE(drm_gem_shmem_test_madvise), KUNIT_CASE(drm_gem_shmem_test_purge), {} diff --git a/drivers/gpu/drm/tiny/sharp-memory.c b/drivers/gpu/drm/tiny/sharp-memory.c index 64272cd0f6e22..cbf69460ebf32 100644 --- a/drivers/gpu/drm/tiny/sharp-memory.c +++ b/drivers/gpu/drm/tiny/sharp-memory.c @@ -541,8 +541,8 @@ static int sharp_memory_probe(struct spi_device *spi) smd = devm_drm_dev_alloc(dev, &sharp_memory_drm_driver, struct sharp_memory_device, drm); - if (!smd) - return -ENOMEM; + if (IS_ERR(smd)) + return PTR_ERR(smd); spi_set_drvdata(spi, smd); diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index d5625dd1e41c8..0052ebe957199 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -76,7 +76,7 @@ fn issue_soft_reset(dev: &Device, iomem: &Devres) -> Result { dev_err!(dev, "GPU reset failed with errno\n"); dev_err!( dev, - "GPU_INT_RAWSTAT is {}\n", + "GPU_IRQ_RAWSTAT is {}\n", regs::GPU_IRQ_RAWSTAT.read(dev, iomem)? ); diff --git a/drivers/gpu/drm/v3d/v3d_drv.c b/drivers/gpu/drm/v3d/v3d_drv.c index c5a3bbbc74c5c..f4da7a94e4016 100644 --- a/drivers/gpu/drm/v3d/v3d_drv.c +++ b/drivers/gpu/drm/v3d/v3d_drv.c @@ -377,6 +377,8 @@ static int v3d_platform_drm_probe(struct platform_device *pdev) if (ret) goto clk_disable; + dma_set_max_seg_size(&pdev->dev, UINT_MAX); + v3d->va_width = 30 + V3D_GET_FIELD(mmu_debug, V3D_MMU_VA_WIDTH); ident1 = V3D_READ(V3D_HUB_IDENT1); diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c index 3057f8baa7d25..e1f18020170ab 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c @@ -1143,7 +1143,7 @@ static int vmw_translate_mob_ptr(struct vmw_private *dev_priv, ret = vmw_user_bo_lookup(sw_context->filp, handle, &vmw_bo); if (ret != 0) { drm_dbg(&dev_priv->drm, "Could not find or use MOB buffer.\n"); - return PTR_ERR(vmw_bo); + return ret; } vmw_bo_placement_set(vmw_bo, VMW_BO_DOMAIN_MOB, VMW_BO_DOMAIN_MOB); ret = vmw_validation_add_bo(sw_context->ctx, vmw_bo); @@ -1199,7 +1199,7 @@ static int vmw_translate_guest_ptr(struct vmw_private *dev_priv, ret = vmw_user_bo_lookup(sw_context->filp, handle, &vmw_bo); if (ret != 0) { drm_dbg(&dev_priv->drm, "Could not find or use GMR region.\n"); - return PTR_ERR(vmw_bo); + return ret; } vmw_bo_placement_set(vmw_bo, VMW_BO_DOMAIN_GMR | VMW_BO_DOMAIN_VRAM, VMW_BO_DOMAIN_GMR | VMW_BO_DOMAIN_VRAM); diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c b/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c index fd4e76486f2d1..45561bc1c9eff 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c @@ -260,6 +260,13 @@ int vmw_bo_dirty_add(struct vmw_bo *vbo) return ret; } +static void vmw_bo_dirty_free(struct kref *kref) +{ + struct vmw_bo_dirty *dirty = container_of(kref, struct vmw_bo_dirty, ref_count); + + kvfree(dirty); +} + /** * vmw_bo_dirty_release - Release a dirty-tracking user from a buffer object * @vbo: The buffer object @@ -274,7 +281,7 @@ void vmw_bo_dirty_release(struct vmw_bo *vbo) { struct vmw_bo_dirty *dirty = vbo->dirty; - if (dirty && kref_put(&dirty->ref_count, (void *)kvfree)) + if (dirty && kref_put(&dirty->ref_count, vmw_bo_dirty_free)) vbo->dirty = NULL; } diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.c b/drivers/gpu/drm/xe/display/xe_fb_pin.c index 1fd4a815e784b..b18d15cc3c53d 100644 --- a/drivers/gpu/drm/xe/display/xe_fb_pin.c +++ b/drivers/gpu/drm/xe/display/xe_fb_pin.c @@ -378,7 +378,7 @@ intel_fb_pin_to_ggtt(const struct drm_framebuffer *fb, { *out_flags = 0; - return __xe_pin_fb_vma(to_intel_framebuffer(fb), view, phys_alignment); + return __xe_pin_fb_vma(to_intel_framebuffer(fb), view, alignment); } void intel_fb_unpin_vma(struct i915_vma *vma, unsigned long flags) diff --git a/drivers/gpu/drm/xe/regs/xe_engine_regs.h b/drivers/gpu/drm/xe/regs/xe_engine_regs.h index f4c3e1187a00a..27fba92301c4e 100644 --- a/drivers/gpu/drm/xe/regs/xe_engine_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_engine_regs.h @@ -96,6 +96,12 @@ #define ENABLE_SEMAPHORE_POLL_BIT REG_BIT(13) #define RING_CMD_CCTL(base) XE_REG((base) + 0xc4, XE_REG_OPTION_MASKED) + +#define CS_MMIO_GROUP_INSTANCE_SELECT(base) XE_REG((base) + 0xcc) +#define SELECTIVE_READ_ADDRESSING REG_BIT(30) +#define SELECTIVE_READ_GROUP REG_GENMASK(29, 23) +#define SELECTIVE_READ_INSTANCE REG_GENMASK(22, 16) + /* * CMD_CCTL read/write fields take a MOCS value and _not_ a table index. * The lsb of each can be considered a separate enabling bit for encryption. diff --git a/drivers/gpu/drm/xe/regs/xe_guc_regs.h b/drivers/gpu/drm/xe/regs/xe_guc_regs.h index 2118f7dec287f..87984713dd126 100644 --- a/drivers/gpu/drm/xe/regs/xe_guc_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_guc_regs.h @@ -90,6 +90,9 @@ #define GUC_SEND_INTERRUPT XE_REG(0xc4c8) #define GUC_SEND_TRIGGER REG_BIT(0) +#define GUC_INTR_CHICKEN XE_REG(0xc50c) +#define DISABLE_SIGNALING_ENGINES REG_BIT(1) + #define GUC_BCS_RCS_IER XE_REG(0xc550) #define GUC_VCS2_VCS1_IER XE_REG(0xc554) #define GUC_WD_VECS_IER XE_REG(0xc558) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index e2e28ff73925b..a270aef7c4980 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -1895,7 +1895,7 @@ static vm_fault_t xe_bo_cpu_fault(struct vm_fault *vmf) int err = 0; int idx; - if (!drm_dev_enter(&xe->drm, &idx)) + if (xe_device_wedged(xe) || !drm_dev_enter(&xe->drm, &idx)) return ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot); ret = xe_bo_cpu_fault_fastpath(vmf, xe, bo, needs_rpm); diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c index 6688b2954d20b..08f379cb5321f 100644 --- a/drivers/gpu/drm/xe/xe_configfs.c +++ b/drivers/gpu/drm/xe/xe_configfs.c @@ -688,6 +688,7 @@ static void xe_config_device_release(struct config_item *item) mutex_destroy(&dev->lock); + kfree(dev->config.ctx_restore_mid_bb[0].cs); kfree(dev->config.ctx_restore_post_bb[0].cs); kfree(dev); } diff --git a/drivers/gpu/drm/xe/xe_configfs.h b/drivers/gpu/drm/xe/xe_configfs.h index c61e0e47ed94c..08cce375ae0f3 100644 --- a/drivers/gpu/drm/xe/xe_configfs.h +++ b/drivers/gpu/drm/xe/xe_configfs.h @@ -19,9 +19,11 @@ void xe_configfs_check_device(struct pci_dev *pdev); bool xe_configfs_get_survivability_mode(struct pci_dev *pdev); u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev); bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev); -u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev, enum xe_engine_class, +u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev, + enum xe_engine_class class, const u32 **cs); -u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev *pdev, enum xe_engine_class, +u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev *pdev, + enum xe_engine_class class, const u32 **cs); #else static inline int xe_configfs_init(void) { return 0; } @@ -30,9 +32,11 @@ static inline void xe_configfs_check_device(struct pci_dev *pdev) { } static inline bool xe_configfs_get_survivability_mode(struct pci_dev *pdev) { return false; } static inline u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev) { return U64_MAX; } static inline bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev) { return false; } -static inline u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev, enum xe_engine_class, +static inline u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev, + enum xe_engine_class class, const u32 **cs) { return 0; } -static inline u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev *pdev, enum xe_engine_class, +static inline u32 xe_configfs_get_ctx_restore_post_bb(struct pci_dev *pdev, + enum xe_engine_class class, const u32 **cs) { return 0; } #endif diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index fe5aadb27b779..0d69cd0e4e798 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -966,6 +966,7 @@ int xe_device_probe(struct xe_device *xe) err_unregister_display: xe_display_unregister(xe); + drm_dev_unregister(&xe->drm); return err; } diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c index 20d9171bd3d0a..2f96983a66b6e 100644 --- a/drivers/gpu/drm/xe/xe_ggtt.c +++ b/drivers/gpu/drm/xe/xe_ggtt.c @@ -365,9 +365,8 @@ static void ggtt_node_remove_work_func(struct work_struct *work) delayed_removal_work); struct xe_device *xe = tile_to_xe(node->ggtt->tile); - xe_pm_runtime_get(xe); + guard(xe_pm_runtime)(xe); ggtt_node_remove(node); - xe_pm_runtime_put(xe); } /** diff --git a/drivers/gpu/drm/xe/xe_gsc_proxy.c b/drivers/gpu/drm/xe/xe_gsc_proxy.c index 464282a89eef3..a6f6f0ea56526 100644 --- a/drivers/gpu/drm/xe/xe_gsc_proxy.c +++ b/drivers/gpu/drm/xe/xe_gsc_proxy.c @@ -435,16 +435,12 @@ static int proxy_channel_alloc(struct xe_gsc *gsc) return 0; } -static void xe_gsc_proxy_remove(void *arg) +static void xe_gsc_proxy_stop(struct xe_gsc *gsc) { - struct xe_gsc *gsc = arg; struct xe_gt *gt = gsc_to_gt(gsc); struct xe_device *xe = gt_to_xe(gt); unsigned int fw_ref = 0; - if (!gsc->proxy.component_added) - return; - /* disable HECI2 IRQs */ xe_pm_runtime_get(xe); fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GSC); @@ -458,6 +454,30 @@ static void xe_gsc_proxy_remove(void *arg) xe_pm_runtime_put(xe); xe_gsc_wait_for_worker_completion(gsc); + gsc->proxy.started = false; +} + +static void xe_gsc_proxy_remove(void *arg) +{ + struct xe_gsc *gsc = arg; + struct xe_gt *gt = gsc_to_gt(gsc); + struct xe_device *xe = gt_to_xe(gt); + + if (!gsc->proxy.component_added) + return; + + /* + * GSC proxy start is an async process that can be ongoing during + * Xe module load/unload. Using devm managed action to register + * xe_gsc_proxy_stop could cause issues if Xe module unload has + * already started when the action is registered, potentially leading + * to the cleanup being called at the wrong time. Therefore, instead + * of registering a separate devm action to undo what is done in + * proxy start, we call it from here, but only if the start has + * completed successfully (tracked with the 'started' flag). + */ + if (gsc->proxy.started) + xe_gsc_proxy_stop(gsc); component_del(xe->drm.dev, &xe_gsc_proxy_component_ops); gsc->proxy.component_added = false; @@ -513,6 +533,7 @@ int xe_gsc_proxy_init(struct xe_gsc *gsc) */ int xe_gsc_proxy_start(struct xe_gsc *gsc) { + struct xe_gt *gt = gsc_to_gt(gsc); int err; /* enable the proxy interrupt in the GSC shim layer */ @@ -524,12 +545,18 @@ int xe_gsc_proxy_start(struct xe_gsc *gsc) */ err = xe_gsc_proxy_request_handler(gsc); if (err) - return err; + goto err_irq_disable; if (!xe_gsc_proxy_init_done(gsc)) { - xe_gt_err(gsc_to_gt(gsc), "GSC FW reports proxy init not completed\n"); - return -EIO; + xe_gt_err(gt, "GSC FW reports proxy init not completed\n"); + err = -EIO; + goto err_irq_disable; } + gsc->proxy.started = true; return 0; + +err_irq_disable: + gsc_proxy_irq_toggle(gsc, false); + return err; } diff --git a/drivers/gpu/drm/xe/xe_gsc_types.h b/drivers/gpu/drm/xe/xe_gsc_types.h index 97c056656df05..5aaa2a75861fd 100644 --- a/drivers/gpu/drm/xe/xe_gsc_types.h +++ b/drivers/gpu/drm/xe/xe_gsc_types.h @@ -58,6 +58,8 @@ struct xe_gsc { struct mutex mutex; /** @proxy.component_added: whether the component has been added */ bool component_added; + /** @proxy.started: whether the proxy has been started */ + bool started; /** @proxy.bo: object to store message to and from the GSC */ struct xe_bo *bo; /** @proxy.to_gsc: map of the memory used to send messages to the GSC */ diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index 61bed3b04dedd..1099335298471 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -187,11 +187,15 @@ static int emit_nop_job(struct xe_gt *gt, struct xe_exec_queue *q) return ret; } +/* Dwords required to emit a RMW of a register */ +#define EMIT_RMW_DW 20 + static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q) { - struct xe_reg_sr *sr = &q->hwe->reg_lrc; + struct xe_hw_engine *hwe = q->hwe; + struct xe_reg_sr *sr = &hwe->reg_lrc; struct xe_reg_sr_entry *entry; - int count_rmw = 0, count = 0, ret; + int count_rmw = 0, count_rmw_mcr = 0, count = 0, ret; unsigned long idx; struct xe_bb *bb; size_t bb_len = 0; @@ -201,6 +205,8 @@ static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q) xa_for_each(&sr->xa, idx, entry) { if (entry->reg.masked || entry->clr_bits == ~0) ++count; + else if (entry->reg.mcr) + ++count_rmw_mcr; else ++count_rmw; } @@ -208,17 +214,35 @@ static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q) if (count) bb_len += count * 2 + 1; - if (count_rmw) - bb_len += count_rmw * 20 + 7; + /* + * RMW of MCR registers is the same as a normal RMW, except an + * additional LRI (3 dwords) is required per register to steer the read + * to a nom-terminated instance. + * + * We could probably shorten the batch slightly by eliding the + * steering for consecutive MCR registers that have the same + * group/instance target, but it's not worth the extra complexity to do + * so. + */ + bb_len += count_rmw * EMIT_RMW_DW; + bb_len += count_rmw_mcr * (EMIT_RMW_DW + 3); + + /* + * After doing all RMW, we need 7 trailing dwords to clean up, + * plus an additional 3 dwords to reset steering if any of the + * registers were MCR. + */ + if (count_rmw || count_rmw_mcr) + bb_len += 7 + (count_rmw_mcr ? 3 : 0); - if (q->hwe->class == XE_ENGINE_CLASS_RENDER) + if (hwe->class == XE_ENGINE_CLASS_RENDER) /* * Big enough to emit all of the context's 3DSTATE via * xe_lrc_emit_hwe_state_instructions() */ - bb_len += xe_gt_lrc_size(gt, q->hwe->class) / sizeof(u32); + bb_len += xe_gt_lrc_size(gt, hwe->class) / sizeof(u32); - xe_gt_dbg(gt, "LRC %s WA job: %zu dwords\n", q->hwe->name, bb_len); + xe_gt_dbg(gt, "LRC %s WA job: %zu dwords\n", hwe->name, bb_len); bb = xe_bb_new(gt, bb_len, false); if (IS_ERR(bb)) @@ -253,13 +277,23 @@ static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q) } } - if (count_rmw) { - /* Emit MI_MATH for each RMW reg: 20dw per reg + 7 trailing dw */ - + if (count_rmw || count_rmw_mcr) { xa_for_each(&sr->xa, idx, entry) { if (entry->reg.masked || entry->clr_bits == ~0) continue; + if (entry->reg.mcr) { + struct xe_reg_mcr reg = { .__reg.raw = entry->reg.raw }; + u8 group, instance; + + xe_gt_mcr_get_nonterminated_steering(gt, reg, &group, &instance); + *cs++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1); + *cs++ = CS_MMIO_GROUP_INSTANCE_SELECT(hwe->mmio_base).addr; + *cs++ = SELECTIVE_READ_ADDRESSING | + REG_FIELD_PREP(SELECTIVE_READ_GROUP, group) | + REG_FIELD_PREP(SELECTIVE_READ_INSTANCE, instance); + } + *cs++ = MI_LOAD_REGISTER_REG | MI_LRR_DST_CS_MMIO; *cs++ = entry->reg.addr; *cs++ = CS_GPR_REG(0, 0).addr; @@ -285,8 +319,9 @@ static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q) *cs++ = CS_GPR_REG(0, 0).addr; *cs++ = entry->reg.addr; - xe_gt_dbg(gt, "REG[%#x] = ~%#x|%#x\n", - entry->reg.addr, entry->clr_bits, entry->set_bits); + xe_gt_dbg(gt, "REG[%#x] = ~%#x|%#x%s\n", + entry->reg.addr, entry->clr_bits, entry->set_bits, + entry->reg.mcr ? " (MCR)" : ""); } /* reset used GPR */ @@ -298,6 +333,13 @@ static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q) *cs++ = 0; *cs++ = CS_GPR_REG(0, 2).addr; *cs++ = 0; + + /* reset steering */ + if (count_rmw_mcr) { + *cs++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1); + *cs++ = CS_MMIO_GROUP_INSTANCE_SELECT(q->hwe->mmio_base).addr; + *cs++ = 0; + } } cs = xe_lrc_emit_hwe_state_instructions(q, cs); diff --git a/drivers/gpu/drm/xe/xe_guc_pc.c b/drivers/gpu/drm/xe/xe_guc_pc.c index 53fdf59524c41..a3e9796e6430f 100644 --- a/drivers/gpu/drm/xe/xe_guc_pc.c +++ b/drivers/gpu/drm/xe/xe_guc_pc.c @@ -1231,6 +1231,36 @@ int xe_guc_pc_set_power_profile(struct xe_guc_pc *pc, const char *buf) return ret; } +static int pc_action_set_dcc(struct xe_guc_pc *pc, bool enable) +{ + int ret; + + ret = pc_action_set_param(pc, + SLPC_PARAM_TASK_ENABLE_DCC, + enable); + if (!ret) + return pc_action_set_param(pc, + SLPC_PARAM_TASK_DISABLE_DCC, + !enable); + else + return ret; +} + +static int pc_modify_defaults(struct xe_guc_pc *pc) +{ + struct xe_device *xe = pc_to_xe(pc); + struct xe_gt *gt = pc_to_gt(pc); + int ret = 0; + + if (xe->info.platform == XE_PANTHERLAKE) { + ret = pc_action_set_dcc(pc, false); + if (unlikely(ret)) + xe_gt_err(gt, "Failed to modify DCC default: %pe\n", ERR_PTR(ret)); + } + + return ret; +} + /** * xe_guc_pc_start - Start GuC's Power Conservation component * @pc: Xe_GuC_PC instance @@ -1288,6 +1318,10 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) ktime_ms_delta(ktime_get(), earlier)); } + ret = pc_modify_defaults(pc); + if (ret) + return ret; + ret = pc_init_freqs(pc); if (ret) goto out; diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 474789bf6506f..ecee50d827108 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -1298,7 +1298,7 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) if (exec_queue_reset(q)) err = -EIO; - if (!exec_queue_destroyed(q)) { + if (!exec_queue_destroyed(q) && xe_uc_fw_is_running(&guc->fw)) { /* * Wait for any pending G2H to flush out before * modifying state @@ -1330,6 +1330,7 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) */ smp_rmb(); ret = wait_event_timeout(guc->ct.wq, + !xe_uc_fw_is_running(&guc->fw) || !exec_queue_pending_disable(q) || xe_guc_read_stopped(guc), HZ * 5); if (!ret || xe_guc_read_stopped(guc)) { diff --git a/drivers/gpu/drm/xe/xe_mmio.c b/drivers/gpu/drm/xe/xe_mmio.c index ef6f3ea573a2c..6752881af093a 100644 --- a/drivers/gpu/drm/xe/xe_mmio.c +++ b/drivers/gpu/drm/xe/xe_mmio.c @@ -260,11 +260,11 @@ u64 xe_mmio_read64_2x32(struct xe_mmio *mmio, struct xe_reg reg) struct xe_reg reg_udw = { .addr = reg.addr + 0x4 }; u32 ldw, udw, oldudw, retries; - reg.addr = xe_mmio_adjusted_addr(mmio, reg.addr); - reg_udw.addr = xe_mmio_adjusted_addr(mmio, reg_udw.addr); - - /* we shouldn't adjust just one register address */ - xe_tile_assert(mmio->tile, reg_udw.addr == reg.addr + 0x4); + /* + * The two dwords of a 64-bit register can never straddle the offset + * adjustment cutoff. + */ + xe_tile_assert(mmio->tile, !in_range(mmio->adj_limit, reg.addr + 1, 7)); oldudw = xe_mmio_read32(mmio, reg_udw); for (retries = 5; retries; --retries) { diff --git a/drivers/gpu/drm/xe/xe_module.h b/drivers/gpu/drm/xe/xe_module.h index 5a3bfea8b7b4c..b668495392701 100644 --- a/drivers/gpu/drm/xe/xe_module.h +++ b/drivers/gpu/drm/xe/xe_module.h @@ -12,7 +12,7 @@ struct xe_modparam { bool force_execlist; bool probe_display; - u32 force_vram_bar_size; + int force_vram_bar_size; int guc_log_level; char *guc_firmware_path; char *huc_firmware_path; diff --git a/drivers/gpu/drm/xe/xe_reg_sr.c b/drivers/gpu/drm/xe/xe_reg_sr.c index fc8447a838c4f..6b9edc7ca4115 100644 --- a/drivers/gpu/drm/xe/xe_reg_sr.c +++ b/drivers/gpu/drm/xe/xe_reg_sr.c @@ -101,10 +101,12 @@ int xe_reg_sr_add(struct xe_reg_sr *sr, *pentry = *e; ret = xa_err(xa_store(&sr->xa, idx, pentry, GFP_KERNEL)); if (ret) - goto fail; + goto fail_free; return 0; +fail_free: + kfree(pentry); fail: xe_gt_err(gt, "discarding save-restore reg %04lx (clear: %08x, set: %08x, masked: %s, mcr: %s): ret=%d\n", diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c index d71837773d6c6..e0082b55e2162 100644 --- a/drivers/gpu/drm/xe/xe_ring_ops.c +++ b/drivers/gpu/drm/xe/xe_ring_ops.c @@ -265,6 +265,9 @@ static void __emit_job_gen12_simple(struct xe_sched_job *job, struct xe_lrc *lrc i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); + /* Don't preempt fence signaling */ + dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; + if (job->user_fence.used) { i = emit_flush_dw(dw, i); i = emit_store_imm_ppgtt_posted(job->user_fence.addr, @@ -328,6 +331,9 @@ static void __emit_job_gen12_video(struct xe_sched_job *job, struct xe_lrc *lrc, i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); + /* Don't preempt fence signaling */ + dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; + if (job->user_fence.used) { i = emit_flush_dw(dw, i); i = emit_store_imm_ppgtt_posted(job->user_fence.addr, @@ -377,6 +383,9 @@ static void __emit_job_gen12_render_compute(struct xe_sched_job *job, i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); + /* Don't preempt fence signaling */ + dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; + i = emit_render_cache_flush(job, dw, i); if (job->user_fence.used) diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 145cd9ffa36b3..59ff911f8aad0 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -2422,8 +2422,17 @@ static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op, if (IS_ERR(vma)) return vma; - if (xe_vma_is_userptr(vma)) + if (xe_vma_is_userptr(vma)) { err = xe_vma_userptr_pin_pages(to_userptr_vma(vma)); + /* + * -EBUSY has dedicated meaning that a user fence + * attached to the VMA is busy, in practice + * xe_vma_userptr_pin_pages can only fail with -EBUSY if + * we are low on memory so convert this to -ENOMEM. + */ + if (err == -EBUSY) + err = -ENOMEM; + } } if (err) { prep_vma_destroy(vm, vma, false); @@ -3170,7 +3179,8 @@ static void op_add_ufence(struct xe_vm *vm, struct xe_vma_op *op, { switch (op->base.op) { case DRM_GPUVA_OP_MAP: - vma_add_ufence(op->map.vma, ufence); + if (!xe_vma_is_cpu_addr_mirror(op->map.vma)) + vma_add_ufence(op->map.vma, ufence); break; case DRM_GPUVA_OP_REMAP: if (op->remap.prev) diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.c b/drivers/gpu/drm/xe/xe_vm_madvise.c index cad3cf627c3f2..fe7e1b45f5c0c 100644 --- a/drivers/gpu/drm/xe/xe_vm_madvise.c +++ b/drivers/gpu/drm/xe/xe_vm_madvise.c @@ -268,8 +268,13 @@ static bool madvise_args_are_sane(struct xe_device *xe, const struct drm_xe_madv break; case DRM_XE_MEM_RANGE_ATTR_PAT: { - u16 coh_mode = xe_pat_index_get_coh_mode(xe, args->pat_index.val); + u16 pat_index, coh_mode; + if (XE_IOCTL_DBG(xe, args->pat_index.val >= xe->pat.n_entries)) + return false; + + pat_index = array_index_nospec(args->pat_index.val, xe->pat.n_entries); + coh_mode = xe_pat_index_get_coh_mode(xe, pat_index); if (XE_IOCTL_DBG(xe, !coh_mode)) return false; diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c index d209434fd7fc2..89472b7362c22 100644 --- a/drivers/gpu/drm/xe/xe_wa.c +++ b/drivers/gpu/drm/xe/xe_wa.c @@ -15,6 +15,7 @@ #include "regs/xe_engine_regs.h" #include "regs/xe_gt_regs.h" +#include "regs/xe_guc_regs.h" #include "regs/xe_regs.h" #include "xe_device_types.h" #include "xe_force_wake.h" @@ -315,6 +316,10 @@ static const struct xe_rtp_entry_sr gt_was[] = { XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F10(0), RAMDFTUNIT_CLKGATE_DIS)), XE_RTP_ENTRY_FLAG(FOREACH_ENGINE), }, + { XE_RTP_NAME("16028005424"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005)), + XE_RTP_ACTIONS(SET(GUC_INTR_CHICKEN, DISABLE_SIGNALING_ENGINES)) + }, }; static const struct xe_rtp_entry_sr engine_was[] = { @@ -567,16 +572,6 @@ static const struct xe_rtp_entry_sr engine_was[] = { FUNC(xe_rtp_match_first_render_or_compute)), XE_RTP_ACTIONS(SET(ROW_CHICKEN, EARLY_EOT_DIS)) }, - { XE_RTP_NAME("14019988906"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), - FUNC(xe_rtp_match_first_render_or_compute)), - XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FLSH_IGNORES_PSD)) - }, - { XE_RTP_NAME("14019877138"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), - FUNC(xe_rtp_match_first_render_or_compute)), - XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT)) - }, { XE_RTP_NAME("14020338487"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), FUNC(xe_rtp_match_first_render_or_compute)), @@ -873,6 +868,14 @@ static const struct xe_rtp_entry_sr lrc_was[] = { XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(SET(WM_CHICKEN3, HIZ_PLANE_COMPRESSION_DIS)) }, + { XE_RTP_NAME("14019988906"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FLSH_IGNORES_PSD)) + }, + { XE_RTP_NAME("14019877138"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT)) + }, { XE_RTP_NAME("14021490052"), XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(SET(FF_MODE, diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index 04420a713be08..03ad8b5c29a4d 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -369,6 +369,7 @@ config HID_ELECOM - EX-G Trackballs (M-XT3DRBK, M-XT3URBK) - DEFT Trackballs (M-DT1DRBK, M-DT1URBK, M-DT2DRBK, M-DT2URBK) - HUGE Trackballs (M-HT1DRBK, M-HT1URBK) + - HUGE Plus Trackball (M-HT1MRBK) config HID_ELO tristate "ELO USB 4000/4500 touchscreen" diff --git a/drivers/hid/hid-apple.c b/drivers/hid/hid-apple.c index 57da4f86a9fa7..233e367cce1d1 100644 --- a/drivers/hid/hid-apple.c +++ b/drivers/hid/hid-apple.c @@ -354,6 +354,7 @@ static const struct apple_key_translation swapped_fn_leftctrl_keys[] = { }; static const struct apple_non_apple_keyboard non_apple_keyboards[] = { + { "SONiX KN85 Keyboard" }, { "SONiX USB DEVICE" }, { "SONiX AK870 PRO" }, { "Keychron" }, diff --git a/drivers/hid/hid-cmedia.c b/drivers/hid/hid-cmedia.c index 528d7f3612157..8bf5649b0c793 100644 --- a/drivers/hid/hid-cmedia.c +++ b/drivers/hid/hid-cmedia.c @@ -99,7 +99,7 @@ static int cmhid_raw_event(struct hid_device *hid, struct hid_report *report, { struct cmhid *cm = hid_get_drvdata(hid); - if (len != CM6533_JD_RAWEV_LEN) + if (len != CM6533_JD_RAWEV_LEN || !(hid->claimed & HID_CLAIMED_INPUT)) goto out; if (memcmp(data+CM6533_JD_SFX_OFFSET, ji_sfx, sizeof(ji_sfx))) goto out; diff --git a/drivers/hid/hid-creative-sb0540.c b/drivers/hid/hid-creative-sb0540.c index b4c8e7a5d3e02..dfd6add353d19 100644 --- a/drivers/hid/hid-creative-sb0540.c +++ b/drivers/hid/hid-creative-sb0540.c @@ -153,7 +153,7 @@ static int creative_sb0540_raw_event(struct hid_device *hid, u64 code, main_code; int key; - if (len != 6) + if (len != 6 || !(hid->claimed & HID_CLAIMED_INPUT)) return 0; /* From daemons/hw_hiddev.c sb0540_rec() in lirc */ diff --git a/drivers/hid/hid-elecom.c b/drivers/hid/hid-elecom.c index 2003d2dcda7cc..37d88ce57f671 100644 --- a/drivers/hid/hid-elecom.c +++ b/drivers/hid/hid-elecom.c @@ -5,6 +5,7 @@ * - EX-G Trackballs (M-XT3DRBK, M-XT3URBK, M-XT4DRBK) * - DEFT Trackballs (M-DT1DRBK, M-DT1URBK, M-DT2DRBK, M-DT2URBK) * - HUGE Trackballs (M-HT1DRBK, M-HT1URBK) + * - HUGE Plus Trackball (M-HT1MRBK) * * Copyright (c) 2010 Richard Nauber * Copyright (c) 2016 Yuxuan Shui @@ -123,12 +124,25 @@ static const __u8 *elecom_report_fixup(struct hid_device *hdev, __u8 *rdesc, */ mouse_button_fixup(hdev, rdesc, *rsize, 22, 30, 24, 16, 8); break; + case USB_DEVICE_ID_ELECOM_M_HT1MRBK: + case USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AB: + case USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AC: + /* + * Report descriptor format: + * 24: button bit count + * 28: padding bit count + * 22: button report size + * 16: button usage maximum + */ + mouse_button_fixup(hdev, rdesc, *rsize, 24, 28, 22, 16, 8); + break; } return rdesc; } static const struct hid_device_id elecom_devices[] = { { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_BM084) }, + { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AC) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XGL20DLBK) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3URBK_00FB) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3URBK_018F) }, @@ -142,6 +156,8 @@ static const struct hid_device_id elecom_devices[] = { { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1URBK_019B) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1DRBK_010D) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1DRBK_011C) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1MRBK) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AB) }, { } }; MODULE_DEVICE_TABLE(hid, elecom_devices); diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h index b75d9d2f4dc73..f5715cf9468fc 100644 --- a/drivers/hid/hid-ids.h +++ b/drivers/hid/hid-ids.h @@ -437,6 +437,7 @@ #define USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_7349 0x7349 #define USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_73F7 0x73f7 #define USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_A001 0xa001 +#define USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_C000 0xc000 #define USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_C002 0xc002 #define USB_VENDOR_ID_EDIFIER 0x2d99 @@ -465,6 +466,9 @@ #define USB_DEVICE_ID_ELECOM_M_HT1URBK_019B 0x019b #define USB_DEVICE_ID_ELECOM_M_HT1DRBK_010D 0x010d #define USB_DEVICE_ID_ELECOM_M_HT1DRBK_011C 0x011c +#define USB_DEVICE_ID_ELECOM_M_HT1MRBK 0x01aa +#define USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AB 0x01ab +#define USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AC 0x01ac #define USB_VENDOR_ID_DREAM_CHEEKY 0x1d34 #define USB_DEVICE_ID_DREAM_CHEEKY_WN 0x0004 @@ -840,6 +844,7 @@ #define USB_DEVICE_ID_LENOVO_X1_TAB3 0x60b5 #define USB_DEVICE_ID_LENOVO_X12_TAB 0x60fe #define USB_DEVICE_ID_LENOVO_X12_TAB2 0x61ae +#define USB_DEVICE_ID_LENOVO_YOGABOOK9I 0x6161 #define USB_DEVICE_ID_LENOVO_OPTICAL_USB_MOUSE_600E 0x600e #define USB_DEVICE_ID_LENOVO_PIXART_USB_MOUSE_608D 0x608d #define USB_DEVICE_ID_LENOVO_PIXART_USB_MOUSE_6019 0x6019 diff --git a/drivers/hid/hid-logitech-hidpp.c b/drivers/hid/hid-logitech-hidpp.c index 9b612f62d0fba..6b463ce112a3c 100644 --- a/drivers/hid/hid-logitech-hidpp.c +++ b/drivers/hid/hid-logitech-hidpp.c @@ -4313,7 +4313,7 @@ static int hidpp_get_report_length(struct hid_device *hdev, int id) re = &(hdev->report_enum[HID_OUTPUT_REPORT]); report = re->report_id_hash[id]; - if (!report) + if (!report || !report->maxfield) return 0; return report->field[0]->report_count + 1; @@ -4665,6 +4665,8 @@ static const struct hid_device_id hidpp_devices[] = { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb037) }, { /* MX Anywhere 3SB mouse over Bluetooth */ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb038) }, + { /* Slim Solar+ K980 Keyboard over Bluetooth */ + HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb391) }, {} }; diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 7d4a25c6de0eb..91f621ceb924b 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -725,6 +725,11 @@ static int magicmouse_input_configured(struct hid_device *hdev, struct magicmouse_sc *msc = hid_get_drvdata(hdev); int ret; + if (!msc->input) { + hid_err(hdev, "magicmouse setup input failed (no input)"); + return -EINVAL; + } + ret = magicmouse_setup_input(msc->input, hdev); if (ret) { hid_err(hdev, "magicmouse setup input failed (%d)\n", ret); diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c index a0c1ad5acb670..af19e089b0122 100644 --- a/drivers/hid/hid-multitouch.c +++ b/drivers/hid/hid-multitouch.c @@ -76,6 +76,8 @@ MODULE_LICENSE("GPL"); #define MT_QUIRK_DISABLE_WAKEUP BIT(21) #define MT_QUIRK_ORIENTATION_INVERT BIT(22) #define MT_QUIRK_APPLE_TOUCHBAR BIT(23) +#define MT_QUIRK_YOGABOOK9I BIT(24) +#define MT_QUIRK_KEEP_LATENCY_ON_CLOSE BIT(25) #define MT_INPUTMODE_TOUCHSCREEN 0x02 #define MT_INPUTMODE_TOUCHPAD 0x03 @@ -211,6 +213,7 @@ static void mt_post_parse(struct mt_device *td, struct mt_application *app); #define MT_CLS_WIN_8_DISABLE_WAKEUP 0x0016 #define MT_CLS_WIN_8_NO_STICKY_FINGERS 0x0017 #define MT_CLS_WIN_8_FORCE_MULTI_INPUT_NSMU 0x0018 +#define MT_CLS_WIN_8_KEEP_LATENCY_ON_CLOSE 0x0019 /* vendor specific classes */ #define MT_CLS_3M 0x0101 @@ -229,6 +232,8 @@ static void mt_post_parse(struct mt_device *td, struct mt_application *app); #define MT_CLS_RAZER_BLADE_STEALTH 0x0112 #define MT_CLS_SMART_TECH 0x0113 #define MT_CLS_APPLE_TOUCHBAR 0x0114 +#define MT_CLS_YOGABOOK9I 0x0115 +#define MT_CLS_EGALAX_P80H84 0x0116 #define MT_CLS_SIS 0x0457 #define MT_DEFAULT_MAXCONTACT 10 @@ -330,6 +335,15 @@ static const struct mt_class mt_classes[] = { MT_QUIRK_CONTACT_CNT_ACCURATE | MT_QUIRK_WIN8_PTP_BUTTONS, .export_all_inputs = true }, + { .name = MT_CLS_WIN_8_KEEP_LATENCY_ON_CLOSE, + .quirks = MT_QUIRK_ALWAYS_VALID | + MT_QUIRK_IGNORE_DUPLICATES | + MT_QUIRK_HOVERING | + MT_QUIRK_CONTACT_CNT_ACCURATE | + MT_QUIRK_STICKY_FINGERS | + MT_QUIRK_WIN8_PTP_BUTTONS | + MT_QUIRK_KEEP_LATENCY_ON_CLOSE, + .export_all_inputs = true }, /* * vendor specific classes @@ -425,6 +439,19 @@ static const struct mt_class mt_classes[] = { .quirks = MT_QUIRK_NOT_SEEN_MEANS_UP | MT_QUIRK_ALWAYS_VALID | MT_QUIRK_CONTACT_CNT_ACCURATE, + }, + { .name = MT_CLS_YOGABOOK9I, + .quirks = MT_QUIRK_ALWAYS_VALID | + MT_QUIRK_FORCE_MULTI_INPUT | + MT_QUIRK_SEPARATE_APP_REPORT | + MT_QUIRK_HOVERING | + MT_QUIRK_YOGABOOK9I, + .export_all_inputs = true + }, + { .name = MT_CLS_EGALAX_P80H84, + .quirks = MT_QUIRK_ALWAYS_VALID | + MT_QUIRK_IGNORE_DUPLICATES | + MT_QUIRK_CONTACT_CNT_ACCURATE, }, { } }; @@ -831,7 +858,8 @@ static int mt_touch_input_mapping(struct hid_device *hdev, struct hid_input *hi, if ((cls->name == MT_CLS_WIN_8 || cls->name == MT_CLS_WIN_8_FORCE_MULTI_INPUT || cls->name == MT_CLS_WIN_8_FORCE_MULTI_INPUT_NSMU || - cls->name == MT_CLS_WIN_8_DISABLE_WAKEUP) && + cls->name == MT_CLS_WIN_8_DISABLE_WAKEUP || + cls->name == MT_CLS_WIN_8_KEEP_LATENCY_ON_CLOSE) && (field->application == HID_DG_TOUCHPAD || field->application == HID_DG_TOUCHSCREEN)) app->quirks |= MT_QUIRK_CONFIDENCE; @@ -1566,6 +1594,38 @@ static void mt_report(struct hid_device *hid, struct hid_report *report) if (rdata && rdata->is_mt_collection) return mt_touch_report(hid, rdata); + /* Lenovo Yoga Book 9i requires consuming and dropping certain bogus reports */ + if (rdata && rdata->application && + (rdata->application->quirks & MT_QUIRK_YOGABOOK9I)) { + + bool all_zero_report = true; + + for (int f = 0; f < report->maxfield && all_zero_report; f++) { + struct hid_field *fld = report->field[f]; + + for (int i = 0; i < fld->report_count; i++) { + unsigned int usage = fld->usage[i].hid; + + if (usage == HID_DG_INRANGE || + usage == HID_DG_TIPSWITCH || + usage == HID_DG_BARRELSWITCH || + usage == HID_DG_BARRELSWITCH2 || + usage == HID_DG_CONTACTID || + usage == HID_DG_TILT_X || + usage == HID_DG_TILT_Y) { + + if (fld->value[i] != 0) { + all_zero_report = false; + break; + } + } + } + } + + if (all_zero_report) + return; + } + if (field && field->hidinput && field->hidinput->input) input_sync(field->hidinput->input); } @@ -1710,7 +1770,8 @@ static int mt_input_configured(struct hid_device *hdev, struct hid_input *hi) int ret; if (td->is_haptic_touchpad && (td->mtclass.name == MT_CLS_WIN_8 || - td->mtclass.name == MT_CLS_WIN_8_FORCE_MULTI_INPUT)) { + td->mtclass.name == MT_CLS_WIN_8_FORCE_MULTI_INPUT || + td->mtclass.name == MT_CLS_WIN_8_KEEP_LATENCY_ON_CLOSE)) { if (hid_haptic_input_configured(hdev, td->haptic, hi) == 0) td->is_haptic_touchpad = false; } else { @@ -1762,6 +1823,30 @@ static int mt_input_configured(struct hid_device *hdev, struct hid_input *hi) break; } + /* Lenovo Yoga Book 9i requires custom naming to allow differentiation in udev */ + if (hi->report && td->mtclass.quirks & MT_QUIRK_YOGABOOK9I) { + switch (hi->report->id) { + case 48: + suffix = "Touchscreen Top"; + break; + case 56: + suffix = "Touchscreen Bottom"; + break; + case 20: + suffix = "Stylus Top"; + break; + case 40: + suffix = "Stylus Bottom"; + break; + case 80: + suffix = "Emulated Touchpad"; + break; + default: + suffix = ""; + break; + } + } + if (suffix) { hi->input->name = devm_kasprintf(&hdev->dev, GFP_KERNEL, "%s %s", hdev->name, suffix); @@ -1999,7 +2084,12 @@ static void mt_on_hid_hw_open(struct hid_device *hdev) static void mt_on_hid_hw_close(struct hid_device *hdev) { - mt_set_modes(hdev, HID_LATENCY_HIGH, TOUCHPAD_REPORT_NONE); + struct mt_device *td = hid_get_drvdata(hdev); + + if (td->mtclass.quirks & MT_QUIRK_KEEP_LATENCY_ON_CLOSE) + mt_set_modes(hdev, HID_LATENCY_NORMAL, TOUCHPAD_REPORT_NONE); + else + mt_set_modes(hdev, HID_LATENCY_HIGH, TOUCHPAD_REPORT_NONE); } /* @@ -2136,8 +2226,12 @@ static const struct hid_device_id mt_devices[] = { { .driver_data = MT_CLS_EGALAX_SERIAL, MT_USB_DEVICE(USB_VENDOR_ID_DWAV, USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_A001) }, - { .driver_data = MT_CLS_EGALAX, + { .driver_data = MT_CLS_EGALAX_SERIAL, MT_USB_DEVICE(USB_VENDOR_ID_DWAV, + USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_C000) }, + { .driver_data = MT_CLS_EGALAX_P80H84, + HID_DEVICE(HID_BUS_ANY, HID_GROUP_MULTITOUCH_WIN_8, + USB_VENDOR_ID_DWAV, USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_C002) }, /* Elan devices */ @@ -2267,6 +2361,12 @@ static const struct hid_device_id mt_devices[] = { USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_X12_TAB2) }, + /* Lenovo Yoga Book 9i */ + { .driver_data = MT_CLS_YOGABOOK9I, + HID_DEVICE(BUS_USB, HID_GROUP_MULTITOUCH_WIN_8, + USB_VENDOR_ID_LENOVO, + USB_DEVICE_ID_LENOVO_YOGABOOK9I) }, + /* Logitech devices */ { .driver_data = MT_CLS_NSMU, HID_DEVICE(BUS_BLUETOOTH, HID_GROUP_MULTITOUCH_WIN_8, @@ -2376,6 +2476,14 @@ static const struct hid_device_id mt_devices[] = { MT_USB_DEVICE(USB_VENDOR_ID_UNITEC, USB_DEVICE_ID_UNITEC_USB_TOUCH_0A19) }, + /* Uniwill touchpads */ + { .driver_data = MT_CLS_WIN_8_KEEP_LATENCY_ON_CLOSE, + HID_DEVICE(BUS_I2C, HID_GROUP_MULTITOUCH_WIN_8, + USB_VENDOR_ID_PIXART, 0x0255) }, + { .driver_data = MT_CLS_WIN_8_KEEP_LATENCY_ON_CLOSE, + HID_DEVICE(BUS_I2C, HID_GROUP_MULTITOUCH_WIN_8, + USB_VENDOR_ID_PIXART, 0x0274) }, + /* VTL panels */ { .driver_data = MT_CLS_VTL, MT_USB_DEVICE(USB_VENDOR_ID_VTL, diff --git a/drivers/hid/hid-pl.c b/drivers/hid/hid-pl.c index 3c8827081deae..dc11d5322fc0f 100644 --- a/drivers/hid/hid-pl.c +++ b/drivers/hid/hid-pl.c @@ -194,9 +194,14 @@ static int pl_probe(struct hid_device *hdev, const struct hid_device_id *id) goto err; } - plff_init(hdev); + ret = plff_init(hdev); + if (ret) + goto stop; return 0; + +stop: + hid_hw_stop(hdev); err: return ret; } diff --git a/drivers/hid/hid-playstation.c b/drivers/hid/hid-playstation.c index e4dfcf26b04e7..2ec6d4445e84b 100644 --- a/drivers/hid/hid-playstation.c +++ b/drivers/hid/hid-playstation.c @@ -774,7 +774,9 @@ ps_gamepad_create(struct hid_device *hdev, #if IS_ENABLED(CONFIG_PLAYSTATION_FF) if (play_effect) { input_set_capability(gamepad, EV_FF, FF_RUMBLE); - input_ff_create_memless(gamepad, NULL, play_effect); + ret = input_ff_create_memless(gamepad, NULL, play_effect); + if (ret) + return ERR_PTR(ret); } #endif diff --git a/drivers/hid/hid-prodikeys.c b/drivers/hid/hid-prodikeys.c index 74bddb2c3e82e..6e413df38358a 100644 --- a/drivers/hid/hid-prodikeys.c +++ b/drivers/hid/hid-prodikeys.c @@ -378,6 +378,10 @@ static int pcmidi_handle_report4(struct pcmidi_snd *pm, u8 *data) bit_mask = (bit_mask << 8) | data[2]; bit_mask = (bit_mask << 8) | data[3]; + /* robustness in case input_mapping hook does not get called */ + if (!pm->input_ep82) + return 0; + /* break keys */ for (bit_index = 0; bit_index < 24; bit_index++) { if (!((0x01 << bit_index) & bit_mask)) { diff --git a/drivers/hid/hid-quirks.c b/drivers/hid/hid-quirks.c index 11438039cdb7f..3217e436c052c 100644 --- a/drivers/hid/hid-quirks.c +++ b/drivers/hid/hid-quirks.c @@ -420,6 +420,7 @@ static const struct hid_device_id hid_have_special_driver[] = { #if IS_ENABLED(CONFIG_HID_ELECOM) { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_BM084) }, { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XGL20DLBK) }, + { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AC) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3URBK_00FB) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3URBK_018F) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3DRBK_00FC) }, @@ -432,6 +433,8 @@ static const struct hid_device_id hid_have_special_driver[] = { { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1URBK_019B) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1DRBK_010D) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1DRBK_011C) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1MRBK) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AB) }, #endif #if IS_ENABLED(CONFIG_HID_ELO) { HID_USB_DEVICE(USB_VENDOR_ID_ELO, 0x0009) }, diff --git a/drivers/hid/hid-zydacron.c b/drivers/hid/hid-zydacron.c index 3bdb26f455925..1aae80f848f50 100644 --- a/drivers/hid/hid-zydacron.c +++ b/drivers/hid/hid-zydacron.c @@ -114,7 +114,7 @@ static int zc_raw_event(struct hid_device *hdev, struct hid_report *report, unsigned key; unsigned short index; - if (report->id == data[0]) { + if (report->id == data[0] && (hdev->claimed & HID_CLAIMED_INPUT)) { /* break keys */ for (index = 0; index < 4; index++) { diff --git a/drivers/hid/i2c-hid/i2c-hid-of-elan.c b/drivers/hid/i2c-hid/i2c-hid-of-elan.c index 0215f217f6d86..b81fcc6ff49ee 100644 --- a/drivers/hid/i2c-hid/i2c-hid-of-elan.c +++ b/drivers/hid/i2c-hid/i2c-hid-of-elan.c @@ -168,6 +168,13 @@ static const struct elan_i2c_hid_chip_data elan_ekth6a12nay_chip_data = { .power_after_backlight = true, }; +static const struct elan_i2c_hid_chip_data focaltech_ft8112_chip_data = { + .post_power_delay_ms = 10, + .post_gpio_reset_on_delay_ms = 150, + .hid_descriptor_address = 0x0001, + .main_supply_name = "vcc33", +}; + static const struct elan_i2c_hid_chip_data ilitek_ili9882t_chip_data = { .post_power_delay_ms = 1, .post_gpio_reset_on_delay_ms = 200, @@ -191,6 +198,7 @@ static const struct elan_i2c_hid_chip_data ilitek_ili2901_chip_data = { static const struct of_device_id elan_i2c_hid_of_match[] = { { .compatible = "elan,ekth6915", .data = &elan_ekth6915_chip_data }, { .compatible = "elan,ekth6a12nay", .data = &elan_ekth6a12nay_chip_data }, + { .compatible = "focaltech,ft8112", .data = &focaltech_ft8112_chip_data }, { .compatible = "ilitek,ili9882t", .data = &ilitek_ili9882t_chip_data }, { .compatible = "ilitek,ili2901", .data = &ilitek_ili2901_chip_data }, { } diff --git a/drivers/hid/intel-ish-hid/ishtp/bus.c b/drivers/hid/intel-ish-hid/ishtp/bus.c index c3915f3a060ea..b890fbf97a75c 100644 --- a/drivers/hid/intel-ish-hid/ishtp/bus.c +++ b/drivers/hid/intel-ish-hid/ishtp/bus.c @@ -730,7 +730,7 @@ void ishtp_bus_remove_all_clients(struct ishtp_device *ishtp_dev, spin_lock_irqsave(&ishtp_dev->cl_list_lock, flags); list_for_each_entry(cl, &ishtp_dev->cl_list, link) { cl->state = ISHTP_CL_DISCONNECTED; - if (warm_reset && cl->device->reference_count) + if (warm_reset && cl->device && cl->device->reference_count) continue; /* diff --git a/drivers/hid/usbhid/hid-pidff.c b/drivers/hid/usbhid/hid-pidff.c index 95377c5f63356..56d6af39ba81e 100644 --- a/drivers/hid/usbhid/hid-pidff.c +++ b/drivers/hid/usbhid/hid-pidff.c @@ -523,9 +523,19 @@ static void pidff_set_effect_report(struct pidff_device *pidff, pidff_set_duration(&pidff->set_effect[PID_DURATION], effect->replay.length); - pidff->set_effect[PID_TRIGGER_BUTTON].value[0] = effect->trigger.button; - pidff_set_time(&pidff->set_effect[PID_TRIGGER_REPEAT_INT], - effect->trigger.interval); + /* Some games set this to random values that can be out of range */ + s32 trigger_button_max = + pidff->set_effect[PID_TRIGGER_BUTTON].field->logical_maximum; + if (effect->trigger.button <= trigger_button_max) { + pidff->set_effect[PID_TRIGGER_BUTTON].value[0] = + effect->trigger.button; + pidff_set_time(&pidff->set_effect[PID_TRIGGER_REPEAT_INT], + effect->trigger.interval); + } else { + pidff->set_effect[PID_TRIGGER_BUTTON].value[0] = 0; + pidff->set_effect[PID_TRIGGER_REPEAT_INT].value[0] = 0; + } + pidff->set_effect[PID_GAIN].value[0] = pidff->set_effect[PID_GAIN].field->logical_maximum; @@ -1442,10 +1452,13 @@ static int pidff_init_fields(struct pidff_device *pidff, struct input_dev *dev) hid_warn(pidff->hid, "unknown ramp effect layout\n"); if (PIDFF_FIND_FIELDS(set_condition, PID_SET_CONDITION, 1)) { - if (test_and_clear_bit(FF_SPRING, dev->ffbit) || - test_and_clear_bit(FF_DAMPER, dev->ffbit) || - test_and_clear_bit(FF_FRICTION, dev->ffbit) || - test_and_clear_bit(FF_INERTIA, dev->ffbit)) + bool test = false; + + test |= test_and_clear_bit(FF_SPRING, dev->ffbit); + test |= test_and_clear_bit(FF_DAMPER, dev->ffbit); + test |= test_and_clear_bit(FF_FRICTION, dev->ffbit); + test |= test_and_clear_bit(FF_INERTIA, dev->ffbit); + if (test) hid_warn(pidff->hid, "unknown condition effect layout\n"); } diff --git a/drivers/hv/mshv_eventfd.c b/drivers/hv/mshv_eventfd.c index 806674722868b..40b7179fee805 100644 --- a/drivers/hv/mshv_eventfd.c +++ b/drivers/hv/mshv_eventfd.c @@ -87,8 +87,9 @@ static void mshv_irqfd_resampler_ack(struct mshv_irq_ack_notifier *mian) idx = srcu_read_lock(&partition->pt_irq_srcu); - hlist_for_each_entry_rcu(irqfd, &resampler->rsmplr_irqfd_list, - irqfd_resampler_hnode) { + hlist_for_each_entry_srcu(irqfd, &resampler->rsmplr_irqfd_list, + irqfd_resampler_hnode, + srcu_read_lock_held(&partition->pt_irq_srcu)) { if (hv_should_clear_interrupt(irqfd->irqfd_lapic_irq.lapic_control.interrupt_type)) hv_call_clear_virtual_interrupt(partition->pt_id); @@ -243,12 +244,13 @@ static void mshv_irqfd_shutdown(struct work_struct *work) { struct mshv_irqfd *irqfd = container_of(work, struct mshv_irqfd, irqfd_shutdown); + u64 cnt; /* * Synchronize with the wait-queue and unhook ourselves to prevent * further events. */ - remove_wait_queue(irqfd->irqfd_wqh, &irqfd->irqfd_wait); + eventfd_ctx_remove_wait_queue(irqfd->irqfd_eventfd_ctx, &irqfd->irqfd_wait, &cnt); if (irqfd->irqfd_resampler) { mshv_irqfd_resampler_shutdown(irqfd); @@ -367,8 +369,6 @@ static void mshv_irqfd_queue_proc(struct file *file, wait_queue_head_t *wqh, struct mshv_irqfd *irqfd = container_of(polltbl, struct mshv_irqfd, irqfd_polltbl); - irqfd->irqfd_wqh = wqh; - /* * TODO: Ensure there isn't already an exclusive, priority waiter, e.g. * that the irqfd isn't already bound to another partition. Only the diff --git a/drivers/hv/mshv_eventfd.h b/drivers/hv/mshv_eventfd.h index 332e7670a3442..464c6b81ab336 100644 --- a/drivers/hv/mshv_eventfd.h +++ b/drivers/hv/mshv_eventfd.h @@ -32,7 +32,6 @@ struct mshv_irqfd { struct mshv_lapic_irq irqfd_lapic_irq; struct hlist_node irqfd_hnode; poll_table irqfd_polltbl; - wait_queue_head_t *irqfd_wqh; wait_queue_entry_t irqfd_wait; struct work_struct irqfd_shutdown; struct mshv_irqfd_resampler *irqfd_resampler; diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index 69591dc7bad26..3ab62277b6be6 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -1306,7 +1307,7 @@ static void vmbus_chan_sched(struct hv_per_cpu_context *hv_cpu) } } -static void vmbus_isr(void) +static void __vmbus_isr(void) { struct hv_per_cpu_context *hv_cpu = this_cpu_ptr(hv_context.cpu_context); @@ -1330,6 +1331,53 @@ static void vmbus_isr(void) add_interrupt_randomness(vmbus_interrupt); } +static DEFINE_PER_CPU(bool, vmbus_irq_pending); +static DEFINE_PER_CPU(struct task_struct *, vmbus_irqd); + +static void vmbus_irqd_wake(void) +{ + struct task_struct *tsk = __this_cpu_read(vmbus_irqd); + + __this_cpu_write(vmbus_irq_pending, true); + wake_up_process(tsk); +} + +static void vmbus_irqd_setup(unsigned int cpu) +{ + sched_set_fifo(current); +} + +static int vmbus_irqd_should_run(unsigned int cpu) +{ + return __this_cpu_read(vmbus_irq_pending); +} + +static void run_vmbus_irqd(unsigned int cpu) +{ + __this_cpu_write(vmbus_irq_pending, false); + __vmbus_isr(); +} + +static bool vmbus_irq_initialized; + +static struct smp_hotplug_thread vmbus_irq_threads = { + .store = &vmbus_irqd, + .setup = vmbus_irqd_setup, + .thread_should_run = vmbus_irqd_should_run, + .thread_fn = run_vmbus_irqd, + .thread_comm = "vmbus_irq/%u", +}; + +static void vmbus_isr(void) +{ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) { + vmbus_irqd_wake(); + } else { + lockdep_hardirq_threaded(); + __vmbus_isr(); + } +} + static irqreturn_t vmbus_percpu_isr(int irq, void *dev_id) { vmbus_isr(); @@ -1375,6 +1423,13 @@ static int vmbus_bus_init(void) * the VMbus interrupt handler. */ + if (IS_ENABLED(CONFIG_PREEMPT_RT) && !vmbus_irq_initialized) { + ret = smpboot_register_percpu_thread(&vmbus_irq_threads); + if (ret) + goto err_kthread; + vmbus_irq_initialized = true; + } + if (vmbus_irq == -1) { hv_setup_vmbus_handler(vmbus_isr); } else { @@ -1449,6 +1504,11 @@ static int vmbus_bus_init(void) free_percpu(vmbus_evt); } err_setup: + if (IS_ENABLED(CONFIG_PREEMPT_RT) && vmbus_irq_initialized) { + smpboot_unregister_percpu_thread(&vmbus_irq_threads); + vmbus_irq_initialized = false; + } +err_kthread: bus_unregister(&hv_bus); return ret; } @@ -2914,6 +2974,10 @@ static void __exit vmbus_exit(void) free_percpu_irq(vmbus_irq, vmbus_evt); free_percpu(vmbus_evt); } + if (IS_ENABLED(CONFIG_PREEMPT_RT) && vmbus_irq_initialized) { + smpboot_unregister_percpu_thread(&vmbus_irq_threads); + vmbus_irq_initialized = false; + } for_each_online_cpu(cpu) { struct hv_per_cpu_context *hv_cpu = per_cpu_ptr(hv_context.cpu_context, cpu); diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig index 2760feb9f83b5..2a71b6e834b08 100644 --- a/drivers/hwmon/Kconfig +++ b/drivers/hwmon/Kconfig @@ -245,12 +245,12 @@ config SENSORS_ADT7475 will be called adt7475. config SENSORS_AHT10 - tristate "Aosong AHT10, AHT20" + tristate "Aosong AHT10, AHT20, DHT20" depends on I2C select CRC8 help - If you say yes here, you get support for the Aosong AHT10 and AHT20 - temperature and humidity sensors + If you say yes here, you get support for the Aosong AHT10, AHT20 and + DHT20 temperature and humidity sensors This driver can also be built as a module. If so, the module will be called aht10. diff --git a/drivers/hwmon/aht10.c b/drivers/hwmon/aht10.c index d1c55e2eb4799..7ba00bafb57d6 100644 --- a/drivers/hwmon/aht10.c +++ b/drivers/hwmon/aht10.c @@ -37,6 +37,10 @@ #define AHT10_CMD_MEAS 0b10101100 #define AHT10_CMD_RST 0b10111010 +#define AHT20_CMD_INIT 0b10111110 + +#define DHT20_CMD_INIT 0b01110001 + /* * Flags in the answer byte/command */ @@ -48,11 +52,12 @@ #define AHT10_MAX_POLL_INTERVAL_LEN 30 -enum aht10_variant { aht10, aht20 }; +enum aht10_variant { aht10, aht20, dht20}; static const struct i2c_device_id aht10_id[] = { { "aht10", aht10 }, { "aht20", aht20 }, + { "dht20", dht20 }, { }, }; MODULE_DEVICE_TABLE(i2c, aht10_id); @@ -77,6 +82,7 @@ MODULE_DEVICE_TABLE(i2c, aht10_id); * AHT10/AHT20 * @crc8: crc8 support flag * @meas_size: measurements data size + * @init_cmd: Initialization command */ struct aht10_data { @@ -92,6 +98,7 @@ struct aht10_data { int humidity; bool crc8; unsigned int meas_size; + u8 init_cmd; }; /* @@ -101,13 +108,13 @@ struct aht10_data { */ static int aht10_init(struct aht10_data *data) { - const u8 cmd_init[] = {AHT10_CMD_INIT, AHT10_CAL_ENABLED | AHT10_MODE_CYC, + const u8 cmd_init[] = {data->init_cmd, AHT10_CAL_ENABLED | AHT10_MODE_CYC, 0x00}; int res; u8 status; struct i2c_client *client = data->client; - res = i2c_master_send(client, cmd_init, 3); + res = i2c_master_send(client, cmd_init, sizeof(cmd_init)); if (res < 0) return res; @@ -352,9 +359,17 @@ static int aht10_probe(struct i2c_client *client) data->meas_size = AHT20_MEAS_SIZE; data->crc8 = true; crc8_populate_msb(crc8_table, AHT20_CRC8_POLY); + data->init_cmd = AHT20_CMD_INIT; + break; + case dht20: + data->meas_size = AHT20_MEAS_SIZE; + data->crc8 = true; + crc8_populate_msb(crc8_table, AHT20_CRC8_POLY); + data->init_cmd = DHT20_CMD_INIT; break; default: data->meas_size = AHT10_MEAS_SIZE; + data->init_cmd = AHT10_CMD_INIT; break; } diff --git a/drivers/hwmon/dell-smm-hwmon.c b/drivers/hwmon/dell-smm-hwmon.c index f3d484a9f708b..768690a597f4d 100644 --- a/drivers/hwmon/dell-smm-hwmon.c +++ b/drivers/hwmon/dell-smm-hwmon.c @@ -1325,6 +1325,13 @@ static const struct dmi_system_id i8k_dmi_table[] __initconst = { DMI_MATCH(DMI_PRODUCT_NAME, "MP061"), }, }, + { + .ident = "Dell OptiPlex 7080", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "OptiPlex 7080"), + }, + }, { .ident = "Dell OptiPlex 7060", .matches = { diff --git a/drivers/hwmon/emc2305.c b/drivers/hwmon/emc2305.c index ceae96c07ac45..67e82021da210 100644 --- a/drivers/hwmon/emc2305.c +++ b/drivers/hwmon/emc2305.c @@ -578,6 +578,7 @@ static int emc2305_of_parse_pwm_child(struct device *dev, data->pwm_output_mask |= EMC2305_OPEN_DRAIN << ch; } + of_node_put(args.np); return 0; } diff --git a/drivers/hwmon/f71882fg.c b/drivers/hwmon/f71882fg.c index df83f9866fbcf..204059d2de6cd 100644 --- a/drivers/hwmon/f71882fg.c +++ b/drivers/hwmon/f71882fg.c @@ -51,6 +51,7 @@ #define SIO_F81866_ID 0x1010 /* Chipset ID */ #define SIO_F71858AD_ID 0x0903 /* Chipset ID */ #define SIO_F81966_ID 0x1502 /* Chipset ID */ +#define SIO_F81968_ID 0x1806 /* Chipset ID */ #define REGION_LENGTH 8 #define ADDR_REG_OFFSET 5 @@ -2570,6 +2571,7 @@ static int __init f71882fg_find(int sioaddr, struct f71882fg_sio_data *sio_data) break; case SIO_F81866_ID: case SIO_F81966_ID: + case SIO_F81968_ID: sio_data->type = f81866a; break; default: @@ -2599,9 +2601,9 @@ static int __init f71882fg_find(int sioaddr, struct f71882fg_sio_data *sio_data) address &= ~(REGION_LENGTH - 1); /* Ignore 3 LSB */ err = address; - pr_info("Found %s chip at %#x, revision %d\n", + pr_info("Found %s chip at %#x, revision %d, devid: %04x\n", f71882fg_names[sio_data->type], (unsigned int)address, - (int)superio_inb(sioaddr, SIO_REG_DEVREV)); + (int)superio_inb(sioaddr, SIO_REG_DEVREV), devid); exit: superio_exit(sioaddr); return err; diff --git a/drivers/hwmon/ibmpex.c b/drivers/hwmon/ibmpex.c index 129f3a9e8fe96..228c5f6c6f383 100644 --- a/drivers/hwmon/ibmpex.c +++ b/drivers/hwmon/ibmpex.c @@ -277,9 +277,6 @@ static ssize_t ibmpex_high_low_store(struct device *dev, { struct ibmpex_bmc_data *data = dev_get_drvdata(dev); - if (!data) - return -ENODEV; - ibmpex_reset_high_low_data(data); return count; @@ -511,9 +508,6 @@ static void ibmpex_bmc_delete(struct ibmpex_bmc_data *data) { int i, j; - hwmon_device_unregister(data->hwmon_dev); - dev_set_drvdata(data->bmc_device, NULL); - device_remove_file(data->bmc_device, &sensor_dev_attr_reset_high_low.dev_attr); device_remove_file(data->bmc_device, &dev_attr_name.attr); @@ -527,7 +521,8 @@ static void ibmpex_bmc_delete(struct ibmpex_bmc_data *data) } list_del(&data->list); - + dev_set_drvdata(data->bmc_device, NULL); + hwmon_device_unregister(data->hwmon_dev); ipmi_destroy_user(data->user); kfree(data->sensors); kfree(data); diff --git a/drivers/hwmon/it87.c b/drivers/hwmon/it87.c index e233aafa8856c..5cfb98a0512f0 100644 --- a/drivers/hwmon/it87.c +++ b/drivers/hwmon/it87.c @@ -3590,10 +3590,13 @@ static int it87_resume(struct device *dev) { struct platform_device *pdev = to_platform_device(dev); struct it87_data *data = dev_get_drvdata(dev); + int err; it87_resume_sio(pdev); - it87_lock(data); + err = it87_lock(data); + if (err) + return err; it87_check_pwm(dev); it87_check_limit_regs(data); diff --git a/drivers/hwmon/max16065.c b/drivers/hwmon/max16065.c index 4c9e7892a73c1..43fbb9b26b102 100644 --- a/drivers/hwmon/max16065.c +++ b/drivers/hwmon/max16065.c @@ -151,27 +151,27 @@ static struct max16065_data *max16065_update_device(struct device *dev) int i; for (i = 0; i < data->num_adc; i++) - data->adc[i] - = max16065_read_adc(client, MAX16065_ADC(i)); + WRITE_ONCE(data->adc[i], + max16065_read_adc(client, MAX16065_ADC(i))); if (data->have_current) { - data->adc[MAX16065_NUM_ADC] - = max16065_read_adc(client, MAX16065_CSP_ADC); - data->curr_sense - = i2c_smbus_read_byte_data(client, - MAX16065_CURR_SENSE); + WRITE_ONCE(data->adc[MAX16065_NUM_ADC], + max16065_read_adc(client, MAX16065_CSP_ADC)); + WRITE_ONCE(data->curr_sense, + i2c_smbus_read_byte_data(client, MAX16065_CURR_SENSE)); } for (i = 0; i < 2; i++) - data->fault[i] - = i2c_smbus_read_byte_data(client, MAX16065_FAULT(i)); + WRITE_ONCE(data->fault[i], + i2c_smbus_read_byte_data(client, MAX16065_FAULT(i))); /* * MAX16067 and MAX16068 have separate undervoltage and * overvoltage alarm bits. Squash them together. */ if (data->chip == max16067 || data->chip == max16068) - data->fault[0] |= data->fault[1]; + WRITE_ONCE(data->fault[0], + data->fault[0] | data->fault[1]); data->last_updated = jiffies; data->valid = true; @@ -185,7 +185,7 @@ static ssize_t max16065_alarm_show(struct device *dev, { struct sensor_device_attribute_2 *attr2 = to_sensor_dev_attr_2(da); struct max16065_data *data = max16065_update_device(dev); - int val = data->fault[attr2->nr]; + int val = READ_ONCE(data->fault[attr2->nr]); if (val < 0) return val; @@ -203,7 +203,7 @@ static ssize_t max16065_input_show(struct device *dev, { struct sensor_device_attribute *attr = to_sensor_dev_attr(da); struct max16065_data *data = max16065_update_device(dev); - int adc = data->adc[attr->index]; + int adc = READ_ONCE(data->adc[attr->index]); if (unlikely(adc < 0)) return adc; @@ -216,7 +216,7 @@ static ssize_t max16065_current_show(struct device *dev, struct device_attribute *da, char *buf) { struct max16065_data *data = max16065_update_device(dev); - int curr_sense = data->curr_sense; + int curr_sense = READ_ONCE(data->curr_sense); if (unlikely(curr_sense < 0)) return curr_sense; diff --git a/drivers/hwmon/max6639.c b/drivers/hwmon/max6639.c index a06346496e1d9..1fc12e1463b58 100644 --- a/drivers/hwmon/max6639.c +++ b/drivers/hwmon/max6639.c @@ -623,7 +623,7 @@ static int max6639_init_client(struct i2c_client *client, return err; /* Fans PWM polarity high by default */ - err = regmap_write(data->regmap, MAX6639_REG_FAN_CONFIG2a(i), 0x00); + err = regmap_write(data->regmap, MAX6639_REG_FAN_CONFIG2a(i), 0x02); if (err) return err; diff --git a/drivers/hwmon/nct6683.c b/drivers/hwmon/nct6683.c index 6cda35388b24c..4a83804140386 100644 --- a/drivers/hwmon/nct6683.c +++ b/drivers/hwmon/nct6683.c @@ -181,6 +181,7 @@ superio_exit(int ioreg) #define NCT6683_CUSTOMER_ID_ASROCK2 0xe1b #define NCT6683_CUSTOMER_ID_ASROCK3 0x1631 #define NCT6683_CUSTOMER_ID_ASROCK4 0x163e +#define NCT6683_CUSTOMER_ID_ASROCK5 0x1621 #define NCT6683_REG_BUILD_YEAR 0x604 #define NCT6683_REG_BUILD_MONTH 0x605 @@ -1242,6 +1243,8 @@ static int nct6683_probe(struct platform_device *pdev) break; case NCT6683_CUSTOMER_ID_ASROCK4: break; + case NCT6683_CUSTOMER_ID_ASROCK5: + break; default: if (!force) return -ENODEV; diff --git a/drivers/hwmon/nct6775-platform.c b/drivers/hwmon/nct6775-platform.c index 407945d2cd6a8..0b529a542b0d4 100644 --- a/drivers/hwmon/nct6775-platform.c +++ b/drivers/hwmon/nct6775-platform.c @@ -1357,6 +1357,7 @@ static const char * const asus_msi_boards[] = { "Pro WS W680-ACE IPMI", "Pro WS W790-ACE", "Pro WS W790E-SAGE SE", + "Pro WS WRX90E-SAGE SE", "ProArt B650-CREATOR", "ProArt B660-CREATOR D4", "ProArt B760-CREATOR D4", diff --git a/drivers/hwmon/nct7363.c b/drivers/hwmon/nct7363.c index e13ab918b1abe..56ac42e4a8653 100644 --- a/drivers/hwmon/nct7363.c +++ b/drivers/hwmon/nct7363.c @@ -351,6 +351,7 @@ static int nct7363_present_pwm_fanin(struct device *dev, if (ret) return ret; + of_node_put(args.np); if (args.args[0] >= NCT7363_PWM_COUNT) return -EINVAL; data->pwm_mask |= BIT(args.args[0]); diff --git a/drivers/hwmon/pmbus/mpq8785.c b/drivers/hwmon/pmbus/mpq8785.c index 1f56aaf4dde80..87bd039c77b9b 100644 --- a/drivers/hwmon/pmbus/mpq8785.c +++ b/drivers/hwmon/pmbus/mpq8785.c @@ -47,6 +47,33 @@ static int mpq8785_identify(struct i2c_client *client, return 0; }; +static int mpq8785_read_byte_data(struct i2c_client *client, int page, int reg) +{ + int ret; + + switch (reg) { + case PMBUS_VOUT_MODE: + ret = pmbus_read_byte_data(client, page, reg); + if (ret < 0) + return ret; + + if ((ret >> 5) == 1) { + /* + * The MPQ8785 chip reports VOUT_MODE as VID mode, but the driver + * treats VID as direct mode. Without this, identification would fail + * due to mode mismatch. + * This override ensures the reported mode matches the driver + * configuration, allowing successful initialization. + */ + return PB_VOUT_MODE_DIRECT; + } + + return ret; + default: + return -ENODATA; + } +} + static int mpm82504_read_word_data(struct i2c_client *client, int page, int phase, int reg) { @@ -129,6 +156,7 @@ static int mpq8785_probe(struct i2c_client *client) break; case mpq8785: info->identify = mpq8785_identify; + info->read_byte_data = mpq8785_read_byte_data; break; default: return -ENODEV; diff --git a/drivers/hwspinlock/omap_hwspinlock.c b/drivers/hwspinlock/omap_hwspinlock.c index 27b47b8623c09..2d8de835bc242 100644 --- a/drivers/hwspinlock/omap_hwspinlock.c +++ b/drivers/hwspinlock/omap_hwspinlock.c @@ -88,7 +88,9 @@ static int omap_hwspinlock_probe(struct platform_device *pdev) * make sure the module is enabled and clocked before reading * the module SYSSTATUS register */ - devm_pm_runtime_enable(&pdev->dev); + ret = devm_pm_runtime_enable(&pdev->dev); + if (ret) + return ret; ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) return ret; diff --git a/drivers/hwtracing/coresight/coresight-etm3x-core.c b/drivers/hwtracing/coresight/coresight-etm3x-core.c index a5e809589d3e3..0c011b7041696 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-core.c @@ -795,16 +795,16 @@ static int __init etm_hp_setup(void) { int ret; - ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ARM_CORESIGHT_STARTING, - "arm/coresight:starting", - etm_starting_cpu, etm_dying_cpu); + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING, + "arm/coresight:starting", + etm_starting_cpu, etm_dying_cpu); if (ret) return ret; - ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN, - "arm/coresight:online", - etm_online_cpu, NULL); + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "arm/coresight:online", + etm_online_cpu, NULL); /* HP dyn state ID returned in ret on success */ if (ret > 0) { diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c index 8d2d0f3f7ded6..10853a83ecb02 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etr.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c @@ -1324,6 +1324,19 @@ static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev) raw_spin_lock_irqsave(&drvdata->spinlock, flags); + /* + * Since the sysfs buffer allocation and the hardware enablement is not + * in the same critical region, it's possible to race with the perf. + */ + if (coresight_get_mode(csdev) == CS_MODE_PERF) { + drvdata->sysfs_buf = NULL; + raw_spin_unlock_irqrestore(&drvdata->spinlock, flags); + + /* Free allocated memory out side of the spinlock */ + tmc_etr_free_sysfs_buf(sysfs_buf); + return -EBUSY; + } + /* * In sysFS mode we can have multiple writers per sink. Since this * sink is already enabled no memory is needed and the HW need not be diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c index 57fbec1259bea..506d69b156f7c 100644 --- a/drivers/i2c/busses/i2c-i801.c +++ b/drivers/i2c/busses/i2c-i801.c @@ -306,9 +306,10 @@ struct i801_priv { /* * If set to true the host controller registers are reserved for - * ACPI AML use. + * ACPI AML use. Needs extra protection by acpi_lock. */ bool acpi_reserved; + struct mutex acpi_lock; }; #define FEATURE_SMBUS_PEC BIT(0) @@ -890,8 +891,11 @@ static s32 i801_access(struct i2c_adapter *adap, u16 addr, int hwpec, ret; struct i801_priv *priv = i2c_get_adapdata(adap); - if (priv->acpi_reserved) + mutex_lock(&priv->acpi_lock); + if (priv->acpi_reserved) { + mutex_unlock(&priv->acpi_lock); return -EBUSY; + } pm_runtime_get_sync(&priv->pci_dev->dev); @@ -931,6 +935,7 @@ static s32 i801_access(struct i2c_adapter *adap, u16 addr, iowrite8(SMBHSTSTS_INUSE_STS | STATUS_FLAGS, SMBHSTSTS(priv)); pm_runtime_put_autosuspend(&priv->pci_dev->dev); + mutex_unlock(&priv->acpi_lock); return ret; } @@ -1459,7 +1464,7 @@ i801_acpi_io_handler(u32 function, acpi_physical_address address, u32 bits, * further access from the driver itself. This device is now owned * by the system firmware. */ - i2c_lock_bus(&priv->adapter, I2C_LOCK_SEGMENT); + mutex_lock(&priv->acpi_lock); if (!priv->acpi_reserved && i801_acpi_is_smbus_ioport(priv, address)) { priv->acpi_reserved = true; @@ -1479,7 +1484,7 @@ i801_acpi_io_handler(u32 function, acpi_physical_address address, u32 bits, else status = acpi_os_write_port(address, (u32)*value, bits); - i2c_unlock_bus(&priv->adapter, I2C_LOCK_SEGMENT); + mutex_unlock(&priv->acpi_lock); return status; } @@ -1539,6 +1544,7 @@ static int i801_probe(struct pci_dev *dev, const struct pci_device_id *id) priv->adapter.dev.parent = &dev->dev; acpi_use_parent_companion(&priv->adapter.dev); priv->adapter.retries = 3; + mutex_init(&priv->acpi_lock); priv->pci_dev = dev; priv->features = id->driver_data; diff --git a/drivers/i2c/busses/i2c-imx-lpi2c.c b/drivers/i2c/busses/i2c-imx-lpi2c.c index d882126c1778c..519a1ac832a41 100644 --- a/drivers/i2c/busses/i2c-imx-lpi2c.c +++ b/drivers/i2c/busses/i2c-imx-lpi2c.c @@ -5,6 +5,7 @@ * Copyright 2016 Freescale Semiconductor, Inc. */ +#include #include #include #include @@ -90,6 +91,7 @@ #define MRDR_RXEMPTY BIT(14) #define MDER_TDDE BIT(0) #define MDER_RDDE BIT(1) +#define MSR_RDF_ASSERTED(x) FIELD_GET(MSR_RDF, (x)) #define SCR_SEN BIT(0) #define SCR_RST BIT(1) @@ -461,7 +463,7 @@ static bool lpi2c_imx_write_txfifo(struct lpi2c_imx_struct *lpi2c_imx, bool atom static bool lpi2c_imx_read_rxfifo(struct lpi2c_imx_struct *lpi2c_imx, bool atomic) { - unsigned int blocklen, remaining; + unsigned int remaining; unsigned int temp, data; do { @@ -472,15 +474,6 @@ static bool lpi2c_imx_read_rxfifo(struct lpi2c_imx_struct *lpi2c_imx, bool atomi lpi2c_imx->rx_buf[lpi2c_imx->delivered++] = data & 0xff; } while (1); - /* - * First byte is the length of remaining packet in the SMBus block - * data read. Add it to msgs->len. - */ - if (lpi2c_imx->block_data) { - blocklen = lpi2c_imx->rx_buf[0]; - lpi2c_imx->msglen += blocklen; - } - remaining = lpi2c_imx->msglen - lpi2c_imx->delivered; if (!remaining) { @@ -493,12 +486,7 @@ static bool lpi2c_imx_read_rxfifo(struct lpi2c_imx_struct *lpi2c_imx, bool atomi lpi2c_imx_set_rx_watermark(lpi2c_imx); /* multiple receive commands */ - if (lpi2c_imx->block_data) { - lpi2c_imx->block_data = 0; - temp = remaining; - temp |= (RECV_DATA << 8); - writel(temp, lpi2c_imx->base + LPI2C_MTDR); - } else if (!(lpi2c_imx->delivered & 0xff)) { + if (!(lpi2c_imx->delivered & 0xff)) { temp = (remaining > CHUNK_DATA ? CHUNK_DATA : remaining) - 1; temp |= (RECV_DATA << 8); writel(temp, lpi2c_imx->base + LPI2C_MTDR); @@ -536,18 +524,77 @@ static int lpi2c_imx_write_atomic(struct lpi2c_imx_struct *lpi2c_imx, return err; } -static void lpi2c_imx_read_init(struct lpi2c_imx_struct *lpi2c_imx, - struct i2c_msg *msgs) +static unsigned int lpi2c_SMBus_block_read_length_byte(struct lpi2c_imx_struct *lpi2c_imx) { - unsigned int temp; + unsigned int data; + + data = readl(lpi2c_imx->base + LPI2C_MRDR); + lpi2c_imx->rx_buf[lpi2c_imx->delivered++] = data & 0xff; + + return data; +} + +static int lpi2c_imx_read_init(struct lpi2c_imx_struct *lpi2c_imx, + struct i2c_msg *msgs) +{ + unsigned int temp, val, block_len; + int ret; lpi2c_imx->rx_buf = msgs->buf; lpi2c_imx->block_data = msgs->flags & I2C_M_RECV_LEN; lpi2c_imx_set_rx_watermark(lpi2c_imx); - temp = msgs->len > CHUNK_DATA ? CHUNK_DATA - 1 : msgs->len - 1; - temp |= (RECV_DATA << 8); - writel(temp, lpi2c_imx->base + LPI2C_MTDR); + + if (!lpi2c_imx->block_data) { + temp = msgs->len > CHUNK_DATA ? CHUNK_DATA - 1 : msgs->len - 1; + temp |= (RECV_DATA << 8); + writel(temp, lpi2c_imx->base + LPI2C_MTDR); + } else { + /* + * The LPI2C controller automatically sends a NACK after the last byte of a + * receive command, unless the next command in MTDR is also a receive command. + * If MTDR is empty when a receive completes, a NACK is sent by default. + * + * To comply with the SMBus block read spec, we start with a 2-byte read: + * The first byte in RXFIFO is the block length. Once this byte arrives, the + * controller immediately updates MTDR with the next read command, ensuring + * continuous ACK instead of NACK. + * + * The second byte is the first block data byte. Therefore, the subsequent + * read command should request (block_len - 1) bytes, since one data byte + * has already been read. + */ + + writel((RECV_DATA << 8) | 0x01, lpi2c_imx->base + LPI2C_MTDR); + + ret = readl_poll_timeout(lpi2c_imx->base + LPI2C_MSR, val, + MSR_RDF_ASSERTED(val), 1, 1000); + if (ret) { + dev_err(&lpi2c_imx->adapter.dev, "SMBus read count failed %d\n", ret); + return ret; + } + + /* Read block length byte and confirm this SMBus transfer meets protocol */ + block_len = lpi2c_SMBus_block_read_length_byte(lpi2c_imx); + if (block_len == 0 || block_len > I2C_SMBUS_BLOCK_MAX) { + dev_err(&lpi2c_imx->adapter.dev, "Invalid SMBus block read length\n"); + return -EPROTO; + } + + /* + * When block_len shows more bytes need to be read, update second read command to + * keep MTDR non-empty and ensuring continuous ACKs. Only update command register + * here. All block bytes will be read out at IRQ handler or lpi2c_imx_read_atomic() + * function. + */ + if (block_len > 1) + writel((RECV_DATA << 8) | (block_len - 2), lpi2c_imx->base + LPI2C_MTDR); + + lpi2c_imx->msglen += block_len; + msgs->len += block_len; + } + + return 0; } static bool lpi2c_imx_read_chunk_atomic(struct lpi2c_imx_struct *lpi2c_imx) @@ -592,6 +639,10 @@ static bool is_use_dma(struct lpi2c_imx_struct *lpi2c_imx, struct i2c_msg *msg) if (!lpi2c_imx->can_use_dma) return false; + /* DMA is not suitable for SMBus block read */ + if (msg->flags & I2C_M_RECV_LEN) + return false; + /* * A system-wide suspend or resume transition is in progress. LPI2C should use PIO to * transfer data to avoid issue caused by no ready DMA HW resource. @@ -609,10 +660,14 @@ static bool is_use_dma(struct lpi2c_imx_struct *lpi2c_imx, struct i2c_msg *msg) static int lpi2c_imx_pio_xfer(struct lpi2c_imx_struct *lpi2c_imx, struct i2c_msg *msg) { + int ret; + reinit_completion(&lpi2c_imx->complete); if (msg->flags & I2C_M_RD) { - lpi2c_imx_read_init(lpi2c_imx, msg); + ret = lpi2c_imx_read_init(lpi2c_imx, msg); + if (ret) + return ret; lpi2c_imx_intctrl(lpi2c_imx, MIER_RDIE | MIER_NDIE); } else { lpi2c_imx_write(lpi2c_imx, msg); @@ -624,8 +679,12 @@ static int lpi2c_imx_pio_xfer(struct lpi2c_imx_struct *lpi2c_imx, static int lpi2c_imx_pio_xfer_atomic(struct lpi2c_imx_struct *lpi2c_imx, struct i2c_msg *msg) { + int ret; + if (msg->flags & I2C_M_RD) { - lpi2c_imx_read_init(lpi2c_imx, msg); + ret = lpi2c_imx_read_init(lpi2c_imx, msg); + if (ret) + return ret; return lpi2c_imx_read_atomic(lpi2c_imx, msg); } diff --git a/drivers/i3c/master.c b/drivers/i3c/master.c index 66513a27e6e77..425e36b36009b 100644 --- a/drivers/i3c/master.c +++ b/drivers/i3c/master.c @@ -620,7 +620,8 @@ static int i3c_set_hotjoin(struct i3c_master_controller *master, bool enable) else ret = master->ops->disable_hotjoin(master); - master->hotjoin = enable; + if (!ret) + master->hotjoin = enable; i3c_bus_normaluse_unlock(&master->bus); @@ -2884,7 +2885,6 @@ int i3c_master_register(struct i3c_master_controller *master, INIT_LIST_HEAD(&master->boardinfo.i3c); device_initialize(&master->dev); - dev_set_name(&master->dev, "i3c-%d", i3cbus->id); master->dev.dma_mask = parent->dma_mask; master->dev.coherent_dma_mask = parent->coherent_dma_mask; @@ -2894,6 +2894,8 @@ int i3c_master_register(struct i3c_master_controller *master, if (ret) goto err_put_dev; + dev_set_name(&master->dev, "i3c-%d", i3cbus->id); + ret = of_populate_i3c_bus(master); if (ret) goto err_put_dev; diff --git a/drivers/i3c/master/dw-i3c-master.c b/drivers/i3c/master/dw-i3c-master.c index 9ceedf09c3b6a..c06595cb74010 100644 --- a/drivers/i3c/master/dw-i3c-master.c +++ b/drivers/i3c/master/dw-i3c-master.c @@ -1094,6 +1094,7 @@ static int dw_i3c_master_i2c_xfers(struct i2c_dev_desc *dev, dev_err(master->dev, "<%s> cannot resume i3c bus master, err: %d\n", __func__, ret); + dw_i3c_master_free_xfer(xfer); return ret; } @@ -1563,6 +1564,8 @@ int dw_i3c_common_probe(struct dw_i3c_master *master, spin_lock_init(&master->xferqueue.lock); INIT_LIST_HEAD(&master->xferqueue.list); + spin_lock_init(&master->devs_lock); + writel(INTR_ALL, master->regs + INTR_STATUS); irq = platform_get_irq(pdev, 0); ret = devm_request_irq(&pdev->dev, irq, diff --git a/drivers/i3c/master/mipi-i3c-hci/dma.c b/drivers/i3c/master/mipi-i3c-hci/dma.c index c401a9425cdc5..951abfea5a6fd 100644 --- a/drivers/i3c/master/mipi-i3c-hci/dma.c +++ b/drivers/i3c/master/mipi-i3c-hci/dma.c @@ -342,6 +342,14 @@ static int hci_dma_init(struct i3c_hci *hci) rh_reg_write(INTR_SIGNAL_ENABLE, regval); ring_ready: + /* + * The MIPI I3C HCI specification does not document reset values for + * RING_OPERATION1 fields and some controllers (e.g. Intel controllers) + * do not reset the values, so ensure the ring pointers are set to zero + * here. + */ + rh_reg_write(RING_OPERATION1, 0); + rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE | RING_CTRL_RUN_STOP); } diff --git a/drivers/i3c/master/mipi-i3c-hci/ext_caps.c b/drivers/i3c/master/mipi-i3c-hci/ext_caps.c index 7714f00ea9cc0..533a495e14c86 100644 --- a/drivers/i3c/master/mipi-i3c-hci/ext_caps.c +++ b/drivers/i3c/master/mipi-i3c-hci/ext_caps.c @@ -272,7 +272,7 @@ int i3c_hci_parse_ext_caps(struct i3c_hci *hci) cap_length = FIELD_GET(CAP_HEADER_LENGTH, cap_header); dev_dbg(&hci->master.dev, "id=0x%02x length=%d", cap_id, cap_length); - if (!cap_length) + if (!cap_id || !cap_length) break; if (curr_cap + cap_length * 4 >= end) { dev_err(&hci->master.dev, diff --git a/drivers/i3c/master/svc-i3c-master.c b/drivers/i3c/master/svc-i3c-master.c index e70a64f2a32fa..93531cb216d16 100644 --- a/drivers/i3c/master/svc-i3c-master.c +++ b/drivers/i3c/master/svc-i3c-master.c @@ -496,8 +496,8 @@ static int svc_i3c_master_handle_ibi_won(struct svc_i3c_master *master, u32 msta static void svc_i3c_master_ibi_isr(struct svc_i3c_master *master) { struct svc_i3c_i2c_dev_data *data; + struct i3c_dev_desc *dev = NULL; unsigned int ibitype, ibiaddr; - struct i3c_dev_desc *dev; u32 status, val; int ret; @@ -590,7 +590,7 @@ static void svc_i3c_master_ibi_isr(struct svc_i3c_master *master) * for the slave to interrupt again. */ if (svc_i3c_master_error(master)) { - if (master->ibi.tbq_slot) { + if (master->ibi.tbq_slot && dev) { data = i3c_dev_get_master_data(dev); i3c_generic_ibi_recycle_slot(data->ibi_pool, master->ibi.tbq_slot); diff --git a/drivers/iio/accel/adxl380.c b/drivers/iio/accel/adxl380.c index 6d82873357cb8..217c5ae56d23a 100644 --- a/drivers/iio/accel/adxl380.c +++ b/drivers/iio/accel/adxl380.c @@ -977,6 +977,7 @@ static irqreturn_t adxl380_irq_handler(int irq, void *p) if (ret) return IRQ_HANDLED; + fifo_entries = rounddown(fifo_entries, st->fifo_set_size); for (i = 0; i < fifo_entries; i += st->fifo_set_size) { ret = regmap_noinc_read(st->regmap, ADXL380_FIFO_DATA, &st->fifo_buf[i], diff --git a/drivers/iio/accel/bma180.c b/drivers/iio/accel/bma180.c index 8925f5279e627..7bc6761f51354 100644 --- a/drivers/iio/accel/bma180.c +++ b/drivers/iio/accel/bma180.c @@ -986,8 +986,9 @@ static int bma180_probe(struct i2c_client *client) } ret = devm_request_irq(dev, client->irq, - iio_trigger_generic_data_rdy_poll, IRQF_TRIGGER_RISING, - "bma180_event", data->trig); + iio_trigger_generic_data_rdy_poll, + IRQF_TRIGGER_RISING | IRQF_NO_THREAD, + "bma180_event", data->trig); if (ret) { dev_err(dev, "unable to request IRQ\n"); goto err_trigger_free; diff --git a/drivers/iio/accel/sca3000.c b/drivers/iio/accel/sca3000.c index bfa8a3f5a92f4..9ef4d6e274669 100644 --- a/drivers/iio/accel/sca3000.c +++ b/drivers/iio/accel/sca3000.c @@ -1489,7 +1489,11 @@ static int sca3000_probe(struct spi_device *spi) if (ret) goto error_free_irq; - return iio_device_register(indio_dev); + ret = iio_device_register(indio_dev); + if (ret) + goto error_free_irq; + + return 0; error_free_irq: if (spi->irq) diff --git a/drivers/iio/adc/ad7766.c b/drivers/iio/adc/ad7766.c index 4d570383ef025..1e6bfe8765ab3 100644 --- a/drivers/iio/adc/ad7766.c +++ b/drivers/iio/adc/ad7766.c @@ -261,7 +261,7 @@ static int ad7766_probe(struct spi_device *spi) * don't enable the interrupt to avoid extra load on the system */ ret = devm_request_irq(&spi->dev, spi->irq, ad7766_irq, - IRQF_TRIGGER_FALLING | IRQF_NO_AUTOEN, + IRQF_TRIGGER_FALLING | IRQF_NO_AUTOEN | IRQF_NO_THREAD, dev_name(&spi->dev), ad7766->trig); if (ret < 0) diff --git a/drivers/iio/gyro/itg3200_buffer.c b/drivers/iio/gyro/itg3200_buffer.c index a624400a239cb..cf97adfa97274 100644 --- a/drivers/iio/gyro/itg3200_buffer.c +++ b/drivers/iio/gyro/itg3200_buffer.c @@ -118,11 +118,9 @@ int itg3200_probe_trigger(struct iio_dev *indio_dev) if (!st->trig) return -ENOMEM; - ret = request_irq(st->i2c->irq, - &iio_trigger_generic_data_rdy_poll, - IRQF_TRIGGER_RISING, - "itg3200_data_rdy", - st->trig); + ret = request_irq(st->i2c->irq, &iio_trigger_generic_data_rdy_poll, + IRQF_TRIGGER_RISING | IRQF_NO_THREAD, + "itg3200_data_rdy", st->trig); if (ret) goto error_free_trig; diff --git a/drivers/iio/gyro/itg3200_core.c b/drivers/iio/gyro/itg3200_core.c index cd8a2dae56cd9..bfe95ec1abda9 100644 --- a/drivers/iio/gyro/itg3200_core.c +++ b/drivers/iio/gyro/itg3200_core.c @@ -93,6 +93,8 @@ static int itg3200_read_raw(struct iio_dev *indio_dev, case IIO_CHAN_INFO_RAW: reg = (u8)chan->address; ret = itg3200_read_reg_s16(indio_dev, reg, val); + if (ret) + return ret; return IIO_VAL_INT; case IIO_CHAN_INFO_SCALE: *val = 0; diff --git a/drivers/iio/gyro/mpu3050-core.c b/drivers/iio/gyro/mpu3050-core.c index 67ae7d1012bc2..ee2fcd20545de 100644 --- a/drivers/iio/gyro/mpu3050-core.c +++ b/drivers/iio/gyro/mpu3050-core.c @@ -1162,10 +1162,8 @@ int mpu3050_common_probe(struct device *dev, mpu3050->regs[1].supply = mpu3050_reg_vlogic; ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(mpu3050->regs), mpu3050->regs); - if (ret) { - dev_err(dev, "Cannot get regulators\n"); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "Cannot get regulators\n"); ret = mpu3050_power_up(mpu3050); if (ret) diff --git a/drivers/iio/imu/bmi270/bmi270_i2c.c b/drivers/iio/imu/bmi270/bmi270_i2c.c index b909a421ad017..b92da4e0776fa 100644 --- a/drivers/iio/imu/bmi270/bmi270_i2c.c +++ b/drivers/iio/imu/bmi270/bmi270_i2c.c @@ -37,6 +37,7 @@ static const struct i2c_device_id bmi270_i2c_id[] = { { "bmi270", (kernel_ulong_t)&bmi270_chip_info }, { } }; +MODULE_DEVICE_TABLE(i2c, bmi270_i2c_id); static const struct acpi_device_id bmi270_acpi_match[] = { /* GPD Win Mini, Aya Neo AIR Pro, OXP Mini Pro, etc. */ @@ -45,12 +46,14 @@ static const struct acpi_device_id bmi270_acpi_match[] = { { "BMI0260", (kernel_ulong_t)&bmi260_chip_info }, { } }; +MODULE_DEVICE_TABLE(acpi, bmi270_acpi_match); static const struct of_device_id bmi270_of_match[] = { { .compatible = "bosch,bmi260", .data = &bmi260_chip_info }, { .compatible = "bosch,bmi270", .data = &bmi270_chip_info }, { } }; +MODULE_DEVICE_TABLE(of, bmi270_of_match); static struct i2c_driver bmi270_i2c_driver = { .driver = { diff --git a/drivers/iio/light/si1145.c b/drivers/iio/light/si1145.c index f8eb251eca8dc..ef0abc4499b74 100644 --- a/drivers/iio/light/si1145.c +++ b/drivers/iio/light/si1145.c @@ -1248,7 +1248,7 @@ static int si1145_probe_trigger(struct iio_dev *indio_dev) ret = devm_request_irq(&client->dev, client->irq, iio_trigger_generic_data_rdy_poll, - IRQF_TRIGGER_FALLING, + IRQF_TRIGGER_FALLING | IRQF_NO_THREAD, "si1145_irq", trig); if (ret < 0) { diff --git a/drivers/iio/magnetometer/ak8975.c b/drivers/iio/magnetometer/ak8975.c index 3fd0171e5d69b..d30315ad85ded 100644 --- a/drivers/iio/magnetometer/ak8975.c +++ b/drivers/iio/magnetometer/ak8975.c @@ -581,7 +581,7 @@ static int ak8975_setup_irq(struct ak8975_data *data) irq = gpiod_to_irq(data->eoc_gpiod); rc = devm_request_irq(&client->dev, irq, ak8975_irq_handler, - IRQF_TRIGGER_RISING | IRQF_ONESHOT, + IRQF_TRIGGER_RISING, dev_name(&client->dev), data); if (rc < 0) { dev_err(&client->dev, "irq %d request failed: %d\n", irq, rc); diff --git a/drivers/iio/pressure/mprls0025pa.c b/drivers/iio/pressure/mprls0025pa.c index 2336f2760eaeb..d4133fef91fac 100644 --- a/drivers/iio/pressure/mprls0025pa.c +++ b/drivers/iio/pressure/mprls0025pa.c @@ -59,7 +59,7 @@ * * Values given to the userspace in sysfs interface: * * raw - press_cnt - * * offset - (-1 * outputmin) - pmin / scale + * * offset - (-1 * outputmin) + pmin / scale * note: With all sensors from the datasheet pmin = 0 * which reduces the offset to (-1 * outputmin) */ @@ -160,8 +160,8 @@ static const struct iio_chan_spec mpr_channels[] = { BIT(IIO_CHAN_INFO_OFFSET), .scan_index = 0, .scan_type = { - .sign = 's', - .realbits = 32, + .sign = 'u', + .realbits = 24, .storagebits = 32, .endianness = IIO_CPU, }, @@ -313,8 +313,7 @@ static int mpr_read_raw(struct iio_dev *indio_dev, return IIO_VAL_INT_PLUS_NANO; case IIO_CHAN_INFO_OFFSET: *val = data->offset; - *val2 = data->offset2; - return IIO_VAL_INT_PLUS_NANO; + return IIO_VAL_INT; default: return -EINVAL; } @@ -330,8 +329,9 @@ int mpr_common_probe(struct device *dev, const struct mpr_ops *ops, int irq) struct mpr_data *data; struct iio_dev *indio_dev; const char *triplet; - s64 scale, offset; + s64 odelta, pdelta; u32 func; + s32 tmp; indio_dev = devm_iio_device_alloc(dev, sizeof(*data)); if (!indio_dev) @@ -405,23 +405,17 @@ int mpr_common_probe(struct device *dev, const struct mpr_ops *ops, int irq) data->outmin = mpr_func_spec[data->function].output_min; data->outmax = mpr_func_spec[data->function].output_max; - /* use 64 bit calculation for preserving a reasonable precision */ - scale = div_s64(((s64)(data->pmax - data->pmin)) * NANO, - data->outmax - data->outmin); - data->scale = div_s64_rem(scale, NANO, &data->scale2); - /* - * multiply with NANO before dividing by scale and later divide by NANO - * again. - */ - offset = ((-1LL) * (s64)data->outmin) * NANO - - div_s64(div_s64((s64)data->pmin * NANO, scale), NANO); - data->offset = div_s64_rem(offset, NANO, &data->offset2); + odelta = data->outmax - data->outmin; + pdelta = data->pmax - data->pmin; + + data->scale = div_s64_rem(div_s64(pdelta * NANO, odelta), NANO, &tmp); + data->scale2 = tmp; + + data->offset = div_s64(odelta * data->pmin, pdelta) - data->outmin; if (data->irq > 0) { - ret = devm_request_irq(dev, data->irq, mpr_eoc_handler, - IRQF_TRIGGER_RISING, - dev_name(dev), - data); + ret = devm_request_irq(dev, data->irq, mpr_eoc_handler, 0, + dev_name(dev), data); if (ret) return dev_err_probe(dev, ret, "request irq %d failed\n", data->irq); diff --git a/drivers/iio/pressure/mprls0025pa.h b/drivers/iio/pressure/mprls0025pa.h index d62a018eaff32..b6944b3051267 100644 --- a/drivers/iio/pressure/mprls0025pa.h +++ b/drivers/iio/pressure/mprls0025pa.h @@ -53,7 +53,6 @@ enum mpr_func_id { * @scale: pressure scale * @scale2: pressure scale, decimal number * @offset: pressure offset - * @offset2: pressure offset, decimal number * @gpiod_reset: reset * @irq: end of conversion irq. used to distinguish between irq mode and * reading in a loop until data is ready @@ -75,7 +74,6 @@ struct mpr_data { int scale; int scale2; int offset; - int offset2; struct gpio_desc *gpiod_reset; int irq; struct completion completion; diff --git a/drivers/iio/pressure/mprls0025pa_spi.c b/drivers/iio/pressure/mprls0025pa_spi.c index d04102f8a4a03..cf17eb2e72083 100644 --- a/drivers/iio/pressure/mprls0025pa_spi.c +++ b/drivers/iio/pressure/mprls0025pa_spi.c @@ -8,6 +8,7 @@ * https://prod-edam.honeywell.com/content/dam/honeywell-edam/sps/siot/en-us/products/sensors/pressure-sensors/board-mount-pressure-sensors/micropressure-mpr-series/documents/sps-siot-mpr-series-datasheet-32332628-ciid-172626.pdf */ +#include #include #include #include @@ -40,17 +41,25 @@ static int mpr_spi_xfer(struct mpr_data *data, const u8 cmd, const u8 pkt_len) { struct spi_device *spi = to_spi_device(data->dev); struct mpr_spi_buf *buf = spi_get_drvdata(spi); - struct spi_transfer xfer; + struct spi_transfer xfers[2] = { }; if (pkt_len > MPR_MEASUREMENT_RD_SIZE) return -EOVERFLOW; buf->tx[0] = cmd; - xfer.tx_buf = buf->tx; - xfer.rx_buf = data->buffer; - xfer.len = pkt_len; - return spi_sync_transfer(spi, &xfer, 1); + /* + * Dummy transfer with no data, just cause a 2.5us+ delay between the CS assert + * and the first clock edge as per the datasheet tHDSS timing requirement. + */ + xfers[0].delay.value = 2500; + xfers[0].delay.unit = SPI_DELAY_UNIT_NSECS; + + xfers[1].tx_buf = buf->tx; + xfers[1].rx_buf = data->buffer; + xfers[1].len = pkt_len; + + return spi_sync_transfer(spi, xfers, ARRAY_SIZE(xfers)); } static const struct mpr_ops mpr_spi_ops = { diff --git a/drivers/iio/test/Kconfig b/drivers/iio/test/Kconfig index 6e65e929791ca..4fc17dd0dcd77 100644 --- a/drivers/iio/test/Kconfig +++ b/drivers/iio/test/Kconfig @@ -8,7 +8,6 @@ config IIO_GTS_KUNIT_TEST tristate "Test IIO gain-time-scale helpers" if !KUNIT_ALL_TESTS depends on KUNIT select IIO_GTS_HELPER - select TEST_KUNIT_DEVICE_HELPERS default KUNIT_ALL_TESTS help build unit tests for the IIO light sensor gain-time-scale helpers. diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c index 81cf3c902e819..78bc7d83edc65 100644 --- a/drivers/infiniband/core/cache.c +++ b/drivers/infiniband/core/cache.c @@ -927,6 +927,13 @@ static int gid_table_setup_one(struct ib_device *ib_dev) if (err) return err; + /* + * Mark the device as ready for GID cache updates. This allows netdev + * event handlers to update the GID cache even before the device is + * fully registered. + */ + ib_device_enable_gid_updates(ib_dev); + rdma_roce_rescan_device(ib_dev); return err; @@ -1537,7 +1544,8 @@ static void ib_cache_event_task(struct work_struct *_work) * the cache. */ ret = ib_cache_update(work->event.device, work->event.element.port_num, - work->event.event == IB_EVENT_GID_CHANGE, + work->event.event == IB_EVENT_GID_CHANGE || + work->event.event == IB_EVENT_CLIENT_REREGISTER, work->event.event == IB_EVENT_PKEY_CHANGE, work->enforce_security); @@ -1638,6 +1646,12 @@ void ib_cache_release_one(struct ib_device *device) void ib_cache_cleanup_one(struct ib_device *device) { + /* + * Clear the GID updates mark first to prevent event handlers from + * accessing the device while it's being torn down. + */ + ib_device_disable_gid_updates(device); + /* The cleanup function waits for all in-progress workqueue * elements and cleans up the GID cache. This function should be * called after the device was removed from the devices list and diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h index 05102769a918a..a2c36666e6fcb 100644 --- a/drivers/infiniband/core/core_priv.h +++ b/drivers/infiniband/core/core_priv.h @@ -100,6 +100,9 @@ void ib_enum_all_roce_netdevs(roce_netdev_filter filter, roce_netdev_callback cb, void *cookie); +void ib_device_enable_gid_updates(struct ib_device *device); +void ib_device_disable_gid_updates(struct ib_device *device); + typedef int (*nldev_callback)(struct ib_device *device, struct sk_buff *skb, struct netlink_callback *cb, diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index e3ba236d7c09f..ac9aaef1e5e61 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -93,6 +93,7 @@ static struct workqueue_struct *ib_unreg_wq; static DEFINE_XARRAY_FLAGS(devices, XA_FLAGS_ALLOC); static DECLARE_RWSEM(devices_rwsem); #define DEVICE_REGISTERED XA_MARK_1 +#define DEVICE_GID_UPDATES XA_MARK_2 static u32 highest_client_id; #define CLIENT_REGISTERED XA_MARK_1 @@ -2441,11 +2442,42 @@ void ib_enum_all_roce_netdevs(roce_netdev_filter filter, unsigned long index; down_read(&devices_rwsem); - xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) + xa_for_each_marked(&devices, index, dev, DEVICE_GID_UPDATES) ib_enum_roce_netdev(dev, filter, filter_cookie, cb, cookie); up_read(&devices_rwsem); } +/** + * ib_device_enable_gid_updates - Mark device as ready for GID cache updates + * @device: Device to mark + * + * Called after GID table is allocated and initialized. After this mark is set, + * netdevice event handlers can update the device's GID cache. This allows + * events that arrive during device registration to be processed, avoiding + * stale GID entries when netdev properties change during the device + * registration process. + */ +void ib_device_enable_gid_updates(struct ib_device *device) +{ + down_write(&devices_rwsem); + xa_set_mark(&devices, device->index, DEVICE_GID_UPDATES); + up_write(&devices_rwsem); +} + +/** + * ib_device_disable_gid_updates - Clear the GID updates mark + * @device: Device to unmark + * + * Called before GID table cleanup to prevent event handlers from accessing + * the device while it's being torn down. + */ +void ib_device_disable_gid_updates(struct ib_device *device) +{ + down_write(&devices_rwsem); + xa_clear_mark(&devices, device->index, DEVICE_GID_UPDATES); + up_write(&devices_rwsem); +} + /* * ib_enum_all_devs - enumerate all ib_devices * @cb: Callback to call for each found ib_device diff --git a/drivers/infiniband/core/iwcm.c b/drivers/infiniband/core/iwcm.c index 62410578dec37..eb942ab9c4055 100644 --- a/drivers/infiniband/core/iwcm.c +++ b/drivers/infiniband/core/iwcm.c @@ -95,7 +95,6 @@ static struct workqueue_struct *iwcm_wq; struct iwcm_work { struct work_struct work; struct iwcm_id_private *cm_id; - struct list_head list; struct iw_cm_event event; struct list_head free_list; }; @@ -178,7 +177,6 @@ static int alloc_work_entries(struct iwcm_id_private *cm_id_priv, int count) return -ENOMEM; } work->cm_id = cm_id_priv; - INIT_LIST_HEAD(&work->list); put_work(work); } return 0; @@ -213,7 +211,6 @@ static void free_cm_id(struct iwcm_id_private *cm_id_priv) static bool iwcm_deref_id(struct iwcm_id_private *cm_id_priv) { if (refcount_dec_and_test(&cm_id_priv->refcount)) { - BUG_ON(!list_empty(&cm_id_priv->work_list)); free_cm_id(cm_id_priv); return true; } @@ -260,7 +257,6 @@ struct iw_cm_id *iw_create_cm_id(struct ib_device *device, refcount_set(&cm_id_priv->refcount, 1); init_waitqueue_head(&cm_id_priv->connect_wait); init_completion(&cm_id_priv->destroy_comp); - INIT_LIST_HEAD(&cm_id_priv->work_list); INIT_LIST_HEAD(&cm_id_priv->work_free_list); return &cm_id_priv->id; @@ -1007,13 +1003,13 @@ static int process_event(struct iwcm_id_private *cm_id_priv, } /* - * Process events on the work_list for the cm_id. If the callback - * function requests that the cm_id be deleted, a flag is set in the - * cm_id flags to indicate that when the last reference is - * removed, the cm_id is to be destroyed. This is necessary to - * distinguish between an object that will be destroyed by the app - * thread asleep on the destroy_comp list vs. an object destroyed - * here synchronously when the last reference is removed. + * Process events for the cm_id. If the callback function requests + * that the cm_id be deleted, a flag is set in the cm_id flags to + * indicate that when the last reference is removed, the cm_id is + * to be destroyed. This is necessary to distinguish between an + * object that will be destroyed by the app thread asleep on the + * destroy_comp list vs. an object destroyed here synchronously + * when the last reference is removed. */ static void cm_work_handler(struct work_struct *_work) { @@ -1024,35 +1020,26 @@ static void cm_work_handler(struct work_struct *_work) int ret = 0; spin_lock_irqsave(&cm_id_priv->lock, flags); - while (!list_empty(&cm_id_priv->work_list)) { - work = list_first_entry(&cm_id_priv->work_list, - struct iwcm_work, list); - list_del_init(&work->list); - levent = work->event; - put_work(work); - spin_unlock_irqrestore(&cm_id_priv->lock, flags); - - if (!test_bit(IWCM_F_DROP_EVENTS, &cm_id_priv->flags)) { - ret = process_event(cm_id_priv, &levent); - if (ret) { - destroy_cm_id(&cm_id_priv->id); - WARN_ON_ONCE(iwcm_deref_id(cm_id_priv)); - } - } else - pr_debug("dropping event %d\n", levent.event); - if (iwcm_deref_id(cm_id_priv)) - return; - spin_lock_irqsave(&cm_id_priv->lock, flags); - } + levent = work->event; + put_work(work); spin_unlock_irqrestore(&cm_id_priv->lock, flags); + + if (!test_bit(IWCM_F_DROP_EVENTS, &cm_id_priv->flags)) { + ret = process_event(cm_id_priv, &levent); + if (ret) { + destroy_cm_id(&cm_id_priv->id); + WARN_ON_ONCE(iwcm_deref_id(cm_id_priv)); + } + } else + pr_debug("dropping event %d\n", levent.event); + if (iwcm_deref_id(cm_id_priv)) + return; } /* * This function is called on interrupt context. Schedule events on * the iwcm_wq thread to allow callback functions to downcall into - * the CM and/or block. Events are queued to a per-CM_ID - * work_list. If this is the first event on the work_list, the work - * element is also queued on the iwcm_wq thread. + * the CM and/or block. * * Each event holds a reference on the cm_id. Until the last posted * event has been delivered and processed, the cm_id cannot be @@ -1094,7 +1081,6 @@ static int cm_event_handler(struct iw_cm_id *cm_id, } refcount_inc(&cm_id_priv->refcount); - list_add_tail(&work->list, &cm_id_priv->work_list); queue_work(iwcm_wq, &work->work); out: spin_unlock_irqrestore(&cm_id_priv->lock, flags); diff --git a/drivers/infiniband/core/iwcm.h b/drivers/infiniband/core/iwcm.h index bf74639be1287..b56fb12edece4 100644 --- a/drivers/infiniband/core/iwcm.h +++ b/drivers/infiniband/core/iwcm.h @@ -50,7 +50,6 @@ struct iwcm_id_private { struct ib_qp *qp; struct completion destroy_comp; wait_queue_head_t connect_wait; - struct list_head work_list; spinlock_t lock; refcount_t refcount; struct list_head work_free_list; diff --git a/drivers/infiniband/core/rw.c b/drivers/infiniband/core/rw.c index 6354ddf2a274c..2522ff1cc462c 100644 --- a/drivers/infiniband/core/rw.c +++ b/drivers/infiniband/core/rw.c @@ -651,34 +651,57 @@ unsigned int rdma_rw_mr_factor(struct ib_device *device, u32 port_num, } EXPORT_SYMBOL(rdma_rw_mr_factor); +/** + * rdma_rw_max_send_wr - compute max Send WRs needed for RDMA R/W contexts + * @dev: RDMA device + * @port_num: port number + * @max_rdma_ctxs: number of rdma_rw_ctx structures + * @create_flags: QP create flags (pass IB_QP_CREATE_INTEGRITY_EN if + * data integrity will be enabled on the QP) + * + * Returns the total number of Send Queue entries needed for + * @max_rdma_ctxs. The result accounts for memory registration and + * invalidation work requests when the device requires them. + * + * ULPs use this to size Send Queues and Send CQs before creating a + * Queue Pair. + */ +unsigned int rdma_rw_max_send_wr(struct ib_device *dev, u32 port_num, + unsigned int max_rdma_ctxs, u32 create_flags) +{ + unsigned int factor = 1; + unsigned int result; + + if (create_flags & IB_QP_CREATE_INTEGRITY_EN || + rdma_rw_can_use_mr(dev, port_num)) + factor += 2; /* reg + inv */ + + if (check_mul_overflow(factor, max_rdma_ctxs, &result)) + return UINT_MAX; + return result; +} +EXPORT_SYMBOL(rdma_rw_max_send_wr); + void rdma_rw_init_qp(struct ib_device *dev, struct ib_qp_init_attr *attr) { - u32 factor; + unsigned int factor = 1; WARN_ON_ONCE(attr->port_num == 0); /* - * Each context needs at least one RDMA READ or WRITE WR. - * - * For some hardware we might need more, eventually we should ask the - * HCA driver for a multiplier here. - */ - factor = 1; - - /* - * If the device needs MRs to perform RDMA READ or WRITE operations, - * we'll need two additional MRs for the registrations and the - * invalidation. + * If the device uses MRs to perform RDMA READ or WRITE operations, + * or if data integrity is enabled, account for registration and + * invalidation work requests. */ if (attr->create_flags & IB_QP_CREATE_INTEGRITY_EN || rdma_rw_can_use_mr(dev, attr->port_num)) - factor += 2; /* inv + reg */ + factor += 2; /* reg + inv */ attr->cap.max_send_wr += factor * attr->cap.max_rdma_ctxs; /* - * But maybe we were just too high in the sky and the device doesn't - * even support all we need, and we'll have to live with what we get.. + * The device might not support all we need, and we'll have to + * live with what we get. */ attr->cap.max_send_wr = min_t(u32, attr->cap.max_send_wr, dev->attrs.max_qp_wr); diff --git a/drivers/infiniband/core/umem_dmabuf.c b/drivers/infiniband/core/umem_dmabuf.c index 0ec2e4120cc94..17b16fe0e49d9 100644 --- a/drivers/infiniband/core/umem_dmabuf.c +++ b/drivers/infiniband/core/umem_dmabuf.c @@ -221,13 +221,11 @@ ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device, err = ib_umem_dmabuf_map_pages(umem_dmabuf); if (err) - goto err_unpin; + goto err_release; dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); return umem_dmabuf; -err_unpin: - dma_buf_unpin(umem_dmabuf->attach); err_release: dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); ib_umem_release(&umem_dmabuf->umem); diff --git a/drivers/infiniband/core/user_mad.c b/drivers/infiniband/core/user_mad.c index fd67fc9fe85a4..2f7e3c4483fc5 100644 --- a/drivers/infiniband/core/user_mad.c +++ b/drivers/infiniband/core/user_mad.c @@ -514,7 +514,8 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf, struct rdma_ah_attr ah_attr; struct ib_ah *ah; __be64 *tid; - int ret, data_len, hdr_len, copy_offset, rmpp_active; + int ret, hdr_len, copy_offset, rmpp_active; + size_t data_len; u8 base_version; if (count < hdr_size(file) + IB_MGMT_RMPP_HDR) @@ -588,7 +589,10 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf, } base_version = ((struct ib_mad_hdr *)&packet->mad.data)->base_version; - data_len = count - hdr_size(file) - hdr_len; + if (check_sub_overflow(count, hdr_size(file) + hdr_len, &data_len)) { + ret = -EINVAL; + goto err_ah; + } packet->msg = ib_create_send_mad(agent, be32_to_cpu(packet->mad.hdr.qpn), packet->mad.hdr.pkey_index, rmpp_active, diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index ce16404cdfb8c..f4616deeca545 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -2049,7 +2049,10 @@ static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs) if (ret) return ret; - user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL); + if (cmd.wqe_size < sizeof(struct ib_uverbs_send_wr)) + return -EINVAL; + + user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL | __GFP_NOWARN); if (!user_wr) return -ENOMEM; @@ -2239,7 +2242,7 @@ ib_uverbs_unmarshall_recv(struct uverbs_req_iter *iter, u32 wr_count, if (ret) return ERR_PTR(ret); - user_wr = kmalloc(wqe_size, GFP_KERNEL); + user_wr = kmalloc(wqe_size, GFP_KERNEL | __GFP_NOWARN); if (!user_wr) return ERR_PTR(-ENOMEM); diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 755bba8d58bbc..5cab7dd70aebf 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -1663,7 +1663,7 @@ static struct efa_mr *efa_alloc_mr(struct ib_pd *ibpd, int access_flags, struct efa_mr *mr; if (udata && udata->inlen && - !ib_is_udata_cleared(udata, 0, sizeof(udata->inlen))) { + !ib_is_udata_cleared(udata, 0, udata->inlen)) { ibdev_dbg(&dev->ibdev, "Incompatible ABI params, udata not cleared\n"); return ERR_PTR(-EINVAL); diff --git a/drivers/infiniband/hw/hns/hns_roce_ah.c b/drivers/infiniband/hw/hns/hns_roce_ah.c index 307c35888b300..3b6c6a6e9f977 100644 --- a/drivers/infiniband/hw/hns/hns_roce_ah.c +++ b/drivers/infiniband/hw/hns/hns_roce_ah.c @@ -61,7 +61,7 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, u8 tclass = get_tclass(grh); u8 priority = 0; u8 tc_mode = 0; - int ret; + int ret = 0; if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 && udata) { ret = -EOPNOTSUPP; @@ -78,19 +78,18 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, ah->av.flowlabel = grh->flow_label; ah->av.udp_sport = get_ah_udp_sport(ah_attr); ah->av.tclass = tclass; + ah->av.sl = rdma_ah_get_sl(ah_attr); - ret = hr_dev->hw->get_dscp(hr_dev, tclass, &tc_mode, &priority); - if (ret == -EOPNOTSUPP) - ret = 0; - - if (ret && grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) - goto err_out; + if (grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { + ret = hr_dev->hw->get_dscp(hr_dev, tclass, &tc_mode, &priority); + if (ret == -EOPNOTSUPP) + ret = 0; + else if (ret) + goto err_out; - if (tc_mode == HNAE3_TC_MAP_MODE_DSCP && - grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) - ah->av.sl = priority; - else - ah->av.sl = rdma_ah_get_sl(ah_attr); + if (tc_mode == HNAE3_TC_MAP_MODE_DSCP) + ah->av.sl = priority; + } if (!check_sl_valid(hr_dev, ah->av.sl)) { ret = -EINVAL; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 63052c0e76133..f895731ad74a0 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -3661,6 +3661,23 @@ static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev, HNS_ROCE_V2_CQ_DEFAULT_INTERVAL); } +static bool left_sw_wc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) +{ + struct hns_roce_qp *hr_qp; + + list_for_each_entry(hr_qp, &hr_cq->sq_list, sq_node) { + if (hr_qp->sq.head != hr_qp->sq.tail) + return true; + } + + list_for_each_entry(hr_qp, &hr_cq->rq_list, rq_node) { + if (hr_qp->rq.head != hr_qp->rq.tail) + return true; + } + + return false; +} + static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) { @@ -3669,6 +3686,12 @@ static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq, struct hns_roce_v2_db cq_db = {}; u32 notify_flag; + if (hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN) { + if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && + left_sw_wc(hr_dev, hr_cq)) + return 1; + return 0; + } /* * flags = 0, then notify_flag : next * flags = 1, then notify flag : solocited @@ -4975,20 +4998,22 @@ static int hns_roce_set_sl(struct ib_qp *ibqp, struct ib_device *ibdev = &hr_dev->ib_dev; int ret; - ret = hns_roce_hw_v2_get_dscp(hr_dev, get_tclass(&attr->ah_attr.grh), - &hr_qp->tc_mode, &hr_qp->priority); - if (ret && ret != -EOPNOTSUPP && - grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { - ibdev_err_ratelimited(ibdev, - "failed to get dscp, ret = %d.\n", ret); - return ret; - } + hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); - if (hr_qp->tc_mode == HNAE3_TC_MAP_MODE_DSCP && - grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) - hr_qp->sl = hr_qp->priority; - else - hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); + if (grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { + ret = hns_roce_hw_v2_get_dscp(hr_dev, + get_tclass(&attr->ah_attr.grh), + &hr_qp->tc_mode, &hr_qp->priority); + if (ret && ret != -EOPNOTSUPP) { + ibdev_err_ratelimited(ibdev, + "failed to get dscp, ret = %d.\n", + ret); + return ret; + } + + if (hr_qp->tc_mode == HNAE3_TC_MAP_MODE_DSCP) + hr_qp->sl = hr_qp->priority; + } if (!check_sl_valid(hr_dev, hr_qp->sl)) return -EINVAL; @@ -6878,7 +6903,8 @@ static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev) INIT_WORK(&hr_dev->ecc_work, fmea_ram_ecc_work); - hr_dev->irq_workq = alloc_ordered_workqueue("hns_roce_irq_workq", 0); + hr_dev->irq_workq = alloc_ordered_workqueue("hns_roce_irq_workq", + WQ_MEM_RECLAIM); if (!hr_dev->irq_workq) { dev_err(dev, "failed to create irq workqueue.\n"); ret = -ENOMEM; diff --git a/drivers/infiniband/hw/ionic/ionic_controlpath.c b/drivers/infiniband/hw/ionic/ionic_controlpath.c index ea12d9b8e125f..83573721af2c0 100644 --- a/drivers/infiniband/hw/ionic/ionic_controlpath.c +++ b/drivers/infiniband/hw/ionic/ionic_controlpath.c @@ -1218,7 +1218,7 @@ int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); struct ionic_vcq *vcq = to_ionic_vcq(ibcq); struct ionic_tbl_buf buf = {}; - struct ionic_cq_resp resp; + struct ionic_cq_resp resp = {}; struct ionic_cq_req req; int udma_idx = 0, rc; diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.c b/drivers/infiniband/hw/ionic/ionic_ibdev.c index 164046d00e5d4..bd4c73e530d08 100644 --- a/drivers/infiniband/hw/ionic/ionic_ibdev.c +++ b/drivers/infiniband/hw/ionic/ionic_ibdev.c @@ -81,6 +81,8 @@ static int ionic_query_port(struct ib_device *ibdev, u32 port, return -EINVAL; ndev = ib_device_get_netdev(ibdev, port); + if (!ndev) + return -ENODEV; if (netif_running(ndev) && netif_carrier_ok(ndev)) { attr->state = IB_PORT_ACTIVE; diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index b8655504da290..5d027c04dba6f 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -5202,7 +5202,7 @@ static int irdma_create_user_ah(struct ib_ah *ibah, #define IRDMA_CREATE_AH_MIN_RESP_LEN offsetofend(struct irdma_create_ah_resp, rsvd) struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah); struct irdma_device *iwdev = to_iwdev(ibah->pd->device); - struct irdma_create_ah_resp uresp; + struct irdma_create_ah_resp uresp = {}; struct irdma_ah *parent_ah; int err; diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index fc1e86f6c4097..5899bd5cb1623 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -557,12 +557,20 @@ static int mlx5_query_port_roce(struct ib_device *device, u32 port_num, * of an error it will still be zeroed out. * Use native port in case of reps */ - if (dev->is_rep) - err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN, - 1, 0); - else - err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN, - mdev_port_num, 0); + if (dev->is_rep) { + struct mlx5_eswitch_rep *rep; + + rep = dev->port[port_num - 1].rep; + if (rep) { + mdev = mlx5_eswitch_get_core_dev(rep->esw); + WARN_ON(!mdev); + } + mdev_port_num = 1; + } + + err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN, + mdev_port_num, 0); + if (err) goto out; ext = !!MLX5_GET_ETH_PROTO(ptys_reg, out, true, eth_proto_capability); @@ -2874,7 +2882,6 @@ static void mlx5_ib_handle_event(struct work_struct *_work) container_of(_work, struct mlx5_ib_event_work, work); struct mlx5_ib_dev *ibdev; struct ib_event ibev; - bool fatal = false; if (work->is_slave) { ibdev = mlx5_ib_get_ibdev_from_mpi(work->mpi); @@ -2885,12 +2892,6 @@ static void mlx5_ib_handle_event(struct work_struct *_work) } switch (work->event) { - case MLX5_DEV_EVENT_SYS_ERROR: - ibev.event = IB_EVENT_DEVICE_FATAL; - mlx5_ib_handle_internal_error(ibdev); - ibev.element.port_num = (u8)(unsigned long)work->param; - fatal = true; - break; case MLX5_EVENT_TYPE_PORT_CHANGE: if (handle_port_change(ibdev, work->param, &ibev)) goto out; @@ -2912,8 +2913,6 @@ static void mlx5_ib_handle_event(struct work_struct *_work) if (ibdev->ib_active) ib_dispatch_event(&ibev); - if (fatal) - ibdev->ib_active = false; out: kfree(work); } @@ -2957,6 +2956,66 @@ static int mlx5_ib_event_slave_port(struct notifier_block *nb, return NOTIFY_OK; } +static void mlx5_ib_handle_sys_error_event(struct work_struct *_work) +{ + struct mlx5_ib_event_work *work = + container_of(_work, struct mlx5_ib_event_work, work); + struct mlx5_ib_dev *ibdev = work->dev; + struct ib_event ibev; + + ibev.event = IB_EVENT_DEVICE_FATAL; + mlx5_ib_handle_internal_error(ibdev); + ibev.element.port_num = (u8)(unsigned long)work->param; + ibev.device = &ibdev->ib_dev; + + if (!rdma_is_port_valid(&ibdev->ib_dev, ibev.element.port_num)) { + mlx5_ib_warn(ibdev, "warning: event on port %d\n", ibev.element.port_num); + goto out; + } + + if (ibdev->ib_active) + ib_dispatch_event(&ibev); + + ibdev->ib_active = false; +out: + kfree(work); +} + +static int mlx5_ib_sys_error_event(struct notifier_block *nb, + unsigned long event, void *param) +{ + struct mlx5_ib_event_work *work; + + if (event != MLX5_DEV_EVENT_SYS_ERROR) + return NOTIFY_DONE; + + work = kmalloc(sizeof(*work), GFP_ATOMIC); + if (!work) + return NOTIFY_DONE; + + INIT_WORK(&work->work, mlx5_ib_handle_sys_error_event); + work->dev = container_of(nb, struct mlx5_ib_dev, sys_error_events); + work->is_slave = false; + work->param = param; + work->event = event; + + queue_work(mlx5_ib_event_wq, &work->work); + + return NOTIFY_OK; +} + +static int mlx5_ib_stage_sys_error_notifier_init(struct mlx5_ib_dev *dev) +{ + dev->sys_error_events.notifier_call = mlx5_ib_sys_error_event; + mlx5_notifier_register(dev->mdev, &dev->sys_error_events); + return 0; +} + +static void mlx5_ib_stage_sys_error_notifier_cleanup(struct mlx5_ib_dev *dev) +{ + mlx5_notifier_unregister(dev->mdev, &dev->sys_error_events); +} + static int mlx5_ib_get_plane_num(struct mlx5_core_dev *mdev, u8 *num_plane) { struct mlx5_hca_vport_context vport_ctx; @@ -4462,12 +4521,16 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) MLX5_HCA_CAP_2_GENERAL_OBJECT_TYPES_RDMA_CTRL) { err = mlx5_ib_init_ucaps(dev); if (err) - return err; + goto err_ucaps; } dev->ib_dev.use_cq_dim = true; return 0; + +err_ucaps: + bitmap_free(dev->var_table.bitmap); + return err; } static const struct ib_device_ops mlx5_ib_dev_port_ops = { @@ -4803,6 +4866,9 @@ static const struct mlx5_ib_profile pf_profile = { STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID, mlx5_ib_devx_init, mlx5_ib_devx_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_SYS_ERROR_NOTIFIER, + mlx5_ib_stage_sys_error_notifier_init, + mlx5_ib_stage_sys_error_notifier_cleanup), STAGE_CREATE(MLX5_IB_STAGE_IB_REG, mlx5_ib_stage_ib_reg_init, mlx5_ib_stage_ib_reg_cleanup), @@ -4860,6 +4926,9 @@ const struct mlx5_ib_profile raw_eth_profile = { STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID, mlx5_ib_devx_init, mlx5_ib_devx_cleanup), + STAGE_CREATE(MLX5_IB_STAGE_SYS_ERROR_NOTIFIER, + mlx5_ib_stage_sys_error_notifier_init, + mlx5_ib_stage_sys_error_notifier_cleanup), STAGE_CREATE(MLX5_IB_STAGE_IB_REG, mlx5_ib_stage_ib_reg_init, mlx5_ib_stage_ib_reg_cleanup), diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 09d82d5f95e35..fbccb0362590b 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -1007,6 +1007,7 @@ enum mlx5_ib_stages { MLX5_IB_STAGE_BFREG, MLX5_IB_STAGE_PRE_IB_REG_UMR, MLX5_IB_STAGE_WHITELIST_UID, + MLX5_IB_STAGE_SYS_ERROR_NOTIFIER, MLX5_IB_STAGE_IB_REG, MLX5_IB_STAGE_DEVICE_NOTIFIER, MLX5_IB_STAGE_POST_IB_REG_UMR, @@ -1165,6 +1166,7 @@ struct mlx5_ib_dev { /* protect accessing data_direct_dev */ struct mutex data_direct_lock; struct notifier_block mdev_events; + struct notifier_block sys_error_events; struct notifier_block lag_events; int num_ports; /* serialize update of capability mask diff --git a/drivers/infiniband/hw/mlx5/std_types.c b/drivers/infiniband/hw/mlx5/std_types.c index 2fcf553044e15..1ee31611b4b3f 100644 --- a/drivers/infiniband/hw/mlx5/std_types.c +++ b/drivers/infiniband/hw/mlx5/std_types.c @@ -195,7 +195,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_GET_DATA_DIRECT_SYSFS_PATH)( int out_len = uverbs_attr_get_len(attrs, MLX5_IB_ATTR_GET_DATA_DIRECT_SYSFS_PATH); u32 dev_path_len; - char *dev_path; + char *dev_path = NULL; int ret; c = to_mucontext(ib_uverbs_get_ucontext(attrs)); @@ -223,9 +223,9 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_GET_DATA_DIRECT_SYSFS_PATH)( ret = uverbs_copy_to(attrs, MLX5_IB_ATTR_GET_DATA_DIRECT_SYSFS_PATH, dev_path, dev_path_len); - kfree(dev_path); end: + kfree(dev_path); mutex_unlock(&dev->data_direct_lock); return ret; } diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c index dd572d76866c2..e095873b381b6 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.c +++ b/drivers/infiniband/hw/mthca/mthca_provider.c @@ -428,6 +428,8 @@ static int mthca_create_srq(struct ib_srq *ibsrq, if (context && ib_copy_to_udata(udata, &srq->srqn, sizeof(__u32))) { mthca_free_srq(to_mdev(ibsrq->device), srq); + mthca_unmap_user_db(to_mdev(ibsrq->device), &context->uar, + context->db_tab, ucmd.db_index); return -EFAULT; } @@ -436,6 +438,7 @@ static int mthca_create_srq(struct ib_srq *ibsrq, static int mthca_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) { + mthca_free_srq(to_mdev(srq->device), to_msrq(srq)); if (udata) { struct mthca_ucontext *context = rdma_udata_to_drv_context( @@ -446,8 +449,6 @@ static int mthca_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) mthca_unmap_user_db(to_mdev(srq->device), &context->uar, context->db_tab, to_msrq(srq)->db_index); } - - mthca_free_srq(to_mdev(srq->device), to_msrq(srq)); return 0; } diff --git a/drivers/infiniband/sw/rxe/rxe_comp.c b/drivers/infiniband/sw/rxe/rxe_comp.c index a5b2b62f596b0..1390e861bd1d7 100644 --- a/drivers/infiniband/sw/rxe/rxe_comp.c +++ b/drivers/infiniband/sw/rxe/rxe_comp.c @@ -119,12 +119,15 @@ void retransmit_timer(struct timer_list *t) rxe_dbg_qp(qp, "retransmit timer fired\n"); + if (!rxe_get(qp)) + return; spin_lock_irqsave(&qp->state_lock, flags); if (qp->valid) { qp->comp.timeout = 1; rxe_sched_task(&qp->send_task); } spin_unlock_irqrestore(&qp->state_lock, flags); + rxe_put(qp); } void rxe_comp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb) diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c index bcb97b3ea58ac..2c486bb616a7c 100644 --- a/drivers/infiniband/sw/rxe/rxe_mr.c +++ b/drivers/infiniband/sw/rxe/rxe_mr.c @@ -72,14 +72,46 @@ void rxe_mr_init_dma(int access, struct rxe_mr *mr) mr->ibmr.type = IB_MR_TYPE_DMA; } +/* + * Convert iova to page_info index. The page_info stores pages of size + * PAGE_SIZE, but MRs can have different page sizes. This function + * handles the conversion for all cases: + * + * 1. mr->page_size > PAGE_SIZE: + * The MR's iova may not be aligned to mr->page_size. We use the + * aligned base (iova & page_mask) as reference, then calculate + * which PAGE_SIZE sub-page the iova falls into. + * + * 2. mr->page_size <= PAGE_SIZE: + * Use simple shift arithmetic since each page_info entry corresponds + * to one or more MR pages. + */ static unsigned long rxe_mr_iova_to_index(struct rxe_mr *mr, u64 iova) { - return (iova >> mr->page_shift) - (mr->ibmr.iova >> mr->page_shift); + int idx; + + if (mr_page_size(mr) > PAGE_SIZE) + idx = (iova - (mr->ibmr.iova & mr->page_mask)) >> PAGE_SHIFT; + else + idx = (iova >> mr->page_shift) - + (mr->ibmr.iova >> mr->page_shift); + + WARN_ON(idx >= mr->nbuf); + return idx; } +/* + * Convert iova to offset within the page_info entry. + * + * For mr_page_size > PAGE_SIZE, the offset is within the system page. + * For mr_page_size <= PAGE_SIZE, the offset is within the MR page size. + */ static unsigned long rxe_mr_iova_to_page_offset(struct rxe_mr *mr, u64 iova) { - return iova & (mr_page_size(mr) - 1); + if (mr_page_size(mr) > PAGE_SIZE) + return iova & (PAGE_SIZE - 1); + else + return iova & (mr_page_size(mr) - 1); } static bool is_pmem_page(struct page *pg) @@ -93,37 +125,69 @@ static bool is_pmem_page(struct page *pg) static int rxe_mr_fill_pages_from_sgt(struct rxe_mr *mr, struct sg_table *sgt) { - XA_STATE(xas, &mr->page_list, 0); struct sg_page_iter sg_iter; struct page *page; bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT); + WARN_ON(mr_page_size(mr) != PAGE_SIZE); + __sg_page_iter_start(&sg_iter, sgt->sgl, sgt->orig_nents, 0); if (!__sg_page_iter_next(&sg_iter)) return 0; - do { - xas_lock(&xas); - while (true) { - page = sg_page_iter_page(&sg_iter); - - if (persistent && !is_pmem_page(page)) { - rxe_dbg_mr(mr, "Page can't be persistent\n"); - xas_set_err(&xas, -EINVAL); - break; - } + while (true) { + page = sg_page_iter_page(&sg_iter); - xas_store(&xas, page); - if (xas_error(&xas)) - break; - xas_next(&xas); - if (!__sg_page_iter_next(&sg_iter)) - break; + if (persistent && !is_pmem_page(page)) { + rxe_dbg_mr(mr, "Page can't be persistent\n"); + return -EINVAL; } - xas_unlock(&xas); - } while (xas_nomem(&xas, GFP_KERNEL)); - return xas_error(&xas); + mr->page_info[mr->nbuf].page = page; + mr->page_info[mr->nbuf].offset = 0; + mr->nbuf++; + + if (!__sg_page_iter_next(&sg_iter)) + break; + } + + return 0; +} + +static int __alloc_mr_page_info(struct rxe_mr *mr, int num_pages) +{ + mr->page_info = kcalloc(num_pages, sizeof(struct rxe_mr_page), + GFP_KERNEL); + if (!mr->page_info) + return -ENOMEM; + + mr->max_allowed_buf = num_pages; + mr->nbuf = 0; + + return 0; +} + +static int alloc_mr_page_info(struct rxe_mr *mr, int num_pages) +{ + int ret; + + WARN_ON(mr->num_buf); + ret = __alloc_mr_page_info(mr, num_pages); + if (ret) + return ret; + + mr->num_buf = num_pages; + + return 0; +} + +static void free_mr_page_info(struct rxe_mr *mr) +{ + if (!mr->page_info) + return; + + kfree(mr->page_info); + mr->page_info = NULL; } int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, @@ -134,8 +198,6 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, rxe_mr_init(access, mr); - xa_init(&mr->page_list); - umem = ib_umem_get(&rxe->ib_dev, start, length, access); if (IS_ERR(umem)) { rxe_dbg_mr(mr, "Unable to pin memory region err = %d\n", @@ -143,46 +205,24 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, return PTR_ERR(umem); } + err = alloc_mr_page_info(mr, ib_umem_num_pages(umem)); + if (err) + goto err2; + err = rxe_mr_fill_pages_from_sgt(mr, &umem->sgt_append.sgt); - if (err) { - ib_umem_release(umem); - return err; - } + if (err) + goto err1; mr->umem = umem; mr->ibmr.type = IB_MR_TYPE_USER; mr->state = RXE_MR_STATE_VALID; return 0; -} - -static int rxe_mr_alloc(struct rxe_mr *mr, int num_buf) -{ - XA_STATE(xas, &mr->page_list, 0); - int i = 0; - int err; - - xa_init(&mr->page_list); - - do { - xas_lock(&xas); - while (i != num_buf) { - xas_store(&xas, XA_ZERO_ENTRY); - if (xas_error(&xas)) - break; - xas_next(&xas); - i++; - } - xas_unlock(&xas); - } while (xas_nomem(&xas, GFP_KERNEL)); - - err = xas_error(&xas); - if (err) - return err; - - mr->num_buf = num_buf; - - return 0; +err1: + free_mr_page_info(mr); +err2: + ib_umem_release(umem); + return err; } int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr) @@ -192,7 +232,7 @@ int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr) /* always allow remote access for FMRs */ rxe_mr_init(RXE_ACCESS_REMOTE, mr); - err = rxe_mr_alloc(mr, max_pages); + err = alloc_mr_page_info(mr, max_pages); if (err) goto err1; @@ -205,26 +245,43 @@ int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr) return err; } +/* + * I) MRs with page_size >= PAGE_SIZE, + * Split a large MR page (mr->page_size) into multiple PAGE_SIZE + * sub-pages and store them in page_info, offset is always 0. + * + * Called when mr->page_size > PAGE_SIZE. Each call to rxe_set_page() + * represents one mr->page_size region, which we must split into + * (mr->page_size >> PAGE_SHIFT) individual pages. + * + * II) MRs with page_size < PAGE_SIZE, + * Save each PAGE_SIZE page and its offset within the system page in page_info. + */ static int rxe_set_page(struct ib_mr *ibmr, u64 dma_addr) { struct rxe_mr *mr = to_rmr(ibmr); - struct page *page = ib_virt_dma_to_page(dma_addr); bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT); - int err; + u32 i, pages_per_mr = mr_page_size(mr) >> PAGE_SHIFT; - if (persistent && !is_pmem_page(page)) { - rxe_dbg_mr(mr, "Page cannot be persistent\n"); - return -EINVAL; - } + pages_per_mr = MAX(1, pages_per_mr); - if (unlikely(mr->nbuf == mr->num_buf)) - return -ENOMEM; + for (i = 0; i < pages_per_mr; i++) { + u64 addr = dma_addr + i * PAGE_SIZE; + struct page *sub_page = ib_virt_dma_to_page(addr); - err = xa_err(xa_store(&mr->page_list, mr->nbuf, page, GFP_KERNEL)); - if (err) - return err; + if (unlikely(mr->nbuf >= mr->max_allowed_buf)) + return -ENOMEM; + + if (persistent && !is_pmem_page(sub_page)) { + rxe_dbg_mr(mr, "Page cannot be persistent\n"); + return -EINVAL; + } + + mr->page_info[mr->nbuf].page = sub_page; + mr->page_info[mr->nbuf].offset = addr & (PAGE_SIZE - 1); + mr->nbuf++; + } - mr->nbuf++; return 0; } @@ -234,6 +291,31 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sgl, struct rxe_mr *mr = to_rmr(ibmr); unsigned int page_size = mr_page_size(mr); + /* + * Ensure page_size and PAGE_SIZE are compatible for mapping. + * We require one to be a multiple of the other for correct + * iova-to-page conversion. + */ + if (!IS_ALIGNED(page_size, PAGE_SIZE) && + !IS_ALIGNED(PAGE_SIZE, page_size)) { + rxe_dbg_mr(mr, "MR page size %u must be compatible with PAGE_SIZE %lu\n", + page_size, PAGE_SIZE); + return -EINVAL; + } + + if (mr_page_size(mr) > PAGE_SIZE) { + /* resize page_info if needed */ + u32 map_mr_pages = (page_size >> PAGE_SHIFT) * mr->num_buf; + + if (map_mr_pages > mr->max_allowed_buf) { + rxe_dbg_mr(mr, "requested pages %u exceed max %u\n", + map_mr_pages, mr->max_allowed_buf); + free_mr_page_info(mr); + if (__alloc_mr_page_info(mr, map_mr_pages)) + return -ENOMEM; + } + } + mr->nbuf = 0; mr->page_shift = ilog2(page_size); mr->page_mask = ~((u64)page_size - 1); @@ -245,30 +327,30 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sgl, static int rxe_mr_copy_xarray(struct rxe_mr *mr, u64 iova, void *addr, unsigned int length, enum rxe_mr_copy_dir dir) { - unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova); - unsigned long index = rxe_mr_iova_to_index(mr, iova); unsigned int bytes; - struct page *page; - void *va; + u8 *va; while (length) { - page = xa_load(&mr->page_list, index); - if (!page) + unsigned long index = rxe_mr_iova_to_index(mr, iova); + struct rxe_mr_page *info = &mr->page_info[index]; + unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova); + + if (!info->page) return -EFAULT; - bytes = min_t(unsigned int, length, - mr_page_size(mr) - page_offset); - va = kmap_local_page(page); + page_offset += info->offset; + bytes = min_t(unsigned int, length, PAGE_SIZE - page_offset); + va = kmap_local_page(info->page); + if (dir == RXE_FROM_MR_OBJ) memcpy(addr, va + page_offset, bytes); else memcpy(va + page_offset, addr, bytes); kunmap_local(va); - page_offset = 0; addr += bytes; + iova += bytes; length -= bytes; - index++; } return 0; @@ -426,9 +508,6 @@ int copy_data( static int rxe_mr_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length) { - unsigned int page_offset; - unsigned long index; - struct page *page; unsigned int bytes; int err; u8 *va; @@ -438,15 +517,17 @@ static int rxe_mr_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int leng return err; while (length > 0) { - index = rxe_mr_iova_to_index(mr, iova); - page = xa_load(&mr->page_list, index); - page_offset = rxe_mr_iova_to_page_offset(mr, iova); - if (!page) + unsigned long index = rxe_mr_iova_to_index(mr, iova); + struct rxe_mr_page *info = &mr->page_info[index]; + unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova); + + if (!info->page) return -EFAULT; - bytes = min_t(unsigned int, length, - mr_page_size(mr) - page_offset); - va = kmap_local_page(page); + page_offset += info->offset; + bytes = min_t(unsigned int, length, PAGE_SIZE - page_offset); + + va = kmap_local_page(info->page); arch_wb_cache_pmem(va + page_offset, bytes); kunmap_local(va); @@ -502,6 +583,7 @@ enum resp_states rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, } else { unsigned long index; int err; + struct rxe_mr_page *info; err = mr_check_range(mr, iova, sizeof(value)); if (err) { @@ -510,9 +592,12 @@ enum resp_states rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, } page_offset = rxe_mr_iova_to_page_offset(mr, iova); index = rxe_mr_iova_to_index(mr, iova); - page = xa_load(&mr->page_list, index); - if (!page) + info = &mr->page_info[index]; + if (!info->page) return RESPST_ERR_RKEY_VIOLATION; + + page_offset += info->offset; + page = info->page; } if (unlikely(page_offset & 0x7)) { @@ -551,6 +636,7 @@ enum resp_states rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value) } else { unsigned long index; int err; + struct rxe_mr_page *info; /* See IBA oA19-28 */ err = mr_check_range(mr, iova, sizeof(value)); @@ -560,9 +646,12 @@ enum resp_states rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value) } page_offset = rxe_mr_iova_to_page_offset(mr, iova); index = rxe_mr_iova_to_index(mr, iova); - page = xa_load(&mr->page_list, index); - if (!page) + info = &mr->page_info[index]; + if (!info->page) return RESPST_ERR_RKEY_VIOLATION; + + page_offset += info->offset; + page = info->page; } /* See IBA A19.4.2 */ @@ -726,5 +815,5 @@ void rxe_mr_cleanup(struct rxe_pool_elem *elem) ib_umem_release(mr->umem); if (mr->ibmr.type != IB_MR_TYPE_DMA) - xa_destroy(&mr->page_list); + free_mr_page_info(mr); } diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c index 373b03f223beb..12d03f390b097 100644 --- a/drivers/infiniband/sw/rxe/rxe_req.c +++ b/drivers/infiniband/sw/rxe/rxe_req.c @@ -102,6 +102,8 @@ void rnr_nak_timer(struct timer_list *t) rxe_dbg_qp(qp, "nak timer fired\n"); + if (!rxe_get(qp)) + return; spin_lock_irqsave(&qp->state_lock, flags); if (qp->valid) { /* request a send queue retry */ @@ -110,6 +112,7 @@ void rnr_nak_timer(struct timer_list *t) rxe_sched_task(&qp->send_task); } spin_unlock_irqrestore(&qp->state_lock, flags); + rxe_put(qp); } static void req_check_sq_drain_done(struct rxe_qp *qp) diff --git a/drivers/infiniband/sw/rxe/rxe_srq.c b/drivers/infiniband/sw/rxe/rxe_srq.c index 2a234f26ac104..c9a7cd38953d3 100644 --- a/drivers/infiniband/sw/rxe/rxe_srq.c +++ b/drivers/infiniband/sw/rxe/rxe_srq.c @@ -77,9 +77,6 @@ int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq, goto err_free; } - srq->rq.queue = q; - init->attr.max_wr = srq->rq.max_wr; - if (uresp) { if (copy_to_user(&uresp->srq_num, &srq->srq_num, sizeof(uresp->srq_num))) { @@ -88,6 +85,9 @@ int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq, } } + srq->rq.queue = q; + init->attr.max_wr = srq->rq.max_wr; + return 0; err_free: diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h index fd48075810dd1..1b8ed1031bd57 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.h +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h @@ -335,6 +335,11 @@ static inline int rkey_is_mw(u32 rkey) return (index >= RXE_MIN_MW_INDEX) && (index <= RXE_MAX_MW_INDEX); } +struct rxe_mr_page { + struct page *page; + unsigned int offset; /* offset in system page */ +}; + struct rxe_mr { struct rxe_pool_elem elem; struct ib_mr ibmr; @@ -351,10 +356,13 @@ struct rxe_mr { unsigned int page_shift; u64 page_mask; + /* size of page_info when mr allocated */ u32 num_buf; + /* real size of page_info */ + u32 max_allowed_buf; u32 nbuf; - struct xarray page_list; + struct rxe_mr_page *page_info; }; static inline unsigned int mr_page_size(struct rxe_mr *mr) diff --git a/drivers/infiniband/sw/siw/siw_qp_rx.c b/drivers/infiniband/sw/siw/siw_qp_rx.c index a10820e338878..e8a88b378d51d 100644 --- a/drivers/infiniband/sw/siw/siw_qp_rx.c +++ b/drivers/infiniband/sw/siw/siw_qp_rx.c @@ -1435,7 +1435,8 @@ int siw_tcp_rx_data(read_descriptor_t *rd_desc, struct sk_buff *skb, } if (unlikely(rv != 0 && rv != -EAGAIN)) { if ((srx->state > SIW_GET_HDR || - qp->rx_fpdu->more_ddp_segs) && run_completion) + (qp->rx_fpdu && qp->rx_fpdu->more_ddp_segs)) && + run_completion) siw_rdmap_complete(qp, rv); siw_dbg_qp(qp, "rx error %d, rx state %d\n", rv, diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c index 2b397a544cb93..8fa1d72bd20a4 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c @@ -1923,7 +1923,7 @@ static int rtrs_rdma_conn_rejected(struct rtrs_clt_con *con, struct rtrs_path *s = con->c.path; const struct rtrs_msg_conn_rsp *msg; const char *rej_msg; - int status, errno; + int status, errno = -ECONNRESET; u8 data_len; status = ev->status; @@ -1945,7 +1945,7 @@ static int rtrs_rdma_conn_rejected(struct rtrs_clt_con *con, status, rej_msg); } - return -ECONNRESET; + return errno; } void rtrs_clt_close_conns(struct rtrs_clt_path *clt_path, bool wait) diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.c b/drivers/infiniband/ulp/rtrs/rtrs-srv.c index 9ecc6343455d6..adb798e2a54ae 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-srv.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.c @@ -208,7 +208,6 @@ static int rdma_write_sg(struct rtrs_srv_op *id) size_t sg_cnt; int err, offset; bool need_inval; - u32 rkey = 0; struct ib_reg_wr rwr; struct ib_sge *plist; struct ib_sge list; @@ -240,11 +239,6 @@ static int rdma_write_sg(struct rtrs_srv_op *id) wr->wr.num_sge = 1; wr->remote_addr = le64_to_cpu(id->rd_msg->desc[0].addr); wr->rkey = le32_to_cpu(id->rd_msg->desc[0].key); - if (rkey == 0) - rkey = wr->rkey; - else - /* Only one key is actually used */ - WARN_ON_ONCE(rkey != wr->rkey); wr->wr.opcode = IB_WR_RDMA_WRITE; wr->wr.wr_cqe = &io_comp_cqe; @@ -277,7 +271,7 @@ static int rdma_write_sg(struct rtrs_srv_op *id) inv_wr.opcode = IB_WR_SEND_WITH_INV; inv_wr.wr_cqe = &io_comp_cqe; inv_wr.send_flags = 0; - inv_wr.ex.invalidate_rkey = rkey; + inv_wr.ex.invalidate_rkey = wr->rkey; } imm_wr.wr.next = NULL; @@ -601,7 +595,7 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path) srv_path->mrs_num++) { struct rtrs_srv_mr *srv_mr = &srv_path->mrs[srv_path->mrs_num]; struct scatterlist *s; - int nr, nr_sgt, chunks; + int nr, nr_sgt, chunks, ind; sgt = &srv_mr->sgt; chunks = chunks_per_mr * srv_path->mrs_num; @@ -631,7 +625,7 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path) } nr = ib_map_mr_sg(mr, sgt->sgl, nr_sgt, NULL, max_chunk_size); - if (nr != nr_sgt) { + if (nr < nr_sgt) { err = nr < 0 ? nr : -EINVAL; goto dereg_mr; } @@ -647,9 +641,24 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path) goto dereg_mr; } } - /* Eventually dma addr for each chunk can be cached */ - for_each_sg(sgt->sgl, s, nr_sgt, i) - srv_path->dma_addr[chunks + i] = sg_dma_address(s); + + /* + * Cache DMA addresses by traversing sg entries. If + * regions were merged, an inner loop is required to + * populate the DMA address array by traversing larger + * regions. + */ + ind = chunks; + for_each_sg(sgt->sgl, s, nr_sgt, i) { + unsigned int dma_len = sg_dma_len(s); + u64 dma_addr = sg_dma_address(s); + u64 dma_addr_end = dma_addr + dma_len; + + do { + srv_path->dma_addr[ind++] = dma_addr; + dma_addr += max_chunk_size; + } while (dma_addr < dma_addr_end); + } ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey)); srv_mr->mr = mr; diff --git a/drivers/input/mouse/synaptics_i2c.c b/drivers/input/mouse/synaptics_i2c.c index a0d707e47d932..29da66af36d74 100644 --- a/drivers/input/mouse/synaptics_i2c.c +++ b/drivers/input/mouse/synaptics_i2c.c @@ -372,7 +372,7 @@ static irqreturn_t synaptics_i2c_irq(int irq, void *dev_id) { struct synaptics_i2c *touch = dev_id; - mod_delayed_work(system_wq, &touch->dwork, 0); + mod_delayed_work(system_dfl_wq, &touch->dwork, 0); return IRQ_HANDLED; } @@ -448,7 +448,7 @@ static void synaptics_i2c_work_handler(struct work_struct *work) * We poll the device once in THREAD_IRQ_SLEEP_SECS and * if error is detected, we try to reset and reconfigure the touchpad. */ - mod_delayed_work(system_wq, &touch->dwork, delay); + mod_delayed_work(system_dfl_wq, &touch->dwork, delay); } static int synaptics_i2c_open(struct input_dev *input) @@ -461,7 +461,7 @@ static int synaptics_i2c_open(struct input_dev *input) return ret; if (polling_req) - mod_delayed_work(system_wq, &touch->dwork, + mod_delayed_work(system_dfl_wq, &touch->dwork, msecs_to_jiffies(NO_DATA_SLEEP_MSECS)); return 0; @@ -615,13 +615,16 @@ static int synaptics_i2c_resume(struct device *dev) int ret; struct i2c_client *client = to_i2c_client(dev); struct synaptics_i2c *touch = i2c_get_clientdata(client); + struct input_dev *input = touch->input; ret = synaptics_i2c_reset_config(client); if (ret) return ret; - mod_delayed_work(system_wq, &touch->dwork, - msecs_to_jiffies(NO_DATA_SLEEP_MSECS)); + guard(mutex)(&input->mutex); + if (input_device_enabled(input)) + mod_delayed_work(system_dfl_wq, &touch->dwork, + msecs_to_jiffies(NO_DATA_SLEEP_MSECS)); return 0; } diff --git a/drivers/interconnect/mediatek/icc-emi.c b/drivers/interconnect/mediatek/icc-emi.c index 7da740b5fa8d6..dfa3a9cd93998 100644 --- a/drivers/interconnect/mediatek/icc-emi.c +++ b/drivers/interconnect/mediatek/icc-emi.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -22,7 +23,9 @@ static int mtk_emi_icc_aggregate(struct icc_node *node, u32 tag, u32 avg_bw, { struct mtk_icc_node *in = node->data; - *agg_avg += avg_bw; + if (check_add_overflow(*agg_avg, avg_bw, agg_avg)) + *agg_avg = U32_MAX; + *agg_peak = max_t(u32, *agg_peak, peak_bw); in->sum_avg = *agg_avg; @@ -40,7 +43,7 @@ static int mtk_emi_icc_set(struct icc_node *src, struct icc_node *dst) if (unlikely(!src->provider)) return -EINVAL; - dev = src->provider->dev; + dev = src->provider->dev->parent; switch (node->ep) { case 0: @@ -97,7 +100,7 @@ int mtk_emi_icc_probe(struct platform_device *pdev) if (!data) return -ENOMEM; - provider->dev = pdev->dev.parent; + provider->dev = dev; provider->set = mtk_emi_icc_set; provider->aggregate = mtk_emi_icc_aggregate; provider->xlate = of_icc_xlate_onecell; diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 9b4b589a54b57..bf77fdf5529f4 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -15,7 +15,6 @@ irqreturn_t amd_iommu_int_thread(int irq, void *data); irqreturn_t amd_iommu_int_thread_evtlog(int irq, void *data); irqreturn_t amd_iommu_int_thread_pprlog(int irq, void *data); irqreturn_t amd_iommu_int_thread_galog(int irq, void *data); -irqreturn_t amd_iommu_int_handler(int irq, void *data); void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type, u8 cntrl_intr, u8 cntrl_log, u32 status_run_mask, u32 status_overflow_mask); diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index a698a2e7ce2a6..b0d919cd1a8fb 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -791,7 +791,7 @@ struct amd_iommu { u32 flags; volatile u64 *cmd_sem; - atomic64_t cmd_sem_val; + u64 cmd_sem_val; /* * Track physical address to directly use it in build_completion_wait() * and avoid adding any special checks and handling for kdump. diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 034edce816d01..76efd74124b33 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1879,7 +1879,7 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, iommu->pci_seg = pci_seg; raw_spin_lock_init(&iommu->lock); - atomic64_set(&iommu->cmd_sem_val, 0); + iommu->cmd_sem_val = 0; /* Add IOMMU to internal data structures */ list_add_tail(&iommu->list, &amd_iommu_list); @@ -2355,12 +2355,8 @@ static int iommu_setup_msi(struct amd_iommu *iommu) if (r) return r; - r = request_threaded_irq(iommu->dev->irq, - amd_iommu_int_handler, - amd_iommu_int_thread, - 0, "AMD-Vi", - iommu); - + r = request_threaded_irq(iommu->dev->irq, NULL, amd_iommu_int_thread, + IRQF_ONESHOT, "AMD-Vi", iommu); if (r) { pci_disable_msi(iommu->dev); return r; @@ -2534,8 +2530,8 @@ static int __iommu_setup_intcapxt(struct amd_iommu *iommu, const char *devname, return irq; } - ret = request_threaded_irq(irq, amd_iommu_int_handler, - thread_fn, 0, devname, iommu); + ret = request_threaded_irq(irq, NULL, thread_fn, IRQF_ONESHOT, devname, + iommu); if (ret) { irq_domain_free_irqs(irq, 1); irq_domain_remove(domain); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 5914bef0c8c19..4beef73139611 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1146,11 +1146,6 @@ irqreturn_t amd_iommu_int_thread(int irq, void *data) return IRQ_HANDLED; } -irqreturn_t amd_iommu_int_handler(int irq, void *data) -{ - return IRQ_WAKE_THREAD; -} - /**************************************************************************** * * IOMMU command queuing functions @@ -1161,7 +1156,12 @@ static int wait_on_sem(struct amd_iommu *iommu, u64 data) { int i = 0; - while (*iommu->cmd_sem != data && i < LOOP_TIMEOUT) { + /* + * cmd_sem holds a monotonically non-decreasing completion sequence + * number. + */ + while ((__s64)(READ_ONCE(*iommu->cmd_sem) - data) < 0 && + i < LOOP_TIMEOUT) { udelay(1); i += 1; } @@ -1386,6 +1386,12 @@ static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd) return iommu_queue_command_sync(iommu, cmd, true); } +static u64 get_cmdsem_val(struct amd_iommu *iommu) +{ + lockdep_assert_held(&iommu->lock); + return ++iommu->cmd_sem_val; +} + /* * This function queues a completion wait command into the command * buffer of an IOMMU @@ -1400,20 +1406,19 @@ static int iommu_completion_wait(struct amd_iommu *iommu) if (!iommu->need_sync) return 0; - data = atomic64_inc_return(&iommu->cmd_sem_val); - build_completion_wait(&cmd, iommu, data); - raw_spin_lock_irqsave(&iommu->lock, flags); + data = get_cmdsem_val(iommu); + build_completion_wait(&cmd, iommu, data); + ret = __iommu_queue_command_sync(iommu, &cmd, false); + raw_spin_unlock_irqrestore(&iommu->lock, flags); + if (ret) - goto out_unlock; + return ret; ret = wait_on_sem(iommu, data); -out_unlock: - raw_spin_unlock_irqrestore(&iommu->lock, flags); - return ret; } @@ -3087,18 +3092,23 @@ static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid) return; build_inv_irt(&cmd, devid); - data = atomic64_inc_return(&iommu->cmd_sem_val); - build_completion_wait(&cmd2, iommu, data); raw_spin_lock_irqsave(&iommu->lock, flags); + data = get_cmdsem_val(iommu); + build_completion_wait(&cmd2, iommu, data); + ret = __iommu_queue_command_sync(iommu, &cmd, true); if (ret) - goto out; + goto out_err; ret = __iommu_queue_command_sync(iommu, &cmd2, false); if (ret) - goto out; + goto out_err; + raw_spin_unlock_irqrestore(&iommu->lock, flags); + wait_on_sem(iommu, data); -out: + return; + +out_err: raw_spin_unlock_irqrestore(&iommu->lock, flags); } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c index 8cd8929bbfdf8..9ec5591565083 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c @@ -165,7 +165,9 @@ static int arm_smmu_attach_dev_nested(struct iommu_domain *domain, * config bit here base this off the EATS value in the STE. If the EATS * is set then the VM must generate ATC flushes. */ - state.disable_ats = !nested_domain->enable_ats; + if (FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(nested_domain->ste[0])) == + STRTAB_STE_0_CFG_S1_TRANS) + state.disable_ats = !nested_domain->enable_ats; ret = arm_smmu_attach_prepare(&state, domain); if (ret) { mutex_unlock(&arm_smmu_asid_lock); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c index d2671bfd37981..b254a94b2003d 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c @@ -38,13 +38,16 @@ enum arm_smmu_test_master_feat { static bool arm_smmu_entry_differs_in_used_bits(const __le64 *entry, const __le64 *used_bits, const __le64 *target, + const __le64 *safe, unsigned int length) { bool differs = false; unsigned int i; for (i = 0; i < length; i++) { - if ((entry[i] & used_bits[i]) != target[i]) + __le64 used = used_bits[i] & ~safe[i]; + + if ((entry[i] & used) != (target[i] & used)) differs = true; } return differs; @@ -56,12 +59,24 @@ arm_smmu_test_writer_record_syncs(struct arm_smmu_entry_writer *writer) struct arm_smmu_test_writer *test_writer = container_of(writer, struct arm_smmu_test_writer, writer); __le64 *entry_used_bits; + __le64 *safe_target; + __le64 *safe_init; entry_used_bits = kunit_kzalloc( test_writer->test, sizeof(*entry_used_bits) * NUM_ENTRY_QWORDS, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test_writer->test, entry_used_bits); + safe_target = kunit_kzalloc(test_writer->test, + sizeof(*safe_target) * NUM_ENTRY_QWORDS, + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test_writer->test, safe_target); + + safe_init = kunit_kzalloc(test_writer->test, + sizeof(*safe_init) * NUM_ENTRY_QWORDS, + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test_writer->test, safe_init); + pr_debug("STE value is now set to: "); print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, test_writer->entry, @@ -79,14 +94,23 @@ arm_smmu_test_writer_record_syncs(struct arm_smmu_entry_writer *writer) * configuration. */ writer->ops->get_used(test_writer->entry, entry_used_bits); + if (writer->ops->get_update_safe) + writer->ops->get_update_safe(test_writer->entry, + test_writer->init_entry, + safe_init); + if (writer->ops->get_update_safe) + writer->ops->get_update_safe(test_writer->entry, + test_writer->target_entry, + safe_target); KUNIT_EXPECT_FALSE( test_writer->test, arm_smmu_entry_differs_in_used_bits( test_writer->entry, entry_used_bits, - test_writer->init_entry, NUM_ENTRY_QWORDS) && + test_writer->init_entry, safe_init, + NUM_ENTRY_QWORDS) && arm_smmu_entry_differs_in_used_bits( test_writer->entry, entry_used_bits, - test_writer->target_entry, + test_writer->target_entry, safe_target, NUM_ENTRY_QWORDS)); } } @@ -106,6 +130,7 @@ arm_smmu_v3_test_debug_print_used_bits(struct arm_smmu_entry_writer *writer, static const struct arm_smmu_entry_writer_ops test_ste_ops = { .sync = arm_smmu_test_writer_record_syncs, .get_used = arm_smmu_get_ste_used, + .get_update_safe = arm_smmu_get_ste_update_safe, }; static const struct arm_smmu_entry_writer_ops test_cd_ops = { diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 9780f40ba3e65..1e47da0ce6b9b 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -487,20 +487,26 @@ static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu) */ static void arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq *cmdq) { - int val; - /* - * We can try to avoid the cmpxchg() loop by simply incrementing the - * lock counter. When held in exclusive state, the lock counter is set - * to INT_MIN so these increments won't hurt as the value will remain - * negative. + * When held in exclusive state, the lock counter is set to INT_MIN + * so these increments won't hurt as the value will remain negative. + * The increment will also signal the exclusive locker that there are + * shared waiters. */ if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0) return; - do { - val = atomic_cond_read_relaxed(&cmdq->lock, VAL >= 0); - } while (atomic_cmpxchg_relaxed(&cmdq->lock, val, val + 1) != val); + /* + * Someone else is holding the lock in exclusive state, so wait + * for them to finish. Since we already incremented the lock counter, + * no exclusive lock can be acquired until we finish. We don't need + * the return value since we only care that the exclusive lock is + * released (i.e. the lock counter is non-negative). + * Once the exclusive locker releases the lock, the sign bit will + * be cleared and our increment will make the lock counter positive, + * allowing us to proceed. + */ + atomic_cond_read_relaxed(&cmdq->lock, VAL > 0); } static void arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq *cmdq) @@ -527,9 +533,14 @@ static bool arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq *cmdq) __ret; \ }) +/* + * Only clear the sign bit when releasing the exclusive lock this will + * allow any shared_lock() waiters to proceed without the possibility + * of entering the exclusive lock in a tight loop. + */ #define arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags) \ ({ \ - atomic_set_release(&cmdq->lock, 0); \ + atomic_fetch_andnot_release(INT_MIN, &cmdq->lock); \ local_irq_restore(flags); \ }) @@ -1082,6 +1093,49 @@ void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits) } EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_ste_used); +VISIBLE_IF_KUNIT +void arm_smmu_get_ste_update_safe(const __le64 *cur, const __le64 *target, + __le64 *safe_bits) +{ + const __le64 eats_s1chk = + FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_S1CHK); + const __le64 eats_trans = + FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_TRANS); + + /* + * When an STE changes EATS_TRANS, the sequencing code in the attach + * logic already will have the PCI cap for ATS disabled. Thus at this + * moment we can expect that the device will not generate ATS queries + * and so we don't care about the sequencing of EATS. The purpose of + * EATS_TRANS is to protect the system from hostile untrusted devices + * that issue ATS when the PCI config space is disabled. However, if + * EATS_TRANS is being changed, then we must have already trusted the + * device as the EATS_TRANS security block is being disabled. + * + * Note: now the EATS_TRANS update is moved to the first entry_set(). + * Changing S2S and EATS might transiently result in S2S=1 and EATS=1 + * which is a bad STE (see "5.2 Stream Table Entry"). In such a case, + * we can't do a hitless update. Also, it should not be added to the + * safe bits with STRTAB_STE_1_EATS_S1CHK, because EATS=0b11 would be + * effectively an errant 0b00 configuration. + */ + if (!((cur[1] | target[1]) & cpu_to_le64(eats_s1chk)) && + !((cur[2] | target[2]) & cpu_to_le64(STRTAB_STE_2_S2S))) + safe_bits[1] |= cpu_to_le64(eats_trans); + + /* + * MEV does not meaningfully impact the operation of the HW, it only + * changes how many fault events are generated, thus we can relax it + * when computing the ordering. The spec notes the device can act like + * MEV=1 anyhow: + * + * Note: Software must expect, and be able to deal with, coalesced + * fault records even when MEV == 0. + */ + safe_bits[1] |= cpu_to_le64(STRTAB_STE_1_MEV); +} +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_ste_update_safe); + /* * Figure out if we can do a hitless update of entry to become target. Returns a * bit mask where 1 indicates that qword needs to be set disruptively. @@ -1094,13 +1148,22 @@ static u8 arm_smmu_entry_qword_diff(struct arm_smmu_entry_writer *writer, { __le64 target_used[NUM_ENTRY_QWORDS] = {}; __le64 cur_used[NUM_ENTRY_QWORDS] = {}; + __le64 safe[NUM_ENTRY_QWORDS] = {}; u8 used_qword_diff = 0; unsigned int i; writer->ops->get_used(entry, cur_used); writer->ops->get_used(target, target_used); + if (writer->ops->get_update_safe) + writer->ops->get_update_safe(entry, target, safe); for (i = 0; i != NUM_ENTRY_QWORDS; i++) { + /* + * Safe is only used for bits that are used by both entries, + * otherwise it is sequenced according to the unused entry. + */ + safe[i] &= target_used[i] & cur_used[i]; + /* * Check that masks are up to date, the make functions are not * allowed to set a bit to 1 if the used function doesn't say it @@ -1109,6 +1172,7 @@ static u8 arm_smmu_entry_qword_diff(struct arm_smmu_entry_writer *writer, WARN_ON_ONCE(target[i] & ~target_used[i]); /* Bits can change because they are not currently being used */ + cur_used[i] &= ~safe[i]; unused_update[i] = (entry[i] & cur_used[i]) | (target[i] & ~cur_used[i]); /* @@ -1121,7 +1185,7 @@ static u8 arm_smmu_entry_qword_diff(struct arm_smmu_entry_writer *writer, return used_qword_diff; } -static bool entry_set(struct arm_smmu_entry_writer *writer, __le64 *entry, +static void entry_set(struct arm_smmu_entry_writer *writer, __le64 *entry, const __le64 *target, unsigned int start, unsigned int len) { @@ -1137,7 +1201,6 @@ static bool entry_set(struct arm_smmu_entry_writer *writer, __le64 *entry, if (changed) writer->ops->sync(writer); - return changed; } /* @@ -1207,12 +1270,9 @@ void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *entry, entry_set(writer, entry, target, 0, 1); } else { /* - * No inuse bit changed. Sanity check that all unused bits are 0 - * in the entry. The target was already sanity checked by - * compute_qword_diff(). + * No inuse bit changed, though safe bits may have changed. */ - WARN_ON_ONCE( - entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS)); + entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS); } } EXPORT_SYMBOL_IF_KUNIT(arm_smmu_write_entry); @@ -1543,6 +1603,7 @@ static void arm_smmu_ste_writer_sync_entry(struct arm_smmu_entry_writer *writer) static const struct arm_smmu_entry_writer_ops arm_smmu_ste_writer_ops = { .sync = arm_smmu_ste_writer_sync_entry, .get_used = arm_smmu_get_ste_used, + .get_update_safe = arm_smmu_get_ste_update_safe, }; static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid, diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index ae23aacc38402..287e223c054d1 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -900,6 +900,8 @@ struct arm_smmu_entry_writer { struct arm_smmu_entry_writer_ops { void (*get_used)(const __le64 *entry, __le64 *used); + void (*get_update_safe)(const __le64 *cur, const __le64 *target, + __le64 *safe_bits); void (*sync)(struct arm_smmu_entry_writer *writer); }; @@ -911,6 +913,8 @@ void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, #if IS_ENABLED(CONFIG_KUNIT) void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits); +void arm_smmu_get_ste_update_safe(const __le64 *cur, const __le64 *target, + __le64 *safe_bits); void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *cur, const __le64 *target); void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits); diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c b/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c index db9b9a8e139c8..4565a58bb213f 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c @@ -228,3 +228,17 @@ struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu) return smmu; } + +int __init arm_smmu_impl_module_init(void) +{ + if (IS_ENABLED(CONFIG_ARM_SMMU_QCOM)) + return qcom_smmu_module_init(); + + return 0; +} + +void __exit arm_smmu_impl_module_exit(void) +{ + if (IS_ENABLED(CONFIG_ARM_SMMU_QCOM)) + qcom_smmu_module_exit(); +} diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index 515771fc8a0b8..f0d4f55b63f65 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -801,10 +801,6 @@ struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu) { const struct device_node *np = smmu->dev->of_node; const struct of_device_id *match; - static u8 tbu_registered; - - if (!tbu_registered++) - platform_driver_register(&qcom_smmu_tbu_driver); #ifdef CONFIG_ACPI if (np == NULL) { @@ -829,3 +825,13 @@ struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu) return smmu; } + +int __init qcom_smmu_module_init(void) +{ + return platform_driver_register(&qcom_smmu_tbu_driver); +} + +void __exit qcom_smmu_module_exit(void) +{ + platform_driver_unregister(&qcom_smmu_tbu_driver); +} diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c index 4ced4b5bee4df..488632b8eeab6 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c @@ -2362,7 +2362,29 @@ static struct platform_driver arm_smmu_driver = { .remove = arm_smmu_device_remove, .shutdown = arm_smmu_device_shutdown, }; -module_platform_driver(arm_smmu_driver); + +static int __init arm_smmu_init(void) +{ + int ret; + + ret = platform_driver_register(&arm_smmu_driver); + if (ret) + return ret; + + ret = arm_smmu_impl_module_init(); + if (ret) + platform_driver_unregister(&arm_smmu_driver); + + return ret; +} +module_init(arm_smmu_init); + +static void __exit arm_smmu_exit(void) +{ + arm_smmu_impl_module_exit(); + platform_driver_unregister(&arm_smmu_driver); +} +module_exit(arm_smmu_exit); MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations"); MODULE_AUTHOR("Will Deacon "); diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.h b/drivers/iommu/arm/arm-smmu/arm-smmu.h index 2dbf3243b5ad2..26d2e33cd328b 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.h +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.h @@ -540,6 +540,11 @@ struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu); struct arm_smmu_device *nvidia_smmu_impl_init(struct arm_smmu_device *smmu); struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu); +int __init arm_smmu_impl_module_init(void); +void __exit arm_smmu_impl_module_exit(void); +int __init qcom_smmu_module_init(void); +void __exit qcom_smmu_module_exit(void); + void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx); int arm_mmu500_reset(struct arm_smmu_device *smmu); diff --git a/drivers/iommu/intel/cache.c b/drivers/iommu/intel/cache.c index 265e7290256b5..385ae5cfb30d4 100644 --- a/drivers/iommu/intel/cache.c +++ b/drivers/iommu/intel/cache.c @@ -363,6 +363,13 @@ static void qi_batch_add_pasid_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qi_batch_increment_index(iommu, batch); } +static bool intel_domain_use_piotlb(struct dmar_domain *domain) +{ + return domain->domain.type == IOMMU_DOMAIN_SVA || + domain->domain.type == IOMMU_DOMAIN_NESTED || + intel_domain_is_fs_paging(domain); +} + static void cache_tag_flush_iotlb(struct dmar_domain *domain, struct cache_tag *tag, unsigned long addr, unsigned long pages, unsigned long mask, int ih) @@ -370,7 +377,7 @@ static void cache_tag_flush_iotlb(struct dmar_domain *domain, struct cache_tag * struct intel_iommu *iommu = tag->iommu; u64 type = DMA_TLB_PSI_FLUSH; - if (intel_domain_is_fs_paging(domain)) { + if (intel_domain_use_piotlb(domain)) { qi_batch_add_piotlb(iommu, tag->domain_id, tag->pasid, addr, pages, ih, domain->qi_batch); return; diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index e236c7ec221f4..49e83c8566a32 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -1722,10 +1722,12 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8 } did = context_domain_id(context); - context_clear_entry(context); + context_clear_present(context); __iommu_flush_cache(iommu, context, sizeof(*context)); spin_unlock(&iommu->lock); intel_context_flush_no_pasid(info, context, did); + context_clear_entry(context); + __iommu_flush_cache(iommu, context, sizeof(*context)); } int __domain_setup_first_level(struct intel_iommu *iommu, struct device *dev, diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index dcc5466d35f93..9198ac7f6bbab 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -969,7 +969,26 @@ static inline unsigned long lvl_to_nr_pages(unsigned int lvl) static inline void context_set_present(struct context_entry *context) { - context->lo |= 1; + u64 val; + + dma_wmb(); + val = READ_ONCE(context->lo) | 1; + WRITE_ONCE(context->lo, val); +} + +/* + * Clear the Present (P) bit (bit 0) of a context table entry. This initiates + * the transition of the entry's ownership from hardware to software. The + * caller is responsible for fulfilling the invalidation handshake recommended + * by the VT-d spec, Section 6.5.3.3 (Guidance to Software for Invalidations). + */ +static inline void context_clear_present(struct context_entry *context) +{ + u64 val; + + val = READ_ONCE(context->lo) & GENMASK_ULL(63, 1); + WRITE_ONCE(context->lo, val); + dma_wmb(); } static inline void context_set_fault_enable(struct context_entry *context) diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 52f678975da74..7a64a55fb5887 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -153,6 +153,9 @@ static struct pasid_entry *intel_pasid_get_entry(struct device *dev, u32 pasid) if (!entries) return NULL; + if (!ecap_coherent(info->iommu->ecap)) + clflush_cache_range(entries, VTD_PAGE_SIZE); + /* * The pasid directory table entry won't be freed after * allocation. No worry about the race with free and @@ -165,10 +168,8 @@ static struct pasid_entry *intel_pasid_get_entry(struct device *dev, u32 pasid) iommu_free_pages(entries); goto retry; } - if (!ecap_coherent(info->iommu->ecap)) { - clflush_cache_range(entries, VTD_PAGE_SIZE); + if (!ecap_coherent(info->iommu->ecap)) clflush_cache_range(&dir[dir_index].val, sizeof(*dir)); - } } return &entries[index]; @@ -218,7 +219,7 @@ devtlb_invalidation_with_pasid(struct intel_iommu *iommu, if (!info || !info->ats_enabled) return; - if (pci_dev_is_disconnected(to_pci_dev(dev))) + if (!pci_device_is_present(to_pci_dev(dev))) return; sid = PCI_DEVID(info->bus, info->devfn); @@ -272,7 +273,7 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev, did = pasid_get_domain_id(pte); pgtt = pasid_pte_get_pgtt(pte); - intel_pasid_clear_entry(dev, pasid, fault_ignore); + pasid_clear_present(pte); spin_unlock(&iommu->lock); if (!ecap_coherent(iommu->ecap)) @@ -286,6 +287,10 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev, iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH); devtlb_invalidation_with_pasid(iommu, dev, pasid); + intel_pasid_clear_entry(dev, pasid, fault_ignore); + if (!ecap_coherent(iommu->ecap)) + clflush_cache_range(pte, sizeof(*pte)); + if (!fault_ignore) intel_iommu_drain_pasid_prq(dev, pasid); } @@ -1023,7 +1028,7 @@ static int device_pasid_table_setup(struct device *dev, u8 bus, u8 devfn) } if (context_copied(iommu, bus, devfn)) { - context_clear_entry(context); + context_clear_present(context); __iommu_flush_cache(iommu, context, sizeof(*context)); /* @@ -1043,6 +1048,9 @@ static int device_pasid_table_setup(struct device *dev, u8 bus, u8 devfn) iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); devtlb_invalidation_with_pasid(iommu, dev, IOMMU_NO_PASID); + context_clear_entry(context); + __iommu_flush_cache(iommu, context, sizeof(*context)); + /* * At this point, the device is supposed to finish reset at * its driver probe stage, so no in-flight DMA will exist, @@ -1106,6 +1114,14 @@ static void __context_flush_dev_iotlb(struct device_domain_info *info) if (!info->ats_enabled) return; + /* + * Skip dev-IOTLB flush for inaccessible PCIe devices to prevent the + * Intel IOMMU from waiting indefinitely for an ATS invalidation that + * cannot complete. + */ + if (!pci_device_is_present(to_pci_dev(info->dev))) + return; + qi_flush_dev_iotlb(info->iommu, PCI_DEVID(info->bus, info->devfn), info->pfsid, info->ats_qdep, 0, MAX_AGAW_PFN_WIDTH); diff --git a/drivers/iommu/intel/pasid.h b/drivers/iommu/intel/pasid.h index a771a77d4239c..637373995be80 100644 --- a/drivers/iommu/intel/pasid.h +++ b/drivers/iommu/intel/pasid.h @@ -233,9 +233,23 @@ static inline void pasid_set_wpe(struct pasid_entry *pe) */ static inline void pasid_set_present(struct pasid_entry *pe) { + dma_wmb(); pasid_set_bits(&pe->val[0], 1 << 0, 1); } +/* + * Clear the Present (P) bit (bit 0) of a scalable-mode PASID table entry. + * This initiates the transition of the entry's ownership from hardware + * to software. The caller is responsible for fulfilling the invalidation + * handshake recommended by the VT-d spec, Section 6.5.3.3 (Guidance to + * Software for Invalidations). + */ +static inline void pasid_clear_present(struct pasid_entry *pe) +{ + pasid_set_bits(&pe->val[0], 1 << 0, 0); + dma_wmb(); +} + /* * Setup Page Walk Snoop bit (Bit 87) of a scalable mode PASID * entry. diff --git a/drivers/irqchip/irq-riscv-imsic-early.c b/drivers/irqchip/irq-riscv-imsic-early.c index 2c4c682627b8c..d1727c343c38f 100644 --- a/drivers/irqchip/irq-riscv-imsic-early.c +++ b/drivers/irqchip/irq-riscv-imsic-early.c @@ -7,6 +7,7 @@ #define pr_fmt(fmt) "riscv-imsic: " fmt #include #include +#include #include #include #include @@ -128,14 +129,8 @@ static void imsic_handle_irq(struct irq_desc *desc) chained_irq_exit(chip, desc); } -static int imsic_starting_cpu(unsigned int cpu) +static void imsic_hw_states_init(void) { - /* Mark per-CPU IMSIC state as online */ - imsic_state_online(); - - /* Enable per-CPU parent interrupt */ - enable_percpu_irq(imsic_parent_irq, irq_get_trigger_type(imsic_parent_irq)); - /* Setup IPIs */ imsic_ipi_starting_cpu(); @@ -147,6 +142,18 @@ static int imsic_starting_cpu(unsigned int cpu) /* Enable local interrupt delivery */ imsic_local_delivery(true); +} + +static int imsic_starting_cpu(unsigned int cpu) +{ + /* Mark per-CPU IMSIC state as online */ + imsic_state_online(); + + /* Enable per-CPU parent interrupt */ + enable_percpu_irq(imsic_parent_irq, irq_get_trigger_type(imsic_parent_irq)); + + /* Initialize the IMSIC registers to enable the interrupt delivery */ + imsic_hw_states_init(); return 0; } @@ -162,6 +169,22 @@ static int imsic_dying_cpu(unsigned int cpu) return 0; } +static int imsic_pm_notifier(struct notifier_block *self, unsigned long cmd, void *v) +{ + switch (cmd) { + case CPU_PM_EXIT: + /* Initialize the IMSIC registers to enable the interrupt delivery */ + imsic_hw_states_init(); + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block imsic_pm_notifier_block = { + .notifier_call = imsic_pm_notifier, +}; + static int __init imsic_early_probe(struct fwnode_handle *fwnode) { struct irq_domain *domain; @@ -199,7 +222,7 @@ static int __init imsic_early_probe(struct fwnode_handle *fwnode) cpuhp_setup_state(CPUHP_AP_IRQ_RISCV_IMSIC_STARTING, "irqchip/riscv/imsic:starting", imsic_starting_cpu, imsic_dying_cpu); - return 0; + return cpu_pm_register_notifier(&imsic_pm_notifier_block); } static int __init imsic_early_dt_init(struct device_node *node, struct device_node *parent) diff --git a/drivers/irqchip/irq-sifive-plic.c b/drivers/irqchip/irq-sifive-plic.c index cbd7697bc1481..0799c15c745d4 100644 --- a/drivers/irqchip/irq-sifive-plic.c +++ b/drivers/irqchip/irq-sifive-plic.c @@ -154,8 +154,13 @@ static void plic_irq_disable(struct irq_data *d) static void plic_irq_eoi(struct irq_data *d) { struct plic_handler *handler = this_cpu_ptr(&plic_handlers); + u32 __iomem *reg; + bool enabled; + + reg = handler->enable_base + (d->hwirq / 32) * sizeof(u32); + enabled = readl(reg) & BIT(d->hwirq % 32); - if (unlikely(irqd_irq_disabled(d))) { + if (unlikely(!enabled)) { plic_toggle(handler, d->hwirq, 1); writel(d->hwirq, handler->hart_base + CONTEXT_CLAIM); plic_toggle(handler, d->hwirq, 0); diff --git a/drivers/leds/leds-expresswire.c b/drivers/leds/leds-expresswire.c index bb69be228a6d3..25c6b159a6ee9 100644 --- a/drivers/leds/leds-expresswire.c +++ b/drivers/leds/leds-expresswire.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -16,37 +17,41 @@ void expresswire_power_off(struct expresswire_common_props *props) { gpiod_set_value_cansleep(props->ctrl_gpio, 0); - usleep_range(props->timing.poweroff_us, props->timing.poweroff_us * 2); + fsleep(props->timing.poweroff_us); } EXPORT_SYMBOL_NS_GPL(expresswire_power_off, "EXPRESSWIRE"); void expresswire_enable(struct expresswire_common_props *props) { + unsigned long flags; + + local_irq_save(flags); + gpiod_set_value(props->ctrl_gpio, 1); udelay(props->timing.detect_delay_us); gpiod_set_value(props->ctrl_gpio, 0); udelay(props->timing.detect_us); gpiod_set_value(props->ctrl_gpio, 1); + + local_irq_restore(flags); } EXPORT_SYMBOL_NS_GPL(expresswire_enable, "EXPRESSWIRE"); -void expresswire_start(struct expresswire_common_props *props) +static void expresswire_start(struct expresswire_common_props *props) { gpiod_set_value(props->ctrl_gpio, 1); udelay(props->timing.data_start_us); } -EXPORT_SYMBOL_NS_GPL(expresswire_start, "EXPRESSWIRE"); -void expresswire_end(struct expresswire_common_props *props) +static void expresswire_end(struct expresswire_common_props *props) { gpiod_set_value(props->ctrl_gpio, 0); udelay(props->timing.end_of_data_low_us); gpiod_set_value(props->ctrl_gpio, 1); udelay(props->timing.end_of_data_high_us); } -EXPORT_SYMBOL_NS_GPL(expresswire_end, "EXPRESSWIRE"); -void expresswire_set_bit(struct expresswire_common_props *props, bool bit) +static void expresswire_set_bit(struct expresswire_common_props *props, bool bit) { if (bit) { gpiod_set_value(props->ctrl_gpio, 0); @@ -60,13 +65,18 @@ void expresswire_set_bit(struct expresswire_common_props *props, bool bit) udelay(props->timing.short_bitset_us); } } -EXPORT_SYMBOL_NS_GPL(expresswire_set_bit, "EXPRESSWIRE"); void expresswire_write_u8(struct expresswire_common_props *props, u8 val) { + unsigned long flags; + + local_irq_save(flags); + expresswire_start(props); for (int i = 7; i >= 0; i--) expresswire_set_bit(props, val & BIT(i)); expresswire_end(props); + + local_irq_restore(flags); } EXPORT_SYMBOL_NS_GPL(expresswire_write_u8, "EXPRESSWIRE"); diff --git a/drivers/leds/rgb/leds-qcom-lpg.c b/drivers/leds/rgb/leds-qcom-lpg.c index e197f548cddb0..a460782dadca4 100644 --- a/drivers/leds/rgb/leds-qcom-lpg.c +++ b/drivers/leds/rgb/leds-qcom-lpg.c @@ -369,7 +369,7 @@ static int lpg_lut_store(struct lpg *lpg, struct led_pattern *pattern, { unsigned int idx; u16 val; - int i; + int i, ret; idx = bitmap_find_next_zero_area(lpg->lut_bitmap, lpg->lut_size, 0, len, 0); @@ -379,8 +379,10 @@ static int lpg_lut_store(struct lpg *lpg, struct led_pattern *pattern, for (i = 0; i < len; i++) { val = pattern[i].brightness; - regmap_bulk_write(lpg->map, lpg->lut_base + LPG_LUT_REG(idx + i), - &val, sizeof(val)); + ret = regmap_bulk_write(lpg->map, lpg->lut_base + LPG_LUT_REG(idx + i), + &val, sizeof(val)); + if (ret) + return ret; } bitmap_set(lpg->lut_bitmap, idx, len); diff --git a/drivers/mailbox/bcm-flexrm-mailbox.c b/drivers/mailbox/bcm-flexrm-mailbox.c index 41f79e51d9e5a..4255fefc3a5a0 100644 --- a/drivers/mailbox/bcm-flexrm-mailbox.c +++ b/drivers/mailbox/bcm-flexrm-mailbox.c @@ -1173,14 +1173,6 @@ static int flexrm_debugfs_stats_show(struct seq_file *file, void *offset) /* ====== FlexRM interrupt handler ===== */ -static irqreturn_t flexrm_irq_event(int irq, void *dev_id) -{ - /* We only have MSI for completions so just wakeup IRQ thread */ - /* Ring related errors will be informed via completion descriptors */ - - return IRQ_WAKE_THREAD; -} - static irqreturn_t flexrm_irq_thread(int irq, void *dev_id) { flexrm_process_completions(dev_id); @@ -1271,10 +1263,8 @@ static int flexrm_startup(struct mbox_chan *chan) ret = -ENODEV; goto fail_free_cmpl_memory; } - ret = request_threaded_irq(ring->irq, - flexrm_irq_event, - flexrm_irq_thread, - 0, dev_name(ring->mbox->dev), ring); + ret = request_threaded_irq(ring->irq, NULL, flexrm_irq_thread, + IRQF_ONESHOT, dev_name(ring->mbox->dev), ring); if (ret) { dev_err(ring->mbox->dev, "failed to request ring%d IRQ\n", ring->num); diff --git a/drivers/mailbox/imx-mailbox.c b/drivers/mailbox/imx-mailbox.c index 6778afc64a048..003f9236c35e0 100644 --- a/drivers/mailbox/imx-mailbox.c +++ b/drivers/mailbox/imx-mailbox.c @@ -122,6 +122,7 @@ struct imx_mu_dcfg { u32 xRR; /* Receive Register0 */ u32 xSR[IMX_MU_xSR_MAX]; /* Status Registers */ u32 xCR[IMX_MU_xCR_MAX]; /* Control Registers */ + bool skip_suspend_flag; }; #define IMX_MU_xSR_GIPn(type, x) (type & IMX_MU_V2 ? BIT(x) : BIT(28 + (3 - (x)))) @@ -988,6 +989,7 @@ static const struct imx_mu_dcfg imx_mu_cfg_imx7ulp = { .xRR = 0x40, .xSR = {0x60, 0x60, 0x60, 0x60}, .xCR = {0x64, 0x64, 0x64, 0x64, 0x64}, + .skip_suspend_flag = true, }; static const struct imx_mu_dcfg imx_mu_cfg_imx8ulp = { @@ -1071,7 +1073,8 @@ static int __maybe_unused imx_mu_suspend_noirq(struct device *dev) priv->xcr[i] = imx_mu_read(priv, priv->dcfg->xCR[i]); } - priv->suspend = true; + if (!priv->dcfg->skip_suspend_flag) + priv->suspend = true; return 0; } @@ -1094,7 +1097,8 @@ static int __maybe_unused imx_mu_resume_noirq(struct device *dev) imx_mu_write(priv, priv->xcr[i], priv->dcfg->xCR[i]); } - priv->suspend = false; + if (!priv->dcfg->skip_suspend_flag) + priv->suspend = false; return 0; } diff --git a/drivers/mailbox/mailbox-mchp-ipc-sbi.c b/drivers/mailbox/mailbox-mchp-ipc-sbi.c index a6e52009a4245..b87bf2fb4b9b9 100644 --- a/drivers/mailbox/mailbox-mchp-ipc-sbi.c +++ b/drivers/mailbox/mailbox-mchp-ipc-sbi.c @@ -174,26 +174,30 @@ static irqreturn_t mchp_ipc_cluster_aggr_isr(int irq, void *data) struct mchp_ipc_msg ipc_msg; struct mchp_ipc_status status_msg; int ret; - unsigned long hartid; u32 i, chan_index, chan_id; + bool found = false; /* Find out the hart that originated the irq */ for_each_online_cpu(i) { - hartid = cpuid_to_hartid_map(i); - if (irq == ipc->cluster_cfg[hartid].irq) + if (irq == ipc->cluster_cfg[i].irq) { + found = true; break; + } } - status_msg.cluster = hartid; - memcpy(ipc->cluster_cfg[hartid].buf_base, &status_msg, sizeof(struct mchp_ipc_status)); + if (unlikely(!found)) + return IRQ_NONE; - ret = mchp_ipc_sbi_send(SBI_EXT_IPC_STATUS, ipc->cluster_cfg[hartid].buf_base_addr); + status_msg.cluster = cpuid_to_hartid_map(i); + memcpy(ipc->cluster_cfg[i].buf_base, &status_msg, sizeof(struct mchp_ipc_status)); + + ret = mchp_ipc_sbi_send(SBI_EXT_IPC_STATUS, ipc->cluster_cfg[i].buf_base_addr); if (ret < 0) { dev_err_ratelimited(ipc->dev, "could not get IHC irq status ret=%d\n", ret); return IRQ_HANDLED; } - memcpy(&status_msg, ipc->cluster_cfg[hartid].buf_base, sizeof(struct mchp_ipc_status)); + memcpy(&status_msg, ipc->cluster_cfg[i].buf_base, sizeof(struct mchp_ipc_status)); /* * Iterate over each bit set in the IHC interrupt status register (IRQ_STATUS) to identify @@ -321,13 +325,6 @@ static int mchp_ipc_startup(struct mbox_chan *chan) goto fail_free_buf_msg_rx; } - if (ret) { - dev_err(ipc->dev, "failed to register interrupt(s)\n"); - goto fail_free_buf_msg_rx; - } - - return ret; - fail_free_buf_msg_rx: kfree(chan_info->msg_buf_rx); fail_free_buf_msg_tx: @@ -385,21 +382,21 @@ static int mchp_ipc_get_cluster_aggr_irq(struct mchp_ipc_sbi_mbox *ipc) if (ret <= 0) continue; - ipc->cluster_cfg[hartid].irq = ret; - ret = devm_request_irq(ipc->dev, ipc->cluster_cfg[hartid].irq, + ipc->cluster_cfg[cpuid].irq = ret; + ret = devm_request_irq(ipc->dev, ipc->cluster_cfg[cpuid].irq, mchp_ipc_cluster_aggr_isr, IRQF_SHARED, "miv-ihc-irq", ipc); if (ret) return ret; - ipc->cluster_cfg[hartid].buf_base = devm_kmalloc(ipc->dev, - sizeof(struct mchp_ipc_status), - GFP_KERNEL); + ipc->cluster_cfg[cpuid].buf_base = devm_kmalloc(ipc->dev, + sizeof(struct mchp_ipc_status), + GFP_KERNEL); - if (!ipc->cluster_cfg[hartid].buf_base) + if (!ipc->cluster_cfg[cpuid].buf_base) return -ENOMEM; - ipc->cluster_cfg[hartid].buf_base_addr = __pa(ipc->cluster_cfg[hartid].buf_base); + ipc->cluster_cfg[cpuid].buf_base_addr = __pa(ipc->cluster_cfg[cpuid].buf_base); irq_found = true; } @@ -419,7 +416,7 @@ static int mchp_ipc_probe(struct platform_device *pdev) ret = sbi_probe_extension(SBI_EXT_MICROCHIP_TECHNOLOGY); if (ret <= 0) - return dev_err_probe(dev, ret, "Microchip SBI extension not detected\n"); + return dev_err_probe(dev, -ENODEV, "Microchip SBI extension not detected\n"); ipc = devm_kzalloc(dev, sizeof(*ipc), GFP_KERNEL); if (!ipc) diff --git a/drivers/mailbox/mailbox.c b/drivers/mailbox/mailbox.c index 2acc6ec229a45..617ba505691d3 100644 --- a/drivers/mailbox/mailbox.c +++ b/drivers/mailbox/mailbox.c @@ -489,12 +489,10 @@ EXPORT_SYMBOL_GPL(mbox_free_channel); static struct mbox_chan *fw_mbox_index_xlate(struct mbox_controller *mbox, const struct fwnode_reference_args *sp) { - int ind = sp->args[0]; - - if (ind >= mbox->num_chans) + if (sp->nargs < 1 || sp->args[0] >= mbox->num_chans) return ERR_PTR(-EINVAL); - return &mbox->chans[ind]; + return &mbox->chans[sp->args[0]]; } /** diff --git a/drivers/mailbox/pcc.c b/drivers/mailbox/pcc.c index ff292b9e0be9e..713022aed2e2f 100644 --- a/drivers/mailbox/pcc.c +++ b/drivers/mailbox/pcc.c @@ -305,22 +305,6 @@ static void pcc_chan_acknowledge(struct pcc_chan_info *pchan) pcc_chan_reg_read_modify_write(&pchan->db); } -static void *write_response(struct pcc_chan_info *pchan) -{ - struct pcc_header pcc_header; - void *buffer; - int data_len; - - memcpy_fromio(&pcc_header, pchan->chan.shmem, - sizeof(pcc_header)); - data_len = pcc_header.length - sizeof(u32) + sizeof(struct pcc_header); - - buffer = pchan->chan.rx_alloc(pchan->chan.mchan->cl, data_len); - if (buffer != NULL) - memcpy_fromio(buffer, pchan->chan.shmem, data_len); - return buffer; -} - /** * pcc_mbox_irq - PCC mailbox interrupt handler * @irq: interrupt number @@ -332,8 +316,6 @@ static irqreturn_t pcc_mbox_irq(int irq, void *p) { struct pcc_chan_info *pchan; struct mbox_chan *chan = p; - struct pcc_header *pcc_header = chan->active_req; - void *handle = NULL; pchan = chan->con_priv; @@ -357,17 +339,7 @@ static irqreturn_t pcc_mbox_irq(int irq, void *p) * required to avoid any possible race in updatation of this flag. */ pchan->chan_in_use = false; - - if (pchan->chan.rx_alloc) - handle = write_response(pchan); - - if (chan->active_req) { - pcc_header = chan->active_req; - if (pcc_header->flags & PCC_CMD_COMPLETION_NOTIFY) - mbox_chan_txdone(chan, 0); - } - - mbox_chan_received_data(chan, handle); + mbox_chan_received_data(chan, NULL); pcc_chan_acknowledge(pchan); @@ -411,24 +383,9 @@ pcc_mbox_request_channel(struct mbox_client *cl, int subspace_id) pcc_mchan = &pchan->chan; pcc_mchan->shmem = acpi_os_ioremap(pcc_mchan->shmem_base_addr, pcc_mchan->shmem_size); - if (!pcc_mchan->shmem) - goto err; - - pcc_mchan->manage_writes = false; - - /* This indicates that the channel is ready to accept messages. - * This needs to happen after the channel has registered - * its callback. There is no access point to do that in - * the mailbox API. That implies that the mailbox client must - * have set the allocate callback function prior to - * sending any messages. - */ - if (pchan->type == ACPI_PCCT_TYPE_EXT_PCC_SLAVE_SUBSPACE) - pcc_chan_reg_read_modify_write(&pchan->cmd_update); - - return pcc_mchan; + if (pcc_mchan->shmem) + return pcc_mchan; -err: mbox_free_channel(chan); return ERR_PTR(-ENXIO); } @@ -459,38 +416,8 @@ void pcc_mbox_free_channel(struct pcc_mbox_chan *pchan) } EXPORT_SYMBOL_GPL(pcc_mbox_free_channel); -static int pcc_write_to_buffer(struct mbox_chan *chan, void *data) -{ - struct pcc_chan_info *pchan = chan->con_priv; - struct pcc_mbox_chan *pcc_mbox_chan = &pchan->chan; - struct pcc_header *pcc_header = data; - - if (!pchan->chan.manage_writes) - return 0; - - /* The PCC header length includes the command field - * but not the other values from the header. - */ - int len = pcc_header->length - sizeof(u32) + sizeof(struct pcc_header); - u64 val; - - pcc_chan_reg_read(&pchan->cmd_complete, &val); - if (!val) { - pr_info("%s pchan->cmd_complete not set", __func__); - return -1; - } - memcpy_toio(pcc_mbox_chan->shmem, data, len); - return 0; -} - - /** - * pcc_send_data - Called from Mailbox Controller code. If - * pchan->chan.rx_alloc is set, then the command complete - * flag is checked and the data is written to the shared - * buffer io memory. - * - * If pchan->chan.rx_alloc is not set, then it is used + * pcc_send_data - Called from Mailbox Controller code. Used * here only to ring the channel doorbell. The PCC client * specific read/write is done in the client driver in * order to maintain atomicity over PCC channel once @@ -506,37 +433,17 @@ static int pcc_send_data(struct mbox_chan *chan, void *data) int ret; struct pcc_chan_info *pchan = chan->con_priv; - ret = pcc_write_to_buffer(chan, data); - if (ret) - return ret; - ret = pcc_chan_reg_read_modify_write(&pchan->cmd_update); if (ret) return ret; ret = pcc_chan_reg_read_modify_write(&pchan->db); - if (!ret && pchan->plat_irq > 0) pchan->chan_in_use = true; return ret; } - -static bool pcc_last_tx_done(struct mbox_chan *chan) -{ - struct pcc_chan_info *pchan = chan->con_priv; - u64 val; - - pcc_chan_reg_read(&pchan->cmd_complete, &val); - if (!val) - return false; - else - return true; -} - - - /** * pcc_startup - Called from Mailbox Controller code. Used here * to request the interrupt. @@ -552,7 +459,7 @@ static int pcc_startup(struct mbox_chan *chan) if (pchan->plat_irq > 0) { irqflags = pcc_chan_plat_irq_can_be_shared(pchan) ? - IRQF_SHARED | IRQF_ONESHOT : 0; + IRQF_SHARED : 0; rc = devm_request_irq(chan->mbox->dev, pchan->plat_irq, pcc_mbox_irq, irqflags, MBOX_IRQ_NAME, chan); if (unlikely(rc)) { @@ -582,7 +489,6 @@ static const struct mbox_chan_ops pcc_chan_ops = { .send_data = pcc_send_data, .startup = pcc_startup, .shutdown = pcc_shutdown, - .last_tx_done = pcc_last_tx_done, }; /** diff --git a/drivers/mailbox/sprd-mailbox.c b/drivers/mailbox/sprd-mailbox.c index ee8539dfcef54..46d0c34177ab9 100644 --- a/drivers/mailbox/sprd-mailbox.c +++ b/drivers/mailbox/sprd-mailbox.c @@ -166,6 +166,11 @@ static irqreturn_t sprd_mbox_inbox_isr(int irq, void *data) return IRQ_NONE; } + /* Clear FIFO delivery and overflow status first */ + writel(fifo_sts & + (SPRD_INBOX_FIFO_DELIVER_MASK | SPRD_INBOX_FIFO_OVERLOW_MASK), + priv->inbox_base + SPRD_MBOX_FIFO_RST); + while (send_sts) { id = __ffs(send_sts); send_sts &= (send_sts - 1); @@ -181,11 +186,6 @@ static irqreturn_t sprd_mbox_inbox_isr(int irq, void *data) mbox_chan_txdone(chan, 0); } - /* Clear FIFO delivery and overflow status */ - writel(fifo_sts & - (SPRD_INBOX_FIFO_DELIVER_MASK | SPRD_INBOX_FIFO_OVERLOW_MASK), - priv->inbox_base + SPRD_MBOX_FIFO_RST); - /* Clear irq status */ writel(SPRD_MBOX_IRQ_CLR, priv->inbox_base + SPRD_MBOX_IRQ_STS); @@ -243,21 +243,19 @@ static int sprd_mbox_startup(struct mbox_chan *chan) /* Select outbox FIFO mode and reset the outbox FIFO status */ writel(0x0, priv->outbox_base + SPRD_MBOX_FIFO_RST); - /* Enable inbox FIFO overflow and delivery interrupt */ - val = readl(priv->inbox_base + SPRD_MBOX_IRQ_MSK); - val &= ~(SPRD_INBOX_FIFO_OVERFLOW_IRQ | SPRD_INBOX_FIFO_DELIVER_IRQ); + /* Enable inbox FIFO delivery interrupt */ + val = SPRD_INBOX_FIFO_IRQ_MASK; + val &= ~SPRD_INBOX_FIFO_DELIVER_IRQ; writel(val, priv->inbox_base + SPRD_MBOX_IRQ_MSK); /* Enable outbox FIFO not empty interrupt */ - val = readl(priv->outbox_base + SPRD_MBOX_IRQ_MSK); + val = SPRD_OUTBOX_FIFO_IRQ_MASK; val &= ~SPRD_OUTBOX_FIFO_NOT_EMPTY_IRQ; writel(val, priv->outbox_base + SPRD_MBOX_IRQ_MSK); /* Enable supplementary outbox as the fundamental one */ if (priv->supp_base) { writel(0x0, priv->supp_base + SPRD_MBOX_FIFO_RST); - val = readl(priv->supp_base + SPRD_MBOX_IRQ_MSK); - val &= ~SPRD_OUTBOX_FIFO_NOT_EMPTY_IRQ; writel(val, priv->supp_base + SPRD_MBOX_IRQ_MSK); } } diff --git a/drivers/mcb/mcb-core.c b/drivers/mcb/mcb-core.c index c1367223e71a0..3d487d75c483d 100644 --- a/drivers/mcb/mcb-core.c +++ b/drivers/mcb/mcb-core.c @@ -85,7 +85,8 @@ static void mcb_remove(struct device *dev) struct mcb_device *mdev = to_mcb_device(dev); struct module *carrier_mod; - mdrv->remove(mdev); + if (mdrv->remove) + mdrv->remove(mdev); carrier_mod = mdev->dev.parent->driver->owner; module_put(carrier_mod); @@ -176,13 +177,13 @@ static const struct device_type mcb_carrier_device_type = { * @owner: The @mcb_driver's module * @mod_name: The name of the @mcb_driver's module * - * Register a @mcb_driver at the system. Perform some sanity checks, if - * the .probe and .remove methods are provided by the driver. + * Register a @mcb_driver at the system. Perform a sanity check, if + * .probe method is provided by the driver. */ int __mcb_register_driver(struct mcb_driver *drv, struct module *owner, const char *mod_name) { - if (!drv->probe || !drv->remove) + if (!drv->probe) return -EINVAL; drv->driver.owner = owner; diff --git a/drivers/md/dm-exception-store.c b/drivers/md/dm-exception-store.c index c3799757bf4a0..88f119a0a2ae0 100644 --- a/drivers/md/dm-exception-store.c +++ b/drivers/md/dm-exception-store.c @@ -116,7 +116,7 @@ int dm_exception_store_type_register(struct dm_exception_store_type *type) if (!__find_exception_store_type(type->name)) list_add(&type->list, &_exception_store_types); else - r = -EEXIST; + r = -EBUSY; spin_unlock(&_lock); return r; diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index 170bf67a2edd9..ba52631052503 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -2411,7 +2411,7 @@ static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); if (unlikely(new_pos != NOT_FOUND) || - unlikely(next_sector < dio->range.logical_sector - dio->range.n_sectors)) { + unlikely(next_sector < dio->range.logical_sector + dio->range.n_sectors)) { remove_range_unlocked(ic, &dio->range); spin_unlock_irq(&ic->endio_wait.lock); queue_work(ic->commit_wq, &ic->commit_work); @@ -3788,14 +3788,27 @@ static void dm_integrity_resume(struct dm_target *ti) struct dm_integrity_c *ic = ti->private; __u64 old_provided_data_sectors = le64_to_cpu(ic->sb->provided_data_sectors); int r; + __le32 flags; DEBUG_print("resume\n"); ic->wrote_to_journal = false; + flags = ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING); + r = sync_rw_sb(ic, REQ_OP_READ); + if (r) + dm_integrity_io_error(ic, "reading superblock", r); + if ((ic->sb->flags & flags) != flags) { + ic->sb->flags |= flags; + r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA); + if (unlikely(r)) + dm_integrity_io_error(ic, "writing superblock", r); + } + if (ic->provided_data_sectors != old_provided_data_sectors) { if (ic->provided_data_sectors > old_provided_data_sectors && ic->mode == 'B' && + ic->sb->flags & cpu_to_le32(SB_FLAG_DIRTY_BITMAP) && ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit) { rw_journal_sectors(ic, REQ_OP_READ, 0, ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); diff --git a/drivers/md/dm-log.c b/drivers/md/dm-log.c index 9d85d045f9d9d..bced5a783ee33 100644 --- a/drivers/md/dm-log.c +++ b/drivers/md/dm-log.c @@ -121,7 +121,7 @@ int dm_dirty_log_type_register(struct dm_dirty_log_type *type) if (!__find_dirty_log_type(type->name)) list_add(&type->list, &_log_types); else - r = -EEXIST; + r = -EBUSY; spin_unlock(&_lock); return r; diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c index aaf4a0a4b0ebb..7d3fdd96f4edf 100644 --- a/drivers/md/dm-mpath.c +++ b/drivers/md/dm-mpath.c @@ -225,6 +225,7 @@ static struct multipath *alloc_multipath(struct dm_target *ti) mutex_init(&m->work_mutex); m->queue_mode = DM_TYPE_NONE; + m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT; m->ti = ti; ti->private = m; @@ -257,7 +258,6 @@ static int alloc_multipath_stage2(struct dm_target *ti, struct multipath *m) set_bit(MPATHF_QUEUE_IO, &m->flags); atomic_set(&m->pg_init_in_progress, 0); atomic_set(&m->pg_init_count, 0); - m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT; init_waitqueue_head(&m->pg_init_wait); init_waitqueue_head(&m->probe_wait); diff --git a/drivers/md/dm-path-selector.c b/drivers/md/dm-path-selector.c index d0b883fabfeb6..2b0ac200f1c02 100644 --- a/drivers/md/dm-path-selector.c +++ b/drivers/md/dm-path-selector.c @@ -107,7 +107,7 @@ int dm_register_path_selector(struct path_selector_type *pst) if (__find_path_selector_type(pst->name)) { kfree(psi); - r = -EEXIST; + r = -EBUSY; } else list_add(&psi->list, &_path_selectors); diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c index a6ca92049c10e..923252fb57aec 100644 --- a/drivers/md/dm-rq.c +++ b/drivers/md/dm-rq.c @@ -109,14 +109,21 @@ static void end_clone_bio(struct bio *clone) */ tio->completed += nr_bytes; + if (!is_last) + return; + /* + * At this moment we know this is the last bio of the cloned request, + * and all cloned bios have been released, so reset the clone request's + * bio pointer to avoid double free. + */ + tio->clone->bio = NULL; + exit: /* * Update the original request. * Do not use blk_mq_end_request() here, because it may complete * the original request before the clone, and break the ordering. */ - if (is_last) - exit: - blk_update_request(tio->orig, BLK_STS_OK, tio->completed); + blk_update_request(tio->orig, BLK_STS_OK, tio->completed); } static struct dm_rq_target_io *tio_from_request(struct request *rq) @@ -278,8 +285,7 @@ static void dm_complete_request(struct request *rq, blk_status_t error) struct dm_rq_target_io *tio = tio_from_request(rq); tio->error = error; - if (likely(!blk_should_fake_timeout(rq->q))) - blk_mq_complete_request(rq); + blk_mq_complete_request(rq); } /* diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c index ad0a60a07b935..81265ed204b54 100644 --- a/drivers/md/dm-table.c +++ b/drivers/md/dm-table.c @@ -1237,9 +1237,6 @@ static int dm_wrappedkey_op_callback(struct dm_target *ti, struct dm_dev *dev, bdev_get_queue(bdev)->crypto_profile; int err = -EOPNOTSUPP; - if (!args->err) - return 0; - switch (args->op) { case DERIVE_SW_SECRET: err = blk_crypto_derive_sw_secret( @@ -1266,9 +1263,7 @@ static int dm_wrappedkey_op_callback(struct dm_target *ti, struct dm_dev *dev, break; } args->err = err; - - /* Try another device in case this fails. */ - return 0; + return 1; /* No need to continue the iteration. */ } static int dm_exec_wrappedkey_op(struct blk_crypto_profile *profile, @@ -1294,14 +1289,13 @@ static int dm_exec_wrappedkey_op(struct blk_crypto_profile *profile, * declared on all underlying devices. Thus, all the underlying devices * should support all wrapped key operations and they should behave * identically, i.e. work with the same keys. So, just executing the - * operation on the first device on which it works suffices for now. + * operation on the first device suffices for now. */ for (i = 0; i < t->num_targets; i++) { ti = dm_table_get_target(t, i); if (!ti->type->iterate_devices) continue; - ti->type->iterate_devices(ti, dm_wrappedkey_op_callback, args); - if (!args->err) + if (ti->type->iterate_devices(ti, dm_wrappedkey_op_callback, args) != 0) break; } out: diff --git a/drivers/md/dm-target.c b/drivers/md/dm-target.c index 8fede41adec00..1fd41289de367 100644 --- a/drivers/md/dm-target.c +++ b/drivers/md/dm-target.c @@ -88,7 +88,7 @@ int dm_register_target(struct target_type *tt) if (__find_target_type(tt->name)) { DMERR("%s: '%s' target already registered", __func__, tt->name); - rv = -EEXIST; + rv = -EBUSY; } else { list_add(&tt->list, &_targets); } diff --git a/drivers/md/dm-unstripe.c b/drivers/md/dm-unstripe.c index e8a9432057dce..17be483595642 100644 --- a/drivers/md/dm-unstripe.c +++ b/drivers/md/dm-unstripe.c @@ -117,7 +117,7 @@ static void unstripe_dtr(struct dm_target *ti) static sector_t map_to_core(struct dm_target *ti, struct bio *bio) { struct unstripe_c *uc = ti->private; - sector_t sector = bio->bi_iter.bi_sector; + sector_t sector = dm_target_offset(ti, bio->bi_iter.bi_sector); sector_t tmp_sector = sector; /* Shift us up to the right "row" on the stripe */ diff --git a/drivers/md/dm-verity-fec.c b/drivers/md/dm-verity-fec.c index e41bde1d3b15b..5365b3987374a 100644 --- a/drivers/md/dm-verity-fec.c +++ b/drivers/md/dm-verity-fec.c @@ -533,9 +533,9 @@ void verity_fec_dtr(struct dm_verity *v) mempool_exit(&f->output_pool); kmem_cache_destroy(f->cache); - if (f->data_bufio) + if (!IS_ERR_OR_NULL(f->data_bufio)) dm_bufio_client_destroy(f->data_bufio); - if (f->bufio) + if (!IS_ERR_OR_NULL(f->bufio)) dm_bufio_client_destroy(f->bufio); if (f->dev) diff --git a/drivers/md/dm-zone.c b/drivers/md/dm-zone.c index 78e17dd4d01b8..f4950b5f766d3 100644 --- a/drivers/md/dm-zone.c +++ b/drivers/md/dm-zone.c @@ -56,7 +56,7 @@ int dm_blk_report_zones(struct gendisk *disk, sector_t sector, { struct mapped_device *md = disk->private_data; struct dm_table *map; - struct dm_table *zone_revalidate_map = md->zone_revalidate_map; + struct dm_table *zone_revalidate_map = READ_ONCE(md->zone_revalidate_map); int srcu_idx, ret = -EIO; bool put_table = false; @@ -66,11 +66,13 @@ int dm_blk_report_zones(struct gendisk *disk, sector_t sector, * Zone revalidation during __bind() is in progress, but this * call is from a different process */ - if (dm_suspended_md(md)) - return -EAGAIN; - map = dm_get_live_table(md, &srcu_idx); put_table = true; + + if (dm_suspended_md(md)) { + ret = -EAGAIN; + goto do_put_table; + } } else { /* Zone revalidation during __bind() */ map = zone_revalidate_map; @@ -80,6 +82,7 @@ int dm_blk_report_zones(struct gendisk *disk, sector_t sector, ret = dm_blk_do_report_zones(md, map, sector, nr_zones, cb, data); +do_put_table: if (put_table) dm_put_live_table(md, srcu_idx); diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 6c83ab940af75..52f01c44e73a6 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -1363,6 +1363,8 @@ void dm_submit_bio_remap(struct bio *clone, struct bio *tgt_clone) if (!tgt_clone) tgt_clone = clone; + bio_clone_blkg_association(tgt_clone, io->orig_bio); + /* * Account io->origin_bio to DM dev on behalf of target * that took ownership of IO with DM_MAPIO_SUBMITTED. diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index 84b7e2af6dbaa..7bb56d0491a2f 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -2453,6 +2453,7 @@ static int __bitmap_resize(struct bitmap *bitmap, sector_t blocks, memcpy(page_address(store.sb_page), page_address(bitmap->storage.sb_page), sizeof(bitmap_super_t)); + mutex_lock(&bitmap->mddev->bitmap_info.mutex); spin_lock_irq(&bitmap->counts.lock); md_bitmap_file_unmap(&bitmap->storage); bitmap->storage = store; @@ -2560,7 +2561,7 @@ static int __bitmap_resize(struct bitmap *bitmap, sector_t blocks, set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY); } spin_unlock_irq(&bitmap->counts.lock); - + mutex_unlock(&bitmap->mddev->bitmap_info.mutex); if (!init) { __bitmap_unplug(bitmap); bitmap->mddev->pers->quiesce(bitmap->mddev, 0); diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c index 11f1e91d387d8..896279988dfd5 100644 --- a/drivers/md/md-cluster.c +++ b/drivers/md/md-cluster.c @@ -549,8 +549,13 @@ static void process_metadata_update(struct mddev *mddev, struct cluster_msg *msg dlm_lock_sync(cinfo->no_new_dev_lockres, DLM_LOCK_CR); - /* daemaon thread must exist */ thread = rcu_dereference_protected(mddev->thread, true); + if (!thread) { + pr_warn("md-cluster: Received metadata update but MD thread is not ready\n"); + dlm_unlock_sync(cinfo->no_new_dev_lockres); + return; + } + wait_event(thread->wqueue, (got_lock = mddev_trylock(mddev)) || test_bit(MD_CLUSTER_HOLDING_MUTEX_FOR_RECVD, &cinfo->state)); diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index 1eb434306162a..bcb6eae1c711f 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -712,8 +712,10 @@ static int llbitmap_suspend_timeout(struct llbitmap *llbitmap, int page_idx) percpu_ref_kill(&pctl->active); if (!wait_event_timeout(pctl->wait, percpu_ref_is_zero(&pctl->active), - llbitmap->mddev->bitmap_info.daemon_sleep * HZ)) + llbitmap->mddev->bitmap_info.daemon_sleep * HZ)) { + percpu_ref_resurrect(&pctl->active); return -ETIMEDOUT; + } return 0; } diff --git a/drivers/md/md.c b/drivers/md/md.c index e04ddcb03981c..92ec4be20db85 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -6716,13 +6716,15 @@ static void __md_stop_writes(struct mddev *mddev) { timer_delete_sync(&mddev->safemode_timer); - if (mddev->pers && mddev->pers->quiesce) { - mddev->pers->quiesce(mddev, 1); - mddev->pers->quiesce(mddev, 0); - } + if (md_is_rdwr(mddev) || !mddev_is_dm(mddev)) { + if (mddev->pers && mddev->pers->quiesce) { + mddev->pers->quiesce(mddev, 1); + mddev->pers->quiesce(mddev, 0); + } - if (md_bitmap_enabled(mddev, true)) - mddev->bitmap_ops->flush(mddev); + if (md_bitmap_enabled(mddev, true)) + mddev->bitmap_ops->flush(mddev); + } if (md_is_rdwr(mddev) && ((!mddev->in_sync && !mddev_is_clustered(mddev)) || diff --git a/drivers/md/md.h b/drivers/md/md.h index fd6e001c1d38f..9d66afb8cc6e6 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -736,8 +736,8 @@ static inline int mddev_trylock(struct mddev *mddev) int ret; ret = mutex_trylock(&mddev->reconfig_mutex); - if (!ret && test_bit(MD_DELETED, &mddev->flags)) { - ret = -ENODEV; + if (ret && test_bit(MD_DELETED, &mddev->flags)) { + ret = 0; mutex_unlock(&mddev->reconfig_mutex); } return ret; diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 592a402330047..ce7fd68869566 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -3253,6 +3253,7 @@ static int raid1_run(struct mddev *mddev) if (!mddev_is_dm(mddev)) { ret = raid1_set_limits(mddev); if (ret) { + md_unregister_thread(mddev, &conf->thread); if (!mddev->private) raid1_free(mddev, conf); return ret; diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 14dcd5142eb47..d58ae150b4502 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -3402,7 +3402,6 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, !test_bit(In_sync, &rdev->flags)) continue; /* This is where we read from */ - any_working = 1; sector = r10_bio->devs[j].addr; if (is_badblock(rdev, sector, max_sync, @@ -3417,6 +3416,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, continue; } } + any_working = 1; bio = r10_bio->devs[0].bio; bio->bi_next = biolist; biolist = bio; diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 41de29206402a..3b711a1198adb 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -3751,9 +3751,14 @@ static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s, struct r5dev *dev = &sh->dev[disk_idx]; struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]], &sh->dev[s->failed_num[1]] }; + struct mddev *mddev = sh->raid_conf->mddev; + bool force_rcw = false; int i; - bool force_rcw = (sh->raid_conf->rmw_level == PARITY_DISABLE_RMW); + if (sh->raid_conf->rmw_level == PARITY_DISABLE_RMW || + (mddev->bitmap_ops && mddev->bitmap_ops->blocks_synced && + !mddev->bitmap_ops->blocks_synced(mddev, sh->sector))) + force_rcw = true; if (test_bit(R5_LOCKED, &dev->flags) || test_bit(R5_UPTODATE, &dev->flags)) @@ -8056,7 +8061,8 @@ static int raid5_run(struct mddev *mddev) goto abort; } - if (log_init(conf, journal_dev, raid5_has_ppl(conf))) + ret = log_init(conf, journal_dev, raid5_has_ppl(conf)); + if (ret) goto abort; return 0; diff --git a/drivers/media/dvb-core/dmxdev.c b/drivers/media/dvb-core/dmxdev.c index 151177e5a06d8..ab34be22cd1aa 100644 --- a/drivers/media/dvb-core/dmxdev.c +++ b/drivers/media/dvb-core/dmxdev.c @@ -168,7 +168,9 @@ static int dvb_dvr_open(struct inode *inode, struct file *file) mutex_unlock(&dmxdev->mutex); return -ENOMEM; } - dvb_ringbuffer_init(&dmxdev->dvr_buffer, mem, DVR_BUFFER_SIZE); + dmxdev->dvr_buffer.data = mem; + dmxdev->dvr_buffer.size = DVR_BUFFER_SIZE; + dvb_ringbuffer_reset(&dmxdev->dvr_buffer); if (dmxdev->may_do_mmap) dvb_vb2_init(&dmxdev->dvr_vb2_ctx, "dvr", file->f_flags & O_NONBLOCK); @@ -397,11 +399,11 @@ static int dvb_dmxdev_section_callback(const u8 *buffer1, size_t buffer1_len, if (dvb_vb2_is_streaming(&dmxdevfilter->vb2_ctx)) { ret = dvb_vb2_fill_buffer(&dmxdevfilter->vb2_ctx, buffer1, buffer1_len, - buffer_flags); + buffer_flags, true); if (ret == buffer1_len) ret = dvb_vb2_fill_buffer(&dmxdevfilter->vb2_ctx, buffer2, buffer2_len, - buffer_flags); + buffer_flags, true); } else { ret = dvb_dmxdev_buffer_write(&dmxdevfilter->buffer, buffer1, buffer1_len); @@ -452,10 +454,10 @@ static int dvb_dmxdev_ts_callback(const u8 *buffer1, size_t buffer1_len, if (dvb_vb2_is_streaming(ctx)) { ret = dvb_vb2_fill_buffer(ctx, buffer1, buffer1_len, - buffer_flags); + buffer_flags, false); if (ret == buffer1_len) ret = dvb_vb2_fill_buffer(ctx, buffer2, buffer2_len, - buffer_flags); + buffer_flags, false); } else { if (buffer->error) { spin_unlock(&dmxdevfilter->dev->lock); diff --git a/drivers/media/dvb-core/dvb_vb2.c b/drivers/media/dvb-core/dvb_vb2.c index 29edaaff7a5c9..7444bbc2f24d9 100644 --- a/drivers/media/dvb-core/dvb_vb2.c +++ b/drivers/media/dvb-core/dvb_vb2.c @@ -249,7 +249,8 @@ int dvb_vb2_is_streaming(struct dvb_vb2_ctx *ctx) int dvb_vb2_fill_buffer(struct dvb_vb2_ctx *ctx, const unsigned char *src, int len, - enum dmx_buffer_flags *buffer_flags) + enum dmx_buffer_flags *buffer_flags, + bool flush) { unsigned long flags = 0; void *vbuf = NULL; @@ -306,7 +307,7 @@ int dvb_vb2_fill_buffer(struct dvb_vb2_ctx *ctx, } } - if (ctx->nonblocking && ctx->buf) { + if (flush && ctx->buf) { vb2_set_plane_payload(&ctx->buf->vb, 0, ll); vb2_buffer_done(&ctx->buf->vb, VB2_BUF_STATE_DONE); list_del(&ctx->buf->list); diff --git a/drivers/media/i2c/adv7180.c b/drivers/media/i2c/adv7180.c index 378f4e6af12cb..5cbc973df684d 100644 --- a/drivers/media/i2c/adv7180.c +++ b/drivers/media/i2c/adv7180.c @@ -507,6 +507,13 @@ static int adv7180_get_frame_interval(struct v4l2_subdev *sd, fi->interval.denominator = 25; } + /* + * If the de-interlacer is active, the chip produces full video frames + * at the field rate. + */ + if (state->field == V4L2_FIELD_NONE) + fi->interval.denominator *= 2; + return 0; } diff --git a/drivers/media/i2c/ccs/ccs-core.c b/drivers/media/i2c/ccs/ccs-core.c index 1c889c878abd3..3f7162f3d1e32 100644 --- a/drivers/media/i2c/ccs/ccs-core.c +++ b/drivers/media/i2c/ccs/ccs-core.c @@ -2346,7 +2346,7 @@ static void ccs_set_compose_scaler(struct v4l2_subdev *subdev, * CCS_LIM(sensor, SCALER_N_MIN) / sel->r.height; max_m = crops[CCS_PAD_SINK]->width * CCS_LIM(sensor, SCALER_N_MIN) - / CCS_LIM(sensor, MIN_X_OUTPUT_SIZE); + / (CCS_LIM(sensor, MIN_X_OUTPUT_SIZE) ?: 1); a = clamp(a, CCS_LIM(sensor, SCALER_M_MIN), CCS_LIM(sensor, SCALER_M_MAX)); @@ -2940,6 +2940,8 @@ static void ccs_cleanup(struct ccs_sensor *sensor) ccs_free_controls(sensor); } +static const struct v4l2_subdev_internal_ops ccs_internal_ops; + static int ccs_init_subdev(struct ccs_sensor *sensor, struct ccs_subdev *ssd, const char *name, unsigned short num_pads, u32 function, @@ -2952,8 +2954,10 @@ static int ccs_init_subdev(struct ccs_sensor *sensor, if (!ssd) return 0; - if (ssd != sensor->src) + if (ssd != sensor->src) { v4l2_subdev_init(&ssd->sd, &ccs_ops); + ssd->sd.internal_ops = &ccs_internal_ops; + } ssd->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE; ssd->sd.entity.function = function; @@ -3062,6 +3066,10 @@ static const struct media_entity_operations ccs_entity_ops = { .link_validate = v4l2_subdev_link_validate, }; +static const struct v4l2_subdev_internal_ops ccs_internal_ops = { + .init_state = ccs_init_state, +}; + static const struct v4l2_subdev_internal_ops ccs_internal_src_ops = { .init_state = ccs_init_state, .registered = ccs_registered, @@ -3425,7 +3433,21 @@ static int ccs_probe(struct i2c_client *client) sensor->scale_m = CCS_LIM(sensor, SCALER_N_MIN); /* prepare PLL configuration input values */ - sensor->pll.bus_type = CCS_PLL_BUS_TYPE_CSI2_DPHY; + switch (sensor->hwcfg.csi_signalling_mode) { + case CCS_CSI_SIGNALING_MODE_CSI_2_CPHY: + sensor->pll.bus_type = CCS_PLL_BUS_TYPE_CSI2_CPHY; + break; + case CCS_CSI_SIGNALING_MODE_CSI_2_DPHY: + case SMIAPP_CSI_SIGNALLING_MODE_CCP2_DATA_CLOCK: + case SMIAPP_CSI_SIGNALLING_MODE_CCP2_DATA_STROBE: + sensor->pll.bus_type = CCS_PLL_BUS_TYPE_CSI2_DPHY; + break; + default: + dev_err(&client->dev, "unsupported signalling mode %u\n", + sensor->hwcfg.csi_signalling_mode); + rval = -EINVAL; + goto out_cleanup; + } sensor->pll.csi2.lanes = sensor->hwcfg.lanes; if (CCS_LIM(sensor, CLOCK_CALCULATION) & CCS_CLOCK_CALCULATION_LANE_SPEED) { diff --git a/drivers/media/i2c/dw9714.c b/drivers/media/i2c/dw9714.c index 1e7ad355a388c..3288de539452e 100644 --- a/drivers/media/i2c/dw9714.c +++ b/drivers/media/i2c/dw9714.c @@ -149,7 +149,7 @@ static int dw9714_power_up(struct dw9714_device *dw9714_dev) gpiod_set_value_cansleep(dw9714_dev->powerdown_gpio, 0); - usleep_range(1000, 2000); + usleep_range(12000, 14000); return 0; } diff --git a/drivers/media/i2c/mt9m114.c b/drivers/media/i2c/mt9m114.c index 51ebbe7ae9969..b1325e2cd1321 100644 --- a/drivers/media/i2c/mt9m114.c +++ b/drivers/media/i2c/mt9m114.c @@ -2360,11 +2360,17 @@ static int mt9m114_parse_dt(struct mt9m114 *sensor) struct fwnode_handle *ep; int ret; + /* + * On ACPI systems the fwnode graph can be initialized by a bridge + * driver, which may not have probed yet. Wait for this. + * + * TODO: Return an error once bridge driver code will have moved + * to the ACPI core. + */ ep = fwnode_graph_get_next_endpoint(fwnode, NULL); - if (!ep) { - dev_err(&sensor->client->dev, "No endpoint found\n"); - return -EINVAL; - } + if (!ep) + return dev_err_probe(&sensor->client->dev, -EPROBE_DEFER, + "waiting for fwnode graph endpoint\n"); sensor->bus_cfg.bus_type = V4L2_MBUS_UNKNOWN; ret = v4l2_fwnode_endpoint_alloc_parse(ep, &sensor->bus_cfg); @@ -2434,7 +2440,7 @@ static int mt9m114_probe(struct i2c_client *client) goto error_ep_free; } - sensor->reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW); + sensor->reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH); if (IS_ERR(sensor->reset)) { ret = PTR_ERR(sensor->reset); dev_err_probe(dev, ret, "Failed to get reset GPIO\n"); diff --git a/drivers/media/i2c/ov01a10.c b/drivers/media/i2c/ov01a10.c index 141cb6f75b555..c1a7373a6311c 100644 --- a/drivers/media/i2c/ov01a10.c +++ b/drivers/media/i2c/ov01a10.c @@ -16,7 +16,7 @@ #include #define OV01A10_LINK_FREQ_400MHZ 400000000ULL -#define OV01A10_SCLK 40000000LL +#define OV01A10_SCLK 80000000LL #define OV01A10_DATA_LANES 1 #define OV01A10_REG_CHIP_ID 0x300a @@ -48,7 +48,7 @@ /* analog gain controls */ #define OV01A10_REG_ANALOG_GAIN 0x3508 #define OV01A10_ANAL_GAIN_MIN 0x100 -#define OV01A10_ANAL_GAIN_MAX 0xffff +#define OV01A10_ANAL_GAIN_MAX 0x3fff #define OV01A10_ANAL_GAIN_STEP 1 /* digital gain controls */ @@ -57,7 +57,7 @@ #define OV01A10_REG_DIGITAL_GAIN_GR 0x3513 #define OV01A10_REG_DIGITAL_GAIN_R 0x3516 #define OV01A10_DGTL_GAIN_MIN 0 -#define OV01A10_DGTL_GAIN_MAX 0x3ffff +#define OV01A10_DGTL_GAIN_MAX 0x3fff #define OV01A10_DGTL_GAIN_STEP 1 #define OV01A10_DGTL_GAIN_DEFAULT 1024 @@ -75,6 +75,15 @@ #define OV01A10_REG_X_WIN 0x3811 #define OV01A10_REG_Y_WIN 0x3813 +/* + * The native ov01a10 bayer-pattern is GBRG, but there was a driver bug enabling + * hflip/mirroring by default resulting in BGGR. Because of this bug Intel's + * proprietary IPU6 userspace stack expects BGGR. So we report BGGR to not break + * userspace and fix things up by shifting the crop window-x coordinate by 1 + * when hflip is *disabled*. + */ +#define OV01A10_MEDIA_BUS_FMT MEDIA_BUS_FMT_SBGGR10_1X10 + struct ov01a10_reg { u16 address; u8 val; @@ -185,14 +194,14 @@ static const struct ov01a10_reg sensor_1280x800_setting[] = { {0x380e, 0x03}, {0x380f, 0x80}, {0x3810, 0x00}, - {0x3811, 0x08}, + {0x3811, 0x09}, {0x3812, 0x00}, {0x3813, 0x08}, {0x3814, 0x01}, {0x3815, 0x01}, {0x3816, 0x01}, {0x3817, 0x01}, - {0x3820, 0xa0}, + {0x3820, 0xa8}, {0x3822, 0x13}, {0x3832, 0x28}, {0x3833, 0x10}, @@ -240,9 +249,8 @@ static const struct ov01a10_reg sensor_1280x800_setting[] = { static const char * const ov01a10_test_pattern_menu[] = { "Disabled", "Color Bar", - "Top-Bottom Darker Color Bar", - "Right-Left Darker Color Bar", - "Color Bar type 4", + "Left-Right Darker Color Bar", + "Bottom-Top Darker Color Bar", }; static const s64 link_freq_menu_items[] = { @@ -397,10 +405,8 @@ static int ov01a10_update_digital_gain(struct ov01a10 *ov01a10, u32 d_gain) static int ov01a10_test_pattern(struct ov01a10 *ov01a10, u32 pattern) { - if (!pattern) - return 0; - - pattern = (pattern - 1) | OV01A10_TEST_PATTERN_ENABLE; + if (pattern) + pattern |= OV01A10_TEST_PATTERN_ENABLE; return ov01a10_write_reg(ov01a10, OV01A10_REG_TEST_PATTERN, 1, pattern); } @@ -411,7 +417,7 @@ static int ov01a10_set_hflip(struct ov01a10 *ov01a10, u32 hflip) int ret; u32 val, offset; - offset = hflip ? 0x9 : 0x8; + offset = hflip ? 0x8 : 0x9; ret = ov01a10_write_reg(ov01a10, OV01A10_REG_X_WIN, 1, offset); if (ret) return ret; @@ -420,8 +426,8 @@ static int ov01a10_set_hflip(struct ov01a10 *ov01a10, u32 hflip) if (ret) return ret; - val = hflip ? val | FIELD_PREP(OV01A10_HFLIP_MASK, 0x1) : - val & ~OV01A10_HFLIP_MASK; + val = hflip ? val & ~OV01A10_HFLIP_MASK : + val | FIELD_PREP(OV01A10_HFLIP_MASK, 0x1); return ov01a10_write_reg(ov01a10, OV01A10_REG_FORMAT1, 1, val); } @@ -610,7 +616,7 @@ static void ov01a10_update_pad_format(const struct ov01a10_mode *mode, { fmt->width = mode->width; fmt->height = mode->height; - fmt->code = MEDIA_BUS_FMT_SBGGR10_1X10; + fmt->code = OV01A10_MEDIA_BUS_FMT; fmt->field = V4L2_FIELD_NONE; fmt->colorspace = V4L2_COLORSPACE_RAW; } @@ -722,7 +728,7 @@ static int ov01a10_set_format(struct v4l2_subdev *sd, h_blank); } - format = v4l2_subdev_state_get_format(sd_state, fmt->stream); + format = v4l2_subdev_state_get_format(sd_state, fmt->pad); *format = fmt->format; return 0; @@ -751,7 +757,7 @@ static int ov01a10_enum_mbus_code(struct v4l2_subdev *sd, if (code->index > 0) return -EINVAL; - code->code = MEDIA_BUS_FMT_SBGGR10_1X10; + code->code = OV01A10_MEDIA_BUS_FMT; return 0; } @@ -761,7 +767,7 @@ static int ov01a10_enum_frame_size(struct v4l2_subdev *sd, struct v4l2_subdev_frame_size_enum *fse) { if (fse->index >= ARRAY_SIZE(supported_modes) || - fse->code != MEDIA_BUS_FMT_SBGGR10_1X10) + fse->code != OV01A10_MEDIA_BUS_FMT) return -EINVAL; fse->min_width = supported_modes[fse->index].width; @@ -855,6 +861,7 @@ static void ov01a10_remove(struct i2c_client *client) struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_async_unregister_subdev(sd); + v4l2_subdev_cleanup(sd); media_entity_cleanup(&sd->entity); v4l2_ctrl_handler_free(sd->ctrl_handler); @@ -925,6 +932,7 @@ static int ov01a10_probe(struct i2c_client *client) err_pm_disable: pm_runtime_disable(dev); pm_runtime_set_suspended(&client->dev); + v4l2_subdev_cleanup(&ov01a10->sd); err_media_entity_cleanup: media_entity_cleanup(&ov01a10->sd.entity); diff --git a/drivers/media/i2c/ov5647.c b/drivers/media/i2c/ov5647.c index e193fef4fcedf..5fb10e02ba6e2 100644 --- a/drivers/media/i2c/ov5647.c +++ b/drivers/media/i2c/ov5647.c @@ -69,11 +69,11 @@ #define OV5647_NATIVE_HEIGHT 1956U #define OV5647_PIXEL_ARRAY_LEFT 16U -#define OV5647_PIXEL_ARRAY_TOP 16U +#define OV5647_PIXEL_ARRAY_TOP 6U #define OV5647_PIXEL_ARRAY_WIDTH 2592U #define OV5647_PIXEL_ARRAY_HEIGHT 1944U -#define OV5647_VBLANK_MIN 4 +#define OV5647_VBLANK_MIN 24 #define OV5647_VTS_MAX 32767 #define OV5647_EXPOSURE_MIN 4 @@ -508,7 +508,7 @@ static const struct ov5647_mode ov5647_modes[] = { { .format = { .code = MEDIA_BUS_FMT_SBGGR10_1X10, - .colorspace = V4L2_COLORSPACE_SRGB, + .colorspace = V4L2_COLORSPACE_RAW, .field = V4L2_FIELD_NONE, .width = 2592, .height = 1944 @@ -529,7 +529,7 @@ static const struct ov5647_mode ov5647_modes[] = { { .format = { .code = MEDIA_BUS_FMT_SBGGR10_1X10, - .colorspace = V4L2_COLORSPACE_SRGB, + .colorspace = V4L2_COLORSPACE_RAW, .field = V4L2_FIELD_NONE, .width = 1920, .height = 1080 @@ -550,7 +550,7 @@ static const struct ov5647_mode ov5647_modes[] = { { .format = { .code = MEDIA_BUS_FMT_SBGGR10_1X10, - .colorspace = V4L2_COLORSPACE_SRGB, + .colorspace = V4L2_COLORSPACE_RAW, .field = V4L2_FIELD_NONE, .width = 1296, .height = 972 @@ -571,7 +571,7 @@ static const struct ov5647_mode ov5647_modes[] = { { .format = { .code = MEDIA_BUS_FMT_SBGGR10_1X10, - .colorspace = V4L2_COLORSPACE_SRGB, + .colorspace = V4L2_COLORSPACE_RAW, .field = V4L2_FIELD_NONE, .width = 640, .height = 480 @@ -582,7 +582,7 @@ static const struct ov5647_mode ov5647_modes[] = { .width = 2560, .height = 1920, }, - .pixel_rate = 55000000, + .pixel_rate = 58333000, .hts = 1852, .vts = 0x1f8, .reg_list = ov5647_640x480_10bpp, @@ -1291,6 +1291,8 @@ static int ov5647_init_controls(struct ov5647 *sensor) v4l2_ctrl_handler_init(&sensor->ctrls, 9); + sensor->ctrls.lock = &sensor->lock; + v4l2_ctrl_new_std(&sensor->ctrls, &ov5647_ctrl_ops, V4L2_CID_AUTOGAIN, 0, 1, 1, 0); @@ -1420,15 +1422,15 @@ static int ov5647_probe(struct i2c_client *client) sensor->mode = OV5647_DEFAULT_MODE; - ret = ov5647_init_controls(sensor); - if (ret) - goto mutex_destroy; - sd = &sensor->sd; v4l2_i2c_subdev_init(sd, client, &ov5647_subdev_ops); sd->internal_ops = &ov5647_subdev_internal_ops; sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE | V4L2_SUBDEV_FL_HAS_EVENTS; + ret = ov5647_init_controls(sensor); + if (ret) + goto mutex_destroy; + sensor->pad.flags = MEDIA_PAD_FL_SOURCE; sd->entity.function = MEDIA_ENT_F_CAM_SENSOR; ret = media_entity_pads_init(&sd->entity, 1, &sensor->pad); diff --git a/drivers/media/i2c/tw9903.c b/drivers/media/i2c/tw9903.c index b996a05e56f28..c3eafd5d5dc82 100644 --- a/drivers/media/i2c/tw9903.c +++ b/drivers/media/i2c/tw9903.c @@ -228,6 +228,7 @@ static int tw9903_probe(struct i2c_client *client) if (write_regs(sd, initial_registers) < 0) { v4l2_err(client, "error initializing TW9903\n"); + v4l2_ctrl_handler_free(hdl); return -EINVAL; } diff --git a/drivers/media/i2c/tw9906.c b/drivers/media/i2c/tw9906.c index 6220f4fddbabc..0ab43fe42d7f4 100644 --- a/drivers/media/i2c/tw9906.c +++ b/drivers/media/i2c/tw9906.c @@ -196,6 +196,7 @@ static int tw9906_probe(struct i2c_client *client) if (write_regs(sd, initial_registers) < 0) { v4l2_err(client, "error initializing TW9906\n"); + v4l2_ctrl_handler_free(hdl); return -EINVAL; } diff --git a/drivers/media/pci/cx23885/cx23885-alsa.c b/drivers/media/pci/cx23885/cx23885-alsa.c index 25dc8d4dc5b73..717fc6c9ef21f 100644 --- a/drivers/media/pci/cx23885/cx23885-alsa.c +++ b/drivers/media/pci/cx23885/cx23885-alsa.c @@ -392,8 +392,10 @@ static int snd_cx23885_hw_params(struct snd_pcm_substream *substream, ret = cx23885_risc_databuffer(chip->pci, &buf->risc, buf->sglist, chip->period_size, chip->num_periods, 1); - if (ret < 0) + if (ret < 0) { + cx23885_alsa_dma_unmap(chip); goto error; + } /* Loop back to start of program */ buf->risc.jmp[0] = cpu_to_le32(RISC_JUMP|RISC_IRQ1|RISC_CNT_INC); diff --git a/drivers/media/pci/cx25821/cx25821-alsa.c b/drivers/media/pci/cx25821/cx25821-alsa.c index a42f0c03a7ca8..f463365163b7e 100644 --- a/drivers/media/pci/cx25821/cx25821-alsa.c +++ b/drivers/media/pci/cx25821/cx25821-alsa.c @@ -535,6 +535,7 @@ static int snd_cx25821_hw_params(struct snd_pcm_substream *substream, chip->period_size, chip->num_periods, 1); if (ret < 0) { pr_info("DEBUG: ERROR after cx25821_risc_databuffer_audio()\n"); + cx25821_alsa_dma_unmap(chip); goto error; } diff --git a/drivers/media/pci/cx25821/cx25821-core.c b/drivers/media/pci/cx25821/cx25821-core.c index 6627fa9166d30..a7336be444748 100644 --- a/drivers/media/pci/cx25821/cx25821-core.c +++ b/drivers/media/pci/cx25821/cx25821-core.c @@ -908,6 +908,7 @@ static int cx25821_dev_setup(struct cx25821_dev *dev) if (!dev->lmmio) { CX25821_ERR("ioremap failed, maybe increasing __VMALLOC_RESERVE in page.h\n"); + release_mem_region(dev->base_io_addr, pci_resource_len(dev->pci, 0)); cx25821_iounmap(dev); return -ENOMEM; } diff --git a/drivers/media/pci/cx88/cx88-alsa.c b/drivers/media/pci/cx88/cx88-alsa.c index 29fb1311e4434..4e574d8390b4d 100644 --- a/drivers/media/pci/cx88/cx88-alsa.c +++ b/drivers/media/pci/cx88/cx88-alsa.c @@ -483,8 +483,10 @@ static int snd_cx88_hw_params(struct snd_pcm_substream *substream, ret = cx88_risc_databuffer(chip->pci, &buf->risc, buf->sglist, chip->period_size, chip->num_periods, 1); - if (ret < 0) + if (ret < 0) { + cx88_alsa_dma_unmap(chip); goto error; + } /* Loop back to start of program */ buf->risc.jmp[0] = cpu_to_le32(RISC_JUMP | RISC_IRQ1 | RISC_CNT_INC); diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c index aa2cf7287477c..8f05987cdb4e7 100644 --- a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c +++ b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c @@ -3,6 +3,7 @@ * Copyright (C) 2013--2024 Intel Corporation */ #include +#include #include #include #include @@ -201,6 +202,8 @@ static int buffer_list_get(struct ipu6_isys_stream *stream, unsigned long flags; unsigned long buf_flag = IPU6_ISYS_BUFFER_LIST_FL_INCOMING; + lockdep_assert_held(&stream->mutex); + bl->nbufs = 0; INIT_LIST_HEAD(&bl->head); @@ -294,9 +297,8 @@ static int ipu6_isys_stream_start(struct ipu6_isys_video *av, struct ipu6_isys_buffer_list __bl; int ret; - mutex_lock(&stream->isys->stream_mutex); + guard(mutex)(&stream->isys->stream_mutex); ret = ipu6_isys_video_set_streaming(av, 1, bl); - mutex_unlock(&stream->isys->stream_mutex); if (ret) goto out_requeue; @@ -637,10 +639,10 @@ static void stop_streaming(struct vb2_queue *q) mutex_lock(&av->isys->stream_mutex); if (stream->nr_streaming == stream->nr_queues && stream->streaming) ipu6_isys_video_set_streaming(av, 0, NULL); + list_del(&aq->node); mutex_unlock(&av->isys->stream_mutex); stream->nr_streaming--; - list_del(&aq->node); stream->streaming = 0; mutex_unlock(&stream->mutex); diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c index f3f3bc0615e5d..b96ae563b82f0 100644 --- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c +++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c @@ -1022,11 +1022,10 @@ int ipu6_isys_video_set_streaming(struct ipu6_isys_video *av, int state, sd->name, r_pad->index, stream_mask); ret = v4l2_subdev_disable_streams(sd, r_pad->index, stream_mask); - if (ret) { + if (ret) dev_err(dev, "stream off %s failed with %d\n", sd->name, ret); - return ret; - } + close_streaming_firmware(av); } else { ret = start_stream_firmware(av, bl); @@ -1052,6 +1051,7 @@ int ipu6_isys_video_set_streaming(struct ipu6_isys_video *av, int state, out_media_entity_stop_streaming_firmware: stop_streaming_firmware(av); + close_streaming_firmware(av); return ret; } diff --git a/drivers/media/pci/intel/ipu6/ipu6-mmu.c b/drivers/media/pci/intel/ipu6/ipu6-mmu.c index 6d1c0b90169d4..85cc6d5b4dd11 100644 --- a/drivers/media/pci/intel/ipu6/ipu6-mmu.c +++ b/drivers/media/pci/intel/ipu6/ipu6-mmu.c @@ -102,7 +102,7 @@ static void page_table_dump(struct ipu6_mmu_info *mmu_info) if (mmu_info->l1_pt[l1_idx] == mmu_info->dummy_l2_pteval) continue; - l2_phys = TBL_PHYS_ADDR(mmu_info->l1_pt[l1_idx];) + l2_phys = TBL_PHYS_ADDR(mmu_info->l1_pt[l1_idx]); dev_dbg(mmu_info->dev, "l1 entry %u; iovas 0x%8.8x-0x%8.8x, at %pap\n", l1_idx, iova, iova + ISP_PAGE_SIZE, &l2_phys); @@ -248,7 +248,7 @@ static u32 *alloc_l2_pt(struct ipu6_mmu_info *mmu_info) dev_dbg(mmu_info->dev, "alloc_l2: get_zeroed_page() = %p\n", pt); - for (i = 0; i < ISP_L1PT_PTES; i++) + for (i = 0; i < ISP_L2PT_PTES; i++) pt[i] = mmu_info->dummy_page_pteval; return pt; diff --git a/drivers/media/pci/intel/ipu6/ipu6.c b/drivers/media/pci/intel/ipu6/ipu6.c index 1f4f20b9c94dc..a2768f44017a5 100644 --- a/drivers/media/pci/intel/ipu6/ipu6.c +++ b/drivers/media/pci/intel/ipu6/ipu6.c @@ -630,21 +630,21 @@ static int ipu6_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) { dev_err_probe(&isp->pdev->dev, ret, "Failed to set MMU hardware\n"); - goto out_ipu6_bus_del_devices; + goto out_ipu6_rpm_put; } ret = ipu6_buttress_map_fw_image(isp->psys, isp->cpd_fw, &isp->psys->fw_sgt); if (ret) { dev_err_probe(&isp->pdev->dev, ret, "failed to map fw image\n"); - goto out_ipu6_bus_del_devices; + goto out_ipu6_rpm_put; } ret = ipu6_cpd_create_pkg_dir(isp->psys, isp->cpd_fw->data); if (ret) { dev_err_probe(&isp->pdev->dev, ret, "failed to create pkg dir\n"); - goto out_ipu6_bus_del_devices; + goto out_ipu6_rpm_put; } ret = devm_request_threaded_irq(dev, pdev->irq, ipu6_buttress_isr, @@ -652,7 +652,7 @@ static int ipu6_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) IRQF_SHARED, IPU6_NAME, isp); if (ret) { dev_err_probe(dev, ret, "Requesting irq failed\n"); - goto out_ipu6_bus_del_devices; + goto out_ipu6_rpm_put; } ret = ipu6_buttress_authenticate(isp); @@ -683,6 +683,8 @@ static int ipu6_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) out_free_irq: devm_free_irq(dev, pdev->irq, isp); +out_ipu6_rpm_put: + pm_runtime_put_sync(&isp->psys->auxdev.dev); out_ipu6_bus_del_devices: if (isp->psys) { ipu6_cpd_free_pkg_dir(isp->psys); diff --git a/drivers/media/pci/mgb4/mgb4_trigger.c b/drivers/media/pci/mgb4/mgb4_trigger.c index d7dddc5c8728e..10c23f0c833d5 100644 --- a/drivers/media/pci/mgb4/mgb4_trigger.c +++ b/drivers/media/pci/mgb4/mgb4_trigger.c @@ -114,7 +114,7 @@ static int probe_trigger(struct iio_dev *indio_dev, int irq) if (!st->trig) return -ENOMEM; - ret = request_irq(irq, &iio_trigger_generic_data_rdy_poll, 0, + ret = request_irq(irq, &iio_trigger_generic_data_rdy_poll, IRQF_NO_THREAD, "mgb4-trigger", st->trig); if (ret) goto error_free_trig; diff --git a/drivers/media/pci/solo6x10/solo6x10-tw28.c b/drivers/media/pci/solo6x10/solo6x10-tw28.c index 1b7c22a9bc94f..8f53946c67928 100644 --- a/drivers/media/pci/solo6x10/solo6x10-tw28.c +++ b/drivers/media/pci/solo6x10/solo6x10-tw28.c @@ -166,7 +166,7 @@ static const u8 tbl_tw2865_pal_template[] = { 0x64, 0x51, 0x40, 0xaf, 0xFF, 0xF0, 0x00, 0xC0, }; -#define is_tw286x(__solo, __id) (!(__solo->tw2815 & (1 << __id))) +#define is_tw286x(__solo, __id) (!((__solo)->tw2815 & (1U << (__id)))) static u8 tw_readbyte(struct solo_dev *solo_dev, int chip_id, u8 tw6x_off, u8 tw_off) @@ -686,6 +686,9 @@ int tw28_set_ctrl_val(struct solo_dev *solo_dev, u32 ctrl, u8 ch, chip_num = ch / 4; ch %= 4; + if (chip_num >= TW_NUM_CHIP) + return -EINVAL; + if (val > 255 || val < 0) return -ERANGE; @@ -758,6 +761,9 @@ int tw28_get_ctrl_val(struct solo_dev *solo_dev, u32 ctrl, u8 ch, chip_num = ch / 4; ch %= 4; + if (chip_num >= TW_NUM_CHIP) + return -EINVAL; + switch (ctrl) { case V4L2_CID_SHARPNESS: /* Only 286x has sharpness */ diff --git a/drivers/media/platform/amphion/vdec.c b/drivers/media/platform/amphion/vdec.c index 32eef2fd1f2a9..6b63e8ca0cb40 100644 --- a/drivers/media/platform/amphion/vdec.c +++ b/drivers/media/platform/amphion/vdec.c @@ -726,6 +726,7 @@ static int vdec_decoder_cmd(struct file *file, void *fh, struct v4l2_decoder_cmd switch (cmd->cmd) { case V4L2_DEC_CMD_START: vdec_cmd_start(inst); + vb2_clear_last_buffer_dequeued(v4l2_m2m_get_dst_vq(inst->fh.m2m_ctx)); break; case V4L2_DEC_CMD_STOP: vdec_cmd_stop(inst); diff --git a/drivers/media/platform/amphion/vpu_v4l2.c b/drivers/media/platform/amphion/vpu_v4l2.c index 47dff9a35bb46..1fb887b9098c6 100644 --- a/drivers/media/platform/amphion/vpu_v4l2.c +++ b/drivers/media/platform/amphion/vpu_v4l2.c @@ -670,7 +670,6 @@ static int vpu_m2m_queue_init(void *priv, struct vb2_queue *src_vq, struct vb2_q src_vq->mem_ops = &vb2_vmalloc_memops; src_vq->drv_priv = inst; src_vq->buf_struct_size = sizeof(struct vpu_vb2_buffer); - src_vq->min_queued_buffers = 1; src_vq->dev = inst->vpu->dev; src_vq->lock = &inst->lock; ret = vb2_queue_init(src_vq); @@ -687,7 +686,6 @@ static int vpu_m2m_queue_init(void *priv, struct vb2_queue *src_vq, struct vb2_q dst_vq->mem_ops = &vb2_vmalloc_memops; dst_vq->drv_priv = inst; dst_vq->buf_struct_size = sizeof(struct vpu_vb2_buffer); - dst_vq->min_queued_buffers = 1; dst_vq->dev = inst->vpu->dev; dst_vq->lock = &inst->lock; ret = vb2_queue_init(dst_vq); diff --git a/drivers/media/platform/chips-media/wave5/wave5-helper.c b/drivers/media/platform/chips-media/wave5/wave5-helper.c index f03ad9c0de221..53a0ac068c2e2 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-helper.c +++ b/drivers/media/platform/chips-media/wave5/wave5-helper.c @@ -27,6 +27,11 @@ const char *state_to_str(enum vpu_instance_state state) } } +int wave5_kfifo_alloc(struct vpu_instance *inst) +{ + return kfifo_alloc(&inst->irq_status, 16 * sizeof(int), GFP_KERNEL); +} + void wave5_cleanup_instance(struct vpu_instance *inst, struct file *filp) { int i; @@ -49,7 +54,7 @@ void wave5_cleanup_instance(struct vpu_instance *inst, struct file *filp) v4l2_fh_del(&inst->v4l2_fh, filp); v4l2_fh_exit(&inst->v4l2_fh); } - list_del_init(&inst->list); + kfifo_free(&inst->irq_status); ida_free(&inst->dev->inst_ida, inst->id); kfree(inst->codec_info); kfree(inst); @@ -61,8 +66,29 @@ int wave5_vpu_release_device(struct file *filp, { struct vpu_instance *inst = file_to_vpu_inst(filp); int ret = 0; + unsigned long flags; v4l2_m2m_ctx_release(inst->v4l2_fh.m2m_ctx); + /* + * To prevent Null reference exception, the existing irq handler were + * separated to two modules. + * One is to queue interrupt reason into the irq handler, + * the other is irq_thread to call the wave5_vpu_dec_finish_decode + * to get decoded frame. + * The list of instances should be protected between all flow of the + * decoding process, but to protect the list in the irq_handler, spin lock + * should be used, and mutex should be used in the irq_thread because spin lock + * is not able to be used because mutex is already being used + * in the wave5_vpu_dec_finish_decode. + * So the spin lock and mutex were used to protect the list in the release function. + */ + ret = mutex_lock_interruptible(&inst->dev->irq_lock); + if (ret) + return ret; + spin_lock_irqsave(&inst->dev->irq_spinlock, flags); + list_del_init(&inst->list); + spin_unlock_irqrestore(&inst->dev->irq_spinlock, flags); + mutex_unlock(&inst->dev->irq_lock); if (inst->state != VPU_INST_STATE_NONE) { u32 fail_res; diff --git a/drivers/media/platform/chips-media/wave5/wave5-helper.h b/drivers/media/platform/chips-media/wave5/wave5-helper.h index 976a402e426ff..d61fdbda359d6 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-helper.h +++ b/drivers/media/platform/chips-media/wave5/wave5-helper.h @@ -33,4 +33,5 @@ void wave5_update_pix_fmt(struct v4l2_pix_format_mplane *pix_mp, unsigned int width, unsigned int height, const struct v4l2_frmsize_stepwise *frmsize); +int wave5_kfifo_alloc(struct vpu_instance *inst); #endif diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c index e3038c18ca362..cff2fa17c3f59 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c @@ -1243,6 +1243,7 @@ static void wave5_vpu_dec_buf_queue_dst(struct vb2_buffer *vb) struct vpu_instance *inst = vb2_get_drv_priv(vb->vb2_queue); struct v4l2_m2m_ctx *m2m_ctx = inst->v4l2_fh.m2m_ctx; + pm_runtime_resume_and_get(inst->dev->dev); vbuf->sequence = inst->queued_dst_buf_num++; if (inst->state == VPU_INST_STATE_PIC_RUN) { @@ -1275,6 +1276,7 @@ static void wave5_vpu_dec_buf_queue_dst(struct vb2_buffer *vb) } else { v4l2_m2m_buf_queue(m2m_ctx, vbuf); } + pm_runtime_put_autosuspend(inst->dev->dev); } static void wave5_vpu_dec_buf_queue(struct vb2_buffer *vb) @@ -1753,8 +1755,10 @@ static int wave5_vpu_open_dec(struct file *filp) spin_lock_init(&inst->state_spinlock); inst->codec_info = kzalloc(sizeof(*inst->codec_info), GFP_KERNEL); - if (!inst->codec_info) + if (!inst->codec_info) { + kfree(inst); return -ENOMEM; + } v4l2_fh_init(&inst->v4l2_fh, vdev); v4l2_fh_add(&inst->v4l2_fh, filp); @@ -1806,6 +1810,11 @@ static int wave5_vpu_open_dec(struct file *filp) inst->xfer_func = V4L2_XFER_FUNC_DEFAULT; init_completion(&inst->irq_done); + ret = wave5_kfifo_alloc(inst); + if (ret) { + dev_err(inst->dev->dev, "failed to allocate fifo\n"); + goto cleanup_inst; + } inst->id = ida_alloc(&inst->dev->inst_ida, GFP_KERNEL); if (inst->id < 0) { @@ -1825,9 +1834,6 @@ static int wave5_vpu_open_dec(struct file *filp) if (ret) goto cleanup_inst; - if (list_empty(&dev->instances)) - pm_runtime_use_autosuspend(inst->dev->dev); - list_add_tail(&inst->list, &dev->instances); mutex_unlock(&dev->dev_lock); diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c index 9bfaa9fb3ceb3..24fc0d0d3f4aa 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c @@ -649,6 +649,8 @@ static int wave5_vpu_enc_encoder_cmd(struct file *file, void *fh, struct v4l2_en m2m_ctx->last_src_buf = v4l2_m2m_last_src_buf(m2m_ctx); m2m_ctx->is_draining = true; + + v4l2_m2m_try_schedule(m2m_ctx); break; case V4L2_ENC_CMD_START: break; @@ -1367,7 +1369,8 @@ static int wave5_vpu_enc_start_streaming(struct vb2_queue *q, unsigned int count if (ret) goto return_buffers; } - if (inst->state == VPU_INST_STATE_OPEN && m2m_ctx->cap_q_ctx.q.streaming) { + if (inst->state == VPU_INST_STATE_OPEN && + (m2m_ctx->cap_q_ctx.q.streaming || q->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)) { ret = initialize_sequence(inst); if (ret) { dev_warn(inst->dev->dev, "Sequence not found: %d\n", ret); @@ -1578,8 +1581,10 @@ static int wave5_vpu_open_enc(struct file *filp) inst->ops = &wave5_vpu_enc_inst_ops; inst->codec_info = kzalloc(sizeof(*inst->codec_info), GFP_KERNEL); - if (!inst->codec_info) + if (!inst->codec_info) { + kfree(inst); return -ENOMEM; + } v4l2_fh_init(&inst->v4l2_fh, vdev); v4l2_fh_add(&inst->v4l2_fh, filp); @@ -1754,6 +1759,11 @@ static int wave5_vpu_open_enc(struct file *filp) inst->frame_rate = 30; init_completion(&inst->irq_done); + ret = wave5_kfifo_alloc(inst); + if (ret) { + dev_err(inst->dev->dev, "failed to allocate fifo\n"); + goto cleanup_inst; + } inst->id = ida_alloc(&inst->dev->inst_ida, GFP_KERNEL); if (inst->id < 0) { @@ -1768,9 +1778,6 @@ static int wave5_vpu_open_enc(struct file *filp) if (ret) goto cleanup_inst; - if (list_empty(&dev->instances)) - pm_runtime_use_autosuspend(inst->dev->dev); - list_add_tail(&inst->list, &dev->instances); mutex_unlock(&dev->dev_lock); diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu.c b/drivers/media/platform/chips-media/wave5/wave5-vpu.c index e1715d3f43b0d..3216b49976447 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpu.c +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu.c @@ -51,8 +51,11 @@ static void wave5_vpu_handle_irq(void *dev_id) u32 seq_done; u32 cmd_done; u32 irq_reason; - struct vpu_instance *inst; + u32 irq_subreason; + struct vpu_instance *inst, *tmp; struct vpu_device *dev = dev_id; + int val; + unsigned long flags; irq_reason = wave5_vdi_read_register(dev, W5_VPU_VINT_REASON); seq_done = wave5_vdi_read_register(dev, W5_RET_SEQ_DONE_INSTANCE_INFO); @@ -60,7 +63,8 @@ static void wave5_vpu_handle_irq(void *dev_id) wave5_vdi_write_register(dev, W5_VPU_VINT_REASON_CLR, irq_reason); wave5_vdi_write_register(dev, W5_VPU_VINT_CLEAR, 0x1); - list_for_each_entry(inst, &dev->instances, list) { + spin_lock_irqsave(&dev->irq_spinlock, flags); + list_for_each_entry_safe(inst, tmp, &dev->instances, list) { if (irq_reason & BIT(INT_WAVE5_INIT_SEQ) || irq_reason & BIT(INT_WAVE5_ENC_SET_PARAM)) { @@ -82,22 +86,54 @@ static void wave5_vpu_handle_irq(void *dev_id) irq_reason & BIT(INT_WAVE5_ENC_PIC)) { if (cmd_done & BIT(inst->id)) { cmd_done &= ~BIT(inst->id); - wave5_vdi_write_register(dev, W5_RET_QUEUE_CMD_DONE_INST, - cmd_done); - inst->ops->finish_process(inst); + if (dev->irq >= 0) { + irq_subreason = + wave5_vdi_read_register(dev, W5_VPU_VINT_REASON); + if (!(irq_subreason & BIT(INT_WAVE5_DEC_PIC))) + wave5_vdi_write_register(dev, + W5_RET_QUEUE_CMD_DONE_INST, + cmd_done); + } + val = BIT(INT_WAVE5_DEC_PIC); + kfifo_in(&inst->irq_status, &val, sizeof(int)); } } + } + spin_unlock_irqrestore(&dev->irq_spinlock, flags); + + if (dev->irq < 0) + up(&dev->irq_sem); +} - wave5_vpu_clear_interrupt(inst, irq_reason); +static irqreturn_t wave5_vpu_irq(int irq, void *dev_id) +{ + struct vpu_device *dev = dev_id; + + if (wave5_vdi_read_register(dev, W5_VPU_VPU_INT_STS)) { + wave5_vpu_handle_irq(dev); + return IRQ_WAKE_THREAD; } + + return IRQ_HANDLED; } static irqreturn_t wave5_vpu_irq_thread(int irq, void *dev_id) { struct vpu_device *dev = dev_id; + struct vpu_instance *inst, *tmp; + int irq_status, ret; - if (wave5_vdi_read_register(dev, W5_VPU_VPU_INT_STS)) - wave5_vpu_handle_irq(dev); + mutex_lock(&dev->irq_lock); + list_for_each_entry_safe(inst, tmp, &dev->instances, list) { + while (kfifo_len(&inst->irq_status)) { + ret = kfifo_out(&inst->irq_status, &irq_status, sizeof(int)); + if (!ret) + break; + + inst->ops->finish_process(inst); + } + } + mutex_unlock(&dev->irq_lock); return IRQ_HANDLED; } @@ -121,6 +157,35 @@ static enum hrtimer_restart wave5_vpu_timer_callback(struct hrtimer *timer) return HRTIMER_RESTART; } +static int irq_thread(void *data) +{ + struct vpu_device *dev = (struct vpu_device *)data; + struct vpu_instance *inst, *tmp; + int irq_status, ret; + + while (!kthread_should_stop()) { + if (down_interruptible(&dev->irq_sem)) + continue; + + if (kthread_should_stop()) + break; + + mutex_lock(&dev->irq_lock); + list_for_each_entry_safe(inst, tmp, &dev->instances, list) { + while (kfifo_len(&inst->irq_status)) { + ret = kfifo_out(&inst->irq_status, &irq_status, sizeof(int)); + if (!ret) + break; + + inst->ops->finish_process(inst); + } + } + mutex_unlock(&dev->irq_lock); + } + + return 0; +} + static int wave5_vpu_load_firmware(struct device *dev, const char *fw_name, u32 *revision) { @@ -224,6 +289,8 @@ static int wave5_vpu_probe(struct platform_device *pdev) mutex_init(&dev->dev_lock); mutex_init(&dev->hw_lock); + mutex_init(&dev->irq_lock); + spin_lock_init(&dev->irq_spinlock); dev_set_drvdata(&pdev->dev, dev); dev->dev = &pdev->dev; @@ -266,9 +333,13 @@ static int wave5_vpu_probe(struct platform_device *pdev) } dev->product = wave5_vpu_get_product_id(dev); + INIT_LIST_HEAD(&dev->instances); + dev->irq = platform_get_irq(pdev, 0); if (dev->irq < 0) { dev_err(&pdev->dev, "failed to get irq resource, falling back to polling\n"); + sema_init(&dev->irq_sem, 1); + dev->irq_thread = kthread_run(irq_thread, dev, "irq thread"); hrtimer_setup(&dev->hrtimer, &wave5_vpu_timer_callback, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); dev->worker = kthread_run_worker(0, "vpu_irq_thread"); @@ -280,7 +351,7 @@ static int wave5_vpu_probe(struct platform_device *pdev) dev->vpu_poll_interval = vpu_poll_interval; kthread_init_work(&dev->work, wave5_vpu_irq_work_fn); } else { - ret = devm_request_threaded_irq(&pdev->dev, dev->irq, NULL, + ret = devm_request_threaded_irq(&pdev->dev, dev->irq, wave5_vpu_irq, wave5_vpu_irq_thread, IRQF_ONESHOT, "vpu_irq", dev); if (ret) { dev_err(&pdev->dev, "Register interrupt handler, fail: %d\n", ret); @@ -288,7 +359,6 @@ static int wave5_vpu_probe(struct platform_device *pdev) } } - INIT_LIST_HEAD(&dev->instances); ret = v4l2_device_register(&pdev->dev, &dev->v4l2_dev); if (ret) { dev_err(&pdev->dev, "v4l2_device_register, fail: %d\n", ret); @@ -322,7 +392,7 @@ static int wave5_vpu_probe(struct platform_device *pdev) dev_info(&pdev->dev, "Product Code: 0x%x\n", dev->product_code); dev_info(&pdev->dev, "Firmware Revision: %u\n", fw_revision); - pm_runtime_set_autosuspend_delay(&pdev->dev, 100); + pm_runtime_set_autosuspend_delay(&pdev->dev, 500); pm_runtime_use_autosuspend(&pdev->dev); pm_runtime_enable(&pdev->dev); wave5_vpu_sleep_wake(&pdev->dev, true, NULL, 0); @@ -351,21 +421,30 @@ static void wave5_vpu_remove(struct platform_device *pdev) { struct vpu_device *dev = dev_get_drvdata(&pdev->dev); + wave5_vpu_enc_unregister_device(dev); + wave5_vpu_dec_unregister_device(dev); + v4l2_device_unregister(&dev->v4l2_dev); + if (dev->irq < 0) { - kthread_destroy_worker(dev->worker); + if (dev->irq_thread) { + kthread_stop(dev->irq_thread); + up(&dev->irq_sem); + dev->irq_thread = NULL; + } + hrtimer_cancel(&dev->hrtimer); + kthread_cancel_work_sync(&dev->work); + kthread_destroy_worker(dev->worker); } - pm_runtime_put_sync(&pdev->dev); + pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_disable(&pdev->dev); mutex_destroy(&dev->dev_lock); mutex_destroy(&dev->hw_lock); + mutex_destroy(&dev->irq_lock); reset_control_assert(dev->resets); clk_bulk_disable_unprepare(dev->num_clks, dev->clks); - wave5_vpu_enc_unregister_device(dev); - wave5_vpu_dec_unregister_device(dev); - v4l2_device_unregister(&dev->v4l2_dev); wave5_vdi_release(&pdev->dev); ida_destroy(&dev->inst_ida); } diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c index e5e879a13e8b8..e94d6ebc9f816 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c +++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c @@ -207,8 +207,6 @@ int wave5_vpu_dec_close(struct vpu_instance *inst, u32 *fail_res) int retry = 0; struct vpu_device *vpu_dev = inst->dev; int i; - int inst_count = 0; - struct vpu_instance *inst_elm; *fail_res = 0; if (!inst->codec_info) @@ -250,11 +248,6 @@ int wave5_vpu_dec_close(struct vpu_instance *inst, u32 *fail_res) wave5_vdi_free_dma_memory(vpu_dev, &p_dec_info->vb_task); - list_for_each_entry(inst_elm, &vpu_dev->instances, list) - inst_count++; - if (inst_count == 1) - pm_runtime_dont_use_autosuspend(vpu_dev->dev); - unlock_and_return: mutex_unlock(&vpu_dev->hw_lock); pm_runtime_put_sync(inst->dev->dev); @@ -720,8 +713,6 @@ int wave5_vpu_enc_close(struct vpu_instance *inst, u32 *fail_res) int ret; int retry = 0; struct vpu_device *vpu_dev = inst->dev; - int inst_count = 0; - struct vpu_instance *inst_elm; *fail_res = 0; if (!inst->codec_info) @@ -764,12 +755,6 @@ int wave5_vpu_enc_close(struct vpu_instance *inst, u32 *fail_res) } wave5_vdi_free_dma_memory(vpu_dev, &p_enc_info->vb_task); - - list_for_each_entry(inst_elm, &vpu_dev->instances, list) - inst_count++; - if (inst_count == 1) - pm_runtime_dont_use_autosuspend(vpu_dev->dev); - mutex_unlock(&vpu_dev->hw_lock); pm_runtime_put_sync(inst->dev->dev); diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h index 45615c15beca3..bc101397204da 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h +++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h @@ -8,6 +8,7 @@ #ifndef VPUAPI_H_INCLUDED #define VPUAPI_H_INCLUDED +#include #include #include #include @@ -747,6 +748,7 @@ struct vpu_device { struct video_device *video_dev_enc; struct mutex dev_lock; /* lock for the src, dst v4l2 queues */ struct mutex hw_lock; /* lock hw configurations */ + struct mutex irq_lock; int irq; enum product_id product; struct vpu_attr attr; @@ -764,7 +766,10 @@ struct vpu_device { struct kthread_worker *worker; int vpu_poll_interval; int num_clks; + struct task_struct *irq_thread; + struct semaphore irq_sem; /* signal to irq_thread when interrupt happens*/ struct reset_control *resets; + spinlock_t irq_spinlock; /* protect instances list */ }; struct vpu_instance; @@ -788,6 +793,7 @@ struct vpu_instance { enum v4l2_ycbcr_encoding ycbcr_enc; enum v4l2_quantization quantization; + struct kfifo irq_status; enum vpu_instance_state state; enum vpu_instance_type type; const struct vpu_instance_ops *ops; diff --git a/drivers/media/platform/mediatek/mdp/mtk_mdp_core.c b/drivers/media/platform/mediatek/mdp/mtk_mdp_core.c index 80fdc6ff57e0e..8432833814f31 100644 --- a/drivers/media/platform/mediatek/mdp/mtk_mdp_core.c +++ b/drivers/media/platform/mediatek/mdp/mtk_mdp_core.c @@ -194,11 +194,17 @@ static int mtk_mdp_probe(struct platform_device *pdev) } mdp->vpu_dev = vpu_get_plat_device(pdev); + if (!mdp->vpu_dev) { + dev_err(&pdev->dev, "Failed to get vpu device\n"); + ret = -ENODEV; + goto err_vpu_get_dev; + } + ret = vpu_wdt_reg_handler(mdp->vpu_dev, mtk_mdp_reset_handler, mdp, VPU_RST_MDP); if (ret) { dev_err(&pdev->dev, "Failed to register reset handler\n"); - goto err_m2m_register; + goto err_reg_handler; } platform_set_drvdata(pdev, mdp); @@ -206,7 +212,7 @@ static int mtk_mdp_probe(struct platform_device *pdev) ret = vb2_dma_contig_set_max_seg_size(&pdev->dev, DMA_BIT_MASK(32)); if (ret) { dev_err(&pdev->dev, "Failed to set vb2 dma mag seg size\n"); - goto err_m2m_register; + goto err_reg_handler; } pm_runtime_enable(dev); @@ -214,6 +220,12 @@ static int mtk_mdp_probe(struct platform_device *pdev) return 0; +err_reg_handler: + platform_device_put(mdp->vpu_dev); + +err_vpu_get_dev: + mtk_mdp_unregister_m2m_device(mdp); + err_m2m_register: v4l2_device_unregister(&mdp->v4l2_dev); @@ -242,6 +254,7 @@ static void mtk_mdp_remove(struct platform_device *pdev) pm_runtime_disable(&pdev->dev); vb2_dma_contig_clear_max_seg_size(&pdev->dev); + platform_device_put(mdp->vpu_dev); mtk_mdp_unregister_m2m_device(mdp); v4l2_device_unregister(&mdp->v4l2_dev); diff --git a/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_stateless.c b/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_stateless.c index d873159b9b306..9eef3ff2b1278 100644 --- a/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_stateless.c +++ b/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_stateless.c @@ -502,6 +502,12 @@ static int mtk_vdec_s_ctrl(struct v4l2_ctrl *ctrl) mtk_v4l2_vdec_err(ctx, "VP9: bit_depth:%d", frame->bit_depth); return -EINVAL; } + + if (!(frame->flags & V4L2_VP9_FRAME_FLAG_X_SUBSAMPLING) || + !(frame->flags & V4L2_VP9_FRAME_FLAG_Y_SUBSAMPLING)) { + mtk_v4l2_vdec_err(ctx, "VP9: only 420 subsampling is supported"); + return -EINVAL; + } break; case V4L2_CID_STATELESS_AV1_SEQUENCE: seq = (struct v4l2_ctrl_av1_sequence *)hdr_ctrl->p_new.p; diff --git a/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc.c b/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc.c index d815e962ab898..0212ff0462fcc 100644 --- a/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc.c +++ b/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc.c @@ -864,7 +864,7 @@ static void vb2ops_venc_buf_queue(struct vb2_buffer *vb) static int vb2ops_venc_start_streaming(struct vb2_queue *q, unsigned int count) { struct mtk_vcodec_enc_ctx *ctx = vb2_get_drv_priv(q); - struct venc_enc_param param; + struct venc_enc_param param = { }; int ret; int i; @@ -1018,7 +1018,7 @@ static int mtk_venc_encode_header(void *priv) int ret; struct vb2_v4l2_buffer *src_buf, *dst_buf; struct mtk_vcodec_mem bs_buf; - struct venc_done_result enc_result; + struct venc_done_result enc_result = { }; dst_buf = v4l2_m2m_dst_buf_remove(ctx->m2m_ctx); if (!dst_buf) { @@ -1139,7 +1139,7 @@ static void mtk_venc_worker(struct work_struct *work) struct vb2_v4l2_buffer *src_buf, *dst_buf; struct venc_frm_buf frm_buf; struct mtk_vcodec_mem bs_buf; - struct venc_done_result enc_result; + struct venc_done_result enc_result = { }; int ret, i; /* check dst_buf, dst_buf may be removed in device_run diff --git a/drivers/media/platform/qcom/iris/iris_buffer.c b/drivers/media/platform/qcom/iris/iris_buffer.c index b89b1ee06cce1..f1f003a787bf2 100644 --- a/drivers/media/platform/qcom/iris/iris_buffer.c +++ b/drivers/media/platform/qcom/iris/iris_buffer.c @@ -351,12 +351,15 @@ static int iris_create_internal_buffer(struct iris_inst *inst, buffer->index = index; buffer->buffer_size = buffers->size; buffer->dma_attrs = DMA_ATTR_WRITE_COMBINE | DMA_ATTR_NO_KERNEL_MAPPING; - list_add_tail(&buffer->list, &buffers->list); buffer->kvaddr = dma_alloc_attrs(core->dev, buffer->buffer_size, &buffer->device_addr, GFP_KERNEL, buffer->dma_attrs); - if (!buffer->kvaddr) + if (!buffer->kvaddr) { + kfree(buffer); return -ENOMEM; + } + + list_add_tail(&buffer->list, &buffers->list); return 0; } diff --git a/drivers/media/platform/qcom/iris/iris_hfi_gen1_command.c b/drivers/media/platform/qcom/iris/iris_hfi_gen1_command.c index 52da7ef7bab08..11815f6f5bacd 100644 --- a/drivers/media/platform/qcom/iris/iris_hfi_gen1_command.c +++ b/drivers/media/platform/qcom/iris/iris_hfi_gen1_command.c @@ -282,7 +282,7 @@ static int iris_hfi_gen1_queue_input_buffer(struct iris_inst *inst, struct iris_ com_ip_pkt.shdr.session_id = inst->session_id; com_ip_pkt.time_stamp_hi = upper_32_bits(buf->timestamp); com_ip_pkt.time_stamp_lo = lower_32_bits(buf->timestamp); - com_ip_pkt.flags = buf->flags; + com_ip_pkt.flags = 0; com_ip_pkt.mark_target = 0; com_ip_pkt.mark_data = 0; com_ip_pkt.offset = buf->data_offset; @@ -441,6 +441,8 @@ static int iris_hfi_gen1_session_unset_buffers(struct iris_inst *inst, struct ir goto exit; ret = iris_wait_for_session_response(inst, false); + if (!ret) + ret = iris_destroy_internal_buffer(inst, buf); exit: kfree(pkt); @@ -733,7 +735,7 @@ static int iris_hfi_gen1_set_resolution(struct iris_inst *inst, u32 plane) struct hfi_framesize fs; int ret; - if (!iris_drc_pending(inst)) { + if (!iris_drc_pending(inst) && !(inst->sub_state & IRIS_INST_SUB_FIRST_IPSC)) { fs.buffer_type = HFI_BUFFER_INPUT; fs.width = inst->fmt_src->fmt.pix_mp.width; fs.height = inst->fmt_src->fmt.pix_mp.height; diff --git a/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c b/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c index b00594bb3c887..ad2c4a7dfed25 100644 --- a/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c +++ b/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c @@ -1036,6 +1036,9 @@ static int iris_hfi_gen2_session_stop(struct iris_inst *inst, u32 plane) struct iris_inst_hfi_gen2 *inst_hfi_gen2 = to_iris_inst_hfi_gen2(inst); int ret = 0; + if (!inst_hfi_gen2->packet) + return -EINVAL; + reinit_completion(&inst->completion); iris_hfi_gen2_packet_session_command(inst, diff --git a/drivers/media/platform/qcom/iris/iris_platform_gen2.c b/drivers/media/platform/qcom/iris/iris_platform_gen2.c index b2d8559dd2b85..c932dc026616e 100644 --- a/drivers/media/platform/qcom/iris/iris_platform_gen2.c +++ b/drivers/media/platform/qcom/iris/iris_platform_gen2.c @@ -1056,6 +1056,7 @@ const struct iris_platform_data sm8750_data = { .get_instance = iris_hfi_gen2_get_instance, .init_hfi_command_ops = iris_hfi_gen2_command_ops_init, .init_hfi_response_ops = iris_hfi_gen2_response_ops_init, + .get_vpu_buffer_size = iris_vpu33_buf_size, .vpu_ops = &iris_vpu35_ops, .set_preset_registers = iris_set_sm8550_preset_registers, .icc_tbl = sm8550_icc_table, @@ -1088,6 +1089,7 @@ const struct iris_platform_data sm8750_data = { .num_vpp_pipe = 4, .max_session_count = 16, .max_core_mbpf = NUM_MBS_8K * 2, + .max_core_mbps = ((7680 * 4320) / 256) * 60, .dec_input_config_params_default = sm8550_vdec_input_config_params_default, .dec_input_config_params_default_size = diff --git a/drivers/media/platform/qcom/iris/iris_vb2.c b/drivers/media/platform/qcom/iris/iris_vb2.c index db8768d8a8f61..bf0b8400996ec 100644 --- a/drivers/media/platform/qcom/iris/iris_vb2.c +++ b/drivers/media/platform/qcom/iris/iris_vb2.c @@ -193,10 +193,14 @@ int iris_vb2_start_streaming(struct vb2_queue *q, unsigned int count) buf_type = iris_v4l2_type_to_driver(q->type); if (inst->domain == DECODER) { - if (inst->state == IRIS_INST_STREAMING) + if (buf_type == BUF_INPUT) + ret = iris_queue_deferred_buffers(inst, BUF_INPUT); + + if (!ret && inst->state == IRIS_INST_STREAMING) { ret = iris_queue_internal_deferred_buffers(inst, BUF_DPB); - if (!ret) - ret = iris_queue_deferred_buffers(inst, buf_type); + if (!ret) + ret = iris_queue_deferred_buffers(inst, BUF_OUTPUT); + } } else { if (inst->state == IRIS_INST_STREAMING) { ret = iris_queue_deferred_buffers(inst, BUF_INPUT); @@ -231,8 +235,6 @@ void iris_vb2_stop_streaming(struct vb2_queue *q) return; mutex_lock(&inst->lock); - if (inst->state == IRIS_INST_ERROR) - goto exit; if (!V4L2_TYPE_IS_OUTPUT(q->type) && !V4L2_TYPE_IS_CAPTURE(q->type)) @@ -243,10 +245,10 @@ void iris_vb2_stop_streaming(struct vb2_queue *q) goto exit; exit: - if (ret) { - iris_helper_buffers_done(inst, q->type, VB2_BUF_STATE_ERROR); + iris_helper_buffers_done(inst, q->type, VB2_BUF_STATE_ERROR); + if (ret) iris_inst_change_state(inst, IRIS_INST_ERROR); - } + mutex_unlock(&inst->lock); } diff --git a/drivers/media/platform/qcom/iris/iris_vdec.c b/drivers/media/platform/qcom/iris/iris_vdec.c index 2be3a85cb3a04..2a8e9fa5e5265 100644 --- a/drivers/media/platform/qcom/iris/iris_vdec.c +++ b/drivers/media/platform/qcom/iris/iris_vdec.c @@ -218,6 +218,14 @@ int iris_vdec_s_fmt(struct iris_inst *inst, struct v4l2_format *f) if (vb2_is_busy(q)) return -EBUSY; + /* Width and height are optional, so fall back to a valid placeholder + * resolution until the real one is decoded from the bitstream. + */ + if (f->fmt.pix_mp.width == 0 && f->fmt.pix_mp.height == 0) { + f->fmt.pix_mp.width = DEFAULT_WIDTH; + f->fmt.pix_mp.height = DEFAULT_HEIGHT; + } + iris_vdec_try_fmt(inst, f); switch (f->type) { diff --git a/drivers/media/platform/qcom/iris/iris_venc.c b/drivers/media/platform/qcom/iris/iris_venc.c index 95e3ceb22d8c3..f34bdae8b60f4 100644 --- a/drivers/media/platform/qcom/iris/iris_venc.c +++ b/drivers/media/platform/qcom/iris/iris_venc.c @@ -384,8 +384,7 @@ int iris_venc_s_param(struct iris_inst *inst, struct v4l2_streamparm *s_parm) struct v4l2_fract *timeperframe = NULL; u32 default_rate = DEFAULT_FPS; bool is_frame_rate = false; - u64 us_per_frame, fps; - u32 max_rate; + u32 fps, max_rate; int ret = 0; @@ -407,23 +406,19 @@ int iris_venc_s_param(struct iris_inst *inst, struct v4l2_streamparm *s_parm) timeperframe->denominator = default_rate; } - us_per_frame = timeperframe->numerator * (u64)USEC_PER_SEC; - do_div(us_per_frame, timeperframe->denominator); - - if (!us_per_frame) + fps = timeperframe->denominator / timeperframe->numerator; + if (!fps) return -EINVAL; - fps = (u64)USEC_PER_SEC; - do_div(fps, us_per_frame); if (fps > max_rate) { ret = -ENOMEM; goto reset_rate; } if (is_frame_rate) - inst->frame_rate = (u32)fps; + inst->frame_rate = fps; else - inst->operating_rate = (u32)fps; + inst->operating_rate = fps; if ((s_parm->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE && vb2_is_streaming(src_q)) || (s_parm->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE && vb2_is_streaming(dst_q))) { diff --git a/drivers/media/platform/qcom/venus/vdec.c b/drivers/media/platform/qcom/venus/vdec.c index 3c5ad94dbaadd..27cb024427b83 100644 --- a/drivers/media/platform/qcom/venus/vdec.c +++ b/drivers/media/platform/qcom/venus/vdec.c @@ -1448,10 +1448,10 @@ static void vdec_buf_done(struct venus_inst *inst, unsigned int buf_type, inst->drain_active = false; inst->codec_state = VENUS_DEC_STATE_STOPPED; } + } else { + if (!bytesused) + state = VB2_BUF_STATE_ERROR; } - - if (!bytesused) - state = VB2_BUF_STATE_ERROR; } else { vbuf->sequence = inst->sequence_out++; } diff --git a/drivers/media/platform/rockchip/rga/rga-buf.c b/drivers/media/platform/rockchip/rga/rga-buf.c index 730bdf98565a5..bb575873f2b24 100644 --- a/drivers/media/platform/rockchip/rga/rga-buf.c +++ b/drivers/media/platform/rockchip/rga/rga-buf.c @@ -80,6 +80,9 @@ static int rga_buf_init(struct vb2_buffer *vb) struct rga_frame *f = rga_get_frame(ctx, vb->vb2_queue->type); size_t n_desc = 0; + if (IS_ERR(f)) + return PTR_ERR(f); + n_desc = DIV_ROUND_UP(f->size, PAGE_SIZE); rbuf->n_desc = n_desc; diff --git a/drivers/media/platform/rockchip/rkisp1/rkisp1-params.c b/drivers/media/platform/rockchip/rkisp1/rkisp1-params.c index f1585f8fa0f47..1c90b1810fcb9 100644 --- a/drivers/media/platform/rockchip/rkisp1/rkisp1-params.c +++ b/drivers/media/platform/rockchip/rkisp1/rkisp1-params.c @@ -409,12 +409,6 @@ static void rkisp1_flt_config(struct rkisp1_params *params, rkisp1_write(params->rkisp1, RKISP1_CIF_ISP_FILT_LUM_WEIGHT, arg->lum_weight); - rkisp1_write(params->rkisp1, RKISP1_CIF_ISP_FILT_MODE, - (arg->mode ? RKISP1_CIF_ISP_FLT_MODE_DNR : 0) | - RKISP1_CIF_ISP_FLT_CHROMA_V_MODE(arg->chr_v_mode) | - RKISP1_CIF_ISP_FLT_CHROMA_H_MODE(arg->chr_h_mode) | - RKISP1_CIF_ISP_FLT_GREEN_STAGE1(arg->grn_stage1)); - /* avoid to override the old enable value */ filt_mode = rkisp1_read(params->rkisp1, RKISP1_CIF_ISP_FILT_MODE); filt_mode &= RKISP1_CIF_ISP_FLT_ENA; diff --git a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-bytecap.c b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-bytecap.c index 1c1b6b48918ee..b18e273ef4a3e 100644 --- a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-bytecap.c +++ b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-bytecap.c @@ -512,6 +512,9 @@ static void dcmipp_bytecap_stop_streaming(struct vb2_queue *vq) /* Disable pipe */ reg_clear(vcap, DCMIPP_P0FSCR, DCMIPP_P0FSCR_PIPEN); + /* Clear any pending interrupts */ + reg_write(vcap, DCMIPP_CMFCR, DCMIPP_CMIER_P0ALL); + spin_lock_irq(&vcap->irqlock); /* Return all queued buffers to vb2 in ERROR state */ diff --git a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-byteproc.c b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-byteproc.c index db76a02a1848a..ec1d773d5ad12 100644 --- a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-byteproc.c +++ b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-byteproc.c @@ -130,11 +130,8 @@ static void dcmipp_byteproc_adjust_compose(struct v4l2_rect *r, r->left = 0; /* Compose is not possible for JPEG or Bayer formats */ - if (fmt->code == MEDIA_BUS_FMT_JPEG_1X8 || - fmt->code == MEDIA_BUS_FMT_SBGGR8_1X8 || - fmt->code == MEDIA_BUS_FMT_SGBRG8_1X8 || - fmt->code == MEDIA_BUS_FMT_SGRBG8_1X8 || - fmt->code == MEDIA_BUS_FMT_SRGGB8_1X8) { + if (fmt->code >= MEDIA_BUS_FMT_SBGGR8_1X8 && + fmt->code <= MEDIA_BUS_FMT_JPEG_1X8) { r->width = fmt->width; r->height = fmt->height; return; diff --git a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-core.c b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-core.c index 1b7bae3266c8d..49398d0777646 100644 --- a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-core.c +++ b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-core.c @@ -526,7 +526,12 @@ static int dcmipp_probe(struct platform_device *pdev) return ret; } - kclk = devm_clk_get(&pdev->dev, "kclk"); + /* + * In case of the DCMIPP has only 1 clock (such as on MP13), the + * clock might not be named. + */ + kclk = devm_clk_get(&pdev->dev, + dcmipp->pipe_cfg->needs_mclk ? "kclk" : NULL); if (IS_ERR(kclk)) return dev_err_probe(&pdev->dev, PTR_ERR(kclk), "Unable to get kclk\n"); diff --git a/drivers/media/platform/ti/omap3isp/isppreview.c b/drivers/media/platform/ti/omap3isp/isppreview.c index e383a57654de8..5c492b31b5160 100644 --- a/drivers/media/platform/ti/omap3isp/isppreview.c +++ b/drivers/media/platform/ti/omap3isp/isppreview.c @@ -1742,22 +1742,17 @@ static void preview_try_format(struct isp_prev_device *prev, switch (pad) { case PREV_PAD_SINK: - /* When reading data from the CCDC, the input size has already - * been mangled by the CCDC output pad so it can be accepted - * as-is. - * - * When reading data from memory, clamp the requested width and - * height. The TRM doesn't specify a minimum input height, make + /* + * Clamp the requested width and height. + * The TRM doesn't specify a minimum input height, make * sure we got enough lines to enable the noise filter and color * filter array interpolation. */ - if (prev->input == PREVIEW_INPUT_MEMORY) { - fmt->width = clamp_t(u32, fmt->width, PREV_MIN_IN_WIDTH, - preview_max_out_width(prev)); - fmt->height = clamp_t(u32, fmt->height, - PREV_MIN_IN_HEIGHT, - PREV_MAX_IN_HEIGHT); - } + fmt->width = clamp_t(u32, fmt->width, PREV_MIN_IN_WIDTH, + preview_max_out_width(prev)); + fmt->height = clamp_t(u32, fmt->height, + PREV_MIN_IN_HEIGHT, + PREV_MAX_IN_HEIGHT); fmt->colorspace = V4L2_COLORSPACE_SRGB; diff --git a/drivers/media/platform/ti/omap3isp/ispvideo.c b/drivers/media/platform/ti/omap3isp/ispvideo.c index 0e7f0bf2b3463..eb33a776f27c9 100644 --- a/drivers/media/platform/ti/omap3isp/ispvideo.c +++ b/drivers/media/platform/ti/omap3isp/ispvideo.c @@ -148,12 +148,12 @@ static unsigned int isp_video_mbus_to_pix(const struct isp_video *video, pix->width = mbus->width; pix->height = mbus->height; - for (i = 0; i < ARRAY_SIZE(formats); ++i) { + for (i = 0; i < ARRAY_SIZE(formats) - 1; ++i) { if (formats[i].code == mbus->code) break; } - if (WARN_ON(i == ARRAY_SIZE(formats))) + if (WARN_ON(i == ARRAY_SIZE(formats) - 1)) return 0; min_bpl = pix->width * formats[i].bpp; @@ -191,7 +191,7 @@ static void isp_video_pix_to_mbus(const struct v4l2_pix_format *pix, /* Skip the last format in the loop so that it will be selected if no * match is found. */ - for (i = 0; i < ARRAY_SIZE(formats) - 1; ++i) { + for (i = 0; i < ARRAY_SIZE(formats) - 2; ++i) { if (formats[i].pixelformat == pix->pixelformat) break; } @@ -1288,6 +1288,7 @@ static const struct v4l2_ioctl_ops isp_video_ioctl_ops = { static int isp_video_open(struct file *file) { struct isp_video *video = video_drvdata(file); + struct v4l2_mbus_framefmt fmt; struct isp_video_fh *handle; struct vb2_queue *queue; int ret = 0; @@ -1330,6 +1331,13 @@ static int isp_video_open(struct file *file) memset(&handle->format, 0, sizeof(handle->format)); handle->format.type = video->type; + handle->format.fmt.pix.width = 720; + handle->format.fmt.pix.height = 480; + handle->format.fmt.pix.pixelformat = V4L2_PIX_FMT_UYVY; + handle->format.fmt.pix.field = V4L2_FIELD_NONE; + handle->format.fmt.pix.colorspace = V4L2_COLORSPACE_SRGB; + isp_video_pix_to_mbus(&handle->format.fmt.pix, &fmt); + isp_video_mbus_to_pix(video, &fmt, &handle->format.fmt.pix); handle->timeperframe.denominator = 1; handle->video = video; diff --git a/drivers/media/platform/verisilicon/rockchip_vpu981_hw_av1_dec.c b/drivers/media/platform/verisilicon/rockchip_vpu981_hw_av1_dec.c index e4703bb6be7c1..e4e21ad373233 100644 --- a/drivers/media/platform/verisilicon/rockchip_vpu981_hw_av1_dec.c +++ b/drivers/media/platform/verisilicon/rockchip_vpu981_hw_av1_dec.c @@ -72,6 +72,14 @@ : AV1_DIV_ROUND_UP_POW2((_value_), (_n_))); \ }) +enum rockchip_av1_tx_mode { + ROCKCHIP_AV1_TX_MODE_ONLY_4X4 = 0, + ROCKCHIP_AV1_TX_MODE_8X8 = 1, + ROCKCHIP_AV1_TX_MODE_16x16 = 2, + ROCKCHIP_AV1_TX_MODE_32x32 = 3, + ROCKCHIP_AV1_TX_MODE_SELECT = 4, +}; + struct rockchip_av1_film_grain { u8 scaling_lut_y[256]; u8 scaling_lut_cb[256]; @@ -373,12 +381,12 @@ int rockchip_vpu981_av1_dec_init(struct hantro_ctx *ctx) return -ENOMEM; av1_dec->global_model.size = GLOBAL_MODEL_SIZE; - av1_dec->tile_info.cpu = dma_alloc_coherent(vpu->dev, AV1_MAX_TILES, + av1_dec->tile_info.cpu = dma_alloc_coherent(vpu->dev, AV1_TILE_INFO_SIZE, &av1_dec->tile_info.dma, GFP_KERNEL); if (!av1_dec->tile_info.cpu) return -ENOMEM; - av1_dec->tile_info.size = AV1_MAX_TILES; + av1_dec->tile_info.size = AV1_TILE_INFO_SIZE; av1_dec->film_grain.cpu = dma_alloc_coherent(vpu->dev, ALIGN(sizeof(struct rockchip_av1_film_grain), 2048), @@ -1396,8 +1404,16 @@ static void rockchip_vpu981_av1_dec_set_cdef(struct hantro_ctx *ctx) u16 luma_sec_strength = 0; u32 chroma_pri_strength = 0; u16 chroma_sec_strength = 0; + bool enable_cdef; int i; + enable_cdef = !(cdef->bits == 0 && + cdef->damping_minus_3 == 0 && + cdef->y_pri_strength[0] == 0 && + cdef->y_sec_strength[0] == 0 && + cdef->uv_pri_strength[0] == 0 && + cdef->uv_sec_strength[0] == 0); + hantro_reg_write(vpu, &av1_enable_cdef, enable_cdef); hantro_reg_write(vpu, &av1_cdef_bits, cdef->bits); hantro_reg_write(vpu, &av1_cdef_damping, cdef->damping_minus_3); @@ -1927,11 +1943,26 @@ static void rockchip_vpu981_av1_dec_set_reference_frames(struct hantro_ctx *ctx) rockchip_vpu981_av1_dec_set_other_frames(ctx); } +static int rockchip_vpu981_av1_get_hardware_tx_mode(enum v4l2_av1_tx_mode tx_mode) +{ + switch (tx_mode) { + case V4L2_AV1_TX_MODE_ONLY_4X4: + return ROCKCHIP_AV1_TX_MODE_ONLY_4X4; + case V4L2_AV1_TX_MODE_LARGEST: + return ROCKCHIP_AV1_TX_MODE_32x32; + case V4L2_AV1_TX_MODE_SELECT: + return ROCKCHIP_AV1_TX_MODE_SELECT; + } + + return ROCKCHIP_AV1_TX_MODE_32x32; +} + static void rockchip_vpu981_av1_dec_set_parameters(struct hantro_ctx *ctx) { struct hantro_dev *vpu = ctx->dev; struct hantro_av1_dec_hw_ctx *av1_dec = &ctx->av1_dec; struct hantro_av1_dec_ctrls *ctrls = &av1_dec->ctrls; + int tx_mode; hantro_reg_write(vpu, &av1_skip_mode, !!(ctrls->frame->flags & V4L2_AV1_FRAME_FLAG_SKIP_MODE_PRESENT)); @@ -1953,8 +1984,6 @@ static void rockchip_vpu981_av1_dec_set_parameters(struct hantro_ctx *ctx) !!(ctrls->frame->flags & V4L2_AV1_FRAME_FLAG_SHOW_FRAME)); hantro_reg_write(vpu, &av1_switchable_motion_mode, !!(ctrls->frame->flags & V4L2_AV1_FRAME_FLAG_IS_MOTION_MODE_SWITCHABLE)); - hantro_reg_write(vpu, &av1_enable_cdef, - !!(ctrls->sequence->flags & V4L2_AV1_SEQUENCE_FLAG_ENABLE_CDEF)); hantro_reg_write(vpu, &av1_allow_masked_compound, !!(ctrls->sequence->flags & V4L2_AV1_SEQUENCE_FLAG_ENABLE_MASKED_COMPOUND)); @@ -1989,7 +2018,7 @@ static void rockchip_vpu981_av1_dec_set_parameters(struct hantro_ctx *ctx) !!(ctrls->frame->quantization.flags & V4L2_AV1_QUANTIZATION_FLAG_DELTA_Q_PRESENT)); - hantro_reg_write(vpu, &av1_idr_pic_e, !ctrls->frame->frame_type); + hantro_reg_write(vpu, &av1_idr_pic_e, IS_INTRA(ctrls->frame->frame_type)); hantro_reg_write(vpu, &av1_quant_base_qindex, ctrls->frame->quantization.base_q_idx); hantro_reg_write(vpu, &av1_bit_depth_y_minus8, ctx->bit_depth - 8); hantro_reg_write(vpu, &av1_bit_depth_c_minus8, ctx->bit_depth - 8); @@ -1999,7 +2028,9 @@ static void rockchip_vpu981_av1_dec_set_parameters(struct hantro_ctx *ctx) !!(ctrls->frame->flags & V4L2_AV1_FRAME_FLAG_ALLOW_HIGH_PRECISION_MV)); hantro_reg_write(vpu, &av1_comp_pred_mode, (ctrls->frame->flags & V4L2_AV1_FRAME_FLAG_REFERENCE_SELECT) ? 2 : 0); - hantro_reg_write(vpu, &av1_transform_mode, (ctrls->frame->tx_mode == 1) ? 3 : 4); + + tx_mode = rockchip_vpu981_av1_get_hardware_tx_mode(ctrls->frame->tx_mode); + hantro_reg_write(vpu, &av1_transform_mode, tx_mode); hantro_reg_write(vpu, &av1_max_cb_size, (ctrls->sequence->flags & V4L2_AV1_SEQUENCE_FLAG_USE_128X128_SUPERBLOCK) ? 7 : 6); diff --git a/drivers/media/radio/radio-keene.c b/drivers/media/radio/radio-keene.c index f3b57f0cb1ec4..c133305fd0194 100644 --- a/drivers/media/radio/radio-keene.c +++ b/drivers/media/radio/radio-keene.c @@ -338,7 +338,6 @@ static int usb_keene_probe(struct usb_interface *intf, if (hdl->error) { retval = hdl->error; - v4l2_ctrl_handler_free(hdl); goto err_v4l2; } retval = v4l2_device_register(&intf->dev, &radio->v4l2_dev); @@ -384,6 +383,7 @@ static int usb_keene_probe(struct usb_interface *intf, err_vdev: v4l2_device_unregister(&radio->v4l2_dev); err_v4l2: + v4l2_ctrl_handler_free(&radio->hdl); kfree(radio->buffer); kfree(radio); err: diff --git a/drivers/media/usb/pvrusb2/pvrusb2-hdw.c b/drivers/media/usb/pvrusb2/pvrusb2-hdw.c index b32bb906a9de2..5807734ae26c6 100644 --- a/drivers/media/usb/pvrusb2/pvrusb2-hdw.c +++ b/drivers/media/usb/pvrusb2/pvrusb2-hdw.c @@ -3709,6 +3709,11 @@ status); "Failed to submit read-control URB status=%d", status); hdw->ctl_read_pend_flag = 0; + if (hdw->ctl_write_pend_flag) { + usb_unlink_urb(hdw->ctl_write_urb); + while (hdw->ctl_write_pend_flag) + wait_for_completion(&hdw->ctl_done); + } goto done; } } diff --git a/drivers/media/usb/uvc/uvc_ctrl.c b/drivers/media/usb/uvc/uvc_ctrl.c index 2905505c240c0..2738ef74c7373 100644 --- a/drivers/media/usb/uvc/uvc_ctrl.c +++ b/drivers/media/usb/uvc/uvc_ctrl.c @@ -1432,7 +1432,7 @@ static bool uvc_ctrl_is_readable(u32 which, struct uvc_control *ctrl, * auto_exposure=1, exposure_time_absolute=251. */ int uvc_ctrl_is_accessible(struct uvc_video_chain *chain, u32 v4l2_id, - const struct v4l2_ext_controls *ctrls, + const struct v4l2_ext_controls *ctrls, u32 which, unsigned long ioctl) { struct uvc_control_mapping *master_map = NULL; @@ -1442,14 +1442,24 @@ int uvc_ctrl_is_accessible(struct uvc_video_chain *chain, u32 v4l2_id, s32 val; int ret; int i; + /* + * There is no need to check the ioctl, all the ioctls except + * VIDIOC_G_EXT_CTRLS use which=V4L2_CTRL_WHICH_CUR_VAL. + */ + bool is_which_min_max = which == V4L2_CTRL_WHICH_MIN_VAL || + which == V4L2_CTRL_WHICH_MAX_VAL; if (__uvc_query_v4l2_class(chain, v4l2_id, 0) >= 0) - return -EACCES; + return is_which_min_max ? -EINVAL : -EACCES; ctrl = uvc_find_control(chain, v4l2_id, &mapping); if (!ctrl) return -EINVAL; + if ((!(ctrl->info.flags & UVC_CTRL_FLAG_GET_MIN) || + !(ctrl->info.flags & UVC_CTRL_FLAG_GET_MAX)) && is_which_min_max) + return -EINVAL; + if (ioctl == VIDIOC_G_EXT_CTRLS) return uvc_ctrl_is_readable(ctrls->which, ctrl, mapping); diff --git a/drivers/media/usb/uvc/uvc_driver.c b/drivers/media/usb/uvc/uvc_driver.c index ee4f54d683496..aa3e8d295e0f5 100644 --- a/drivers/media/usb/uvc/uvc_driver.c +++ b/drivers/media/usb/uvc/uvc_driver.c @@ -165,28 +165,17 @@ static struct uvc_entity *uvc_entity_by_reference(struct uvc_device *dev, return NULL; } -static struct uvc_streaming *uvc_stream_by_id(struct uvc_device *dev, int id) +static struct uvc_streaming *uvc_stream_for_terminal(struct uvc_device *dev, + struct uvc_entity *term) { - struct uvc_streaming *stream, *last_stream; - unsigned int count = 0; + u16 id = UVC_HARDWARE_ENTITY_ID(term->id); + struct uvc_streaming *stream; list_for_each_entry(stream, &dev->streams, list) { - count += 1; - last_stream = stream; if (stream->header.bTerminalLink == id) return stream; } - /* - * If the streaming entity is referenced by an invalid ID, notify the - * user and use heuristics to guess the correct entity. - */ - if (count == 1 && id == UVC_INVALID_ENTITY_ID) { - dev_warn(&dev->intf->dev, - "UVC non compliance: Invalid USB header. The streaming entity has an invalid ID, guessing the correct one."); - return last_stream; - } - return NULL; } @@ -823,10 +812,12 @@ static struct uvc_entity *uvc_alloc_new_entity(struct uvc_device *dev, u16 type, } /* Per UVC 1.1+ spec 3.7.2, the ID is unique. */ - if (uvc_entity_by_id(dev, id)) { - dev_err(&dev->intf->dev, "Found multiple Units with ID %u\n", id); + if (uvc_entity_by_id(dev, UVC_HARDWARE_ENTITY_ID(id))) + dev_err(&dev->intf->dev, "Found multiple Units with ID %u\n", + UVC_HARDWARE_ENTITY_ID(id)); + + if (uvc_entity_by_id(dev, id)) id = UVC_INVALID_ENTITY_ID; - } extra_size = roundup(extra_size, sizeof(*entity->pads)); if (num_pads) @@ -982,6 +973,7 @@ static int uvc_parse_standard_control(struct uvc_device *dev, struct usb_host_interface *alts = dev->intf->cur_altsetting; unsigned int i, n, p, len; const char *type_name; + unsigned int id; u16 type; switch (buffer[2]) { @@ -1120,8 +1112,28 @@ static int uvc_parse_standard_control(struct uvc_device *dev, return 0; } + id = buffer[3]; + + /* + * Some devices, such as the Grandstream GUV3100, exhibit entity + * ID collisions between units and streaming output terminals. + * Move streaming output terminals to their own ID namespace by + * setting bit UVC_TERM_OUTPUT (15), above the ID's 8-bit value. + * The bit is ignored in uvc_stream_for_terminal() when looking + * up the streaming interface for the terminal. + * + * This hack is safe to enable unconditionally, as the ID is not + * used for any other purpose (streaming output terminals have + * no controls and are never referenced as sources in UVC + * descriptors). Other types output terminals can have controls, + * so limit usage of this separate namespace to streaming output + * terminals. + */ + if (type & UVC_TT_STREAMING) + id |= UVC_TERM_OUTPUT; + term = uvc_alloc_new_entity(dev, type | UVC_TERM_OUTPUT, - buffer[3], 1, 0); + id, 1, 0); if (IS_ERR(term)) return PTR_ERR(term); @@ -2118,8 +2130,8 @@ static int uvc_register_terms(struct uvc_device *dev, if (UVC_ENTITY_TYPE(term) != UVC_TT_STREAMING) continue; - stream = uvc_stream_by_id(dev, term->id); - if (stream == NULL) { + stream = uvc_stream_for_terminal(dev, term); + if (!stream) { dev_info(&dev->intf->dev, "No streaming interface found for terminal %u.", term->id); diff --git a/drivers/media/usb/uvc/uvc_queue.c b/drivers/media/usb/uvc/uvc_queue.c index 790184c9843d2..e838c6c1893a6 100644 --- a/drivers/media/usb/uvc/uvc_queue.c +++ b/drivers/media/usb/uvc/uvc_queue.c @@ -177,18 +177,20 @@ static int uvc_start_streaming_video(struct vb2_queue *vq, unsigned int count) ret = uvc_pm_get(stream->dev); if (ret) - return ret; + goto err_buffers; queue->buf_used = 0; ret = uvc_video_start_streaming(stream); - if (ret == 0) - return 0; + if (ret) + goto err_pm; - uvc_pm_put(stream->dev); + return 0; +err_pm: + uvc_pm_put(stream->dev); +err_buffers: uvc_queue_return_buffers(queue, UVC_BUF_STATE_QUEUED); - return ret; } diff --git a/drivers/media/usb/uvc/uvc_v4l2.c b/drivers/media/usb/uvc/uvc_v4l2.c index 9e4a251eca880..30c160daed8cb 100644 --- a/drivers/media/usb/uvc/uvc_v4l2.c +++ b/drivers/media/usb/uvc/uvc_v4l2.c @@ -765,14 +765,15 @@ static int uvc_ioctl_query_ext_ctrl(struct file *file, void *priv, static int uvc_ctrl_check_access(struct uvc_video_chain *chain, struct v4l2_ext_controls *ctrls, - unsigned long ioctl) + u32 which, unsigned long ioctl) { struct v4l2_ext_control *ctrl = ctrls->controls; unsigned int i; int ret = 0; for (i = 0; i < ctrls->count; ++ctrl, ++i) { - ret = uvc_ctrl_is_accessible(chain, ctrl->id, ctrls, ioctl); + ret = uvc_ctrl_is_accessible(chain, ctrl->id, ctrls, which, + ioctl); if (ret) break; } @@ -806,7 +807,7 @@ static int uvc_ioctl_g_ext_ctrls(struct file *file, void *priv, which = V4L2_CTRL_WHICH_CUR_VAL; } - ret = uvc_ctrl_check_access(chain, ctrls, VIDIOC_G_EXT_CTRLS); + ret = uvc_ctrl_check_access(chain, ctrls, which, VIDIOC_G_EXT_CTRLS); if (ret < 0) return ret; @@ -840,7 +841,8 @@ static int uvc_ioctl_s_try_ext_ctrls(struct uvc_fh *handle, if (!ctrls->count) return 0; - ret = uvc_ctrl_check_access(chain, ctrls, ioctl); + ret = uvc_ctrl_check_access(chain, ctrls, V4L2_CTRL_WHICH_CUR_VAL, + ioctl); if (ret < 0) return ret; diff --git a/drivers/media/usb/uvc/uvc_video.c b/drivers/media/usb/uvc/uvc_video.c index 2094e059d7d39..ec76595f3c4be 100644 --- a/drivers/media/usb/uvc/uvc_video.c +++ b/drivers/media/usb/uvc/uvc_video.c @@ -1812,7 +1812,7 @@ static int uvc_alloc_urb_buffers(struct uvc_streaming *stream, npackets = UVC_MAX_PACKETS; /* Retry allocations until one succeed. */ - for (; npackets > 1; npackets /= 2) { + for (; npackets > 0; npackets /= 2) { stream->urb_size = psize * npackets; for (i = 0; i < UVC_URBS; ++i) { @@ -1837,6 +1837,7 @@ static int uvc_alloc_urb_buffers(struct uvc_streaming *stream, uvc_dbg(stream->dev, VIDEO, "Failed to allocate URB buffers (%u bytes per packet)\n", psize); + stream->urb_size = 0; return 0; } diff --git a/drivers/media/usb/uvc/uvcvideo.h b/drivers/media/usb/uvc/uvcvideo.h index ed7bad31f75ca..8480d65ecb85e 100644 --- a/drivers/media/usb/uvc/uvcvideo.h +++ b/drivers/media/usb/uvc/uvcvideo.h @@ -41,7 +41,8 @@ #define UVC_EXT_GPIO_UNIT 0x7ffe #define UVC_EXT_GPIO_UNIT_ID 0x100 -#define UVC_INVALID_ENTITY_ID 0xffff +#define UVC_HARDWARE_ENTITY_ID(id) ((id) & 0xff) +#define UVC_INVALID_ENTITY_ID 0xffff /* ------------------------------------------------------------------------ * Driver specific constants. @@ -786,7 +787,7 @@ int uvc_ctrl_get(struct uvc_video_chain *chain, u32 which, struct v4l2_ext_control *xctrl); int uvc_ctrl_set(struct uvc_fh *handle, struct v4l2_ext_control *xctrl); int uvc_ctrl_is_accessible(struct uvc_video_chain *chain, u32 v4l2_id, - const struct v4l2_ext_controls *ctrls, + const struct v4l2_ext_controls *ctrls, u32 which, unsigned long ioctl); int uvc_xu_ctrl_query(struct uvc_video_chain *chain, diff --git a/drivers/media/v4l2-core/v4l2-async.c b/drivers/media/v4l2-core/v4l2-async.c index ee884a8221fbd..1c08bba9ecb91 100644 --- a/drivers/media/v4l2-core/v4l2-async.c +++ b/drivers/media/v4l2-core/v4l2-async.c @@ -343,7 +343,6 @@ static int v4l2_async_match_notify(struct v4l2_async_notifier *notifier, struct v4l2_subdev *sd, struct v4l2_async_connection *asc) { - struct v4l2_async_notifier *subdev_notifier; bool registered = false; int ret; @@ -389,6 +388,25 @@ static int v4l2_async_match_notify(struct v4l2_async_notifier *notifier, dev_dbg(notifier_dev(notifier), "v4l2-async: %s bound (ret %d)\n", dev_name(sd->dev), ret); + return 0; + +err_call_unbind: + v4l2_async_nf_call_unbind(notifier, sd, asc); + list_del(&asc->asc_subdev_entry); + +err_unregister_subdev: + if (registered) + v4l2_device_unregister_subdev(sd); + + return ret; +} + +static int +v4l2_async_nf_try_subdev_notifier(struct v4l2_async_notifier *notifier, + struct v4l2_subdev *sd) +{ + struct v4l2_async_notifier *subdev_notifier; + /* * See if the sub-device has a notifier. If not, return here. */ @@ -404,16 +422,6 @@ static int v4l2_async_match_notify(struct v4l2_async_notifier *notifier, subdev_notifier->parent = notifier; return v4l2_async_nf_try_all_subdevs(subdev_notifier); - -err_call_unbind: - v4l2_async_nf_call_unbind(notifier, sd, asc); - list_del(&asc->asc_subdev_entry); - -err_unregister_subdev: - if (registered) - v4l2_device_unregister_subdev(sd); - - return ret; } /* Test all async sub-devices in a notifier for a match. */ @@ -445,6 +453,10 @@ v4l2_async_nf_try_all_subdevs(struct v4l2_async_notifier *notifier) if (ret < 0) return ret; + ret = v4l2_async_nf_try_subdev_notifier(notifier, sd); + if (ret < 0) + return ret; + /* * v4l2_async_match_notify() may lead to registering a * new notifier and thus changing the async subdevs @@ -829,7 +841,11 @@ int __v4l2_async_register_subdev(struct v4l2_subdev *sd, struct module *module) ret = v4l2_async_match_notify(notifier, v4l2_dev, sd, asc); if (ret) - goto err_unbind; + goto err_unlock; + + ret = v4l2_async_nf_try_subdev_notifier(notifier, sd); + if (ret) + goto err_unbind_one; ret = v4l2_async_nf_try_complete(notifier); if (ret) @@ -853,9 +869,10 @@ int __v4l2_async_register_subdev(struct v4l2_subdev *sd, struct module *module) if (subdev_notifier) v4l2_async_nf_unbind_all_subdevs(subdev_notifier); - if (asc) - v4l2_async_unbind_subdev_one(notifier, asc); +err_unbind_one: + v4l2_async_unbind_subdev_one(notifier, asc); +err_unlock: mutex_unlock(&list_lock); sd->owner = NULL; diff --git a/drivers/memory/mtk-smi.c b/drivers/memory/mtk-smi.c index 733e22f695ab7..3609bfd3c64be 100644 --- a/drivers/memory/mtk-smi.c +++ b/drivers/memory/mtk-smi.c @@ -674,6 +674,7 @@ static int mtk_smi_larb_probe(struct platform_device *pdev) err_pm_disable: pm_runtime_disable(dev); device_link_remove(dev, larb->smi_common_dev); + put_device(larb->smi_common_dev); return ret; } @@ -684,6 +685,7 @@ static void mtk_smi_larb_remove(struct platform_device *pdev) device_link_remove(&pdev->dev, larb->smi_common_dev); pm_runtime_disable(&pdev->dev); component_del(&pdev->dev, &mtk_smi_larb_component_ops); + put_device(larb->smi_common_dev); } static int __maybe_unused mtk_smi_larb_resume(struct device *dev) @@ -917,6 +919,7 @@ static void mtk_smi_common_remove(struct platform_device *pdev) if (common->plat->type == MTK_SMI_GEN2_SUB_COMM) device_link_remove(&pdev->dev, common->smi_common_dev); pm_runtime_disable(&pdev->dev); + put_device(common->smi_common_dev); } static int __maybe_unused mtk_smi_common_resume(struct device *dev) diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig index 6cec1858947bf..55a9fea95195b 100644 --- a/drivers/mfd/Kconfig +++ b/drivers/mfd/Kconfig @@ -406,6 +406,17 @@ config MFD_CS47L92 help Support for Cirrus Logic CS42L92, CS47L92 and CS47L93 Smart Codecs +config MFD_TN48M_CPLD + tristate "Delta Networks TN48M switch CPLD driver" + depends on I2C + depends on ARCH_MVEBU || COMPILE_TEST + select MFD_SIMPLE_MFD_I2C + help + Select this option to enable support for Delta Networks TN48M switch + CPLD. It consists of reset and GPIO drivers. CPLD provides GPIOS-s + for the SFP slots as well as power supply related information. + SFP support depends on the GPIO driver being selected. + config PMIC_DA903X bool "Dialog Semiconductor DA9030/DA9034 PMIC Support" depends on I2C=y diff --git a/drivers/mfd/arizona-core.c b/drivers/mfd/arizona-core.c index 85ff8717d8504..91975536d14d2 100644 --- a/drivers/mfd/arizona-core.c +++ b/drivers/mfd/arizona-core.c @@ -1100,7 +1100,7 @@ int arizona_dev_init(struct arizona *arizona) } else if (val & 0x01) { ret = wm5102_clear_write_sequencer(arizona); if (ret) - return ret; + goto err_reset; } break; default: diff --git a/drivers/mfd/da9052-spi.c b/drivers/mfd/da9052-spi.c index 80fc5c0cac2fb..be5f2b34e18ae 100644 --- a/drivers/mfd/da9052-spi.c +++ b/drivers/mfd/da9052-spi.c @@ -37,7 +37,7 @@ static int da9052_spi_probe(struct spi_device *spi) spi_set_drvdata(spi, da9052); config = da9052_regmap_config; - config.write_flag_mask = 1; + config.read_flag_mask = 1; config.reg_bits = 7; config.pad_bits = 1; config.val_bits = 8; diff --git a/drivers/mfd/intel-lpss-pci.c b/drivers/mfd/intel-lpss-pci.c index 8d92c895d3aef..713a5bfb1a3c2 100644 --- a/drivers/mfd/intel-lpss-pci.c +++ b/drivers/mfd/intel-lpss-pci.c @@ -437,6 +437,19 @@ static const struct pci_device_id intel_lpss_pci_ids[] = { { PCI_VDEVICE(INTEL, 0x5ac4), (kernel_ulong_t)&bxt_spi_info }, { PCI_VDEVICE(INTEL, 0x5ac6), (kernel_ulong_t)&bxt_spi_info }, { PCI_VDEVICE(INTEL, 0x5aee), (kernel_ulong_t)&bxt_uart_info }, + /* NVL-S */ + { PCI_VDEVICE(INTEL, 0x6e28), (kernel_ulong_t)&bxt_uart_info }, + { PCI_VDEVICE(INTEL, 0x6e29), (kernel_ulong_t)&bxt_uart_info }, + { PCI_VDEVICE(INTEL, 0x6e2a), (kernel_ulong_t)&tgl_spi_info }, + { PCI_VDEVICE(INTEL, 0x6e2b), (kernel_ulong_t)&tgl_spi_info }, + { PCI_VDEVICE(INTEL, 0x6e4c), (kernel_ulong_t)&ehl_i2c_info }, + { PCI_VDEVICE(INTEL, 0x6e4d), (kernel_ulong_t)&ehl_i2c_info }, + { PCI_VDEVICE(INTEL, 0x6e4e), (kernel_ulong_t)&ehl_i2c_info }, + { PCI_VDEVICE(INTEL, 0x6e4f), (kernel_ulong_t)&ehl_i2c_info }, + { PCI_VDEVICE(INTEL, 0x6e5c), (kernel_ulong_t)&bxt_uart_info }, + { PCI_VDEVICE(INTEL, 0x6e5e), (kernel_ulong_t)&tgl_spi_info }, + { PCI_VDEVICE(INTEL, 0x6e7a), (kernel_ulong_t)&ehl_i2c_info }, + { PCI_VDEVICE(INTEL, 0x6e7b), (kernel_ulong_t)&ehl_i2c_info }, /* ARL-H */ { PCI_VDEVICE(INTEL, 0x7725), (kernel_ulong_t)&bxt_uart_info }, { PCI_VDEVICE(INTEL, 0x7726), (kernel_ulong_t)&bxt_uart_info }, diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index e6cdae221f1d4..3228e79c86eb5 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -413,6 +413,7 @@ static int apple_smc_probe(struct platform_device *pdev) if (!smc) return -ENOMEM; + mutex_init(&smc->mutex); smc->dev = &pdev->dev; smc->sram_base = devm_platform_get_and_ioremap_resource(pdev, 1, &smc->sram); if (IS_ERR(smc->sram_base)) diff --git a/drivers/mfd/mfd-core.c b/drivers/mfd/mfd-core.c index 7d14a1e7631ee..c55223ce4327a 100644 --- a/drivers/mfd/mfd-core.c +++ b/drivers/mfd/mfd-core.c @@ -22,6 +22,7 @@ #include static LIST_HEAD(mfd_of_node_list); +static DEFINE_MUTEX(mfd_of_node_mutex); struct mfd_of_node_entry { struct list_head list; @@ -105,9 +106,11 @@ static int mfd_match_of_node_to_dev(struct platform_device *pdev, u64 of_node_addr; /* Skip if OF node has previously been allocated to a device */ - list_for_each_entry(of_entry, &mfd_of_node_list, list) - if (of_entry->np == np) - return -EAGAIN; + scoped_guard(mutex, &mfd_of_node_mutex) { + list_for_each_entry(of_entry, &mfd_of_node_list, list) + if (of_entry->np == np) + return -EAGAIN; + } if (!cell->use_of_reg) /* No of_reg defined - allocate first free compatible match */ @@ -129,7 +132,8 @@ static int mfd_match_of_node_to_dev(struct platform_device *pdev, of_entry->dev = &pdev->dev; of_entry->np = np; - list_add_tail(&of_entry->list, &mfd_of_node_list); + scoped_guard(mutex, &mfd_of_node_mutex) + list_add_tail(&of_entry->list, &mfd_of_node_list); of_node_get(np); device_set_node(&pdev->dev, of_fwnode_handle(np)); @@ -286,11 +290,13 @@ static int mfd_add_device(struct device *parent, int id, if (cell->swnode) device_remove_software_node(&pdev->dev); fail_of_entry: - list_for_each_entry_safe(of_entry, tmp, &mfd_of_node_list, list) - if (of_entry->dev == &pdev->dev) { - list_del(&of_entry->list); - kfree(of_entry); - } + scoped_guard(mutex, &mfd_of_node_mutex) { + list_for_each_entry_safe(of_entry, tmp, &mfd_of_node_list, list) + if (of_entry->dev == &pdev->dev) { + list_del(&of_entry->list); + kfree(of_entry); + } + } fail_alias: regulator_bulk_unregister_supply_alias(&pdev->dev, cell->parent_supplies, @@ -360,11 +366,13 @@ static int mfd_remove_devices_fn(struct device *dev, void *data) if (cell->swnode) device_remove_software_node(&pdev->dev); - list_for_each_entry_safe(of_entry, tmp, &mfd_of_node_list, list) - if (of_entry->dev == &pdev->dev) { - list_del(&of_entry->list); - kfree(of_entry); - } + scoped_guard(mutex, &mfd_of_node_mutex) { + list_for_each_entry_safe(of_entry, tmp, &mfd_of_node_list, list) + if (of_entry->dev == &pdev->dev) { + list_del(&of_entry->list); + kfree(of_entry); + } + } regulator_bulk_unregister_supply_alias(dev, cell->parent_supplies, cell->num_parent_supplies); diff --git a/drivers/mfd/omap-usb-host.c b/drivers/mfd/omap-usb-host.c index a77b6fc790f2e..4d29a6e2ed87a 100644 --- a/drivers/mfd/omap-usb-host.c +++ b/drivers/mfd/omap-usb-host.c @@ -819,8 +819,10 @@ static void usbhs_omap_remove(struct platform_device *pdev) { pm_runtime_disable(&pdev->dev); - /* remove children */ - device_for_each_child(&pdev->dev, NULL, usbhs_omap_remove_child); + if (pdev->dev.of_node) + of_platform_depopulate(&pdev->dev); + else + device_for_each_child(&pdev->dev, NULL, usbhs_omap_remove_child); } static const struct dev_pm_ops usbhsomap_dev_pm_ops = { diff --git a/drivers/mfd/qcom-pm8xxx.c b/drivers/mfd/qcom-pm8xxx.c index 1149f7102a365..0cf374c015ce7 100644 --- a/drivers/mfd/qcom-pm8xxx.c +++ b/drivers/mfd/qcom-pm8xxx.c @@ -577,17 +577,11 @@ static int pm8xxx_probe(struct platform_device *pdev) return rc; } -static int pm8xxx_remove_child(struct device *dev, void *unused) -{ - platform_device_unregister(to_platform_device(dev)); - return 0; -} - static void pm8xxx_remove(struct platform_device *pdev) { struct pm_irq_chip *chip = platform_get_drvdata(pdev); - device_for_each_child(&pdev->dev, NULL, pm8xxx_remove_child); + of_platform_depopulate(&pdev->dev); irq_domain_remove(chip->irqdomain); } diff --git a/drivers/mfd/simple-mfd-i2c.c b/drivers/mfd/simple-mfd-i2c.c index 0a607a1e3ca1d..9f911afafc25e 100644 --- a/drivers/mfd/simple-mfd-i2c.c +++ b/drivers/mfd/simple-mfd-i2c.c @@ -110,6 +110,7 @@ static const struct simple_mfd_data spacemit_p1 = { }; static const struct of_device_id simple_mfd_i2c_of_match[] = { + { .compatible = "delta,tn48m-cpld" }, { .compatible = "fsl,ls1028aqds-fpga" }, { .compatible = "fsl,lx2160aqds-fpga" }, { .compatible = "fsl,lx2160ardb-fpga" }, diff --git a/drivers/mfd/tps65219.c b/drivers/mfd/tps65219.c index 65a952555218d..7275dcdb7c44f 100644 --- a/drivers/mfd/tps65219.c +++ b/drivers/mfd/tps65219.c @@ -498,6 +498,15 @@ static int tps65219_probe(struct i2c_client *client) return ret; } + if (chip_id == TPS65214) { + ret = i2c_smbus_write_byte_data(client, TPS65214_REG_LOCK, + TPS65214_LOCK_ACCESS_CMD); + if (ret) { + dev_err(tps->dev, "Failed to unlock registers %d\n", ret); + return ret; + } + } + ret = devm_regmap_add_irq_chip(tps->dev, tps->regmap, client->irq, IRQF_ONESHOT, 0, pmic->irq_chip, &tps->irq_data); diff --git a/drivers/misc/bcm-vk/bcm_vk_msg.c b/drivers/misc/bcm-vk/bcm_vk_msg.c index 1f42d1d5a630a..665a3888708ac 100644 --- a/drivers/misc/bcm-vk/bcm_vk_msg.c +++ b/drivers/misc/bcm-vk/bcm_vk_msg.c @@ -1010,6 +1010,9 @@ ssize_t bcm_vk_read(struct file *p_file, struct device *dev = &vk->pdev->dev; struct bcm_vk_msg_chan *chan = &vk->to_h_msg_chan; struct bcm_vk_wkent *entry = NULL, *iter; + struct vk_msg_blk tmp_msg; + u32 tmp_usr_msg_id; + u32 tmp_blks; u32 q_num; u32 rsp_length; @@ -1034,6 +1037,9 @@ ssize_t bcm_vk_read(struct file *p_file, entry = iter; } else { /* buffer not big enough */ + tmp_msg = iter->to_h_msg[0]; + tmp_usr_msg_id = iter->usr_msg_id; + tmp_blks = iter->to_h_blks; rc = -EMSGSIZE; } goto read_loop_exit; @@ -1052,14 +1058,12 @@ ssize_t bcm_vk_read(struct file *p_file, bcm_vk_free_wkent(dev, entry); } else if (rc == -EMSGSIZE) { - struct vk_msg_blk tmp_msg = entry->to_h_msg[0]; - /* * in this case, return just the first block, so * that app knows what size it is looking for. */ - set_msg_id(&tmp_msg, entry->usr_msg_id); - tmp_msg.size = entry->to_h_blks - 1; + set_msg_id(&tmp_msg, tmp_usr_msg_id); + tmp_msg.size = tmp_blks - 1; if (copy_to_user(buf, &tmp_msg, VK_MSGQ_BLK_SIZE) != 0) { dev_err(dev, "Error return 1st block in -EMSGSIZE\n"); rc = -EFAULT; diff --git a/drivers/misc/eeprom/eeprom_93xx46.c b/drivers/misc/eeprom/eeprom_93xx46.c index 9cae6f530679b..5230e910a1d11 100644 --- a/drivers/misc/eeprom/eeprom_93xx46.c +++ b/drivers/misc/eeprom/eeprom_93xx46.c @@ -45,6 +45,7 @@ struct eeprom_93xx46_platform_data { #define OP_START 0x4 #define OP_WRITE (OP_START | 0x1) #define OP_READ (OP_START | 0x2) +/* The following addresses are offset for the 1K EEPROM variant in 16-bit mode */ #define ADDR_EWDS 0x00 #define ADDR_ERAL 0x20 #define ADDR_EWEN 0x30 @@ -191,10 +192,7 @@ static int eeprom_93xx46_ew(struct eeprom_93xx46_dev *edev, int is_on) bits = edev->addrlen + 3; cmd_addr = OP_START << edev->addrlen; - if (edev->pdata->flags & EE_ADDR8) - cmd_addr |= (is_on ? ADDR_EWEN : ADDR_EWDS) << 1; - else - cmd_addr |= (is_on ? ADDR_EWEN : ADDR_EWDS); + cmd_addr |= (is_on ? ADDR_EWEN : ADDR_EWDS) << (edev->addrlen - 6); if (has_quirk_instruction_length(edev)) { cmd_addr <<= 2; @@ -328,10 +326,7 @@ static int eeprom_93xx46_eral(struct eeprom_93xx46_dev *edev) bits = edev->addrlen + 3; cmd_addr = OP_START << edev->addrlen; - if (edev->pdata->flags & EE_ADDR8) - cmd_addr |= ADDR_ERAL << 1; - else - cmd_addr |= ADDR_ERAL; + cmd_addr |= ADDR_ERAL << (edev->addrlen - 6); if (has_quirk_instruction_length(edev)) { cmd_addr <<= 2; diff --git a/drivers/misc/ti_fpc202.c b/drivers/misc/ti_fpc202.c index 7964e46c74482..8eb2b5ac98506 100644 --- a/drivers/misc/ti_fpc202.c +++ b/drivers/misc/ti_fpc202.c @@ -309,7 +309,6 @@ static void fpc202_remove_port(struct fpc202_priv *priv, int port_id) static int fpc202_probe(struct i2c_client *client) { struct device *dev = &client->dev; - struct device_node *i2c_handle; struct fpc202_priv *priv; int ret, port_id; @@ -357,7 +356,7 @@ static int fpc202_probe(struct i2c_client *client) bitmap_zero(priv->probed_ports, FPC202_NUM_PORTS); - for_each_child_of_node(dev->of_node, i2c_handle) { + for_each_child_of_node_scoped(dev->of_node, i2c_handle) { ret = of_property_read_u32(i2c_handle, "reg", &port_id); if (ret) { if (ret == -EINVAL) diff --git a/drivers/most/core.c b/drivers/most/core.c index da319d108ea1d..40d63e38fef54 100644 --- a/drivers/most/core.c +++ b/drivers/most/core.c @@ -1282,19 +1282,28 @@ int most_register_interface(struct most_interface *iface) int id; struct most_channel *c; - if (!iface || !iface->enqueue || !iface->configure || - !iface->poison_channel || (iface->num_channels > MAX_CHANNELS)) + if (!iface) return -EINVAL; + device_initialize(iface->dev); + + if (!iface->enqueue || !iface->configure || !iface->poison_channel || + (iface->num_channels > MAX_CHANNELS)) { + put_device(iface->dev); + return -EINVAL; + } + id = ida_alloc(&mdev_id, GFP_KERNEL); if (id < 0) { dev_err(iface->dev, "Failed to allocate device ID\n"); + put_device(iface->dev); return id; } iface->p = kzalloc(sizeof(*iface->p), GFP_KERNEL); if (!iface->p) { ida_free(&mdev_id, id); + put_device(iface->dev); return -ENOMEM; } @@ -1304,7 +1313,7 @@ int most_register_interface(struct most_interface *iface) iface->dev->bus = &mostbus; iface->dev->groups = interface_attr_groups; dev_set_drvdata(iface->dev, iface); - if (device_register(iface->dev)) { + if (device_add(iface->dev)) { dev_err(iface->dev, "Failed to register interface device\n"); kfree(iface->p); put_device(iface->dev); diff --git a/drivers/mtd/devices/mtd_intel_dg.c b/drivers/mtd/devices/mtd_intel_dg.c index b438ee5aacc34..114e69135b8d9 100644 --- a/drivers/mtd/devices/mtd_intel_dg.c +++ b/drivers/mtd/devices/mtd_intel_dg.c @@ -738,6 +738,7 @@ static int intel_dg_mtd_probe(struct auxiliary_device *aux_dev, kref_init(&nvm->refcnt); mutex_init(&nvm->lock); + nvm->nregions = nregions; for (n = 0, i = 0; i < INTEL_DG_NVM_REGIONS; i++) { if (!invm->regions[i].name) @@ -745,13 +746,15 @@ static int intel_dg_mtd_probe(struct auxiliary_device *aux_dev, char *name = kasprintf(GFP_KERNEL, "%s.%s", dev_name(&aux_dev->dev), invm->regions[i].name); - if (!name) - continue; + if (!name) { + ret = -ENOMEM; + goto err; + } + nvm->regions[n].name = name; nvm->regions[n].id = i; n++; } - nvm->nregions = n; /* in case where kasprintf fail */ nvm->base = devm_ioremap_resource(device, &invm->bar); if (IS_ERR(nvm->base)) { diff --git a/drivers/mtd/nand/raw/cadence-nand-controller.c b/drivers/mtd/nand/raw/cadence-nand-controller.c index 32ed38b893947..21eabedc2ed1c 100644 --- a/drivers/mtd/nand/raw/cadence-nand-controller.c +++ b/drivers/mtd/nand/raw/cadence-nand-controller.c @@ -1015,7 +1015,7 @@ static int cadence_nand_cdma_send(struct cdns_nand_ctrl *cdns_ctrl, } /* Send SDMA command and wait for finish. */ -static u32 +static int cadence_nand_cdma_send_and_wait(struct cdns_nand_ctrl *cdns_ctrl, u8 thread) { diff --git a/drivers/mtd/nand/raw/pl35x-nand-controller.c b/drivers/mtd/nand/raw/pl35x-nand-controller.c index 11bd90e3f18cb..7f012b7c3eaec 100644 --- a/drivers/mtd/nand/raw/pl35x-nand-controller.c +++ b/drivers/mtd/nand/raw/pl35x-nand-controller.c @@ -976,6 +976,7 @@ static int pl35x_nand_attach_chip(struct nand_chip *chip) fallthrough; case NAND_ECC_ENGINE_TYPE_NONE: case NAND_ECC_ENGINE_TYPE_SOFT: + chip->ecc.write_page_raw = nand_monolithic_write_page_raw; break; case NAND_ECC_ENGINE_TYPE_ON_HOST: ret = pl35x_nand_init_hw_ecc_controller(nfc, chip); diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c index f92133b8e1a60..697877584a285 100644 --- a/drivers/mtd/nand/spi/core.c +++ b/drivers/mtd/nand/spi/core.c @@ -859,6 +859,14 @@ static void spinand_cont_read_init(struct spinand_device *spinand) (engine_type == NAND_ECC_ENGINE_TYPE_ON_DIE || engine_type == NAND_ECC_ENGINE_TYPE_NONE)) { spinand->cont_read_possible = true; + + /* + * Ensure continuous read is disabled on probe. + * Some devices retain this state across soft reset, + * which leaves the OOB area inaccessible and results + * in false positive returns from spinand_isbad(). + */ + spinand_cont_read_enable(spinand, false); } } diff --git a/drivers/mtd/parsers/ofpart_core.c b/drivers/mtd/parsers/ofpart_core.c index abfa687989182..09961c6f39496 100644 --- a/drivers/mtd/parsers/ofpart_core.c +++ b/drivers/mtd/parsers/ofpart_core.c @@ -77,6 +77,7 @@ static int parse_fixed_partitions(struct mtd_info *master, of_id = of_match_node(parse_ofpart_match_table, ofpart_node); if (dedicated && !of_id) { /* The 'partitions' subnode might be used by another parser */ + of_node_put(ofpart_node); return 0; } @@ -91,12 +92,18 @@ static int parse_fixed_partitions(struct mtd_info *master, nr_parts++; } - if (nr_parts == 0) + if (nr_parts == 0) { + if (dedicated) + of_node_put(ofpart_node); return 0; + } parts = kcalloc(nr_parts, sizeof(*parts), GFP_KERNEL); - if (!parts) + if (!parts) { + if (dedicated) + of_node_put(ofpart_node); return -ENOMEM; + } i = 0; for_each_child_of_node(ofpart_node, pp) { @@ -175,6 +182,9 @@ static int parse_fixed_partitions(struct mtd_info *master, if (quirks && quirks->post_parse) quirks->post_parse(master, parts, nr_parts); + if (dedicated) + of_node_put(ofpart_node); + *pparts = parts; return nr_parts; @@ -183,6 +193,8 @@ static int parse_fixed_partitions(struct mtd_info *master, master->name, pp, mtd_node); ret = -EINVAL; ofpart_none: + if (dedicated) + of_node_put(ofpart_node); of_node_put(pp); kfree(parts); return ret; diff --git a/drivers/mtd/parsers/tplink_safeloader.c b/drivers/mtd/parsers/tplink_safeloader.c index e358a029dc70c..4fcaf92d22e4f 100644 --- a/drivers/mtd/parsers/tplink_safeloader.c +++ b/drivers/mtd/parsers/tplink_safeloader.c @@ -116,6 +116,7 @@ static int mtd_parser_tplink_safeloader_parse(struct mtd_info *mtd, return idx; err_free: + kfree(buf); for (idx -= 1; idx >= 0; idx--) kfree(parts[idx].name); err_free_parts: diff --git a/drivers/mux/mmio.c b/drivers/mux/mmio.c index 9993ce38a818d..5b0171d19d430 100644 --- a/drivers/mux/mmio.c +++ b/drivers/mux/mmio.c @@ -58,7 +58,7 @@ static int mux_mmio_probe(struct platform_device *pdev) if (IS_ERR(base)) regmap = ERR_PTR(-ENODEV); else - regmap = regmap_init_mmio(dev, base, &mux_mmio_regmap_cfg); + regmap = devm_regmap_init_mmio(dev, base, &mux_mmio_regmap_cfg); /* Fallback to checking the parent node on "real" errors. */ if (IS_ERR(regmap) && regmap != ERR_PTR(-EPROBE_DEFER)) { regmap = dev_get_regmap(dev->parent, NULL); diff --git a/drivers/net/arcnet/com20020-pci.c b/drivers/net/arcnet/com20020-pci.c index 0472bcdff1307..b5729d6c0b47c 100644 --- a/drivers/net/arcnet/com20020-pci.c +++ b/drivers/net/arcnet/com20020-pci.c @@ -115,6 +115,8 @@ static const struct attribute_group com20020_state_group = { .attrs = com20020_state_attrs, }; +static struct com20020_pci_card_info card_info_2p5mbit; + static void com20020pci_remove(struct pci_dev *pdev); static int com20020pci_probe(struct pci_dev *pdev, @@ -140,7 +142,7 @@ static int com20020pci_probe(struct pci_dev *pdev, ci = (struct com20020_pci_card_info *)id->driver_data; if (!ci) - return -EINVAL; + ci = &card_info_2p5mbit; priv->ci = ci; mm = &ci->misc_map; @@ -347,6 +349,18 @@ static struct com20020_pci_card_info card_info_5mbit = { .flags = ARC_IS_5MBIT, }; +static struct com20020_pci_card_info card_info_2p5mbit = { + .name = "ARC-PCI", + .devcount = 1, + .chan_map_tbl = { + { + .bar = 2, + .offset = 0x00, + .size = 0x08, + }, + }, +}; + static struct com20020_pci_card_info card_info_sohard = { .name = "SOHARD SH ARC-PCI", .devcount = 1, diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index 51733fb29bd77..55f98d6254af8 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -324,7 +324,7 @@ static bool bond_sk_check(struct bonding *bond) } } -bool bond_xdp_check(struct bonding *bond, int mode) +bool __bond_xdp_check(int mode, int xmit_policy) { switch (mode) { case BOND_MODE_ROUNDROBIN: @@ -335,7 +335,7 @@ bool bond_xdp_check(struct bonding *bond, int mode) /* vlan+srcmac is not supported with XDP as in most cases the 802.1q * payload is not in the packet due to hardware offload. */ - if (bond->params.xmit_policy != BOND_XMIT_POLICY_VLAN_SRCMAC) + if (xmit_policy != BOND_XMIT_POLICY_VLAN_SRCMAC) return true; fallthrough; default: @@ -343,6 +343,11 @@ bool bond_xdp_check(struct bonding *bond, int mode) } } +bool bond_xdp_check(struct bonding *bond, int mode) +{ + return __bond_xdp_check(mode, bond->params.xmit_policy); +} + /*---------------------------------- VLAN -----------------------------------*/ /* In the following 2 functions, bond_vlan_rx_add_vid and bond_vlan_rx_kill_vid, @@ -791,26 +796,29 @@ static int bond_update_speed_duplex(struct slave *slave) struct ethtool_link_ksettings ecmd; int res; - slave->speed = SPEED_UNKNOWN; - slave->duplex = DUPLEX_UNKNOWN; - res = __ethtool_get_link_ksettings(slave_dev, &ecmd); if (res < 0) - return 1; + goto speed_duplex_unknown; if (ecmd.base.speed == 0 || ecmd.base.speed == ((__u32)-1)) - return 1; + goto speed_duplex_unknown; switch (ecmd.base.duplex) { case DUPLEX_FULL: case DUPLEX_HALF: break; default: - return 1; + goto speed_duplex_unknown; } slave->speed = ecmd.base.speed; slave->duplex = ecmd.base.duplex; return 0; + +speed_duplex_unknown: + slave->speed = SPEED_UNKNOWN; + slave->duplex = DUPLEX_UNKNOWN; + + return 1; } const char *bond_slave_link_status(s8 link) @@ -4402,9 +4410,13 @@ static int bond_close(struct net_device *bond_dev) bond_work_cancel_all(bond); bond->send_peer_notif = 0; + WRITE_ONCE(bond->recv_probe, NULL); + + /* Wait for any in-flight RX handlers */ + synchronize_net(); + if (bond_is_lb(bond)) bond_alb_deinitialize(bond); - bond->recv_probe = NULL; if (BOND_MODE(bond) == BOND_MODE_8023AD && bond->params.broadcast_neighbor) diff --git a/drivers/net/bonding/bond_options.c b/drivers/net/bonding/bond_options.c index f1c6e9d8f6167..adc216df43459 100644 --- a/drivers/net/bonding/bond_options.c +++ b/drivers/net/bonding/bond_options.c @@ -1574,6 +1574,8 @@ static int bond_option_fail_over_mac_set(struct bonding *bond, static int bond_option_xmit_hash_policy_set(struct bonding *bond, const struct bond_opt_value *newval) { + if (bond->xdp_prog && !__bond_xdp_check(BOND_MODE(bond), newval->value)) + return -EOPNOTSUPP; netdev_dbg(bond->dev, "Setting xmit hash policy to %s (%llu)\n", newval->string, newval->value); bond->params.xmit_policy = newval->value; diff --git a/drivers/net/caif/caif_serial.c b/drivers/net/caif/caif_serial.c index c398ac42eae90..b90890030751f 100644 --- a/drivers/net/caif/caif_serial.c +++ b/drivers/net/caif/caif_serial.c @@ -284,6 +284,7 @@ static void ser_release(struct work_struct *work) { struct list_head list; struct ser_device *ser, *tmp; + struct tty_struct *tty; spin_lock(&ser_lock); list_replace_init(&ser_release_list, &list); @@ -292,9 +293,11 @@ static void ser_release(struct work_struct *work) if (!list_empty(&list)) { rtnl_lock(); list_for_each_entry_safe(ser, tmp, &list, node) { + tty = ser->tty; dev_close(ser->dev); unregister_netdevice(ser->dev); debugfs_deinit(ser); + tty_kref_put(tty); } rtnl_unlock(); } @@ -355,8 +358,6 @@ static void ldisc_close(struct tty_struct *tty) { struct ser_device *ser = tty->disc_data; - tty_kref_put(ser->tty); - spin_lock(&ser_lock); list_move(&ser->node, &ser_release_list); spin_unlock(&ser_lock); diff --git a/drivers/net/can/spi/mcp251x.c b/drivers/net/can/spi/mcp251x.c index b797e08499d70..b46262e791301 100644 --- a/drivers/net/can/spi/mcp251x.c +++ b/drivers/net/can/spi/mcp251x.c @@ -1214,6 +1214,7 @@ static int mcp251x_open(struct net_device *net) { struct mcp251x_priv *priv = netdev_priv(net); struct spi_device *spi = priv->spi; + bool release_irq = false; unsigned long flags = 0; int ret; @@ -1257,12 +1258,24 @@ static int mcp251x_open(struct net_device *net) return 0; out_free_irq: - free_irq(spi->irq, priv); + /* The IRQ handler might be running, and if so it will be waiting + * for the lock. But free_irq() must wait for the handler to finish + * so calling it here would deadlock. + * + * Setting priv->force_quit will let the handler exit right away + * without any access to the hardware. This make it safe to call + * free_irq() after the lock is released. + */ + priv->force_quit = 1; + release_irq = true; + mcp251x_hw_sleep(spi); out_close: mcp251x_power_enable(priv->transceiver, 0); close_candev(net); mutex_unlock(&priv->mcp_lock); + if (release_irq) + free_irq(spi->irq, priv); return ret; } diff --git a/drivers/net/can/usb/ems_usb.c b/drivers/net/can/usb/ems_usb.c index fac8ac79df59f..d8c881130e900 100644 --- a/drivers/net/can/usb/ems_usb.c +++ b/drivers/net/can/usb/ems_usb.c @@ -445,6 +445,11 @@ static void ems_usb_read_bulk_callback(struct urb *urb) start = CPC_HEADER_SIZE; while (msg_count) { + if (start + CPC_MSG_HEADER_LEN > urb->actual_length) { + netdev_err(netdev, "format error\n"); + break; + } + msg = (struct ems_cpc_msg *)&ibuf[start]; switch (msg->type) { @@ -474,7 +479,7 @@ static void ems_usb_read_bulk_callback(struct urb *urb) start += CPC_MSG_HEADER_LEN + msg->length; msg_count--; - if (start > urb->transfer_buffer_length) { + if (start > urb->actual_length) { netdev_err(netdev, "format error\n"); break; } diff --git a/drivers/net/can/usb/etas_es58x/es58x_core.c b/drivers/net/can/usb/etas_es58x/es58x_core.c index 6eeba9baa1317..1e44bd1dfee9b 100644 --- a/drivers/net/can/usb/etas_es58x/es58x_core.c +++ b/drivers/net/can/usb/etas_es58x/es58x_core.c @@ -1461,12 +1461,18 @@ static void es58x_read_bulk_callback(struct urb *urb) } resubmit_urb: + usb_anchor_urb(urb, &es58x_dev->rx_urbs); ret = usb_submit_urb(urb, GFP_ATOMIC); + if (!ret) + return; + + usb_unanchor_urb(urb); + if (ret == -ENODEV) { for (i = 0; i < es58x_dev->num_can_ch; i++) if (es58x_dev->netdev[i]) netif_device_detach(es58x_dev->netdev[i]); - } else if (ret) + } else dev_err_ratelimited(dev, "Failed resubmitting read bulk urb: %pe\n", ERR_PTR(ret)); diff --git a/drivers/net/can/usb/f81604.c b/drivers/net/can/usb/f81604.c index e0cfa1460b0b8..6b8b2795c0188 100644 --- a/drivers/net/can/usb/f81604.c +++ b/drivers/net/can/usb/f81604.c @@ -413,6 +413,7 @@ static void f81604_read_bulk_callback(struct urb *urb) { struct f81604_can_frame *frame = urb->transfer_buffer; struct net_device *netdev = urb->context; + struct f81604_port_priv *priv = netdev_priv(netdev); int ret; if (!netif_device_present(netdev)) @@ -445,10 +446,15 @@ static void f81604_read_bulk_callback(struct urb *urb) f81604_process_rx_packet(netdev, frame); resubmit_urb: + usb_anchor_urb(urb, &priv->urbs_anchor); ret = usb_submit_urb(urb, GFP_ATOMIC); + if (!ret) + return; + usb_unanchor_urb(urb); + if (ret == -ENODEV) netif_device_detach(netdev); - else if (ret) + else netdev_err(netdev, "%s: failed to resubmit read bulk urb: %pe\n", __func__, ERR_PTR(ret)); @@ -620,6 +626,12 @@ static void f81604_read_int_callback(struct urb *urb) netdev_info(netdev, "%s: Int URB aborted: %pe\n", __func__, ERR_PTR(urb->status)); + if (urb->actual_length < sizeof(*data)) { + netdev_warn(netdev, "%s: short int URB: %u < %zu\n", + __func__, urb->actual_length, sizeof(*data)); + goto resubmit_urb; + } + switch (urb->status) { case 0: /* success */ break; @@ -646,10 +658,15 @@ static void f81604_read_int_callback(struct urb *urb) f81604_handle_tx(priv, data); resubmit_urb: + usb_anchor_urb(urb, &priv->urbs_anchor); ret = usb_submit_urb(urb, GFP_ATOMIC); + if (!ret) + return; + usb_unanchor_urb(urb); + if (ret == -ENODEV) netif_device_detach(netdev); - else if (ret) + else netdev_err(netdev, "%s: failed to resubmit int urb: %pe\n", __func__, ERR_PTR(ret)); } @@ -874,9 +891,27 @@ static void f81604_write_bulk_callback(struct urb *urb) if (!netif_device_present(netdev)) return; - if (urb->status) - netdev_info(netdev, "%s: Tx URB error: %pe\n", __func__, - ERR_PTR(urb->status)); + if (!urb->status) + return; + + switch (urb->status) { + case -ENOENT: + case -ECONNRESET: + case -ESHUTDOWN: + return; + default: + break; + } + + if (net_ratelimit()) + netdev_err(netdev, "%s: Tx URB error: %pe\n", __func__, + ERR_PTR(urb->status)); + + can_free_echo_skb(netdev, 0, NULL); + netdev->stats.tx_dropped++; + netdev->stats.tx_errors++; + + netif_wake_queue(netdev); } static void f81604_clear_reg_work(struct work_struct *work) diff --git a/drivers/net/can/usb/ucan.c b/drivers/net/can/usb/ucan.c index 07406daf7c88e..6c90b4a7d955a 100644 --- a/drivers/net/can/usb/ucan.c +++ b/drivers/net/can/usb/ucan.c @@ -749,7 +749,7 @@ static void ucan_read_bulk_callback(struct urb *urb) len = le16_to_cpu(m->len); /* check sanity (length of content) */ - if (urb->actual_length - pos < len) { + if ((len == 0) || (urb->actual_length - pos < len)) { netdev_warn(up->netdev, "invalid message (short; no data; l:%d)\n", urb->actual_length); diff --git a/drivers/net/dsa/realtek/rtl8365mb.c b/drivers/net/dsa/realtek/rtl8365mb.c index 964a56ee16cc9..d06b384d47643 100644 --- a/drivers/net/dsa/realtek/rtl8365mb.c +++ b/drivers/net/dsa/realtek/rtl8365mb.c @@ -769,7 +769,7 @@ static int rtl8365mb_phy_ocp_write(struct realtek_priv *priv, int phy, out: rtl83xx_unlock(priv); - return 0; + return ret; } static int rtl8365mb_phy_read(struct realtek_priv *priv, int phy, int regnum) diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-common.h b/drivers/net/ethernet/amd/xgbe/xgbe-common.h index 62b01de93db49..826c5caa70d71 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-common.h +++ b/drivers/net/ethernet/amd/xgbe/xgbe-common.h @@ -431,7 +431,7 @@ #define MAC_SSIR_SSINC_INDEX 16 #define MAC_SSIR_SSINC_WIDTH 8 #define MAC_TCR_SS_INDEX 29 -#define MAC_TCR_SS_WIDTH 2 +#define MAC_TCR_SS_WIDTH 3 #define MAC_TCR_TE_INDEX 0 #define MAC_TCR_TE_WIDTH 1 #define MAC_TCR_VNE_INDEX 24 diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c index ba5e728ae6308..89ece3dbd773a 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c @@ -1089,7 +1089,6 @@ int xgbe_powerdown(struct net_device *netdev, unsigned int caller) { struct xgbe_prv_data *pdata = netdev_priv(netdev); struct xgbe_hw_if *hw_if = &pdata->hw_if; - unsigned long flags; DBGPR("-->xgbe_powerdown\n"); @@ -1100,8 +1099,6 @@ int xgbe_powerdown(struct net_device *netdev, unsigned int caller) return -EINVAL; } - spin_lock_irqsave(&pdata->lock, flags); - if (caller == XGMAC_DRIVER_CONTEXT) netif_device_detach(netdev); @@ -1117,8 +1114,6 @@ int xgbe_powerdown(struct net_device *netdev, unsigned int caller) pdata->power_down = 1; - spin_unlock_irqrestore(&pdata->lock, flags); - DBGPR("<--xgbe_powerdown\n"); return 0; @@ -1128,7 +1123,6 @@ int xgbe_powerup(struct net_device *netdev, unsigned int caller) { struct xgbe_prv_data *pdata = netdev_priv(netdev); struct xgbe_hw_if *hw_if = &pdata->hw_if; - unsigned long flags; DBGPR("-->xgbe_powerup\n"); @@ -1139,8 +1133,6 @@ int xgbe_powerup(struct net_device *netdev, unsigned int caller) return -EINVAL; } - spin_lock_irqsave(&pdata->lock, flags); - pdata->power_down = 0; xgbe_napi_enable(pdata, 0); @@ -1155,8 +1147,6 @@ int xgbe_powerup(struct net_device *netdev, unsigned int caller) xgbe_start_timers(pdata); - spin_unlock_irqrestore(&pdata->lock, flags); - DBGPR("<--xgbe_powerup\n"); return 0; diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-main.c b/drivers/net/ethernet/amd/xgbe/xgbe-main.c index d1f0419edb234..7d45ea22a02e2 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-main.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-main.c @@ -76,7 +76,6 @@ struct xgbe_prv_data *xgbe_alloc_pdata(struct device *dev) pdata->netdev = netdev; pdata->dev = dev; - spin_lock_init(&pdata->lock); spin_lock_init(&pdata->xpcs_lock); mutex_init(&pdata->rss_mutex); spin_lock_init(&pdata->tstamp_lock); diff --git a/drivers/net/ethernet/amd/xgbe/xgbe.h b/drivers/net/ethernet/amd/xgbe/xgbe.h index e8bbb68059013..6fec51a065e22 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe.h +++ b/drivers/net/ethernet/amd/xgbe/xgbe.h @@ -1003,9 +1003,6 @@ struct xgbe_prv_data { unsigned int pp3; unsigned int pp4; - /* Overall device lock */ - spinlock_t lock; - /* XPCS indirect addressing lock */ spinlock_t xpcs_lock; unsigned int xpcs_window_def_reg; diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c b/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c index 198f49b40dbf0..2994f10446a63 100644 --- a/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c +++ b/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c @@ -442,7 +442,7 @@ __bnge_hwrm_reserve_pf_rings(struct bnge_dev *bd, struct bnge_hw_rings *hwr) struct hwrm_func_cfg_input *req; u32 enables = 0; - if (bnge_hwrm_req_init(bd, req, HWRM_FUNC_QCFG)) + if (bnge_hwrm_req_init(bd, req, HWRM_FUNC_CFG)) return NULL; req->fid = cpu_to_le16(0xffff); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 95c774d98da66..2b9c039d347d5 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -6201,6 +6201,9 @@ int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp, int rc; set_bit(BNXT_FLTR_FW_DELETED, &fltr->base.state); + if (!test_bit(BNXT_STATE_OPEN, &bp->state)) + return 0; + rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_FREE); if (rc) return rc; @@ -10809,12 +10812,10 @@ void bnxt_del_one_rss_ctx(struct bnxt *bp, struct bnxt_rss_ctx *rss_ctx, struct bnxt_ntuple_filter *ntp_fltr; int i; - if (netif_running(bp->dev)) { - bnxt_hwrm_vnic_free_one(bp, &rss_ctx->vnic); - for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++) { - if (vnic->fw_rss_cos_lb_ctx[i] != INVALID_HW_RING_ID) - bnxt_hwrm_vnic_ctx_free_one(bp, vnic, i); - } + bnxt_hwrm_vnic_free_one(bp, &rss_ctx->vnic); + for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++) { + if (vnic->fw_rss_cos_lb_ctx[i] != INVALID_HW_RING_ID) + bnxt_hwrm_vnic_ctx_free_one(bp, vnic, i); } if (!all) return; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index 41686a6f84b58..df4f0d15dd3d8 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -1340,16 +1340,17 @@ static int bnxt_add_ntuple_cls_rule(struct bnxt *bp, struct bnxt_l2_filter *l2_fltr; struct bnxt_flow_masks *fmasks; struct flow_keys *fkeys; - u32 idx, ring; + u32 idx; int rc; - u8 vf; if (!bp->vnic_info) return -EAGAIN; - vf = ethtool_get_flow_spec_ring_vf(fs->ring_cookie); - ring = ethtool_get_flow_spec_ring(fs->ring_cookie); - if ((fs->flow_type & (FLOW_MAC_EXT | FLOW_EXT)) || vf) + if (fs->flow_type & (FLOW_MAC_EXT | FLOW_EXT)) + return -EOPNOTSUPP; + + if (fs->ring_cookie != RX_CLS_FLOW_DISC && + ethtool_get_flow_spec_ring_vf(fs->ring_cookie)) return -EOPNOTSUPP; if (flow_type == IP_USER_FLOW) { @@ -1475,7 +1476,7 @@ static int bnxt_add_ntuple_cls_rule(struct bnxt *bp, if (fs->ring_cookie == RX_CLS_FLOW_DISC) new_fltr->base.flags |= BNXT_ACT_DROP; else - new_fltr->base.rxq = ring; + new_fltr->base.rxq = ethtool_get_flow_spec_ring(fs->ring_cookie); __set_bit(BNXT_FLTR_VALID, &new_fltr->base.state); rc = bnxt_insert_ntp_filter(bp, new_fltr, idx); if (!rc) { diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index 064fccdcf6994..90550055c71c1 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -741,14 +741,12 @@ static void macb_mac_link_up(struct phylink_config *config, if (rx_pause) ctrl |= MACB_BIT(PAE); - /* Initialize rings & buffers as clearing MACB_BIT(TE) in link down - * cleared the pipeline and control registers. - */ - macb_init_buffers(bp); - - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + queue->tx_head = 0; + queue->tx_tail = 0; queue_writel(queue, IER, bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); + } } macb_or_gem_writel(bp, NCFGR, ctrl); @@ -2991,6 +2989,7 @@ static int macb_open(struct net_device *dev) } bp->macbgem_ops.mog_init_rings(bp); + macb_init_buffers(bp); for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { napi_enable(&queue->napi_rx); diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c index 4ee970f3bad6e..ee0154337a9c5 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c @@ -1199,12 +1199,12 @@ static struct sock *chtls_recv_sock(struct sock *lsk, struct ipv6_pinfo *newnp = inet6_sk(newsk); struct ipv6_pinfo *np = inet6_sk(lsk); - inet_sk(newsk)->pinet6 = &newtcp6sk->inet6; + newinet->pinet6 = &newtcp6sk->inet6; + newinet->ipv6_fl_list = NULL; memcpy(newnp, np, sizeof(struct ipv6_pinfo)); newsk->sk_v6_daddr = treq->ir_v6_rmt_addr; newsk->sk_v6_rcv_saddr = treq->ir_v6_loc_addr; inet6_sk(newsk)->saddr = treq->ir_v6_loc_addr; - newnp->ipv6_fl_list = NULL; newnp->pktoptions = NULL; newsk->sk_bound_dev_if = treq->ir_iif; newinet->inet_opt = NULL; diff --git a/drivers/net/ethernet/ec_bhf.c b/drivers/net/ethernet/ec_bhf.c index 67275aa4f65b2..0c86cbb0313c3 100644 --- a/drivers/net/ethernet/ec_bhf.c +++ b/drivers/net/ethernet/ec_bhf.c @@ -423,7 +423,7 @@ static int ec_bhf_open(struct net_device *net_dev) error_rx_free: dma_free_coherent(dev, priv->rx_buf.alloc_len, priv->rx_buf.alloc, - priv->rx_buf.alloc_len); + priv->rx_buf.alloc_phys); out: return err; } diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c index 66240c340492c..e212a014c8d41 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c @@ -1533,7 +1533,7 @@ static irqreturn_t dpaa2_switch_irq0_handler_thread(int irq_num, void *arg) if_id = (status & 0xFFFF0000) >> 16; if (if_id >= ethsw->sw_attr.num_ifs) { dev_err(dev, "Invalid if_id %d in IRQ status\n", if_id); - goto out; + goto out_clear; } port_priv = ethsw->ports[if_id]; @@ -1553,6 +1553,7 @@ static irqreturn_t dpaa2_switch_irq0_handler_thread(int irq_num, void *arg) dpaa2_switch_port_connect_mac(port_priv); } +out_clear: err = dpsw_clear_irq_status(ethsw->mc_io, 0, ethsw->dpsw_handle, DPSW_IRQ_INDEX_IF, status); if (err) @@ -3034,6 +3035,13 @@ static int dpaa2_switch_init(struct fsl_mc_device *sw_dev) goto err_close; } + if (ethsw->sw_attr.num_ifs >= DPSW_MAX_IF) { + dev_err(dev, "DPSW num_ifs %u exceeds max %u\n", + ethsw->sw_attr.num_ifs, DPSW_MAX_IF); + err = -EINVAL; + goto err_close; + } + err = dpsw_get_api_version(ethsw->mc_io, 0, ðsw->major, ðsw->minor); diff --git a/drivers/net/ethernet/freescale/enetc/enetc.c b/drivers/net/ethernet/freescale/enetc/enetc.c index b6e3fb0401619..d97a76718dd89 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc.c +++ b/drivers/net/ethernet/freescale/enetc/enetc.c @@ -3458,7 +3458,7 @@ static int enetc_int_vector_init(struct enetc_ndev_priv *priv, int i, priv->rx_ring[i] = bdr; err = __xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0, - ENETC_RXB_DMA_SIZE_XDP); + ENETC_RXB_TRUESIZE); if (err) goto free_vector; diff --git a/drivers/net/ethernet/google/gve/gve_tx_dqo.c b/drivers/net/ethernet/google/gve/gve_tx_dqo.c index 6f1d515673d25..5fbee184f0e7a 100644 --- a/drivers/net/ethernet/google/gve/gve_tx_dqo.c +++ b/drivers/net/ethernet/google/gve/gve_tx_dqo.c @@ -167,6 +167,25 @@ gve_free_pending_packet(struct gve_tx_ring *tx, } } +static void gve_unmap_packet(struct device *dev, + struct gve_tx_pending_packet_dqo *pkt) +{ + int i; + + if (!pkt->num_bufs) + return; + + /* SKB linear portion is guaranteed to be mapped */ + dma_unmap_single(dev, dma_unmap_addr(pkt, dma[0]), + dma_unmap_len(pkt, len[0]), DMA_TO_DEVICE); + for (i = 1; i < pkt->num_bufs; i++) { + netmem_dma_unmap_page_attrs(dev, dma_unmap_addr(pkt, dma[i]), + dma_unmap_len(pkt, len[i]), + DMA_TO_DEVICE, 0); + } + pkt->num_bufs = 0; +} + /* gve_tx_free_desc - Cleans up all pending tx requests and buffers. */ static void gve_tx_clean_pending_packets(struct gve_tx_ring *tx) @@ -176,21 +195,12 @@ static void gve_tx_clean_pending_packets(struct gve_tx_ring *tx) for (i = 0; i < tx->dqo.num_pending_packets; i++) { struct gve_tx_pending_packet_dqo *cur_state = &tx->dqo.pending_packets[i]; - int j; - - for (j = 0; j < cur_state->num_bufs; j++) { - if (j == 0) { - dma_unmap_single(tx->dev, - dma_unmap_addr(cur_state, dma[j]), - dma_unmap_len(cur_state, len[j]), - DMA_TO_DEVICE); - } else { - dma_unmap_page(tx->dev, - dma_unmap_addr(cur_state, dma[j]), - dma_unmap_len(cur_state, len[j]), - DMA_TO_DEVICE); - } - } + + if (tx->dqo.qpl) + gve_free_tx_qpl_bufs(tx, cur_state); + else + gve_unmap_packet(tx->dev, cur_state); + if (cur_state->skb) { dev_consume_skb_any(cur_state->skb); cur_state->skb = NULL; @@ -1158,22 +1168,6 @@ static void remove_from_list(struct gve_tx_ring *tx, } } -static void gve_unmap_packet(struct device *dev, - struct gve_tx_pending_packet_dqo *pkt) -{ - int i; - - /* SKB linear portion is guaranteed to be mapped */ - dma_unmap_single(dev, dma_unmap_addr(pkt, dma[0]), - dma_unmap_len(pkt, len[0]), DMA_TO_DEVICE); - for (i = 1; i < pkt->num_bufs; i++) { - netmem_dma_unmap_page_attrs(dev, dma_unmap_addr(pkt, dma[i]), - dma_unmap_len(pkt, len[i]), - DMA_TO_DEVICE, 0); - } - pkt->num_bufs = 0; -} - /* Completion types and expected behavior: * No Miss compl + Packet compl = Packet completed normally. * Miss compl + Re-inject compl = Packet completed normally. diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index e976a88b952f0..c8eba180250e7 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -1048,13 +1048,13 @@ static void hns3_init_tx_spare_buffer(struct hns3_enet_ring *ring) int order; if (!alloc_size) - return; + goto not_init; order = get_order(alloc_size); if (order > MAX_PAGE_ORDER) { if (net_ratelimit()) dev_warn(ring_to_dev(ring), "failed to allocate tx spare buffer, exceed to max order\n"); - return; + goto not_init; } tx_spare = devm_kzalloc(ring_to_dev(ring), sizeof(*tx_spare), @@ -1092,6 +1092,13 @@ static void hns3_init_tx_spare_buffer(struct hns3_enet_ring *ring) devm_kfree(ring_to_dev(ring), tx_spare); devm_kzalloc_error: ring->tqp->handle->kinfo.tx_spare_buf_size = 0; +not_init: + /* When driver init or reset_init, the ring->tx_spare is always NULL; + * but when called from hns3_set_ringparam, it's usually not NULL, and + * will be restored if hns3_init_all_ring() failed. So it's safe to set + * ring->tx_spare to NULL here. + */ + ring->tx_spare = NULL; } /* Use hns3_tx_spare_space() to make sure there is enough buffer diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h index 416e02e7b995f..bc333d8710ac1 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h @@ -727,8 +727,8 @@ struct hclge_fd_tcam_config_3_cmd { #define HCLGE_FD_AD_DROP_B 0 #define HCLGE_FD_AD_DIRECT_QID_B 1 -#define HCLGE_FD_AD_QID_S 2 -#define HCLGE_FD_AD_QID_M GENMASK(11, 2) +#define HCLGE_FD_AD_QID_L_S 2 +#define HCLGE_FD_AD_QID_L_M GENMASK(11, 2) #define HCLGE_FD_AD_USE_COUNTER_B 12 #define HCLGE_FD_AD_COUNTER_NUM_S 13 #define HCLGE_FD_AD_COUNTER_NUM_M GENMASK(19, 13) @@ -741,6 +741,7 @@ struct hclge_fd_tcam_config_3_cmd { #define HCLGE_FD_AD_TC_OVRD_B 16 #define HCLGE_FD_AD_TC_SIZE_S 17 #define HCLGE_FD_AD_TC_SIZE_M GENMASK(20, 17) +#define HCLGE_FD_AD_QID_H_B 21 struct hclge_fd_ad_config_cmd { u8 stage; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index 5cc5ee9dcd982..54d0a9ba7879b 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -5679,11 +5679,13 @@ static int hclge_fd_ad_config(struct hclge_dev *hdev, u8 stage, int loc, hnae3_set_field(ad_data, HCLGE_FD_AD_TC_SIZE_M, HCLGE_FD_AD_TC_SIZE_S, (u32)action->tc_size); } + hnae3_set_bit(ad_data, HCLGE_FD_AD_QID_H_B, + action->queue_id >= HCLGE_TQP_MAX_SIZE_DEV_V2 ? 1 : 0); ad_data <<= 32; hnae3_set_bit(ad_data, HCLGE_FD_AD_DROP_B, action->drop_packet); hnae3_set_bit(ad_data, HCLGE_FD_AD_DIRECT_QID_B, action->forward_to_direct_queue); - hnae3_set_field(ad_data, HCLGE_FD_AD_QID_M, HCLGE_FD_AD_QID_S, + hnae3_set_field(ad_data, HCLGE_FD_AD_QID_L_M, HCLGE_FD_AD_QID_L_S, action->queue_id); hnae3_set_bit(ad_data, HCLGE_FD_AD_USE_COUNTER_B, action->use_counter); hnae3_set_field(ad_data, HCLGE_FD_AD_COUNTER_NUM_M, diff --git a/drivers/net/ethernet/intel/e1000e/defines.h b/drivers/net/ethernet/intel/e1000e/defines.h index ba331899d1861..d4a1041e456dc 100644 --- a/drivers/net/ethernet/intel/e1000e/defines.h +++ b/drivers/net/ethernet/intel/e1000e/defines.h @@ -33,6 +33,7 @@ /* Extended Device Control */ #define E1000_CTRL_EXT_LPCD 0x00000004 /* LCD Power Cycle Done */ +#define E1000_CTRL_EXT_DPG_EN 0x00000008 /* Dynamic Power Gating Enable */ #define E1000_CTRL_EXT_SDP3_DATA 0x00000080 /* Value of SW Definable Pin 3 */ #define E1000_CTRL_EXT_FORCE_SMBUS 0x00000800 /* Force SMBus mode */ #define E1000_CTRL_EXT_EE_RST 0x00002000 /* Reinitialize from EEPROM */ diff --git a/drivers/net/ethernet/intel/e1000e/ich8lan.c b/drivers/net/ethernet/intel/e1000e/ich8lan.c index df4e7d781cb1c..f9328caefe44b 100644 --- a/drivers/net/ethernet/intel/e1000e/ich8lan.c +++ b/drivers/net/ethernet/intel/e1000e/ich8lan.c @@ -4925,6 +4925,15 @@ static s32 e1000_reset_hw_ich8lan(struct e1000_hw *hw) reg |= E1000_KABGTXD_BGSQLBIAS; ew32(KABGTXD, reg); + /* The hardware reset value of the DPG_EN bit is 1. + * Clear DPG_EN to prevent unexpected autonomous power gating. + */ + if (hw->mac.type >= e1000_pch_ptp) { + reg = er32(CTRL_EXT); + reg &= ~E1000_CTRL_EXT_DPG_EN; + ew32(CTRL_EXT, reg); + } + return 0; } diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index d3bc3207054f9..598739220dfb9 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -75,7 +75,13 @@ static const struct pci_device_id i40e_pci_tbl[] = { {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0}, {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0}, {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0}, - {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_B), 0}, + /* + * This ID conflicts with ipw2200, but the devices can be differentiated + * because i40e devices use PCI_CLASS_NETWORK_ETHERNET and ipw2200 + * devices use PCI_CLASS_NETWORK_OTHER. + */ + {PCI_DEVICE(PCI_VENDOR_ID_INTEL, I40E_DEV_ID_10G_B), + PCI_CLASS_NETWORK_ETHERNET << 8, 0xffff00, 0}, {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0}, {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0}, {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0}, @@ -3563,6 +3569,7 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring) u16 pf_q = vsi->base_queue + ring->queue_index; struct i40e_hw *hw = &vsi->back->hw; struct i40e_hmc_obj_rxq rx_ctx; + u32 xdp_frame_sz; int err = 0; bool ok; @@ -3572,49 +3579,47 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring) memset(&rx_ctx, 0, sizeof(rx_ctx)); ring->rx_buf_len = vsi->rx_buf_len; + xdp_frame_sz = i40e_rx_pg_size(ring) / 2; /* XDP RX-queue info only needed for RX rings exposed to XDP */ if (ring->vsi->type != I40E_VSI_MAIN) goto skip; - if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) { - err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, - ring->queue_index, - ring->q_vector->napi.napi_id, - ring->rx_buf_len); - if (err) - return err; - } - ring->xsk_pool = i40e_xsk_pool(ring); if (ring->xsk_pool) { - xdp_rxq_info_unreg(&ring->xdp_rxq); + xdp_frame_sz = xsk_pool_get_rx_frag_step(ring->xsk_pool); ring->rx_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool); err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, ring->queue_index, ring->q_vector->napi.napi_id, - ring->rx_buf_len); + xdp_frame_sz); if (err) return err; err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, MEM_TYPE_XSK_BUFF_POOL, NULL); if (err) - return err; + goto unreg_xdp; dev_info(&vsi->back->pdev->dev, "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n", ring->queue_index); } else { + err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, + ring->queue_index, + ring->q_vector->napi.napi_id, + xdp_frame_sz); + if (err) + return err; err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, MEM_TYPE_PAGE_SHARED, NULL); if (err) - return err; + goto unreg_xdp; } skip: - xdp_init_buff(&ring->xdp, i40e_rx_pg_size(ring) / 2, &ring->xdp_rxq); + xdp_init_buff(&ring->xdp, xdp_frame_sz, &ring->xdp_rxq); rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len, BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT)); @@ -3648,7 +3653,8 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring) dev_info(&vsi->back->pdev->dev, "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", ring->queue_index, pf_q, err); - return -ENOMEM; + err = -ENOMEM; + goto unreg_xdp; } /* set the context in the HMC */ @@ -3657,7 +3663,8 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring) dev_info(&vsi->back->pdev->dev, "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", ring->queue_index, pf_q, err); - return -ENOMEM; + err = -ENOMEM; + goto unreg_xdp; } /* configure Rx buffer alignment */ @@ -3665,7 +3672,8 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring) if (I40E_2K_TOO_SMALL_WITH_PADDING) { dev_info(&vsi->back->pdev->dev, "2k Rx buffer is too small to fit standard MTU and skb_shared_info\n"); - return -EOPNOTSUPP; + err = -EOPNOTSUPP; + goto unreg_xdp; } clear_ring_build_skb_enabled(ring); } else { @@ -3695,6 +3703,11 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring) } return 0; +unreg_xdp: + if (ring->vsi->type == I40E_VSI_MAIN) + xdp_rxq_info_unreg(&ring->xdp_rxq); + + return err; } /** diff --git a/drivers/net/ethernet/intel/i40e/i40e_trace.h b/drivers/net/ethernet/intel/i40e/i40e_trace.h index 759f3d1c4c8f0..dde0ccd789ed1 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_trace.h +++ b/drivers/net/ethernet/intel/i40e/i40e_trace.h @@ -88,7 +88,7 @@ TRACE_EVENT(i40e_napi_poll, __entry->rx_clean_complete = rx_clean_complete; __entry->tx_clean_complete = tx_clean_complete; __entry->irq_num = q->irq_num; - __entry->curr_cpu = get_cpu(); + __entry->curr_cpu = smp_processor_id(); __assign_str(qname); __assign_str(dev_name); __assign_bitmask(irq_affinity, cpumask_bits(&q->affinity_mask), diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c index cc0b9efc2637a..816179c7e2712 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c @@ -1470,6 +1470,9 @@ void i40e_clean_rx_ring(struct i40e_ring *rx_ring) if (!rx_ring->rx_bi) return; + if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) + xdp_rxq_info_unreg(&rx_ring->xdp_rxq); + if (rx_ring->xsk_pool) { i40e_xsk_clean_rx_ring(rx_ring); goto skip_free; @@ -1527,8 +1530,6 @@ void i40e_clean_rx_ring(struct i40e_ring *rx_ring) void i40e_free_rx_resources(struct i40e_ring *rx_ring) { i40e_clean_rx_ring(rx_ring); - if (rx_ring->vsi->type == I40E_VSI_MAIN) - xdp_rxq_info_unreg(&rx_ring->xdp_rxq); rx_ring->xdp_prog = NULL; kfree(rx_ring->rx_bi); rx_ring->rx_bi = NULL; diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c index 4b0fc8f354bc9..53a0366fbf998 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_main.c +++ b/drivers/net/ethernet/intel/iavf/iavf_main.c @@ -2797,7 +2797,22 @@ static void iavf_init_config_adapter(struct iavf_adapter *adapter) netdev->watchdog_timeo = 5 * HZ; netdev->min_mtu = ETH_MIN_MTU; - netdev->max_mtu = LIBIE_MAX_MTU; + + /* PF/VF API: vf_res->max_mtu is max frame size (not MTU). + * Convert to MTU. + */ + if (!adapter->vf_res->max_mtu) { + netdev->max_mtu = LIBIE_MAX_MTU; + } else if (adapter->vf_res->max_mtu < LIBETH_RX_LL_LEN + ETH_MIN_MTU || + adapter->vf_res->max_mtu > + LIBETH_RX_LL_LEN + LIBIE_MAX_MTU) { + netdev_warn_once(adapter->netdev, + "invalid max frame size %d from PF, using default MTU %d", + adapter->vf_res->max_mtu, LIBIE_MAX_MTU); + netdev->max_mtu = LIBIE_MAX_MTU; + } else { + netdev->max_mtu = adapter->vf_res->max_mtu - LIBETH_RX_LL_LEN; + } if (!is_valid_ether_addr(adapter->hw.mac.addr)) { dev_info(&pdev->dev, "Invalid MAC address %pM, using random\n", diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h index a23ccd4ba08d2..6886188043764 100644 --- a/drivers/net/ethernet/intel/ice/ice.h +++ b/drivers/net/ethernet/intel/ice/ice.h @@ -988,6 +988,7 @@ int ice_schedule_reset(struct ice_pf *pf, enum ice_reset_req reset); void ice_print_link_msg(struct ice_vsi *vsi, bool isup); int ice_plug_aux_dev(struct ice_pf *pf); void ice_unplug_aux_dev(struct ice_pf *pf); +void ice_rdma_finalize_setup(struct ice_pf *pf); int ice_init_rdma(struct ice_pf *pf); void ice_deinit_rdma(struct ice_pf *pf); bool ice_is_wol_supported(struct ice_hw *hw); diff --git a/drivers/net/ethernet/intel/ice/ice_common.c b/drivers/net/ethernet/intel/ice/ice_common.c index eb148c8d9e083..95160c8dc1bba 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.c +++ b/drivers/net/ethernet/intel/ice/ice_common.c @@ -6428,7 +6428,7 @@ int ice_lldp_fltr_add_remove(struct ice_hw *hw, struct ice_vsi *vsi, bool add) struct ice_aqc_lldp_filter_ctrl *cmd; struct libie_aq_desc desc; - if (vsi->type != ICE_VSI_PF || !ice_fw_supports_lldp_fltr_ctrl(hw)) + if (!ice_fw_supports_lldp_fltr_ctrl(hw)) return -EOPNOTSUPP; cmd = libie_aq_raw(&desc); diff --git a/drivers/net/ethernet/intel/ice/ice_idc.c b/drivers/net/ethernet/intel/ice/ice_idc.c index 420d45c2558b6..ded029aa71d7d 100644 --- a/drivers/net/ethernet/intel/ice/ice_idc.c +++ b/drivers/net/ethernet/intel/ice/ice_idc.c @@ -360,6 +360,39 @@ void ice_unplug_aux_dev(struct ice_pf *pf) auxiliary_device_uninit(adev); } +/** + * ice_rdma_finalize_setup - Complete RDMA setup after VSI is ready + * @pf: ptr to ice_pf + * + * Sets VSI-dependent information and plugs aux device. + * Must be called after ice_init_rdma(), ice_vsi_rebuild(), and + * ice_dcb_rebuild() complete. + */ +void ice_rdma_finalize_setup(struct ice_pf *pf) +{ + struct device *dev = ice_pf_to_dev(pf); + struct iidc_rdma_priv_dev_info *privd; + int ret; + + if (!ice_is_rdma_ena(pf) || !pf->cdev_info) + return; + + privd = pf->cdev_info->iidc_priv; + if (!privd || !pf->vsi || !pf->vsi[0] || !pf->vsi[0]->netdev) + return; + + /* Assign VSI info now that VSI is valid */ + privd->netdev = pf->vsi[0]->netdev; + privd->vport_id = pf->vsi[0]->vsi_num; + + /* Update QoS info after DCB has been rebuilt */ + ice_setup_dcb_qos_info(pf, &privd->qos_info); + + ret = ice_plug_aux_dev(pf); + if (ret) + dev_warn(dev, "Failed to plug RDMA aux device: %d\n", ret); +} + /** * ice_init_rdma - initializes PF for RDMA use * @pf: ptr to ice_pf @@ -398,22 +431,14 @@ int ice_init_rdma(struct ice_pf *pf) } cdev->iidc_priv = privd; - privd->netdev = pf->vsi[0]->netdev; privd->hw_addr = (u8 __iomem *)pf->hw.hw_addr; cdev->pdev = pf->pdev; - privd->vport_id = pf->vsi[0]->vsi_num; pf->cdev_info->rdma_protocol |= IIDC_RDMA_PROTOCOL_ROCEV2; - ice_setup_dcb_qos_info(pf, &privd->qos_info); - ret = ice_plug_aux_dev(pf); - if (ret) - goto err_plug_aux_dev; + return 0; -err_plug_aux_dev: - pf->cdev_info->adev = NULL; - xa_erase(&ice_aux_id, pf->aux_idx); err_alloc_xa: kfree(privd); err_privd_alloc: @@ -432,7 +457,6 @@ void ice_deinit_rdma(struct ice_pf *pf) if (!ice_is_rdma_ena(pf)) return; - ice_unplug_aux_dev(pf); xa_erase(&ice_aux_id, pf->aux_idx); kfree(pf->cdev_info->iidc_priv); kfree(pf->cdev_info); diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index f2b91f7f87861..a4ae032f2161b 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -5147,6 +5147,9 @@ int ice_load(struct ice_pf *pf) if (err) goto err_init_rdma; + /* Finalize RDMA: VSI already created, assign info and plug device */ + ice_rdma_finalize_setup(pf); + ice_service_task_restart(pf); clear_bit(ICE_DOWN, pf->state); @@ -5178,6 +5181,7 @@ void ice_unload(struct ice_pf *pf) devl_assert_locked(priv_to_devlink(pf)); + ice_unplug_aux_dev(pf); ice_deinit_rdma(pf); ice_deinit_features(pf); ice_tc_indir_block_unregister(vsi); @@ -5604,6 +5608,7 @@ static int ice_suspend(struct device *dev) */ disabled = ice_service_task_stop(pf); + ice_unplug_aux_dev(pf); ice_deinit_rdma(pf); /* Already suspended?, then there is nothing to do */ @@ -7809,7 +7814,7 @@ static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) ice_health_clear(pf); - ice_plug_aux_dev(pf); + ice_rdma_finalize_setup(pf); if (ice_is_feature_supported(pf, ICE_F_SRIOV_LAG)) ice_lag_rebuild(pf); diff --git a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c index 3e191cf528b69..6c0a9296eccc8 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c +++ b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c @@ -291,9 +291,6 @@ static int idpf_del_flow_steer(struct net_device *netdev, vport_config = vport->adapter->vport_config[np->vport_idx]; user_config = &vport_config->user_config; - if (!idpf_sideband_action_ena(vport, fsp)) - return -EOPNOTSUPP; - rule = kzalloc(struct_size(rule, rule_info, 1), GFP_KERNEL); if (!rule) return -ENOMEM; diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index a48088eb9b822..c859665b2dc89 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -2326,7 +2326,7 @@ void idpf_wait_for_sw_marker_completion(const struct idpf_tx_queue *txq) do { struct idpf_splitq_4b_tx_compl_desc *tx_desc; - struct idpf_tx_queue *target; + struct idpf_tx_queue *target = NULL; u32 ctype_gen, id; tx_desc = flow ? &complq->comp[ntc].common : @@ -2346,14 +2346,14 @@ void idpf_wait_for_sw_marker_completion(const struct idpf_tx_queue *txq) target = complq->txq_grp->txqs[id]; idpf_queue_clear(SW_MARKER, target); - if (target == txq) - break; next: if (unlikely(++ntc == complq->desc_count)) { ntc = 0; gen_flag = !gen_flag; } + if (target == txq) + break; } while (time_before(jiffies, timeout)); idpf_queue_assign(GEN_CHK, complq, gen_flag); @@ -4038,7 +4038,7 @@ static int idpf_vport_intr_req_irq(struct idpf_vport *vport) continue; name = kasprintf(GFP_KERNEL, "%s-%s-%s-%d", drv_name, if_name, - vec_name, vidx); + vec_name, vector); err = request_irq(irq_num, idpf_vport_intr_clean_queues, 0, name, q_vector); diff --git a/drivers/net/ethernet/intel/igb/igb_xsk.c b/drivers/net/ethernet/intel/igb/igb_xsk.c index 30ce5fbb5b776..ce4a7b58cad2f 100644 --- a/drivers/net/ethernet/intel/igb/igb_xsk.c +++ b/drivers/net/ethernet/intel/igb/igb_xsk.c @@ -524,6 +524,16 @@ bool igb_xmit_zc(struct igb_ring *tx_ring, struct xsk_buff_pool *xsk_pool) return nb_pkts < budget; } +static u32 igb_sw_irq_prep(struct igb_q_vector *q_vector) +{ + u32 eics = 0; + + if (!napi_if_scheduled_mark_missed(&q_vector->napi)) + eics = q_vector->eims_value; + + return eics; +} + int igb_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) { struct igb_adapter *adapter = netdev_priv(dev); @@ -542,20 +552,32 @@ int igb_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) ring = adapter->tx_ring[qid]; - if (test_bit(IGB_RING_FLAG_TX_DISABLED, &ring->flags)) - return -ENETDOWN; - if (!READ_ONCE(ring->xsk_pool)) return -EINVAL; - if (!napi_if_scheduled_mark_missed(&ring->q_vector->napi)) { + if (flags & XDP_WAKEUP_TX) { + if (test_bit(IGB_RING_FLAG_TX_DISABLED, &ring->flags)) + return -ENETDOWN; + + eics |= igb_sw_irq_prep(ring->q_vector); + } + + if (flags & XDP_WAKEUP_RX) { + /* If IGB_FLAG_QUEUE_PAIRS is active, the q_vector + * and NAPI is shared between RX and TX. + * If NAPI is already running it would be marked as missed + * from the TX path, making this RX call a NOP + */ + ring = adapter->rx_ring[qid]; + eics |= igb_sw_irq_prep(ring->q_vector); + } + + if (eics) { /* Cause software interrupt */ - if (adapter->flags & IGB_FLAG_HAS_MSIX) { - eics |= ring->q_vector->eims_value; + if (adapter->flags & IGB_FLAG_HAS_MSIX) wr32(E1000_EICS, eics); - } else { + else wr32(E1000_ICS, E1000_ICS_RXDMT0); - } } return 0; diff --git a/drivers/net/ethernet/intel/libie/fwlog.c b/drivers/net/ethernet/intel/libie/fwlog.c index f39cc11cb7c56..5d890d9d3c4d5 100644 --- a/drivers/net/ethernet/intel/libie/fwlog.c +++ b/drivers/net/ethernet/intel/libie/fwlog.c @@ -1051,6 +1051,10 @@ void libie_fwlog_deinit(struct libie_fwlog *fwlog) { int status; + /* if FW logging isn't supported it means no configuration was done */ + if (!libie_fwlog_supported(fwlog)) + return; + /* make sure FW logging is disabled to not put the FW in a weird state * for the next driver load */ diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c b/drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c index b5805969404fa..01e82d0b6b2cd 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c @@ -307,7 +307,7 @@ static void octep_setup_iq_regs_cn93_pf(struct octep_device *oct, int iq_no) } /* Setup registers for a hardware Rx Queue */ -static void octep_setup_oq_regs_cn93_pf(struct octep_device *oct, int oq_no) +static int octep_setup_oq_regs_cn93_pf(struct octep_device *oct, int oq_no) { u64 reg_val; u64 oq_ctl = 0ULL; @@ -355,6 +355,7 @@ static void octep_setup_oq_regs_cn93_pf(struct octep_device *oct, int oq_no) reg_val = ((u64)time_threshold << 32) | CFG_GET_OQ_INTR_PKT(oct->conf); octep_write_csr64(oct, CN93_SDP_R_OUT_INT_LEVELS(oq_no), reg_val); + return 0; } /* Setup registers for a PF mailbox */ @@ -696,14 +697,26 @@ static void octep_enable_interrupts_cn93_pf(struct octep_device *oct) /* Disable all interrupts */ static void octep_disable_interrupts_cn93_pf(struct octep_device *oct) { - u64 intr_mask = 0ULL; + u64 reg_val, intr_mask = 0ULL; int srn, num_rings, i; srn = CFG_GET_PORTS_PF_SRN(oct->conf); num_rings = CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); - for (i = 0; i < num_rings; i++) - intr_mask |= (0x1ULL << (srn + i)); + for (i = 0; i < num_rings; i++) { + intr_mask |= BIT_ULL(srn + i); + reg_val = octep_read_csr64(oct, + CN93_SDP_R_IN_INT_LEVELS(srn + i)); + reg_val &= ~CN93_INT_ENA_BIT; + octep_write_csr64(oct, + CN93_SDP_R_IN_INT_LEVELS(srn + i), reg_val); + + reg_val = octep_read_csr64(oct, + CN93_SDP_R_OUT_INT_LEVELS(srn + i)); + reg_val &= ~CN93_INT_ENA_BIT; + octep_write_csr64(oct, + CN93_SDP_R_OUT_INT_LEVELS(srn + i), reg_val); + } octep_write_csr64(oct, CN93_SDP_EPF_IRERR_RINT_ENA_W1C, intr_mask); octep_write_csr64(oct, CN93_SDP_EPF_ORERR_RINT_ENA_W1C, intr_mask); diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_cnxk_pf.c b/drivers/net/ethernet/marvell/octeon_ep/octep_cnxk_pf.c index 5de0b5ecbc5fd..09a3f1d0645b8 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_cnxk_pf.c +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_cnxk_pf.c @@ -8,6 +8,7 @@ #include #include #include +#include #include "octep_config.h" #include "octep_main.h" @@ -327,12 +328,14 @@ static void octep_setup_iq_regs_cnxk_pf(struct octep_device *oct, int iq_no) } /* Setup registers for a hardware Rx Queue */ -static void octep_setup_oq_regs_cnxk_pf(struct octep_device *oct, int oq_no) +static int octep_setup_oq_regs_cnxk_pf(struct octep_device *oct, int oq_no) { - u64 reg_val; - u64 oq_ctl = 0ULL; - u32 time_threshold = 0; struct octep_oq *oq = oct->oq[oq_no]; + unsigned long t_out_jiffies; + u32 time_threshold = 0; + u64 oq_ctl = 0ULL; + u64 reg_ba_val; + u64 reg_val; oq_no += CFG_GET_PORTS_PF_SRN(oct->conf); reg_val = octep_read_csr64(oct, CNXK_SDP_R_OUT_CONTROL(oq_no)); @@ -343,6 +346,36 @@ static void octep_setup_oq_regs_cnxk_pf(struct octep_device *oct, int oq_no) reg_val = octep_read_csr64(oct, CNXK_SDP_R_OUT_CONTROL(oq_no)); } while (!(reg_val & CNXK_R_OUT_CTL_IDLE)); } + octep_write_csr64(oct, CNXK_SDP_R_OUT_WMARK(oq_no), oq->max_count); + /* Wait for WMARK to get applied */ + usleep_range(10, 15); + + octep_write_csr64(oct, CNXK_SDP_R_OUT_SLIST_BADDR(oq_no), + oq->desc_ring_dma); + octep_write_csr64(oct, CNXK_SDP_R_OUT_SLIST_RSIZE(oq_no), + oq->max_count); + reg_ba_val = octep_read_csr64(oct, CNXK_SDP_R_OUT_SLIST_BADDR(oq_no)); + + if (reg_ba_val != oq->desc_ring_dma) { + t_out_jiffies = jiffies + 10 * HZ; + do { + if (reg_ba_val == ULLONG_MAX) + return -EFAULT; + octep_write_csr64(oct, + CNXK_SDP_R_OUT_SLIST_BADDR(oq_no), + oq->desc_ring_dma); + octep_write_csr64(oct, + CNXK_SDP_R_OUT_SLIST_RSIZE(oq_no), + oq->max_count); + reg_ba_val = + octep_read_csr64(oct, + CNXK_SDP_R_OUT_SLIST_BADDR(oq_no)); + } while ((reg_ba_val != oq->desc_ring_dma) && + time_before(jiffies, t_out_jiffies)); + + if (reg_ba_val != oq->desc_ring_dma) + return -EAGAIN; + } reg_val &= ~(CNXK_R_OUT_CTL_IMODE); reg_val &= ~(CNXK_R_OUT_CTL_ROR_P); @@ -356,10 +389,6 @@ static void octep_setup_oq_regs_cnxk_pf(struct octep_device *oct, int oq_no) reg_val |= (CNXK_R_OUT_CTL_ES_P); octep_write_csr64(oct, CNXK_SDP_R_OUT_CONTROL(oq_no), reg_val); - octep_write_csr64(oct, CNXK_SDP_R_OUT_SLIST_BADDR(oq_no), - oq->desc_ring_dma); - octep_write_csr64(oct, CNXK_SDP_R_OUT_SLIST_RSIZE(oq_no), - oq->max_count); oq_ctl = octep_read_csr64(oct, CNXK_SDP_R_OUT_CONTROL(oq_no)); @@ -385,6 +414,7 @@ static void octep_setup_oq_regs_cnxk_pf(struct octep_device *oct, int oq_no) reg_val &= ~0xFFFFFFFFULL; reg_val |= CFG_GET_OQ_WMARK(oct->conf); octep_write_csr64(oct, CNXK_SDP_R_OUT_WMARK(oq_no), reg_val); + return 0; } /* Setup registers for a PF mailbox */ @@ -720,14 +750,26 @@ static void octep_enable_interrupts_cnxk_pf(struct octep_device *oct) /* Disable all interrupts */ static void octep_disable_interrupts_cnxk_pf(struct octep_device *oct) { - u64 intr_mask = 0ULL; + u64 reg_val, intr_mask = 0ULL; int srn, num_rings, i; srn = CFG_GET_PORTS_PF_SRN(oct->conf); num_rings = CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); - for (i = 0; i < num_rings; i++) - intr_mask |= (0x1ULL << (srn + i)); + for (i = 0; i < num_rings; i++) { + intr_mask |= BIT_ULL(srn + i); + reg_val = octep_read_csr64(oct, + CNXK_SDP_R_IN_INT_LEVELS(srn + i)); + reg_val &= ~CNXK_INT_ENA_BIT; + octep_write_csr64(oct, + CNXK_SDP_R_IN_INT_LEVELS(srn + i), reg_val); + + reg_val = octep_read_csr64(oct, + CNXK_SDP_R_OUT_INT_LEVELS(srn + i)); + reg_val &= ~CNXK_INT_ENA_BIT; + octep_write_csr64(oct, + CNXK_SDP_R_OUT_INT_LEVELS(srn + i), reg_val); + } octep_write_csr64(oct, CNXK_SDP_EPF_IRERR_RINT_ENA_W1C, intr_mask); octep_write_csr64(oct, CNXK_SDP_EPF_ORERR_RINT_ENA_W1C, intr_mask); diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_main.c b/drivers/net/ethernet/marvell/octeon_ep/octep_main.c index 57db7ea2f5be9..16f52d4b11e91 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_main.c +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_main.c @@ -555,28 +555,43 @@ static void octep_clean_irqs(struct octep_device *oct) } /** - * octep_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue. + * octep_update_pkt() - Update IQ/OQ IN/OUT_CNT registers. * * @iq: Octeon Tx queue data structure. * @oq: Octeon Rx queue data structure. */ -static void octep_enable_ioq_irq(struct octep_iq *iq, struct octep_oq *oq) +static void octep_update_pkt(struct octep_iq *iq, struct octep_oq *oq) { - u32 pkts_pend = oq->pkts_pending; + u32 pkts_pend = READ_ONCE(oq->pkts_pending); + u32 last_pkt_count = READ_ONCE(oq->last_pkt_count); + u32 pkts_processed = READ_ONCE(iq->pkts_processed); + u32 pkt_in_done = READ_ONCE(iq->pkt_in_done); netdev_dbg(iq->netdev, "enabling intr for Q-%u\n", iq->q_no); - if (iq->pkts_processed) { - writel(iq->pkts_processed, iq->inst_cnt_reg); - iq->pkt_in_done -= iq->pkts_processed; - iq->pkts_processed = 0; + if (pkts_processed) { + writel(pkts_processed, iq->inst_cnt_reg); + readl(iq->inst_cnt_reg); + WRITE_ONCE(iq->pkt_in_done, (pkt_in_done - pkts_processed)); + WRITE_ONCE(iq->pkts_processed, 0); } - if (oq->last_pkt_count - pkts_pend) { - writel(oq->last_pkt_count - pkts_pend, oq->pkts_sent_reg); - oq->last_pkt_count = pkts_pend; + if (last_pkt_count - pkts_pend) { + writel(last_pkt_count - pkts_pend, oq->pkts_sent_reg); + readl(oq->pkts_sent_reg); + WRITE_ONCE(oq->last_pkt_count, pkts_pend); } /* Flush the previous wrties before writing to RESEND bit */ - wmb(); + smp_wmb(); +} + +/** + * octep_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue. + * + * @iq: Octeon Tx queue data structure. + * @oq: Octeon Rx queue data structure. + */ +static void octep_enable_ioq_irq(struct octep_iq *iq, struct octep_oq *oq) +{ writeq(1UL << OCTEP_OQ_INTR_RESEND_BIT, oq->pkts_sent_reg); writeq(1UL << OCTEP_IQ_INTR_RESEND_BIT, iq->inst_cnt_reg); } @@ -602,7 +617,8 @@ static int octep_napi_poll(struct napi_struct *napi, int budget) if (tx_pending || rx_done >= budget) return budget; - napi_complete(napi); + octep_update_pkt(ioq_vector->iq, ioq_vector->oq); + napi_complete_done(napi, rx_done); octep_enable_ioq_irq(ioq_vector->iq, ioq_vector->oq); return rx_done; } diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_main.h b/drivers/net/ethernet/marvell/octeon_ep/octep_main.h index 81ac4267811c8..35d0ff289a70d 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_main.h +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_main.h @@ -77,7 +77,7 @@ struct octep_pci_win_regs { struct octep_hw_ops { void (*setup_iq_regs)(struct octep_device *oct, int q); - void (*setup_oq_regs)(struct octep_device *oct, int q); + int (*setup_oq_regs)(struct octep_device *oct, int q); void (*setup_mbox_regs)(struct octep_device *oct, int mbox); irqreturn_t (*mbox_intr_handler)(void *ioq_vector); diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cn9k_pf.h b/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cn9k_pf.h index ca473502d7a02..95f1dfff90cce 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cn9k_pf.h +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cn9k_pf.h @@ -386,5 +386,6 @@ #define CN93_PEM_BAR4_INDEX 7 #define CN93_PEM_BAR4_INDEX_SIZE 0x400000ULL #define CN93_PEM_BAR4_INDEX_OFFSET (CN93_PEM_BAR4_INDEX * CN93_PEM_BAR4_INDEX_SIZE) +#define CN93_INT_ENA_BIT BIT_ULL(62) #endif /* _OCTEP_REGS_CN9K_PF_H_ */ diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cnxk_pf.h b/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cnxk_pf.h index e637d7c8224d4..4d172a552f80c 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cnxk_pf.h +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cnxk_pf.h @@ -412,5 +412,6 @@ #define CNXK_PEM_BAR4_INDEX 7 #define CNXK_PEM_BAR4_INDEX_SIZE 0x400000ULL #define CNXK_PEM_BAR4_INDEX_OFFSET (CNXK_PEM_BAR4_INDEX * CNXK_PEM_BAR4_INDEX_SIZE) +#define CNXK_INT_ENA_BIT BIT_ULL(62) #endif /* _OCTEP_REGS_CNXK_PF_H_ */ diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c b/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c index 82b6b19e76b47..74de19166488f 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c @@ -12,6 +12,8 @@ #include "octep_config.h" #include "octep_main.h" +static void octep_oq_free_ring_buffers(struct octep_oq *oq); + static void octep_oq_reset_indices(struct octep_oq *oq) { oq->host_read_idx = 0; @@ -170,11 +172,15 @@ static int octep_setup_oq(struct octep_device *oct, int q_no) goto oq_fill_buff_err; octep_oq_reset_indices(oq); - oct->hw_ops.setup_oq_regs(oct, q_no); + if (oct->hw_ops.setup_oq_regs(oct, q_no)) + goto oq_setup_err; + oct->num_oqs++; return 0; +oq_setup_err: + octep_oq_free_ring_buffers(oq); oq_fill_buff_err: vfree(oq->buff_info); oq->buff_info = NULL; @@ -318,10 +324,16 @@ static int octep_oq_check_hw_for_pkts(struct octep_device *oct, struct octep_oq *oq) { u32 pkt_count, new_pkts; + u32 last_pkt_count, pkts_pending; pkt_count = readl(oq->pkts_sent_reg); - new_pkts = pkt_count - oq->last_pkt_count; + last_pkt_count = READ_ONCE(oq->last_pkt_count); + new_pkts = pkt_count - last_pkt_count; + if (pkt_count < last_pkt_count) { + dev_err(oq->dev, "OQ-%u pkt_count(%u) < oq->last_pkt_count(%u)\n", + oq->q_no, pkt_count, last_pkt_count); + } /* Clear the hardware packets counter register if the rx queue is * being processed continuously with-in a single interrupt and * reached half its max value. @@ -332,8 +344,9 @@ static int octep_oq_check_hw_for_pkts(struct octep_device *oct, pkt_count = readl(oq->pkts_sent_reg); new_pkts += pkt_count; } - oq->last_pkt_count = pkt_count; - oq->pkts_pending += new_pkts; + WRITE_ONCE(oq->last_pkt_count, pkt_count); + pkts_pending = READ_ONCE(oq->pkts_pending); + WRITE_ONCE(oq->pkts_pending, (pkts_pending + new_pkts)); return new_pkts; } @@ -408,7 +421,7 @@ static int __octep_oq_process_rx(struct octep_device *oct, u16 rx_ol_flags; u32 read_idx; - read_idx = oq->host_read_idx; + read_idx = READ_ONCE(oq->host_read_idx); rx_bytes = 0; desc_used = 0; for (pkt = 0; pkt < pkts_to_process; pkt++) { @@ -493,7 +506,7 @@ static int __octep_oq_process_rx(struct octep_device *oct, napi_gro_receive(oq->napi, skb); } - oq->host_read_idx = read_idx; + WRITE_ONCE(oq->host_read_idx, read_idx); oq->refill_count += desc_used; oq->stats->packets += pkt; oq->stats->bytes += rx_bytes; @@ -516,22 +529,26 @@ int octep_oq_process_rx(struct octep_oq *oq, int budget) { u32 pkts_available, pkts_processed, total_pkts_processed; struct octep_device *oct = oq->octep_dev; + u32 pkts_pending; pkts_available = 0; pkts_processed = 0; total_pkts_processed = 0; while (total_pkts_processed < budget) { /* update pending count only when current one exhausted */ - if (oq->pkts_pending == 0) + pkts_pending = READ_ONCE(oq->pkts_pending); + if (pkts_pending == 0) octep_oq_check_hw_for_pkts(oct, oq); + pkts_pending = READ_ONCE(oq->pkts_pending); pkts_available = min(budget - total_pkts_processed, - oq->pkts_pending); + pkts_pending); if (!pkts_available) break; pkts_processed = __octep_oq_process_rx(oct, oq, pkts_available); - oq->pkts_pending -= pkts_processed; + pkts_pending = READ_ONCE(oq->pkts_pending); + WRITE_ONCE(oq->pkts_pending, (pkts_pending - pkts_processed)); total_pkts_processed += pkts_processed; } diff --git a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cn9k.c b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cn9k.c index 88937fce75f14..4c769b27c2789 100644 --- a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cn9k.c +++ b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cn9k.c @@ -196,7 +196,7 @@ static void octep_vf_setup_iq_regs_cn93(struct octep_vf_device *oct, int iq_no) } /* Setup registers for a hardware Rx Queue */ -static void octep_vf_setup_oq_regs_cn93(struct octep_vf_device *oct, int oq_no) +static int octep_vf_setup_oq_regs_cn93(struct octep_vf_device *oct, int oq_no) { struct octep_vf_oq *oq = oct->oq[oq_no]; u32 time_threshold = 0; @@ -239,6 +239,7 @@ static void octep_vf_setup_oq_regs_cn93(struct octep_vf_device *oct, int oq_no) time_threshold = CFG_GET_OQ_INTR_TIME(oct->conf); reg_val = ((u64)time_threshold << 32) | CFG_GET_OQ_INTR_PKT(oct->conf); octep_vf_write_csr64(oct, CN93_VF_SDP_R_OUT_INT_LEVELS(oq_no), reg_val); + return 0; } /* Setup registers for a VF mailbox */ diff --git a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cnxk.c b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cnxk.c index 1f79dfad42c62..a968b93a67943 100644 --- a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cnxk.c +++ b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_cnxk.c @@ -199,11 +199,13 @@ static void octep_vf_setup_iq_regs_cnxk(struct octep_vf_device *oct, int iq_no) } /* Setup registers for a hardware Rx Queue */ -static void octep_vf_setup_oq_regs_cnxk(struct octep_vf_device *oct, int oq_no) +static int octep_vf_setup_oq_regs_cnxk(struct octep_vf_device *oct, int oq_no) { struct octep_vf_oq *oq = oct->oq[oq_no]; + unsigned long t_out_jiffies; u32 time_threshold = 0; u64 oq_ctl = ULL(0); + u64 reg_ba_val; u64 reg_val; reg_val = octep_vf_read_csr64(oct, CNXK_VF_SDP_R_OUT_CONTROL(oq_no)); @@ -214,6 +216,38 @@ static void octep_vf_setup_oq_regs_cnxk(struct octep_vf_device *oct, int oq_no) reg_val = octep_vf_read_csr64(oct, CNXK_VF_SDP_R_OUT_CONTROL(oq_no)); } while (!(reg_val & CNXK_VF_R_OUT_CTL_IDLE)); } + octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_WMARK(oq_no), + oq->max_count); + /* Wait for WMARK to get applied */ + usleep_range(10, 15); + + octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_SLIST_BADDR(oq_no), + oq->desc_ring_dma); + octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_SLIST_RSIZE(oq_no), + oq->max_count); + reg_ba_val = octep_vf_read_csr64(oct, + CNXK_VF_SDP_R_OUT_SLIST_BADDR(oq_no)); + if (reg_ba_val != oq->desc_ring_dma) { + t_out_jiffies = jiffies + 10 * HZ; + do { + if (reg_ba_val == ULLONG_MAX) + return -EFAULT; + octep_vf_write_csr64(oct, + CNXK_VF_SDP_R_OUT_SLIST_BADDR + (oq_no), oq->desc_ring_dma); + octep_vf_write_csr64(oct, + CNXK_VF_SDP_R_OUT_SLIST_RSIZE + (oq_no), oq->max_count); + reg_ba_val = + octep_vf_read_csr64(oct, + CNXK_VF_SDP_R_OUT_SLIST_BADDR + (oq_no)); + } while ((reg_ba_val != oq->desc_ring_dma) && + time_before(jiffies, t_out_jiffies)); + + if (reg_ba_val != oq->desc_ring_dma) + return -EAGAIN; + } reg_val &= ~(CNXK_VF_R_OUT_CTL_IMODE); reg_val &= ~(CNXK_VF_R_OUT_CTL_ROR_P); @@ -227,8 +261,6 @@ static void octep_vf_setup_oq_regs_cnxk(struct octep_vf_device *oct, int oq_no) reg_val |= (CNXK_VF_R_OUT_CTL_ES_P); octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_CONTROL(oq_no), reg_val); - octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_SLIST_BADDR(oq_no), oq->desc_ring_dma); - octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_SLIST_RSIZE(oq_no), oq->max_count); oq_ctl = octep_vf_read_csr64(oct, CNXK_VF_SDP_R_OUT_CONTROL(oq_no)); /* Clear the ISIZE and BSIZE (22-0) */ @@ -250,6 +282,7 @@ static void octep_vf_setup_oq_regs_cnxk(struct octep_vf_device *oct, int oq_no) reg_val &= ~GENMASK_ULL(31, 0); reg_val |= CFG_GET_OQ_WMARK(oct->conf); octep_vf_write_csr64(oct, CNXK_VF_SDP_R_OUT_WMARK(oq_no), reg_val); + return 0; } /* Setup registers for a VF mailbox */ diff --git a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.c b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.c index 1d9760b4b8f47..a3c359124887e 100644 --- a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.c +++ b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.c @@ -286,28 +286,45 @@ static void octep_vf_clean_irqs(struct octep_vf_device *oct) } /** - * octep_vf_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue. + * octep_vf_update_pkt() - Update IQ/OQ IN/OUT_CNT registers. * * @iq: Octeon Tx queue data structure. * @oq: Octeon Rx queue data structure. */ -static void octep_vf_enable_ioq_irq(struct octep_vf_iq *iq, struct octep_vf_oq *oq) + +static void octep_vf_update_pkt(struct octep_vf_iq *iq, struct octep_vf_oq *oq) { - u32 pkts_pend = oq->pkts_pending; + u32 pkts_pend = READ_ONCE(oq->pkts_pending); + u32 last_pkt_count = READ_ONCE(oq->last_pkt_count); + u32 pkts_processed = READ_ONCE(iq->pkts_processed); + u32 pkt_in_done = READ_ONCE(iq->pkt_in_done); netdev_dbg(iq->netdev, "enabling intr for Q-%u\n", iq->q_no); - if (iq->pkts_processed) { - writel(iq->pkts_processed, iq->inst_cnt_reg); - iq->pkt_in_done -= iq->pkts_processed; - iq->pkts_processed = 0; + if (pkts_processed) { + writel(pkts_processed, iq->inst_cnt_reg); + readl(iq->inst_cnt_reg); + WRITE_ONCE(iq->pkt_in_done, (pkt_in_done - pkts_processed)); + WRITE_ONCE(iq->pkts_processed, 0); } - if (oq->last_pkt_count - pkts_pend) { - writel(oq->last_pkt_count - pkts_pend, oq->pkts_sent_reg); - oq->last_pkt_count = pkts_pend; + if (last_pkt_count - pkts_pend) { + writel(last_pkt_count - pkts_pend, oq->pkts_sent_reg); + readl(oq->pkts_sent_reg); + WRITE_ONCE(oq->last_pkt_count, pkts_pend); } /* Flush the previous wrties before writing to RESEND bit */ smp_wmb(); +} + +/** + * octep_vf_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue. + * + * @iq: Octeon Tx queue data structure. + * @oq: Octeon Rx queue data structure. + */ +static void octep_vf_enable_ioq_irq(struct octep_vf_iq *iq, + struct octep_vf_oq *oq) +{ writeq(1UL << OCTEP_VF_OQ_INTR_RESEND_BIT, oq->pkts_sent_reg); writeq(1UL << OCTEP_VF_IQ_INTR_RESEND_BIT, iq->inst_cnt_reg); } @@ -333,6 +350,7 @@ static int octep_vf_napi_poll(struct napi_struct *napi, int budget) if (tx_pending || rx_done >= budget) return budget; + octep_vf_update_pkt(ioq_vector->iq, ioq_vector->oq); if (likely(napi_complete_done(napi, rx_done))) octep_vf_enable_ioq_irq(ioq_vector->iq, ioq_vector->oq); diff --git a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.h b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.h index b9f13506f4620..c74cd2369e90d 100644 --- a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.h +++ b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_main.h @@ -55,7 +55,7 @@ struct octep_vf_mmio { struct octep_vf_hw_ops { void (*setup_iq_regs)(struct octep_vf_device *oct, int q); - void (*setup_oq_regs)(struct octep_vf_device *oct, int q); + int (*setup_oq_regs)(struct octep_vf_device *oct, int q); void (*setup_mbox_regs)(struct octep_vf_device *oct, int mbox); irqreturn_t (*non_ioq_intr_handler)(void *ioq_vector); diff --git a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c index d70c8be3cfc40..b579d5b545c46 100644 --- a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c +++ b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c @@ -12,6 +12,8 @@ #include "octep_vf_config.h" #include "octep_vf_main.h" +static void octep_vf_oq_free_ring_buffers(struct octep_vf_oq *oq); + static void octep_vf_oq_reset_indices(struct octep_vf_oq *oq) { oq->host_read_idx = 0; @@ -171,11 +173,15 @@ static int octep_vf_setup_oq(struct octep_vf_device *oct, int q_no) goto oq_fill_buff_err; octep_vf_oq_reset_indices(oq); - oct->hw_ops.setup_oq_regs(oct, q_no); + if (oct->hw_ops.setup_oq_regs(oct, q_no)) + goto oq_setup_err; + oct->num_oqs++; return 0; +oq_setup_err: + octep_vf_oq_free_ring_buffers(oq); oq_fill_buff_err: vfree(oq->buff_info); oq->buff_info = NULL; @@ -319,9 +325,16 @@ static int octep_vf_oq_check_hw_for_pkts(struct octep_vf_device *oct, struct octep_vf_oq *oq) { u32 pkt_count, new_pkts; + u32 last_pkt_count, pkts_pending; pkt_count = readl(oq->pkts_sent_reg); - new_pkts = pkt_count - oq->last_pkt_count; + last_pkt_count = READ_ONCE(oq->last_pkt_count); + new_pkts = pkt_count - last_pkt_count; + + if (pkt_count < last_pkt_count) { + dev_err(oq->dev, "OQ-%u pkt_count(%u) < oq->last_pkt_count(%u)\n", + oq->q_no, pkt_count, last_pkt_count); + } /* Clear the hardware packets counter register if the rx queue is * being processed continuously with-in a single interrupt and @@ -333,8 +346,9 @@ static int octep_vf_oq_check_hw_for_pkts(struct octep_vf_device *oct, pkt_count = readl(oq->pkts_sent_reg); new_pkts += pkt_count; } - oq->last_pkt_count = pkt_count; - oq->pkts_pending += new_pkts; + WRITE_ONCE(oq->last_pkt_count, pkt_count); + pkts_pending = READ_ONCE(oq->pkts_pending); + WRITE_ONCE(oq->pkts_pending, (pkts_pending + new_pkts)); return new_pkts; } @@ -363,7 +377,7 @@ static int __octep_vf_oq_process_rx(struct octep_vf_device *oct, struct sk_buff *skb; u32 read_idx; - read_idx = oq->host_read_idx; + read_idx = READ_ONCE(oq->host_read_idx); rx_bytes = 0; desc_used = 0; for (pkt = 0; pkt < pkts_to_process; pkt++) { @@ -457,7 +471,7 @@ static int __octep_vf_oq_process_rx(struct octep_vf_device *oct, napi_gro_receive(oq->napi, skb); } - oq->host_read_idx = read_idx; + WRITE_ONCE(oq->host_read_idx, read_idx); oq->refill_count += desc_used; oq->stats->packets += pkt; oq->stats->bytes += rx_bytes; @@ -480,22 +494,26 @@ int octep_vf_oq_process_rx(struct octep_vf_oq *oq, int budget) { u32 pkts_available, pkts_processed, total_pkts_processed; struct octep_vf_device *oct = oq->octep_vf_dev; + u32 pkts_pending; pkts_available = 0; pkts_processed = 0; total_pkts_processed = 0; while (total_pkts_processed < budget) { /* update pending count only when current one exhausted */ - if (oq->pkts_pending == 0) + pkts_pending = READ_ONCE(oq->pkts_pending); + if (pkts_pending == 0) octep_vf_oq_check_hw_for_pkts(oct, oq); + pkts_pending = READ_ONCE(oq->pkts_pending); pkts_available = min(budget - total_pkts_processed, - oq->pkts_pending); + pkts_pending); if (!pkts_available) break; pkts_processed = __octep_vf_oq_process_rx(oct, oq, pkts_available); - oq->pkts_pending -= pkts_processed; + pkts_pending = READ_ONCE(oq->pkts_pending); + WRITE_ONCE(oq->pkts_pending, (pkts_pending - pkts_processed)); total_pkts_processed += pkts_processed; } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c index ec0e11c77cbf2..81b55f1416e0d 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c @@ -1823,6 +1823,8 @@ static int cgx_lmac_exit(struct cgx *cgx) cgx->mac_ops->mac_pause_frm_config(cgx, lmac->lmac_id, false); cgx_configure_interrupt(cgx, lmac, lmac->lmac_id, true); kfree(lmac->mac_to_index_bmap.bmap); + rvu_free_bitmap(&lmac->rx_fc_pfvf_bmap); + rvu_free_bitmap(&lmac->tx_fc_pfvf_bmap); kfree(lmac->name); kfree(lmac); } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c index 747fbdf2a908f..8530df8b3fdaf 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c @@ -3632,11 +3632,22 @@ static void rvu_remove(struct pci_dev *pdev) devm_kfree(&pdev->dev, rvu); } +static void rvu_shutdown(struct pci_dev *pdev) +{ + struct rvu *rvu = pci_get_drvdata(pdev); + + if (!rvu) + return; + + rvu_clear_rvum_blk_revid(rvu); +} + static struct pci_driver rvu_driver = { .name = DRV_NAME, .id_table = rvu_id_table, .probe = rvu_probe, .remove = rvu_remove, + .shutdown = rvu_shutdown, }; static int __init rvu_init_module(void) diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c index 828316211b245..c9c65ac69ead8 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c @@ -4932,12 +4932,18 @@ static int rvu_nix_block_init(struct rvu *rvu, struct nix_hw *nix_hw) /* Set chan/link to backpressure TL3 instead of TL2 */ rvu_write64(rvu, blkaddr, NIX_AF_PSE_CHANNEL_LEVEL, 0x01); - /* Disable SQ manager's sticky mode operation (set TM6 = 0) + /* Disable SQ manager's sticky mode operation (set TM6 = 0, TM11 = 0) * This sticky mode is known to cause SQ stalls when multiple - * SQs are mapped to same SMQ and transmitting pkts at a time. + * SQs are mapped to same SMQ and transmitting pkts simultaneously. + * NIX PSE may deadlock when there are any sticky to non-sticky + * transmission. Hence disable it (TM5 = 0). */ cfg = rvu_read64(rvu, blkaddr, NIX_AF_SQM_DBG_CTL_STATUS); - cfg &= ~BIT_ULL(15); + cfg &= ~(BIT_ULL(15) | BIT_ULL(14) | BIT_ULL(23)); + /* NIX may drop credits when condition clocks are turned off. + * Hence enable control flow clk (set TM9 = 1). + */ + cfg |= BIT_ULL(21); rvu_write64(rvu, blkaddr, NIX_AF_SQM_DBG_CTL_STATUS, cfg); ltdefs = rvu->kpu.lt_def; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c index c7c70429eb6c1..8658cb2143dfc 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c @@ -1042,32 +1042,35 @@ void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_ACTION(index, bank), *(u64 *)&action); - /* update the VF flow rule action with the VF default entry action */ - if (mcam_index < 0) - npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc, - *(u64 *)&action); - /* update the action change in default rule */ pfvf = rvu_get_pfvf(rvu, pcifunc); if (pfvf->def_ucast_rule) pfvf->def_ucast_rule->rx_action = action; - index = npc_get_nixlf_mcam_index(mcam, pcifunc, - nixlf, NIXLF_PROMISC_ENTRY); + if (mcam_index < 0) { + /* update the VF flow rule action with the VF default + * entry action + */ + npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc, + *(u64 *)&action); - /* If PF's promiscuous entry is enabled, - * Set RSS action for that entry as well - */ - npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, blkaddr, - alg_idx); + index = npc_get_nixlf_mcam_index(mcam, pcifunc, + nixlf, NIXLF_PROMISC_ENTRY); - index = npc_get_nixlf_mcam_index(mcam, pcifunc, - nixlf, NIXLF_ALLMULTI_ENTRY); - /* If PF's allmulti entry is enabled, - * Set RSS action for that entry as well - */ - npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, blkaddr, - alg_idx); + /* If PF's promiscuous entry is enabled, + * Set RSS action for that entry as well + */ + npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, + blkaddr, alg_idx); + + index = npc_get_nixlf_mcam_index(mcam, pcifunc, + nixlf, NIXLF_ALLMULTI_ENTRY); + /* If PF's allmulti entry is enabled, + * Set RSS action for that entry as well + */ + npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, + blkaddr, alg_idx); + } } void npc_enadis_default_mce_entry(struct rvu *rvu, u16 pcifunc, diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c index a7a7bc0e1b675..bbf25769f4994 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c @@ -3321,6 +3321,7 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id) err_sriov_cleannup: otx2_sriov_vfcfg_cleanup(pf); err_pf_sriov_init: + otx2_unregister_dl(pf); otx2_shutdown_tc(pf); err_mcam_flow_del: otx2_mcam_flow_del(pf); diff --git a/drivers/net/ethernet/marvell/skge.c b/drivers/net/ethernet/marvell/skge.c index 05349a0b2db1c..cf4e26d337bb5 100644 --- a/drivers/net/ethernet/marvell/skge.c +++ b/drivers/net/ethernet/marvell/skge.c @@ -78,7 +78,6 @@ static const struct pci_device_id skge_id_table[] = { { PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, 0x4320) }, /* SK-98xx V2.0 */ { PCI_DEVICE(PCI_VENDOR_ID_DLINK, 0x4b01) }, /* D-Link DGE-530T (rev.B) */ { PCI_DEVICE(PCI_VENDOR_ID_DLINK, 0x4c00) }, /* D-Link DGE-530T */ - { PCI_DEVICE(PCI_VENDOR_ID_DLINK, 0x4302) }, /* D-Link DGE-530T Rev C1 */ { PCI_DEVICE(PCI_VENDOR_ID_MARVELL, 0x4320) }, /* Marvell Yukon 88E8001/8003/8010 */ { PCI_DEVICE(PCI_VENDOR_ID_MARVELL, 0x5005) }, /* Belkin */ { PCI_DEVICE(PCI_VENDOR_ID_CNET, 0x434E) }, /* CNet PowerG-2000 */ diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c index e68997a29191b..8d3e15bc867d2 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -3749,12 +3749,21 @@ static int mtk_xdp_setup(struct net_device *dev, struct bpf_prog *prog, mtk_stop(dev); old_prog = rcu_replace_pointer(eth->prog, prog, lockdep_rtnl_is_held()); + + if (netif_running(dev) && need_update) { + int err; + + err = mtk_open(dev); + if (err) { + rcu_assign_pointer(eth->prog, old_prog); + + return err; + } + } + if (old_prog) bpf_prog_put(old_prog); - if (netif_running(dev) && need_update) - return mtk_open(dev); - return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index b34b85539f3b1..5bced924a24fb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -179,7 +179,8 @@ static inline u16 mlx5_min_rx_wqes(int wq_type, u32 wq_size) } /* Use this function to get max num channels (rxqs/txqs) only to create netdev */ -static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev) +static inline unsigned int +mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev) { return is_kdump_kernel() ? MLX5E_MIN_NUM_CHANNELS : diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c index c93ee969ea647..ec715b158a342 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c @@ -448,22 +448,8 @@ static void mlx5e_ptpsq_unhealthy_work(struct work_struct *work) { struct mlx5e_ptpsq *ptpsq = container_of(work, struct mlx5e_ptpsq, report_unhealthy_work); - struct mlx5e_txqsq *sq = &ptpsq->txqsq; - - /* Recovering the PTP SQ means re-enabling NAPI, which requires the - * netdev instance lock. However, SQ closing has to wait for this work - * task to finish while also holding the same lock. So either get the - * lock or find that the SQ is no longer enabled and thus this work is - * not relevant anymore. - */ - while (!netdev_trylock(sq->netdev)) { - if (!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)) - return; - msleep(20); - } mlx5e_reporter_tx_ptpsq_unhealthy(ptpsq); - netdev_unlock(sq->netdev); } static int mlx5e_ptp_open_txqsq(struct mlx5e_ptp *c, u32 tisn, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c index b1415992ffa24..a09a7c05820d6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright (c) 2019 Mellanox Technologies. +#include + #include "health.h" #include "params.h" #include "txrx.h" @@ -177,6 +179,16 @@ static int mlx5e_rx_reporter_timeout_recover(void *ctx) rq = ctx; priv = rq->priv; + /* Acquire netdev instance lock to synchronize with channel close and + * reopen flows. Either successfully obtain the lock, or detect that + * channels are closing for another reason, making this work no longer + * necessary. + */ + while (!netdev_trylock(rq->netdev)) { + if (!test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &rq->priv->state)) + return 0; + msleep(20); + } mutex_lock(&priv->state_lock); eq = rq->cq.mcq.eq; @@ -186,6 +198,7 @@ static int mlx5e_rx_reporter_timeout_recover(void *ctx) clear_bit(MLX5E_SQ_STATE_ENABLED, &rq->icosq->state); mutex_unlock(&priv->state_lock); + netdev_unlock(rq->netdev); return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c index 9e2cf191ed308..9f6454102cf79 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c @@ -1,6 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2019 Mellanox Technologies. */ +#include + #include "health.h" #include "en/ptp.h" #include "en/devlink.h" @@ -78,6 +80,18 @@ static int mlx5e_tx_reporter_err_cqe_recover(void *ctx) if (!test_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state)) return 0; + /* Recovering queues means re-enabling NAPI, which requires the netdev + * instance lock. However, SQ closing flows have to wait for work tasks + * to finish while also holding the netdev instance lock. So either get + * the lock or find that the SQ is no longer enabled and thus this work + * is not relevant anymore. + */ + while (!netdev_trylock(dev)) { + if (!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)) + return 0; + msleep(20); + } + err = mlx5_core_query_sq_state(mdev, sq->sqn, &state); if (err) { netdev_err(dev, "Failed to query SQ 0x%x state. err = %d\n", @@ -113,9 +127,11 @@ static int mlx5e_tx_reporter_err_cqe_recover(void *ctx) else mlx5e_trigger_napi_sched(sq->cq.napi); + netdev_unlock(dev); return 0; out: clear_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state); + netdev_unlock(dev); return err; } @@ -136,10 +152,24 @@ static int mlx5e_tx_reporter_timeout_recover(void *ctx) sq = to_ctx->sq; eq = sq->cq.mcq.eq; priv = sq->priv; + + /* Recovering the TX queues implies re-enabling NAPI, which requires + * the netdev instance lock. + * However, channel closing flows have to wait for this work to finish + * while holding the same lock. So either get the lock or find that + * channels are being closed for other reason and this work is not + * relevant anymore. + */ + while (!netdev_trylock(sq->netdev)) { + if (!test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state)) + return 0; + msleep(20); + } + err = mlx5e_health_channel_eq_recover(sq->netdev, eq, sq->cq.ch_stats); if (!err) { to_ctx->status = 0; /* this sq recovered */ - return err; + goto out; } mutex_lock(&priv->state_lock); @@ -147,7 +177,7 @@ static int mlx5e_tx_reporter_timeout_recover(void *ctx) mutex_unlock(&priv->state_lock); if (!err) { to_ctx->status = 1; /* all channels recovered */ - return err; + goto out; } to_ctx->status = err; @@ -155,7 +185,8 @@ static int mlx5e_tx_reporter_timeout_recover(void *ctx) netdev_err(priv->netdev, "mlx5e_safe_reopen_channels failed recovering from a tx_timeout, err(%d).\n", err); - +out: + netdev_unlock(sq->netdev); return err; } @@ -172,10 +203,22 @@ static int mlx5e_tx_reporter_ptpsq_unhealthy_recover(void *ctx) return 0; priv = ptpsq->txqsq.priv; + netdev = priv->netdev; + + /* Recovering the PTP SQ means re-enabling NAPI, which requires the + * netdev instance lock. However, SQ closing has to wait for this work + * task to finish while also holding the same lock. So either get the + * lock or find that the SQ is no longer enabled and thus this work is + * not relevant anymore. + */ + while (!netdev_trylock(netdev)) { + if (!test_bit(MLX5E_SQ_STATE_ENABLED, &ptpsq->txqsq.state)) + return 0; + msleep(20); + } mutex_lock(&priv->state_lock); chs = &priv->channels; - netdev = priv->netdev; carrier_ok = netif_carrier_ok(netdev); netif_carrier_off(netdev); @@ -192,6 +235,7 @@ static int mlx5e_tx_reporter_ptpsq_unhealthy_recover(void *ctx) netif_carrier_on(netdev); mutex_unlock(&priv->state_lock); + netdev_unlock(netdev); return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 9c7064187ed0f..f03507a522b4f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -259,7 +259,6 @@ static void mlx5e_ipsec_init_limits(struct mlx5e_ipsec_sa_entry *sa_entry, static void mlx5e_ipsec_init_macs(struct mlx5e_ipsec_sa_entry *sa_entry, struct mlx5_accel_esp_xfrm_attrs *attrs) { - struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); struct mlx5e_ipsec_addr *addrs = &attrs->addrs; struct net_device *netdev = sa_entry->dev; struct xfrm_state *x = sa_entry->x; @@ -276,7 +275,7 @@ static void mlx5e_ipsec_init_macs(struct mlx5e_ipsec_sa_entry *sa_entry, attrs->type != XFRM_DEV_OFFLOAD_PACKET) return; - mlx5_query_mac_address(mdev, addr); + ether_addr_copy(addr, netdev->dev_addr); switch (attrs->dir) { case XFRM_DEV_OFFLOAD_IN: src = attrs->dmac; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 59e17b41c3a67..cb993ad2d9ad9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -688,19 +688,7 @@ static void mlx5e_rq_timeout_work(struct work_struct *timeout_work) struct mlx5e_rq, rx_timeout_work); - /* Acquire netdev instance lock to synchronize with channel close and - * reopen flows. Either successfully obtain the lock, or detect that - * channels are closing for another reason, making this work no longer - * necessary. - */ - while (!netdev_trylock(rq->netdev)) { - if (!test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &rq->priv->state)) - return; - msleep(20); - } - mlx5e_reporter_rx_timeout(rq); - netdev_unlock(rq->netdev); } static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq) @@ -1997,20 +1985,7 @@ void mlx5e_tx_err_cqe_work(struct work_struct *recover_work) struct mlx5e_txqsq *sq = container_of(recover_work, struct mlx5e_txqsq, recover_work); - /* Recovering queues means re-enabling NAPI, which requires the netdev - * instance lock. However, SQ closing flows have to wait for work tasks - * to finish while also holding the netdev instance lock. So either get - * the lock or find that the SQ is no longer enabled and thus this work - * is not relevant anymore. - */ - while (!netdev_trylock(sq->netdev)) { - if (!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)) - return; - msleep(20); - } - mlx5e_reporter_tx_err_cqe(sq); - netdev_unlock(sq->netdev); } static struct dim_cq_moder mlx5e_get_def_tx_moderation(u8 cq_period_mode) @@ -5102,19 +5077,6 @@ static void mlx5e_tx_timeout_work(struct work_struct *work) struct net_device *netdev = priv->netdev; int i; - /* Recovering the TX queues implies re-enabling NAPI, which requires - * the netdev instance lock. - * However, channel closing flows have to wait for this work to finish - * while holding the same lock. So either get the lock or find that - * channels are being closed for other reason and this work is not - * relevant anymore. - */ - while (!netdev_trylock(netdev)) { - if (!test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state)) - return; - msleep(20); - } - for (i = 0; i < netdev->real_num_tx_queues; i++) { struct netdev_queue *dev_queue = netdev_get_tx_queue(netdev, i); @@ -5127,8 +5089,6 @@ static void mlx5e_tx_timeout_work(struct work_struct *work) /* break if tried to reopened channels */ break; } - - netdev_unlock(netdev); } static void mlx5e_tx_timeout(struct net_device *dev, unsigned int txqueue) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index 784130cdf6c07..8c0e812f13c3f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -3854,6 +3854,8 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, if (mode == DEVLINK_ESWITCH_MODE_LEGACY) esw->dev->priv.flags |= MLX5_PRIV_FLAGS_SWITCH_LEGACY; + if (mlx5_mode == MLX5_ESWITCH_OFFLOADS) + esw->dev->priv.flags &= ~MLX5_PRIV_FLAGS_SWITCH_LEGACY; mlx5_eswitch_disable_locked(esw); if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) { if (mlx5_devlink_trap_get_num_active(esw->dev)) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c index 8ec04a5f434dd..2d6019a32669c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c @@ -1652,8 +1652,12 @@ void mlx5_lag_disable_change(struct mlx5_core_dev *dev) mutex_lock(&ldev->lock); ldev->mode_changes_in_progress++; - if (__mlx5_lag_is_active(ldev)) - mlx5_disable_lag(ldev); + if (__mlx5_lag_is_active(ldev)) { + if (ldev->mode == MLX5_LAG_MODE_MPESW) + mlx5_lag_disable_mpesw(ldev); + else + mlx5_disable_lag(ldev); + } mutex_unlock(&ldev->lock); mlx5_devcom_comp_unlock(dev->priv.hca_devcom_comp); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c index 2d86af8f0d9b8..c217998604fdb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c @@ -65,7 +65,7 @@ static int mlx5_mpesw_metadata_set(struct mlx5_lag *ldev) return err; } -static int enable_mpesw(struct mlx5_lag *ldev) +static int mlx5_lag_enable_mpesw(struct mlx5_lag *ldev) { struct mlx5_core_dev *dev0; int err; @@ -124,7 +124,7 @@ static int enable_mpesw(struct mlx5_lag *ldev) return err; } -static void disable_mpesw(struct mlx5_lag *ldev) +void mlx5_lag_disable_mpesw(struct mlx5_lag *ldev) { if (ldev->mode == MLX5_LAG_MODE_MPESW) { mlx5_mpesw_metadata_cleanup(ldev); @@ -150,9 +150,9 @@ static void mlx5_mpesw_work(struct work_struct *work) } if (mpesww->op == MLX5_MPESW_OP_ENABLE) - mpesww->result = enable_mpesw(ldev); + mpesww->result = mlx5_lag_enable_mpesw(ldev); else if (mpesww->op == MLX5_MPESW_OP_DISABLE) - disable_mpesw(ldev); + mlx5_lag_disable_mpesw(ldev); unlock: mutex_unlock(&ldev->lock); mlx5_devcom_comp_unlock(devcom); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.h b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.h index 02520f27a033c..46de93ed790de 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.h @@ -31,5 +31,10 @@ int mlx5_lag_mpesw_do_mirred(struct mlx5_core_dev *mdev, bool mlx5_lag_is_mpesw(struct mlx5_core_dev *dev); void mlx5_lag_mpesw_disable(struct mlx5_core_dev *dev); int mlx5_lag_mpesw_enable(struct mlx5_core_dev *dev); +#ifdef CONFIG_MLX5_ESWITCH +void mlx5_lag_disable_mpesw(struct mlx5_lag *ldev); +#else +static inline void mlx5_lag_disable_mpesw(struct mlx5_lag *ldev) {} +#endif /* CONFIG_MLX5_ESWITCH */ #endif /* __MLX5_LAG_MPESW_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c index a2fc937d54617..172862a70c70d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c @@ -193,7 +193,9 @@ static int mlx5_sriov_enable(struct pci_dev *pdev, int num_vfs) err = pci_enable_sriov(pdev, num_vfs); if (err) { mlx5_core_warn(dev, "pci_enable_sriov failed : %d\n", err); + devl_lock(devlink); mlx5_device_disable_sriov(dev, num_vfs, true, true); + devl_unlock(devlink); } return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_dbg.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_dbg.c index 030a5776c9374..a4c19af1775f1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_dbg.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_dbg.c @@ -1050,8 +1050,8 @@ static int dr_dump_domain_all(struct seq_file *file, struct mlx5dr_domain *dmn) struct mlx5dr_table *tbl; int ret; - mutex_lock(&dmn->dump_info.dbg_mutex); mlx5dr_domain_lock(dmn); + mutex_lock(&dmn->dump_info.dbg_mutex); ret = dr_dump_domain(file, dmn); if (ret < 0) @@ -1064,8 +1064,8 @@ static int dr_dump_domain_all(struct seq_file *file, struct mlx5dr_domain *dmn) } unlock_mutex: - mlx5dr_domain_unlock(dmn); mutex_unlock(&dmn->dump_info.dbg_mutex); + mlx5dr_domain_unlock(dmn); return ret; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/wc.c b/drivers/net/ethernet/mellanox/mlx5/core/wc.c index 05e5fd777d4f4..8701b7b6a2d51 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/wc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/wc.c @@ -2,6 +2,7 @@ // Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. #include +#include #include #include "lib/clock.h" #include "mlx5_core.h" @@ -14,7 +15,7 @@ #define TEST_WC_NUM_WQES 255 #define TEST_WC_LOG_CQ_SZ (order_base_2(TEST_WC_NUM_WQES)) #define TEST_WC_SQ_LOG_WQ_SZ TEST_WC_LOG_CQ_SZ -#define TEST_WC_POLLING_MAX_TIME_JIFFIES msecs_to_jiffies(100) +#define TEST_WC_POLLING_MAX_TIME_USEC (100 * USEC_PER_MSEC) struct mlx5_wc_cq { /* data path - accessed per cqe */ @@ -358,7 +359,6 @@ static int mlx5_wc_poll_cq(struct mlx5_wc_sq *sq) static void mlx5_core_test_wc(struct mlx5_core_dev *mdev) { unsigned int offset = 0; - unsigned long expires; struct mlx5_wc_sq *sq; int i, err; @@ -388,13 +388,9 @@ static void mlx5_core_test_wc(struct mlx5_core_dev *mdev) mlx5_wc_post_nop(sq, &offset, true); - expires = jiffies + TEST_WC_POLLING_MAX_TIME_JIFFIES; - do { - err = mlx5_wc_poll_cq(sq); - if (err) - usleep_range(2, 10); - } while (mdev->wc_state == MLX5_WC_STATE_UNINITIALIZED && - time_is_after_jiffies(expires)); + poll_timeout_us(mlx5_wc_poll_cq(sq), + mdev->wc_state != MLX5_WC_STATE_UNINITIALIZED, 10, + TEST_WC_POLLING_MAX_TIME_USEC, false); mlx5_wc_destroy_sq(sq); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c index 95fac020eb93c..08aed4103323e 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c @@ -1142,6 +1142,9 @@ static int fbnic_set_cls_rule_ins(struct fbnic_net *fbn, return -EINVAL; } + dest |= FIELD_PREP(FBNIC_RPC_ACT_TBL0_DMA_HINT, + FBNIC_RCD_HDR_AL_DMA_HINT_L4); + /* Write action table values */ act_tcam->dest = dest; act_tcam->rss_en_mask = fbnic_flow_hash_2_rss_en_mask(fbn, hash_idx); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.c b/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.c index 85a883dba385f..d8a9a7d7c2375 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.c @@ -51,8 +51,6 @@ int fbnic_fw_log_init(struct fbnic_dev *fbd) log->data_start = data; log->data_end = data + FBNIC_FW_LOG_SIZE; - fbnic_fw_log_enable(fbd, true); - return 0; } @@ -63,7 +61,6 @@ void fbnic_fw_log_free(struct fbnic_dev *fbd) if (!fbnic_fw_log_ready(fbd)) return; - fbnic_fw_log_disable(fbd); INIT_LIST_HEAD(&log->entries); log->size = 0; vfree(log->data_start); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c index e95be0e7bd9e0..cbedaa037cfa8 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c @@ -262,6 +262,23 @@ static int fbnic_set_mac(struct net_device *netdev, void *p) return 0; } +static int fbnic_change_mtu(struct net_device *dev, int new_mtu) +{ + struct fbnic_net *fbn = netdev_priv(dev); + + if (fbnic_check_split_frames(fbn->xdp_prog, new_mtu, fbn->hds_thresh)) { + dev_err(&dev->dev, + "MTU %d is larger than HDS threshold %d in XDP mode\n", + new_mtu, fbn->hds_thresh); + + return -EINVAL; + } + + WRITE_ONCE(dev->mtu, new_mtu); + + return 0; +} + void fbnic_clear_rx_mode(struct fbnic_dev *fbd) { struct net_device *netdev = fbd->netdev; @@ -533,6 +550,7 @@ static const struct net_device_ops fbnic_netdev_ops = { .ndo_start_xmit = fbnic_xmit_frame, .ndo_features_check = fbnic_features_check, .ndo_set_mac_address = fbnic_set_mac, + .ndo_change_mtu = fbnic_change_mtu, .ndo_set_rx_mode = fbnic_set_rx_mode, .ndo_get_stats64 = fbnic_get_stats64, .ndo_bpf = fbnic_bpf, @@ -790,6 +808,8 @@ struct net_device *fbnic_netdev_alloc(struct fbnic_dev *fbd) netdev->hw_enc_features |= netdev->features; netdev->features |= NETIF_F_NTUPLE; + netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_RX_SG; + netdev->min_mtu = IPV6_MIN_MTU; netdev->max_mtu = FBNIC_MAX_JUMBO_FRAME_SIZE - ETH_HLEN; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.h b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.h index b0a87c57910f2..e6ca23a9957d9 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.h @@ -73,6 +73,8 @@ struct fbnic_net { /* Time stamping filter config */ struct kernel_hwtstamp_config hwtstamp_config; + + bool tx_pause; }; int __fbnic_open(struct fbnic_net *fbn); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c index 0fa90baad5f88..698b8a85afb31 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c @@ -303,11 +303,17 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto free_irqs; } + err = fbnic_fw_log_init(fbd); + if (err) + dev_warn(fbd->dev, + "Unable to initialize firmware log buffer: %d\n", + err); + err = fbnic_fw_request_mbx(fbd); if (err) { dev_err(&pdev->dev, "Firmware mailbox initialization failure\n"); - goto free_irqs; + goto free_fw_log; } /* Send the request to enable the FW logging to host. Note if this @@ -315,11 +321,7 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) * possible the FW is just too old to support the logging and needs * to be updated. */ - err = fbnic_fw_log_init(fbd); - if (err) - dev_warn(fbd->dev, - "Unable to initialize firmware log buffer: %d\n", - err); + fbnic_fw_log_enable(fbd, true); fbnic_devlink_register(fbd); fbnic_devlink_otp_check(fbd, "error detected during probe"); @@ -363,6 +365,8 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) * firmware updates for fixes. */ return 0; +free_fw_log: + fbnic_fw_log_free(fbd); free_irqs: fbnic_free_irqs(fbd); err_destroy_health: @@ -397,8 +401,9 @@ static void fbnic_remove(struct pci_dev *pdev) fbnic_hwmon_unregister(fbd); fbnic_dbg_fbd_exit(fbd); fbnic_devlink_unregister(fbd); - fbnic_fw_log_free(fbd); + fbnic_fw_log_disable(fbd); fbnic_fw_free_mbx(fbd); + fbnic_fw_log_free(fbd); fbnic_free_irqs(fbd); fbnic_devlink_health_destroy(fbd); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_phylink.c b/drivers/net/ethernet/meta/fbnic/fbnic_phylink.c index 7ce3fdd252828..62701923cfe96 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_phylink.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_phylink.c @@ -208,6 +208,9 @@ fbnic_phylink_mac_link_up(struct phylink_config *config, struct fbnic_net *fbn = netdev_priv(netdev); struct fbnic_dev *fbd = fbn->fbd; + fbn->tx_pause = tx_pause; + fbnic_config_drop_mode(fbn, tx_pause); + fbd->mac->link_up(fbd, tx_pause, rx_pause); } diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c index 7f31e890031c0..42a186db43ea9 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c @@ -338,9 +338,8 @@ void fbnic_rss_reinit(struct fbnic_dev *fbd, struct fbnic_net *fbn) else if (tstamp_mask & (1u << flow_type)) dest |= FBNIC_RPC_ACT_TBL0_TS_ENA; - if (act1_value[flow_type] & FBNIC_RPC_TCAM_ACT1_L4_VALID) - dest |= FIELD_PREP(FBNIC_RPC_ACT_TBL0_DMA_HINT, - FBNIC_RCD_HDR_AL_DMA_HINT_L4); + dest |= FIELD_PREP(FBNIC_RPC_ACT_TBL0_DMA_HINT, + FBNIC_RCD_HDR_AL_DMA_HINT_L4); rss_en_mask = fbnic_flow_hash_2_rss_en_mask(fbn, flow_type); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c index b1e8ce89870f7..fbdf79b6ad2de 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c @@ -2573,20 +2573,44 @@ static void fbnic_enable_bdq(struct fbnic_ring *hpq, struct fbnic_ring *ppq) } static void fbnic_config_drop_mode_rcq(struct fbnic_napi_vector *nv, - struct fbnic_ring *rcq) + struct fbnic_ring *rcq, bool tx_pause, + bool hdr_split) { + struct fbnic_net *fbn = netdev_priv(nv->napi.dev); u32 drop_mode, rcq_ctl; - drop_mode = FBNIC_QUEUE_RDE_CTL0_DROP_IMMEDIATE; + if (!tx_pause && fbn->num_rx_queues > 1) + drop_mode = FBNIC_QUEUE_RDE_CTL0_DROP_IMMEDIATE; + else + drop_mode = FBNIC_QUEUE_RDE_CTL0_DROP_NEVER; /* Specify packet layout */ rcq_ctl = FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_DROP_MODE_MASK, drop_mode) | FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_MIN_HROOM_MASK, FBNIC_RX_HROOM) | - FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_MIN_TROOM_MASK, FBNIC_RX_TROOM); + FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_MIN_TROOM_MASK, FBNIC_RX_TROOM) | + FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_EN_HDR_SPLIT, hdr_split); fbnic_ring_wr32(rcq, FBNIC_QUEUE_RDE_CTL0, rcq_ctl); } +void fbnic_config_drop_mode(struct fbnic_net *fbn, bool txp) +{ + bool hds; + int i, t; + + hds = fbn->hds_thresh < FBNIC_HDR_BYTES_MIN; + + for (i = 0; i < fbn->num_napi; i++) { + struct fbnic_napi_vector *nv = fbn->napi[i]; + + for (t = 0; t < nv->rxt_count; t++) { + struct fbnic_q_triad *qt = &nv->qt[nv->txt_count + t]; + + fbnic_config_drop_mode_rcq(nv, &qt->cmpl, txp, hds); + } + } +} + static void fbnic_config_rim_threshold(struct fbnic_ring *rcq, u16 nv_idx, u32 rx_desc) { u32 threshold; @@ -2633,20 +2657,18 @@ static void fbnic_enable_rcq(struct fbnic_napi_vector *nv, { struct fbnic_net *fbn = netdev_priv(nv->napi.dev); u32 log_size = fls(rcq->size_mask); - u32 hds_thresh = fbn->hds_thresh; u32 rcq_ctl = 0; - - fbnic_config_drop_mode_rcq(nv, rcq); + bool hdr_split; + u32 hds_thresh; /* Force lower bound on MAX_HEADER_BYTES. Below this, all frames should * be split at L4. It would also result in the frames being split at * L2/L3 depending on the frame size. */ - if (fbn->hds_thresh < FBNIC_HDR_BYTES_MIN) { - rcq_ctl = FBNIC_QUEUE_RDE_CTL0_EN_HDR_SPLIT; - hds_thresh = FBNIC_HDR_BYTES_MIN; - } + hdr_split = fbn->hds_thresh < FBNIC_HDR_BYTES_MIN; + fbnic_config_drop_mode_rcq(nv, rcq, fbn->tx_pause, hdr_split); + hds_thresh = max(fbn->hds_thresh, FBNIC_HDR_BYTES_MIN); rcq_ctl |= FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PADLEN_MASK, FBNIC_RX_PAD) | FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_MAX_HDR_MASK, hds_thresh) | FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PAYLD_OFF_MASK, @@ -2699,7 +2721,6 @@ static void __fbnic_nv_enable(struct fbnic_napi_vector *nv) &nv->napi); fbnic_enable_bdq(&qt->sub0, &qt->sub1); - fbnic_config_drop_mode_rcq(nv, &qt->cmpl); fbnic_enable_rcq(nv, &qt->cmpl); } } diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h index ca37da5a0b179..51a98f27d5d91 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h @@ -66,7 +66,7 @@ struct fbnic_net; (4096 - FBNIC_RX_HROOM - FBNIC_RX_TROOM - FBNIC_RX_PAD) #define FBNIC_HDS_THRESH_DEFAULT \ (1536 - FBNIC_RX_PAD) -#define FBNIC_HDR_BYTES_MIN 128 +#define FBNIC_HDR_BYTES_MIN 256 struct fbnic_pkt_buff { struct xdp_buff buff; @@ -184,6 +184,7 @@ void fbnic_reset_netif_queues(struct fbnic_net *fbn); irqreturn_t fbnic_msix_clean_rings(int irq, void *data); void fbnic_napi_enable(struct fbnic_net *fbn); void fbnic_napi_disable(struct fbnic_net *fbn); +void fbnic_config_drop_mode(struct fbnic_net *fbn, bool tx_pause); void fbnic_enable(struct fbnic_net *fbn); void fbnic_disable(struct fbnic_net *fbn); void fbnic_flush(struct fbnic_net *fbn); diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c b/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c index 2f168700f63c1..8b2e07821a950 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c @@ -576,7 +576,7 @@ static int sparx5_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) static struct ptp_clock_info sparx5_ptp_clock_info = { .owner = THIS_MODULE, .name = "sparx5 ptp", - .max_adj = 200000, + .max_adj = 10000000, .gettime64 = sparx5_ptp_gettime64, .settime64 = sparx5_ptp_settime64, .adjtime = sparx5_ptp_adjtime, diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_qos.h b/drivers/net/ethernet/microchip/sparx5/sparx5_qos.h index 1231a80335d7b..04f76f1e23f60 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_qos.h +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_qos.h @@ -35,7 +35,7 @@ #define SPX5_SE_BURST_UNIT 4096 /* Dwrr */ -#define SPX5_DWRR_COST_MAX 63 +#define SPX5_DWRR_COST_MAX 31 struct sparx5_shaper { u32 mode; diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c index 43f034e180c41..e22a98a9c9856 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -1866,7 +1866,10 @@ static void mana_gd_cleanup(struct pci_dev *pdev) mana_gd_remove_irqs(pdev); - destroy_workqueue(gc->service_wq); + if (gc->service_wq) { + destroy_workqueue(gc->service_wq); + gc->service_wq = NULL; + } dev_dbg(&pdev->dev, "mana gdma cleanup successful\n"); } diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c index 0142fd98392c2..5712ec4f644a4 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_en.c +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c @@ -3580,7 +3580,9 @@ void mana_rdma_remove(struct gdma_dev *gd) } WRITE_ONCE(gd->rdma_teardown, true); - flush_workqueue(gc->service_wq); + + if (gc->service_wq) + flush_workqueue(gc->service_wq); if (gd->adev) remove_adev(gd); diff --git a/drivers/net/ethernet/mscc/ocelot_net.c b/drivers/net/ethernet/mscc/ocelot_net.c index 469784d3a1a67..1b82693204640 100644 --- a/drivers/net/ethernet/mscc/ocelot_net.c +++ b/drivers/net/ethernet/mscc/ocelot_net.c @@ -551,44 +551,81 @@ static int ocelot_port_stop(struct net_device *dev) return 0; } -static netdev_tx_t ocelot_port_xmit(struct sk_buff *skb, struct net_device *dev) +static bool ocelot_xmit_timestamp(struct ocelot *ocelot, int port, + struct sk_buff *skb, u32 *rew_op) { - struct ocelot_port_private *priv = netdev_priv(dev); - struct ocelot_port *ocelot_port = &priv->port; - struct ocelot *ocelot = ocelot_port->ocelot; - int port = priv->port.index; - u32 rew_op = 0; - - if (!static_branch_unlikely(&ocelot_fdma_enabled) && - !ocelot_can_inject(ocelot, 0)) - return NETDEV_TX_BUSY; - - /* Check if timestamping is needed */ if (ocelot->ptp && (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { struct sk_buff *clone = NULL; if (ocelot_port_txtstamp_request(ocelot, port, skb, &clone)) { kfree_skb(skb); - return NETDEV_TX_OK; + return false; } if (clone) OCELOT_SKB_CB(skb)->clone = clone; - rew_op = ocelot_ptp_rew_op(skb); + *rew_op = ocelot_ptp_rew_op(skb); } - if (static_branch_unlikely(&ocelot_fdma_enabled)) { - ocelot_fdma_inject_frame(ocelot, port, rew_op, skb, dev); - } else { - ocelot_port_inject_frame(ocelot, port, 0, rew_op, skb); + return true; +} + +static netdev_tx_t ocelot_port_xmit_fdma(struct sk_buff *skb, + struct net_device *dev) +{ + struct ocelot_port_private *priv = netdev_priv(dev); + struct ocelot_port *ocelot_port = &priv->port; + struct ocelot *ocelot = ocelot_port->ocelot; + int port = priv->port.index; + u32 rew_op = 0; + + if (!ocelot_xmit_timestamp(ocelot, port, skb, &rew_op)) + return NETDEV_TX_OK; + + ocelot_fdma_inject_frame(ocelot, port, rew_op, skb, dev); + + return NETDEV_TX_OK; +} - consume_skb(skb); +static netdev_tx_t ocelot_port_xmit_inj(struct sk_buff *skb, + struct net_device *dev) +{ + struct ocelot_port_private *priv = netdev_priv(dev); + struct ocelot_port *ocelot_port = &priv->port; + struct ocelot *ocelot = ocelot_port->ocelot; + int port = priv->port.index; + u32 rew_op = 0; + + ocelot_lock_inj_grp(ocelot, 0); + + if (!ocelot_can_inject(ocelot, 0)) { + ocelot_unlock_inj_grp(ocelot, 0); + return NETDEV_TX_BUSY; } + if (!ocelot_xmit_timestamp(ocelot, port, skb, &rew_op)) { + ocelot_unlock_inj_grp(ocelot, 0); + return NETDEV_TX_OK; + } + + ocelot_port_inject_frame(ocelot, port, 0, rew_op, skb); + + ocelot_unlock_inj_grp(ocelot, 0); + + consume_skb(skb); + return NETDEV_TX_OK; } +static netdev_tx_t ocelot_port_xmit(struct sk_buff *skb, struct net_device *dev) +{ + if (static_branch_unlikely(&ocelot_fdma_enabled)) + return ocelot_port_xmit_fdma(skb, dev); + + return ocelot_port_xmit_inj(skb, dev); +} + enum ocelot_action_type { OCELOT_MACT_LEARN, OCELOT_MACT_FORGET, diff --git a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c index 7be30a8df2685..2f0cdbd4e2ac9 100644 --- a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c +++ b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c @@ -688,6 +688,9 @@ static int myri10ge_get_firmware_capabilities(struct myri10ge_priv *mgp) /* probe for IPv6 TSO support */ mgp->features = NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_TSO; + cmd.data0 = 0, + cmd.data1 = 0, + cmd.data2 = 0, status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_TSO6_HDR_SIZE, &cmd, 0); if (status == 0) { @@ -806,6 +809,7 @@ static int myri10ge_update_mac_address(struct myri10ge_priv *mgp, | (addr[2] << 8) | addr[3]); cmd.data1 = ((addr[4] << 8) | (addr[5])); + cmd.data2 = 0; status = myri10ge_send_cmd(mgp, MXGEFW_SET_MAC_ADDRESS, &cmd, 0); return status; @@ -817,6 +821,9 @@ static int myri10ge_change_pause(struct myri10ge_priv *mgp, int pause) int status, ctl; ctl = pause ? MXGEFW_ENABLE_FLOW_CONTROL : MXGEFW_DISABLE_FLOW_CONTROL; + cmd.data0 = 0, + cmd.data1 = 0, + cmd.data2 = 0, status = myri10ge_send_cmd(mgp, ctl, &cmd, 0); if (status) { @@ -834,6 +841,9 @@ myri10ge_change_promisc(struct myri10ge_priv *mgp, int promisc, int atomic) int status, ctl; ctl = promisc ? MXGEFW_ENABLE_PROMISC : MXGEFW_DISABLE_PROMISC; + cmd.data0 = 0; + cmd.data1 = 0; + cmd.data2 = 0; status = myri10ge_send_cmd(mgp, ctl, &cmd, atomic); if (status) netdev_err(mgp->dev, "Failed to set promisc mode\n"); @@ -1946,6 +1956,8 @@ static int myri10ge_allocate_rings(struct myri10ge_slice_state *ss) /* get ring sizes */ slice = ss - mgp->ss; cmd.data0 = slice; + cmd.data1 = 0; + cmd.data2 = 0; status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd, 0); tx_ring_size = cmd.data0; cmd.data0 = slice; @@ -2238,12 +2250,16 @@ static int myri10ge_get_txrx(struct myri10ge_priv *mgp, int slice) status = 0; if (slice == 0 || (mgp->dev->real_num_tx_queues > 1)) { cmd.data0 = slice; + cmd.data1 = 0; + cmd.data2 = 0; status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_OFFSET, &cmd, 0); ss->tx.lanai = (struct mcp_kreq_ether_send __iomem *) (mgp->sram + cmd.data0); } cmd.data0 = slice; + cmd.data1 = 0; + cmd.data2 = 0; status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SMALL_RX_OFFSET, &cmd, 0); ss->rx_small.lanai = (struct mcp_kreq_ether_recv __iomem *) @@ -2312,6 +2328,7 @@ static int myri10ge_open(struct net_device *dev) if (mgp->num_slices > 1) { cmd.data0 = mgp->num_slices; cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; + cmd.data2 = 0; if (mgp->dev->real_num_tx_queues > 1) cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES, @@ -2414,6 +2431,8 @@ static int myri10ge_open(struct net_device *dev) /* now give firmware buffers sizes, and MTU */ cmd.data0 = dev->mtu + ETH_HLEN + VLAN_HLEN; + cmd.data1 = 0; + cmd.data2 = 0; status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_MTU, &cmd, 0); cmd.data0 = mgp->small_bytes; status |= @@ -2472,7 +2491,6 @@ static int myri10ge_open(struct net_device *dev) static int myri10ge_close(struct net_device *dev) { struct myri10ge_priv *mgp = netdev_priv(dev); - struct myri10ge_cmd cmd; int status, old_down_cnt; int i; @@ -2491,8 +2509,13 @@ static int myri10ge_close(struct net_device *dev) netif_tx_stop_all_queues(dev); if (mgp->rebooted == 0) { + struct myri10ge_cmd cmd; + old_down_cnt = mgp->down_cnt; mb(); + cmd.data0 = 0; + cmd.data1 = 0; + cmd.data2 = 0; status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_DOWN, &cmd, 0); if (status) @@ -2956,6 +2979,9 @@ static void myri10ge_set_multicast_list(struct net_device *dev) /* Disable multicast filtering */ + cmd.data0 = 0; + cmd.data1 = 0; + cmd.data2 = 0; err = myri10ge_send_cmd(mgp, MXGEFW_ENABLE_ALLMULTI, &cmd, 1); if (err != 0) { netdev_err(dev, "Failed MXGEFW_ENABLE_ALLMULTI, error status: %d\n", diff --git a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c index 2d9efadb5d2ae..347b0aff100b9 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c @@ -263,9 +263,10 @@ static int ionic_get_link_ksettings(struct net_device *netdev, /* This means there's no module plugged in */ break; default: - dev_info(lif->ionic->dev, "unknown xcvr type pid=%d / 0x%x\n", - idev->port_info->status.xcvr.pid, - idev->port_info->status.xcvr.pid); + dev_dbg_ratelimited(lif->ionic->dev, + "unknown xcvr type pid=%d / 0x%x\n", + idev->port_info->status.xcvr.pid, + idev->port_info->status.xcvr.pid); break; } diff --git a/drivers/net/ethernet/renesas/rswitch_l2.c b/drivers/net/ethernet/renesas/rswitch_l2.c index 4a69ec77d69c6..9433cd8adced9 100644 --- a/drivers/net/ethernet/renesas/rswitch_l2.c +++ b/drivers/net/ethernet/renesas/rswitch_l2.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Renesas Ethernet Switch device driver * - * Copyright (C) 2025 Renesas Electronics Corporation + * Copyright (C) 2025 - 2026 Renesas Electronics Corporation */ #include @@ -60,6 +60,7 @@ static void rswitch_update_l2_hw_learning(struct rswitch_private *priv) static void rswitch_update_l2_hw_forwarding(struct rswitch_private *priv) { struct rswitch_device *rdev; + bool new_forwarding_offload; unsigned int fwd_mask; /* calculate fwd_mask with zeroes in bits corresponding to ports that @@ -73,8 +74,9 @@ static void rswitch_update_l2_hw_forwarding(struct rswitch_private *priv) } rswitch_for_all_ports(priv, rdev) { - if ((rdev_for_l2_offload(rdev) && rdev->forwarding_requested) || - rdev->forwarding_offloaded) { + new_forwarding_offload = (rdev_for_l2_offload(rdev) && rdev->forwarding_requested); + + if (new_forwarding_offload || rdev->forwarding_offloaded) { /* Update allowed offload destinations even for ports * with L2 offload enabled earlier. * @@ -84,13 +86,10 @@ static void rswitch_update_l2_hw_forwarding(struct rswitch_private *priv) priv->addr + FWPC2(rdev->port)); } - if (rdev_for_l2_offload(rdev) && - rdev->forwarding_requested && - !rdev->forwarding_offloaded) { + if (new_forwarding_offload && !rdev->forwarding_offloaded) rswitch_change_l2_hw_offloading(rdev, true, false); - } else if (rdev->forwarding_offloaded) { + else if (!new_forwarding_offload && rdev->forwarding_offloaded) rswitch_change_l2_hw_offloading(rdev, false, false); - } } } diff --git a/drivers/net/ethernet/stmicro/stmmac/common.h b/drivers/net/ethernet/stmicro/stmmac/common.h index 8f34c9ad457f0..9e012720a69fa 100644 --- a/drivers/net/ethernet/stmicro/stmmac/common.h +++ b/drivers/net/ethernet/stmicro/stmmac/common.h @@ -43,6 +43,11 @@ #define DWXGMAC_ID 0x76 #define DWXLGMAC_ID 0x27 +static inline bool dwmac_is_xmac(enum dwmac_core_type core_type) +{ + return core_type == DWMAC_CORE_GMAC4 || core_type == DWMAC_CORE_XGMAC; +} + #define STMMAC_CHAN0 0 /* Always supported and default for all chips */ /* TX and RX Descriptor Length, these need to be power of two. @@ -376,7 +381,6 @@ enum request_irq_err { REQ_IRQ_ERR_SFTY, REQ_IRQ_ERR_SFTY_UE, REQ_IRQ_ERR_SFTY_CE, - REQ_IRQ_ERR_LPI, REQ_IRQ_ERR_WOL, REQ_IRQ_ERR_MAC, REQ_IRQ_ERR_NO, diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c index e8539cad4602e..1d30f2fb984f1 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c @@ -109,7 +109,7 @@ static int dwc_eth_dwmac_config_dt(struct platform_device *pdev, } /* dwc-qos needs GMAC4, AAL, TSO and PMT */ - plat_dat->has_gmac4 = 1; + plat_dat->core_type = DWMAC_CORE_GMAC4; plat_dat->dma_cfg->aal = 1; plat_dat->flags |= STMMAC_FLAG_TSO_EN; plat_dat->pmt = 1; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c index e74d00984b889..47fda982d6b15 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c @@ -565,7 +565,7 @@ static void common_default_data(struct plat_stmmacenet_data *plat) { /* clk_csr_i = 20-35MHz & MDC = clk_csr_i/16 */ plat->clk_csr = STMMAC_CSR_20_35M; - plat->has_gmac = 1; + plat->core_type = DWMAC_CORE_GMAC; plat->force_sf_dma_mode = 1; plat->mdio_bus_data->needs_reset = true; @@ -612,8 +612,7 @@ static int intel_mgbe_common_data(struct pci_dev *pdev, plat->pdev = pdev; plat->phy_addr = -1; plat->clk_csr = STMMAC_CSR_250_300M; - plat->has_gmac = 0; - plat->has_gmac4 = 1; + plat->core_type = DWMAC_CORE_GMAC4; plat->force_sf_dma_mode = 0; plat->flags |= (STMMAC_FLAG_TSO_EN | STMMAC_FLAG_SPH_DISABLE); @@ -760,7 +759,6 @@ static int intel_mgbe_common_data(struct pci_dev *pdev, /* Setup MSI vector offset specific to Intel mGbE controller */ plat->msi_mac_vec = 29; - plat->msi_lpi_vec = 28; plat->msi_sfty_ce_vec = 27; plat->msi_sfty_ue_vec = 26; plat->msi_rx_base_vec = 0; @@ -1218,8 +1216,6 @@ static int stmmac_config_multi_msi(struct pci_dev *pdev, res->irq = pci_irq_vector(pdev, plat->msi_mac_vec); if (plat->msi_wol_vec < STMMAC_MSI_VEC_MAX) res->wol_irq = pci_irq_vector(pdev, plat->msi_wol_vec); - if (plat->msi_lpi_vec < STMMAC_MSI_VEC_MAX) - res->lpi_irq = pci_irq_vector(pdev, plat->msi_lpi_vec); if (plat->msi_sfty_ce_vec < STMMAC_MSI_VEC_MAX) res->sfty_ce_irq = pci_irq_vector(pdev, plat->msi_sfty_ce_vec); if (plat->msi_sfty_ue_vec < STMMAC_MSI_VEC_MAX) @@ -1335,7 +1331,6 @@ static int intel_eth_pci_probe(struct pci_dev *pdev, */ plat->msi_mac_vec = STMMAC_MSI_VEC_MAX; plat->msi_wol_vec = STMMAC_MSI_VEC_MAX; - plat->msi_lpi_vec = STMMAC_MSI_VEC_MAX; plat->msi_sfty_ce_vec = STMMAC_MSI_VEC_MAX; plat->msi_sfty_ue_vec = STMMAC_MSI_VEC_MAX; plat->msi_rx_base_vec = STMMAC_MSI_VEC_MAX; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-ipq806x.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-ipq806x.c index ca4035cbb55b6..c05f85534f0ca 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-ipq806x.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-ipq806x.c @@ -473,7 +473,7 @@ static int ipq806x_gmac_probe(struct platform_device *pdev) return err; } - plat_dat->has_gmac = true; + plat_dat->core_type = DWMAC_CORE_GMAC; plat_dat->bsp_priv = gmac; plat_dat->set_clk_tx_rate = ipq806x_gmac_set_clk_tx_rate; plat_dat->multicast_filter_bins = 0; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c index 592aa9d636e50..ab431bf9b25f2 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c @@ -90,9 +90,9 @@ static void loongson_default_data(struct pci_dev *pdev, /* Get bus_id, this can be overwritten later */ plat->bus_id = pci_dev_id(pdev); - /* clk_csr_i = 20-35MHz & MDC = clk_csr_i/16 */ - plat->clk_csr = STMMAC_CSR_20_35M; - plat->has_gmac = 1; + /* clk_csr_i = 100-150MHz & MDC = clk_csr_i/62 */ + plat->clk_csr = STMMAC_CSR_100_150M; + plat->core_type = DWMAC_CORE_GMAC; plat->force_sf_dma_mode = 1; /* Set default value for multicast hash bins */ @@ -465,13 +465,6 @@ static int loongson_dwmac_dt_config(struct pci_dev *pdev, res->wol_irq = res->irq; } - res->lpi_irq = of_irq_get_byname(np, "eth_lpi"); - if (res->lpi_irq < 0) { - dev_err(&pdev->dev, "IRQ eth_lpi not found\n"); - ret = -ENODEV; - goto err_put_node; - } - ret = device_get_phy_mode(&pdev->dev); if (ret < 0) { dev_err(&pdev->dev, "phy_mode not found\n"); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-lpc18xx.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-lpc18xx.c index 2562a6d036a2a..6fffc9dfbae55 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-lpc18xx.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-lpc18xx.c @@ -41,7 +41,7 @@ static int lpc18xx_dwmac_probe(struct platform_device *pdev) if (IS_ERR(plat_dat)) return PTR_ERR(plat_dat); - plat_dat->has_gmac = true; + plat_dat->core_type = DWMAC_CORE_GMAC; reg = syscon_regmap_lookup_by_compatible("nxp,lpc1850-creg"); if (IS_ERR(reg)) { diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c index 7117559ddaf14..594e7ff82f2f1 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c @@ -96,7 +96,6 @@ struct ethqos_emac_driver_data { bool rgmii_config_loopback_en; bool has_emac_ge_3; const char *link_clk_name; - bool has_integrated_pcs; u32 dma_addr_width; struct dwmac4_addrs dwmac4_addrs; bool needs_sgmii_loopback; @@ -282,7 +281,6 @@ static const struct ethqos_emac_driver_data emac_v4_0_0_data = { .rgmii_config_loopback_en = false, .has_emac_ge_3 = true, .link_clk_name = "phyaux", - .has_integrated_pcs = true, .needs_sgmii_loopback = true, .dma_addr_width = 36, .dwmac4_addrs = { @@ -852,7 +850,7 @@ static int qcom_ethqos_probe(struct platform_device *pdev) plat_dat->fix_mac_speed = ethqos_fix_mac_speed; plat_dat->dump_debug_regs = rgmii_dump; plat_dat->ptp_clk_freq_config = ethqos_ptp_clk_freq_config; - plat_dat->has_gmac4 = 1; + plat_dat->core_type = DWMAC_CORE_GMAC4; if (ethqos->has_emac_ge_3) plat_dat->dwmac4_addrs = &data->dwmac4_addrs; plat_dat->pmt = 1; @@ -860,8 +858,6 @@ static int qcom_ethqos_probe(struct platform_device *pdev) plat_dat->flags |= STMMAC_FLAG_TSO_EN; if (of_device_is_compatible(np, "qcom,qcs404-ethqos")) plat_dat->flags |= STMMAC_FLAG_RX_CLK_RUNS_IN_LPI; - if (data->has_integrated_pcs) - plat_dat->flags |= STMMAC_FLAG_HAS_INTEGRATED_PCS; if (data->dma_addr_width) plat_dat->host_dma_width = data->dma_addr_width; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c index 0786816e05f04..643578266dfca 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c @@ -1751,8 +1751,8 @@ static int rk_gmac_probe(struct platform_device *pdev) /* If the stmmac is not already selected as gmac4, * then make sure we fallback to gmac. */ - if (!plat_dat->has_gmac4) { - plat_dat->has_gmac = true; + if (plat_dat->core_type != DWMAC_CORE_GMAC4) { + plat_dat->core_type = DWMAC_CORE_GMAC; plat_dat->rx_fifo_size = 4096; plat_dat->tx_fifo_size = 2048; } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-s32.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-s32.c index 221539d760bc8..ee095ac132037 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-s32.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-s32.c @@ -146,7 +146,7 @@ static int s32_dwmac_probe(struct platform_device *pdev) gmac->ioaddr = res.addr; /* S32CC core feature set */ - plat->has_gmac4 = true; + plat->core_type = DWMAC_CORE_GMAC4; plat->pmt = 1; plat->flags |= STMMAC_FLAG_SPH_DISABLE; plat->rx_fifo_size = 20480; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c index 354f01184e6cc..2ff5db6d41ca0 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c @@ -497,7 +497,7 @@ static int socfpga_dwmac_probe(struct platform_device *pdev) plat_dat->pcs_init = socfpga_dwmac_pcs_init; plat_dat->pcs_exit = socfpga_dwmac_pcs_exit; plat_dat->select_pcs = socfpga_dwmac_select_pcs; - plat_dat->has_gmac = true; + plat_dat->core_type = DWMAC_CORE_GMAC; plat_dat->riwt_off = 1; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c index 1eadcf5d1ad63..7f560d78209d1 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c @@ -136,7 +136,7 @@ static int sun7i_gmac_probe(struct platform_device *pdev) /* platform data specifying hardware features and callbacks. * hardware features were copied from Allwinner drivers. */ plat_dat->tx_coe = 1; - plat_dat->has_gmac = true; + plat_dat->core_type = DWMAC_CORE_GMAC; plat_dat->bsp_priv = gmac; plat_dat->init = sun7i_gmac_init; plat_dat->exit = sun7i_gmac_exit; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c index dc903b846b1bf..d765acbe37548 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c @@ -308,7 +308,7 @@ static int tegra_mgbe_probe(struct platform_device *pdev) goto disable_clks; } - plat->has_xgmac = 1; + plat->core_type = DWMAC_CORE_XGMAC; plat->flags |= STMMAC_FLAG_TSO_EN; plat->pmt = 1; plat->bsp_priv = mgbe; diff --git a/drivers/net/ethernet/stmicro/stmmac/hwif.c b/drivers/net/ethernet/stmicro/stmmac/hwif.c index 3f7c765dcb797..00083ce525492 100644 --- a/drivers/net/ethernet/stmicro/stmmac/hwif.c +++ b/drivers/net/ethernet/stmicro/stmmac/hwif.c @@ -106,9 +106,7 @@ int stmmac_reset(struct stmmac_priv *priv, void __iomem *ioaddr) } static const struct stmmac_hwif_entry { - bool gmac; - bool gmac4; - bool xgmac; + enum dwmac_core_type core_type; u32 min_id; u32 dev_id; const struct stmmac_regs_off regs; @@ -127,9 +125,7 @@ static const struct stmmac_hwif_entry { } stmmac_hw[] = { /* NOTE: New HW versions shall go to the end of this table */ { - .gmac = false, - .gmac4 = false, - .xgmac = false, + .core_type = DWMAC_CORE_MAC100, .min_id = 0, .regs = { .ptp_off = PTP_GMAC3_X_OFFSET, @@ -146,9 +142,7 @@ static const struct stmmac_hwif_entry { .setup = dwmac100_setup, .quirks = stmmac_dwmac1_quirks, }, { - .gmac = true, - .gmac4 = false, - .xgmac = false, + .core_type = DWMAC_CORE_GMAC, .min_id = 0, .regs = { .ptp_off = PTP_GMAC3_X_OFFSET, @@ -165,9 +159,7 @@ static const struct stmmac_hwif_entry { .setup = dwmac1000_setup, .quirks = stmmac_dwmac1_quirks, }, { - .gmac = false, - .gmac4 = true, - .xgmac = false, + .core_type = DWMAC_CORE_GMAC4, .min_id = 0, .regs = { .ptp_off = PTP_GMAC4_OFFSET, @@ -187,9 +179,7 @@ static const struct stmmac_hwif_entry { .setup = dwmac4_setup, .quirks = stmmac_dwmac4_quirks, }, { - .gmac = false, - .gmac4 = true, - .xgmac = false, + .core_type = DWMAC_CORE_GMAC4, .min_id = DWMAC_CORE_4_00, .regs = { .ptp_off = PTP_GMAC4_OFFSET, @@ -210,9 +200,7 @@ static const struct stmmac_hwif_entry { .setup = dwmac4_setup, .quirks = NULL, }, { - .gmac = false, - .gmac4 = true, - .xgmac = false, + .core_type = DWMAC_CORE_GMAC4, .min_id = DWMAC_CORE_4_10, .regs = { .ptp_off = PTP_GMAC4_OFFSET, @@ -233,9 +221,7 @@ static const struct stmmac_hwif_entry { .setup = dwmac4_setup, .quirks = NULL, }, { - .gmac = false, - .gmac4 = true, - .xgmac = false, + .core_type = DWMAC_CORE_GMAC4, .min_id = DWMAC_CORE_5_10, .regs = { .ptp_off = PTP_GMAC4_OFFSET, @@ -256,9 +242,7 @@ static const struct stmmac_hwif_entry { .setup = dwmac4_setup, .quirks = NULL, }, { - .gmac = false, - .gmac4 = false, - .xgmac = true, + .core_type = DWMAC_CORE_XGMAC, .min_id = DWXGMAC_CORE_2_10, .dev_id = DWXGMAC_ID, .regs = { @@ -280,9 +264,7 @@ static const struct stmmac_hwif_entry { .setup = dwxgmac2_setup, .quirks = NULL, }, { - .gmac = false, - .gmac4 = false, - .xgmac = true, + .core_type = DWMAC_CORE_XGMAC, .min_id = DWXLGMAC_CORE_2_00, .dev_id = DWXLGMAC_ID, .regs = { @@ -308,20 +290,18 @@ static const struct stmmac_hwif_entry { int stmmac_hwif_init(struct stmmac_priv *priv) { - bool needs_xgmac = priv->plat->has_xgmac; - bool needs_gmac4 = priv->plat->has_gmac4; - bool needs_gmac = priv->plat->has_gmac; + enum dwmac_core_type core_type = priv->plat->core_type; const struct stmmac_hwif_entry *entry; struct mac_device_info *mac; bool needs_setup = true; u32 id, dev_id = 0; int i, ret; - if (needs_gmac) { + if (core_type == DWMAC_CORE_GMAC) { id = stmmac_get_id(priv, GMAC_VERSION); - } else if (needs_gmac4 || needs_xgmac) { + } else if (dwmac_is_xmac(core_type)) { id = stmmac_get_id(priv, GMAC4_VERSION); - if (needs_xgmac) + if (core_type == DWMAC_CORE_XGMAC) dev_id = stmmac_get_dev_id(priv, GMAC4_VERSION); } else { id = 0; @@ -331,14 +311,16 @@ int stmmac_hwif_init(struct stmmac_priv *priv) priv->synopsys_id = id; /* Lets assume some safe values first */ - priv->ptpaddr = priv->ioaddr + - (needs_gmac4 ? PTP_GMAC4_OFFSET : PTP_GMAC3_X_OFFSET); - priv->mmcaddr = priv->ioaddr + - (needs_gmac4 ? MMC_GMAC4_OFFSET : MMC_GMAC3_X_OFFSET); - if (needs_gmac4) + if (core_type == DWMAC_CORE_GMAC4) { + priv->ptpaddr = priv->ioaddr + PTP_GMAC4_OFFSET; + priv->mmcaddr = priv->ioaddr + MMC_GMAC4_OFFSET; priv->estaddr = priv->ioaddr + EST_GMAC4_OFFSET; - else if (needs_xgmac) - priv->estaddr = priv->ioaddr + EST_XGMAC_OFFSET; + } else { + priv->ptpaddr = priv->ioaddr + PTP_GMAC3_X_OFFSET; + priv->mmcaddr = priv->ioaddr + MMC_GMAC3_X_OFFSET; + if (core_type == DWMAC_CORE_XGMAC) + priv->estaddr = priv->ioaddr + EST_XGMAC_OFFSET; + } /* Check for HW specific setup first */ if (priv->plat->setup) { @@ -355,16 +337,12 @@ int stmmac_hwif_init(struct stmmac_priv *priv) for (i = ARRAY_SIZE(stmmac_hw) - 1; i >= 0; i--) { entry = &stmmac_hw[i]; - if (needs_gmac ^ entry->gmac) - continue; - if (needs_gmac4 ^ entry->gmac4) - continue; - if (needs_xgmac ^ entry->xgmac) + if (core_type != entry->core_type) continue; /* Use synopsys_id var because some setups can override this */ if (priv->synopsys_id < entry->min_id) continue; - if (needs_xgmac && (dev_id ^ entry->dev_id)) + if (core_type == DWMAC_CORE_XGMAC && (dev_id ^ entry->dev_id)) continue; /* Only use generic HW helpers if needed */ @@ -400,6 +378,7 @@ int stmmac_hwif_init(struct stmmac_priv *priv) } dev_err(priv->device, "Failed to find HW IF (id=0x%x, gmac=%d/%d)\n", - id, needs_gmac, needs_gmac4); + id, core_type == DWMAC_CORE_GMAC, + core_type == DWMAC_CORE_GMAC4); return -EINVAL; } diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac.h b/drivers/net/ethernet/stmicro/stmmac/stmmac.h index 7ca5477be390b..865531d6cd3b8 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac.h +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac.h @@ -29,7 +29,6 @@ struct stmmac_resources { void __iomem *addr; u8 mac[ETH_ALEN]; int wol_irq; - int lpi_irq; int irq; int sfty_irq; int sfty_ce_irq; @@ -291,7 +290,6 @@ struct stmmac_priv { int wol_irq; u32 gmii_address_bus_config; struct timer_list eee_ctrl_timer; - int lpi_irq; u32 tx_lpi_timer; bool tx_lpi_clk_stop; bool eee_enabled; @@ -318,6 +316,7 @@ struct stmmac_priv { void __iomem *ptpaddr; void __iomem *estaddr; unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)]; + unsigned int num_double_vlans; int sfty_irq; int sfty_ce_irq; int sfty_ue_irq; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_est.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_est.c index 4b513d27a9889..afc516059b897 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_est.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_est.c @@ -53,7 +53,7 @@ static int est_configure(struct stmmac_priv *priv, struct stmmac_est *cfg, } ctrl = readl(est_addr + EST_CONTROL); - if (priv->plat->has_xgmac) { + if (priv->plat->core_type == DWMAC_CORE_XGMAC) { ctrl &= ~EST_XGMAC_PTOV; ctrl |= ((NSEC_PER_SEC / ptp_rate) * EST_XGMAC_PTOV_MUL) << EST_XGMAC_PTOV_SHIFT; @@ -148,7 +148,7 @@ static void est_irq_status(struct stmmac_priv *priv, struct net_device *dev, } if (status & EST_BTRE) { - if (priv->plat->has_xgmac) { + if (priv->plat->core_type == DWMAC_CORE_XGMAC) { btrl = FIELD_GET(EST_XGMAC_BTRL, status); btrl_max = FIELD_MAX(EST_XGMAC_BTRL); } else { diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c index 39fa1ec92f82f..81d4039e1c082 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c @@ -303,9 +303,10 @@ static void stmmac_ethtool_getdrvinfo(struct net_device *dev, { struct stmmac_priv *priv = netdev_priv(dev); - if (priv->plat->has_gmac || priv->plat->has_gmac4) + if (priv->plat->core_type == DWMAC_CORE_GMAC || + priv->plat->core_type == DWMAC_CORE_GMAC4) strscpy(info->driver, GMAC_ETHTOOL_NAME, sizeof(info->driver)); - else if (priv->plat->has_xgmac) + else if (priv->plat->core_type == DWMAC_CORE_XGMAC) strscpy(info->driver, XGMAC_ETHTOOL_NAME, sizeof(info->driver)); else strscpy(info->driver, MAC100_ETHTOOL_NAME, @@ -322,47 +323,6 @@ static int stmmac_ethtool_get_link_ksettings(struct net_device *dev, { struct stmmac_priv *priv = netdev_priv(dev); - if (!(priv->plat->flags & STMMAC_FLAG_HAS_INTEGRATED_PCS) && - (priv->hw->pcs & STMMAC_PCS_RGMII || - priv->hw->pcs & STMMAC_PCS_SGMII)) { - u32 supported, advertising, lp_advertising; - - if (!priv->xstats.pcs_link) { - cmd->base.speed = SPEED_UNKNOWN; - cmd->base.duplex = DUPLEX_UNKNOWN; - return 0; - } - cmd->base.duplex = priv->xstats.pcs_duplex; - - cmd->base.speed = priv->xstats.pcs_speed; - - /* Encoding of PSE bits is defined in 802.3z, 37.2.1.4 */ - - ethtool_convert_link_mode_to_legacy_u32( - &supported, cmd->link_modes.supported); - ethtool_convert_link_mode_to_legacy_u32( - &advertising, cmd->link_modes.advertising); - ethtool_convert_link_mode_to_legacy_u32( - &lp_advertising, cmd->link_modes.lp_advertising); - - /* Reg49[3] always set because ANE is always supported */ - cmd->base.autoneg = ADVERTISED_Autoneg; - supported |= SUPPORTED_Autoneg; - advertising |= ADVERTISED_Autoneg; - lp_advertising |= ADVERTISED_Autoneg; - - cmd->base.port = PORT_OTHER; - - ethtool_convert_legacy_u32_to_link_mode( - cmd->link_modes.supported, supported); - ethtool_convert_legacy_u32_to_link_mode( - cmd->link_modes.advertising, advertising); - ethtool_convert_legacy_u32_to_link_mode( - cmd->link_modes.lp_advertising, lp_advertising); - - return 0; - } - return phylink_ethtool_ksettings_get(priv->phylink, cmd); } @@ -372,20 +332,6 @@ stmmac_ethtool_set_link_ksettings(struct net_device *dev, { struct stmmac_priv *priv = netdev_priv(dev); - if (!(priv->plat->flags & STMMAC_FLAG_HAS_INTEGRATED_PCS) && - (priv->hw->pcs & STMMAC_PCS_RGMII || - priv->hw->pcs & STMMAC_PCS_SGMII)) { - /* Only support ANE */ - if (cmd->base.autoneg != AUTONEG_ENABLE) - return -EINVAL; - - mutex_lock(&priv->lock); - stmmac_pcs_ctrl_ane(priv, 1, priv->hw->ps, 0); - mutex_unlock(&priv->lock); - - return 0; - } - return phylink_ethtool_ksettings_set(priv->phylink, cmd); } @@ -406,9 +352,9 @@ static int stmmac_ethtool_get_regs_len(struct net_device *dev) { struct stmmac_priv *priv = netdev_priv(dev); - if (priv->plat->has_xgmac) + if (priv->plat->core_type == DWMAC_CORE_XGMAC) return XGMAC_REGSIZE * 4; - else if (priv->plat->has_gmac4) + else if (priv->plat->core_type == DWMAC_CORE_GMAC4) return GMAC4_REG_SPACE_SIZE; return REG_SPACE_SIZE; } @@ -423,12 +369,12 @@ static void stmmac_ethtool_gregs(struct net_device *dev, stmmac_dump_dma_regs(priv, priv->ioaddr, reg_space); /* Copy DMA registers to where ethtool expects them */ - if (priv->plat->has_gmac4) { + if (priv->plat->core_type == DWMAC_CORE_GMAC4) { /* GMAC4 dumps its DMA registers at its DMA_CHAN_BASE_ADDR */ memcpy(®_space[ETHTOOL_DMA_OFFSET], ®_space[GMAC4_DMA_CHAN_BASE_ADDR / 4], NUM_DWMAC4_DMA_REGS * 4); - } else if (!priv->plat->has_xgmac) { + } else if (priv->plat->core_type != DWMAC_CORE_XGMAC) { memcpy(®_space[ETHTOOL_DMA_OFFSET], ®_space[DMA_BUS_MODE / 4], NUM_DWMAC1000_DMA_REGS * 4); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 0dd17179c85d2..eeeb9f50c265c 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -139,6 +139,7 @@ static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue); static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue); static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, u32 rxmode, u32 chan); +static int stmmac_vlan_restore(struct stmmac_priv *priv); #ifdef CONFIG_DEBUG_FS static const struct net_device_ops stmmac_netdev_ops; @@ -446,7 +447,7 @@ static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, if (!priv->hwts_rx_en) return; /* For GMAC4, the valid timestamp is from CTX next desc. */ - if (priv->plat->has_gmac4 || priv->plat->has_xgmac) + if (dwmac_is_xmac(priv->plat->core_type)) desc = np; /* Check if timestamp is available */ @@ -697,7 +698,7 @@ static int stmmac_hwtstamp_get(struct net_device *dev, static int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags) { - bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; + bool xmac = dwmac_is_xmac(priv->plat->core_type); struct timespec64 now; u32 sec_inc = 0; u64 temp = 0; @@ -746,7 +747,7 @@ static int stmmac_init_tstamp_counter(struct stmmac_priv *priv, */ static int stmmac_init_timestamping(struct stmmac_priv *priv) { - bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; + bool xmac = dwmac_is_xmac(priv->plat->core_type); int ret; if (priv->plat->ptp_clk_freq_config) @@ -2398,7 +2399,7 @@ static void stmmac_dma_operation_mode(struct stmmac_priv *priv) txfifosz = priv->dma_cap.tx_fifo_size; /* Split up the shared Tx/Rx FIFO memory on DW QoS Eth and DW XGMAC */ - if (priv->plat->has_gmac4 || priv->plat->has_xgmac) { + if (dwmac_is_xmac(priv->plat->core_type)) { rxfifosz /= rx_channels_count; txfifosz /= tx_channels_count; } @@ -3579,10 +3580,6 @@ static void stmmac_free_irq(struct net_device *dev, free_irq(priv->sfty_ce_irq, dev); fallthrough; case REQ_IRQ_ERR_SFTY_CE: - if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) - free_irq(priv->lpi_irq, dev); - fallthrough; - case REQ_IRQ_ERR_LPI: if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) free_irq(priv->wol_irq, dev); fallthrough; @@ -3640,24 +3637,6 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev) } } - /* Request the LPI IRQ in case of another line - * is used for LPI - */ - if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { - int_name = priv->int_name_lpi; - sprintf(int_name, "%s:%s", dev->name, "lpi"); - ret = request_irq(priv->lpi_irq, - stmmac_mac_interrupt, - 0, int_name, dev); - if (unlikely(ret < 0)) { - netdev_err(priv->dev, - "%s: alloc lpi MSI %d (error: %d)\n", - __func__, priv->lpi_irq, ret); - irq_err = REQ_IRQ_ERR_LPI; - goto irq_error; - } - } - /* Request the common Safety Feature Correctible/Uncorrectible * Error line in case of another line is used */ @@ -3797,19 +3776,6 @@ static int stmmac_request_irq_single(struct net_device *dev) } } - /* Request the IRQ lines */ - if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { - ret = request_irq(priv->lpi_irq, stmmac_interrupt, - IRQF_SHARED, dev->name, dev); - if (unlikely(ret < 0)) { - netdev_err(priv->dev, - "%s: ERROR: allocating the LPI IRQ %d (%d)\n", - __func__, priv->lpi_irq, ret); - irq_err = REQ_IRQ_ERR_LPI; - goto irq_error; - } - } - /* Request the common Safety Feature Correctible/Uncorrectible * Error line in case of another line is used */ @@ -3967,6 +3933,8 @@ static int __stmmac_open(struct net_device *dev, /* We may have called phylink_speed_down before */ phylink_speed_up(priv->phylink); + stmmac_vlan_restore(priv); + ret = stmmac_request_irq(dev); if (ret) goto irq_error; @@ -4514,7 +4482,8 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) if (skb_is_gso(skb) && priv->tso) { if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) return stmmac_tso_xmit(skb, dev); - if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4)) + if (priv->plat->core_type == DWMAC_CORE_GMAC4 && + (gso & SKB_GSO_UDP_L4)) return stmmac_tso_xmit(skb, dev); } @@ -4879,13 +4848,27 @@ static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, if (!priv->sph) return 0; - /* Not last descriptor */ - if (status & rx_not_ls) + /* For GMAC4, when split header is enabled, in some rare cases, the + * hardware does not fill buf2 of the first descriptor with payload. + * Thus we cannot assume buf2 is always fully filled if it is not + * the last descriptor. Otherwise, the length of buf2 of the second + * descriptor will be calculated wrong and cause an oops. + * + * If this is the last descriptor, 'plen' is the length of the + * received packet that was transferred to system memory. + * Otherwise, it is the accumulated number of bytes that have been + * transferred for the current packet. + * + * Thus 'plen - len' always gives the correct length of buf2. + */ + + /* Not GMAC4 and not last descriptor */ + if (priv->plat->core_type != DWMAC_CORE_GMAC4 && (status & rx_not_ls)) return priv->dma_conf.dma_buf_sz; + /* GMAC4 or last descriptor */ plen = stmmac_get_rx_frame_len(priv, p, coe); - /* Last descriptor */ return plen - len; } @@ -5984,7 +5967,7 @@ static void stmmac_common_interrupt(struct stmmac_priv *priv) u32 queue; bool xmac; - xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; + xmac = dwmac_is_xmac(priv->plat->core_type); queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt; if (priv->irq_wake) @@ -5998,7 +5981,7 @@ static void stmmac_common_interrupt(struct stmmac_priv *priv) stmmac_fpe_irq_status(priv); /* To handle GMAC own interrupts */ - if ((priv->plat->has_gmac) || xmac) { + if (priv->plat->core_type == DWMAC_CORE_GMAC || xmac) { int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats); if (unlikely(status)) { @@ -6012,15 +5995,6 @@ static void stmmac_common_interrupt(struct stmmac_priv *priv) for (queue = 0; queue < queues_count; queue++) stmmac_host_mtl_irq_status(priv, priv->hw, queue); - /* PCS link status */ - if (priv->hw->pcs && - !(priv->plat->flags & STMMAC_FLAG_HAS_INTEGRATED_PCS)) { - if (priv->xstats.pcs_link) - netif_carrier_on(priv->dev); - else - netif_carrier_off(priv->dev); - } - stmmac_timestamp_interrupt(priv, priv); } } @@ -6368,7 +6342,7 @@ static int stmmac_dma_cap_show(struct seq_file *seq, void *v) (priv->dma_cap.mbps_1000) ? "Y" : "N"); seq_printf(seq, "\tHalf duplex: %s\n", (priv->dma_cap.half_duplex) ? "Y" : "N"); - if (priv->plat->has_xgmac) { + if (priv->plat->core_type == DWMAC_CORE_XGMAC) { seq_printf(seq, "\tNumber of Additional MAC address registers: %d\n", priv->dma_cap.multi_addr); @@ -6392,7 +6366,7 @@ static int stmmac_dma_cap_show(struct seq_file *seq, void *v) (priv->dma_cap.time_stamp) ? "Y" : "N"); seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n", (priv->dma_cap.atime_stamp) ? "Y" : "N"); - if (priv->plat->has_xgmac) + if (priv->plat->core_type == DWMAC_CORE_XGMAC) seq_printf(seq, "\tTimestamp System Time Source: %s\n", dwxgmac_timestamp_source[priv->dma_cap.tssrc]); seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n", @@ -6401,7 +6375,7 @@ static int stmmac_dma_cap_show(struct seq_file *seq, void *v) seq_printf(seq, "\tChecksum Offload in TX: %s\n", (priv->dma_cap.tx_coe) ? "Y" : "N"); if (priv->synopsys_id >= DWMAC_CORE_4_00 || - priv->plat->has_xgmac) { + priv->plat->core_type == DWMAC_CORE_XGMAC) { seq_printf(seq, "\tIP Checksum Offload in RX: %s\n", (priv->dma_cap.rx_coe) ? "Y" : "N"); } else { @@ -6604,6 +6578,9 @@ static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double) hash = 0; } + if (!netif_running(priv->dev)) + return 0; + return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double); } @@ -6613,6 +6590,7 @@ static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double) static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid) { struct stmmac_priv *priv = netdev_priv(ndev); + unsigned int num_double_vlans; bool is_double = false; int ret; @@ -6624,7 +6602,8 @@ static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid is_double = true; set_bit(vid, priv->active_vlans); - ret = stmmac_vlan_update(priv, is_double); + num_double_vlans = priv->num_double_vlans + is_double; + ret = stmmac_vlan_update(priv, num_double_vlans); if (ret) { clear_bit(vid, priv->active_vlans); goto err_pm_put; @@ -6632,9 +6611,15 @@ static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid if (priv->hw->num_vlan) { ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); - if (ret) + if (ret) { + clear_bit(vid, priv->active_vlans); + stmmac_vlan_update(priv, priv->num_double_vlans); goto err_pm_put; + } } + + priv->num_double_vlans = num_double_vlans; + err_pm_put: pm_runtime_put(priv->device); @@ -6647,6 +6632,7 @@ static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid) { struct stmmac_priv *priv = netdev_priv(ndev); + unsigned int num_double_vlans; bool is_double = false; int ret; @@ -6658,14 +6644,23 @@ static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vi is_double = true; clear_bit(vid, priv->active_vlans); + num_double_vlans = priv->num_double_vlans - is_double; + ret = stmmac_vlan_update(priv, num_double_vlans); + if (ret) { + set_bit(vid, priv->active_vlans); + goto del_vlan_error; + } if (priv->hw->num_vlan) { ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); - if (ret) + if (ret) { + set_bit(vid, priv->active_vlans); + stmmac_vlan_update(priv, priv->num_double_vlans); goto del_vlan_error; + } } - ret = stmmac_vlan_update(priv, is_double); + priv->num_double_vlans = num_double_vlans; del_vlan_error: pm_runtime_put(priv->device); @@ -6673,6 +6668,23 @@ static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vi return ret; } +static int stmmac_vlan_restore(struct stmmac_priv *priv) +{ + int ret; + + if (!(priv->dev->features & NETIF_F_VLAN_FEATURES)) + return 0; + + if (priv->hw->num_vlan) + stmmac_restore_hw_vlan_rx_fltr(priv, priv->dev, priv->hw); + + ret = stmmac_vlan_update(priv, priv->num_double_vlans); + if (ret) + netdev_err(priv->dev, "Failed to restore VLANs\n"); + + return ret; +} + static int stmmac_bpf(struct net_device *dev, struct netdev_bpf *bpf) { struct stmmac_priv *priv = netdev_priv(dev); @@ -7253,8 +7265,9 @@ static int stmmac_hw_init(struct stmmac_priv *priv) * has to be disable and this can be done by passing the * riwt_off field from the platform. */ - if (((priv->synopsys_id >= DWMAC_CORE_3_50) || - (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) { + if ((priv->synopsys_id >= DWMAC_CORE_3_50 || + priv->plat->core_type == DWMAC_CORE_XGMAC) && + !priv->plat->riwt_off) { priv->use_riwt = 1; dev_info(priv->device, "Enable RX Mitigation via HW Watchdog Timer\n"); @@ -7368,7 +7381,7 @@ static int stmmac_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp) return -ENODATA; /* For GMAC4, the valid timestamp is from CTX next desc. */ - if (priv->plat->has_gmac4 || priv->plat->has_xgmac) + if (dwmac_is_xmac(priv->plat->core_type)) desc_contains_ts = ndesc; /* Check if timestamp is available */ @@ -7438,7 +7451,6 @@ int stmmac_dvr_probe(struct device *device, priv->dev->irq = res->irq; priv->wol_irq = res->wol_irq; - priv->lpi_irq = res->lpi_irq; priv->sfty_irq = res->sfty_irq; priv->sfty_ce_irq = res->sfty_ce_irq; priv->sfty_ue_irq = res->sfty_ue_irq; @@ -7524,7 +7536,7 @@ int stmmac_dvr_probe(struct device *device, if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; - if (priv->plat->has_gmac4) + if (priv->plat->core_type == DWMAC_CORE_GMAC4) ndev->hw_features |= NETIF_F_GSO_UDP_L4; priv->tso = true; dev_info(priv->device, "TSO feature enabled\n"); @@ -7577,7 +7589,7 @@ int stmmac_dvr_probe(struct device *device, #ifdef STMMAC_VLAN_TAG_USED /* Both mac100 and gmac support receive VLAN tag detection */ ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX; - if (priv->plat->has_gmac4 || priv->plat->has_xgmac) { + if (dwmac_is_xmac(priv->plat->core_type)) { ndev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; priv->hw->hw_vlan_en = true; } @@ -7605,7 +7617,7 @@ int stmmac_dvr_probe(struct device *device, /* MTU range: 46 - hw-specific max */ ndev->min_mtu = ETH_ZLEN - ETH_HLEN; - if (priv->plat->has_xgmac) + if (priv->plat->core_type == DWMAC_CORE_XGMAC) ndev->max_mtu = XGMAC_JUMBO_LEN; else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00)) ndev->max_mtu = JUMBO_LEN; @@ -7906,10 +7918,10 @@ int stmmac_resume(struct device *dev) stmmac_init_coalesce(priv); phylink_rx_clk_stop_block(priv->phylink); stmmac_set_rx_mode(ndev); - - stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw); phylink_rx_clk_stop_unblock(priv->phylink); + stmmac_vlan_restore(priv); + stmmac_enable_all_queues(priv); stmmac_enable_all_dma_irq(priv); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c index f408737f6fc73..2b55b02de380d 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c @@ -301,7 +301,7 @@ static int stmmac_mdio_read_c22(struct mii_bus *bus, int phyaddr, int phyreg) struct stmmac_priv *priv = netdev_priv(bus->priv); u32 cmd; - if (priv->plat->has_gmac4) + if (priv->plat->core_type == DWMAC_CORE_GMAC4) cmd = MII_GMAC4_READ; else cmd = 0; @@ -344,7 +344,7 @@ static int stmmac_mdio_write_c22(struct mii_bus *bus, int phyaddr, int phyreg, struct stmmac_priv *priv = netdev_priv(bus->priv); u32 cmd; - if (priv->plat->has_gmac4) + if (priv->plat->core_type == DWMAC_CORE_GMAC4) cmd = MII_GMAC4_WRITE; else cmd = MII_ADDR_GWRITE; @@ -417,7 +417,7 @@ int stmmac_mdio_reset(struct mii_bus *bus) * on MDC, so perform a dummy mdio read. To be updated for GMAC4 * if needed. */ - if (!priv->plat->has_gmac4) + if (priv->plat->core_type != DWMAC_CORE_GMAC4) writel(0, priv->ioaddr + mii_address); #endif return 0; @@ -528,7 +528,7 @@ static u32 stmmac_clk_csr_set(struct stmmac_priv *priv) value = 0; } - if (priv->plat->has_xgmac) { + if (priv->plat->core_type == DWMAC_CORE_XGMAC) { if (clk_rate > 400000000) value = 0x5; else if (clk_rate > 350000000) @@ -600,7 +600,7 @@ int stmmac_mdio_register(struct net_device *ndev) new_bus->name = "stmmac"; - if (priv->plat->has_xgmac) { + if (priv->plat->core_type == DWMAC_CORE_XGMAC) { new_bus->read = &stmmac_xgmac2_mdio_read_c22; new_bus->write = &stmmac_xgmac2_mdio_write_c22; new_bus->read_c45 = &stmmac_xgmac2_mdio_read_c45; @@ -621,7 +621,7 @@ int stmmac_mdio_register(struct net_device *ndev) } else { new_bus->read = &stmmac_mdio_read_c22; new_bus->write = &stmmac_mdio_write_c22; - if (priv->plat->has_gmac4) { + if (priv->plat->core_type == DWMAC_CORE_GMAC4) { new_bus->read_c45 = &stmmac_mdio_read_c45; new_bus->write_c45 = &stmmac_mdio_write_c45; } @@ -649,7 +649,7 @@ int stmmac_mdio_register(struct net_device *ndev) } /* Looks like we need a dummy read for XGMAC only and C45 PHYs */ - if (priv->plat->has_xgmac) + if (priv->plat->core_type == DWMAC_CORE_XGMAC) stmmac_xgmac2_mdio_read_c45(new_bus, 0, 0, 0); /* If fixed-link is set, skip PHY scanning */ diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c index 4e3aa611fda83..94b3a3b272706 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c @@ -23,7 +23,7 @@ static void common_default_data(struct plat_stmmacenet_data *plat) { /* clk_csr_i = 20-35MHz & MDC = clk_csr_i/16 */ plat->clk_csr = STMMAC_CSR_20_35M; - plat->has_gmac = 1; + plat->core_type = DWMAC_CORE_GMAC; plat->force_sf_dma_mode = 1; plat->mdio_bus_data->needs_reset = true; @@ -76,7 +76,7 @@ static int snps_gmac5_default_data(struct pci_dev *pdev, int i; plat->clk_csr = STMMAC_CSR_250_300M; - plat->has_gmac4 = 1; + plat->core_type = DWMAC_CORE_GMAC4; plat->force_sf_dma_mode = 1; plat->flags |= STMMAC_FLAG_TSO_EN; plat->pmt = 1; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c index 27bcaae07a7f2..0cb51935c4056 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c @@ -552,12 +552,12 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac) &pdev->dev, plat->unicast_filter_entries); plat->multicast_filter_bins = dwmac1000_validate_mcast_bins( &pdev->dev, plat->multicast_filter_bins); - plat->has_gmac = 1; + plat->core_type = DWMAC_CORE_GMAC; plat->pmt = 1; } if (of_device_is_compatible(np, "snps,dwmac-3.40a")) { - plat->has_gmac = 1; + plat->core_type = DWMAC_CORE_GMAC; plat->enh_desc = 1; plat->tx_coe = 1; plat->bugged_jumbo = 1; @@ -565,8 +565,7 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac) } if (of_device_compatible_match(np, stmmac_gmac4_compats)) { - plat->has_gmac4 = 1; - plat->has_gmac = 0; + plat->core_type = DWMAC_CORE_GMAC4; plat->pmt = 1; if (of_property_read_bool(np, "snps,tso")) plat->flags |= STMMAC_FLAG_TSO_EN; @@ -580,7 +579,7 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac) } if (of_device_is_compatible(np, "snps,dwxgmac")) { - plat->has_xgmac = 1; + plat->core_type = DWMAC_CORE_XGMAC; plat->pmt = 1; if (of_property_read_bool(np, "snps,tso")) plat->flags |= STMMAC_FLAG_TSO_EN; @@ -762,14 +761,6 @@ int stmmac_get_platform_resources(struct platform_device *pdev, stmmac_res->wol_irq = stmmac_res->irq; } - stmmac_res->lpi_irq = - platform_get_irq_byname_optional(pdev, "eth_lpi"); - if (stmmac_res->lpi_irq < 0) { - if (stmmac_res->lpi_irq == -EPROBE_DEFER) - return -EPROBE_DEFER; - dev_info(&pdev->dev, "IRQ eth_lpi not found\n"); - } - stmmac_res->sfty_irq = platform_get_irq_byname_optional(pdev, "sfty"); if (stmmac_res->sfty_irq < 0) { diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c index 993ff4e87e557..3e30172fa1294 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c @@ -57,7 +57,7 @@ static int stmmac_adjust_time(struct ptp_clock_info *ptp, s64 delta) bool xmac, est_rst = false; int ret; - xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; + xmac = dwmac_is_xmac(priv->plat->core_type); if (delta < 0) { neg_adj = 1; @@ -344,7 +344,7 @@ void stmmac_ptp_register(struct stmmac_priv *priv) /* Calculate the clock domain crossing (CDC) error if necessary */ priv->plat->cdc_error_adj = 0; - if (priv->plat->has_gmac4) + if (priv->plat->core_type == DWMAC_CORE_GMAC4) priv->plat->cdc_error_adj = (2 * NSEC_PER_SEC) / priv->plat->clk_ptp_rate; /* Update the ptp clock parameters based on feature discovery, when diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_vlan.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_vlan.c index b18404dd5a8be..e24efe3bfedbe 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_vlan.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_vlan.c @@ -76,7 +76,9 @@ static int vlan_add_hw_rx_fltr(struct net_device *dev, } hw->vlan_filter[0] = vid; - vlan_write_single(dev, vid); + + if (netif_running(dev)) + vlan_write_single(dev, vid); return 0; } @@ -97,12 +99,15 @@ static int vlan_add_hw_rx_fltr(struct net_device *dev, return -EPERM; } - ret = vlan_write_filter(dev, hw, index, val); + if (netif_running(dev)) { + ret = vlan_write_filter(dev, hw, index, val); + if (ret) + return ret; + } - if (!ret) - hw->vlan_filter[index] = val; + hw->vlan_filter[index] = val; - return ret; + return 0; } static int vlan_del_hw_rx_fltr(struct net_device *dev, @@ -115,7 +120,9 @@ static int vlan_del_hw_rx_fltr(struct net_device *dev, if (hw->num_vlan == 1) { if ((hw->vlan_filter[0] & VLAN_TAG_VID) == vid) { hw->vlan_filter[0] = 0; - vlan_write_single(dev, 0); + + if (netif_running(dev)) + vlan_write_single(dev, 0); } return 0; } @@ -124,25 +131,23 @@ static int vlan_del_hw_rx_fltr(struct net_device *dev, for (i = 0; i < hw->num_vlan; i++) { if ((hw->vlan_filter[i] & VLAN_TAG_DATA_VEN) && ((hw->vlan_filter[i] & VLAN_TAG_DATA_VID) == vid)) { - ret = vlan_write_filter(dev, hw, i, 0); - if (!ret) - hw->vlan_filter[i] = 0; - else - return ret; + if (netif_running(dev)) { + ret = vlan_write_filter(dev, hw, i, 0); + if (ret) + return ret; + } + + hw->vlan_filter[i] = 0; } } - return ret; + return 0; } static void vlan_restore_hw_rx_fltr(struct net_device *dev, struct mac_device_info *hw) { - void __iomem *ioaddr = hw->pcsr; - u32 value; - u32 hash; - u32 val; int i; /* Single Rx VLAN Filter */ @@ -152,19 +157,8 @@ static void vlan_restore_hw_rx_fltr(struct net_device *dev, } /* Extended Rx VLAN Filter Enable */ - for (i = 0; i < hw->num_vlan; i++) { - if (hw->vlan_filter[i] & VLAN_TAG_DATA_VEN) { - val = hw->vlan_filter[i]; - vlan_write_filter(dev, hw, i, val); - } - } - - hash = readl(ioaddr + VLAN_HASH_TABLE); - if (hash & VLAN_VLHT) { - value = readl(ioaddr + VLAN_TAG); - value |= VLAN_VTHM; - writel(value, ioaddr + VLAN_TAG); - } + for (i = 0; i < hw->num_vlan; i++) + vlan_write_filter(dev, hw, i, hw->vlan_filter[i]); } static void vlan_update_hash(struct mac_device_info *hw, u32 hash, @@ -183,6 +177,10 @@ static void vlan_update_hash(struct mac_device_info *hw, u32 hash, value |= VLAN_EDVLP; value |= VLAN_ESVL; value |= VLAN_DOVLTC; + } else { + value &= ~VLAN_EDVLP; + value &= ~VLAN_ESVL; + value &= ~VLAN_DOVLTC; } writel(value, ioaddr + VLAN_TAG); @@ -193,6 +191,10 @@ static void vlan_update_hash(struct mac_device_info *hw, u32 hash, value |= VLAN_EDVLP; value |= VLAN_ESVL; value |= VLAN_DOVLTC; + } else { + value &= ~VLAN_EDVLP; + value &= ~VLAN_ESVL; + value &= ~VLAN_DOVLTC; } writel(value | perfect_match, ioaddr + VLAN_TAG); diff --git a/drivers/net/ethernet/sun/sunhme.c b/drivers/net/ethernet/sun/sunhme.c index 48f0a96c0e9e3..6669980829980 100644 --- a/drivers/net/ethernet/sun/sunhme.c +++ b/drivers/net/ethernet/sun/sunhme.c @@ -2551,6 +2551,9 @@ static int happy_meal_sbus_probe_one(struct platform_device *op, int is_qfe) goto err_out_clear_quattro; } + /* BIGMAC may have bogus sizes */ + if ((op->resource[3].end - op->resource[3].start) >= BMAC_REG_SIZE) + op->resource[3].end = op->resource[3].start + BMAC_REG_SIZE - 1; hp->bigmacregs = devm_platform_ioremap_resource(op, 3); if (IS_ERR(hp->bigmacregs)) { dev_err(&op->dev, "Cannot map BIGMAC registers.\n"); diff --git a/drivers/net/ethernet/ti/Kconfig b/drivers/net/ethernet/ti/Kconfig index fe5b2926d8ab0..c60b04921c62c 100644 --- a/drivers/net/ethernet/ti/Kconfig +++ b/drivers/net/ethernet/ti/Kconfig @@ -192,6 +192,7 @@ config TI_ICSSG_PRUETH depends on NET_SWITCHDEV depends on ARCH_K3 && OF && TI_K3_UDMA_GLUE_LAYER depends on PTP_1588_CLOCK_OPTIONAL + depends on HSR || !HSR help Support dual Gigabit Ethernet ports over the ICSSG PRU Subsystem. This subsystem is available starting with the AM65 platform. diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c index 110eb2da8dbc1..77c2cf61c1fb4 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c +++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c @@ -391,7 +391,7 @@ static void am65_cpsw_nuss_ndo_slave_set_rx_mode(struct net_device *ndev) cpsw_ale_set_allmulti(common->ale, ndev->flags & IFF_ALLMULTI, port->port_id); - port_mask = ALE_PORT_HOST; + port_mask = BIT(port->port_id) | ALE_PORT_HOST; /* Clear all mcast from ALE */ cpsw_ale_flush_multicast(common->ale, port_mask, -1); diff --git a/drivers/net/ethernet/ti/cpsw_ale.c b/drivers/net/ethernet/ti/cpsw_ale.c index fbe35af615a6f..9632ad3741de1 100644 --- a/drivers/net/ethernet/ti/cpsw_ale.c +++ b/drivers/net/ethernet/ti/cpsw_ale.c @@ -455,14 +455,13 @@ static void cpsw_ale_flush_mcast(struct cpsw_ale *ale, u32 *ale_entry, ale->port_mask_bits); if ((mask & port_mask) == 0) return; /* ports dont intersect, not interested */ - mask &= ~port_mask; + mask &= (~port_mask | ALE_PORT_HOST); - /* free if only remaining port is host port */ - if (mask) + if (mask == 0x0 || mask == ALE_PORT_HOST) + cpsw_ale_set_entry_type(ale_entry, ALE_TYPE_FREE); + else cpsw_ale_set_port_mask(ale_entry, mask, ale->port_mask_bits); - else - cpsw_ale_set_entry_type(ale_entry, ALE_TYPE_FREE); } int cpsw_ale_flush_multicast(struct cpsw_ale *ale, int port_mask, int vid) diff --git a/drivers/net/ethernet/ti/cpsw_new.c b/drivers/net/ethernet/ti/cpsw_new.c index 371a099ac4e63..fd3931d667021 100644 --- a/drivers/net/ethernet/ti/cpsw_new.c +++ b/drivers/net/ethernet/ti/cpsw_new.c @@ -1472,7 +1472,7 @@ static void cpsw_unregister_ports(struct cpsw_common *cpsw) for (i = 0; i < cpsw->data.slaves; i++) { ndev = cpsw->slaves[i].ndev; - if (!ndev) + if (!ndev || ndev->reg_state != NETREG_REGISTERED) continue; priv = netdev_priv(ndev); @@ -1494,7 +1494,6 @@ static int cpsw_register_ports(struct cpsw_common *cpsw) if (ret) { dev_err(cpsw->dev, "cpsw: err registering net device%d\n", i); - cpsw->slaves[i].ndev = NULL; break; } } @@ -2001,7 +2000,7 @@ static int cpsw_probe(struct platform_device *pdev) /* setup netdevs */ ret = cpsw_create_ports(cpsw); if (ret) - goto clean_unregister_netdev; + goto clean_cpts; /* Grab RX and TX IRQs. Note that we also have RX_THRESHOLD and * MISC IRQs which are always kept disabled with this driver so @@ -2015,14 +2014,14 @@ static int cpsw_probe(struct platform_device *pdev) 0, dev_name(dev), cpsw); if (ret < 0) { dev_err(dev, "error attaching irq (%d)\n", ret); - goto clean_unregister_netdev; + goto clean_cpts; } ret = devm_request_irq(dev, cpsw->irqs_table[1], cpsw_tx_interrupt, 0, dev_name(dev), cpsw); if (ret < 0) { dev_err(dev, "error attaching irq (%d)\n", ret); - goto clean_unregister_netdev; + goto clean_cpts; } if (!cpsw->cpts) @@ -2032,7 +2031,7 @@ static int cpsw_probe(struct platform_device *pdev) 0, dev_name(&pdev->dev), cpsw); if (ret < 0) { dev_err(dev, "error attaching misc irq (%d)\n", ret); - goto clean_unregister_netdev; + goto clean_cpts; } /* Enable misc CPTS evnt_pend IRQ */ @@ -2041,7 +2040,7 @@ static int cpsw_probe(struct platform_device *pdev) skip_cpts: ret = cpsw_register_notifiers(cpsw); if (ret) - goto clean_unregister_netdev; + goto clean_cpts; ret = cpsw_register_devlink(cpsw); if (ret) @@ -2063,8 +2062,6 @@ static int cpsw_probe(struct platform_device *pdev) clean_unregister_notifiers: cpsw_unregister_notifiers(cpsw); -clean_unregister_netdev: - cpsw_unregister_ports(cpsw); clean_cpts: cpts_release(cpsw->cpts); cpdma_ctlr_destroy(cpsw->dma); diff --git a/drivers/net/ethernet/ti/icssg/icssg_prueth.c b/drivers/net/ethernet/ti/icssg/icssg_prueth.c index e42d0fdefee12..07489564270b2 100644 --- a/drivers/net/ethernet/ti/icssg/icssg_prueth.c +++ b/drivers/net/ethernet/ti/icssg/icssg_prueth.c @@ -270,6 +270,14 @@ static int prueth_emac_common_start(struct prueth *prueth) if (ret) goto disable_class; + /* Reset link state to force reconfiguration in + * emac_adjust_link(). Without this, if the link was already up + * before restart, emac_adjust_link() won't detect any state + * change and will skip critical configuration like writing + * speed to firmware. + */ + emac->link = 0; + mutex_lock(&emac->ndev->phydev->lock); emac_adjust_link(emac->ndev); mutex_unlock(&emac->ndev->phydev->lock); diff --git a/drivers/net/ethernet/xscale/ixp4xx_eth.c b/drivers/net/ethernet/xscale/ixp4xx_eth.c index e1e7f65553e76..b0faa0f1780d0 100644 --- a/drivers/net/ethernet/xscale/ixp4xx_eth.c +++ b/drivers/net/ethernet/xscale/ixp4xx_eth.c @@ -403,15 +403,12 @@ static int ixp4xx_hwtstamp_set(struct net_device *netdev, int ret; int ch; - if (!cpu_is_ixp46x()) - return -EOPNOTSUPP; - if (!netif_running(netdev)) return -EINVAL; ret = ixp46x_ptp_find(&port->timesync_regs, &port->phc_index); if (ret) - return ret; + return -EOPNOTSUPP; ch = PORT2CHANNEL(port); regs = port->timesync_regs; diff --git a/drivers/net/ethernet/xscale/ptp_ixp46x.c b/drivers/net/ethernet/xscale/ptp_ixp46x.c index 94203eb46e6b0..93c64db22a696 100644 --- a/drivers/net/ethernet/xscale/ptp_ixp46x.c +++ b/drivers/net/ethernet/xscale/ptp_ixp46x.c @@ -232,6 +232,9 @@ static struct ixp_clock ixp_clock; int ixp46x_ptp_find(struct ixp46x_ts_regs *__iomem *regs, int *phc_index) { + if (!cpu_is_ixp46x()) + return -ENODEV; + *regs = ixp_clock.regs; *phc_index = ptp_clock_index(ixp_clock.ptp_clock); diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index c509228be84d1..4433b8e95b6ac 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -1572,6 +1572,11 @@ int macvlan_common_newlink(struct net_device *dev, if (create) macvlan_port_destroy(port->dev); } + /* @dev might have been made visible before an error was detected. + * Make sure to observe an RCU grace period before our caller + * (rtnl_newlink()) frees it. + */ + synchronize_net(); return err; } EXPORT_SYMBOL_GPL(macvlan_common_newlink); diff --git a/drivers/net/mctp/mctp-i2c.c b/drivers/net/mctp/mctp-i2c.c index f782d93f826ef..8043b57bdf250 100644 --- a/drivers/net/mctp/mctp-i2c.c +++ b/drivers/net/mctp/mctp-i2c.c @@ -242,6 +242,12 @@ static int mctp_i2c_slave_cb(struct i2c_client *client, return 0; switch (event) { + case I2C_SLAVE_READ_REQUESTED: + case I2C_SLAVE_READ_PROCESSED: + /* MCTP I2C transport only uses writes */ + midev->rx_pos = 0; + *val = 0xff; + break; case I2C_SLAVE_WRITE_RECEIVED: if (midev->rx_pos < MCTP_I2C_BUFSZ) { midev->rx_buffer[midev->rx_pos] = *val; @@ -279,6 +285,9 @@ static int mctp_i2c_recv(struct mctp_i2c_dev *midev) size_t recvlen; int status; + if (midev->rx_pos == 0) + return 0; + /* + 1 for the PEC */ if (midev->rx_pos < MCTP_I2C_MINLEN + 1) { ndev->stats.rx_length_errors++; diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c index bb6e03a929565..020252961c906 100644 --- a/drivers/net/netconsole.c +++ b/drivers/net/netconsole.c @@ -1537,7 +1537,8 @@ static void send_msg_no_fragmentation(struct netconsole_target *nt, if (release_len) { release = init_utsname()->release; - scnprintf(nt->buf, MAX_PRINT_CHUNK, "%s,%s", release, msg); + scnprintf(nt->buf, MAX_PRINT_CHUNK, "%s,%.*s", release, + msg_len, msg); msg_len += release_len; } else { memcpy(nt->buf, msg, msg_len); diff --git a/drivers/net/ovpn/io.c b/drivers/net/ovpn/io.c index 3e9e7f8444b34..955c9a37e1f8d 100644 --- a/drivers/net/ovpn/io.c +++ b/drivers/net/ovpn/io.c @@ -355,6 +355,7 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev) struct ovpn_priv *ovpn = netdev_priv(dev); struct sk_buff *segments, *curr, *next; struct sk_buff_head skb_list; + unsigned int tx_bytes = 0; struct ovpn_peer *peer; __be16 proto; int ret; @@ -365,7 +366,27 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev) /* verify IP header size in network packet */ proto = ovpn_ip_check_protocol(skb); if (unlikely(!proto || skb->protocol != proto)) - goto drop; + goto drop_no_peer; + + /* retrieve peer serving the destination IP of this packet */ + peer = ovpn_peer_get_by_dst(ovpn, skb); + if (unlikely(!peer)) { + switch (skb->protocol) { + case htons(ETH_P_IP): + net_dbg_ratelimited("%s: no peer to send data to dst=%pI4\n", + netdev_name(ovpn->dev), + &ip_hdr(skb)->daddr); + break; + case htons(ETH_P_IPV6): + net_dbg_ratelimited("%s: no peer to send data to dst=%pI6c\n", + netdev_name(ovpn->dev), + &ipv6_hdr(skb)->daddr); + break; + } + goto drop_no_peer; + } + /* dst was needed for peer selection - it can now be dropped */ + skb_dst_drop(skb); if (skb_is_gso(skb)) { segments = skb_gso_segment(skb, 0); @@ -394,36 +415,28 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev) continue; } + /* only count what we actually send */ + tx_bytes += curr->len; __skb_queue_tail(&skb_list, curr); } - skb_list.prev->next = NULL; - /* retrieve peer serving the destination IP of this packet */ - peer = ovpn_peer_get_by_dst(ovpn, skb); - if (unlikely(!peer)) { - switch (skb->protocol) { - case htons(ETH_P_IP): - net_dbg_ratelimited("%s: no peer to send data to dst=%pI4\n", - netdev_name(ovpn->dev), - &ip_hdr(skb)->daddr); - break; - case htons(ETH_P_IPV6): - net_dbg_ratelimited("%s: no peer to send data to dst=%pI6c\n", - netdev_name(ovpn->dev), - &ipv6_hdr(skb)->daddr); - break; - } - goto drop; + /* no segments survived: don't jump to 'drop' because we already + * incremented the counter for each failure in the loop + */ + if (unlikely(skb_queue_empty(&skb_list))) { + ovpn_peer_put(peer); + return NETDEV_TX_OK; } - /* dst was needed for peer selection - it can now be dropped */ - skb_dst_drop(skb); + skb_list.prev->next = NULL; - ovpn_peer_stats_increment_tx(&peer->vpn_stats, skb->len); + ovpn_peer_stats_increment_tx(&peer->vpn_stats, tx_bytes); ovpn_send(ovpn, skb_list.next, peer); return NETDEV_TX_OK; drop: + ovpn_peer_put(peer); +drop_no_peer: dev_dstats_tx_dropped(ovpn->dev); skb_tx_error(skb); kfree_skb_list(skb); diff --git a/drivers/net/ovpn/socket.c b/drivers/net/ovpn/socket.c index 9750871ab65ce..448cee3b3f9fa 100644 --- a/drivers/net/ovpn/socket.c +++ b/drivers/net/ovpn/socket.c @@ -200,6 +200,22 @@ struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer) ovpn_sock->sk = sk; kref_init(&ovpn_sock->refcount); + /* TCP sockets are per-peer, therefore they are linked to their unique + * peer + */ + if (sk->sk_protocol == IPPROTO_TCP) { + INIT_WORK(&ovpn_sock->tcp_tx_work, ovpn_tcp_tx_work); + ovpn_sock->peer = peer; + ovpn_peer_hold(peer); + } else if (sk->sk_protocol == IPPROTO_UDP) { + /* in UDP we only link the ovpn instance since the socket is + * shared among multiple peers + */ + ovpn_sock->ovpn = peer->ovpn; + netdev_hold(peer->ovpn->dev, &ovpn_sock->dev_tracker, + GFP_KERNEL); + } + /* the newly created ovpn_socket is holding reference to sk, * therefore we increase its refcounter. * @@ -212,29 +228,16 @@ struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer) ret = ovpn_socket_attach(ovpn_sock, sock, peer); if (ret < 0) { + if (sk->sk_protocol == IPPROTO_TCP) + ovpn_peer_put(peer); + else if (sk->sk_protocol == IPPROTO_UDP) + netdev_put(peer->ovpn->dev, &ovpn_sock->dev_tracker); + sock_put(sk); kfree(ovpn_sock); ovpn_sock = ERR_PTR(ret); - goto sock_release; - } - - /* TCP sockets are per-peer, therefore they are linked to their unique - * peer - */ - if (sk->sk_protocol == IPPROTO_TCP) { - INIT_WORK(&ovpn_sock->tcp_tx_work, ovpn_tcp_tx_work); - ovpn_sock->peer = peer; - ovpn_peer_hold(peer); - } else if (sk->sk_protocol == IPPROTO_UDP) { - /* in UDP we only link the ovpn instance since the socket is - * shared among multiple peers - */ - ovpn_sock->ovpn = peer->ovpn; - netdev_hold(peer->ovpn->dev, &ovpn_sock->dev_tracker, - GFP_KERNEL); } - rcu_assign_sk_user_data(sk, ovpn_sock); sock_release: release_sock(sk); return ovpn_sock; diff --git a/drivers/net/ovpn/tcp.c b/drivers/net/ovpn/tcp.c index 0d7f30360d874..5499c1572f3e2 100644 --- a/drivers/net/ovpn/tcp.c +++ b/drivers/net/ovpn/tcp.c @@ -70,37 +70,56 @@ static void ovpn_tcp_to_userspace(struct ovpn_peer *peer, struct sock *sk, peer->tcp.sk_cb.sk_data_ready(sk); } -static void ovpn_tcp_rcv(struct strparser *strp, struct sk_buff *skb) +static struct sk_buff *ovpn_tcp_skb_packet(const struct ovpn_peer *peer, + struct sk_buff *orig_skb, + const int pkt_len, const int pkt_off) { - struct ovpn_peer *peer = container_of(strp, struct ovpn_peer, tcp.strp); - struct strp_msg *msg = strp_msg(skb); - size_t pkt_len = msg->full_len - 2; - size_t off = msg->offset + 2; - u8 opcode; + struct sk_buff *ovpn_skb; + int err; - /* ensure skb->data points to the beginning of the openvpn packet */ - if (!pskb_pull(skb, off)) { - net_warn_ratelimited("%s: packet too small for peer %u\n", - netdev_name(peer->ovpn->dev), peer->id); + /* create a new skb with only the content of the current packet */ + ovpn_skb = netdev_alloc_skb(peer->ovpn->dev, pkt_len); + if (unlikely(!ovpn_skb)) goto err; - } - /* strparser does not trim the skb for us, therefore we do it now */ - if (pskb_trim(skb, pkt_len) != 0) { - net_warn_ratelimited("%s: trimming skb failed for peer %u\n", + skb_copy_header(ovpn_skb, orig_skb); + err = skb_copy_bits(orig_skb, pkt_off, skb_put(ovpn_skb, pkt_len), + pkt_len); + if (unlikely(err)) { + net_warn_ratelimited("%s: skb_copy_bits failed for peer %u\n", netdev_name(peer->ovpn->dev), peer->id); + kfree_skb(ovpn_skb); goto err; } - /* we need the first 4 bytes of data to be accessible + consume_skb(orig_skb); + return ovpn_skb; +err: + kfree_skb(orig_skb); + return NULL; +} + +static void ovpn_tcp_rcv(struct strparser *strp, struct sk_buff *skb) +{ + struct ovpn_peer *peer = container_of(strp, struct ovpn_peer, tcp.strp); + struct strp_msg *msg = strp_msg(skb); + int pkt_len = msg->full_len - 2; + u8 opcode; + + /* we need at least 4 bytes of data in the packet * to extract the opcode and the key ID later on */ - if (!pskb_may_pull(skb, OVPN_OPCODE_SIZE)) { + if (unlikely(pkt_len < OVPN_OPCODE_SIZE)) { net_warn_ratelimited("%s: packet too small to fetch opcode for peer %u\n", netdev_name(peer->ovpn->dev), peer->id); goto err; } + /* extract the packet into a new skb */ + skb = ovpn_tcp_skb_packet(peer, skb, pkt_len, msg->offset + 2); + if (unlikely(!skb)) + goto err; + /* DATA_V2 packets are handled in kernel, the rest goes to user space */ opcode = ovpn_opcode_from_skb(skb, 0); if (unlikely(opcode != OVPN_DATA_V2)) { @@ -113,7 +132,7 @@ static void ovpn_tcp_rcv(struct strparser *strp, struct sk_buff *skb) /* The packet size header must be there when sending the packet * to userspace, therefore we put it back */ - skb_push(skb, 2); + *(__be16 *)__skb_push(skb, sizeof(u16)) = htons(pkt_len); ovpn_tcp_to_userspace(peer, strp->sk, skb); return; } @@ -199,7 +218,19 @@ void ovpn_tcp_socket_detach(struct ovpn_socket *ovpn_sock) sk->sk_data_ready = peer->tcp.sk_cb.sk_data_ready; sk->sk_write_space = peer->tcp.sk_cb.sk_write_space; sk->sk_prot = peer->tcp.sk_cb.prot; - sk->sk_socket->ops = peer->tcp.sk_cb.ops; + + /* tcp_close() may race this function and could set + * sk->sk_socket to NULL. It does so by invoking + * sock_orphan(), which holds sk_callback_lock before + * doing the assignment. + * + * For this reason we acquire the same lock to avoid + * sk_socket to disappear under our feet + */ + write_lock_bh(&sk->sk_callback_lock); + if (sk->sk_socket) + sk->sk_socket->ops = peer->tcp.sk_cb.ops; + write_unlock_bh(&sk->sk_callback_lock); rcu_assign_sk_user_data(sk, NULL); } @@ -487,6 +518,7 @@ int ovpn_tcp_socket_attach(struct ovpn_socket *ovpn_sock, /* make sure no pre-existing encapsulation handler exists */ if (ovpn_sock->sk->sk_user_data) return -EBUSY; + rcu_assign_sk_user_data(ovpn_sock->sk, ovpn_sock); /* only a fully connected socket is expected. Connection should be * handled in userspace @@ -495,13 +527,14 @@ int ovpn_tcp_socket_attach(struct ovpn_socket *ovpn_sock, net_err_ratelimited("%s: provided TCP socket is not in ESTABLISHED state: %d\n", netdev_name(peer->ovpn->dev), ovpn_sock->sk->sk_state); - return -EINVAL; + ret = -EINVAL; + goto err; } ret = strp_init(&peer->tcp.strp, ovpn_sock->sk, &cb); if (ret < 0) { DEBUG_NET_WARN_ON_ONCE(1); - return ret; + goto err; } INIT_WORK(&peer->tcp.defer_del_work, ovpn_tcp_peer_del_work); @@ -536,6 +569,9 @@ int ovpn_tcp_socket_attach(struct ovpn_socket *ovpn_sock, strp_check_rcv(&peer->tcp.strp); return 0; +err: + rcu_assign_sk_user_data(ovpn_sock->sk, NULL); + return ret; } static void ovpn_tcp_close(struct sock *sk, long timeout) diff --git a/drivers/net/ovpn/udp.c b/drivers/net/ovpn/udp.c index d6a0f7a0b75d7..272b535ecaad4 100644 --- a/drivers/net/ovpn/udp.c +++ b/drivers/net/ovpn/udp.c @@ -386,6 +386,7 @@ int ovpn_udp_socket_attach(struct ovpn_socket *ovpn_sock, struct socket *sock, struct ovpn_priv *ovpn) { struct udp_tunnel_sock_cfg cfg = { + .sk_user_data = ovpn_sock, .encap_type = UDP_ENCAP_OVPNINUDP, .encap_rcv = ovpn_udp_encap_recv, .encap_destroy = ovpn_udp_encap_destroy, diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 7a67c900e79a5..2353d6eced68d 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -1751,8 +1751,6 @@ int phy_attach_direct(struct net_device *dev, struct phy_device *phydev, goto error; phy_resume(phydev); - if (!phydev->is_on_sfp_module) - phy_led_triggers_register(phydev); /** * If the external phy used by current mac interface is managed by @@ -1867,9 +1865,6 @@ void phy_detach(struct phy_device *phydev) phydev->phy_link_change = NULL; phydev->phylink = NULL; - if (!phydev->is_on_sfp_module) - phy_led_triggers_unregister(phydev); - if (phydev->mdio.dev.driver) module_put(phydev->mdio.dev.driver->owner); @@ -3500,16 +3495,27 @@ static int phy_probe(struct device *dev) /* Set the state to READY by default */ phydev->state = PHY_READY; + /* Register the PHY LED triggers */ + if (!phydev->is_on_sfp_module) + phy_led_triggers_register(phydev); + /* Get the LEDs from the device tree, and instantiate standard * LEDs for them. */ - if (IS_ENABLED(CONFIG_PHYLIB_LEDS) && !phy_driver_is_genphy(phydev)) + if (IS_ENABLED(CONFIG_PHYLIB_LEDS) && !phy_driver_is_genphy(phydev)) { err = of_phy_leds(phydev); + if (err) + goto out; + } + + return 0; out: + if (!phydev->is_on_sfp_module) + phy_led_triggers_unregister(phydev); + /* Re-assert the reset signal on error */ - if (err) - phy_device_reset(phydev, 1); + phy_device_reset(phydev, 1); return err; } @@ -3523,6 +3529,9 @@ static int phy_remove(struct device *dev) if (IS_ENABLED(CONFIG_PHYLIB_LEDS) && !phy_driver_is_genphy(phydev)) phy_leds_unregister(phydev); + if (!phydev->is_on_sfp_module) + phy_led_triggers_unregister(phydev); + phydev->state = PHY_DOWN; sfp_bus_del_upstream(phydev->sfp_bus); diff --git a/drivers/net/phy/qcom/qca807x.c b/drivers/net/phy/qcom/qca807x.c index 1be8295a95cb5..511cde345e089 100644 --- a/drivers/net/phy/qcom/qca807x.c +++ b/drivers/net/phy/qcom/qca807x.c @@ -375,7 +375,7 @@ static int qca807x_gpio_get(struct gpio_chip *gc, unsigned int offset) reg = QCA807X_MMD7_LED_FORCE_CTRL(offset); val = phy_read_mmd(priv->phy, MDIO_MMD_AN, reg); - return FIELD_GET(QCA807X_GPIO_FORCE_MODE_MASK, val); + return !!FIELD_GET(QCA807X_GPIO_FORCE_MODE_MASK, val); } static int qca807x_gpio_set(struct gpio_chip *gc, unsigned int offset, int value) diff --git a/drivers/net/phy/sfp.c b/drivers/net/phy/sfp.c index 3e023723887c4..43aefdd8b70f7 100644 --- a/drivers/net/phy/sfp.c +++ b/drivers/net/phy/sfp.c @@ -532,9 +532,13 @@ static const struct sfp_quirk sfp_quirks[] = { SFP_QUIRK("HUAWEI", "MA5671A", sfp_quirk_2500basex, sfp_fixup_ignore_tx_fault), - // Lantech 8330-262D-E can operate at 2500base-X, but incorrectly report - // 2500MBd NRZ in their EEPROM + // Lantech 8330-262D-E and 8330-265D can operate at 2500base-X, but + // incorrectly report 2500MBd NRZ in their EEPROM. + // Some 8330-265D modules have inverted LOS, while all of them report + // normal LOS in EEPROM. Therefore we need to ignore LOS entirely. SFP_QUIRK_S("Lantech", "8330-262D-E", sfp_quirk_2500basex), + SFP_QUIRK("Lantech", "8330-265D", sfp_quirk_2500basex, + sfp_fixup_ignore_los), SFP_QUIRK_S("UBNT", "UF-INSTANT", sfp_quirk_ubnt_uf_instant), diff --git a/drivers/net/team/team_core.c b/drivers/net/team/team_core.c index 2fd3469d10464..6ec6708c52e2d 100644 --- a/drivers/net/team/team_core.c +++ b/drivers/net/team/team_core.c @@ -982,63 +982,6 @@ static void team_port_disable(struct team *team, team_lower_state_changed(port); } -#define TEAM_VLAN_FEATURES (NETIF_F_HW_CSUM | NETIF_F_SG | \ - NETIF_F_FRAGLIST | NETIF_F_GSO_SOFTWARE | \ - NETIF_F_HIGHDMA | NETIF_F_LRO | \ - NETIF_F_GSO_ENCAP_ALL) - -#define TEAM_ENC_FEATURES (NETIF_F_HW_CSUM | NETIF_F_SG | \ - NETIF_F_RXCSUM | NETIF_F_GSO_SOFTWARE) - -static void __team_compute_features(struct team *team) -{ - struct team_port *port; - netdev_features_t vlan_features = TEAM_VLAN_FEATURES; - netdev_features_t enc_features = TEAM_ENC_FEATURES; - unsigned short max_hard_header_len = ETH_HLEN; - unsigned int dst_release_flag = IFF_XMIT_DST_RELEASE | - IFF_XMIT_DST_RELEASE_PERM; - - rcu_read_lock(); - if (list_empty(&team->port_list)) - goto done; - - vlan_features = netdev_base_features(vlan_features); - enc_features = netdev_base_features(enc_features); - - list_for_each_entry_rcu(port, &team->port_list, list) { - vlan_features = netdev_increment_features(vlan_features, - port->dev->vlan_features, - TEAM_VLAN_FEATURES); - enc_features = - netdev_increment_features(enc_features, - port->dev->hw_enc_features, - TEAM_ENC_FEATURES); - - dst_release_flag &= port->dev->priv_flags; - if (port->dev->hard_header_len > max_hard_header_len) - max_hard_header_len = port->dev->hard_header_len; - } -done: - rcu_read_unlock(); - - team->dev->vlan_features = vlan_features; - team->dev->hw_enc_features = enc_features | NETIF_F_GSO_ENCAP_ALL | - NETIF_F_HW_VLAN_CTAG_TX | - NETIF_F_HW_VLAN_STAG_TX; - team->dev->hard_header_len = max_hard_header_len; - - team->dev->priv_flags &= ~IFF_XMIT_DST_RELEASE; - if (dst_release_flag == (IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM)) - team->dev->priv_flags |= IFF_XMIT_DST_RELEASE; -} - -static void team_compute_features(struct team *team) -{ - __team_compute_features(team); - netdev_change_features(team->dev); -} - static int team_port_enter(struct team *team, struct team_port *port) { int err = 0; @@ -1306,7 +1249,7 @@ static int team_port_add(struct team *team, struct net_device *port_dev, port->index = -1; list_add_tail_rcu(&port->list, &team->port_list); team_port_enable(team, port); - __team_compute_features(team); + netdev_compute_master_upper_features(team->dev, true); __team_port_change_port_added(port, !!netif_oper_up(port_dev)); __team_options_change_check(team); @@ -1348,7 +1291,7 @@ static int team_port_add(struct team *team, struct net_device *port_dev, static void __team_port_change_port_removed(struct team_port *port); -static int team_port_del(struct team *team, struct net_device *port_dev) +static int team_port_del(struct team *team, struct net_device *port_dev, bool unregister) { struct net_device *dev = team->dev; struct team_port *port; @@ -1386,10 +1329,16 @@ static int team_port_del(struct team *team, struct net_device *port_dev) __team_port_change_port_removed(port); team_port_set_orig_dev_addr(port); - dev_set_mtu(port_dev, port->orig.mtu); + if (unregister) { + netdev_lock_ops(port_dev); + __netif_set_mtu(port_dev, port->orig.mtu); + netdev_unlock_ops(port_dev); + } else { + dev_set_mtu(port_dev, port->orig.mtu); + } kfree_rcu(port, rcu); netdev_info(dev, "Port device %s removed\n", portname); - __team_compute_features(team); + netdev_compute_master_upper_features(team->dev, true); return 0; } @@ -1690,7 +1639,7 @@ static void team_uninit(struct net_device *dev) ASSERT_RTNL(); list_for_each_entry_safe(port, tmp, &team->port_list, list) - team_port_del(team, port->dev); + team_port_del(team, port->dev, false); __team_change_mode(team, NULL); /* cleanup */ __team_options_unregister(team, team_options, ARRAY_SIZE(team_options)); @@ -1977,33 +1926,28 @@ static int team_add_slave(struct net_device *dev, struct net_device *port_dev, struct netlink_ext_ack *extack) { struct team *team = netdev_priv(dev); - int err; ASSERT_RTNL(); - err = team_port_add(team, port_dev, extack); - - if (!err) - netdev_change_features(dev); - - return err; + return team_port_add(team, port_dev, extack); } static int team_del_slave(struct net_device *dev, struct net_device *port_dev) { struct team *team = netdev_priv(dev); - int err; ASSERT_RTNL(); - err = team_port_del(team, port_dev); + return team_port_del(team, port_dev, false); +} - if (err) - return err; +static int team_del_slave_on_unregister(struct net_device *dev, struct net_device *port_dev) +{ + struct team *team = netdev_priv(dev); - netdev_change_features(dev); + ASSERT_RTNL(); - return err; + return team_port_del(team, port_dev, true); } static netdev_features_t team_fix_features(struct net_device *dev, @@ -2197,7 +2141,7 @@ static void team_setup(struct net_device *dev) dev->features |= NETIF_F_GRO; - dev->hw_features = TEAM_VLAN_FEATURES | + dev->hw_features = MASTER_UPPER_DEV_VLAN_FEATURES | NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_STAG_RX | @@ -2996,12 +2940,12 @@ static int team_device_event(struct notifier_block *unused, !!netif_oper_up(port->dev)); break; case NETDEV_UNREGISTER: - team_del_slave(port->team->dev, dev); + team_del_slave_on_unregister(port->team->dev, dev); break; case NETDEV_FEAT_CHANGE: if (!port->team->notifier_ctx) { port->team->notifier_ctx = true; - team_compute_features(port->team); + netdev_compute_master_upper_features(port->team->dev, true); port->team->notifier_ctx = false; } break; @@ -3069,3 +3013,4 @@ MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Jiri Pirko "); MODULE_DESCRIPTION("Ethernet team device driver"); MODULE_ALIAS_RTNL_LINK(DRV_NAME); +MODULE_IMPORT_NS("NETDEV_INTERNAL"); diff --git a/drivers/net/usb/Kconfig b/drivers/net/usb/Kconfig index 856e648d804e0..da0f6a138f4fc 100644 --- a/drivers/net/usb/Kconfig +++ b/drivers/net/usb/Kconfig @@ -319,7 +319,6 @@ config USB_NET_DM9601 config USB_NET_SR9700 tristate "CoreChip-sz SR9700 based USB 1.1 10/100 ethernet devices" depends on USB_USBNET - select CRC32 help This option adds support for CoreChip-sz SR9700 based USB 1.1 10/100 Ethernet adapters. diff --git a/drivers/net/usb/catc.c b/drivers/net/usb/catc.c index 6759388692f8e..3c824340ffb06 100644 --- a/drivers/net/usb/catc.c +++ b/drivers/net/usb/catc.c @@ -64,6 +64,16 @@ static const char driver_name[] = "catc"; #define CTRL_QUEUE 16 /* Max control requests in flight (power of two) */ #define RX_PKT_SZ 1600 /* Max size of receive packet for F5U011 */ +/* + * USB endpoints. + */ + +enum catc_usb_ep { + CATC_USB_EP_CONTROL = 0, + CATC_USB_EP_BULK = 1, + CATC_USB_EP_INT_IN = 2, +}; + /* * Control requests. */ @@ -772,6 +782,13 @@ static int catc_probe(struct usb_interface *intf, const struct usb_device_id *id u8 broadcast[ETH_ALEN]; u8 *macbuf; int pktsz, ret = -ENOMEM; + static const u8 bulk_ep_addr[] = { + CATC_USB_EP_BULK | USB_DIR_OUT, + CATC_USB_EP_BULK | USB_DIR_IN, + 0}; + static const u8 int_ep_addr[] = { + CATC_USB_EP_INT_IN | USB_DIR_IN, + 0}; macbuf = kmalloc(ETH_ALEN, GFP_KERNEL); if (!macbuf) @@ -784,6 +801,14 @@ static int catc_probe(struct usb_interface *intf, const struct usb_device_id *id goto fail_mem; } + /* Verify that all required endpoints are present */ + if (!usb_check_bulk_endpoints(intf, bulk_ep_addr) || + !usb_check_int_endpoints(intf, int_ep_addr)) { + dev_err(dev, "Missing or invalid endpoints\n"); + ret = -ENODEV; + goto fail_mem; + } + netdev = alloc_etherdev(sizeof(struct catc)); if (!netdev) goto fail_mem; @@ -828,14 +853,14 @@ static int catc_probe(struct usb_interface *intf, const struct usb_device_id *id usb_fill_control_urb(catc->ctrl_urb, usbdev, usb_sndctrlpipe(usbdev, 0), NULL, NULL, 0, catc_ctrl_done, catc); - usb_fill_bulk_urb(catc->tx_urb, usbdev, usb_sndbulkpipe(usbdev, 1), - NULL, 0, catc_tx_done, catc); + usb_fill_bulk_urb(catc->tx_urb, usbdev, usb_sndbulkpipe(usbdev, CATC_USB_EP_BULK), + NULL, 0, catc_tx_done, catc); - usb_fill_bulk_urb(catc->rx_urb, usbdev, usb_rcvbulkpipe(usbdev, 1), - catc->rx_buf, pktsz, catc_rx_done, catc); + usb_fill_bulk_urb(catc->rx_urb, usbdev, usb_rcvbulkpipe(usbdev, CATC_USB_EP_BULK), + catc->rx_buf, pktsz, catc_rx_done, catc); - usb_fill_int_urb(catc->irq_urb, usbdev, usb_rcvintpipe(usbdev, 2), - catc->irq_buf, 2, catc_irq_done, catc, 1); + usb_fill_int_urb(catc->irq_urb, usbdev, usb_rcvintpipe(usbdev, CATC_USB_EP_INT_IN), + catc->irq_buf, 2, catc_irq_done, catc, 1); if (!catc->is_f5u011) { u32 *buf; diff --git a/drivers/net/usb/kalmia.c b/drivers/net/usb/kalmia.c index 613fc6910f148..ee9c48f7f68f9 100644 --- a/drivers/net/usb/kalmia.c +++ b/drivers/net/usb/kalmia.c @@ -132,11 +132,18 @@ kalmia_bind(struct usbnet *dev, struct usb_interface *intf) { int status; u8 ethernet_addr[ETH_ALEN]; + static const u8 ep_addr[] = { + 1 | USB_DIR_IN, + 2 | USB_DIR_OUT, + 0}; /* Don't bind to AT command interface */ if (intf->cur_altsetting->desc.bInterfaceClass != USB_CLASS_VENDOR_SPEC) return -EINVAL; + if (!usb_check_bulk_endpoints(intf, ep_addr)) + return -ENODEV; + dev->in = usb_rcvbulkpipe(dev->udev, 0x81 & USB_ENDPOINT_NUMBER_MASK); dev->out = usb_sndbulkpipe(dev->udev, 0x02 & USB_ENDPOINT_NUMBER_MASK); dev->status = NULL; diff --git a/drivers/net/usb/kaweth.c b/drivers/net/usb/kaweth.c index c9efb7df892ec..cb2472b59e104 100644 --- a/drivers/net/usb/kaweth.c +++ b/drivers/net/usb/kaweth.c @@ -765,7 +765,6 @@ static void kaweth_set_rx_mode(struct net_device *net) netdev_dbg(net, "Setting Rx mode to %d\n", packet_filter_bitmap); - netif_stop_queue(net); if (net->flags & IFF_PROMISC) { packet_filter_bitmap |= KAWETH_PACKET_FILTER_PROMISCUOUS; @@ -775,7 +774,6 @@ static void kaweth_set_rx_mode(struct net_device *net) } kaweth->packet_filter_bitmap = packet_filter_bitmap; - netif_wake_queue(net); } /**************************************************************** @@ -885,6 +883,13 @@ static int kaweth_probe( const eth_addr_t bcast_addr = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; int result = 0; int rv = -EIO; + static const u8 bulk_ep_addr[] = { + 1 | USB_DIR_IN, + 2 | USB_DIR_OUT, + 0}; + static const u8 int_ep_addr[] = { + 3 | USB_DIR_IN, + 0}; dev_dbg(dev, "Kawasaki Device Probe (Device number:%d): 0x%4.4x:0x%4.4x:0x%4.4x\n", @@ -898,6 +903,12 @@ static int kaweth_probe( (int)udev->descriptor.bLength, (int)udev->descriptor.bDescriptorType); + if (!usb_check_bulk_endpoints(intf, bulk_ep_addr) || + !usb_check_int_endpoints(intf, int_ep_addr)) { + dev_err(dev, "couldn't find required endpoints\n"); + return -ENODEV; + } + netdev = alloc_etherdev(sizeof(*kaweth)); if (!netdev) return -ENOMEM; diff --git a/drivers/net/usb/lan78xx.c b/drivers/net/usb/lan78xx.c index 00397a8073934..065588c9cfa65 100644 --- a/drivers/net/usb/lan78xx.c +++ b/drivers/net/usb/lan78xx.c @@ -2094,8 +2094,6 @@ static int lan78xx_mdio_init(struct lan78xx_net *dev) dev->mdiobus->phy_mask = ~(1 << 1); break; case ID_REV_CHIP_ID_7801_: - /* scan thru PHYAD[2..0] */ - dev->mdiobus->phy_mask = ~(0xFF); break; } diff --git a/drivers/net/usb/pegasus.c b/drivers/net/usb/pegasus.c index c514483134f05..475b066081c7f 100644 --- a/drivers/net/usb/pegasus.c +++ b/drivers/net/usb/pegasus.c @@ -31,6 +31,17 @@ static const char driver_name[] = "pegasus"; BMSR_100FULL | BMSR_ANEGCAPABLE) #define CARRIER_CHECK_DELAY (2 * HZ) +/* + * USB endpoints. + */ + +enum pegasus_usb_ep { + PEGASUS_USB_EP_CONTROL = 0, + PEGASUS_USB_EP_BULK_IN = 1, + PEGASUS_USB_EP_BULK_OUT = 2, + PEGASUS_USB_EP_INT_IN = 3, +}; + static bool loopback; static bool mii_mode; static char *devid; @@ -545,7 +556,7 @@ static void read_bulk_callback(struct urb *urb) goto tl_sched; goon: usb_fill_bulk_urb(pegasus->rx_urb, pegasus->usb, - usb_rcvbulkpipe(pegasus->usb, 1), + usb_rcvbulkpipe(pegasus->usb, PEGASUS_USB_EP_BULK_IN), pegasus->rx_skb->data, PEGASUS_MTU, read_bulk_callback, pegasus); rx_status = usb_submit_urb(pegasus->rx_urb, GFP_ATOMIC); @@ -585,7 +596,7 @@ static void rx_fixup(struct tasklet_struct *t) return; } usb_fill_bulk_urb(pegasus->rx_urb, pegasus->usb, - usb_rcvbulkpipe(pegasus->usb, 1), + usb_rcvbulkpipe(pegasus->usb, PEGASUS_USB_EP_BULK_IN), pegasus->rx_skb->data, PEGASUS_MTU, read_bulk_callback, pegasus); try_again: @@ -713,7 +724,7 @@ static netdev_tx_t pegasus_start_xmit(struct sk_buff *skb, ((__le16 *) pegasus->tx_buff)[0] = cpu_to_le16(l16); skb_copy_from_linear_data(skb, pegasus->tx_buff + 2, skb->len); usb_fill_bulk_urb(pegasus->tx_urb, pegasus->usb, - usb_sndbulkpipe(pegasus->usb, 2), + usb_sndbulkpipe(pegasus->usb, PEGASUS_USB_EP_BULK_OUT), pegasus->tx_buff, count, write_bulk_callback, pegasus); if ((res = usb_submit_urb(pegasus->tx_urb, GFP_ATOMIC))) { @@ -804,8 +815,19 @@ static void unlink_all_urbs(pegasus_t *pegasus) static int alloc_urbs(pegasus_t *pegasus) { + static const u8 bulk_ep_addr[] = { + 1 | USB_DIR_IN, + 2 | USB_DIR_OUT, + 0}; + static const u8 int_ep_addr[] = { + 3 | USB_DIR_IN, + 0}; int res = -ENOMEM; + if (!usb_check_bulk_endpoints(pegasus->intf, bulk_ep_addr) || + !usb_check_int_endpoints(pegasus->intf, int_ep_addr)) + return -ENODEV; + pegasus->rx_urb = usb_alloc_urb(0, GFP_KERNEL); if (!pegasus->rx_urb) { return res; @@ -840,7 +862,7 @@ static int pegasus_open(struct net_device *net) set_registers(pegasus, EthID, 6, net->dev_addr); usb_fill_bulk_urb(pegasus->rx_urb, pegasus->usb, - usb_rcvbulkpipe(pegasus->usb, 1), + usb_rcvbulkpipe(pegasus->usb, PEGASUS_USB_EP_BULK_IN), pegasus->rx_skb->data, PEGASUS_MTU, read_bulk_callback, pegasus); if ((res = usb_submit_urb(pegasus->rx_urb, GFP_KERNEL))) { @@ -851,7 +873,7 @@ static int pegasus_open(struct net_device *net) } usb_fill_int_urb(pegasus->intr_urb, pegasus->usb, - usb_rcvintpipe(pegasus->usb, 3), + usb_rcvintpipe(pegasus->usb, PEGASUS_USB_EP_INT_IN), pegasus->intr_buff, sizeof(pegasus->intr_buff), intr_callback, pegasus, pegasus->intr_interval); if ((res = usb_submit_urb(pegasus->intr_urb, GFP_KERNEL))) { @@ -1136,16 +1158,31 @@ static int pegasus_probe(struct usb_interface *intf, pegasus_t *pegasus; int dev_index = id - pegasus_ids; int res = -ENOMEM; + static const u8 bulk_ep_addr[] = { + PEGASUS_USB_EP_BULK_IN | USB_DIR_IN, + PEGASUS_USB_EP_BULK_OUT | USB_DIR_OUT, + 0}; + static const u8 int_ep_addr[] = { + PEGASUS_USB_EP_INT_IN | USB_DIR_IN, + 0}; if (pegasus_blacklisted(dev)) return -ENODEV; + /* Verify that all required endpoints are present */ + if (!usb_check_bulk_endpoints(intf, bulk_ep_addr) || + !usb_check_int_endpoints(intf, int_ep_addr)) { + dev_err(&intf->dev, "Missing or invalid endpoints\n"); + return -ENODEV; + } + net = alloc_etherdev(sizeof(struct pegasus)); if (!net) goto out; pegasus = netdev_priv(net); pegasus->dev_index = dev_index; + pegasus->intf = intf; res = alloc_urbs(pegasus); if (res < 0) { @@ -1157,7 +1194,6 @@ static int pegasus_probe(struct usb_interface *intf, INIT_DELAYED_WORK(&pegasus->carrier_check, check_carrier); - pegasus->intf = intf; pegasus->usb = dev; pegasus->net = net; diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c index 6a43054d5171f..da8de7b1a4891 100644 --- a/drivers/net/usb/r8152.c +++ b/drivers/net/usb/r8152.c @@ -2449,6 +2449,8 @@ static int r8152_tx_agg_fill(struct r8152 *tp, struct tx_agg *agg) ret = usb_submit_urb(agg->urb, GFP_ATOMIC); if (ret < 0) usb_autopm_put_interface_async(tp->intf); + else + netif_trans_update(tp->netdev); out_tx_fill: return ret; diff --git a/drivers/net/usb/sr9700.c b/drivers/net/usb/sr9700.c index 820c4c5069792..a5d364fbc3639 100644 --- a/drivers/net/usb/sr9700.c +++ b/drivers/net/usb/sr9700.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include "sr9700.h" @@ -265,31 +264,15 @@ static const struct ethtool_ops sr9700_ethtool_ops = { static void sr9700_set_multicast(struct net_device *netdev) { struct usbnet *dev = netdev_priv(netdev); - /* We use the 20 byte dev->data for our 8 byte filter buffer - * to avoid allocating memory that is tricky to free later - */ - u8 *hashes = (u8 *)&dev->data; /* rx_ctl setting : enable, disable_long, disable_crc */ u8 rx_ctl = RCR_RXEN | RCR_DIS_CRC | RCR_DIS_LONG; - memset(hashes, 0x00, SR_MCAST_SIZE); - /* broadcast address */ - hashes[SR_MCAST_SIZE - 1] |= SR_MCAST_ADDR_FLAG; - if (netdev->flags & IFF_PROMISC) { + if (netdev->flags & IFF_PROMISC) rx_ctl |= RCR_PRMSC; - } else if (netdev->flags & IFF_ALLMULTI || - netdev_mc_count(netdev) > SR_MCAST_MAX) { - rx_ctl |= RCR_RUNT; - } else if (!netdev_mc_empty(netdev)) { - struct netdev_hw_addr *ha; - - netdev_for_each_mc_addr(ha, netdev) { - u32 crc = ether_crc(ETH_ALEN, ha->addr) >> 26; - hashes[crc >> 3] |= 1 << (crc & 0x7); - } - } + else if (netdev->flags & IFF_ALLMULTI || !netdev_mc_empty(netdev)) + /* The chip has no multicast filter */ + rx_ctl |= RCR_ALL; - sr_write_async(dev, SR_MAR, SR_MCAST_SIZE, hashes); sr_write_reg_async(dev, SR_RCR, rx_ctl); } diff --git a/drivers/net/usb/sr9700.h b/drivers/net/usb/sr9700.h index ea2b4de621c86..c479908f7d823 100644 --- a/drivers/net/usb/sr9700.h +++ b/drivers/net/usb/sr9700.h @@ -104,9 +104,7 @@ #define WCR_LINKEN (1 << 5) /* Physical Address Reg */ #define SR_PAR 0x10 /* 0x10 ~ 0x15 6 bytes for PAR */ -/* Multicast Address Reg */ -#define SR_MAR 0x16 /* 0x16 ~ 0x1D 8 bytes for MAR */ -/* 0x1e unused */ +/* 0x16 --> 0x1E unused */ /* Phy Reset Reg */ #define SR_PRR 0x1F #define PRR_PHY_RST (1 << 0) @@ -161,9 +159,6 @@ /* parameters */ #define SR_SHARE_TIMEOUT 1000 #define SR_EEPROM_LEN 256 -#define SR_MCAST_SIZE 8 -#define SR_MCAST_ADDR_FLAG 0x80 -#define SR_MCAST_MAX 64 #define SR_TX_OVERHEAD 2 /* 2bytes header */ #define SR_RX_OVERHEAD 7 /* 3bytes header + 4crc tail */ diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c index e957aa12a8a44..2a140be86bafc 100644 --- a/drivers/net/vxlan/vxlan_core.c +++ b/drivers/net/vxlan/vxlan_core.c @@ -2130,6 +2130,11 @@ static bool route_shortcircuit(struct net_device *dev, struct sk_buff *skb) { struct ipv6hdr *pip6; + /* check if nd_tbl is not initiliazed due to + * ipv6.disable=1 set during boot + */ + if (!ipv6_stub->nd_tbl) + return false; if (!pskb_may_pull(skb, sizeof(struct ipv6hdr))) return false; pip6 = ipv6_hdr(skb); diff --git a/drivers/net/wan/farsync.c b/drivers/net/wan/farsync.c index 5b01642ca44e0..6b2d1e63855e8 100644 --- a/drivers/net/wan/farsync.c +++ b/drivers/net/wan/farsync.c @@ -2550,6 +2550,8 @@ fst_remove_one(struct pci_dev *pdev) fst_disable_intr(card); free_irq(card->irq, card); + tasklet_kill(&fst_tx_task); + tasklet_kill(&fst_int_task); iounmap(card->ctlmem); iounmap(card->mem); diff --git a/drivers/net/wan/fsl_ucc_hdlc.c b/drivers/net/wan/fsl_ucc_hdlc.c index f999798a56127..dff84731343cc 100644 --- a/drivers/net/wan/fsl_ucc_hdlc.c +++ b/drivers/net/wan/fsl_ucc_hdlc.c @@ -790,18 +790,14 @@ static void uhdlc_memclean(struct ucc_hdlc_private *priv) if (priv->rx_buffer) { dma_free_coherent(priv->dev, - RX_BD_RING_LEN * MAX_RX_BUF_LENGTH, + (RX_BD_RING_LEN + TX_BD_RING_LEN) * MAX_RX_BUF_LENGTH, priv->rx_buffer, priv->dma_rx_addr); priv->rx_buffer = NULL; priv->dma_rx_addr = 0; - } - if (priv->tx_buffer) { - dma_free_coherent(priv->dev, - TX_BD_RING_LEN * MAX_RX_BUF_LENGTH, - priv->tx_buffer, priv->dma_tx_addr); priv->tx_buffer = NULL; priv->dma_tx_addr = 0; + } } diff --git a/drivers/net/wireless/ath/ath10k/sdio.c b/drivers/net/wireless/ath/ath10k/sdio.c index c06d50db40b81..00d0556dafefd 100644 --- a/drivers/net/wireless/ath/ath10k/sdio.c +++ b/drivers/net/wireless/ath/ath10k/sdio.c @@ -2487,7 +2487,11 @@ void ath10k_sdio_fw_crashed_dump(struct ath10k *ar) if (fast_dump) ath10k_bmi_start(ar); + mutex_lock(&ar->dump_mutex); + + spin_lock_bh(&ar->data_lock); ar->stats.fw_crash_counter++; + spin_unlock_bh(&ar->data_lock); ath10k_sdio_disable_intrs(ar); @@ -2505,6 +2509,8 @@ void ath10k_sdio_fw_crashed_dump(struct ath10k *ar) ath10k_sdio_enable_intrs(ar); + mutex_unlock(&ar->dump_mutex); + ath10k_core_start_recovery(ar); } diff --git a/drivers/net/wireless/ath/ath10k/wmi.c b/drivers/net/wireless/ath/ath10k/wmi.c index b4aad6604d6d9..ce22141e5efd9 100644 --- a/drivers/net/wireless/ath/ath10k/wmi.c +++ b/drivers/net/wireless/ath/ath10k/wmi.c @@ -5289,8 +5289,6 @@ ath10k_wmi_event_peer_sta_ps_state_chg(struct ath10k *ar, struct sk_buff *skb) struct ath10k_sta *arsta; u8 peer_addr[ETH_ALEN]; - lockdep_assert_held(&ar->data_lock); - ev = (struct wmi_peer_sta_ps_state_chg_event *)skb->data; ether_addr_copy(peer_addr, ev->peer_macaddr.addr); @@ -5305,7 +5303,9 @@ ath10k_wmi_event_peer_sta_ps_state_chg(struct ath10k *ar, struct sk_buff *skb) } arsta = (struct ath10k_sta *)sta->drv_priv; + spin_lock_bh(&ar->data_lock); arsta->peer_ps_state = __le32_to_cpu(ev->peer_ps_state); + spin_unlock_bh(&ar->data_lock); exit: rcu_read_unlock(); diff --git a/drivers/net/wireless/ath/ath11k/core.c b/drivers/net/wireless/ath/ath11k/core.c index a4a3a65c77526..712d0a434e5b7 100644 --- a/drivers/net/wireless/ath/ath11k/core.c +++ b/drivers/net/wireless/ath/ath11k/core.c @@ -993,6 +993,34 @@ static const struct dmi_system_id ath11k_pm_quirk_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "21F9"), }, }, + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* Z13 G1 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21D2"), + }, + }, + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* Z13 G1 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21D3"), + }, + }, + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* Z16 G1 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21D4"), + }, + }, + { + .driver_data = (void *)ATH11K_PM_WOW, + .matches = { /* Z16 G1 */ + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21D5"), + }, + }, {} }; diff --git a/drivers/net/wireless/ath/ath11k/core.h b/drivers/net/wireless/ath/ath11k/core.h index f8fcd897ebd26..fac4e716c67d7 100644 --- a/drivers/net/wireless/ath/ath11k/core.h +++ b/drivers/net/wireless/ath/ath11k/core.h @@ -1328,6 +1328,9 @@ static inline void ath11k_core_create_firmware_path(struct ath11k_base *ab, { const char *fw_name = ath11k_core_get_usecase_firmware(ab); + if (!fw_name) + fw_name = ath11k_core_get_usecase_firmware(ab); + if (fw_name && strncmp(filename, "board", 5)) snprintf(buf, buf_len, "%s/%s/%s/%s", ATH11K_FW_DIR, ab->hw_params.fw.dir, fw_name, filename); diff --git a/drivers/net/wireless/ath/ath11k/mhi.c b/drivers/net/wireless/ath/ath11k/mhi.c index acd76e9392d31..d2c44f7f9b622 100644 --- a/drivers/net/wireless/ath/ath11k/mhi.c +++ b/drivers/net/wireless/ath/ath11k/mhi.c @@ -34,7 +34,6 @@ static const struct mhi_channel_config ath11k_mhi_channels_qca6390[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, }, { .num = 21, @@ -48,7 +47,6 @@ static const struct mhi_channel_config ath11k_mhi_channels_qca6390[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = true, }, }; @@ -99,7 +97,6 @@ static const struct mhi_channel_config ath11k_mhi_channels_qcn9074[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = false, }, { .num = 21, @@ -113,7 +110,6 @@ static const struct mhi_channel_config ath11k_mhi_channels_qcn9074[] = { .lpm_notify = false, .offload_channel = false, .doorbell_mode_switch = false, - .auto_queue = true, }, }; diff --git a/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.c b/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.c index 48b010a1b7566..4f749d473d0e1 100644 --- a/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.c +++ b/drivers/net/wireless/ath/ath12k/debugfs_htt_stats.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -29,8 +29,10 @@ print_array_to_buf_index(u8 *buf, u32 offset, const char *header, u32 stats_inde " %u:%u,", stats_index++, le32_to_cpu(array[i])); } /* To overwrite the last trailing comma */ - index--; - *(buf + offset + index) = '\0'; + if (array_len > 0) { + index--; + *(buf + offset + index) = '\0'; + } if (footer) { index += scnprintf(buf + offset + index, diff --git a/drivers/net/wireless/ath/ath12k/mac.c b/drivers/net/wireless/ath/ath12k/mac.c index 5cd10752b22e1..b6778537f7dcc 100644 --- a/drivers/net/wireless/ath/ath12k/mac.c +++ b/drivers/net/wireless/ath/ath12k/mac.c @@ -4281,8 +4281,10 @@ ath12k_mac_op_change_vif_links(struct ieee80211_hw *hw, if (WARN_ON(!arvif)) return -EINVAL; - if (!arvif->is_created) + if (!arvif->is_created) { + ath12k_mac_unassign_link_vif(arvif); continue; + } if (WARN_ON(!arvif->ar)) return -EINVAL; diff --git a/drivers/net/wireless/ath/ath12k/mhi.c b/drivers/net/wireless/ath/ath12k/mhi.c index 45c0f66dcc5ea..5d1675ce82868 100644 --- a/drivers/net/wireless/ath/ath12k/mhi.c +++ b/drivers/net/wireless/ath/ath12k/mhi.c @@ -18,6 +18,132 @@ #define OTP_VALID_DUALMAC_BOARD_ID_MASK 0x1000 #define MHI_CB_INVALID 0xff +static const struct mhi_channel_config ath12k_mhi_channels_qcn9274[] = { + { + .num = 20, + .name = "IPCR", + .num_elements = 32, + .event_ring = 1, + .dir = DMA_TO_DEVICE, + .ee_mask = 0x4, + .pollcfg = 0, + .doorbell = MHI_DB_BRST_DISABLE, + .lpm_notify = false, + .offload_channel = false, + .doorbell_mode_switch = false, + }, + { + .num = 21, + .name = "IPCR", + .num_elements = 32, + .event_ring = 1, + .dir = DMA_FROM_DEVICE, + .ee_mask = 0x4, + .pollcfg = 0, + .doorbell = MHI_DB_BRST_DISABLE, + .lpm_notify = false, + .offload_channel = false, + .doorbell_mode_switch = false, + }, +}; + +static struct mhi_event_config ath12k_mhi_events_qcn9274[] = { + { + .num_elements = 32, + .irq_moderation_ms = 0, + .irq = 1, + .data_type = MHI_ER_CTRL, + .mode = MHI_DB_BRST_DISABLE, + .hardware_event = false, + .client_managed = false, + .offload_channel = false, + }, + { + .num_elements = 256, + .irq_moderation_ms = 1, + .irq = 2, + .mode = MHI_DB_BRST_DISABLE, + .priority = 1, + .hardware_event = false, + .client_managed = false, + .offload_channel = false, + }, +}; + +const struct mhi_controller_config ath12k_mhi_config_qcn9274 = { + .max_channels = 30, + .timeout_ms = 10000, + .use_bounce_buf = false, + .buf_len = 0, + .num_channels = ARRAY_SIZE(ath12k_mhi_channels_qcn9274), + .ch_cfg = ath12k_mhi_channels_qcn9274, + .num_events = ARRAY_SIZE(ath12k_mhi_events_qcn9274), + .event_cfg = ath12k_mhi_events_qcn9274, +}; + +static const struct mhi_channel_config ath12k_mhi_channels_wcn7850[] = { + { + .num = 20, + .name = "IPCR", + .num_elements = 64, + .event_ring = 1, + .dir = DMA_TO_DEVICE, + .ee_mask = 0x4, + .pollcfg = 0, + .doorbell = MHI_DB_BRST_DISABLE, + .lpm_notify = false, + .offload_channel = false, + .doorbell_mode_switch = false, + }, + { + .num = 21, + .name = "IPCR", + .num_elements = 64, + .event_ring = 1, + .dir = DMA_FROM_DEVICE, + .ee_mask = 0x4, + .pollcfg = 0, + .doorbell = MHI_DB_BRST_DISABLE, + .lpm_notify = false, + .offload_channel = false, + .doorbell_mode_switch = false, + }, +}; + +static struct mhi_event_config ath12k_mhi_events_wcn7850[] = { + { + .num_elements = 32, + .irq_moderation_ms = 0, + .irq = 1, + .mode = MHI_DB_BRST_DISABLE, + .data_type = MHI_ER_CTRL, + .hardware_event = false, + .client_managed = false, + .offload_channel = false, + }, + { + .num_elements = 256, + .irq_moderation_ms = 1, + .irq = 2, + .mode = MHI_DB_BRST_DISABLE, + .priority = 1, + .hardware_event = false, + .client_managed = false, + .offload_channel = false, + }, +}; + +const struct mhi_controller_config ath12k_mhi_config_wcn7850 = { + .max_channels = 128, + .timeout_ms = 2000, + .use_bounce_buf = false, + .buf_len = 8192, + .num_channels = ARRAY_SIZE(ath12k_mhi_channels_wcn7850), + .ch_cfg = ath12k_mhi_channels_wcn7850, + .num_events = ARRAY_SIZE(ath12k_mhi_events_wcn7850), + .event_cfg = ath12k_mhi_events_wcn7850, +}; + void ath12k_mhi_set_mhictrl_reset(struct ath12k_base *ab) { u32 val; diff --git a/drivers/net/wireless/ath/ath9k/Kconfig b/drivers/net/wireless/ath/ath9k/Kconfig index 0c47be06c153b..47d570a5ca6a1 100644 --- a/drivers/net/wireless/ath/ath9k/Kconfig +++ b/drivers/net/wireless/ath/ath9k/Kconfig @@ -47,7 +47,7 @@ config ATH9K_PCI config ATH9K_AHB bool "Atheros ath9k AHB bus support" - depends on ATH9K + depends on ATH9K && OF default n help This option enables the AHB bus support in ath9k. diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index 6a3f187320fc4..13952dfeb3e30 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -951,11 +951,10 @@ int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) goto out; /* try to attach to the target device */ - sdiodev->bus = brcmf_sdio_probe(sdiodev); - if (IS_ERR(sdiodev->bus)) { - ret = PTR_ERR(sdiodev->bus); + ret = brcmf_sdio_probe(sdiodev); + if (ret) goto out; - } + brcmf_sdiod_host_fixup(sdiodev->func2->card->host); out: if (ret) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 8cf9d7e7c3f70..4e6ed02c15913 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -4445,7 +4445,7 @@ brcmf_sdio_prepare_fw_request(struct brcmf_sdio *bus) return fwreq; } -struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) +int brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) { int ret; struct brcmf_sdio *bus; @@ -4551,11 +4551,12 @@ struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) goto fail; } - return bus; + return 0; fail: brcmf_sdio_remove(bus); - return ERR_PTR(ret); + sdiodev->bus = NULL; + return ret; } /* Detach and free everything */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h index 0d18ed15b4032..80180d5c6c879 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h @@ -358,7 +358,7 @@ void brcmf_sdiod_freezer_uncount(struct brcmf_sdio_dev *sdiodev); int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev); int brcmf_sdiod_remove(struct brcmf_sdio_dev *sdiodev); -struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev); +int brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev); void brcmf_sdio_remove(struct brcmf_sdio *bus); void brcmf_sdio_isr(struct brcmf_sdio *bus, bool in_isr); diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2200.c b/drivers/net/wireless/intel/ipw2x00/ipw2200.c index 24a5624ef2071..3876010c2ce8f 100644 --- a/drivers/net/wireless/intel/ipw2x00/ipw2200.c +++ b/drivers/net/wireless/intel/ipw2x00/ipw2200.c @@ -11387,7 +11387,13 @@ static const struct pci_device_id card_ids[] = { {PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2754, 0, 0, 0}, {PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2761, 0, 0, 0}, {PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2762, 0, 0, 0}, - {PCI_VDEVICE(INTEL, 0x104f), 0}, + /* + * This ID conflicts with i40e, but the devices can be differentiated + * because i40e devices use PCI_CLASS_NETWORK_ETHERNET and ipw2200 + * devices use PCI_CLASS_NETWORK_OTHER. + */ + {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x104f), + PCI_CLASS_NETWORK_OTHER << 8, 0xffff00, 0}, {PCI_VDEVICE(INTEL, 0x4220), 0}, /* BG */ {PCI_VDEVICE(INTEL, 0x4221), 0}, /* BG */ {PCI_VDEVICE(INTEL, 0x4223), 0}, /* ABG */ diff --git a/drivers/net/wireless/intel/iwlegacy/3945-mac.c b/drivers/net/wireless/intel/iwlegacy/3945-mac.c index 104748fcdc33e..54991f31c52c5 100644 --- a/drivers/net/wireless/intel/iwlegacy/3945-mac.c +++ b/drivers/net/wireless/intel/iwlegacy/3945-mac.c @@ -3224,7 +3224,9 @@ il3945_store_measurement(struct device *d, struct device_attribute *attr, D_INFO("Invoking measurement of type %d on " "channel %d (for '%s')\n", type, params.channel, buf); + mutex_lock(&il->mutex); il3945_get_measurement(il, ¶ms, type); + mutex_unlock(&il->mutex); return count; } diff --git a/drivers/net/wireless/intel/iwlegacy/4965-mac.c b/drivers/net/wireless/intel/iwlegacy/4965-mac.c index 3588dec75ebdd..57fa866efd9f8 100644 --- a/drivers/net/wireless/intel/iwlegacy/4965-mac.c +++ b/drivers/net/wireless/intel/iwlegacy/4965-mac.c @@ -4606,7 +4606,9 @@ il4965_store_tx_power(struct device *d, struct device_attribute *attr, if (ret) IL_INFO("%s is not in decimal form.\n", buf); else { + mutex_lock(&il->mutex); ret = il_set_tx_power(il, val, false); + mutex_unlock(&il->mutex); if (ret) IL_ERR("failed setting tx power (0x%08x).\n", ret); else diff --git a/drivers/net/wireless/intel/iwlwifi/fw/smem.c b/drivers/net/wireless/intel/iwlwifi/fw/smem.c index 90fd69b4860c1..344ddde85b189 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/smem.c +++ b/drivers/net/wireless/intel/iwlwifi/fw/smem.c @@ -6,6 +6,7 @@ */ #include "iwl-drv.h" #include "runtime.h" +#include "dbg.h" #include "fw/api/commands.h" static void iwl_parse_shared_mem_22000(struct iwl_fw_runtime *fwrt, @@ -17,7 +18,9 @@ static void iwl_parse_shared_mem_22000(struct iwl_fw_runtime *fwrt, u8 api_ver = iwl_fw_lookup_notif_ver(fwrt->fw, SYSTEM_GROUP, SHARED_MEM_CFG_CMD, 0); - if (WARN_ON(lmac_num > ARRAY_SIZE(mem_cfg->lmac_smem))) + /* Note: notification has 3 entries, but we only expect 2 */ + if (IWL_FW_CHECK(fwrt, lmac_num > ARRAY_SIZE(fwrt->smem_cfg.lmac), + "FW advertises %d LMACs\n", lmac_num)) return; fwrt->smem_cfg.num_lmacs = lmac_num; @@ -26,7 +29,8 @@ static void iwl_parse_shared_mem_22000(struct iwl_fw_runtime *fwrt, fwrt->smem_cfg.rxfifo2_size = le32_to_cpu(mem_cfg->rxfifo2_size); if (api_ver >= 4 && - !WARN_ON_ONCE(iwl_rx_packet_payload_len(pkt) < sizeof(*mem_cfg))) { + !IWL_FW_CHECK(fwrt, iwl_rx_packet_payload_len(pkt) < sizeof(*mem_cfg), + "bad shared mem notification size\n")) { fwrt->smem_cfg.rxfifo2_control_size = le32_to_cpu(mem_cfg->rxfifo2_control_size); } diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c index 2a7e7417d7d84..0f2db3ed58530 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c @@ -980,7 +980,9 @@ int iwl_mld_assign_vif_chanctx(struct ieee80211_hw *hw, { struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw); struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link); - unsigned int n_active = iwl_mld_count_active_links(mld, vif); + struct iwl_mld_link *temp_mld_link; + struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif); + u16 final_active_links = 0; int ret; lockdep_assert_wiphy(mld->wiphy); @@ -988,10 +990,7 @@ int iwl_mld_assign_vif_chanctx(struct ieee80211_hw *hw, if (WARN_ON(!mld_link)) return -EINVAL; - /* if the assigned one was not counted yet, count it now */ if (!rcu_access_pointer(mld_link->chan_ctx)) { - n_active++; - /* Track addition of non-BSS link */ if (ieee80211_vif_type_p2p(vif) != NL80211_IFTYPE_STATION) { ret = iwl_mld_emlsr_check_non_bss_block(mld, 1); @@ -1012,17 +1011,25 @@ int iwl_mld_assign_vif_chanctx(struct ieee80211_hw *hw, rcu_assign_pointer(mld_link->chan_ctx, ctx); - if (n_active > 1) { - struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif); + /* We cannot rely on vif->active_links at this stage as it contains + * both the removed links and the newly added links. + * Therefore, we create our own bitmap of the final active links, + * which does not include the removed links. + */ + for_each_mld_vif_valid_link(mld_vif, temp_mld_link) { + if (rcu_access_pointer(temp_mld_link->chan_ctx)) + final_active_links |= BIT(link_id); + } + if (hweight16(final_active_links) > 1) { /* Indicate to mac80211 that EML is enabled */ vif->driver_flags |= IEEE80211_VIF_EML_ACTIVE; mld_vif->emlsr.last_entry_ts = jiffies; - if (vif->active_links & BIT(mld_vif->emlsr.selected_links)) + if (final_active_links == mld_vif->emlsr.selected_links) mld_vif->emlsr.primary = mld_vif->emlsr.selected_primary; else - mld_vif->emlsr.primary = __ffs(vif->active_links); + mld_vif->emlsr.primary = __ffs(final_active_links); iwl_dbg_tlv_time_point(&mld->fwrt, IWL_FW_INI_TIME_ESR_LINK_UP, NULL); diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mlo.c b/drivers/net/wireless/intel/iwlwifi/mld/mlo.c index 241a6271d13d6..bf70e71aa5143 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/mlo.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/mlo.c @@ -851,9 +851,9 @@ iwl_mld_emlsr_pair_state(struct ieee80211_vif *vif, if (c_low->chan->center_freq > c_high->chan->center_freq) swap(c_low, c_high); - c_low_upper_edge = c_low->chan->center_freq + + c_low_upper_edge = c_low->center_freq1 + cfg80211_chandef_get_width(c_low) / 2; - c_high_lower_edge = c_high->chan->center_freq - + c_high_lower_edge = c_high->center_freq1 - cfg80211_chandef_get_width(c_high) / 2; if (a->chandef->chan->band == NL80211_BAND_5GHZ && diff --git a/drivers/net/wireless/intel/iwlwifi/mld/tx.c b/drivers/net/wireless/intel/iwlwifi/mld/tx.c index 3b4b575aadaa5..e3fb4fc4f452e 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/tx.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/tx.c @@ -345,6 +345,11 @@ u8 iwl_mld_get_lowest_rate(struct iwl_mld *mld, iwl_mld_get_basic_rates_and_band(mld, vif, info, &basic_rates, &band); + if (band >= NUM_NL80211_BANDS) { + WARN_ON(vif->type != NL80211_IFTYPE_NAN); + return IWL_FIRST_OFDM_RATE; + } + sband = mld->hw->wiphy->bands[band]; for_each_set_bit(i, &basic_rates, BITS_PER_LONG) { u16 hw = sband->bitrates[i].hw_value; diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c index 867807abde664..49ffc4ecee855 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c @@ -1761,6 +1761,20 @@ void iwl_mvm_probe_resp_data_notif(struct iwl_mvm *mvm, mvmvif = iwl_mvm_vif_from_mac80211(vif); + /* + * len_low should be 2 + n*13 (where n is the number of descriptors. + * 13 is the size of a NoA descriptor). We can have either one or two + * descriptors. + */ + if (IWL_FW_CHECK(mvm, notif->noa_active && + notif->noa_attr.len_low != 2 + + sizeof(struct ieee80211_p2p_noa_desc) && + notif->noa_attr.len_low != 2 + + sizeof(struct ieee80211_p2p_noa_desc) * 2, + "Invalid noa_attr.len_low (%d)\n", + notif->noa_attr.len_low)) + return; + new_data = kzalloc(sizeof(*new_data), GFP_KERNEL); if (!new_data) return; diff --git a/drivers/net/wireless/marvell/libertas/if_usb.c b/drivers/net/wireless/marvell/libertas/if_usb.c index b3c4040257a67..924ab93b7b671 100644 --- a/drivers/net/wireless/marvell/libertas/if_usb.c +++ b/drivers/net/wireless/marvell/libertas/if_usb.c @@ -426,6 +426,8 @@ static int usb_tx_block(struct if_usb_card *cardp, uint8_t *payload, uint16_t nb goto tx_ret; } + usb_kill_urb(cardp->tx_urb); + usb_fill_bulk_urb(cardp->tx_urb, cardp->udev, usb_sndbulkpipe(cardp->udev, cardp->ep_out), diff --git a/drivers/net/wireless/marvell/libertas/main.c b/drivers/net/wireless/marvell/libertas/main.c index d44e02c6fe385..dd97f1b61f4d1 100644 --- a/drivers/net/wireless/marvell/libertas/main.c +++ b/drivers/net/wireless/marvell/libertas/main.c @@ -799,8 +799,8 @@ static void lbs_free_adapter(struct lbs_private *priv) { lbs_free_cmd_buffer(priv); kfifo_free(&priv->event_fifo); - timer_delete(&priv->command_timer); - timer_delete(&priv->tx_lockup_timer); + timer_delete_sync(&priv->command_timer); + timer_delete_sync(&priv->tx_lockup_timer); } static const struct net_device_ops lbs_netdev_ops = { diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c index 0db00efe88b0b..837bd0f136fa1 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c @@ -411,6 +411,7 @@ mt76_connac2_mac_write_txwi_80211(struct mt76_dev *dev, __le32 *txwi, u32 val; if (ieee80211_is_action(fc) && + skb->len >= IEEE80211_MIN_ACTION_SIZE + 1 + 1 + 2 && mgmt->u.action.category == WLAN_CATEGORY_BACK && mgmt->u.action.u.addba_req.action_code == WLAN_ACTION_ADDBA_REQ) { u16 capab = le16_to_cpu(mgmt->u.action.u.addba_req.capab); diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c index 1e44e96f034e9..e880f3820a1ad 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c @@ -667,6 +667,7 @@ mt7925_mac_write_txwi_80211(struct mt76_dev *dev, __le32 *txwi, u32 val; if (ieee80211_is_action(fc) && + skb->len >= IEEE80211_MIN_ACTION_SIZE + 1 && mgmt->u.action.category == WLAN_CATEGORY_BACK && mgmt->u.action.u.addba_req.action_code == WLAN_ACTION_ADDBA_REQ) tid = MT_TX_ADDBA; diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c index 502136691a69e..0958961d2758e 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c @@ -799,6 +799,7 @@ mt7996_mac_write_txwi_80211(struct mt7996_dev *dev, __le32 *txwi, u32 val; if (ieee80211_is_action(fc) && + skb->len >= IEEE80211_MIN_ACTION_SIZE + 1 && mgmt->u.action.category == WLAN_CATEGORY_BACK && mgmt->u.action.u.addba_req.action_code == WLAN_ACTION_ADDBA_REQ) { if (is_mt7990(&dev->mt76)) diff --git a/drivers/net/wireless/realtek/rtw88/main.c b/drivers/net/wireless/realtek/rtw88/main.c index d93d21656f26c..dde2ea6a00e06 100644 --- a/drivers/net/wireless/realtek/rtw88/main.c +++ b/drivers/net/wireless/realtek/rtw88/main.c @@ -730,10 +730,10 @@ void rtw_set_rx_freq_band(struct rtw_rx_pkt_stat *pkt_stat, u8 channel) } EXPORT_SYMBOL(rtw_set_rx_freq_band); -void rtw_set_dtim_period(struct rtw_dev *rtwdev, int dtim_period) +void rtw_set_dtim_period(struct rtw_dev *rtwdev, u8 dtim_period) { rtw_write32_set(rtwdev, REG_TCR, BIT_TCR_UPDATE_TIMIE); - rtw_write8(rtwdev, REG_DTIM_COUNTER_ROOT, dtim_period - 1); + rtw_write8(rtwdev, REG_DTIM_COUNTER_ROOT, dtim_period ? dtim_period - 1 : 0); } void rtw_update_channel(struct rtw_dev *rtwdev, u8 center_channel, @@ -1658,14 +1658,41 @@ static u16 rtw_get_max_scan_ie_len(struct rtw_dev *rtwdev) return len; } +static struct ieee80211_supported_band * +rtw_sband_dup(struct rtw_dev *rtwdev, + const struct ieee80211_supported_band *sband) +{ + struct ieee80211_supported_band *dup; + + dup = devm_kmemdup(rtwdev->dev, sband, sizeof(*sband), GFP_KERNEL); + if (!dup) + return NULL; + + dup->channels = devm_kmemdup_array(rtwdev->dev, sband->channels, + sband->n_channels, + sizeof(*sband->channels), + GFP_KERNEL); + if (!dup->channels) + return NULL; + + dup->bitrates = devm_kmemdup_array(rtwdev->dev, sband->bitrates, + sband->n_bitrates, + sizeof(*sband->bitrates), + GFP_KERNEL); + if (!dup->bitrates) + return NULL; + + return dup; +} + static void rtw_set_supported_band(struct ieee80211_hw *hw, const struct rtw_chip_info *chip) { - struct rtw_dev *rtwdev = hw->priv; struct ieee80211_supported_band *sband; + struct rtw_dev *rtwdev = hw->priv; if (chip->band & RTW_BAND_2G) { - sband = kmemdup(&rtw_band_2ghz, sizeof(*sband), GFP_KERNEL); + sband = rtw_sband_dup(rtwdev, &rtw_band_2ghz); if (!sband) goto err_out; if (chip->ht_supported) @@ -1674,7 +1701,7 @@ static void rtw_set_supported_band(struct ieee80211_hw *hw, } if (chip->band & RTW_BAND_5G) { - sband = kmemdup(&rtw_band_5ghz, sizeof(*sband), GFP_KERNEL); + sband = rtw_sband_dup(rtwdev, &rtw_band_5ghz); if (!sband) goto err_out; if (chip->ht_supported) @@ -1690,13 +1717,6 @@ static void rtw_set_supported_band(struct ieee80211_hw *hw, rtw_err(rtwdev, "failed to set supported band\n"); } -static void rtw_unset_supported_band(struct ieee80211_hw *hw, - const struct rtw_chip_info *chip) -{ - kfree(hw->wiphy->bands[NL80211_BAND_2GHZ]); - kfree(hw->wiphy->bands[NL80211_BAND_5GHZ]); -} - static void rtw_vif_smps_iter(void *data, u8 *mac, struct ieee80211_vif *vif) { @@ -2320,10 +2340,7 @@ EXPORT_SYMBOL(rtw_register_hw); void rtw_unregister_hw(struct rtw_dev *rtwdev, struct ieee80211_hw *hw) { - const struct rtw_chip_info *chip = rtwdev->chip; - ieee80211_unregister_hw(hw); - rtw_unset_supported_band(hw, chip); rtw_debugfs_deinit(rtwdev); rtw_led_deinit(rtwdev); } diff --git a/drivers/net/wireless/realtek/rtw88/main.h b/drivers/net/wireless/realtek/rtw88/main.h index 43ed6d6b42919..1ab70214ce36e 100644 --- a/drivers/net/wireless/realtek/rtw88/main.h +++ b/drivers/net/wireless/realtek/rtw88/main.h @@ -2226,7 +2226,7 @@ enum nl80211_band rtw_hw_to_nl80211_band(enum rtw_supported_band hw_band) } void rtw_set_rx_freq_band(struct rtw_rx_pkt_stat *pkt_stat, u8 channel); -void rtw_set_dtim_period(struct rtw_dev *rtwdev, int dtim_period); +void rtw_set_dtim_period(struct rtw_dev *rtwdev, u8 dtim_period); void rtw_get_channel_params(struct cfg80211_chan_def *chandef, struct rtw_channel_params *ch_param); bool check_hw_ready(struct rtw_dev *rtwdev, u32 addr, u32 mask, u32 target); diff --git a/drivers/net/wireless/realtek/rtw88/rtw8821cu.c b/drivers/net/wireless/realtek/rtw88/rtw8821cu.c index 7a0fffc359e25..8cd09d66655db 100644 --- a/drivers/net/wireless/realtek/rtw88/rtw8821cu.c +++ b/drivers/net/wireless/realtek/rtw88/rtw8821cu.c @@ -37,6 +37,8 @@ static const struct usb_device_id rtw_8821cu_id_table[] = { .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* Edimax */ { USB_DEVICE_AND_INTERFACE_INFO(0x7392, 0xd811, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* Edimax */ + { USB_DEVICE_AND_INTERFACE_INFO(0x2c4e, 0x0105, 0xff, 0xff, 0xff), + .driver_info = (kernel_ulong_t)&(rtw8821c_hw_spec) }, /* Mercusys */ {}, }; MODULE_DEVICE_TABLE(usb, rtw_8821cu_id_table); diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822b.c b/drivers/net/wireless/realtek/rtw88/rtw8822b.c index 89b6485b229a8..4d88cc2f41485 100644 --- a/drivers/net/wireless/realtek/rtw88/rtw8822b.c +++ b/drivers/net/wireless/realtek/rtw88/rtw8822b.c @@ -1005,7 +1005,8 @@ static int rtw8822b_set_antenna(struct rtw_dev *rtwdev, hal->antenna_tx = antenna_tx; hal->antenna_rx = antenna_rx; - rtw8822b_config_trx_mode(rtwdev, antenna_tx, antenna_rx, false); + if (test_bit(RTW_FLAG_POWERON, rtwdev->flags)) + rtw8822b_config_trx_mode(rtwdev, antenna_tx, antenna_rx, false); return 0; } diff --git a/drivers/net/wireless/realtek/rtw89/chan.c b/drivers/net/wireless/realtek/rtw89/chan.c index 86f1b39a967fe..8fe6a7ef738f7 100644 --- a/drivers/net/wireless/realtek/rtw89/chan.c +++ b/drivers/net/wireless/realtek/rtw89/chan.c @@ -2608,17 +2608,20 @@ bool rtw89_mcc_detect_go_bcn(struct rtw89_dev *rtwdev, static void rtw89_mcc_detect_connection(struct rtw89_dev *rtwdev, struct rtw89_mcc_role *role) { + struct rtw89_vif_link *rtwvif_link = role->rtwvif_link; struct ieee80211_vif *vif; bool start_detect; int ret; ret = rtw89_core_send_nullfunc(rtwdev, role->rtwvif_link, true, false, RTW89_MCC_PROBE_TIMEOUT); - if (ret) + if (ret && + READ_ONCE(rtwvif_link->sync_bcn_tsf) == rtwvif_link->last_sync_bcn_tsf) role->probe_count++; else role->probe_count = 0; + rtwvif_link->last_sync_bcn_tsf = READ_ONCE(rtwvif_link->sync_bcn_tsf); if (role->probe_count < RTW89_MCC_PROBE_MAX_TRIES) return; diff --git a/drivers/net/wireless/realtek/rtw89/core.c b/drivers/net/wireless/realtek/rtw89/core.c index 917b2adede61d..018857d3569a8 100644 --- a/drivers/net/wireless/realtek/rtw89/core.c +++ b/drivers/net/wireless/realtek/rtw89/core.c @@ -2655,7 +2655,7 @@ static void rtw89_core_bcn_track_assoc(struct rtw89_dev *rtwdev, rcu_read_lock(); bss_conf = rtw89_vif_rcu_dereference_link(rtwvif_link, true); - beacon_int = bss_conf->beacon_int; + beacon_int = bss_conf->beacon_int ?: 100; dtim = bss_conf->dtim_period; rcu_read_unlock(); @@ -2685,9 +2685,7 @@ static void rtw89_core_bcn_track_reset(struct rtw89_dev *rtwdev) memset(&rtwdev->bcn_track, 0, sizeof(rtwdev->bcn_track)); } -static void rtw89_vif_rx_bcn_stat(struct rtw89_dev *rtwdev, - struct ieee80211_bss_conf *bss_conf, - struct sk_buff *skb) +static void rtw89_vif_rx_bcn_stat(struct rtw89_dev *rtwdev, struct sk_buff *skb) { #define RTW89_APPEND_TSF_2GHZ 384 #define RTW89_APPEND_TSF_5GHZ 52 @@ -2696,7 +2694,7 @@ static void rtw89_vif_rx_bcn_stat(struct rtw89_dev *rtwdev, struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb); struct rtw89_beacon_stat *bcn_stat = &rtwdev->phystat.bcn_stat; struct rtw89_beacon_track_info *bcn_track = &rtwdev->bcn_track; - u32 bcn_intvl_us = ieee80211_tu_to_usec(bss_conf->beacon_int); + u32 bcn_intvl_us = ieee80211_tu_to_usec(bcn_track->beacon_int); u64 tsf = le64_to_cpu(mgmt->u.beacon.timestamp); u8 wp, num = bcn_stat->num; u16 append; @@ -2704,6 +2702,10 @@ static void rtw89_vif_rx_bcn_stat(struct rtw89_dev *rtwdev, if (!RTW89_CHK_FW_FEATURE(BEACON_TRACKING, &rtwdev->fw)) return; + /* Skip if not yet associated */ + if (!bcn_intvl_us) + return; + switch (rx_status->band) { default: case NL80211_BAND_2GHZ: @@ -2791,7 +2793,7 @@ static void rtw89_vif_rx_stats_iter(void *data, u8 *mac, pkt_stat->beacon_rate = desc_info->data_rate; pkt_stat->beacon_len = skb->len; - rtw89_vif_rx_bcn_stat(rtwdev, bss_conf, skb); + rtw89_vif_rx_bcn_stat(rtwdev, skb); } if (!ether_addr_equal(bss_conf->addr, hdr->addr1)) @@ -5109,7 +5111,7 @@ static void rtw89_init_eht_cap(struct rtw89_dev *rtwdev, u8 val, val_mcs13; int sts = 8; - if (chip->chip_gen == RTW89_CHIP_AX) + if (chip->chip_gen == RTW89_CHIP_AX || hal->no_eht) return; if (hal->no_mcs_12_13) diff --git a/drivers/net/wireless/realtek/rtw89/core.h b/drivers/net/wireless/realtek/rtw89/core.h index 928c8c84c9644..c3839d49f4420 100644 --- a/drivers/net/wireless/realtek/rtw89/core.h +++ b/drivers/net/wireless/realtek/rtw89/core.h @@ -4996,6 +4996,7 @@ struct rtw89_hal { bool support_cckpd; bool support_igi; bool no_mcs_12_13; + bool no_eht; atomic_t roc_chanctx_idx; u8 roc_link_index; diff --git a/drivers/net/wireless/realtek/rtw89/debug.c b/drivers/net/wireless/realtek/rtw89/debug.c index 3dc7981c510fd..a82df3814069c 100644 --- a/drivers/net/wireless/realtek/rtw89/debug.c +++ b/drivers/net/wireless/realtek/rtw89/debug.c @@ -824,10 +824,6 @@ static ssize_t __print_txpwr_map(struct rtw89_dev *rtwdev, char *buf, size_t buf s8 *bufp, tmp; int ret; - bufp = vzalloc(map->addr_to - map->addr_from + 4); - if (!bufp) - return -ENOMEM; - if (path_num == 1) max_valid_addr = map->addr_to_1ss; else @@ -836,6 +832,10 @@ static ssize_t __print_txpwr_map(struct rtw89_dev *rtwdev, char *buf, size_t buf if (max_valid_addr == 0) return -EOPNOTSUPP; + bufp = vzalloc(map->addr_to - map->addr_from + 4); + if (!bufp) + return -ENOMEM; + for (addr = map->addr_from; addr <= max_valid_addr; addr += 4) { ret = rtw89_mac_txpwr_read32(rtwdev, RTW89_PHY_0, addr, &val); if (ret) diff --git a/drivers/net/wireless/realtek/rtw89/fw.c b/drivers/net/wireless/realtek/rtw89/fw.c index 080c4f8a655a8..c9851aafe649e 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.c +++ b/drivers/net/wireless/realtek/rtw89/fw.c @@ -8042,6 +8042,7 @@ int rtw89_hw_scan_start(struct rtw89_dev *rtwdev, struct cfg80211_scan_request *req = &scan_req->req; const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, rtwvif_link->chanctx_idx); + struct ieee80211_vif *vif = rtwvif_link_to_vif(rtwvif_link); struct rtw89_vif *rtwvif = rtwvif_link->rtwvif; struct rtw89_chanctx_pause_parm pause_parm = { .rsn = RTW89_CHANCTX_PAUSE_REASON_HW_SCAN, @@ -8071,6 +8072,8 @@ int rtw89_hw_scan_start(struct rtw89_dev *rtwdev, if (req->flags & NL80211_SCAN_FLAG_RANDOM_ADDR) get_random_mask_addr(mac_addr, req->mac_addr, req->mac_addr_mask); + else if (ieee80211_vif_is_mld(vif)) + ether_addr_copy(mac_addr, vif->addr); else ether_addr_copy(mac_addr, rtwvif_link->mac_addr); diff --git a/drivers/net/wireless/realtek/rtw89/fw.h b/drivers/net/wireless/realtek/rtw89/fw.h index ddebf79720687..47e5cbec306d2 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.h +++ b/drivers/net/wireless/realtek/rtw89/fw.h @@ -42,6 +42,10 @@ struct rtw89_c2hreg_phycap { #define RTW89_C2HREG_PHYCAP_W0_BW GENMASK(31, 24) #define RTW89_C2HREG_PHYCAP_W1_TX_NSS GENMASK(7, 0) #define RTW89_C2HREG_PHYCAP_W1_PROT GENMASK(15, 8) +#define RTW89_C2HREG_PHYCAP_W1_PROT_11N 1 +#define RTW89_C2HREG_PHYCAP_W1_PROT_11AC 2 +#define RTW89_C2HREG_PHYCAP_W1_PROT_11AX 3 +#define RTW89_C2HREG_PHYCAP_W1_PROT_11BE 4 #define RTW89_C2HREG_PHYCAP_W1_NIC GENMASK(23, 16) #define RTW89_C2HREG_PHYCAP_W1_WL_FUNC GENMASK(31, 24) #define RTW89_C2HREG_PHYCAP_W2_HW_TYPE GENMASK(7, 0) diff --git a/drivers/net/wireless/realtek/rtw89/mac.c b/drivers/net/wireless/realtek/rtw89/mac.c index fd11b8fb3c899..71194ea68bcee 100644 --- a/drivers/net/wireless/realtek/rtw89/mac.c +++ b/drivers/net/wireless/realtek/rtw89/mac.c @@ -2999,6 +2999,7 @@ static int rtw89_mac_setup_phycap_part0(struct rtw89_dev *rtwdev) struct rtw89_efuse *efuse = &rtwdev->efuse; struct rtw89_mac_c2h_info c2h_info = {}; struct rtw89_hal *hal = &rtwdev->hal; + u8 protocol; u8 tx_nss; u8 rx_nss; u8 tx_ant; @@ -3046,6 +3047,10 @@ static int rtw89_mac_setup_phycap_part0(struct rtw89_dev *rtwdev) rtw89_debug(rtwdev, RTW89_DBG_FW, "TX path diversity=%d\n", hal->tx_path_diversity); rtw89_debug(rtwdev, RTW89_DBG_FW, "Antenna diversity=%d\n", hal->ant_diversity); + protocol = u32_get_bits(phycap->w1, RTW89_C2HREG_PHYCAP_W1_PROT); + if (protocol < RTW89_C2HREG_PHYCAP_W1_PROT_11BE) + hal->no_eht = true; + return 0; } @@ -4262,6 +4267,7 @@ static void rtw89_mac_bcn_drop(struct rtw89_dev *rtwdev, #define BCN_HOLD_DEF 200 #define BCN_MASK_DEF 0 #define TBTT_ERLY_DEF 5 +#define TBTT_AGG_DEF 1 #define BCN_SET_UNIT 32 #define BCN_ERLY_SET_DLY (10 * 2) @@ -4565,6 +4571,16 @@ static void rtw89_mac_port_cfg_tbtt_early(struct rtw89_dev *rtwdev, B_AX_TBTTERLY_MASK, TBTT_ERLY_DEF); } +static void rtw89_mac_port_cfg_tbtt_agg(struct rtw89_dev *rtwdev, + struct rtw89_vif_link *rtwvif_link) +{ + const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; + const struct rtw89_port_reg *p = mac->port_base; + + rtw89_write16_port_mask(rtwdev, rtwvif_link, p->tbtt_agg, + B_AX_TBTT_AGG_NUM_MASK, TBTT_AGG_DEF); +} + static void rtw89_mac_port_cfg_bss_color(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_link) { @@ -4825,6 +4841,7 @@ int rtw89_mac_port_update(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvi rtw89_mac_port_cfg_bcn_hold_time(rtwdev, rtwvif_link); rtw89_mac_port_cfg_bcn_mask_area(rtwdev, rtwvif_link); rtw89_mac_port_cfg_tbtt_early(rtwdev, rtwvif_link); + rtw89_mac_port_cfg_tbtt_agg(rtwdev, rtwvif_link); rtw89_mac_port_cfg_bss_color(rtwdev, rtwvif_link); rtw89_mac_port_cfg_mbssid(rtwdev, rtwvif_link); rtw89_mac_port_cfg_func_en(rtwdev, rtwvif_link, true); @@ -7016,6 +7033,7 @@ const struct rtw89_mac_gen_def rtw89_mac_gen_ax = { .check_mac_en = rtw89_mac_check_mac_en_ax, .sys_init = sys_init_ax, .trx_init = trx_init_ax, + .err_imr_ctrl = err_imr_ctrl_ax, .hci_func_en = rtw89_mac_hci_func_en_ax, .dmac_func_pre_en = rtw89_mac_dmac_func_pre_en_ax, .dle_func_en = dle_func_en_ax, diff --git a/drivers/net/wireless/realtek/rtw89/mac.h b/drivers/net/wireless/realtek/rtw89/mac.h index 25fe5e5c8a979..51e37c183a35e 100644 --- a/drivers/net/wireless/realtek/rtw89/mac.h +++ b/drivers/net/wireless/realtek/rtw89/mac.h @@ -999,6 +999,7 @@ struct rtw89_mac_gen_def { enum rtw89_mac_hwmod_sel sel); int (*sys_init)(struct rtw89_dev *rtwdev); int (*trx_init)(struct rtw89_dev *rtwdev); + void (*err_imr_ctrl)(struct rtw89_dev *rtwdev, bool en); void (*hci_func_en)(struct rtw89_dev *rtwdev); void (*dmac_func_pre_en)(struct rtw89_dev *rtwdev); void (*dle_func_en)(struct rtw89_dev *rtwdev, bool enable); diff --git a/drivers/net/wireless/realtek/rtw89/mac80211.c b/drivers/net/wireless/realtek/rtw89/mac80211.c index 7b04183a3a5dd..474be7a5e49c7 100644 --- a/drivers/net/wireless/realtek/rtw89/mac80211.c +++ b/drivers/net/wireless/realtek/rtw89/mac80211.c @@ -127,6 +127,7 @@ static int __rtw89_ops_add_iface_link(struct rtw89_dev *rtwdev, rtwvif_link->reg_6ghz_power = RTW89_REG_6GHZ_POWER_DFLT; rtwvif_link->rand_tsf_done = false; rtwvif_link->detect_bcn_count = 0; + rtwvif_link->last_sync_bcn_tsf = 0; rcu_read_lock(); diff --git a/drivers/net/wireless/realtek/rtw89/mac_be.c b/drivers/net/wireless/realtek/rtw89/mac_be.c index ef69672b6862e..e5a61c628b731 100644 --- a/drivers/net/wireless/realtek/rtw89/mac_be.c +++ b/drivers/net/wireless/realtek/rtw89/mac_be.c @@ -1168,7 +1168,7 @@ static int resp_pktctl_init_be(struct rtw89_dev *rtwdev, u8 mac_idx) reg = rtw89_mac_reg_by_idx(rtwdev, R_BE_RESP_CSI_RESERVED_PAGE, mac_idx); rtw89_write32_mask(rtwdev, reg, B_BE_CSI_RESERVED_START_PAGE_MASK, qt_cfg.pktid); - rtw89_write32_mask(rtwdev, reg, B_BE_CSI_RESERVED_PAGE_NUM_MASK, qt_cfg.pg_num); + rtw89_write32_mask(rtwdev, reg, B_BE_CSI_RESERVED_PAGE_NUM_MASK, qt_cfg.pg_num + 1); return 0; } @@ -2594,6 +2594,7 @@ const struct rtw89_mac_gen_def rtw89_mac_gen_be = { .check_mac_en = rtw89_mac_check_mac_en_be, .sys_init = sys_init_be, .trx_init = trx_init_be, + .err_imr_ctrl = err_imr_ctrl_be, .hci_func_en = rtw89_mac_hci_func_en_be, .dmac_func_pre_en = rtw89_mac_dmac_func_pre_en_be, .dle_func_en = dle_func_en_be, diff --git a/drivers/net/wireless/realtek/rtw89/pci.c b/drivers/net/wireless/realtek/rtw89/pci.c index 0ee5f85794476..6be1849b0c4d2 100644 --- a/drivers/net/wireless/realtek/rtw89/pci.c +++ b/drivers/net/wireless/realtek/rtw89/pci.c @@ -604,8 +604,15 @@ static void rtw89_pci_release_rpp(struct rtw89_dev *rtwdev, void *rpp) info->parse_rpp(rtwdev, rpp, &rpp_info); - if (rpp_info.txch == RTW89_TXCH_CH12) { - rtw89_warn(rtwdev, "should no fwcmd release report\n"); + if (unlikely(rpp_info.txch >= RTW89_TXCH_NUM || + info->tx_dma_ch_mask & BIT(rpp_info.txch))) { + rtw89_warn(rtwdev, "should no release report on txch %d\n", + rpp_info.txch); + return; + } + + if (unlikely(rpp_info.seq >= RTW89_PCI_TXWD_NUM_MAX)) { + rtw89_warn(rtwdev, "invalid seq %d\n", rpp_info.seq); return; } @@ -4584,6 +4591,7 @@ static int __maybe_unused rtw89_pci_resume(struct device *dev) rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1, B_AX_SEL_REQ_ENTR_L1); } + rtw89_pci_hci_ldo(rtwdev); rtw89_pci_l2_hci_ldo(rtwdev); rtw89_pci_basic_cfg(rtwdev, true); diff --git a/drivers/net/wireless/realtek/rtw89/regd.c b/drivers/net/wireless/realtek/rtw89/regd.c index 58582f8d2b74c..8211d1277699c 100644 --- a/drivers/net/wireless/realtek/rtw89/regd.c +++ b/drivers/net/wireless/realtek/rtw89/regd.c @@ -1142,6 +1142,7 @@ static int rtw89_reg_6ghz_power_recalc(struct rtw89_dev *rtwdev, } } else { rtwvif_link->reg_6ghz_power = RTW89_REG_6GHZ_POWER_DFLT; + dflt = true; } rcu_read_unlock(); diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852bu.c b/drivers/net/wireless/realtek/rtw89/rtw8852bu.c index 0694272f7ffae..add5987110b33 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852bu.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852bu.c @@ -30,6 +30,8 @@ static const struct usb_device_id rtw_8852bu_id_table[] = { .driver_info = (kernel_ulong_t)&rtw89_8852bu_info }, { USB_DEVICE_AND_INTERFACE_INFO(0x0db0, 0x6931, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)&rtw89_8852bu_info }, + { USB_DEVICE_AND_INTERFACE_INFO(0x0db0, 0xf0c8, 0xff, 0xff, 0xff), + .driver_info = (kernel_ulong_t)&rtw89_8852bu_info }, { USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x3327, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)&rtw89_8852bu_info }, { USB_DEVICE_AND_INTERFACE_INFO(0x3574, 0x6121, 0xff, 0xff, 0xff), diff --git a/drivers/net/wireless/realtek/rtw89/rtw8922a.c b/drivers/net/wireless/realtek/rtw89/rtw8922a.c index 6aa19ad259aca..730e5d0d65750 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8922a.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8922a.c @@ -636,16 +636,30 @@ static int rtw8922a_read_efuse_rf(struct rtw89_dev *rtwdev, u8 *log_map) static int rtw8922a_read_efuse(struct rtw89_dev *rtwdev, u8 *log_map, enum rtw89_efuse_block block) { + struct rtw89_efuse *efuse = &rtwdev->efuse; + int ret; + switch (block) { case RTW89_EFUSE_BLOCK_HCI_DIG_PCIE_SDIO: - return rtw8922a_read_efuse_pci_sdio(rtwdev, log_map); + ret = rtw8922a_read_efuse_pci_sdio(rtwdev, log_map); + break; case RTW89_EFUSE_BLOCK_HCI_DIG_USB: - return rtw8922a_read_efuse_usb(rtwdev, log_map); + ret = rtw8922a_read_efuse_usb(rtwdev, log_map); + break; case RTW89_EFUSE_BLOCK_RF: - return rtw8922a_read_efuse_rf(rtwdev, log_map); + ret = rtw8922a_read_efuse_rf(rtwdev, log_map); + break; default: - return 0; + ret = 0; + break; + } + + if (!ret && is_zero_ether_addr(efuse->addr)) { + rtw89_info(rtwdev, "efuse mac address is zero, using random mac\n"); + eth_random_addr(efuse->addr); } + + return ret; } #define THM_TRIM_POSITIVE_MASK BIT(6) @@ -1756,6 +1770,32 @@ static int rtw8922a_ctrl_rx_path_tmac(struct rtw89_dev *rtwdev, } #define DIGITAL_PWR_COMP_REG_NUM 22 +static const u32 rtw8922a_digital_pwr_comp_2g_s0_val[][DIGITAL_PWR_COMP_REG_NUM] = { + {0x012C0064, 0x04B00258, 0x00432710, 0x019000A7, 0x06400320, + 0x0D05091D, 0x14D50FA0, 0x00000000, 0x01010000, 0x00000101, + 0x01010101, 0x02020201, 0x02010000, 0x03030202, 0x00000303, + 0x03020101, 0x06060504, 0x01010000, 0x06050403, 0x01000606, + 0x05040202, 0x07070706}, + {0x012C0064, 0x04B00258, 0x00432710, 0x019000A7, 0x06400320, + 0x0D05091D, 0x14D50FA0, 0x00000000, 0x01010100, 0x00000101, + 0x01000000, 0x01010101, 0x01010000, 0x02020202, 0x00000404, + 0x03020101, 0x04040303, 0x02010000, 0x03030303, 0x00000505, + 0x03030201, 0x05050303}, +}; + +static const u32 rtw8922a_digital_pwr_comp_2g_s1_val[][DIGITAL_PWR_COMP_REG_NUM] = { + {0x012C0064, 0x04B00258, 0x00432710, 0x019000A7, 0x06400320, + 0x0D05091D, 0x14D50FA0, 0x01010000, 0x01010101, 0x00000101, + 0x01010100, 0x01010101, 0x01010000, 0x02020202, 0x01000202, + 0x02020101, 0x03030202, 0x02010000, 0x05040403, 0x01000606, + 0x05040302, 0x07070605}, + {0x012C0064, 0x04B00258, 0x00432710, 0x019000A7, 0x06400320, + 0x0D05091D, 0x14D50FA0, 0x00000000, 0x01010100, 0x00000101, + 0x01010000, 0x02020201, 0x02010100, 0x03030202, 0x01000404, + 0x04030201, 0x05050404, 0x01010100, 0x04030303, 0x01000505, + 0x03030101, 0x05050404}, +}; + static const u32 rtw8922a_digital_pwr_comp_val[][DIGITAL_PWR_COMP_REG_NUM] = { {0x012C0096, 0x044C02BC, 0x00322710, 0x015E0096, 0x03C8028A, 0x0BB80708, 0x17701194, 0x02020100, 0x03030303, 0x01000303, @@ -1770,7 +1810,7 @@ static const u32 rtw8922a_digital_pwr_comp_val[][DIGITAL_PWR_COMP_REG_NUM] = { }; static void rtw8922a_set_digital_pwr_comp(struct rtw89_dev *rtwdev, - bool enable, u8 nss, + u8 band, u8 nss, enum rtw89_rf_path path) { static const u32 ltpc_t0[2] = {R_BE_LTPC_T0_PATH0, R_BE_LTPC_T0_PATH1}; @@ -1778,14 +1818,25 @@ static void rtw8922a_set_digital_pwr_comp(struct rtw89_dev *rtwdev, u32 addr, val; u32 i; - if (nss == 1) - digital_pwr_comp = rtw8922a_digital_pwr_comp_val[0]; - else - digital_pwr_comp = rtw8922a_digital_pwr_comp_val[1]; + if (nss == 1) { + if (band == RTW89_BAND_2G) + digital_pwr_comp = path == RF_PATH_A ? + rtw8922a_digital_pwr_comp_2g_s0_val[0] : + rtw8922a_digital_pwr_comp_2g_s1_val[0]; + else + digital_pwr_comp = rtw8922a_digital_pwr_comp_val[0]; + } else { + if (band == RTW89_BAND_2G) + digital_pwr_comp = path == RF_PATH_A ? + rtw8922a_digital_pwr_comp_2g_s0_val[1] : + rtw8922a_digital_pwr_comp_2g_s1_val[1]; + else + digital_pwr_comp = rtw8922a_digital_pwr_comp_val[1]; + } addr = ltpc_t0[path]; for (i = 0; i < DIGITAL_PWR_COMP_REG_NUM; i++, addr += 4) { - val = enable ? digital_pwr_comp[i] : 0; + val = digital_pwr_comp[i]; rtw89_phy_write32(rtwdev, addr, val); } } @@ -1794,7 +1845,7 @@ static void rtw8922a_digital_pwr_comp(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx) { const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_CHANCTX_0); - bool enable = chan->band_type != RTW89_BAND_2G; + u8 band = chan->band_type; u8 path; if (rtwdev->mlo_dbcc_mode == MLO_1_PLUS_1_1RF) { @@ -1802,10 +1853,10 @@ static void rtw8922a_digital_pwr_comp(struct rtw89_dev *rtwdev, path = RF_PATH_A; else path = RF_PATH_B; - rtw8922a_set_digital_pwr_comp(rtwdev, enable, 1, path); + rtw8922a_set_digital_pwr_comp(rtwdev, band, 1, path); } else { - rtw8922a_set_digital_pwr_comp(rtwdev, enable, 2, RF_PATH_A); - rtw8922a_set_digital_pwr_comp(rtwdev, enable, 2, RF_PATH_B); + rtw8922a_set_digital_pwr_comp(rtwdev, band, 2, RF_PATH_A); + rtw8922a_set_digital_pwr_comp(rtwdev, band, 2, RF_PATH_B); } } diff --git a/drivers/net/wireless/realtek/rtw89/ser.c b/drivers/net/wireless/realtek/rtw89/ser.c index f99e179f7ff9f..7fdc69578da31 100644 --- a/drivers/net/wireless/realtek/rtw89/ser.c +++ b/drivers/net/wireless/realtek/rtw89/ser.c @@ -431,6 +431,14 @@ static void hal_send_m4_event(struct rtw89_ser *ser) rtw89_mac_set_err_status(rtwdev, MAC_AX_ERR_L1_RCVY_EN); } +static void hal_enable_err_imr(struct rtw89_ser *ser) +{ + struct rtw89_dev *rtwdev = container_of(ser, struct rtw89_dev, ser); + const struct rtw89_mac_gen_def *mac = rtwdev->chip->mac_def; + + mac->err_imr_ctrl(rtwdev, true); +} + /* state handler */ static void ser_idle_st_hdl(struct rtw89_ser *ser, u8 evt) { @@ -552,6 +560,8 @@ static void ser_do_hci_st_hdl(struct rtw89_ser *ser, u8 evt) break; case SER_EV_MAC_RESET_DONE: + hal_enable_err_imr(ser); + ser_state_goto(ser, SER_IDLE_ST); break; diff --git a/drivers/net/wireless/realtek/rtw89/wow.c b/drivers/net/wireless/realtek/rtw89/wow.c index 5faa51ad896a2..f34cd863d1009 100644 --- a/drivers/net/wireless/realtek/rtw89/wow.c +++ b/drivers/net/wireless/realtek/rtw89/wow.c @@ -809,6 +809,10 @@ static void rtw89_wow_show_wakeup_reason(struct rtw89_dev *rtwdev) reason = rtw89_read8(rtwdev, wow_reason_reg); switch (reason) { + case RTW89_WOW_RSN_RX_DISASSOC: + wakeup.disconnect = true; + rtw89_debug(rtwdev, RTW89_DBG_WOW, "WOW: Rx disassoc\n"); + break; case RTW89_WOW_RSN_RX_DEAUTH: wakeup.disconnect = true; rtw89_debug(rtwdev, RTW89_DBG_WOW, "WOW: Rx deauth\n"); diff --git a/drivers/net/wireless/realtek/rtw89/wow.h b/drivers/net/wireless/realtek/rtw89/wow.h index d2ba6cebc2a6b..71e07f482174f 100644 --- a/drivers/net/wireless/realtek/rtw89/wow.h +++ b/drivers/net/wireless/realtek/rtw89/wow.h @@ -33,6 +33,7 @@ enum rtw89_wake_reason { RTW89_WOW_RSN_RX_PTK_REKEY = 0x1, RTW89_WOW_RSN_RX_GTK_REKEY = 0x2, + RTW89_WOW_RSN_RX_DISASSOC = 0x4, RTW89_WOW_RSN_RX_DEAUTH = 0x8, RTW89_WOW_RSN_DISCONNECT = 0x10, RTW89_WOW_RSN_RX_MAGIC_PKT = 0x21, diff --git a/drivers/net/wireless/rsi/rsi_91x_mac80211.c b/drivers/net/wireless/rsi/rsi_91x_mac80211.c index 8c8e074a3a705..c7ae8031436ae 100644 --- a/drivers/net/wireless/rsi/rsi_91x_mac80211.c +++ b/drivers/net/wireless/rsi/rsi_91x_mac80211.c @@ -668,7 +668,7 @@ static int rsi_mac80211_config(struct ieee80211_hw *hw, struct rsi_hw *adapter = hw->priv; struct rsi_common *common = adapter->priv; struct ieee80211_conf *conf = &hw->conf; - int status = -EOPNOTSUPP; + int status = 0; mutex_lock(&common->mutex); diff --git a/drivers/net/wireless/st/cw1200/pm.c b/drivers/net/wireless/st/cw1200/pm.c index 2002e3f9fe45b..b656afe65db07 100644 --- a/drivers/net/wireless/st/cw1200/pm.c +++ b/drivers/net/wireless/st/cw1200/pm.c @@ -264,12 +264,14 @@ int cw1200_wow_suspend(struct ieee80211_hw *hw, struct cfg80211_wowlan *wowlan) wiphy_err(priv->hw->wiphy, "PM request failed: %d. WoW is disabled.\n", ret); cw1200_wow_resume(hw); + mutex_unlock(&priv->conf_mutex); return -EBUSY; } /* Force resume if event is coming from the device. */ if (atomic_read(&priv->bh_rx)) { cw1200_wow_resume(hw); + mutex_unlock(&priv->conf_mutex); return -EAGAIN; } diff --git a/drivers/net/wireless/ti/wlcore/main.c b/drivers/net/wireless/ti/wlcore/main.c index 6116a8522d960..bdb06584d7e45 100644 --- a/drivers/net/wireless/ti/wlcore/main.c +++ b/drivers/net/wireless/ti/wlcore/main.c @@ -1880,6 +1880,8 @@ static int __maybe_unused wl1271_op_resume(struct ieee80211_hw *hw) wl->wow_enabled); WARN_ON(!wl->wow_enabled); + mutex_lock(&wl->mutex); + ret = pm_runtime_force_resume(wl->dev); if (ret < 0) { wl1271_error("ELP wakeup failure!"); @@ -1896,8 +1898,6 @@ static int __maybe_unused wl1271_op_resume(struct ieee80211_hw *hw) run_irq_work = true; spin_unlock_irqrestore(&wl->wl_lock, flags); - mutex_lock(&wl->mutex); - /* test the recovery flag before calling any SDIO functions */ pending_recovery = test_bit(WL1271_FLAG_RECOVERY_IN_PROGRESS, &wl->flags); diff --git a/drivers/net/wwan/mhi_wwan_mbim.c b/drivers/net/wwan/mhi_wwan_mbim.c index f8bc9a39bfa30..1d7e3ad900c12 100644 --- a/drivers/net/wwan/mhi_wwan_mbim.c +++ b/drivers/net/wwan/mhi_wwan_mbim.c @@ -98,7 +98,8 @@ static struct mhi_mbim_link *mhi_mbim_get_link_rcu(struct mhi_mbim_context *mbim static int mhi_mbim_get_link_mux_id(struct mhi_controller *cntrl) { if (strcmp(cntrl->name, "foxconn-dw5934e") == 0 || - strcmp(cntrl->name, "foxconn-t99w640") == 0) + strcmp(cntrl->name, "foxconn-t99w640") == 0 || + strcmp(cntrl->name, "foxconn-t99w760") == 0) return WDS_BIND_MUX_DATA_PORT_MUX_ID; return 0; diff --git a/drivers/net/xen-netback/xenbus.c b/drivers/net/xen-netback/xenbus.c index a78a25b872409..61b547aab286a 100644 --- a/drivers/net/xen-netback/xenbus.c +++ b/drivers/net/xen-netback/xenbus.c @@ -735,10 +735,11 @@ static void connect(struct backend_info *be) */ requested_num_queues = xenbus_read_unsigned(dev->otherend, "multi-queue-num-queues", 1); - if (requested_num_queues > xenvif_max_queues) { + if (requested_num_queues > xenvif_max_queues || + requested_num_queues == 0) { /* buggy or malicious guest */ xenbus_dev_fatal(dev, -EINVAL, - "guest requested %u queues, exceeding the maximum of %u.", + "guest requested %u queues, but valid range is 1 - %u.", requested_num_queues, xenvif_max_queues); return; } diff --git a/drivers/nfc/nxp-nci/i2c.c b/drivers/nfc/nxp-nci/i2c.c index 049662ffdf972..6a5ce8ff91f0b 100644 --- a/drivers/nfc/nxp-nci/i2c.c +++ b/drivers/nfc/nxp-nci/i2c.c @@ -305,7 +305,7 @@ static int nxp_nci_i2c_probe(struct i2c_client *client) r = request_threaded_irq(client->irq, NULL, nxp_nci_i2c_irq_thread_fn, - IRQF_TRIGGER_RISING | IRQF_ONESHOT, + IRQF_ONESHOT, NXP_NCI_I2C_DRIVER_NAME, phy); if (r < 0) nfc_err(&client->dev, "Unable to register IRQ handler\n"); diff --git a/drivers/nfc/pn533/usb.c b/drivers/nfc/pn533/usb.c index 018a80674f06e..0f12f86ebb023 100644 --- a/drivers/nfc/pn533/usb.c +++ b/drivers/nfc/pn533/usb.c @@ -628,6 +628,7 @@ static void pn533_usb_disconnect(struct usb_interface *interface) usb_free_urb(phy->out_urb); usb_free_urb(phy->ack_urb); kfree(phy->ack_buffer); + usb_put_dev(phy->udev); nfc_info(&interface->dev, "NXP PN533 NFC device disconnected\n"); } diff --git a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c index f851397b65d6e..0536521fa6ccc 100644 --- a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c +++ b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c @@ -1202,7 +1202,8 @@ static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev) sndev->mmio_self_ctrl); sndev->nr_lut_mw = ioread16(&sndev->mmio_self_ctrl->lut_table_entries); - sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw); + if (sndev->nr_lut_mw) + sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw); dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut\n", sndev->nr_direct_mw, sndev->nr_lut_mw); @@ -1212,7 +1213,8 @@ static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev) sndev->peer_nr_lut_mw = ioread16(&sndev->mmio_peer_ctrl->lut_table_entries); - sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw); + if (sndev->peer_nr_lut_mw) + sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw); dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut\n", sndev->peer_nr_direct_mw, sndev->peer_nr_lut_mw); @@ -1314,6 +1316,12 @@ static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev) for (i = 0; i < sndev->nr_lut_mw; i++) { int idx = sndev->nr_direct_mw + i; + if (idx >= MAX_MWS) { + dev_err(&sndev->stdev->dev, + "Total number of MW cannot be bigger than %d", MAX_MWS); + break; + } + sndev->self_shared->mw_sizes[idx] = LUT_SIZE; } } diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c index 71d4bb25f7fdd..4d00263ebc934 100644 --- a/drivers/ntb/ntb_transport.c +++ b/drivers/ntb/ntb_transport.c @@ -1236,9 +1236,9 @@ static int ntb_transport_init_queue(struct ntb_transport_ctx *nt, qp->tx_max_entry = tx_size / qp->tx_max_frame; if (nt->debugfs_node_dir) { - char debugfs_name[4]; + char debugfs_name[8]; - snprintf(debugfs_name, 4, "qp%d", qp_num); + snprintf(debugfs_name, sizeof(debugfs_name), "qp%d", qp_num); qp->debugfs_dir = debugfs_create_dir(debugfs_name, nt->debugfs_node_dir); diff --git a/drivers/nvdimm/nd_virtio.c b/drivers/nvdimm/nd_virtio.c index c3f07be4aa22a..af82385be7c6a 100644 --- a/drivers/nvdimm/nd_virtio.c +++ b/drivers/nvdimm/nd_virtio.c @@ -44,6 +44,8 @@ static int virtio_pmem_flush(struct nd_region *nd_region) unsigned long flags; int err, err1; + guard(mutex)(&vpmem->flush_lock); + /* * Don't bother to submit the request to the device if the device is * not activated. @@ -53,7 +55,6 @@ static int virtio_pmem_flush(struct nd_region *nd_region) return -EIO; } - might_sleep(); req_data = kmalloc(sizeof(*req_data), GFP_KERNEL); if (!req_data) return -ENOMEM; diff --git a/drivers/nvdimm/virtio_pmem.c b/drivers/nvdimm/virtio_pmem.c index 2396d19ce5496..77b1966619059 100644 --- a/drivers/nvdimm/virtio_pmem.c +++ b/drivers/nvdimm/virtio_pmem.c @@ -64,6 +64,7 @@ static int virtio_pmem_probe(struct virtio_device *vdev) goto out_err; } + mutex_init(&vpmem->flush_lock); vpmem->vdev = vdev; vdev->priv = vpmem; err = init_vq(vpmem); diff --git a/drivers/nvdimm/virtio_pmem.h b/drivers/nvdimm/virtio_pmem.h index 0dddefe594c46..f72cf17f9518f 100644 --- a/drivers/nvdimm/virtio_pmem.h +++ b/drivers/nvdimm/virtio_pmem.h @@ -13,6 +13,7 @@ #include #include #include +#include #include struct virtio_pmem_request { @@ -35,6 +36,9 @@ struct virtio_pmem { /* Virtio pmem request queue */ struct virtqueue *req_vq; + /* Serialize flush requests to the device. */ + struct mutex flush_lock; + /* nvdimm bus registers virtio pmem device */ struct nvdimm_bus *nvdimm_bus; struct nvdimm_bus_descriptor nd_desc; diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index f1f719351f3f2..2ba7244fdaf1c 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -4865,6 +4865,13 @@ int nvme_alloc_admin_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set, if (ret) return ret; + /* + * If a previous admin queue exists (e.g., from before a reset), + * put it now before allocating a new one to avoid orphaning it. + */ + if (ctrl->admin_q) + blk_put_queue(ctrl->admin_q); + ctrl->admin_q = blk_mq_alloc_queue(set, &lim, NULL); if (IS_ERR(ctrl->admin_q)) { ret = PTR_ERR(ctrl->admin_q); diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index e35eccacee8c8..81ccdd91f7790 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -1310,13 +1310,11 @@ void nvme_mpath_remove_disk(struct nvme_ns_head *head) if (!list_empty(&head->list)) goto out; - if (head->delayed_removal_secs) { - /* - * Ensure that no one could remove this module while the head - * remove work is pending. - */ - if (!try_module_get(THIS_MODULE)) - goto out; + /* + * Ensure that no one could remove this module while the head + * remove work is pending. + */ + if (head->delayed_removal_secs && try_module_get(THIS_MODULE)) { mod_delayed_work(nvme_wq, &head->remove_work, head->delayed_removal_secs * HZ); } else { diff --git a/drivers/nvme/host/pr.c b/drivers/nvme/host/pr.c index ca6a74607b139..fe7dbe2648158 100644 --- a/drivers/nvme/host/pr.c +++ b/drivers/nvme/host/pr.c @@ -228,7 +228,8 @@ static int nvme_pr_resv_report(struct block_device *bdev, void *data, static int nvme_pr_read_keys(struct block_device *bdev, struct pr_keys *keys_info) { - u32 rse_len, num_keys = keys_info->num_keys; + size_t rse_len; + u32 num_keys = keys_info->num_keys; struct nvme_reservation_status_ext *rse; int ret, i; bool eds; @@ -238,7 +239,10 @@ static int nvme_pr_read_keys(struct block_device *bdev, * enough to get enough keys to fill the return keys buffer. */ rse_len = struct_size(rse, regctl_eds, num_keys); - rse = kzalloc(rse_len, GFP_KERNEL); + if (rse_len > U32_MAX) + return -EINVAL; + + rse = kvzalloc(rse_len, GFP_KERNEL); if (!rse) return -ENOMEM; @@ -263,7 +267,7 @@ static int nvme_pr_read_keys(struct block_device *bdev, } free_rse: - kfree(rse); + kvfree(rse); return ret; } diff --git a/drivers/nvme/target/fcloop.c b/drivers/nvme/target/fcloop.c index 5dffcc5becae8..305ab7ee6e760 100644 --- a/drivers/nvme/target/fcloop.c +++ b/drivers/nvme/target/fcloop.c @@ -492,6 +492,7 @@ fcloop_t2h_xmt_ls_rsp(struct nvme_fc_local_port *localport, struct fcloop_rport *rport = remoteport->private; struct nvmet_fc_target_port *targetport = rport->targetport; struct fcloop_tport *tport; + int ret = 0; if (!targetport) { /* @@ -501,12 +502,18 @@ fcloop_t2h_xmt_ls_rsp(struct nvme_fc_local_port *localport, * We end up here from delete association exchange: * nvmet_fc_xmt_disconnect_assoc sends an async request. * - * Return success because this is what LLDDs do; silently - * drop the response. + * Return success when remoteport is still online because this + * is what LLDDs do and silently drop the response. Otherwise, + * return with error to signal upper layer to perform the lsrsp + * resource cleanup. */ - lsrsp->done(lsrsp); + if (remoteport->port_state == FC_OBJSTATE_ONLINE) + lsrsp->done(lsrsp); + else + ret = -ENODEV; + kmem_cache_free(lsreq_cache, tls_req); - return 0; + return ret; } memcpy(lsreq->rspaddr, lsrsp->rspbuf, diff --git a/drivers/nvmem/Kconfig b/drivers/nvmem/Kconfig index e0d88d3199c11..11b098705ec62 100644 --- a/drivers/nvmem/Kconfig +++ b/drivers/nvmem/Kconfig @@ -30,7 +30,7 @@ source "drivers/nvmem/layouts/Kconfig" config NVMEM_AN8855_EFUSE tristate "Airoha AN8855 eFuse support" - depends on MFD_AIROHA_AN8855 || COMPILE_TEST + depends on COMPILE_TEST help Say y here to enable support for reading eFuses on Airoha AN8855 Switch. These are e.g. used to store factory programmed diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c index 387c88c552595..ff68fd5ad3d6f 100644 --- a/drivers/nvmem/core.c +++ b/drivers/nvmem/core.c @@ -831,6 +831,7 @@ static int nvmem_add_cells_from_dt(struct nvmem_device *nvmem, struct device_nod kfree(info.name); if (ret) { of_node_put(child); + of_node_put(info.np); return ret; } } diff --git a/drivers/of/kexec.c b/drivers/of/kexec.c index 1ee2d31816aeb..c4cf3552c0183 100644 --- a/drivers/of/kexec.c +++ b/drivers/of/kexec.c @@ -128,7 +128,6 @@ int __init ima_get_kexec_buffer(void **addr, size_t *size) { int ret, len; unsigned long tmp_addr; - unsigned long start_pfn, end_pfn; size_t tmp_size; const void *prop; @@ -144,17 +143,9 @@ int __init ima_get_kexec_buffer(void **addr, size_t *size) if (!tmp_size) return -ENOENT; - /* - * Calculate the PFNs for the buffer and ensure - * they are with in addressable memory. - */ - start_pfn = PHYS_PFN(tmp_addr); - end_pfn = PHYS_PFN(tmp_addr + tmp_size - 1); - if (!page_is_ram(start_pfn) || !page_is_ram(end_pfn)) { - pr_warn("IMA buffer at 0x%lx, size = 0x%zx beyond memory\n", - tmp_addr, tmp_size); - return -EINVAL; - } + ret = ima_validate_range(tmp_addr, tmp_size); + if (ret) + return ret; *addr = __va(tmp_addr); *size = tmp_size; diff --git a/drivers/of/unittest.c b/drivers/of/unittest.c index 3b773aaf9d050..9c184e93f50c6 100644 --- a/drivers/of/unittest.c +++ b/drivers/of/unittest.c @@ -804,11 +804,13 @@ static void __init of_unittest_property_copy(void) new = __of_prop_dup(&p1, GFP_KERNEL); unittest(new && propcmp(&p1, new), "empty property didn't copy correctly\n"); - __of_prop_free(new); + if (new) + __of_prop_free(new); new = __of_prop_dup(&p2, GFP_KERNEL); unittest(new && propcmp(&p2, new), "non-empty property didn't copy correctly\n"); - __of_prop_free(new); + if (new) + __of_prop_free(new); #endif } diff --git a/drivers/opp/core.c b/drivers/opp/core.c index bba4f7daff8cb..775d4a36f2f54 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -241,7 +241,7 @@ unsigned int dev_pm_opp_get_level(struct dev_pm_opp *opp) { if (IS_ERR_OR_NULL(opp) || !opp->available) { pr_err("%s: Invalid parameters\n", __func__); - return 0; + return U32_MAX; } return opp->level; diff --git a/drivers/pci/bus.c b/drivers/pci/bus.c index 3669755e3b1ba..ac8a7512ba26f 100644 --- a/drivers/pci/bus.c +++ b/drivers/pci/bus.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -359,6 +360,31 @@ void pci_bus_add_device(struct pci_dev *dev) /* Save config space for error recoverability */ pci_save_state(dev); + /* + * If the PCI device is associated with a pwrctrl device with a + * power supply, create a device link between the PCI device and + * pwrctrl device. This ensures that pwrctrl drivers are probed + * before PCI client drivers. + */ + pdev = of_find_device_by_node(dn); + if (pdev) { + if (of_pci_supply_present(dn)) { + if (!device_link_add(&dev->dev, &pdev->dev, + DL_FLAG_AUTOREMOVE_CONSUMER)) { + pci_err(dev, "failed to add device link to power control device %s\n", + pdev->name); + } + } + put_device(&pdev->dev); + } + + /* + * Enable runtime PM, which potentially allows the device to + * suspend immediately, only after the PCI state has been + * configured completely. + */ + pm_runtime_enable(&dev->dev); + if (!dn || of_device_is_available(dn)) pci_dev_allow_binding(dev); diff --git a/drivers/pci/controller/cadence/pci-j721e.c b/drivers/pci/controller/cadence/pci-j721e.c index 5bc5ab20aa6d9..0413d163cfea0 100644 --- a/drivers/pci/controller/cadence/pci-j721e.c +++ b/drivers/pci/controller/cadence/pci-j721e.c @@ -479,7 +479,6 @@ static int j721e_pcie_probe(struct platform_device *pdev) struct cdns_pcie_ep *ep = NULL; struct gpio_desc *gpiod; void __iomem *base; - struct clk *clk; u32 num_lanes; u32 mode; int ret; @@ -603,19 +602,13 @@ static int j721e_pcie_probe(struct platform_device *pdev) goto err_get_sync; } - clk = devm_clk_get_optional(dev, "pcie_refclk"); - if (IS_ERR(clk)) { - ret = dev_err_probe(dev, PTR_ERR(clk), "failed to get pcie_refclk\n"); + pcie->refclk = devm_clk_get_optional_enabled(dev, "pcie_refclk"); + if (IS_ERR(pcie->refclk)) { + ret = dev_err_probe(dev, PTR_ERR(pcie->refclk), + "failed to enable pcie_refclk\n"); goto err_pcie_setup; } - ret = clk_prepare_enable(clk); - if (ret) { - dev_err_probe(dev, ret, "failed to enable pcie_refclk\n"); - goto err_pcie_setup; - } - pcie->refclk = clk; - /* * Section 2.2 of the PCI Express Card Electromechanical * Specification (Revision 5.1) mandates that the deassertion @@ -628,10 +621,10 @@ static int j721e_pcie_probe(struct platform_device *pdev) gpiod_set_value_cansleep(gpiod, 1); } - ret = cdns_pcie_host_setup(rc); - if (ret < 0) { - clk_disable_unprepare(pcie->refclk); - goto err_pcie_setup; + if (IS_ENABLED(CONFIG_PCI_J721E_HOST)) { + ret = cdns_pcie_host_setup(rc); + if (ret < 0) + goto err_pcie_setup; } break; @@ -642,9 +635,11 @@ static int j721e_pcie_probe(struct platform_device *pdev) goto err_get_sync; } - ret = cdns_pcie_ep_setup(ep); - if (ret < 0) - goto err_pcie_setup; + if (IS_ENABLED(CONFIG_PCI_J721E_EP)) { + ret = cdns_pcie_ep_setup(ep); + if (ret < 0) + goto err_pcie_setup; + } break; } @@ -669,17 +664,17 @@ static void j721e_pcie_remove(struct platform_device *pdev) struct cdns_pcie_ep *ep; struct cdns_pcie_rc *rc; - if (pcie->mode == PCI_MODE_RC) { + if (IS_ENABLED(CONFIG_PCI_J721E_HOST) && + pcie->mode == PCI_MODE_RC) { rc = container_of(cdns_pcie, struct cdns_pcie_rc, pcie); cdns_pcie_host_disable(rc); - } else { + } else if (IS_ENABLED(CONFIG_PCI_J721E_EP)) { ep = container_of(cdns_pcie, struct cdns_pcie_ep, pcie); cdns_pcie_ep_disable(ep); } gpiod_set_value_cansleep(pcie->reset_gpio, 0); - clk_disable_unprepare(pcie->refclk); cdns_pcie_disable_phy(cdns_pcie); j721e_pcie_disable_link_irq(pcie); pm_runtime_put(dev); @@ -739,10 +734,12 @@ static int j721e_pcie_resume_noirq(struct device *dev) gpiod_set_value_cansleep(pcie->reset_gpio, 1); } - ret = cdns_pcie_host_link_setup(rc); - if (ret < 0) { - clk_disable_unprepare(pcie->refclk); - return ret; + if (IS_ENABLED(CONFIG_PCI_J721E_HOST)) { + ret = cdns_pcie_host_link_setup(rc); + if (ret < 0) { + clk_disable_unprepare(pcie->refclk); + return ret; + } } /* @@ -752,10 +749,12 @@ static int j721e_pcie_resume_noirq(struct device *dev) for (enum cdns_pcie_rp_bar bar = RP_BAR0; bar <= RP_NO_BAR; bar++) rc->avail_ib_bar[bar] = true; - ret = cdns_pcie_host_init(rc); - if (ret) { - clk_disable_unprepare(pcie->refclk); - return ret; + if (IS_ENABLED(CONFIG_PCI_J721E_HOST)) { + ret = cdns_pcie_host_init(rc); + if (ret) { + clk_disable_unprepare(pcie->refclk); + return ret; + } } } diff --git a/drivers/pci/controller/cadence/pcie-cadence.c b/drivers/pci/controller/cadence/pcie-cadence.c index bd683d0fecb22..d614452861f77 100644 --- a/drivers/pci/controller/cadence/pcie-cadence.c +++ b/drivers/pci/controller/cadence/pcie-cadence.c @@ -13,13 +13,13 @@ u8 cdns_pcie_find_capability(struct cdns_pcie *pcie, u8 cap) { return PCI_FIND_NEXT_CAP(cdns_pcie_read_cfg, PCI_CAPABILITY_LIST, - cap, pcie); + cap, NULL, pcie); } EXPORT_SYMBOL_GPL(cdns_pcie_find_capability); u16 cdns_pcie_find_ext_capability(struct cdns_pcie *pcie, u8 cap) { - return PCI_FIND_NEXT_EXT_CAP(cdns_pcie_read_cfg, 0, cap, pcie); + return PCI_FIND_NEXT_EXT_CAP(cdns_pcie_read_cfg, 0, cap, NULL, pcie); } EXPORT_SYMBOL_GPL(cdns_pcie_find_ext_capability); diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index 4668fc9648bff..a42164c870548 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -52,6 +52,8 @@ #define IMX95_PCIE_REF_CLKEN BIT(23) #define IMX95_PCIE_PHY_CR_PARA_SEL BIT(9) #define IMX95_PCIE_SS_RW_REG_1 0xf4 +#define IMX95_PCIE_CLKREQ_OVERRIDE_EN BIT(8) +#define IMX95_PCIE_CLKREQ_OVERRIDE_VAL BIT(9) #define IMX95_PCIE_SYS_AUX_PWR_DET BIT(31) #define IMX95_PE0_GEN_CTRL_1 0x1050 @@ -114,6 +116,7 @@ enum imx_pcie_variants { #define IMX_PCIE_FLAG_BROKEN_SUSPEND BIT(9) #define IMX_PCIE_FLAG_HAS_LUT BIT(10) #define IMX_PCIE_FLAG_8GT_ECN_ERR051586 BIT(11) +#define IMX_PCIE_FLAG_SKIP_L23_READY BIT(12) #define imx_check_flag(pci, val) (pci->drvdata->flags & val) @@ -706,6 +709,22 @@ static int imx7d_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable) return 0; } +static void imx95_pcie_clkreq_override(struct imx_pcie *imx_pcie, bool enable) +{ + regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_SS_RW_REG_1, + IMX95_PCIE_CLKREQ_OVERRIDE_EN, + enable ? IMX95_PCIE_CLKREQ_OVERRIDE_EN : 0); + regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_SS_RW_REG_1, + IMX95_PCIE_CLKREQ_OVERRIDE_VAL, + enable ? IMX95_PCIE_CLKREQ_OVERRIDE_VAL : 0); +} + +static int imx95_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable) +{ + imx95_pcie_clkreq_override(imx_pcie, enable); + return 0; +} + static int imx_pcie_clk_enable(struct imx_pcie *imx_pcie) { struct dw_pcie *pci = imx_pcie->pci; @@ -1777,6 +1796,8 @@ static int imx_pcie_probe(struct platform_device *pdev) */ imx_pcie_add_lut_by_rid(imx_pcie, 0); } else { + if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_SKIP_L23_READY)) + pci->pp.skip_l23_ready = true; pci->pp.use_atu_msg = true; ret = dw_pcie_host_init(&pci->pp); if (ret < 0) @@ -1838,6 +1859,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .variant = IMX6QP, .flags = IMX_PCIE_FLAG_IMX_PHY | IMX_PCIE_FLAG_SPEED_CHANGE_WORKAROUND | + IMX_PCIE_FLAG_SKIP_L23_READY | IMX_PCIE_FLAG_SUPPORTS_SUSPEND, .dbi_length = 0x200, .gpr = "fsl,imx6q-iomuxc-gpr", @@ -1854,6 +1876,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .variant = IMX7D, .flags = IMX_PCIE_FLAG_SUPPORTS_SUSPEND | IMX_PCIE_FLAG_HAS_APP_RESET | + IMX_PCIE_FLAG_SKIP_L23_READY | IMX_PCIE_FLAG_HAS_PHY_RESET, .gpr = "fsl,imx7d-iomuxc-gpr", .mode_off[0] = IOMUXC_GPR12, @@ -1913,6 +1936,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .core_reset = imx95_pcie_core_reset, .init_phy = imx95_pcie_init_phy, .wait_pll_lock = imx95_pcie_wait_for_phy_pll_lock, + .enable_ref_clk = imx95_pcie_enable_ref_clk, }, [IMX8MQ_EP] = { .variant = IMX8MQ_EP, @@ -1969,6 +1993,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .core_reset = imx95_pcie_core_reset, .wait_pll_lock = imx95_pcie_wait_for_phy_pll_lock, .epc_features = &imx95_pcie_epc_features, + .enable_ref_clk = imx95_pcie_enable_ref_clk, .mode = DW_PCIE_EP_TYPE, }, }; diff --git a/drivers/pci/controller/dwc/pcie-designware-debugfs.c b/drivers/pci/controller/dwc/pcie-designware-debugfs.c index 0fbf86c0b97e0..df98fee69892b 100644 --- a/drivers/pci/controller/dwc/pcie-designware-debugfs.c +++ b/drivers/pci/controller/dwc/pcie-designware-debugfs.c @@ -485,6 +485,8 @@ static const char *ltssm_status_string(enum dw_pcie_ltssm ltssm) DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_RCVRY_EQ1); DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_RCVRY_EQ2); DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_RCVRY_EQ3); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_L1_1); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_L1_2); default: str = "DW_PCIE_LTSSM_UNKNOWN"; break; diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c index 7f2112c2fb215..e2e18beb2951d 100644 --- a/drivers/pci/controller/dwc/pcie-designware-ep.c +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c @@ -72,47 +72,15 @@ EXPORT_SYMBOL_GPL(dw_pcie_ep_reset_bar); static u8 dw_pcie_ep_find_capability(struct dw_pcie_ep *ep, u8 func_no, u8 cap) { return PCI_FIND_NEXT_CAP(dw_pcie_ep_read_cfg, PCI_CAPABILITY_LIST, - cap, ep, func_no); + cap, NULL, ep, func_no); } -/** - * dw_pcie_ep_hide_ext_capability - Hide a capability from the linked list - * @pci: DWC PCI device - * @prev_cap: Capability preceding the capability that should be hidden - * @cap: Capability that should be hidden - * - * Return: 0 if success, errno otherwise. - */ -int dw_pcie_ep_hide_ext_capability(struct dw_pcie *pci, u8 prev_cap, u8 cap) +static u16 dw_pcie_ep_find_ext_capability(struct dw_pcie_ep *ep, + u8 func_no, u8 cap) { - u16 prev_cap_offset, cap_offset; - u32 prev_cap_header, cap_header; - - prev_cap_offset = dw_pcie_find_ext_capability(pci, prev_cap); - if (!prev_cap_offset) - return -EINVAL; - - prev_cap_header = dw_pcie_readl_dbi(pci, prev_cap_offset); - cap_offset = PCI_EXT_CAP_NEXT(prev_cap_header); - cap_header = dw_pcie_readl_dbi(pci, cap_offset); - - /* cap must immediately follow prev_cap. */ - if (PCI_EXT_CAP_ID(cap_header) != cap) - return -EINVAL; - - /* Clear next ptr. */ - prev_cap_header &= ~GENMASK(31, 20); - - /* Set next ptr to next ptr of cap. */ - prev_cap_header |= cap_header & GENMASK(31, 20); - - dw_pcie_dbi_ro_wr_en(pci); - dw_pcie_writel_dbi(pci, prev_cap_offset, prev_cap_header); - dw_pcie_dbi_ro_wr_dis(pci); - - return 0; + return PCI_FIND_NEXT_EXT_CAP(dw_pcie_ep_read_cfg, 0, + cap, NULL, ep, func_no); } -EXPORT_SYMBOL_GPL(dw_pcie_ep_hide_ext_capability); static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no, u8 vfunc_no, struct pci_epf_header *hdr) @@ -217,22 +185,22 @@ static void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, ep->bar_to_atu[bar] = 0; } -static unsigned int dw_pcie_ep_get_rebar_offset(struct dw_pcie *pci, +static unsigned int dw_pcie_ep_get_rebar_offset(struct dw_pcie_ep *ep, u8 func_no, enum pci_barno bar) { u32 reg, bar_index; unsigned int offset, nbars; int i; - offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR); + offset = dw_pcie_ep_find_ext_capability(ep, func_no, PCI_EXT_CAP_ID_REBAR); if (!offset) return offset; - reg = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL); + reg = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, reg); for (i = 0; i < nbars; i++, offset += PCI_REBAR_CTRL) { - reg = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL); + reg = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); bar_index = FIELD_GET(PCI_REBAR_CTRL_BAR_IDX, reg); if (bar_index == bar) return offset; @@ -253,7 +221,7 @@ static int dw_pcie_ep_set_bar_resizable(struct dw_pcie_ep *ep, u8 func_no, u32 rebar_cap, rebar_ctrl; int ret; - rebar_offset = dw_pcie_ep_get_rebar_offset(pci, bar); + rebar_offset = dw_pcie_ep_get_rebar_offset(ep, func_no, bar); if (!rebar_offset) return -EINVAL; @@ -283,16 +251,16 @@ static int dw_pcie_ep_set_bar_resizable(struct dw_pcie_ep *ep, u8 func_no, * 1 MB to 128 TB. Bits 31:16 in PCI_REBAR_CTRL define "supported sizes" * bits for sizes 256 TB to 8 EB. Disallow sizes 256 TB to 8 EB. */ - rebar_ctrl = dw_pcie_readl_dbi(pci, rebar_offset + PCI_REBAR_CTRL); + rebar_ctrl = dw_pcie_ep_readl_dbi(ep, func_no, rebar_offset + PCI_REBAR_CTRL); rebar_ctrl &= ~GENMASK(31, 16); - dw_pcie_writel_dbi(pci, rebar_offset + PCI_REBAR_CTRL, rebar_ctrl); + dw_pcie_ep_writel_dbi(ep, func_no, rebar_offset + PCI_REBAR_CTRL, rebar_ctrl); /* * The "selected size" (bits 13:8) in PCI_REBAR_CTRL are automatically * updated when writing PCI_REBAR_CAP, see "Figure 3-26 Resizable BAR * Example for 32-bit Memory BAR0" in DWC EP databook 5.96a. */ - dw_pcie_writel_dbi(pci, rebar_offset + PCI_REBAR_CAP, rebar_cap); + dw_pcie_ep_writel_dbi(ep, func_no, rebar_offset + PCI_REBAR_CAP, rebar_cap); dw_pcie_dbi_ro_wr_dis(pci); @@ -793,6 +761,9 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, writel(msg_data, ep->msi_mem + offset); + /* flush posted write before unmap */ + readl(ep->msi_mem + offset); + dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); return 0; @@ -835,20 +806,17 @@ void dw_pcie_ep_deinit(struct dw_pcie_ep *ep) } EXPORT_SYMBOL_GPL(dw_pcie_ep_deinit); -static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci) +static void dw_pcie_ep_init_rebar_registers(struct dw_pcie_ep *ep, u8 func_no) { - struct dw_pcie_ep *ep = &pci->ep; unsigned int offset; unsigned int nbars; enum pci_barno bar; u32 reg, i, val; - offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR); - - dw_pcie_dbi_ro_wr_en(pci); + offset = dw_pcie_ep_find_ext_capability(ep, func_no, PCI_EXT_CAP_ID_REBAR); if (offset) { - reg = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL); + reg = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, reg); /* @@ -869,16 +837,28 @@ static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci) * the controller when RESBAR_CAP_REG is written, which * is why RESBAR_CAP_REG is written here. */ - val = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL); + val = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); bar = FIELD_GET(PCI_REBAR_CTRL_BAR_IDX, val); if (ep->epf_bar[bar]) pci_epc_bar_size_to_rebar_cap(ep->epf_bar[bar]->size, &val); else val = BIT(4); - dw_pcie_writel_dbi(pci, offset + PCI_REBAR_CAP, val); + dw_pcie_ep_writel_dbi(ep, func_no, offset + PCI_REBAR_CAP, val); } } +} + +static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci) +{ + struct dw_pcie_ep *ep = &pci->ep; + u8 funcs = ep->epc->max_functions; + u8 func_no; + + dw_pcie_dbi_ro_wr_en(pci); + + for (func_no = 0; func_no < funcs; func_no++) + dw_pcie_ep_init_rebar_registers(ep, func_no); dw_pcie_setup(pci); dw_pcie_dbi_ro_wr_dis(pci); diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c index 460048812e068..e622e59f621d4 100644 --- a/drivers/pci/controller/dwc/pcie-designware-host.c +++ b/drivers/pci/controller/dwc/pcie-designware-host.c @@ -664,14 +664,8 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp) goto err_remove_edma; } - /* - * Note: Skip the link up delay only when a Link Up IRQ is present. - * If there is no Link Up IRQ, we should not bypass the delay - * because that would require users to manually rescan for devices. - */ - if (!pp->use_linkup_irq) - /* Ignore errors, the link may come up later */ - dw_pcie_wait_for_link(pci); + /* Ignore errors, the link may come up later */ + dw_pcie_wait_for_link(pci); ret = pci_host_probe(bridge); if (ret) @@ -942,7 +936,14 @@ static int dw_pcie_iatu_setup(struct dw_pcie_rp *pp) dev_warn(pci->dev, "Ranges exceed outbound iATU size (%d)\n", pci->num_ob_windows); - pp->msg_atu_index = i; + if (pp->use_atu_msg) { + if (pci->num_ob_windows > ++i) { + pp->msg_atu_index = i; + } else { + dev_err(pci->dev, "Cannot add outbound window for MSG TLP\n"); + return -ENOMEM; + } + } i = 0; resource_list_for_each_entry(entry, &pp->bridge->dma_ranges) { @@ -1080,6 +1081,8 @@ int dw_pcie_setup_rc(struct dw_pcie_rp *pp) PCI_COMMAND_MASTER | PCI_COMMAND_SERR; dw_pcie_writel_dbi(pci, PCI_COMMAND, val); + dw_pcie_hide_unsupported_l1ss(pci); + dw_pcie_config_presets(pp); /* * If the platform provides its own child bus config accesses, it means @@ -1145,8 +1148,12 @@ static int dw_pcie_pme_turn_off(struct dw_pcie *pci) int dw_pcie_suspend_noirq(struct dw_pcie *pci) { + u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + int ret = 0; u32 val; - int ret; + + if (!dw_pcie_link_up(pci)) + goto stop_link; if (!pci_host_common_can_enter_d3cold(pci->pp.bridge)) return 0; @@ -1159,6 +1166,16 @@ int dw_pcie_suspend_noirq(struct dw_pcie *pci) return ret; } + /* + * Some SoCs do not support reading the LTSSM register after + * PME_Turn_Off broadcast. For those SoCs, skip waiting for L2/L3 Ready + * state and wait 10ms as recommended in PCIe spec r6.0, sec 5.3.3.2.1. + */ + if (pci->pp.skip_l23_ready) { + mdelay(PCIE_PME_TO_L2_TIMEOUT_US/1000); + goto stop_link; + } + ret = read_poll_timeout(dw_pcie_get_ltssm, val, val == DW_PCIE_LTSSM_L2_IDLE || val <= DW_PCIE_LTSSM_DETECT_WAIT, @@ -1177,6 +1194,7 @@ int dw_pcie_suspend_noirq(struct dw_pcie *pci) */ udelay(1); +stop_link: dw_pcie_stop_link(pci); if (pci->pp.ops->deinit) pci->pp.ops->deinit(&pci->pp); diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c index 06eca858eb1b3..345365ea97c74 100644 --- a/drivers/pci/controller/dwc/pcie-designware.c +++ b/drivers/pci/controller/dwc/pcie-designware.c @@ -226,16 +226,69 @@ void dw_pcie_version_detect(struct dw_pcie *pci) u8 dw_pcie_find_capability(struct dw_pcie *pci, u8 cap) { return PCI_FIND_NEXT_CAP(dw_pcie_read_cfg, PCI_CAPABILITY_LIST, cap, - pci); + NULL, pci); } EXPORT_SYMBOL_GPL(dw_pcie_find_capability); u16 dw_pcie_find_ext_capability(struct dw_pcie *pci, u8 cap) { - return PCI_FIND_NEXT_EXT_CAP(dw_pcie_read_cfg, 0, cap, pci); + return PCI_FIND_NEXT_EXT_CAP(dw_pcie_read_cfg, 0, cap, NULL, pci); } EXPORT_SYMBOL_GPL(dw_pcie_find_ext_capability); +void dw_pcie_remove_capability(struct dw_pcie *pci, u8 cap) +{ + u8 cap_pos, pre_pos, next_pos; + u16 reg; + + cap_pos = PCI_FIND_NEXT_CAP(dw_pcie_read_cfg, PCI_CAPABILITY_LIST, cap, + &pre_pos, pci); + if (!cap_pos) + return; + + reg = dw_pcie_readw_dbi(pci, cap_pos); + next_pos = (reg & 0xff00) >> 8; + + dw_pcie_dbi_ro_wr_en(pci); + if (pre_pos == PCI_CAPABILITY_LIST) + dw_pcie_writeb_dbi(pci, PCI_CAPABILITY_LIST, next_pos); + else + dw_pcie_writeb_dbi(pci, pre_pos + 1, next_pos); + dw_pcie_dbi_ro_wr_dis(pci); +} +EXPORT_SYMBOL_GPL(dw_pcie_remove_capability); + +void dw_pcie_remove_ext_capability(struct dw_pcie *pci, u8 cap) +{ + int cap_pos, next_pos, pre_pos; + u32 pre_header, header; + + cap_pos = PCI_FIND_NEXT_EXT_CAP(dw_pcie_read_cfg, 0, cap, &pre_pos, pci); + if (!cap_pos) + return; + + header = dw_pcie_readl_dbi(pci, cap_pos); + /* + * If the first cap at offset PCI_CFG_SPACE_SIZE is removed, + * only set it's capid to zero as it cannot be skipped. + */ + if (cap_pos == PCI_CFG_SPACE_SIZE) { + dw_pcie_dbi_ro_wr_en(pci); + dw_pcie_writel_dbi(pci, cap_pos, header & 0xffff0000); + dw_pcie_dbi_ro_wr_dis(pci); + return; + } + + pre_header = dw_pcie_readl_dbi(pci, pre_pos); + next_pos = PCI_EXT_CAP_NEXT(header); + + dw_pcie_dbi_ro_wr_en(pci); + dw_pcie_writel_dbi(pci, pre_pos, + (pre_header & 0xfffff) | (next_pos << 20)); + dw_pcie_dbi_ro_wr_dis(pci); +} +EXPORT_SYMBOL_GPL(dw_pcie_remove_ext_capability); + static u16 __dw_pcie_find_vsec_capability(struct dw_pcie *pci, u16 vendor_id, u16 vsec_id) { @@ -246,7 +299,7 @@ static u16 __dw_pcie_find_vsec_capability(struct dw_pcie *pci, u16 vendor_id, return 0; while ((vsec = PCI_FIND_NEXT_EXT_CAP(dw_pcie_read_cfg, vsec, - PCI_EXT_CAP_ID_VNDR, pci))) { + PCI_EXT_CAP_ID_VNDR, NULL, pci))) { header = dw_pcie_readl_dbi(pci, vsec + PCI_VNDR_HEADER); if (PCI_VNDR_HEADER_ID(header) == vsec_id) return vsec; @@ -1083,6 +1136,30 @@ void dw_pcie_edma_remove(struct dw_pcie *pci) dw_edma_remove(&pci->edma); } +void dw_pcie_hide_unsupported_l1ss(struct dw_pcie *pci) +{ + u16 l1ss; + u32 l1ss_cap; + + if (pci->l1ss_support) + return; + + l1ss = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS); + if (!l1ss) + return; + + /* + * Unless the driver claims "l1ss_support", don't advertise L1 PM + * Substates because they require CLKREQ# and possibly other + * device-specific configuration. + */ + l1ss_cap = dw_pcie_readl_dbi(pci, l1ss + PCI_L1SS_CAP); + l1ss_cap &= ~(PCI_L1SS_CAP_PCIPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_1 | + PCI_L1SS_CAP_PCIPM_L1_2 | PCI_L1SS_CAP_ASPM_L1_2 | + PCI_L1SS_CAP_L1_PM_SS); + dw_pcie_writel_dbi(pci, l1ss + PCI_L1SS_CAP, l1ss_cap); +} + void dw_pcie_setup(struct dw_pcie *pci) { u32 val; diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index 263d04fd95350..57ca6b8de55bc 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -381,6 +381,10 @@ enum dw_pcie_ltssm { DW_PCIE_LTSSM_RCVRY_EQ2 = 0x22, DW_PCIE_LTSSM_RCVRY_EQ3 = 0x23, + /* Vendor glue drivers provide pseudo L1 substates from get_ltssm() */ + DW_PCIE_LTSSM_L1_1 = 0x141, + DW_PCIE_LTSSM_L1_2 = 0x142, + DW_PCIE_LTSSM_UNKNOWN = 0xFFFFFFFF, }; @@ -427,11 +431,11 @@ struct dw_pcie_rp { bool use_atu_msg; int msg_atu_index; struct resource *msg_res; - bool use_linkup_irq; struct pci_eq_presets presets; struct pci_config_window *cfg; bool ecam_enabled; bool native_ecam; + bool skip_l23_ready; }; struct dw_pcie_ep_ops { @@ -517,6 +521,7 @@ struct dw_pcie { int max_link_speed; u8 n_fts[2]; struct dw_edma_chip edma; + bool l1ss_support; /* L1 PM Substates support */ struct clk_bulk_data app_clks[DW_PCIE_NUM_APP_CLKS]; struct clk_bulk_data core_clks[DW_PCIE_NUM_CORE_CLKS]; struct reset_control_bulk_data app_rsts[DW_PCIE_NUM_APP_RSTS]; @@ -553,6 +558,8 @@ void dw_pcie_version_detect(struct dw_pcie *pci); u8 dw_pcie_find_capability(struct dw_pcie *pci, u8 cap); u16 dw_pcie_find_ext_capability(struct dw_pcie *pci, u8 cap); +void dw_pcie_remove_capability(struct dw_pcie *pci, u8 cap); +void dw_pcie_remove_ext_capability(struct dw_pcie *pci, u8 cap); u16 dw_pcie_find_rasdes_capability(struct dw_pcie *pci); u16 dw_pcie_find_ptm_capability(struct dw_pcie *pci); @@ -574,6 +581,7 @@ int dw_pcie_prog_ep_inbound_atu(struct dw_pcie *pci, u8 func_no, int index, int type, u64 parent_bus_addr, u8 bar, size_t size); void dw_pcie_disable_atu(struct dw_pcie *pci, u32 dir, int index); +void dw_pcie_hide_unsupported_l1ss(struct dw_pcie *pci); void dw_pcie_setup(struct dw_pcie *pci); void dw_pcie_iatu_detect(struct dw_pcie *pci); int dw_pcie_edma_detect(struct dw_pcie *pci); @@ -881,7 +889,6 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 func_no, u16 interrupt_num); void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar); -int dw_pcie_ep_hide_ext_capability(struct dw_pcie *pci, u8 prev_cap, u8 cap); struct dw_pcie_ep_func * dw_pcie_ep_get_func_from_ep(struct dw_pcie_ep *ep, u8 func_no); #else @@ -939,12 +946,6 @@ static inline void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar) { } -static inline int dw_pcie_ep_hide_ext_capability(struct dw_pcie *pci, - u8 prev_cap, u8 cap) -{ - return 0; -} - static inline struct dw_pcie_ep_func * dw_pcie_ep_get_func_from_ep(struct dw_pcie_ep *ep, u8 func_no) { diff --git a/drivers/pci/controller/dwc/pcie-dw-rockchip.c b/drivers/pci/controller/dwc/pcie-dw-rockchip.c index 3e2752c7dd096..c2c36290be061 100644 --- a/drivers/pci/controller/dwc/pcie-dw-rockchip.c +++ b/drivers/pci/controller/dwc/pcie-dw-rockchip.c @@ -62,6 +62,17 @@ /* Interrupt Mask Register Related to Miscellaneous Operation */ #define PCIE_CLIENT_INTR_MASK_MISC 0x24 +/* Power Management Control Register */ +#define PCIE_CLIENT_POWER_CON 0x2c +#define PCIE_CLKREQ_READY FIELD_PREP_WM16(BIT(0), 1) +#define PCIE_CLKREQ_NOT_READY FIELD_PREP_WM16(BIT(0), 0) +#define PCIE_CLKREQ_PULL_DOWN FIELD_PREP_WM16(GENMASK(13, 12), 1) + +/* RASDES TBA information */ +#define PCIE_CLIENT_CDM_RASDES_TBA_INFO_CMN 0x154 +#define PCIE_CLIENT_CDM_RASDES_TBA_L1_1 BIT(4) +#define PCIE_CLIENT_CDM_RASDES_TBA_L1_2 BIT(5) + /* Hot Reset Control Register */ #define PCIE_CLIENT_HOT_RESET_CTRL 0x180 #define PCIE_LTSSM_APP_DLY2_EN BIT(1) @@ -74,6 +85,8 @@ #define PCIE_LINKUP_MASK GENMASK(17, 16) #define PCIE_LTSSM_STATUS_MASK GENMASK(5, 0) +#define PCIE_TYPE0_HDR_DBI2_OFFSET 0x100000 + struct rockchip_pcie { struct dw_pcie pci; void __iomem *apb_base; @@ -85,6 +98,7 @@ struct rockchip_pcie { struct regulator *vpcie3v3; struct irq_domain *irq_domain; const struct rockchip_pcie_of_data *data; + bool supports_clkreq; }; struct rockchip_pcie_of_data { @@ -175,11 +189,26 @@ static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip) return 0; } -static u32 rockchip_pcie_get_ltssm(struct rockchip_pcie *rockchip) +static u32 rockchip_pcie_get_ltssm_reg(struct rockchip_pcie *rockchip) { return rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_LTSSM_STATUS); } +static enum dw_pcie_ltssm rockchip_pcie_get_ltssm(struct dw_pcie *pci) +{ + struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); + u32 val = rockchip_pcie_readl_apb(rockchip, + PCIE_CLIENT_CDM_RASDES_TBA_INFO_CMN); + + if (val & PCIE_CLIENT_CDM_RASDES_TBA_L1_1) + return DW_PCIE_LTSSM_L1_1; + + if (val & PCIE_CLIENT_CDM_RASDES_TBA_L1_2) + return DW_PCIE_LTSSM_L1_2; + + return rockchip_pcie_get_ltssm_reg(rockchip) & PCIE_LTSSM_STATUS_MASK; +} + static void rockchip_pcie_enable_ltssm(struct rockchip_pcie *rockchip) { rockchip_pcie_writel_apb(rockchip, PCIE_CLIENT_ENABLE_LTSSM, @@ -195,11 +224,40 @@ static void rockchip_pcie_disable_ltssm(struct rockchip_pcie *rockchip) static bool rockchip_pcie_link_up(struct dw_pcie *pci) { struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); - u32 val = rockchip_pcie_get_ltssm(rockchip); + u32 val = rockchip_pcie_get_ltssm_reg(rockchip); return FIELD_GET(PCIE_LINKUP_MASK, val) == PCIE_LINKUP; } +/* + * See e.g. section '11.6.6.4 L1 Substate' in the RK3588 TRM V1.0 for the steps + * needed to support L1 substates. Currently, just enable L1 substates for RC + * mode if CLKREQ# is properly connected and supports-clkreq is present in DT. + * For EP mode, there are more things should be done to actually save power in + * L1 substates, so disable L1 substates until there is proper support. + */ +static void rockchip_pcie_configure_l1ss(struct dw_pcie *pci) +{ + struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); + + /* Enable L1 substates if CLKREQ# is properly connected */ + if (rockchip->supports_clkreq) { + rockchip_pcie_writel_apb(rockchip, PCIE_CLKREQ_READY, + PCIE_CLIENT_POWER_CON); + pci->l1ss_support = true; + return; + } + + /* + * Otherwise, assert CLKREQ# unconditionally. Since + * pci->l1ss_support is not set, the DWC core will prevent L1 + * Substates support from being advertised. + */ + rockchip_pcie_writel_apb(rockchip, + PCIE_CLKREQ_PULL_DOWN | PCIE_CLKREQ_NOT_READY, + PCIE_CLIENT_POWER_CON); +} + static void rockchip_pcie_enable_l0s(struct dw_pcie *pci) { u32 cap, lnkcap; @@ -257,6 +315,8 @@ static int rockchip_pcie_host_init(struct dw_pcie_rp *pp) if (irq < 0) return irq; + pci->dbi_base2 = pci->dbi_base + PCIE_TYPE0_HDR_DBI2_OFFSET; + ret = rockchip_pcie_init_irq_domain(rockchip); if (ret < 0) dev_err(dev, "failed to init irq domain\n"); @@ -264,8 +324,13 @@ static int rockchip_pcie_host_init(struct dw_pcie_rp *pp) irq_set_chained_handler_and_data(irq, rockchip_pcie_intx_handler, rockchip); + rockchip_pcie_configure_l1ss(pci); rockchip_pcie_enable_l0s(pci); + /* Disable Root Ports BAR0 and BAR1 as they report bogus size */ + dw_pcie_writel_dbi2(pci, PCI_BASE_ADDRESS_0, 0x0); + dw_pcie_writel_dbi2(pci, PCI_BASE_ADDRESS_1, 0x0); + return 0; } @@ -291,9 +356,7 @@ static void rockchip_pcie_ep_hide_broken_ats_cap_rk3588(struct dw_pcie_ep *ep) if (!of_device_is_compatible(dev->of_node, "rockchip,rk3588-pcie-ep")) return; - if (dw_pcie_ep_hide_ext_capability(pci, PCI_EXT_CAP_ID_SECPCI, - PCI_EXT_CAP_ID_ATS)) - dev_err(dev, "failed to hide ATS capability\n"); + dw_pcie_remove_ext_capability(pci, PCI_EXT_CAP_ID_ATS); } static void rockchip_pcie_ep_init(struct dw_pcie_ep *ep) @@ -412,6 +475,9 @@ static int rockchip_pcie_resource_get(struct platform_device *pdev, return dev_err_probe(&pdev->dev, PTR_ERR(rockchip->rst), "failed to get reset lines\n"); + rockchip->supports_clkreq = of_property_read_bool(pdev->dev.of_node, + "supports-clkreq"); + return 0; } @@ -446,36 +512,9 @@ static const struct dw_pcie_ops dw_pcie_ops = { .link_up = rockchip_pcie_link_up, .start_link = rockchip_pcie_start_link, .stop_link = rockchip_pcie_stop_link, + .get_ltssm = rockchip_pcie_get_ltssm, }; -static irqreturn_t rockchip_pcie_rc_sys_irq_thread(int irq, void *arg) -{ - struct rockchip_pcie *rockchip = arg; - struct dw_pcie *pci = &rockchip->pci; - struct dw_pcie_rp *pp = &pci->pp; - struct device *dev = pci->dev; - u32 reg; - - reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_MISC); - rockchip_pcie_writel_apb(rockchip, reg, PCIE_CLIENT_INTR_STATUS_MISC); - - dev_dbg(dev, "PCIE_CLIENT_INTR_STATUS_MISC: %#x\n", reg); - dev_dbg(dev, "LTSSM_STATUS: %#x\n", rockchip_pcie_get_ltssm(rockchip)); - - if (reg & PCIE_RDLH_LINK_UP_CHGED) { - if (rockchip_pcie_link_up(pci)) { - msleep(PCIE_RESET_CONFIG_WAIT_MS); - dev_dbg(dev, "Received Link up event. Starting enumeration!\n"); - /* Rescan the bus to enumerate endpoint devices */ - pci_lock_rescan_remove(); - pci_rescan_bus(pp->bridge->bus); - pci_unlock_rescan_remove(); - } - } - - return IRQ_HANDLED; -} - static irqreturn_t rockchip_pcie_ep_sys_irq_thread(int irq, void *arg) { struct rockchip_pcie *rockchip = arg; @@ -487,7 +526,7 @@ static irqreturn_t rockchip_pcie_ep_sys_irq_thread(int irq, void *arg) rockchip_pcie_writel_apb(rockchip, reg, PCIE_CLIENT_INTR_STATUS_MISC); dev_dbg(dev, "PCIE_CLIENT_INTR_STATUS_MISC: %#x\n", reg); - dev_dbg(dev, "LTSSM_STATUS: %#x\n", rockchip_pcie_get_ltssm(rockchip)); + dev_dbg(dev, "LTSSM_STATUS: %#x\n", rockchip_pcie_get_ltssm_reg(rockchip)); if (reg & PCIE_LINK_REQ_RST_NOT_INT) { dev_dbg(dev, "hot reset or link-down reset\n"); @@ -508,29 +547,14 @@ static irqreturn_t rockchip_pcie_ep_sys_irq_thread(int irq, void *arg) return IRQ_HANDLED; } -static int rockchip_pcie_configure_rc(struct platform_device *pdev, - struct rockchip_pcie *rockchip) +static int rockchip_pcie_configure_rc(struct rockchip_pcie *rockchip) { - struct device *dev = &pdev->dev; struct dw_pcie_rp *pp; - int irq, ret; u32 val; if (!IS_ENABLED(CONFIG_PCIE_ROCKCHIP_DW_HOST)) return -ENODEV; - irq = platform_get_irq_byname(pdev, "sys"); - if (irq < 0) - return irq; - - ret = devm_request_threaded_irq(dev, irq, NULL, - rockchip_pcie_rc_sys_irq_thread, - IRQF_ONESHOT, "pcie-sys-rc", rockchip); - if (ret) { - dev_err(dev, "failed to request PCIe sys IRQ\n"); - return ret; - } - /* LTSSM enable control mode */ val = FIELD_PREP_WM16(PCIE_LTSSM_ENABLE_ENHANCE, 1); rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_HOT_RESET_CTRL); @@ -541,19 +565,8 @@ static int rockchip_pcie_configure_rc(struct platform_device *pdev, pp = &rockchip->pci.pp; pp->ops = &rockchip_pcie_host_ops; - pp->use_linkup_irq = true; - - ret = dw_pcie_host_init(pp); - if (ret) { - dev_err(dev, "failed to initialize host\n"); - return ret; - } - - /* unmask DLL up/down indicator */ - val = FIELD_PREP_WM16(PCIE_RDLH_LINK_UP_CHGED, 0); - rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_INTR_MASK_MISC); - return ret; + return dw_pcie_host_init(pp); } static int rockchip_pcie_configure_ep(struct platform_device *pdev, @@ -679,7 +692,7 @@ static int rockchip_pcie_probe(struct platform_device *pdev) switch (data->mode) { case DW_PCIE_RC_TYPE: - ret = rockchip_pcie_configure_rc(pdev, rockchip); + ret = rockchip_pcie_configure_rc(rockchip); if (ret) goto deinit_clk; break; diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index ec6a7090e4250..83e973623f55b 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -57,9 +57,6 @@ #define PARF_AXI_MSTR_WR_ADDR_HALT_V2 0x1a8 #define PARF_Q2A_FLUSH 0x1ac #define PARF_LTSSM 0x1b0 -#define PARF_INT_ALL_STATUS 0x224 -#define PARF_INT_ALL_CLEAR 0x228 -#define PARF_INT_ALL_MASK 0x22c #define PARF_SID_OFFSET 0x234 #define PARF_BDF_TRANSLATE_CFG 0x24c #define PARF_DBI_BASE_ADDR_V2 0x350 @@ -137,10 +134,6 @@ #define LTSSM_EN BIT(8) #define PARF_LTSSM_STATE_MASK GENMASK(5, 0) -/* PARF_INT_ALL_{STATUS/CLEAR/MASK} register fields */ -#define PARF_INT_ALL_LINK_UP BIT(13) -#define PARF_INT_MSI_DEV_0_7 GENMASK(30, 23) - /* PARF_NO_SNOOP_OVERRIDE register fields */ #define WR_NO_SNOOP_OVERRIDE_EN BIT(1) #define RD_NO_SNOOP_OVERRIDE_EN BIT(3) @@ -1015,6 +1008,8 @@ static int qcom_pcie_init_2_7_0(struct qcom_pcie *pcie) val &= ~REQ_NOT_ENTR_L1; writel(val, pcie->parf + PARF_PM_CTRL); + pci->l1ss_support = true; + val = readl(pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT_V2); val |= EN; writel(val, pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT_V2); @@ -1653,32 +1648,6 @@ static void qcom_pcie_init_debugfs(struct qcom_pcie *pcie) qcom_pcie_link_transition_count); } -static irqreturn_t qcom_pcie_global_irq_thread(int irq, void *data) -{ - struct qcom_pcie *pcie = data; - struct dw_pcie_rp *pp = &pcie->pci->pp; - struct device *dev = pcie->pci->dev; - u32 status = readl_relaxed(pcie->parf + PARF_INT_ALL_STATUS); - - writel_relaxed(status, pcie->parf + PARF_INT_ALL_CLEAR); - - if (FIELD_GET(PARF_INT_ALL_LINK_UP, status)) { - msleep(PCIE_RESET_CONFIG_WAIT_MS); - dev_dbg(dev, "Received Link up event. Starting enumeration!\n"); - /* Rescan the bus to enumerate endpoint devices */ - pci_lock_rescan_remove(); - pci_rescan_bus(pp->bridge->bus); - pci_unlock_rescan_remove(); - - qcom_pcie_icc_opp_update(pcie); - } else { - dev_WARN_ONCE(dev, 1, "Received unknown event. INT_STATUS: 0x%08x\n", - status); - } - - return IRQ_HANDLED; -} - static void qcom_pci_free_msi(void *ptr) { struct dw_pcie_rp *pp = (struct dw_pcie_rp *)ptr; @@ -1823,8 +1792,7 @@ static int qcom_pcie_probe(struct platform_device *pdev) struct dw_pcie_rp *pp; struct resource *res; struct dw_pcie *pci; - int ret, irq; - char *name; + int ret; pcie_cfg = of_device_get_match_data(dev); if (!pcie_cfg) { @@ -1975,37 +1943,12 @@ static int qcom_pcie_probe(struct platform_device *pdev) platform_set_drvdata(pdev, pcie); - irq = platform_get_irq_byname_optional(pdev, "global"); - if (irq > 0) - pp->use_linkup_irq = true; - ret = dw_pcie_host_init(pp); if (ret) { dev_err_probe(dev, ret, "cannot initialize host\n"); goto err_phy_exit; } - name = devm_kasprintf(dev, GFP_KERNEL, "qcom_pcie_global_irq%d", - pci_domain_nr(pp->bridge->bus)); - if (!name) { - ret = -ENOMEM; - goto err_host_deinit; - } - - if (irq > 0) { - ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, - qcom_pcie_global_irq_thread, - IRQF_ONESHOT, name, pcie); - if (ret) { - dev_err_probe(&pdev->dev, ret, - "Failed to request Global IRQ\n"); - goto err_host_deinit; - } - - writel_relaxed(PARF_INT_ALL_LINK_UP | PARF_INT_MSI_DEV_0_7, - pcie->parf + PARF_INT_ALL_MASK); - } - qcom_pcie_icc_opp_update(pcie); if (pcie->mhi) @@ -2013,8 +1956,6 @@ static int qcom_pcie_probe(struct platform_device *pdev) return 0; -err_host_deinit: - dw_pcie_host_deinit(pp); err_phy_exit: list_for_each_entry_safe(port, tmp, &pcie->ports, list) { phy_exit(port->phy); diff --git a/drivers/pci/controller/dwc/pcie-sophgo.c b/drivers/pci/controller/dwc/pcie-sophgo.c index ad4baaa34ffa1..044088898819e 100644 --- a/drivers/pci/controller/dwc/pcie-sophgo.c +++ b/drivers/pci/controller/dwc/pcie-sophgo.c @@ -161,6 +161,22 @@ static void sophgo_pcie_msi_enable(struct dw_pcie_rp *pp) raw_spin_unlock_irqrestore(&pp->lock, flags); } +static void sophgo_pcie_disable_l0s_l1(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + u32 offset, val; + + offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + + dw_pcie_dbi_ro_wr_en(pci); + + val = dw_pcie_readl_dbi(pci, PCI_EXP_LNKCAP + offset); + val &= ~(PCI_EXP_LNKCAP_ASPM_L0S | PCI_EXP_LNKCAP_ASPM_L1); + dw_pcie_writel_dbi(pci, PCI_EXP_LNKCAP + offset, val); + + dw_pcie_dbi_ro_wr_dis(pci); +} + static int sophgo_pcie_host_init(struct dw_pcie_rp *pp) { int irq; @@ -171,6 +187,8 @@ static int sophgo_pcie_host_init(struct dw_pcie_rp *pp) irq_set_chained_handler_and_data(irq, sophgo_pcie_intx_handler, pp); + sophgo_pcie_disable_l0s_l1(pp); + sophgo_pcie_msi_enable(pp); return 0; diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c index 10e74458e667b..3934757baa30c 100644 --- a/drivers/pci/controller/dwc/pcie-tegra194.c +++ b/drivers/pci/controller/dwc/pcie-tegra194.c @@ -703,6 +703,9 @@ static void init_host_aspm(struct tegra_pcie_dw *pcie) val |= (pcie->aspm_pwr_on_t << 19); dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val); + if (pcie->supports_clkreq) + pci->l1ss_support = true; + /* Program L0s and L1 entrance latencies */ val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR); val &= ~PORT_AFR_L0S_ENTRANCE_LAT_MASK; diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c index 24cc30a2ab6c6..e0bf667c2b4c6 100644 --- a/drivers/pci/controller/pcie-mediatek.c +++ b/drivers/pci/controller/pcie-mediatek.c @@ -575,8 +575,10 @@ static int mtk_pcie_init_irq_domain(struct mtk_pcie_port *port, if (IS_ENABLED(CONFIG_PCI_MSI)) { ret = mtk_pcie_allocate_msi_domains(port); - if (ret) + if (ret) { + irq_domain_remove(port->irq_domain); return ret; + } } return 0; diff --git a/drivers/pci/controller/pcie-xilinx.c b/drivers/pci/controller/pcie-xilinx.c index 937ea6ae1ac48..4aa139abac16e 100644 --- a/drivers/pci/controller/pcie-xilinx.c +++ b/drivers/pci/controller/pcie-xilinx.c @@ -302,9 +302,10 @@ static int xilinx_allocate_msi_domains(struct xilinx_pcie *pcie) return 0; } -static void xilinx_free_msi_domains(struct xilinx_pcie *pcie) +static void xilinx_free_irq_domains(struct xilinx_pcie *pcie) { irq_domain_remove(pcie->msi_domain); + irq_domain_remove(pcie->leg_domain); } /* INTx Functions */ @@ -480,8 +481,10 @@ static int xilinx_pcie_init_irq_domain(struct xilinx_pcie *pcie) phys_addr_t pa = ALIGN_DOWN(virt_to_phys(pcie), SZ_4K); ret = xilinx_allocate_msi_domains(pcie); - if (ret) + if (ret) { + irq_domain_remove(pcie->leg_domain); return ret; + } pcie_write(pcie, upper_32_bits(pa), XILINX_PCIE_REG_MSIBASE1); pcie_write(pcie, lower_32_bits(pa), XILINX_PCIE_REG_MSIBASE2); @@ -600,7 +603,7 @@ static int xilinx_pcie_probe(struct platform_device *pdev) err = pci_host_probe(bridge); if (err) - xilinx_free_msi_domains(pcie); + xilinx_free_irq_domains(pcie); return err; } diff --git a/drivers/pci/endpoint/pci-ep-cfs.c b/drivers/pci/endpoint/pci-ep-cfs.c index 43feb6139fa36..8b392a8363bb1 100644 --- a/drivers/pci/endpoint/pci-ep-cfs.c +++ b/drivers/pci/endpoint/pci-ep-cfs.c @@ -68,8 +68,8 @@ static int pci_secondary_epc_epf_link(struct config_item *epf_item, return 0; } -static void pci_secondary_epc_epf_unlink(struct config_item *epc_item, - struct config_item *epf_item) +static void pci_secondary_epc_epf_unlink(struct config_item *epf_item, + struct config_item *epc_item) { struct pci_epf_group *epf_group = to_pci_epf_group(epf_item->ci_parent); struct pci_epc_group *epc_group = to_pci_epc_group(epc_item); @@ -132,8 +132,8 @@ static int pci_primary_epc_epf_link(struct config_item *epf_item, return 0; } -static void pci_primary_epc_epf_unlink(struct config_item *epc_item, - struct config_item *epf_item) +static void pci_primary_epc_epf_unlink(struct config_item *epf_item, + struct config_item *epc_item) { struct pci_epf_group *epf_group = to_pci_epf_group(epf_item->ci_parent); struct pci_epc_group *epc_group = to_pci_epc_group(epc_item); diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c index 77dee43b78583..c6dc1b44bf602 100644 --- a/drivers/pci/iov.c +++ b/drivers/pci/iov.c @@ -495,7 +495,9 @@ static ssize_t sriov_numvfs_store(struct device *dev, if (num_vfs == 0) { /* disable VFs */ + pci_lock_rescan_remove(); ret = pdev->driver->sriov_configure(pdev, 0); + pci_unlock_rescan_remove(); goto exit; } @@ -507,7 +509,9 @@ static ssize_t sriov_numvfs_store(struct device *dev, goto exit; } + pci_lock_rescan_remove(); ret = pdev->driver->sriov_configure(pdev, num_vfs); + pci_unlock_rescan_remove(); if (ret < 0) goto exit; @@ -629,18 +633,15 @@ static int sriov_add_vfs(struct pci_dev *dev, u16 num_vfs) if (dev->no_vf_scan) return 0; - pci_lock_rescan_remove(); for (i = 0; i < num_vfs; i++) { rc = pci_iov_add_virtfn(dev, i); if (rc) goto failed; } - pci_unlock_rescan_remove(); return 0; failed: while (i--) pci_iov_remove_virtfn(dev, i); - pci_unlock_rescan_remove(); return rc; } @@ -765,10 +766,8 @@ static void sriov_del_vfs(struct pci_dev *dev) struct pci_sriov *iov = dev->sriov; int i; - pci_lock_rescan_remove(); for (i = 0; i < iov->num_VFs; i++) pci_iov_remove_virtfn(dev, i); - pci_unlock_rescan_remove(); } static void sriov_disable(struct pci_dev *dev) diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index 34d664139f48f..e010ecd9f90dd 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -737,7 +737,7 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, ret = msix_setup_interrupts(dev, entries, nvec, affd); if (ret) - goto out_disable; + goto out_unmap; /* Disable INTX */ pci_intx_for_msi(dev, 0); @@ -758,6 +758,8 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, pcibios_free_irq(dev); return 0; +out_unmap: + iounmap(dev->msix_base); out_disable: dev->msix_enabled = 0; pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE, 0); diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c index 78e108e47254a..49e395eb013d3 100644 --- a/drivers/pci/p2pdma.c +++ b/drivers/pci/p2pdma.c @@ -147,11 +147,19 @@ static int p2pmem_alloc_mmap(struct file *filp, struct kobject *kobj, * we have just allocated the page no one else should be * using it. */ - VM_WARN_ON_ONCE_PAGE(!page_ref_count(page), page); + VM_WARN_ON_ONCE_PAGE(page_ref_count(page), page); set_page_count(page, 1); ret = vm_insert_page(vma, vaddr, page); if (ret) { gen_pool_free(p2pdma->pool, (uintptr_t)kaddr, len); + + /* + * Reset the page count. We don't use put_page() + * because we don't want to trigger the + * p2pdma_folio_free() path. + */ + set_page_count(page, 0); + percpu_ref_put(ref); return ret; } percpu_ref_get(ref); diff --git a/drivers/pci/pci-acpi.c b/drivers/pci/pci-acpi.c index 9369377725fa0..0162acfb57896 100644 --- a/drivers/pci/pci-acpi.c +++ b/drivers/pci/pci-acpi.c @@ -271,21 +271,6 @@ static acpi_status decode_type1_hpx_record(union acpi_object *record, return AE_OK; } -static bool pcie_root_rcb_set(struct pci_dev *dev) -{ - struct pci_dev *rp = pcie_find_root_port(dev); - u16 lnkctl; - - if (!rp) - return false; - - pcie_capability_read_word(rp, PCI_EXP_LNKCTL, &lnkctl); - if (lnkctl & PCI_EXP_LNKCTL_RCB) - return true; - - return false; -} - /* _HPX PCI Express Setting Record (Type 2) */ struct hpx_type2 { u32 revision; @@ -311,6 +296,7 @@ static void program_hpx_type2(struct pci_dev *dev, struct hpx_type2 *hpx) { int pos; u32 reg32; + const struct pci_host_bridge *host; if (!hpx) return; @@ -318,6 +304,15 @@ static void program_hpx_type2(struct pci_dev *dev, struct hpx_type2 *hpx) if (!pci_is_pcie(dev)) return; + host = pci_find_host_bridge(dev->bus); + + /* + * Only do the _HPX Type 2 programming if OS owns PCIe native + * hotplug but not AER. + */ + if (!host->native_pcie_hotplug || host->native_aer) + return; + if (hpx->revision > 1) { pci_warn(dev, "PCIe settings rev %d not supported\n", hpx->revision); @@ -325,33 +320,27 @@ static void program_hpx_type2(struct pci_dev *dev, struct hpx_type2 *hpx) } /* - * Don't allow _HPX to change MPS or MRRS settings. We manage - * those to make sure they're consistent with the rest of the - * platform. + * We only allow _HPX to program DEVCTL bits related to AER, namely + * PCI_EXP_DEVCTL_CERE, PCI_EXP_DEVCTL_NFERE, PCI_EXP_DEVCTL_FERE, + * and PCI_EXP_DEVCTL_URRE. + * + * The rest of DEVCTL is managed by the OS to make sure it's + * consistent with the rest of the platform. */ - hpx->pci_exp_devctl_and |= PCI_EXP_DEVCTL_PAYLOAD | - PCI_EXP_DEVCTL_READRQ; - hpx->pci_exp_devctl_or &= ~(PCI_EXP_DEVCTL_PAYLOAD | - PCI_EXP_DEVCTL_READRQ); + hpx->pci_exp_devctl_and |= ~PCI_EXP_AER_FLAGS; + hpx->pci_exp_devctl_or &= PCI_EXP_AER_FLAGS; /* Initialize Device Control Register */ pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL, ~hpx->pci_exp_devctl_and, hpx->pci_exp_devctl_or); - /* Initialize Link Control Register */ + /* Log if _HPX attempts to modify Link Control Register */ if (pcie_cap_has_lnkctl(dev)) { - - /* - * If the Root Port supports Read Completion Boundary of - * 128, set RCB to 128. Otherwise, clear it. - */ - hpx->pci_exp_lnkctl_and |= PCI_EXP_LNKCTL_RCB; - hpx->pci_exp_lnkctl_or &= ~PCI_EXP_LNKCTL_RCB; - if (pcie_root_rcb_set(dev)) - hpx->pci_exp_lnkctl_or |= PCI_EXP_LNKCTL_RCB; - - pcie_capability_clear_and_set_word(dev, PCI_EXP_LNKCTL, - ~hpx->pci_exp_lnkctl_and, hpx->pci_exp_lnkctl_or); + if (hpx->pci_exp_lnkctl_and != 0xffff || + hpx->pci_exp_lnkctl_or != 0) + pci_info(dev, "_HPX attempts Link Control setting (AND %#06x OR %#06x)\n", + hpx->pci_exp_lnkctl_and, + hpx->pci_exp_lnkctl_or); } /* Find Advanced Error Reporting Enhanced Capability */ diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c index 327b21c48614d..b4111c92c9572 100644 --- a/drivers/pci/pci-driver.c +++ b/drivers/pci/pci-driver.c @@ -1652,6 +1652,14 @@ static int pci_dma_configure(struct device *dev) ret = acpi_dma_configure(dev, acpi_get_dma_attr(adev)); } + /* + * Attempt to enable ACS regardless of capability because some Root + * Ports (e.g. those quirked with *_intel_pch_acs_*) do not have + * the standard ACS capability but still support ACS via those + * quirks. + */ + pci_enable_acs(to_pci_dev(dev)); + pci_put_host_bridge_device(bridge); /* @drv may not be valid when we're called from the IOMMU layer */ diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 2f0da5dbbba40..e128696d5b761 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -426,7 +426,7 @@ static int pci_dev_str_match(struct pci_dev *dev, const char *p, static u8 __pci_find_next_cap(struct pci_bus *bus, unsigned int devfn, u8 pos, int cap) { - return PCI_FIND_NEXT_CAP(pci_bus_read_config, pos, cap, bus, devfn); + return PCI_FIND_NEXT_CAP(pci_bus_read_config, pos, cap, NULL, bus, devfn); } u8 pci_find_next_capability(struct pci_dev *dev, u8 pos, int cap) @@ -531,7 +531,7 @@ u16 pci_find_next_ext_capability(struct pci_dev *dev, u16 start, int cap) return 0; return PCI_FIND_NEXT_EXT_CAP(pci_bus_read_config, start, cap, - dev->bus, dev->devfn); + NULL, dev->bus, dev->devfn); } EXPORT_SYMBOL_GPL(pci_find_next_ext_capability); @@ -600,7 +600,7 @@ static u8 __pci_find_next_ht_cap(struct pci_dev *dev, u8 pos, int ht_cap) mask = HT_5BIT_CAP_MASK; pos = PCI_FIND_NEXT_CAP(pci_bus_read_config, pos, - PCI_CAP_ID_HT, dev->bus, dev->devfn); + PCI_CAP_ID_HT, NULL, dev->bus, dev->devfn); while (pos) { rc = pci_read_config_byte(dev, pos + 3, &cap); if (rc != PCIBIOS_SUCCESSFUL) @@ -611,7 +611,7 @@ static u8 __pci_find_next_ht_cap(struct pci_dev *dev, u8 pos, int ht_cap) pos = PCI_FIND_NEXT_CAP(pci_bus_read_config, pos + PCI_CAP_LIST_NEXT, - PCI_CAP_ID_HT, dev->bus, + PCI_CAP_ID_HT, NULL, dev->bus, dev->devfn); } @@ -1015,7 +1015,7 @@ static void pci_std_enable_acs(struct pci_dev *dev, struct pci_acs *caps) * pci_enable_acs - enable ACS if hardware support it * @dev: the PCI device */ -static void pci_enable_acs(struct pci_dev *dev) +void pci_enable_acs(struct pci_dev *dev) { struct pci_acs caps; bool enable_acs = false; @@ -1488,6 +1488,9 @@ static int pci_set_low_power_state(struct pci_dev *dev, pci_power_t state, bool || (state == PCI_D2 && !dev->d2_support)) return -EIO; + if (dev->current_state == state) + return 0; + pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); if (PCI_POSSIBLE_ERROR(pmcsr)) { pci_err(dev, "Unable to change power state from %s to %s, device inaccessible\n", @@ -3222,8 +3225,14 @@ void pci_pm_init(struct pci_dev *dev) poweron: pci_pm_power_up_and_verify_state(dev); pm_runtime_forbid(&dev->dev); + + /* + * Runtime PM will be enabled for the device when it has been fully + * configured, but since its parent and suppliers may suspend in + * the meantime, prevent them from doing so by changing the + * device's runtime PM status to "active". + */ pm_runtime_set_active(&dev->dev); - pm_runtime_enable(&dev->dev); } static unsigned long pci_ea_flags(struct pci_dev *dev, u8 prop) @@ -3674,14 +3683,6 @@ bool pci_acs_path_enabled(struct pci_dev *start, void pci_acs_init(struct pci_dev *dev) { dev->acs_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS); - - /* - * Attempt to enable ACS regardless of capability because some Root - * Ports (e.g. those quirked with *_intel_pch_acs_*) do not have - * the standard ACS capability but still support ACS via those - * quirks. - */ - pci_enable_acs(dev); } void pci_rebar_init(struct pci_dev *pdev) @@ -5435,10 +5436,9 @@ static int pci_bus_trylock(struct pci_bus *bus) /* Do any devices on or below this slot prevent a bus reset? */ static bool pci_slot_resettable(struct pci_slot *slot) { - struct pci_dev *dev; + struct pci_dev *dev, *bridge = slot->bus->self; - if (slot->bus->self && - (slot->bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)) + if (bridge && (bridge->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)) return false; list_for_each_entry(dev, &slot->bus->devices, bus_list) { @@ -5455,7 +5455,10 @@ static bool pci_slot_resettable(struct pci_slot *slot) /* Lock devices from the top of the tree down */ static void pci_slot_lock(struct pci_slot *slot) { - struct pci_dev *dev; + struct pci_dev *dev, *bridge = slot->bus->self; + + if (bridge) + pci_dev_lock(bridge); list_for_each_entry(dev, &slot->bus->devices, bus_list) { if (!dev->slot || dev->slot != slot) @@ -5470,7 +5473,7 @@ static void pci_slot_lock(struct pci_slot *slot) /* Unlock devices from the bottom of the tree up */ static void pci_slot_unlock(struct pci_slot *slot) { - struct pci_dev *dev; + struct pci_dev *dev, *bridge = slot->bus->self; list_for_each_entry(dev, &slot->bus->devices, bus_list) { if (!dev->slot || dev->slot != slot) @@ -5480,21 +5483,25 @@ static void pci_slot_unlock(struct pci_slot *slot) else pci_dev_unlock(dev); } + + if (bridge) + pci_dev_unlock(bridge); } /* Return 1 on successful lock, 0 on contention */ static int pci_slot_trylock(struct pci_slot *slot) { - struct pci_dev *dev; + struct pci_dev *dev, *bridge = slot->bus->self; + + if (bridge && !pci_dev_trylock(bridge)) + return 0; list_for_each_entry(dev, &slot->bus->devices, bus_list) { if (!dev->slot || dev->slot != slot) continue; if (dev->subordinate) { - if (!pci_bus_trylock(dev->subordinate)) { - pci_dev_unlock(dev); + if (!pci_bus_trylock(dev->subordinate)) goto unlock; - } } else if (!pci_dev_trylock(dev)) goto unlock; } @@ -5510,6 +5517,9 @@ static int pci_slot_trylock(struct pci_slot *slot) else pci_dev_unlock(dev); } + + if (bridge) + pci_dev_unlock(bridge); return 0; } @@ -6714,7 +6724,7 @@ static void of_pci_bus_release_domain_nr(struct device *parent, int domain_nr) return; /* Release domain from IDA where it was allocated. */ - if (of_get_pci_domain_nr(parent->of_node) == domain_nr) + if (parent && of_get_pci_domain_nr(parent->of_node) == domain_nr) ida_free(&pci_domain_nr_static_ida, domain_nr); else ida_free(&pci_domain_nr_dynamic_ida, domain_nr); diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 36f8c0985430a..5510c103be2d3 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -88,6 +88,9 @@ struct pcie_tlp_log; #define PCI_BUS_BRIDGE_MEM_WINDOW 1 #define PCI_BUS_BRIDGE_PREF_MEM_WINDOW 2 +#define PCI_EXP_AER_FLAGS (PCI_EXP_DEVCTL_CERE | PCI_EXP_DEVCTL_NFERE | \ + PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE) + extern const unsigned char pcie_link_speed[]; extern bool pci_early_dump; @@ -103,17 +106,21 @@ bool pcie_cap_has_rtctl(const struct pci_dev *dev); * @read_cfg: Function pointer for reading PCI config space * @start: Starting position to begin search * @cap: Capability ID to find + * @prev_ptr: Pointer to store position of preceding capability (optional) * @args: Arguments to pass to read_cfg function * - * Search the capability list in PCI config space to find @cap. + * Search the capability list in PCI config space to find @cap. If + * found, update *prev_ptr with the position of the preceding capability + * (if prev_ptr != NULL) * Implements TTL (time-to-live) protection against infinite loops. * * Return: Position of the capability if found, 0 otherwise. */ -#define PCI_FIND_NEXT_CAP(read_cfg, start, cap, args...) \ +#define PCI_FIND_NEXT_CAP(read_cfg, start, cap, prev_ptr, args...) \ ({ \ int __ttl = PCI_FIND_CAP_TTL; \ - u8 __id, __found_pos = 0; \ + u8 __id, __found_pos = 0; \ + u8 __prev_pos = (start); \ u8 __pos = (start); \ u16 __ent; \ \ @@ -132,9 +139,12 @@ bool pcie_cap_has_rtctl(const struct pci_dev *dev); \ if (__id == (cap)) { \ __found_pos = __pos; \ + if (prev_ptr != NULL) \ + *(u8 *)prev_ptr = __prev_pos; \ break; \ } \ \ + __prev_pos = __pos; \ __pos = FIELD_GET(PCI_CAP_LIST_NEXT_MASK, __ent); \ } \ __found_pos; \ @@ -146,21 +156,26 @@ bool pcie_cap_has_rtctl(const struct pci_dev *dev); * @read_cfg: Function pointer for reading PCI config space * @start: Starting position to begin search (0 for initial search) * @cap: Extended capability ID to find + * @prev_ptr: Pointer to store position of preceding capability (optional) * @args: Arguments to pass to read_cfg function * * Search the extended capability list in PCI config space to find @cap. + * If found, update *prev_ptr with the position of the preceding capability + * (if prev_ptr != NULL) * Implements TTL protection against infinite loops using a calculated * maximum search count. * * Return: Position of the capability if found, 0 otherwise. */ -#define PCI_FIND_NEXT_EXT_CAP(read_cfg, start, cap, args...) \ +#define PCI_FIND_NEXT_EXT_CAP(read_cfg, start, cap, prev_ptr, args...) \ ({ \ u16 __pos = (start) ?: PCI_CFG_SPACE_SIZE; \ u16 __found_pos = 0; \ + u16 __prev_pos; \ int __ttl, __ret; \ u32 __header; \ \ + __prev_pos = __pos; \ __ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8; \ while (__ttl-- > 0 && __pos >= PCI_CFG_SPACE_SIZE) { \ __ret = read_cfg##_dword(args, __pos, &__header); \ @@ -172,9 +187,12 @@ bool pcie_cap_has_rtctl(const struct pci_dev *dev); \ if (PCI_EXT_CAP_ID(__header) == (cap) && __pos != start) {\ __found_pos = __pos; \ + if (prev_ptr != NULL) \ + *(u16 *)prev_ptr = __prev_pos; \ break; \ } \ \ + __prev_pos = __pos; \ __pos = PCI_EXT_CAP_NEXT(__header); \ } \ __found_pos; \ @@ -917,6 +935,7 @@ static inline resource_size_t pci_resource_alignment(struct pci_dev *dev, } void pci_acs_init(struct pci_dev *dev); +void pci_enable_acs(struct pci_dev *dev); #ifdef CONFIG_PCI_QUIRKS int pci_dev_specific_acs_enabled(struct pci_dev *dev, u16 acs_flags); int pci_dev_specific_enable_acs(struct pci_dev *dev); diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c index 0b5ed4722ac32..29504173425cd 100644 --- a/drivers/pci/pcie/aer.c +++ b/drivers/pci/pcie/aer.c @@ -238,9 +238,6 @@ void pcie_ecrc_get_policy(char *str) } #endif /* CONFIG_PCIE_ECRC */ -#define PCI_EXP_AER_FLAGS (PCI_EXP_DEVCTL_CERE | PCI_EXP_DEVCTL_NFERE | \ - PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE) - int pcie_aer_is_native(struct pci_dev *dev) { struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); @@ -1606,6 +1603,20 @@ static void aer_disable_irq(struct pci_dev *pdev) pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, reg32); } +static int clear_status_iter(struct pci_dev *dev, void *data) +{ + u16 devctl; + + /* Skip if pci_enable_pcie_error_reporting() hasn't been called yet */ + pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &devctl); + if (!(devctl & PCI_EXP_AER_FLAGS)) + return 0; + + pci_aer_clear_status(dev); + pcie_clear_device_status(dev); + return 0; +} + /** * aer_enable_rootport - enable Root Port's interrupts when receiving messages * @rpc: pointer to a Root Port data structure @@ -1627,9 +1638,19 @@ static void aer_enable_rootport(struct aer_rpc *rpc) pcie_capability_clear_word(pdev, PCI_EXP_RTCTL, SYSTEM_ERROR_INTR_ON_MESG_MASK); - /* Clear error status */ + /* Clear error status of this Root Port or RCEC */ pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, ®32); pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, reg32); + + /* Clear error status of agents reporting to this Root Port or RCEC */ + if (reg32 & AER_ERR_STATUS_MASK) { + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_RC_EC) + pcie_walk_rcec(pdev, clear_status_iter, NULL); + else if (pdev->subordinate) + pci_walk_bus(pdev->subordinate, clear_status_iter, + NULL); + } + pci_read_config_dword(pdev, aer + PCI_ERR_COR_STATUS, ®32); pci_write_config_dword(pdev, aer + PCI_ERR_COR_STATUS, reg32); pci_read_config_dword(pdev, aer + PCI_ERR_UNCOR_STATUS, ®32); diff --git a/drivers/pci/pcie/bwctrl.c b/drivers/pci/pcie/bwctrl.c index 36f939f23d34e..4ae92c9f912a8 100644 --- a/drivers/pci/pcie/bwctrl.c +++ b/drivers/pci/pcie/bwctrl.c @@ -250,6 +250,9 @@ static int pcie_bwnotif_probe(struct pcie_device *srv) struct pci_dev *port = srv->port; int ret; + if (port->no_bw_notif) + return -ENODEV; + /* Can happen if we run out of bus numbers during enumeration. */ if (!port->subordinate) return -ENODEV; diff --git a/drivers/pci/pcie/portdrv.c b/drivers/pci/pcie/portdrv.c index 38a41ccf79b9a..a0991da482136 100644 --- a/drivers/pci/pcie/portdrv.c +++ b/drivers/pci/pcie/portdrv.c @@ -557,10 +557,10 @@ static int pcie_port_remove_service(struct device *dev) pciedev = to_pcie_device(dev); driver = to_service_driver(dev->driver); - if (driver && driver->remove) { + if (driver && driver->remove) driver->remove(pciedev); - put_device(dev); - } + + put_device(dev); return 0; } diff --git a/drivers/pci/pcie/ptm.c b/drivers/pci/pcie/ptm.c index 65e4b008be00d..41d370f082ee4 100644 --- a/drivers/pci/pcie/ptm.c +++ b/drivers/pci/pcie/ptm.c @@ -519,8 +519,10 @@ struct pci_ptm_debugfs *pcie_ptm_create_debugfs(struct device *dev, void *pdata, return NULL; dirname = devm_kasprintf(dev, GFP_KERNEL, "pcie_ptm_%s", dev_name(dev)); - if (!dirname) + if (!dirname) { + kfree(ptm_debugfs); return NULL; + } ptm_debugfs->debugfs = debugfs_create_dir(dirname, NULL); ptm_debugfs->pdata = pdata; @@ -551,6 +553,7 @@ void pcie_ptm_destroy_debugfs(struct pci_ptm_debugfs *ptm_debugfs) mutex_destroy(&ptm_debugfs->lock); debugfs_remove_recursive(ptm_debugfs->debugfs); + kfree(ptm_debugfs); } EXPORT_SYMBOL_GPL(pcie_ptm_destroy_debugfs); #endif diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 4ba6b7b5ebfef..fb0eb6b993ce1 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -287,8 +287,7 @@ int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type, if ((sizeof(pci_bus_addr_t) < 8 || sizeof(resource_size_t) < 8) && sz64 > 0x100000000ULL) { res->flags |= IORESOURCE_UNSET | IORESOURCE_DISABLED; - res->start = 0; - res->end = 0; + resource_set_range(res, 0, 0); pci_err(dev, "%s: can't handle BAR larger than 4GB (size %#010llx)\n", res_name, (unsigned long long)sz64); goto out; @@ -297,8 +296,7 @@ int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type, if ((sizeof(pci_bus_addr_t) < 8) && l) { /* Above 32-bit boundary; try to reallocate */ res->flags |= IORESOURCE_UNSET; - res->start = 0; - res->end = sz64 - 1; + resource_set_range(res, 0, sz64); pci_info(dev, "%s: can't handle BAR above 4GB (bus address %#010llx)\n", res_name, (unsigned long long)l64); goto out; @@ -2251,7 +2249,8 @@ int pci_configure_extended_tags(struct pci_dev *dev, void *ign) u16 ctl; int ret; - if (!pci_is_pcie(dev)) + /* PCI_EXP_DEVCTL_EXT_TAG is RsvdP in VFs */ + if (!pci_is_pcie(dev) || dev->is_virtfn) return 0; ret = pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap); @@ -2380,6 +2379,37 @@ static void pci_configure_serr(struct pci_dev *dev) } } +static void pci_configure_rcb(struct pci_dev *dev) +{ + struct pci_dev *rp; + u16 rp_lnkctl; + + /* + * Per PCIe r7.0, sec 7.5.3.7, RCB is only meaningful in Root Ports + * (where it is read-only), Endpoints, and Bridges. It may only be + * set for Endpoints and Bridges if it is set in the Root Port. For + * Endpoints, it is 'RsvdP' for Virtual Functions. + */ + if (!pci_is_pcie(dev) || + pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT || + pci_pcie_type(dev) == PCI_EXP_TYPE_UPSTREAM || + pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM || + pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC || + dev->is_virtfn) + return; + + /* Root Port often not visible to virtualized guests */ + rp = pcie_find_root_port(dev); + if (!rp) + return; + + pcie_capability_read_word(rp, PCI_EXP_LNKCTL, &rp_lnkctl); + pcie_capability_clear_and_set_word(dev, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_RCB, + (rp_lnkctl & PCI_EXP_LNKCTL_RCB) ? + PCI_EXP_LNKCTL_RCB : 0); +} + static void pci_configure_device(struct pci_dev *dev) { pci_configure_mps(dev); @@ -2389,6 +2419,7 @@ static void pci_configure_device(struct pci_dev *dev) pci_configure_aspm_l1ss(dev); pci_configure_eetlp_prefix(dev); pci_configure_serr(dev); + pci_configure_rcb(dev); pci_acpi_program_hp_params(dev); } diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 59671b4e1af90..77689f8505e73 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -1359,6 +1359,16 @@ static void quirk_transparent_bridge(struct pci_dev *dev) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82380FB, quirk_transparent_bridge); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_TOSHIBA, 0x605, quirk_transparent_bridge); +/* + * Enabling Link Bandwidth Management Interrupts (BW notifications) can cause + * boot hangs on P45. + */ +static void quirk_p45_bw_notifications(struct pci_dev *dev) +{ + dev->no_bw_notif = 1; +} +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e21, quirk_p45_bw_notifications); + /* * Common misconfiguration of the MediaGX/Geode PCI master that will reduce * PCI bandwidth from 70MB/s to 25MB/s. See the GXM/GXLV/GX1 datasheets @@ -3799,6 +3809,14 @@ static void quirk_no_bus_reset(struct pci_dev *dev) dev->dev_flags |= PCI_DEV_FLAGS_NO_BUS_RESET; } +/* + * After asserting Secondary Bus Reset to downstream devices via a GB10 + * Root Port, the link may not retrain correctly. + * https://lore.kernel.org/r/20251113084441.2124737-1-Johnny-CC.Chang@mediatek.com + */ +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_NVIDIA, 0x22CE, quirk_no_bus_reset); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_NVIDIA, 0x22D0, quirk_no_bus_reset); + /* * Some NVIDIA GPU devices do not work with bus reset, SBR needs to be * prevented for those affected devices. @@ -3842,6 +3860,16 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_CAVIUM, 0xa100, quirk_no_bus_reset); */ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_TI, 0xb005, quirk_no_bus_reset); +/* + * Reports from users making use of PCI device assignment with ASM1164 + * controllers indicate an issue with bus reset where the device fails to + * retrain. The issue appears more common in configurations with multiple + * controllers. The device does indicate PM reset support (NoSoftRst-), + * therefore this still leaves a viable reset method. + * https://forum.proxmox.com/threads/problems-with-pcie-passthrough-with-two-identical-devices.149003/ + */ +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ASMEDIA, 0x1164, quirk_no_bus_reset); + static void quirk_no_pm_reset(struct pci_dev *dev) { /* @@ -5158,6 +5186,10 @@ static const struct pci_dev_acs_enabled { { PCI_VENDOR_ID_QCOM, 0x0401, pci_quirk_qcom_rp_acs }, /* QCOM SA8775P root port */ { PCI_VENDOR_ID_QCOM, 0x0115, pci_quirk_qcom_rp_acs }, + /* QCOM Hamoa root port */ + { PCI_VENDOR_ID_QCOM, 0x0111, pci_quirk_qcom_rp_acs }, + /* QCOM Glymur root port */ + { PCI_VENDOR_ID_QCOM, 0x0120, pci_quirk_qcom_rp_acs }, /* HXT SD4800 root ports. The ACS design is same as QCOM QDF2xxx */ { PCI_VENDOR_ID_HXT, 0x0401, pci_quirk_qcom_rp_acs }, /* Intel PCH root ports */ @@ -5632,6 +5664,7 @@ static void quirk_no_ext_tags(struct pci_dev *pdev) pci_walk_bus(bridge->bus, pci_configure_extended_tags, NULL); } DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_3WARE, 0x1004, quirk_no_ext_tags); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_3WARE, 0x1005, quirk_no_ext_tags); DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SERVERWORKS, 0x0132, quirk_no_ext_tags); DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SERVERWORKS, 0x0140, quirk_no_ext_tags); DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SERVERWORKS, 0x0141, quirk_no_ext_tags); @@ -6239,6 +6272,10 @@ DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_PERICOM, 0x2303, pci_fixup_pericom_acs_store_forward); DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_PERICOM, 0x2303, pci_fixup_pericom_acs_store_forward); +DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_PERICOM, 0xb404, + pci_fixup_pericom_acs_store_forward); +DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_PERICOM, 0xb404, + pci_fixup_pericom_acs_store_forward); static void nvidia_ion_ahci_fixup(struct pci_dev *pdev) { diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c index 5ba878f15db35..c2d640164f697 100644 --- a/drivers/pci/setup-bus.c +++ b/drivers/pci/setup-bus.c @@ -14,6 +14,7 @@ * tighter packing. Prefetchable range support. */ +#include #include #include #include @@ -221,14 +222,21 @@ static struct resource *pbus_select_window_for_type(struct pci_bus *bus, switch (iores_type) { case IORESOURCE_IO: - return pci_bus_resource_n(bus, PCI_BUS_BRIDGE_IO_WINDOW); + win = pci_bus_resource_n(bus, PCI_BUS_BRIDGE_IO_WINDOW); + if (win && (win->flags & IORESOURCE_IO)) + return win; + return NULL; case IORESOURCE_MEM: mmio = pci_bus_resource_n(bus, PCI_BUS_BRIDGE_MEM_WINDOW); mmio_pref = pci_bus_resource_n(bus, PCI_BUS_BRIDGE_PREF_MEM_WINDOW); - if (!(type & IORESOURCE_PREFETCH) || - !(mmio_pref->flags & IORESOURCE_MEM)) + if (mmio && !(mmio->flags & IORESOURCE_MEM)) + mmio = NULL; + if (mmio_pref && !(mmio_pref->flags & IORESOURCE_MEM)) + mmio_pref = NULL; + + if (!(type & IORESOURCE_PREFETCH) || !mmio_pref) return mmio; if ((type & IORESOURCE_MEM_64) || @@ -452,7 +460,7 @@ static void reassign_resources_sorted(struct list_head *realloc_head, "%s %pR: ignoring failure in optional allocation\n", res_name, res); } - } else if (add_size > 0) { + } else if (add_size > 0 || !IS_ALIGNED(res->start, align)) { res->flags |= add_res->flags & (IORESOURCE_STARTALIGN|IORESOURCE_SIZEALIGN); if (pci_reassign_resource(dev, idx, add_size, align)) @@ -1066,16 +1074,13 @@ static resource_size_t calculate_memsize(resource_size_t size, resource_size_t min_size, resource_size_t add_size, resource_size_t children_add_size, - resource_size_t old_size, resource_size_t align) { if (size < min_size) size = min_size; - if (old_size == 1) - old_size = 0; size = max(size, add_size) + children_add_size; - return ALIGN(max(size, old_size), align); + return ALIGN(size, align); } resource_size_t __weak pcibios_window_alignment(struct pci_bus *bus, @@ -1223,66 +1228,43 @@ static inline resource_size_t calculate_mem_align(resource_size_t *aligns, return min_align; } -/** - * pbus_upstream_space_available - Check no upstream resource limits allocation - * @bus: The bus - * @res: The resource to help select the correct bridge window - * @size: The size required from the bridge window - * @align: Required alignment for the resource - * - * Check that @size can fit inside the upstream bridge resources that are - * already assigned. Select the upstream bridge window based on the type of - * @res. - * - * Return: %true if enough space is available on all assigned upstream +/* + * Calculate bridge window head alignment that leaves no gaps in between * resources. */ -static bool pbus_upstream_space_available(struct pci_bus *bus, - struct resource *res, - resource_size_t size, - resource_size_t align) -{ - struct resource_constraint constraint = { - .max = RESOURCE_SIZE_MAX, - .align = align, - }; - struct pci_bus *downstream = bus; - - while ((bus = bus->parent)) { - if (pci_is_root_bus(bus)) - break; +static resource_size_t calculate_head_align(resource_size_t *aligns, + int max_order) +{ + resource_size_t head_align = 1; + resource_size_t remainder = 0; + int order; - res = pbus_select_window(bus, res); - if (!res) - return false; - if (!res->parent) - continue; + /* Take the largest alignment as the starting point. */ + head_align <<= max_order + __ffs(SZ_1M); - if (resource_size(res) >= size) { - struct resource gap = {}; + for (order = max_order - 1; order >= 0; order--) { + resource_size_t align1 = 1; - if (find_resource_space(res, &gap, size, &constraint) == 0) { - gap.flags = res->flags; - pci_dbg(bus->self, - "Assigned bridge window %pR to %pR free space at %pR\n", - res, &bus->busn_res, &gap); - return true; - } - } + align1 <<= order + __ffs(SZ_1M); - if (bus->self) { - pci_info(bus->self, - "Assigned bridge window %pR to %pR cannot fit 0x%llx required for %s bridging to %pR\n", - res, &bus->busn_res, - (unsigned long long)size, - pci_name(downstream->self), - &downstream->busn_res); - } + /* + * Account smaller resources with alignment < max_order that + * could be used to fill head room if alignment less than + * max_order is used. + */ + remainder += aligns[order]; - return false; + /* + * Test if head fill is enough to satisfy the alignment of + * the larger resources after reducing the alignment. + */ + while ((head_align > align1) && (remainder >= head_align / 2)) { + head_align /= 2; + remainder -= head_align; + } } - return true; + return head_align; } /** @@ -1310,14 +1292,12 @@ static void pbus_size_mem(struct pci_bus *bus, unsigned long type, { struct pci_dev *dev; resource_size_t min_align, win_align, align, size, size0, size1 = 0; - resource_size_t aligns[28]; /* Alignments from 1MB to 128TB */ + resource_size_t aligns[28] = {}; /* Alignments from 1MB to 128TB */ int order, max_order; struct resource *b_res = pbus_select_window_for_type(bus, type); resource_size_t children_add_size = 0; resource_size_t children_add_align = 0; resource_size_t add_align = 0; - resource_size_t relaxed_align; - resource_size_t old_size; if (!b_res) return; @@ -1326,7 +1306,6 @@ static void pbus_size_mem(struct pci_bus *bus, unsigned long type, if (b_res->parent) return; - memset(aligns, 0, sizeof(aligns)); max_order = 0; size = 0; @@ -1371,12 +1350,8 @@ static void pbus_size_mem(struct pci_bus *bus, unsigned long type, continue; } size += max(r_size, align); - /* - * Exclude ranges with size > align from calculation of - * the alignment. - */ - if (r_size <= align) - aligns[order] += align; + + aligns[order] += align; if (order > max_order) max_order = order; @@ -1388,44 +1363,20 @@ static void pbus_size_mem(struct pci_bus *bus, unsigned long type, } } - old_size = resource_size(b_res); win_align = window_alignment(bus, b_res->flags); - min_align = calculate_mem_align(aligns, max_order); + min_align = calculate_head_align(aligns, max_order); min_align = max(min_align, win_align); - size0 = calculate_memsize(size, min_size, 0, 0, old_size, min_align); + size0 = calculate_memsize(size, min_size, 0, 0, win_align); if (size0) { resource_set_range(b_res, min_align, size0); b_res->flags &= ~IORESOURCE_DISABLED; } - if (bus->self && size0 && - !pbus_upstream_space_available(bus, b_res, size0, min_align)) { - relaxed_align = 1ULL << (max_order + __ffs(SZ_1M)); - relaxed_align = max(relaxed_align, win_align); - min_align = min(min_align, relaxed_align); - size0 = calculate_memsize(size, min_size, 0, 0, old_size, win_align); - resource_set_range(b_res, min_align, size0); - pci_info(bus->self, "bridge window %pR to %pR requires relaxed alignment rules\n", - b_res, &bus->busn_res); - } - if (realloc_head && (add_size > 0 || children_add_size > 0)) { add_align = max(min_align, add_align); size1 = calculate_memsize(size, min_size, add_size, children_add_size, - old_size, add_align); - - if (bus->self && size1 && - !pbus_upstream_space_available(bus, b_res, size1, add_align)) { - relaxed_align = 1ULL << (max_order + __ffs(SZ_1M)); - relaxed_align = max(relaxed_align, win_align); - min_align = min(min_align, relaxed_align); - size1 = calculate_memsize(size, min_size, add_size, children_add_size, - old_size, win_align); - pci_info(bus->self, - "bridge window %pR to %pR requires relaxed alignment rules\n", - b_res, &bus->busn_res); - } + win_align); } if (!size0 && !size1) { @@ -1438,12 +1389,13 @@ static void pbus_size_mem(struct pci_bus *bus, unsigned long type, resource_set_range(b_res, min_align, size0); b_res->flags |= IORESOURCE_STARTALIGN; - if (bus->self && size1 > size0 && realloc_head) { + if (bus->self && realloc_head && (size1 > size0 || add_align > min_align)) { b_res->flags &= ~IORESOURCE_DISABLED; - add_to_list(realloc_head, bus->self, b_res, size1-size0, add_align); + add_size = size1 > size0 ? size1 - size0 : 0; + add_to_list(realloc_head, bus->self, b_res, add_size, add_align); pci_info(bus->self, "bridge window %pR to %pR add_size %llx add_align %llx\n", b_res, &bus->busn_res, - (unsigned long long) (size1 - size0), + (unsigned long long) add_size, (unsigned long long) add_align); } } @@ -1729,6 +1681,8 @@ static void pci_claim_bridge_resources(struct pci_dev *dev) if (!r->flags || r->parent) continue; + if (r->flags & IORESOURCE_DISABLED) + continue; pci_claim_bridge_resource(dev, i); } diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c index 23245352a3fc0..4fbafc4b79843 100644 --- a/drivers/perf/arm-cmn.c +++ b/drivers/perf/arm-cmn.c @@ -210,6 +210,7 @@ enum cmn_model { enum cmn_part { PART_CMN600 = 0x434, PART_CMN650 = 0x436, + PART_CMN600AE = 0x438, PART_CMN700 = 0x43c, PART_CI700 = 0x43a, PART_CMN_S3 = 0x43e, @@ -2266,6 +2267,9 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) reg = readq_relaxed(cfg_region + CMN_CFGM_PERIPH_ID_01); part = FIELD_GET(CMN_CFGM_PID0_PART_0, reg); part |= FIELD_GET(CMN_CFGM_PID1_PART_1, reg) << 8; + /* 600AE is close enough that it's not really worth more complexity */ + if (part == PART_CMN600AE) + part = PART_CMN600; if (cmn->part && cmn->part != part) dev_warn(cmn->dev, "Firmware binding mismatch: expected part number 0x%x, found 0x%x\n", @@ -2418,6 +2422,15 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) arm_cmn_init_node_info(cmn, reg & CMN_CHILD_NODE_ADDR, dn); dn->portid_bits = xp->portid_bits; dn->deviceid_bits = xp->deviceid_bits; + /* + * Logical IDs are assigned from 0 per node type, so as + * soon as we see one bigger than expected, we can assume + * there are more than we can cope with. + */ + if (dn->logid > CMN_MAX_NODES_PER_EVENT) { + dev_err(cmn->dev, "Node ID invalid for supported CMN versions: %d\n", dn->logid); + return -ENODEV; + } switch (dn->type) { case CMN_TYPE_DTC: @@ -2467,7 +2480,7 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) break; /* Something has gone horribly wrong */ default: - dev_err(cmn->dev, "invalid device node type: 0x%x\n", dn->type); + dev_err(cmn->dev, "Device node type invalid for supported CMN versions: 0x%x\n", dn->type); return -ENODEV; } } @@ -2495,6 +2508,10 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) cmn->mesh_x = cmn->num_xps; cmn->mesh_y = cmn->num_xps / cmn->mesh_x; + if (max(cmn->mesh_x, cmn->mesh_y) > CMN_MAX_DIMENSION) { + dev_err(cmn->dev, "Mesh size invalid for supported CMN versions: %dx%d\n", cmn->mesh_x, cmn->mesh_y); + return -ENODEV; + } /* 1x1 config plays havoc with XP event encodings */ if (cmn->num_xps == 1) dev_warn(cmn->dev, "1x1 config not fully supported, translate XP events manually\n"); diff --git a/drivers/perf/arm_spe_pmu.c b/drivers/perf/arm_spe_pmu.c index fa50645feddad..e4e4e63c64c42 100644 --- a/drivers/perf/arm_spe_pmu.c +++ b/drivers/perf/arm_spe_pmu.c @@ -105,6 +105,8 @@ struct arm_spe_pmu { /* Keep track of our dynamic hotplug state */ static enum cpuhp_state arm_spe_pmu_online; +static void arm_spe_pmu_stop(struct perf_event *event, int flags); + enum arm_spe_pmu_buf_fault_action { SPE_PMU_BUF_FAULT_ACT_SPURIOUS, SPE_PMU_BUF_FAULT_ACT_FATAL, @@ -582,8 +584,8 @@ static u64 arm_spe_pmu_next_off(struct perf_output_handle *handle) return limit; } -static void arm_spe_perf_aux_output_begin(struct perf_output_handle *handle, - struct perf_event *event) +static int arm_spe_perf_aux_output_begin(struct perf_output_handle *handle, + struct perf_event *event) { u64 base, limit; struct arm_spe_pmu_buf *buf; @@ -597,7 +599,6 @@ static void arm_spe_perf_aux_output_begin(struct perf_output_handle *handle, /* Start a new aux session */ buf = perf_aux_output_begin(handle, event); if (!buf) { - event->hw.state |= PERF_HES_STOPPED; /* * We still need to clear the limit pointer, since the * profiler might only be disabled by virtue of a fault. @@ -617,6 +618,7 @@ static void arm_spe_perf_aux_output_begin(struct perf_output_handle *handle, out_write_limit: write_sysreg_s(limit, SYS_PMBLIMITR_EL1); + return (limit & PMBLIMITR_EL1_E) ? 0 : -EIO; } static void arm_spe_perf_aux_output_end(struct perf_output_handle *handle) @@ -756,7 +758,10 @@ static irqreturn_t arm_spe_pmu_irq_handler(int irq, void *dev) * when we get to it. */ if (!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)) { - arm_spe_perf_aux_output_begin(handle, event); + if (arm_spe_perf_aux_output_begin(handle, event)) { + arm_spe_pmu_stop(event, PERF_EF_UPDATE); + break; + } isb(); } break; @@ -851,9 +856,10 @@ static void arm_spe_pmu_start(struct perf_event *event, int flags) struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle); hwc->state = 0; - arm_spe_perf_aux_output_begin(handle, event); - if (hwc->state) + if (arm_spe_perf_aux_output_begin(handle, event)) { + arm_spe_pmu_stop(event, 0); return; + } reg = arm_spe_event_to_pmsfcr(event); write_sysreg_s(reg, SYS_PMSFCR_EL1); diff --git a/drivers/perf/cxl_pmu.c b/drivers/perf/cxl_pmu.c index d094030220bf2..68a54d97d2a8a 100644 --- a/drivers/perf/cxl_pmu.c +++ b/drivers/perf/cxl_pmu.c @@ -877,7 +877,7 @@ static int cxl_pmu_probe(struct device *dev) if (!irq_name) return -ENOMEM; - rc = devm_request_irq(dev, irq, cxl_pmu_irq, IRQF_SHARED | IRQF_ONESHOT, + rc = devm_request_irq(dev, irq, cxl_pmu_irq, IRQF_SHARED | IRQF_NO_THREAD, irq_name, info); if (rc) return rc; diff --git a/drivers/phy/cadence/phy-cadence-torrent.c b/drivers/phy/cadence/phy-cadence-torrent.c index 37fa4bad6bd72..877f22177c699 100644 --- a/drivers/phy/cadence/phy-cadence-torrent.c +++ b/drivers/phy/cadence/phy-cadence-torrent.c @@ -397,6 +397,7 @@ struct cdns_torrent_refclk_driver { struct clk_hw hw; struct regmap_field *cmn_fields[REFCLK_OUT_NUM_CMN_CONFIG]; struct clk_init_data clk_data; + u8 parent_index; }; #define to_cdns_torrent_refclk_driver(_hw) \ @@ -3326,11 +3327,29 @@ static const struct cdns_torrent_vals sgmii_qsgmii_xcvr_diag_ln_vals = { .num_regs = ARRAY_SIZE(sgmii_qsgmii_xcvr_diag_ln_regs), }; +static void cdns_torrent_refclk_driver_suspend(struct cdns_torrent_phy *cdns_phy) +{ + struct clk_hw *hw = cdns_phy->clk_hw_data->hws[CDNS_TORRENT_REFCLK_DRIVER]; + struct cdns_torrent_refclk_driver *refclk_driver = to_cdns_torrent_refclk_driver(hw); + + refclk_driver->parent_index = cdns_torrent_refclk_driver_get_parent(hw); +} + +static int cdns_torrent_refclk_driver_resume(struct cdns_torrent_phy *cdns_phy) +{ + struct clk_hw *hw = cdns_phy->clk_hw_data->hws[CDNS_TORRENT_REFCLK_DRIVER]; + struct cdns_torrent_refclk_driver *refclk_driver = to_cdns_torrent_refclk_driver(hw); + + return cdns_torrent_refclk_driver_set_parent(hw, refclk_driver->parent_index); +} + static int cdns_torrent_phy_suspend_noirq(struct device *dev) { struct cdns_torrent_phy *cdns_phy = dev_get_drvdata(dev); int i; + cdns_torrent_refclk_driver_suspend(cdns_phy); + reset_control_assert(cdns_phy->phy_rst); reset_control_assert(cdns_phy->apb_rst); for (i = 0; i < cdns_phy->nsubnodes; i++) @@ -3352,6 +3371,10 @@ static int cdns_torrent_phy_resume_noirq(struct device *dev) int node = cdns_phy->nsubnodes; int ret, i; + ret = cdns_torrent_refclk_driver_resume(cdns_phy); + if (ret) + return ret; + ret = cdns_torrent_clk(cdns_phy); if (ret) return ret; diff --git a/drivers/phy/freescale/phy-fsl-imx8mq-usb.c b/drivers/phy/freescale/phy-fsl-imx8mq-usb.c index f6cac4c049c43..8e7b6e10e1f0b 100644 --- a/drivers/phy/freescale/phy-fsl-imx8mq-usb.c +++ b/drivers/phy/freescale/phy-fsl-imx8mq-usb.c @@ -662,6 +662,8 @@ static int imx8mq_usb_phy_probe(struct platform_device *pdev) if (!imx_phy) return -ENOMEM; + platform_set_drvdata(pdev, imx_phy); + imx_phy->clk = devm_clk_get(dev, "phy"); if (IS_ERR(imx_phy->clk)) { dev_err(dev, "failed to get imx8mq usb phy clock\n"); @@ -711,6 +713,7 @@ static struct platform_driver imx8mq_usb_phy_driver = { .driver = { .name = "imx8mq-usb-phy", .of_match_table = imx8mq_usb_phy_of_match, + .suppress_bind_attrs = true, } }; module_platform_driver(imx8mq_usb_phy_driver); diff --git a/drivers/phy/freescale/phy-fsl-imx8qm-hsio.c b/drivers/phy/freescale/phy-fsl-imx8qm-hsio.c index 977d21d753a59..279b8ac7822df 100644 --- a/drivers/phy/freescale/phy-fsl-imx8qm-hsio.c +++ b/drivers/phy/freescale/phy-fsl-imx8qm-hsio.c @@ -251,7 +251,7 @@ static void imx_hsio_configure_clk_pad(struct phy *phy) struct imx_hsio_lane *lane = phy_get_drvdata(phy); struct imx_hsio_priv *priv = lane->priv; - if (strncmp(priv->refclk_pad, "output", 6) == 0) { + if (priv->refclk_pad && strncmp(priv->refclk_pad, "output", 6) == 0) { pll = true; regmap_update_bits(priv->misc, HSIO_CTRL0, HSIO_IOB_A_0_TXOE | HSIO_IOB_A_0_M1M0_MASK, diff --git a/drivers/phy/marvell/phy-mvebu-cp110-utmi.c b/drivers/phy/marvell/phy-mvebu-cp110-utmi.c index 59903f86b13f5..dd3e515a8e865 100644 --- a/drivers/phy/marvell/phy-mvebu-cp110-utmi.c +++ b/drivers/phy/marvell/phy-mvebu-cp110-utmi.c @@ -338,7 +338,7 @@ static int mvebu_cp110_utmi_phy_probe(struct platform_device *pdev) return -ENOMEM; } - port->dr_mode = of_usb_get_dr_mode_by_phy(child, -1); + port->dr_mode = of_usb_get_dr_mode_by_phy(child, 0); if ((port->dr_mode != USB_DR_MODE_HOST) && (port->dr_mode != USB_DR_MODE_PERIPHERAL)) { dev_err(&pdev->dev, diff --git a/drivers/phy/qualcomm/phy-qcom-edp.c b/drivers/phy/qualcomm/phy-qcom-edp.c index ca9bb9d70e29e..06a08c9ea0f70 100644 --- a/drivers/phy/qualcomm/phy-qcom-edp.c +++ b/drivers/phy/qualcomm/phy-qcom-edp.c @@ -1096,7 +1096,7 @@ static int qcom_edp_phy_probe(struct platform_device *pdev) edp->num_clks = devm_clk_bulk_get_all(dev, &edp->clks); if (edp->num_clks < 0) - return dev_err_probe(dev, edp->num_clks, "failed to parse clocks\n"); + return dev_err_probe(dev, edp->num_clks, "failed to get clocks\n"); edp->supplies[0].supply = "vdda-phy"; edp->supplies[1].supply = "vdda-pll"; diff --git a/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c b/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c index 29de2f7bdae8a..cafa618d70fdc 100644 --- a/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c +++ b/drivers/phy/rockchip/phy-rockchip-samsung-hdptx.c @@ -414,6 +414,8 @@ struct rk_hdptx_phy { static const struct ropll_config ropll_tmds_cfg[] = { { 594000000ULL, 124, 124, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 62, 1, 16, 5, 0, 1, 1, 0, 0x20, 0x0c, 1, 0x0e, 0, 0, }, + { 461101250ULL, 97, 97, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 71, 1, 53, 2, 6, + 35, 1, 0, 0x20, 0x0c, 1, 0x0e, 0, 0, }, { 371250000ULL, 155, 155, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 62, 1, 16, 5, 0, 1, 1, 0, 0x20, 0x0c, 1, 0x0e, 0, 0, }, { 297000000ULL, 124, 124, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 62, 1, 16, 5, 0, diff --git a/drivers/phy/ti/phy-j721e-wiz.c b/drivers/phy/ti/phy-j721e-wiz.c index a8b440c6c46bb..ba31b0a1f7f79 100644 --- a/drivers/phy/ti/phy-j721e-wiz.c +++ b/drivers/phy/ti/phy-j721e-wiz.c @@ -393,6 +393,7 @@ struct wiz { struct clk *output_clks[WIZ_MAX_OUTPUT_CLOCKS]; struct clk_onecell_data clk_data; const struct wiz_data *data; + int mux_sel_status[WIZ_MUX_NUM_CLOCKS]; }; static int wiz_reset(struct wiz *wiz) @@ -1654,11 +1655,25 @@ static void wiz_remove(struct platform_device *pdev) pm_runtime_disable(dev); } +static int wiz_suspend_noirq(struct device *dev) +{ + struct wiz *wiz = dev_get_drvdata(dev); + int i; + + for (i = 0; i < WIZ_MUX_NUM_CLOCKS; i++) + regmap_field_read(wiz->mux_sel_field[i], &wiz->mux_sel_status[i]); + + return 0; +} + static int wiz_resume_noirq(struct device *dev) { struct device_node *node = dev->of_node; struct wiz *wiz = dev_get_drvdata(dev); - int ret; + int ret, i; + + for (i = 0; i < WIZ_MUX_NUM_CLOCKS; i++) + regmap_field_write(wiz->mux_sel_field[i], wiz->mux_sel_status[i]); /* Enable supplemental Control override if available */ if (wiz->sup_legacy_clk_override) @@ -1680,7 +1695,7 @@ static int wiz_resume_noirq(struct device *dev) return ret; } -static DEFINE_NOIRQ_DEV_PM_OPS(wiz_pm_ops, NULL, wiz_resume_noirq); +static DEFINE_NOIRQ_DEV_PM_OPS(wiz_pm_ops, wiz_suspend_noirq, wiz_resume_noirq); static struct platform_driver wiz_driver = { .probe = wiz_probe, diff --git a/drivers/pinctrl/cirrus/pinctrl-cs42l43.c b/drivers/pinctrl/cirrus/pinctrl-cs42l43.c index a8f82104a3842..227c37c360e19 100644 --- a/drivers/pinctrl/cirrus/pinctrl-cs42l43.c +++ b/drivers/pinctrl/cirrus/pinctrl-cs42l43.c @@ -574,10 +574,9 @@ static int cs42l43_pin_probe(struct platform_device *pdev) if (child) { ret = devm_add_action_or_reset(&pdev->dev, cs42l43_fwnode_put, child); - if (ret) { - fwnode_handle_put(child); + if (ret) return ret; - } + if (!child->dev) child->dev = priv->dev; fwnode = child; diff --git a/drivers/pinctrl/intel/Kconfig b/drivers/pinctrl/intel/Kconfig index 248c2e558ff32..3ebf072371457 100644 --- a/drivers/pinctrl/intel/Kconfig +++ b/drivers/pinctrl/intel/Kconfig @@ -52,7 +52,10 @@ config PINCTRL_ALDERLAKE select PINCTRL_INTEL help This pinctrl driver provides an interface that allows configuring - of Intel Alder Lake PCH pins and using them as GPIOs. + PCH pins of the following platforms and using them as GPIOs: + - Alder Lake HX, N, and S + - Raptor Lake HX, E, and S + - Twin Lake config PINCTRL_BROXTON tristate "Intel Broxton pinctrl and GPIO driver" @@ -136,15 +139,17 @@ config PINCTRL_METEORLAKE select PINCTRL_INTEL help This pinctrl driver provides an interface that allows configuring - of Intel Meteor Lake pins and using them as GPIOs. + SoC pins of the following platforms and using them as GPIOs: + - Arrow Lake (all variants) + - Meteor Lake (all variants) config PINCTRL_METEORPOINT tristate "Intel Meteor Point pinctrl and GPIO driver" select PINCTRL_INTEL help - Meteor Point is the PCH of Intel Meteor Lake. This pinctrl driver - provides an interface that allows configuring of PCH pins and - using them as GPIOs. + This pinctrl driver provides an interface that allows configuring + PCH pins of the following platforms and using them as GPIOs: + - Arrow Lake HX and S config PINCTRL_SUNRISEPOINT tristate "Intel Sunrisepoint pinctrl and GPIO driver" @@ -159,7 +164,11 @@ config PINCTRL_TIGERLAKE select PINCTRL_INTEL help This pinctrl driver provides an interface that allows configuring - of Intel Tiger Lake PCH pins and using them as GPIOs. + PCH pins of the following platforms and using them as GPIOs: + - Alder Lake H, P, PS, and U + - Raptor Lake H, P, PS, PX, and U + - Rocket Lake S + - Tiger Lake (all variants) source "drivers/pinctrl/intel/Kconfig.tng" endmenu diff --git a/drivers/pinctrl/mediatek/mtk-eint.c b/drivers/pinctrl/mediatek/mtk-eint.c index 9f175c73613f8..2ea0902b4f660 100644 --- a/drivers/pinctrl/mediatek/mtk-eint.c +++ b/drivers/pinctrl/mediatek/mtk-eint.c @@ -539,24 +539,32 @@ int mtk_eint_do_init(struct mtk_eint *eint, struct mtk_eint_pin *eint_pin) } } - eint->pin_list = devm_kmalloc(eint->dev, eint->nbase * sizeof(u16 *), GFP_KERNEL); + eint->pin_list = devm_kcalloc(eint->dev, eint->nbase, + sizeof(*eint->pin_list), GFP_KERNEL); if (!eint->pin_list) goto err_pin_list; - eint->wake_mask = devm_kmalloc(eint->dev, eint->nbase * sizeof(u32 *), GFP_KERNEL); + eint->wake_mask = devm_kcalloc(eint->dev, eint->nbase, + sizeof(*eint->wake_mask), GFP_KERNEL); if (!eint->wake_mask) goto err_wake_mask; - eint->cur_mask = devm_kmalloc(eint->dev, eint->nbase * sizeof(u32 *), GFP_KERNEL); + eint->cur_mask = devm_kcalloc(eint->dev, eint->nbase, + sizeof(*eint->cur_mask), GFP_KERNEL); if (!eint->cur_mask) goto err_cur_mask; for (i = 0; i < eint->nbase; i++) { - eint->pin_list[i] = devm_kzalloc(eint->dev, eint->base_pin_num[i] * sizeof(u16), + eint->pin_list[i] = devm_kzalloc(eint->dev, + eint->base_pin_num[i] * sizeof(**eint->pin_list), GFP_KERNEL); port = DIV_ROUND_UP(eint->base_pin_num[i], 32); - eint->wake_mask[i] = devm_kzalloc(eint->dev, port * sizeof(u32), GFP_KERNEL); - eint->cur_mask[i] = devm_kzalloc(eint->dev, port * sizeof(u32), GFP_KERNEL); + eint->wake_mask[i] = devm_kzalloc(eint->dev, + port * sizeof(**eint->wake_mask), + GFP_KERNEL); + eint->cur_mask[i] = devm_kzalloc(eint->dev, + port * sizeof(**eint->cur_mask), + GFP_KERNEL); if (!eint->pin_list[i] || !eint->wake_mask[i] || !eint->cur_mask[i]) goto err_eint; } @@ -592,12 +600,9 @@ int mtk_eint_do_init(struct mtk_eint *eint, struct mtk_eint_pin *eint_pin) err_eint: for (i = 0; i < eint->nbase; i++) { - if (eint->cur_mask[i]) - devm_kfree(eint->dev, eint->cur_mask[i]); - if (eint->wake_mask[i]) - devm_kfree(eint->dev, eint->wake_mask[i]); - if (eint->pin_list[i]) - devm_kfree(eint->dev, eint->pin_list[i]); + devm_kfree(eint->dev, eint->cur_mask[i]); + devm_kfree(eint->dev, eint->wake_mask[i]); + devm_kfree(eint->dev, eint->pin_list[i]); } devm_kfree(eint->dev, eint->cur_mask); err_cur_mask: diff --git a/drivers/pinctrl/meson/pinctrl-amlogic-a4.c b/drivers/pinctrl/meson/pinctrl-amlogic-a4.c index d9e3a8d5932a8..e2293a872dcb7 100644 --- a/drivers/pinctrl/meson/pinctrl-amlogic-a4.c +++ b/drivers/pinctrl/meson/pinctrl-amlogic-a4.c @@ -24,6 +24,7 @@ #include #include "../core.h" +#include "../pinctrl-utils.h" #include "../pinconf.h" #define gpio_chip_to_bank(chip) \ @@ -672,11 +673,78 @@ static void aml_pin_dbg_show(struct pinctrl_dev *pcdev, struct seq_file *s, seq_printf(s, " %s", dev_name(pcdev->dev)); } +static int aml_dt_node_to_map_pinmux(struct pinctrl_dev *pctldev, + struct device_node *np, + struct pinctrl_map **map, + unsigned int *num_maps) +{ + struct device *dev = pctldev->dev; + unsigned long *configs = NULL; + unsigned int num_configs = 0; + struct property *prop; + unsigned int reserved_maps; + int reserve; + int ret; + + prop = of_find_property(np, "pinmux", NULL); + if (!prop) { + dev_info(dev, "Missing pinmux property\n"); + return -ENOENT; + } + + struct device_node *pnode __free(device_node) = of_get_parent(np); + if (!pnode) { + dev_info(dev, "Missing function node\n"); + return -EINVAL; + } + + reserved_maps = 0; + *map = NULL; + *num_maps = 0; + + ret = pinconf_generic_parse_dt_config(np, pctldev, &configs, + &num_configs); + if (ret < 0) { + dev_err(dev, "%pOF: could not parse node property\n", np); + return ret; + } + + reserve = 1; + if (num_configs) + reserve++; + + ret = pinctrl_utils_reserve_map(pctldev, map, &reserved_maps, + num_maps, reserve); + if (ret < 0) + goto exit; + + ret = pinctrl_utils_add_map_mux(pctldev, map, + &reserved_maps, num_maps, np->name, + pnode->name); + if (ret < 0) + goto exit; + + if (num_configs) { + ret = pinctrl_utils_add_map_configs(pctldev, map, &reserved_maps, + num_maps, np->name, configs, + num_configs, PIN_MAP_TYPE_CONFIGS_GROUP); + if (ret < 0) + goto exit; + } + +exit: + kfree(configs); + if (ret) + pinctrl_utils_free_map(pctldev, *map, *num_maps); + + return ret; +} + static const struct pinctrl_ops aml_pctrl_ops = { .get_groups_count = aml_get_groups_count, .get_group_name = aml_get_group_name, .get_group_pins = aml_get_group_pins, - .dt_node_to_map = pinconf_generic_dt_node_to_map_pinmux, + .dt_node_to_map = aml_dt_node_to_map_pinmux, .dt_free_map = pinconf_generic_dt_free_map, .pin_dbg_show = aml_pin_dbg_show, }; @@ -725,8 +793,9 @@ static u32 aml_bank_pins(struct device_node *np) if (of_parse_phandle_with_fixed_args(np, "gpio-ranges", 3, 0, &of_args)) return 0; - else - return of_args.args[2]; + + of_node_put(of_args.np); + return of_args.args[2]; } static int aml_bank_number(struct device_node *np) @@ -736,8 +805,9 @@ static int aml_bank_number(struct device_node *np) if (of_parse_phandle_with_fixed_args(np, "gpio-ranges", 3, 0, &of_args)) return -EINVAL; - else - return of_args.args[1] >> 8; + + of_node_put(of_args.np); + return of_args.args[1] >> 8; } static unsigned int aml_count_pins(struct device_node *np) @@ -893,7 +963,7 @@ static const struct gpio_chip aml_gpio_template = { .direction_input = aml_gpio_direction_input, .direction_output = aml_gpio_direction_output, .get_direction = aml_gpio_get_direction, - .can_sleep = false, + .can_sleep = true, }; static void init_bank_register_bit(struct aml_pinctrl *info, diff --git a/drivers/pinctrl/pinconf-generic.c b/drivers/pinctrl/pinconf-generic.c index 5de6ff62c69bd..cad29abe4050a 100644 --- a/drivers/pinctrl/pinconf-generic.c +++ b/drivers/pinctrl/pinconf-generic.c @@ -356,75 +356,6 @@ int pinconf_generic_parse_dt_config(struct device_node *np, } EXPORT_SYMBOL_GPL(pinconf_generic_parse_dt_config); -int pinconf_generic_dt_node_to_map_pinmux(struct pinctrl_dev *pctldev, - struct device_node *np, - struct pinctrl_map **map, - unsigned int *num_maps) -{ - struct device *dev = pctldev->dev; - struct device_node *pnode; - unsigned long *configs = NULL; - unsigned int num_configs = 0; - struct property *prop; - unsigned int reserved_maps; - int reserve; - int ret; - - prop = of_find_property(np, "pinmux", NULL); - if (!prop) { - dev_info(dev, "Missing pinmux property\n"); - return -ENOENT; - } - - pnode = of_get_parent(np); - if (!pnode) { - dev_info(dev, "Missing function node\n"); - return -EINVAL; - } - - reserved_maps = 0; - *map = NULL; - *num_maps = 0; - - ret = pinconf_generic_parse_dt_config(np, pctldev, &configs, - &num_configs); - if (ret < 0) { - dev_err(dev, "%pOF: could not parse node property\n", np); - return ret; - } - - reserve = 1; - if (num_configs) - reserve++; - - ret = pinctrl_utils_reserve_map(pctldev, map, &reserved_maps, - num_maps, reserve); - if (ret < 0) - goto exit; - - ret = pinctrl_utils_add_map_mux(pctldev, map, - &reserved_maps, num_maps, np->name, - pnode->name); - if (ret < 0) - goto exit; - - if (num_configs) { - ret = pinctrl_utils_add_map_configs(pctldev, map, &reserved_maps, - num_maps, np->name, configs, - num_configs, PIN_MAP_TYPE_CONFIGS_GROUP); - if (ret < 0) - goto exit; - } - -exit: - kfree(configs); - if (ret) - pinctrl_utils_free_map(pctldev, *map, *num_maps); - - return ret; -} -EXPORT_SYMBOL_GPL(pinconf_generic_dt_node_to_map_pinmux); - int pinconf_generic_dt_subnode_to_map(struct pinctrl_dev *pctldev, struct device_node *np, struct pinctrl_map **map, unsigned int *reserved_maps, unsigned int *num_maps, diff --git a/drivers/pinctrl/pinctrl-equilibrium.c b/drivers/pinctrl/pinctrl-equilibrium.c index 2d04829b29c99..ba1c867b7b891 100644 --- a/drivers/pinctrl/pinctrl-equilibrium.c +++ b/drivers/pinctrl/pinctrl-equilibrium.c @@ -23,7 +23,7 @@ #define PIN_NAME_LEN 10 #define PAD_REG_OFF 0x100 -static void eqbr_gpio_disable_irq(struct irq_data *d) +static void eqbr_irq_mask(struct irq_data *d) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); struct eqbr_gpio_ctrl *gctrl = gpiochip_get_data(gc); @@ -36,7 +36,7 @@ static void eqbr_gpio_disable_irq(struct irq_data *d) gpiochip_disable_irq(gc, offset); } -static void eqbr_gpio_enable_irq(struct irq_data *d) +static void eqbr_irq_unmask(struct irq_data *d) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); struct eqbr_gpio_ctrl *gctrl = gpiochip_get_data(gc); @@ -50,7 +50,7 @@ static void eqbr_gpio_enable_irq(struct irq_data *d) raw_spin_unlock_irqrestore(&gctrl->lock, flags); } -static void eqbr_gpio_ack_irq(struct irq_data *d) +static void eqbr_irq_ack(struct irq_data *d) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); struct eqbr_gpio_ctrl *gctrl = gpiochip_get_data(gc); @@ -62,10 +62,17 @@ static void eqbr_gpio_ack_irq(struct irq_data *d) raw_spin_unlock_irqrestore(&gctrl->lock, flags); } -static void eqbr_gpio_mask_ack_irq(struct irq_data *d) +static void eqbr_irq_mask_ack(struct irq_data *d) { - eqbr_gpio_disable_irq(d); - eqbr_gpio_ack_irq(d); + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct eqbr_gpio_ctrl *gctrl = gpiochip_get_data(gc); + unsigned int offset = irqd_to_hwirq(d); + unsigned long flags; + + raw_spin_lock_irqsave(&gctrl->lock, flags); + writel(BIT(offset), gctrl->membase + GPIO_IRNENCLR); + writel(BIT(offset), gctrl->membase + GPIO_IRNCR); + raw_spin_unlock_irqrestore(&gctrl->lock, flags); } static inline void eqbr_cfg_bit(void __iomem *addr, @@ -92,7 +99,7 @@ static int eqbr_irq_type_cfg(struct gpio_irq_type *type, return 0; } -static int eqbr_gpio_set_irq_type(struct irq_data *d, unsigned int type) +static int eqbr_irq_set_type(struct irq_data *d, unsigned int type) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); struct eqbr_gpio_ctrl *gctrl = gpiochip_get_data(gc); @@ -166,11 +173,11 @@ static void eqbr_irq_handler(struct irq_desc *desc) static const struct irq_chip eqbr_irq_chip = { .name = "gpio_irq", - .irq_mask = eqbr_gpio_disable_irq, - .irq_unmask = eqbr_gpio_enable_irq, - .irq_ack = eqbr_gpio_ack_irq, - .irq_mask_ack = eqbr_gpio_mask_ack_irq, - .irq_set_type = eqbr_gpio_set_irq_type, + .irq_ack = eqbr_irq_ack, + .irq_mask = eqbr_irq_mask, + .irq_mask_ack = eqbr_irq_mask_ack, + .irq_unmask = eqbr_irq_unmask, + .irq_set_type = eqbr_irq_set_type, .flags = IRQCHIP_IMMUTABLE, GPIOCHIP_IRQ_RESOURCE_HELPERS, }; @@ -846,6 +853,7 @@ static int pinbank_init(struct device_node *np, bank->pin_base = spec.args[1]; bank->nr_pins = spec.args[2]; + of_node_put(spec.np); bank->aval_pinmap = readl(bank->membase + REG_AVAIL); bank->id = id; diff --git a/drivers/pinctrl/pinctrl-k230.c b/drivers/pinctrl/pinctrl-k230.c index d716f23d837f7..20f7c0f70eb77 100644 --- a/drivers/pinctrl/pinctrl-k230.c +++ b/drivers/pinctrl/pinctrl-k230.c @@ -65,6 +65,7 @@ struct k230_pmx_func { }; struct k230_pinctrl { + struct device *dev; struct pinctrl_desc pctl; struct pinctrl_dev *pctl_dev; struct regmap *regmap_base; @@ -470,7 +471,7 @@ static int k230_pinctrl_parse_groups(struct device_node *np, struct k230_pinctrl *info, unsigned int index) { - struct device *dev = info->pctl_dev->dev; + struct device *dev = info->dev; const __be32 *list; int size, i, ret; @@ -511,7 +512,7 @@ static int k230_pinctrl_parse_functions(struct device_node *np, struct k230_pinctrl *info, unsigned int index) { - struct device *dev = info->pctl_dev->dev; + struct device *dev = info->dev; struct k230_pmx_func *func; struct k230_pin_group *grp; static unsigned int idx, i; @@ -596,6 +597,8 @@ static int k230_pinctrl_probe(struct platform_device *pdev) if (!info) return -ENOMEM; + info->dev = dev; + pctl = &info->pctl; pctl->name = "k230-pinctrl"; diff --git a/drivers/pinctrl/pinctrl-single.c b/drivers/pinctrl/pinctrl-single.c index 998f23d6c3179..d85e6c1f63218 100644 --- a/drivers/pinctrl/pinctrl-single.c +++ b/drivers/pinctrl/pinctrl-single.c @@ -1359,6 +1359,7 @@ static int pcs_add_gpio_func(struct device_node *node, struct pcs_device *pcs) } range = devm_kzalloc(pcs->dev, sizeof(*range), GFP_KERNEL); if (!range) { + of_node_put(gpiospec.np); ret = -ENOMEM; break; } @@ -1368,6 +1369,7 @@ static int pcs_add_gpio_func(struct device_node *node, struct pcs_device *pcs) mutex_lock(&pcs->mutex); list_add_tail(&range->node, &pcs->gpiofuncs); mutex_unlock(&pcs->mutex); + of_node_put(gpiospec.np); } return ret; } diff --git a/drivers/pinctrl/qcom/pinctrl-qcs615.c b/drivers/pinctrl/qcom/pinctrl-qcs615.c index 4dfa820d4e77c..f1c827ddbfbfa 100644 --- a/drivers/pinctrl/qcom/pinctrl-qcs615.c +++ b/drivers/pinctrl/qcom/pinctrl-qcs615.c @@ -1067,6 +1067,7 @@ static const struct msm_pinctrl_soc_data qcs615_tlmm = { .ntiles = ARRAY_SIZE(qcs615_tiles), .wakeirq_map = qcs615_pdc_map, .nwakeirq_map = ARRAY_SIZE(qcs615_pdc_map), + .wakeirq_dual_edge_errata = true, }; static const struct of_device_id qcs615_tlmm_of_match[] = { diff --git a/drivers/pinctrl/qcom/pinctrl-sm8250-lpass-lpi.c b/drivers/pinctrl/qcom/pinctrl-sm8250-lpass-lpi.c index 64494a86490e2..c27452eece3e6 100644 --- a/drivers/pinctrl/qcom/pinctrl-sm8250-lpass-lpi.c +++ b/drivers/pinctrl/qcom/pinctrl-sm8250-lpass-lpi.c @@ -73,7 +73,7 @@ static const char * const i2s1_ws_groups[] = { "gpio7" }; static const char * const i2s1_data_groups[] = { "gpio8", "gpio9" }; static const char * const wsa_swr_clk_groups[] = { "gpio10" }; static const char * const wsa_swr_data_groups[] = { "gpio11" }; -static const char * const i2s2_data_groups[] = { "gpio12", "gpio12" }; +static const char * const i2s2_data_groups[] = { "gpio12", "gpio13" }; static const struct lpi_pingroup sm8250_groups[] = { LPI_PINGROUP(0, 0, swr_tx_clk, qua_mi2s_sclk, _, _), diff --git a/drivers/pinctrl/renesas/pinctrl-rzt2h.c b/drivers/pinctrl/renesas/pinctrl-rzt2h.c index 3872638f5ebb3..3161b2469c362 100644 --- a/drivers/pinctrl/renesas/pinctrl-rzt2h.c +++ b/drivers/pinctrl/renesas/pinctrl-rzt2h.c @@ -51,6 +51,7 @@ #define PFC_MASK GENMASK_ULL(5, 0) #define PFC_PIN_MASK(pin) (PFC_MASK << ((pin) * 8)) +#define PFC_FUNC_INTERRUPT 0 /* * Use 16 lower bits [15:0] for pin identifier @@ -486,6 +487,7 @@ static int rzt2h_gpio_get_direction(struct gpio_chip *chip, unsigned int offset) struct rzt2h_pinctrl *pctrl = gpiochip_get_data(chip); u8 port = RZT2H_PIN_ID_TO_PORT(offset); u8 bit = RZT2H_PIN_ID_TO_PIN(offset); + u64 reg64; u16 reg; int ret; @@ -493,8 +495,25 @@ static int rzt2h_gpio_get_direction(struct gpio_chip *chip, unsigned int offset) if (ret) return ret; - if (rzt2h_pinctrl_readb(pctrl, port, PMC(port)) & BIT(bit)) + guard(spinlock_irqsave)(&pctrl->lock); + + if (rzt2h_pinctrl_readb(pctrl, port, PMC(port)) & BIT(bit)) { + /* + * When a GPIO is being requested as an IRQ, the pinctrl + * framework expects to be able to read the GPIO's direction. + * IRQ function is separate from GPIO, and enabling it takes the + * pin out of GPIO mode. + * At this point, .child_to_parent_hwirq() has already been + * called to enable the IRQ function. + * Default to input direction for IRQ function. + */ + reg64 = rzt2h_pinctrl_readq(pctrl, port, PFC(port)); + reg64 = (reg64 >> (bit * 8)) & PFC_MASK; + if (reg64 == PFC_FUNC_INTERRUPT) + return GPIO_LINE_DIRECTION_IN; + return -EINVAL; + } reg = rzt2h_pinctrl_readw(pctrl, port, PM(port)); reg = (reg >> (bit * 2)) & PM_MASK; diff --git a/drivers/platform/chrome/cros_ec_lightbar.c b/drivers/platform/chrome/cros_ec_lightbar.c index 87634f6921b78..6b028615ee245 100644 --- a/drivers/platform/chrome/cros_ec_lightbar.c +++ b/drivers/platform/chrome/cros_ec_lightbar.c @@ -119,7 +119,7 @@ static int get_lightbar_version(struct cros_ec_dev *ec, param = (struct ec_params_lightbar *)msg->data; param->cmd = LIGHTBAR_CMD_VERSION; msg->outsize = sizeof(param->cmd); - msg->result = sizeof(resp->version); + msg->insize = sizeof(resp->version); ret = cros_ec_cmd_xfer_status(ec->ec_dev, msg); if (ret < 0 && ret != -EINVAL) { ret = 0; diff --git a/drivers/platform/chrome/cros_typec_switch.c b/drivers/platform/chrome/cros_typec_switch.c index 8d7c34abb0a12..d8a28d4e51a85 100644 --- a/drivers/platform/chrome/cros_typec_switch.c +++ b/drivers/platform/chrome/cros_typec_switch.c @@ -230,20 +230,20 @@ static int cros_typec_register_switches(struct cros_typec_switch_data *sdata) adev = to_acpi_device_node(fwnode); if (!adev) { - dev_err(fwnode->dev, "Couldn't get ACPI device handle\n"); + dev_err(dev, "Couldn't get ACPI device handle for %pfwP\n", fwnode); ret = -ENODEV; goto err_switch; } ret = acpi_evaluate_integer(adev->handle, "_ADR", NULL, &index); if (ACPI_FAILURE(ret)) { - dev_err(fwnode->dev, "_ADR wasn't evaluated\n"); + dev_err(dev, "_ADR wasn't evaluated for %pfwP\n", fwnode); ret = -ENODATA; goto err_switch; } if (index >= EC_USB_PD_MAX_PORTS) { - dev_err(fwnode->dev, "Invalid port index number: %llu\n", index); + dev_err(dev, "%pfwP: Invalid port index number: %llu\n", fwnode, index); ret = -EINVAL; goto err_switch; } diff --git a/drivers/platform/x86/amd/pmc/pmc-quirks.c b/drivers/platform/x86/amd/pmc/pmc-quirks.c index 404e62ad293a9..ed285afaf9b0d 100644 --- a/drivers/platform/x86/amd/pmc/pmc-quirks.c +++ b/drivers/platform/x86/amd/pmc/pmc-quirks.c @@ -302,6 +302,13 @@ static const struct dmi_system_id fwbug_list[] = { DMI_MATCH(DMI_BOARD_NAME, "XxKK4NAx_XxSP4NAx"), } }, + { + .ident = "MECHREVO Wujie 15X Pro", + .driver_data = &quirk_spurious_8042, + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "WUJIE Series-X5SP4NAG"), + } + }, {} }; diff --git a/drivers/platform/x86/amd/pmf/core.c b/drivers/platform/x86/amd/pmf/core.c index bc544a4a5266e..e787480f4df26 100644 --- a/drivers/platform/x86/amd/pmf/core.c +++ b/drivers/platform/x86/amd/pmf/core.c @@ -314,6 +314,61 @@ int amd_pmf_init_metrics_table(struct amd_pmf_dev *dev) return 0; } +static int amd_pmf_reinit_ta(struct amd_pmf_dev *pdev) +{ + bool status; + int ret, i; + + for (i = 0; i < ARRAY_SIZE(amd_pmf_ta_uuid); i++) { + ret = amd_pmf_tee_init(pdev, &amd_pmf_ta_uuid[i]); + if (ret) { + dev_err(pdev->dev, "TEE init failed for UUID[%d] ret: %d\n", i, ret); + return ret; + } + + ret = amd_pmf_start_policy_engine(pdev); + dev_dbg(pdev->dev, "start policy engine ret: %d (UUID idx: %d)\n", ret, i); + status = ret == TA_PMF_TYPE_SUCCESS; + if (status) + break; + amd_pmf_tee_deinit(pdev); + } + + return 0; +} + +static int amd_pmf_restore_handler(struct device *dev) +{ + struct amd_pmf_dev *pdev = dev_get_drvdata(dev); + int ret; + + if (pdev->buf) { + ret = amd_pmf_set_dram_addr(pdev, false); + if (ret) + return ret; + } + + if (pdev->smart_pc_enabled) + amd_pmf_reinit_ta(pdev); + + return 0; +} + +static int amd_pmf_freeze_handler(struct device *dev) +{ + struct amd_pmf_dev *pdev = dev_get_drvdata(dev); + + if (!pdev->smart_pc_enabled) + return 0; + + cancel_delayed_work_sync(&pdev->pb_work); + /* Clear all TEE resources */ + amd_pmf_tee_deinit(pdev); + pdev->session_id = 0; + + return 0; +} + static int amd_pmf_suspend_handler(struct device *dev) { struct amd_pmf_dev *pdev = dev_get_drvdata(dev); @@ -347,7 +402,12 @@ static int amd_pmf_resume_handler(struct device *dev) return 0; } -static DEFINE_SIMPLE_DEV_PM_OPS(amd_pmf_pm, amd_pmf_suspend_handler, amd_pmf_resume_handler); +static const struct dev_pm_ops amd_pmf_pm = { + .suspend = amd_pmf_suspend_handler, + .resume = amd_pmf_resume_handler, + .freeze = amd_pmf_freeze_handler, + .restore = amd_pmf_restore_handler, +}; static void amd_pmf_init_features(struct amd_pmf_dev *dev) { diff --git a/drivers/platform/x86/amd/pmf/pmf.h b/drivers/platform/x86/amd/pmf/pmf.h index bd19f2a6bc786..2da1885d87915 100644 --- a/drivers/platform/x86/amd/pmf/pmf.h +++ b/drivers/platform/x86/amd/pmf/pmf.h @@ -122,6 +122,12 @@ struct cookie_header { typedef void (*apmf_event_handler_t)(acpi_handle handle, u32 event, void *data); +static const uuid_t amd_pmf_ta_uuid[] __used = { UUID_INIT(0xd9b39bf2, 0x66bd, 0x4154, 0xaf, 0xb8, + 0x8a, 0xcc, 0x2b, 0x2b, 0x60, 0xd6), + UUID_INIT(0x6fd93b77, 0x3fb8, 0x524d, 0xb1, 0x2d, + 0xc5, 0x29, 0xb1, 0x3d, 0x85, 0x43), + }; + /* APTS PMF BIOS Interface */ struct amd_pmf_apts_output { u16 table_version; @@ -888,4 +894,8 @@ void amd_pmf_populate_ta_inputs(struct amd_pmf_dev *dev, struct ta_pmf_enact_tab void amd_pmf_dump_ta_inputs(struct amd_pmf_dev *dev, struct ta_pmf_enact_table *in); int amd_pmf_invoke_cmd_enact(struct amd_pmf_dev *dev); +int amd_pmf_tee_init(struct amd_pmf_dev *dev, const uuid_t *uuid); +void amd_pmf_tee_deinit(struct amd_pmf_dev *dev); +int amd_pmf_start_policy_engine(struct amd_pmf_dev *dev); + #endif /* PMF_H */ diff --git a/drivers/platform/x86/amd/pmf/tee-if.c b/drivers/platform/x86/amd/pmf/tee-if.c index 6e8116bef4f6a..9030459352375 100644 --- a/drivers/platform/x86/amd/pmf/tee-if.c +++ b/drivers/platform/x86/amd/pmf/tee-if.c @@ -27,12 +27,6 @@ module_param(pb_side_load, bool, 0444); MODULE_PARM_DESC(pb_side_load, "Sideload policy binaries debug policy failures"); #endif -static const uuid_t amd_pmf_ta_uuid[] = { UUID_INIT(0xd9b39bf2, 0x66bd, 0x4154, 0xaf, 0xb8, 0x8a, - 0xcc, 0x2b, 0x2b, 0x60, 0xd6), - UUID_INIT(0x6fd93b77, 0x3fb8, 0x524d, 0xb1, 0x2d, 0xc5, - 0x29, 0xb1, 0x3d, 0x85, 0x43), - }; - static const char *amd_pmf_uevent_as_str(unsigned int state) { switch (state) { @@ -312,7 +306,7 @@ static void amd_pmf_invoke_cmd(struct work_struct *work) schedule_delayed_work(&dev->pb_work, msecs_to_jiffies(pb_actions_ms)); } -static int amd_pmf_start_policy_engine(struct amd_pmf_dev *dev) +int amd_pmf_start_policy_engine(struct amd_pmf_dev *dev) { struct cookie_header *header; int res; @@ -468,7 +462,7 @@ static int amd_pmf_register_input_device(struct amd_pmf_dev *dev) return 0; } -static int amd_pmf_tee_init(struct amd_pmf_dev *dev, const uuid_t *uuid) +int amd_pmf_tee_init(struct amd_pmf_dev *dev, const uuid_t *uuid) { u32 size; int ret; @@ -516,7 +510,7 @@ static int amd_pmf_tee_init(struct amd_pmf_dev *dev, const uuid_t *uuid) return ret; } -static void amd_pmf_tee_deinit(struct amd_pmf_dev *dev) +void amd_pmf_tee_deinit(struct amd_pmf_dev *dev) { if (!dev->tee_ctx) return; diff --git a/drivers/platform/x86/classmate-laptop.c b/drivers/platform/x86/classmate-laptop.c index 6b1b8e444e241..74d3eb83f56a6 100644 --- a/drivers/platform/x86/classmate-laptop.c +++ b/drivers/platform/x86/classmate-laptop.c @@ -207,7 +207,12 @@ static ssize_t cmpc_accel_sensitivity_show_v4(struct device *dev, acpi = to_acpi_device(dev); inputdev = dev_get_drvdata(&acpi->dev); + if (!inputdev) + return -ENXIO; + accel = dev_get_drvdata(&inputdev->dev); + if (!accel) + return -ENXIO; return sysfs_emit(buf, "%d\n", accel->sensitivity); } @@ -224,7 +229,12 @@ static ssize_t cmpc_accel_sensitivity_store_v4(struct device *dev, acpi = to_acpi_device(dev); inputdev = dev_get_drvdata(&acpi->dev); + if (!inputdev) + return -ENXIO; + accel = dev_get_drvdata(&inputdev->dev); + if (!accel) + return -ENXIO; r = kstrtoul(buf, 0, &sensitivity); if (r) @@ -256,7 +266,12 @@ static ssize_t cmpc_accel_g_select_show_v4(struct device *dev, acpi = to_acpi_device(dev); inputdev = dev_get_drvdata(&acpi->dev); + if (!inputdev) + return -ENXIO; + accel = dev_get_drvdata(&inputdev->dev); + if (!accel) + return -ENXIO; return sysfs_emit(buf, "%d\n", accel->g_select); } @@ -273,7 +288,12 @@ static ssize_t cmpc_accel_g_select_store_v4(struct device *dev, acpi = to_acpi_device(dev); inputdev = dev_get_drvdata(&acpi->dev); + if (!inputdev) + return -ENXIO; + accel = dev_get_drvdata(&inputdev->dev); + if (!accel) + return -ENXIO; r = kstrtoul(buf, 0, &g_select); if (r) @@ -302,6 +322,8 @@ static int cmpc_accel_open_v4(struct input_dev *input) acpi = to_acpi_device(input->dev.parent); accel = dev_get_drvdata(&input->dev); + if (!accel) + return -ENXIO; cmpc_accel_set_sensitivity_v4(acpi->handle, accel->sensitivity); cmpc_accel_set_g_select_v4(acpi->handle, accel->g_select); @@ -549,7 +571,12 @@ static ssize_t cmpc_accel_sensitivity_show(struct device *dev, acpi = to_acpi_device(dev); inputdev = dev_get_drvdata(&acpi->dev); + if (!inputdev) + return -ENXIO; + accel = dev_get_drvdata(&inputdev->dev); + if (!accel) + return -ENXIO; return sysfs_emit(buf, "%d\n", accel->sensitivity); } @@ -566,7 +593,12 @@ static ssize_t cmpc_accel_sensitivity_store(struct device *dev, acpi = to_acpi_device(dev); inputdev = dev_get_drvdata(&acpi->dev); + if (!inputdev) + return -ENXIO; + accel = dev_get_drvdata(&inputdev->dev); + if (!accel) + return -ENXIO; r = kstrtoul(buf, 0, &sensitivity); if (r) diff --git a/drivers/platform/x86/dell/alienware-wmi-wmax.c b/drivers/platform/x86/dell/alienware-wmi-wmax.c index 01af6dde9057f..fdae40689f178 100644 --- a/drivers/platform/x86/dell/alienware-wmi-wmax.c +++ b/drivers/platform/x86/dell/alienware-wmi-wmax.c @@ -175,7 +175,7 @@ static const struct dmi_system_id awcc_dmi_table[] __initconst = { DMI_MATCH(DMI_SYS_VENDOR, "Alienware"), DMI_MATCH(DMI_PRODUCT_NAME, "Alienware m18"), }, - .driver_data = &generic_quirks, + .driver_data = &g_series_quirks, }, { .ident = "Alienware x15", diff --git a/drivers/platform/x86/dell/dell-wmi-base.c b/drivers/platform/x86/dell/dell-wmi-base.c index 28076929d6af5..907f1da01c8db 100644 --- a/drivers/platform/x86/dell/dell-wmi-base.c +++ b/drivers/platform/x86/dell/dell-wmi-base.c @@ -80,6 +80,12 @@ static const struct dmi_system_id dell_wmi_smbios_list[] __initconst = { static const struct key_entry dell_wmi_keymap_type_0000[] = { { KE_IGNORE, 0x003a, { KEY_CAPSLOCK } }, + /* Audio mute toggle */ + { KE_KEY, 0x0109, { KEY_MUTE } }, + + /* Mic mute toggle */ + { KE_KEY, 0x0150, { KEY_MICMUTE } }, + /* Meta key lock */ { KE_IGNORE, 0xe000, { KEY_RIGHTMETA } }, diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/passwordattr-interface.c b/drivers/platform/x86/dell/dell-wmi-sysman/passwordattr-interface.c index 86ec962aace9b..e586f7957946b 100644 --- a/drivers/platform/x86/dell/dell-wmi-sysman/passwordattr-interface.c +++ b/drivers/platform/x86/dell/dell-wmi-sysman/passwordattr-interface.c @@ -93,7 +93,6 @@ int set_new_password(const char *password_type, const char *new) if (ret < 0) goto out; - print_hex_dump_bytes("set new password data: ", DUMP_PREFIX_NONE, buffer, buffer_size); ret = call_password_interface(wmi_priv.password_attr_wdev, buffer, buffer_size); /* on success copy the new password to current password */ if (!ret) diff --git a/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c index f346aad8e9d89..af4d1920d4880 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c +++ b/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c @@ -94,8 +94,11 @@ int hp_alloc_enumeration_data(void) bioscfg_drv.enumeration_instances_count = hp_get_instance_count(HP_WMI_BIOS_ENUMERATION_GUID); - bioscfg_drv.enumeration_data = kcalloc(bioscfg_drv.enumeration_instances_count, - sizeof(*bioscfg_drv.enumeration_data), GFP_KERNEL); + if (!bioscfg_drv.enumeration_instances_count) + return -EINVAL; + bioscfg_drv.enumeration_data = kvcalloc(bioscfg_drv.enumeration_instances_count, + sizeof(*bioscfg_drv.enumeration_data), GFP_KERNEL); + if (!bioscfg_drv.enumeration_data) { bioscfg_drv.enumeration_instances_count = 0; return -ENOMEM; @@ -444,6 +447,6 @@ void hp_exit_enumeration_attributes(void) } bioscfg_drv.enumeration_instances_count = 0; - kfree(bioscfg_drv.enumeration_data); + kvfree(bioscfg_drv.enumeration_data); bioscfg_drv.enumeration_data = NULL; } diff --git a/drivers/platform/x86/hp/hp-wmi.c b/drivers/platform/x86/hp/hp-wmi.c index ad9d9f97960f2..dfe45692c956a 100644 --- a/drivers/platform/x86/hp/hp-wmi.c +++ b/drivers/platform/x86/hp/hp-wmi.c @@ -53,6 +53,66 @@ MODULE_ALIAS("wmi:5FB7F034-2C63-45E9-BE91-3D44E2C707E4"); #define zero_if_sup(tmp) (zero_insize_support?0:sizeof(tmp)) // use when zero insize is required +enum hp_thermal_profile_omen_v0 { + HP_OMEN_V0_THERMAL_PROFILE_DEFAULT = 0x00, + HP_OMEN_V0_THERMAL_PROFILE_PERFORMANCE = 0x01, + HP_OMEN_V0_THERMAL_PROFILE_COOL = 0x02, +}; + +enum hp_thermal_profile_omen_v1 { + HP_OMEN_V1_THERMAL_PROFILE_DEFAULT = 0x30, + HP_OMEN_V1_THERMAL_PROFILE_PERFORMANCE = 0x31, + HP_OMEN_V1_THERMAL_PROFILE_COOL = 0x50, +}; + +enum hp_thermal_profile_omen_flags { + HP_OMEN_EC_FLAGS_TURBO = 0x04, + HP_OMEN_EC_FLAGS_NOTIMER = 0x02, + HP_OMEN_EC_FLAGS_JUSTSET = 0x01, +}; + +enum hp_thermal_profile_victus { + HP_VICTUS_THERMAL_PROFILE_DEFAULT = 0x00, + HP_VICTUS_THERMAL_PROFILE_PERFORMANCE = 0x01, + HP_VICTUS_THERMAL_PROFILE_QUIET = 0x03, +}; + +enum hp_thermal_profile_victus_s { + HP_VICTUS_S_THERMAL_PROFILE_DEFAULT = 0x00, + HP_VICTUS_S_THERMAL_PROFILE_PERFORMANCE = 0x01, +}; + +enum hp_thermal_profile { + HP_THERMAL_PROFILE_PERFORMANCE = 0x00, + HP_THERMAL_PROFILE_DEFAULT = 0x01, + HP_THERMAL_PROFILE_COOL = 0x02, + HP_THERMAL_PROFILE_QUIET = 0x03, +}; + +struct thermal_profile_params { + u8 performance; + u8 balanced; + u8 low_power; +}; + +static const struct thermal_profile_params victus_s_thermal_params = { + .performance = HP_VICTUS_S_THERMAL_PROFILE_PERFORMANCE, + .balanced = HP_VICTUS_S_THERMAL_PROFILE_DEFAULT, + .low_power = HP_VICTUS_S_THERMAL_PROFILE_DEFAULT, +}; + +static const struct thermal_profile_params omen_v1_thermal_params = { + .performance = HP_OMEN_V1_THERMAL_PROFILE_PERFORMANCE, + .balanced = HP_OMEN_V1_THERMAL_PROFILE_DEFAULT, + .low_power = HP_OMEN_V1_THERMAL_PROFILE_DEFAULT, +}; + +/* + * A generic pointer for the currently-active board's thermal profile + * parameters. + */ +static struct thermal_profile_params *active_thermal_profile_params; + /* DMI board names of devices that should use the omen specific path for * thermal profiles. * This was obtained by taking a look in the windows omen command center @@ -93,12 +153,40 @@ static const char * const victus_thermal_profile_boards[] = { }; /* DMI Board names of Victus 16-r and Victus 16-s laptops */ -static const char * const victus_s_thermal_profile_boards[] = { - "8BBE", "8BD4", "8BD5", - "8C78", "8C99", "8C9C", - "8D41", +static const struct dmi_system_id victus_s_thermal_profile_boards[] __initconst = { + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BBE") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BD4") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8BD5") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8C78") }, + .driver_data = (void *)&omen_v1_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8C99") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8C9C") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8D41") }, + .driver_data = (void *)&victus_s_thermal_params, + }, + {}, }; +static bool is_victus_s_board; + enum hp_wmi_radio { HPWMI_WIFI = 0x0, HPWMI_BLUETOOTH = 0x1, @@ -219,42 +307,6 @@ enum hp_wireless2_bits { HPWMI_POWER_FW_OR_HW = HPWMI_POWER_BIOS | HPWMI_POWER_HARD, }; -enum hp_thermal_profile_omen_v0 { - HP_OMEN_V0_THERMAL_PROFILE_DEFAULT = 0x00, - HP_OMEN_V0_THERMAL_PROFILE_PERFORMANCE = 0x01, - HP_OMEN_V0_THERMAL_PROFILE_COOL = 0x02, -}; - -enum hp_thermal_profile_omen_v1 { - HP_OMEN_V1_THERMAL_PROFILE_DEFAULT = 0x30, - HP_OMEN_V1_THERMAL_PROFILE_PERFORMANCE = 0x31, - HP_OMEN_V1_THERMAL_PROFILE_COOL = 0x50, -}; - -enum hp_thermal_profile_omen_flags { - HP_OMEN_EC_FLAGS_TURBO = 0x04, - HP_OMEN_EC_FLAGS_NOTIMER = 0x02, - HP_OMEN_EC_FLAGS_JUSTSET = 0x01, -}; - -enum hp_thermal_profile_victus { - HP_VICTUS_THERMAL_PROFILE_DEFAULT = 0x00, - HP_VICTUS_THERMAL_PROFILE_PERFORMANCE = 0x01, - HP_VICTUS_THERMAL_PROFILE_QUIET = 0x03, -}; - -enum hp_thermal_profile_victus_s { - HP_VICTUS_S_THERMAL_PROFILE_DEFAULT = 0x00, - HP_VICTUS_S_THERMAL_PROFILE_PERFORMANCE = 0x01, -}; - -enum hp_thermal_profile { - HP_THERMAL_PROFILE_PERFORMANCE = 0x00, - HP_THERMAL_PROFILE_DEFAULT = 0x01, - HP_THERMAL_PROFILE_COOL = 0x02, - HP_THERMAL_PROFILE_QUIET = 0x03, -}; - #define IS_HWBLOCKED(x) ((x & HPWMI_POWER_FW_OR_HW) != HPWMI_POWER_FW_OR_HW) #define IS_SWBLOCKED(x) !(x & HPWMI_POWER_SOFT) @@ -1575,15 +1627,8 @@ static int platform_profile_victus_set_ec(enum platform_profile_option profile) static bool is_victus_s_thermal_profile(void) { - const char *board_name; - - board_name = dmi_get_system_info(DMI_BOARD_NAME); - if (!board_name) - return false; - - return match_string(victus_s_thermal_profile_boards, - ARRAY_SIZE(victus_s_thermal_profile_boards), - board_name) >= 0; + /* Initialised in driver init, hence safe to use here */ + return is_victus_s_board; } static int victus_s_gpu_thermal_profile_get(bool *ctgp_enable, @@ -1666,25 +1711,30 @@ static int victus_s_set_cpu_pl1_pl2(u8 pl1, u8 pl2) static int platform_profile_victus_s_set_ec(enum platform_profile_option profile) { + struct thermal_profile_params *params; bool gpu_ctgp_enable, gpu_ppab_enable; u8 gpu_dstate; /* Test shows 1 = 100%, 2 = 50%, 3 = 25%, 4 = 12.5% */ int err, tp; + params = active_thermal_profile_params; + if (!params) + return -ENODEV; + switch (profile) { case PLATFORM_PROFILE_PERFORMANCE: - tp = HP_VICTUS_S_THERMAL_PROFILE_PERFORMANCE; + tp = params->performance; gpu_ctgp_enable = true; gpu_ppab_enable = true; gpu_dstate = 1; break; case PLATFORM_PROFILE_BALANCED: - tp = HP_VICTUS_S_THERMAL_PROFILE_DEFAULT; + tp = params->balanced; gpu_ctgp_enable = false; gpu_ppab_enable = true; gpu_dstate = 1; break; case PLATFORM_PROFILE_LOW_POWER: - tp = HP_VICTUS_S_THERMAL_PROFILE_DEFAULT; + tp = params->low_power; gpu_ctgp_enable = false; gpu_ppab_enable = false; gpu_dstate = 1; @@ -2221,6 +2271,26 @@ static int hp_wmi_hwmon_init(void) return 0; } +static void __init setup_active_thermal_profile_params(void) +{ + const struct dmi_system_id *id; + + /* + * Currently only victus_s devices use the + * active_thermal_profile_params + */ + id = dmi_first_match(victus_s_thermal_profile_boards); + if (id) { + /* + * Marking this boolean is required to ensure that + * is_victus_s_thermal_profile() behaves like a valid + * wrapper. + */ + is_victus_s_board = true; + active_thermal_profile_params = id->driver_data; + } +} + static int __init hp_wmi_init(void) { int event_capable = wmi_has_guid(HPWMI_EVENT_GUID); @@ -2248,6 +2318,11 @@ static int __init hp_wmi_init(void) goto err_destroy_input; } + /* + * Setup active board's thermal profile parameters before + * starting platform driver probe. + */ + setup_active_thermal_profile_params(); err = platform_driver_probe(&hp_wmi_driver, hp_wmi_bios_setup); if (err) goto err_unregister_device; diff --git a/drivers/platform/x86/intel/int0002_vgpio.c b/drivers/platform/x86/intel/int0002_vgpio.c index 6f5629dc3f8db..562e880256436 100644 --- a/drivers/platform/x86/intel/int0002_vgpio.c +++ b/drivers/platform/x86/intel/int0002_vgpio.c @@ -206,8 +206,8 @@ static int int0002_probe(struct platform_device *pdev) * FIXME: augment this if we managed to pull handling of shared * IRQs into gpiolib. */ - ret = devm_request_irq(dev, irq, int0002_irq, - IRQF_ONESHOT | IRQF_SHARED, "INT0002", chip); + ret = devm_request_irq(dev, irq, int0002_irq, IRQF_SHARED, "INT0002", + chip); if (ret) { dev_err(dev, "Error requesting IRQ %d: %d\n", irq, ret); return ret; diff --git a/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c index 34bff2f65a835..13b11c3a2ec4e 100644 --- a/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c +++ b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c @@ -612,6 +612,9 @@ static long isst_if_core_power_state(void __user *argp) return -EINVAL; if (core_power.get_set) { + if (power_domain_info->write_blocked) + return -EPERM; + _write_cp_info("cp_enable", core_power.enable, SST_CP_CONTROL_OFFSET, SST_CP_ENABLE_START, SST_CP_ENABLE_WIDTH, SST_MUL_FACTOR_NONE) _write_cp_info("cp_prio_type", core_power.priority_type, SST_CP_CONTROL_OFFSET, @@ -1720,55 +1723,67 @@ EXPORT_SYMBOL_NS_GPL(tpmi_sst_dev_remove, "INTEL_TPMI_SST"); void tpmi_sst_dev_suspend(struct auxiliary_device *auxdev) { struct tpmi_sst_struct *tpmi_sst = auxiliary_get_drvdata(auxdev); - struct tpmi_per_power_domain_info *power_domain_info; + struct tpmi_per_power_domain_info *power_domain_info, *pd_info; struct oobmsm_plat_info *plat_info; void __iomem *cp_base; + int num_resources, i; plat_info = tpmi_get_platform_data(auxdev); if (!plat_info) return; power_domain_info = tpmi_sst->power_domain_info[plat_info->partition]; + num_resources = tpmi_sst->number_of_power_domains[plat_info->partition]; - cp_base = power_domain_info->sst_base + power_domain_info->sst_header.cp_offset; - power_domain_info->saved_sst_cp_control = readq(cp_base + SST_CP_CONTROL_OFFSET); - - memcpy_fromio(power_domain_info->saved_clos_configs, cp_base + SST_CLOS_CONFIG_0_OFFSET, - sizeof(power_domain_info->saved_clos_configs)); + for (i = 0; i < num_resources; i++) { + pd_info = &power_domain_info[i]; + if (!pd_info || !pd_info->sst_base) + continue; - memcpy_fromio(power_domain_info->saved_clos_assocs, cp_base + SST_CLOS_ASSOC_0_OFFSET, - sizeof(power_domain_info->saved_clos_assocs)); + cp_base = pd_info->sst_base + pd_info->sst_header.cp_offset; + pd_info->saved_sst_cp_control = readq(cp_base + SST_CP_CONTROL_OFFSET); + memcpy_fromio(pd_info->saved_clos_configs, cp_base + SST_CLOS_CONFIG_0_OFFSET, + sizeof(pd_info->saved_clos_configs)); + memcpy_fromio(pd_info->saved_clos_assocs, cp_base + SST_CLOS_ASSOC_0_OFFSET, + sizeof(pd_info->saved_clos_assocs)); - power_domain_info->saved_pp_control = readq(power_domain_info->sst_base + - power_domain_info->sst_header.pp_offset + - SST_PP_CONTROL_OFFSET); + pd_info->saved_pp_control = readq(pd_info->sst_base + + pd_info->sst_header.pp_offset + + SST_PP_CONTROL_OFFSET); + } } EXPORT_SYMBOL_NS_GPL(tpmi_sst_dev_suspend, "INTEL_TPMI_SST"); void tpmi_sst_dev_resume(struct auxiliary_device *auxdev) { struct tpmi_sst_struct *tpmi_sst = auxiliary_get_drvdata(auxdev); - struct tpmi_per_power_domain_info *power_domain_info; + struct tpmi_per_power_domain_info *power_domain_info, *pd_info; struct oobmsm_plat_info *plat_info; void __iomem *cp_base; + int num_resources, i; plat_info = tpmi_get_platform_data(auxdev); if (!plat_info) return; power_domain_info = tpmi_sst->power_domain_info[plat_info->partition]; + num_resources = tpmi_sst->number_of_power_domains[plat_info->partition]; - cp_base = power_domain_info->sst_base + power_domain_info->sst_header.cp_offset; - writeq(power_domain_info->saved_sst_cp_control, cp_base + SST_CP_CONTROL_OFFSET); - - memcpy_toio(cp_base + SST_CLOS_CONFIG_0_OFFSET, power_domain_info->saved_clos_configs, - sizeof(power_domain_info->saved_clos_configs)); + for (i = 0; i < num_resources; i++) { + pd_info = &power_domain_info[i]; + if (!pd_info || !pd_info->sst_base) + continue; - memcpy_toio(cp_base + SST_CLOS_ASSOC_0_OFFSET, power_domain_info->saved_clos_assocs, - sizeof(power_domain_info->saved_clos_assocs)); + cp_base = pd_info->sst_base + pd_info->sst_header.cp_offset; + writeq(pd_info->saved_sst_cp_control, cp_base + SST_CP_CONTROL_OFFSET); + memcpy_toio(cp_base + SST_CLOS_CONFIG_0_OFFSET, pd_info->saved_clos_configs, + sizeof(pd_info->saved_clos_configs)); + memcpy_toio(cp_base + SST_CLOS_ASSOC_0_OFFSET, pd_info->saved_clos_assocs, + sizeof(pd_info->saved_clos_assocs)); - writeq(power_domain_info->saved_pp_control, power_domain_info->sst_base + - power_domain_info->sst_header.pp_offset + SST_PP_CONTROL_OFFSET); + writeq(pd_info->saved_pp_control, power_domain_info->sst_base + + pd_info->sst_header.pp_offset + SST_PP_CONTROL_OFFSET); + } } EXPORT_SYMBOL_NS_GPL(tpmi_sst_dev_resume, "INTEL_TPMI_SST"); diff --git a/drivers/platform/x86/lenovo/thinkpad_acpi.c b/drivers/platform/x86/lenovo/thinkpad_acpi.c index cc19fe520ea96..075543cd0e77e 100644 --- a/drivers/platform/x86/lenovo/thinkpad_acpi.c +++ b/drivers/platform/x86/lenovo/thinkpad_acpi.c @@ -9525,14 +9525,16 @@ static int tpacpi_battery_get(int what, int battery, int *ret) { switch (what) { case THRESHOLD_START: - if ACPI_FAILURE(tpacpi_battery_acpi_eval(GET_START, ret, battery)) + if (!battery_info.batteries[battery].start_support || + ACPI_FAILURE(tpacpi_battery_acpi_eval(GET_START, ret, battery))) return -ENODEV; /* The value is in the low 8 bits of the response */ *ret = *ret & 0xFF; return 0; case THRESHOLD_STOP: - if ACPI_FAILURE(tpacpi_battery_acpi_eval(GET_STOP, ret, battery)) + if (!battery_info.batteries[battery].stop_support || + ACPI_FAILURE(tpacpi_battery_acpi_eval(GET_STOP, ret, battery))) return -ENODEV; /* Value is in lower 8 bits */ *ret = *ret & 0xFF; diff --git a/drivers/platform/x86/panasonic-laptop.c b/drivers/platform/x86/panasonic-laptop.c index 255317e6fec88..937f1a5b78edf 100644 --- a/drivers/platform/x86/panasonic-laptop.c +++ b/drivers/platform/x86/panasonic-laptop.c @@ -1089,7 +1089,7 @@ static int acpi_pcc_hotkey_add(struct acpi_device *device) PLATFORM_DEVID_NONE, NULL, 0); if (IS_ERR(pcc->platform)) { result = PTR_ERR(pcc->platform); - goto out_backlight; + goto out_sysfs; } result = device_create_file(&pcc->platform->dev, &dev_attr_cdpower); @@ -1105,6 +1105,8 @@ static int acpi_pcc_hotkey_add(struct acpi_device *device) out_platform: platform_device_unregister(pcc->platform); +out_sysfs: + sysfs_remove_group(&device->dev.kobj, &pcc_attr_group); out_backlight: backlight_device_unregister(pcc->backlight); out_input: diff --git a/drivers/power/reset/nvmem-reboot-mode.c b/drivers/power/reset/nvmem-reboot-mode.c index 5d73dde585b1f..becbe33f04b30 100644 --- a/drivers/power/reset/nvmem-reboot-mode.c +++ b/drivers/power/reset/nvmem-reboot-mode.c @@ -10,6 +10,7 @@ #include #include #include +#include struct nvmem_reboot_mode { struct reboot_mode_driver reboot; @@ -19,17 +20,21 @@ struct nvmem_reboot_mode { static int nvmem_reboot_mode_write(struct reboot_mode_driver *reboot, u64 magic) { struct nvmem_reboot_mode *nvmem_rbm; - u32 magic_32; + size_t buf_len; + void *buf; int ret; - if (magic > U32_MAX) - return -EINVAL; + nvmem_rbm = container_of(reboot, struct nvmem_reboot_mode, reboot); - magic_32 = magic; + buf = nvmem_cell_read(nvmem_rbm->cell, &buf_len); + if (IS_ERR(buf)) + return PTR_ERR(buf); + kfree(buf); - nvmem_rbm = container_of(reboot, struct nvmem_reboot_mode, reboot); + if (buf_len > sizeof(magic)) + return -EINVAL; - ret = nvmem_cell_write(nvmem_rbm->cell, &magic_32, sizeof(magic_32)); + ret = nvmem_cell_write(nvmem_rbm->cell, &magic, buf_len); if (ret < 0) dev_err(reboot->dev, "update reboot mode bits failed\n"); diff --git a/drivers/power/reset/tdx-ec-poweroff.c b/drivers/power/reset/tdx-ec-poweroff.c index 3302a127fce52..8040aa03d74d4 100644 --- a/drivers/power/reset/tdx-ec-poweroff.c +++ b/drivers/power/reset/tdx-ec-poweroff.c @@ -8,7 +8,10 @@ */ #include +#include +#include #include +#include #include #include #include @@ -31,6 +34,8 @@ #define EC_REG_MAX 0xD0 +#define EC_CMD_TIMEOUT_MS 1000 + static const struct regmap_range volatile_ranges[] = { regmap_reg_range(EC_CMD_REG, EC_CMD_REG), }; @@ -75,6 +80,13 @@ static int tdx_ec_power_off(struct sys_off_data *data) err = tdx_ec_cmd(regmap, EC_CMD_POWEROFF); + if (err) { + dev_err(data->dev, "Failed to send power off command\n"); + } else { + mdelay(EC_CMD_TIMEOUT_MS); + WARN_ONCE(1, "Unable to power off system\n"); + } + return err ? NOTIFY_BAD : NOTIFY_DONE; } @@ -85,6 +97,13 @@ static int tdx_ec_restart(struct sys_off_data *data) err = tdx_ec_cmd(regmap, EC_CMD_RESET); + if (err) { + dev_err(data->dev, "Failed to send restart command\n"); + } else { + mdelay(EC_CMD_TIMEOUT_MS); + WARN_ONCE(1, "Unable to restart system\n"); + } + return err ? NOTIFY_BAD : NOTIFY_DONE; } diff --git a/drivers/power/sequencing/core.c b/drivers/power/sequencing/core.c index 190564e559885..1fcf0af7cc0bb 100644 --- a/drivers/power/sequencing/core.c +++ b/drivers/power/sequencing/core.c @@ -914,8 +914,10 @@ int pwrseq_power_on(struct pwrseq_desc *desc) if (target->post_enable) { ret = target->post_enable(pwrseq); if (ret) { - pwrseq_unit_disable(pwrseq, unit); - desc->powered_on = false; + scoped_guard(mutex, &pwrseq->state_lock) { + pwrseq_unit_disable(pwrseq, unit); + desc->powered_on = false; + } } } diff --git a/drivers/power/supply/ab8500_charger.c b/drivers/power/supply/ab8500_charger.c index 5f4537766e5b9..1813fbdfa1c1f 100644 --- a/drivers/power/supply/ab8500_charger.c +++ b/drivers/power/supply/ab8500_charger.c @@ -3466,26 +3466,6 @@ static int ab8500_charger_probe(struct platform_device *pdev) return ret; } - /* Request interrupts */ - for (i = 0; i < ARRAY_SIZE(ab8500_charger_irq); i++) { - irq = platform_get_irq_byname(pdev, ab8500_charger_irq[i].name); - if (irq < 0) - return irq; - - ret = devm_request_threaded_irq(dev, - irq, NULL, ab8500_charger_irq[i].isr, - IRQF_SHARED | IRQF_NO_SUSPEND | IRQF_ONESHOT, - ab8500_charger_irq[i].name, di); - - if (ret != 0) { - dev_err(dev, "failed to request %s IRQ %d: %d\n" - , ab8500_charger_irq[i].name, irq, ret); - return ret; - } - dev_dbg(dev, "Requested %s IRQ %d: %d\n", - ab8500_charger_irq[i].name, irq, ret); - } - /* initialize lock */ spin_lock_init(&di->usb_state.usb_lock); mutex_init(&di->usb_ipt_crnt_lock); @@ -3614,6 +3594,26 @@ static int ab8500_charger_probe(struct platform_device *pdev) return PTR_ERR(di->usb_chg.psy); } + /* Request interrupts */ + for (i = 0; i < ARRAY_SIZE(ab8500_charger_irq); i++) { + irq = platform_get_irq_byname(pdev, ab8500_charger_irq[i].name); + if (irq < 0) + return irq; + + ret = devm_request_threaded_irq(dev, + irq, NULL, ab8500_charger_irq[i].isr, + IRQF_SHARED | IRQF_NO_SUSPEND | IRQF_ONESHOT, + ab8500_charger_irq[i].name, di); + + if (ret != 0) { + dev_err(dev, "failed to request %s IRQ %d: %d\n" + , ab8500_charger_irq[i].name, irq, ret); + return ret; + } + dev_dbg(dev, "Requested %s IRQ %d: %d\n", + ab8500_charger_irq[i].name, irq, ret); + } + /* * Check what battery we have, since we always have the USB * psy, use that as a handle. diff --git a/drivers/power/supply/act8945a_charger.c b/drivers/power/supply/act8945a_charger.c index 3901a02f326a5..9dec4486b1439 100644 --- a/drivers/power/supply/act8945a_charger.c +++ b/drivers/power/supply/act8945a_charger.c @@ -597,14 +597,6 @@ static int act8945a_charger_probe(struct platform_device *pdev) return irq ?: -ENXIO; } - ret = devm_request_irq(&pdev->dev, irq, act8945a_status_changed, - IRQF_TRIGGER_FALLING, "act8945a_interrupt", - charger); - if (ret) { - dev_err(&pdev->dev, "failed to request nIRQ pin IRQ\n"); - return ret; - } - charger->desc.name = "act8945a-charger"; charger->desc.get_property = act8945a_charger_get_property; charger->desc.properties = act8945a_charger_props; @@ -625,6 +617,14 @@ static int act8945a_charger_probe(struct platform_device *pdev) return PTR_ERR(charger->psy); } + ret = devm_request_irq(&pdev->dev, irq, act8945a_status_changed, + IRQF_TRIGGER_FALLING, "act8945a_interrupt", + charger); + if (ret) { + dev_err(&pdev->dev, "failed to request nIRQ pin IRQ\n"); + return ret; + } + platform_set_drvdata(pdev, charger); INIT_WORK(&charger->work, act8945a_work); diff --git a/drivers/power/supply/bq256xx_charger.c b/drivers/power/supply/bq256xx_charger.c index ae14162f017a9..d3de4f8b80db1 100644 --- a/drivers/power/supply/bq256xx_charger.c +++ b/drivers/power/supply/bq256xx_charger.c @@ -1741,6 +1741,12 @@ static int bq256xx_probe(struct i2c_client *client) usb_register_notifier(bq->usb3_phy, &bq->usb_nb); } + ret = bq256xx_power_supply_init(bq, &psy_cfg, dev); + if (ret) { + dev_err(dev, "Failed to register power supply\n"); + return ret; + } + if (client->irq) { ret = devm_request_threaded_irq(dev, client->irq, NULL, bq256xx_irq_handler_thread, @@ -1753,12 +1759,6 @@ static int bq256xx_probe(struct i2c_client *client) } } - ret = bq256xx_power_supply_init(bq, &psy_cfg, dev); - if (ret) { - dev_err(dev, "Failed to register power supply\n"); - return ret; - } - ret = bq256xx_hw_init(bq); if (ret) { dev_err(dev, "Cannot initialize the chip.\n"); diff --git a/drivers/power/supply/bq25980_charger.c b/drivers/power/supply/bq25980_charger.c index 723858d62d141..73f06f09f134c 100644 --- a/drivers/power/supply/bq25980_charger.c +++ b/drivers/power/supply/bq25980_charger.c @@ -1241,6 +1241,12 @@ static int bq25980_probe(struct i2c_client *client) return ret; } + ret = bq25980_power_supply_init(bq, dev); + if (ret) { + dev_err(dev, "Failed to register power supply\n"); + return ret; + } + if (client->irq) { ret = devm_request_threaded_irq(dev, client->irq, NULL, bq25980_irq_handler_thread, @@ -1251,12 +1257,6 @@ static int bq25980_probe(struct i2c_client *client) return ret; } - ret = bq25980_power_supply_init(bq, dev); - if (ret) { - dev_err(dev, "Failed to register power supply\n"); - return ret; - } - ret = bq25980_hw_init(bq); if (ret) { dev_err(dev, "Cannot initialize the chip.\n"); diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c index 19445e39651c7..45f0e39b8c2dd 100644 --- a/drivers/power/supply/bq27xxx_battery.c +++ b/drivers/power/supply/bq27xxx_battery.c @@ -1172,7 +1172,7 @@ static inline int bq27xxx_write(struct bq27xxx_device_info *di, int reg_index, return -EINVAL; if (!di->bus.write) - return -EPERM; + return -EOPNOTSUPP; ret = di->bus.write(di, di->regs[reg_index], value, single); if (ret < 0) @@ -1191,7 +1191,7 @@ static inline int bq27xxx_read_block(struct bq27xxx_device_info *di, int reg_ind return -EINVAL; if (!di->bus.read_bulk) - return -EPERM; + return -EOPNOTSUPP; ret = di->bus.read_bulk(di, di->regs[reg_index], data, len); if (ret < 0) @@ -1210,7 +1210,7 @@ static inline int bq27xxx_write_block(struct bq27xxx_device_info *di, int reg_in return -EINVAL; if (!di->bus.write_bulk) - return -EPERM; + return -EOPNOTSUPP; ret = di->bus.write_bulk(di, di->regs[reg_index], data, len); if (ret < 0) diff --git a/drivers/power/supply/cpcap-battery.c b/drivers/power/supply/cpcap-battery.c index 8106d1edcbc26..507fdc1c866d5 100644 --- a/drivers/power/supply/cpcap-battery.c +++ b/drivers/power/supply/cpcap-battery.c @@ -1122,10 +1122,6 @@ static int cpcap_battery_probe(struct platform_device *pdev) platform_set_drvdata(pdev, ddata); - error = cpcap_battery_init_interrupts(pdev, ddata); - if (error) - return error; - error = cpcap_battery_init_iio(ddata); if (error) return error; @@ -1142,6 +1138,10 @@ static int cpcap_battery_probe(struct platform_device *pdev) return error; } + error = cpcap_battery_init_interrupts(pdev, ddata); + if (error) + return error; + atomic_set(&ddata->active, 1); error = cpcap_battery_calibrate(ddata); diff --git a/drivers/power/supply/goldfish_battery.c b/drivers/power/supply/goldfish_battery.c index 479195e35d734..5aa24e4dc4455 100644 --- a/drivers/power/supply/goldfish_battery.c +++ b/drivers/power/supply/goldfish_battery.c @@ -224,12 +224,6 @@ static int goldfish_battery_probe(struct platform_device *pdev) if (data->irq < 0) return -ENODEV; - ret = devm_request_irq(&pdev->dev, data->irq, - goldfish_battery_interrupt, - IRQF_SHARED, pdev->name, data); - if (ret) - return ret; - psy_cfg.drv_data = data; data->ac = devm_power_supply_register(&pdev->dev, @@ -244,6 +238,12 @@ static int goldfish_battery_probe(struct platform_device *pdev) if (IS_ERR(data->battery)) return PTR_ERR(data->battery); + ret = devm_request_irq(&pdev->dev, data->irq, + goldfish_battery_interrupt, + IRQF_SHARED, pdev->name, data); + if (ret) + return ret; + GOLDFISH_BATTERY_WRITE(data, BATTERY_INT_ENABLE, BATTERY_INT_MASK); return 0; } diff --git a/drivers/power/supply/pm8916_bms_vm.c b/drivers/power/supply/pm8916_bms_vm.c index 5120be086e6ff..de5d571c03e21 100644 --- a/drivers/power/supply/pm8916_bms_vm.c +++ b/drivers/power/supply/pm8916_bms_vm.c @@ -167,15 +167,6 @@ static int pm8916_bms_vm_battery_probe(struct platform_device *pdev) if (ret < 0) return -EINVAL; - irq = platform_get_irq_byname(pdev, "fifo"); - if (irq < 0) - return irq; - - ret = devm_request_threaded_irq(dev, irq, NULL, pm8916_bms_vm_fifo_update_done_irq, - IRQF_ONESHOT, "pm8916_vm_bms", bat); - if (ret) - return ret; - ret = regmap_bulk_read(bat->regmap, bat->reg + PM8916_PERPH_TYPE, &tmp, 2); if (ret) goto comm_error; @@ -220,6 +211,15 @@ static int pm8916_bms_vm_battery_probe(struct platform_device *pdev) if (ret) return dev_err_probe(dev, ret, "Unable to get battery info\n"); + irq = platform_get_irq_byname(pdev, "fifo"); + if (irq < 0) + return irq; + + ret = devm_request_threaded_irq(dev, irq, NULL, pm8916_bms_vm_fifo_update_done_irq, + IRQF_ONESHOT, "pm8916_vm_bms", bat); + if (ret) + return ret; + platform_set_drvdata(pdev, bat); return 0; diff --git a/drivers/power/supply/pm8916_lbc.c b/drivers/power/supply/pm8916_lbc.c index c74b75b1b2676..6b631012a7959 100644 --- a/drivers/power/supply/pm8916_lbc.c +++ b/drivers/power/supply/pm8916_lbc.c @@ -274,15 +274,6 @@ static int pm8916_lbc_charger_probe(struct platform_device *pdev) return dev_err_probe(dev, -EINVAL, "Wrong amount of reg values: %d (4 expected)\n", len); - irq = platform_get_irq_byname(pdev, "usb_vbus"); - if (irq < 0) - return irq; - - ret = devm_request_threaded_irq(dev, irq, NULL, pm8916_lbc_charger_state_changed_irq, - IRQF_ONESHOT, "pm8916_lbc", chg); - if (ret) - return ret; - ret = device_property_read_u32_array(dev, "reg", chg->reg, len); if (ret) return ret; @@ -332,6 +323,10 @@ static int pm8916_lbc_charger_probe(struct platform_device *pdev) if (ret) return dev_err_probe(dev, ret, "Unable to get battery info\n"); + irq = platform_get_irq_byname(pdev, "usb_vbus"); + if (irq < 0) + return irq; + chg->edev = devm_extcon_dev_allocate(dev, pm8916_lbc_charger_cable); if (IS_ERR(chg->edev)) return PTR_ERR(chg->edev); @@ -340,6 +335,11 @@ static int pm8916_lbc_charger_probe(struct platform_device *pdev) if (ret < 0) return dev_err_probe(dev, ret, "failed to register extcon device\n"); + ret = devm_request_threaded_irq(dev, irq, NULL, pm8916_lbc_charger_state_changed_irq, + IRQF_ONESHOT, "pm8916_lbc", chg); + if (ret) + return ret; + ret = regmap_read(chg->regmap, chg->reg[LBC_USB] + PM8916_INT_RT_STS, &tmp); if (ret) goto comm_error; diff --git a/drivers/power/supply/qcom_battmgr.c b/drivers/power/supply/qcom_battmgr.c index e6f01e0122e1c..ff77dba29a3ef 100644 --- a/drivers/power/supply/qcom_battmgr.c +++ b/drivers/power/supply/qcom_battmgr.c @@ -1244,7 +1244,8 @@ static unsigned int qcom_battmgr_sc8280xp_parse_technology(const char *chemistry if ((!strncmp(chemistry, "LIO", BATTMGR_CHEMISTRY_LEN)) || (!strncmp(chemistry, "OOI", BATTMGR_CHEMISTRY_LEN))) return POWER_SUPPLY_TECHNOLOGY_LION; - if (!strncmp(chemistry, "LIP", BATTMGR_CHEMISTRY_LEN)) + if (!strncmp(chemistry, "LIP", BATTMGR_CHEMISTRY_LEN) || + !strncmp(chemistry, "LiP", BATTMGR_CHEMISTRY_LEN)) return POWER_SUPPLY_TECHNOLOGY_LIPO; pr_err("Unknown battery technology '%s'\n", chemistry); diff --git a/drivers/power/supply/rt9455_charger.c b/drivers/power/supply/rt9455_charger.c index 1ffe7f02932f6..5130d2395e88f 100644 --- a/drivers/power/supply/rt9455_charger.c +++ b/drivers/power/supply/rt9455_charger.c @@ -1663,6 +1663,15 @@ static int rt9455_probe(struct i2c_client *client) rt9455_charger_config.supplied_to = rt9455_charger_supplied_to; rt9455_charger_config.num_supplicants = ARRAY_SIZE(rt9455_charger_supplied_to); + + info->charger = devm_power_supply_register(dev, &rt9455_charger_desc, + &rt9455_charger_config); + if (IS_ERR(info->charger)) { + dev_err(dev, "Failed to register charger\n"); + ret = PTR_ERR(info->charger); + goto put_usb_notifier; + } + ret = devm_request_threaded_irq(dev, client->irq, NULL, rt9455_irq_handler_thread, IRQF_TRIGGER_LOW | IRQF_ONESHOT, @@ -1678,14 +1687,6 @@ static int rt9455_probe(struct i2c_client *client) goto put_usb_notifier; } - info->charger = devm_power_supply_register(dev, &rt9455_charger_desc, - &rt9455_charger_config); - if (IS_ERR(info->charger)) { - dev_err(dev, "Failed to register charger\n"); - ret = PTR_ERR(info->charger); - goto put_usb_notifier; - } - return 0; put_usb_notifier: diff --git a/drivers/power/supply/sbs-battery.c b/drivers/power/supply/sbs-battery.c index 943c82ee978f4..43c48196c1674 100644 --- a/drivers/power/supply/sbs-battery.c +++ b/drivers/power/supply/sbs-battery.c @@ -1174,24 +1174,6 @@ static int sbs_probe(struct i2c_client *client) i2c_set_clientdata(client, chip); - if (!chip->gpio_detect) - goto skip_gpio; - - irq = gpiod_to_irq(chip->gpio_detect); - if (irq <= 0) { - dev_warn(&client->dev, "Failed to get gpio as irq: %d\n", irq); - goto skip_gpio; - } - - rc = devm_request_threaded_irq(&client->dev, irq, NULL, sbs_irq, - IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING | IRQF_ONESHOT, - dev_name(&client->dev), chip); - if (rc) { - dev_warn(&client->dev, "Failed to request irq: %d\n", rc); - goto skip_gpio; - } - -skip_gpio: /* * Before we register, we might need to make sure we can actually talk * to the battery. @@ -1217,6 +1199,24 @@ static int sbs_probe(struct i2c_client *client) return dev_err_probe(&client->dev, PTR_ERR(chip->power_supply), "Failed to register power supply\n"); + if (!chip->gpio_detect) + goto out; + + irq = gpiod_to_irq(chip->gpio_detect); + if (irq <= 0) { + dev_warn(&client->dev, "Failed to get gpio as irq: %d\n", irq); + goto out; + } + + rc = devm_request_threaded_irq(&client->dev, irq, NULL, sbs_irq, + IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING | IRQF_ONESHOT, + dev_name(&client->dev), chip); + if (rc) { + dev_warn(&client->dev, "Failed to request irq: %d\n", rc); + goto out; + } + +out: dev_info(&client->dev, "%s: battery gas gauge device registered\n", client->name); diff --git a/drivers/power/supply/wm97xx_battery.c b/drivers/power/supply/wm97xx_battery.c index b3b0c37a9dd2d..f00722c88c6fe 100644 --- a/drivers/power/supply/wm97xx_battery.c +++ b/drivers/power/supply/wm97xx_battery.c @@ -178,12 +178,6 @@ static int wm97xx_bat_probe(struct platform_device *dev) "failed to get charge GPIO\n"); if (charge_gpiod) { gpiod_set_consumer_name(charge_gpiod, "BATT CHRG"); - ret = request_irq(gpiod_to_irq(charge_gpiod), - wm97xx_chrg_irq, 0, - "AC Detect", dev); - if (ret) - return dev_err_probe(&dev->dev, ret, - "failed to request GPIO irq\n"); props++; /* POWER_SUPPLY_PROP_STATUS */ } @@ -199,10 +193,8 @@ static int wm97xx_bat_probe(struct platform_device *dev) props++; /* POWER_SUPPLY_PROP_VOLTAGE_MIN */ prop = kcalloc(props, sizeof(*prop), GFP_KERNEL); - if (!prop) { - ret = -ENOMEM; - goto err3; - } + if (!prop) + return -ENOMEM; prop[i++] = POWER_SUPPLY_PROP_PRESENT; if (charge_gpiod) @@ -236,15 +228,27 @@ static int wm97xx_bat_probe(struct platform_device *dev) schedule_work(&bat_work); } else { ret = PTR_ERR(bat_psy); - goto err4; + goto free; + } + + if (charge_gpiod) { + ret = request_irq(gpiod_to_irq(charge_gpiod), wm97xx_chrg_irq, + 0, "AC Detect", dev); + if (ret) { + dev_err_probe(&dev->dev, ret, + "failed to request GPIO irq\n"); + goto unregister; + } } return 0; -err4: + +unregister: + power_supply_unregister(bat_psy); + +free: kfree(prop); -err3: - if (charge_gpiod) - free_irq(gpiod_to_irq(charge_gpiod), dev); + return ret; } diff --git a/drivers/powercap/intel_rapl_msr.c b/drivers/powercap/intel_rapl_msr.c index c6b9a7debc354..e827247683dbf 100644 --- a/drivers/powercap/intel_rapl_msr.c +++ b/drivers/powercap/intel_rapl_msr.c @@ -140,6 +140,7 @@ static int rapl_msr_write_raw(int cpu, struct reg_action *ra) /* List of verified CPUs. */ static const struct x86_cpu_id pl4_support_ids[] = { + X86_MATCH_VFM(INTEL_ICELAKE_L, NULL), X86_MATCH_VFM(INTEL_TIGERLAKE_L, NULL), X86_MATCH_VFM(INTEL_ALDERLAKE, NULL), X86_MATCH_VFM(INTEL_ALDERLAKE_L, NULL), diff --git a/drivers/powercap/intel_rapl_tpmi.c b/drivers/powercap/intel_rapl_tpmi.c index 82201bf4685d4..34c0bd1edd61a 100644 --- a/drivers/powercap/intel_rapl_tpmi.c +++ b/drivers/powercap/intel_rapl_tpmi.c @@ -157,7 +157,7 @@ static int parse_one_domain(struct tpmi_rapl_package *trp, u32 offset) tpmi_domain_flags = tpmi_domain_header >> 32 & 0xffff; if (tpmi_domain_version == TPMI_VERSION_INVALID) { - pr_warn(FW_BUG "Invalid version\n"); + pr_debug("Invalid version, other instances may be valid\n"); return -ENODEV; } diff --git a/drivers/ptp/ptp_vmclock.c b/drivers/ptp/ptp_vmclock.c index b3a83b03d9c14..cbbfc494680c7 100644 --- a/drivers/ptp/ptp_vmclock.c +++ b/drivers/ptp/ptp_vmclock.c @@ -591,6 +591,7 @@ static int vmclock_probe(struct platform_device *pdev) static const struct acpi_device_id vmclock_acpi_ids[] = { { "AMZNC10C", 0 }, + { "VMCLOCK", 0 }, {} }; MODULE_DEVICE_TABLE(acpi, vmclock_acpi_ids); diff --git a/drivers/pwm/pwm-tiehrpwm.c b/drivers/pwm/pwm-tiehrpwm.c index 7a86cb090f76f..2533c95b0ba9d 100644 --- a/drivers/pwm/pwm-tiehrpwm.c +++ b/drivers/pwm/pwm-tiehrpwm.c @@ -237,8 +237,6 @@ static int ehrpwm_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, if (period_cycles < 1) period_cycles = 1; - pm_runtime_get_sync(pwmchip_parent(chip)); - /* Update clock prescaler values */ ehrpwm_modify(pc->mmio_base, TBCTL, TBCTL_CLKDIV_MASK, tb_divval); @@ -290,8 +288,6 @@ static int ehrpwm_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, if (!(duty_cycles > period_cycles)) ehrpwm_write(pc->mmio_base, cmp_reg, duty_cycles); - pm_runtime_put_sync(pwmchip_parent(chip)); - return 0; } @@ -378,6 +374,8 @@ static int ehrpwm_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, int err; bool enabled = pwm->state.enabled; + guard(pm_runtime_active)(pwmchip_parent(chip)); + if (state->polarity != pwm->state.polarity) { if (enabled) { ehrpwm_pwm_disable(chip, pwm); diff --git a/drivers/rapidio/rio-scan.c b/drivers/rapidio/rio-scan.c index c12941f71e2cb..dcd6619a4b027 100644 --- a/drivers/rapidio/rio-scan.c +++ b/drivers/rapidio/rio-scan.c @@ -854,7 +854,8 @@ static struct rio_net *rio_scan_alloc_net(struct rio_mport *mport, if (idtab == NULL) { pr_err("RIO: failed to allocate destID table\n"); - rio_free_net(net); + kfree(net); + mport->net = NULL; net = NULL; } else { net->enum_data = idtab; diff --git a/drivers/ras/ras.c b/drivers/ras/ras.c index 2a5b5a9fdcb36..03df3db623346 100644 --- a/drivers/ras/ras.c +++ b/drivers/ras/ras.c @@ -72,7 +72,11 @@ void log_arm_hw_error(struct cper_sec_proc_arm *err, const u8 sev) ctx_err = (u8 *)ctx_info; for (n = 0; n < err->context_info_num; n++) { - sz = sizeof(struct cper_arm_ctx_info) + ctx_info->size; + sz = sizeof(struct cper_arm_ctx_info); + + if (sz + (long)ctx_info - (long)err >= err->section_length) + sz += ctx_info->size; + ctx_info = (struct cper_arm_ctx_info *)((long)ctx_info + sz); ctx_len += sz; } diff --git a/drivers/regulator/bq257xx-regulator.c b/drivers/regulator/bq257xx-regulator.c index fc1ccede44688..dab8f1ab44503 100644 --- a/drivers/regulator/bq257xx-regulator.c +++ b/drivers/regulator/bq257xx-regulator.c @@ -115,11 +115,10 @@ static void bq257xx_reg_dt_parse_gpio(struct platform_device *pdev) return; subchild = of_get_child_by_name(child, pdata->desc.of_match); + of_node_put(child); if (!subchild) return; - of_node_put(child); - pdata->otg_en_gpio = devm_fwnode_gpiod_get_index(&pdev->dev, of_fwnode_handle(subchild), "enable", 0, diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c index c62f3358ddfd0..765bd1b5deb3a 100644 --- a/drivers/regulator/core.c +++ b/drivers/regulator/core.c @@ -1431,6 +1431,33 @@ static int set_machine_constraints(struct regulator_dev *rdev) int ret = 0; const struct regulator_ops *ops = rdev->desc->ops; + /* + * If there is no mechanism for controlling the regulator then + * flag it as always_on so we don't end up duplicating checks + * for this so much. Note that we could control the state of + * a supply to control the output on a regulator that has no + * direct control. + */ + if (!rdev->ena_pin && !ops->enable) { + if (rdev->supply_name && !rdev->supply) + return -EPROBE_DEFER; + + if (rdev->supply) + rdev->constraints->always_on = + rdev->supply->rdev->constraints->always_on; + else + rdev->constraints->always_on = true; + } + + /* + * If we want to enable this regulator, make sure that we know the + * supplying regulator. + */ + if (rdev->constraints->always_on || rdev->constraints->boot_on) { + if (rdev->supply_name && !rdev->supply) + return -EPROBE_DEFER; + } + ret = machine_constraints_voltage(rdev, rdev->constraints); if (ret != 0) return ret; @@ -1596,37 +1623,15 @@ static int set_machine_constraints(struct regulator_dev *rdev) } } - /* - * If there is no mechanism for controlling the regulator then - * flag it as always_on so we don't end up duplicating checks - * for this so much. Note that we could control the state of - * a supply to control the output on a regulator that has no - * direct control. - */ - if (!rdev->ena_pin && !ops->enable) { - if (rdev->supply_name && !rdev->supply) - return -EPROBE_DEFER; - - if (rdev->supply) - rdev->constraints->always_on = - rdev->supply->rdev->constraints->always_on; - else - rdev->constraints->always_on = true; - } - /* If the constraints say the regulator should be on at this point * and we have control then make sure it is enabled. */ if (rdev->constraints->always_on || rdev->constraints->boot_on) { bool supply_enabled = false; - /* If we want to enable this regulator, make sure that we know - * the supplying regulator. - */ - if (rdev->supply_name && !rdev->supply) - return -EPROBE_DEFER; - - /* If supplying regulator has already been enabled, + /* We have ensured a potential supply has been resolved above. + * + * If supplying regulator has already been enabled, * it's not intended to have use_count increment * when rdev is only boot-on. */ @@ -1833,8 +1838,6 @@ static const struct file_operations constraint_flags_fops = { #endif }; -#define REG_STR_SIZE 64 - static void link_and_create_debugfs(struct regulator *regulator, struct regulator_dev *rdev, struct device *dev) { diff --git a/drivers/remoteproc/imx_dsp_rproc.c b/drivers/remoteproc/imx_dsp_rproc.c index 6e78a01755c7b..c0c2bda4964a7 100644 --- a/drivers/remoteproc/imx_dsp_rproc.c +++ b/drivers/remoteproc/imx_dsp_rproc.c @@ -1026,9 +1026,11 @@ static int imx_dsp_rproc_load(struct rproc *rproc, const struct firmware *fw) * Clear buffers after pm rumtime for internal ocram is not * accessible if power and clock are not enabled. */ - list_for_each_entry(carveout, &rproc->carveouts, node) { - if (carveout->va) - memset(carveout->va, 0, carveout->len); + if (rproc->state == RPROC_OFFLINE) { + list_for_each_entry(carveout, &rproc->carveouts, node) { + if (carveout->va) + memset(carveout->va, 0, carveout->len); + } } ret = imx_dsp_rproc_elf_load_segments(rproc, fw); @@ -1301,6 +1303,15 @@ static int imx_dsp_suspend(struct device *dev) if (rproc->state != RPROC_RUNNING) goto out; + /* + * No channel available for sending messages; + * indicates no mailboxes present, so trigger PM runtime suspend + */ + if (!priv->tx_ch) { + dev_dbg(dev, "No initialized mbox tx channel, suspend directly.\n"); + goto out; + } + reinit_completion(&priv->pm_comp); /* Tell DSP that suspend is happening */ diff --git a/drivers/remoteproc/imx_rproc.c b/drivers/remoteproc/imx_rproc.c index 8424e6ea5569b..7ef99eac37f1a 100644 --- a/drivers/remoteproc/imx_rproc.c +++ b/drivers/remoteproc/imx_rproc.c @@ -608,6 +608,10 @@ imx_rproc_elf_find_loaded_rsc_table(struct rproc *rproc, const struct firmware * { struct imx_rproc *priv = rproc->priv; + /* No resource table in the firmware */ + if (!rproc->table_ptr) + return NULL; + if (priv->rsc_table) return (struct resource_table *)priv->rsc_table; diff --git a/drivers/remoteproc/mtk_scp.c b/drivers/remoteproc/mtk_scp.c index 8206a17664818..2aeb0ded165cf 100644 --- a/drivers/remoteproc/mtk_scp.c +++ b/drivers/remoteproc/mtk_scp.c @@ -282,7 +282,7 @@ static irqreturn_t scp_irq_handler(int irq, void *priv) struct mtk_scp *scp = priv; int ret; - ret = clk_prepare_enable(scp->clk); + ret = clk_enable(scp->clk); if (ret) { dev_err(scp->dev, "failed to enable clocks\n"); return IRQ_NONE; @@ -290,7 +290,7 @@ static irqreturn_t scp_irq_handler(int irq, void *priv) scp->data->scp_irq_handler(scp); - clk_disable_unprepare(scp->clk); + clk_disable(scp->clk); return IRQ_HANDLED; } @@ -664,7 +664,7 @@ static int scp_load(struct rproc *rproc, const struct firmware *fw) struct device *dev = scp->dev; int ret; - ret = clk_prepare_enable(scp->clk); + ret = clk_enable(scp->clk); if (ret) { dev_err(dev, "failed to enable clocks\n"); return ret; @@ -679,7 +679,7 @@ static int scp_load(struct rproc *rproc, const struct firmware *fw) ret = scp_elf_load_segments(rproc, fw); leave: - clk_disable_unprepare(scp->clk); + clk_disable(scp->clk); return ret; } @@ -690,14 +690,14 @@ static int scp_parse_fw(struct rproc *rproc, const struct firmware *fw) struct device *dev = scp->dev; int ret; - ret = clk_prepare_enable(scp->clk); + ret = clk_enable(scp->clk); if (ret) { dev_err(dev, "failed to enable clocks\n"); return ret; } ret = scp_ipi_init(scp, fw); - clk_disable_unprepare(scp->clk); + clk_disable(scp->clk); return ret; } @@ -708,7 +708,7 @@ static int scp_start(struct rproc *rproc) struct scp_run *run = &scp->run; int ret; - ret = clk_prepare_enable(scp->clk); + ret = clk_enable(scp->clk); if (ret) { dev_err(dev, "failed to enable clocks\n"); return ret; @@ -733,14 +733,14 @@ static int scp_start(struct rproc *rproc) goto stop; } - clk_disable_unprepare(scp->clk); + clk_disable(scp->clk); dev_info(dev, "SCP is ready. FW version %s\n", run->fw_ver); return 0; stop: scp->data->scp_reset_assert(scp); - clk_disable_unprepare(scp->clk); + clk_disable(scp->clk); return ret; } @@ -908,7 +908,7 @@ static int scp_stop(struct rproc *rproc) struct mtk_scp *scp = rproc->priv; int ret; - ret = clk_prepare_enable(scp->clk); + ret = clk_enable(scp->clk); if (ret) { dev_err(scp->dev, "failed to enable clocks\n"); return ret; @@ -916,12 +916,29 @@ static int scp_stop(struct rproc *rproc) scp->data->scp_reset_assert(scp); scp->data->scp_stop(scp); - clk_disable_unprepare(scp->clk); + clk_disable(scp->clk); return 0; } +static int scp_prepare(struct rproc *rproc) +{ + struct mtk_scp *scp = rproc->priv; + + return clk_prepare(scp->clk); +} + +static int scp_unprepare(struct rproc *rproc) +{ + struct mtk_scp *scp = rproc->priv; + + clk_unprepare(scp->clk); + return 0; +} + static const struct rproc_ops scp_ops = { + .prepare = scp_prepare, + .unprepare = scp_unprepare, .start = scp_start, .stop = scp_stop, .load = scp_load, diff --git a/drivers/remoteproc/mtk_scp_ipi.c b/drivers/remoteproc/mtk_scp_ipi.c index c068227e251e7..7a37e273b3af8 100644 --- a/drivers/remoteproc/mtk_scp_ipi.c +++ b/drivers/remoteproc/mtk_scp_ipi.c @@ -171,7 +171,7 @@ int scp_ipi_send(struct mtk_scp *scp, u32 id, void *buf, unsigned int len, WARN_ON(len > scp_sizes->ipi_share_buffer_size) || WARN_ON(!buf)) return -EINVAL; - ret = clk_prepare_enable(scp->clk); + ret = clk_enable(scp->clk); if (ret) { dev_err(scp->dev, "failed to enable clock\n"); return ret; @@ -211,7 +211,7 @@ int scp_ipi_send(struct mtk_scp *scp, u32 id, void *buf, unsigned int len, unlock_mutex: mutex_unlock(&scp->send_lock); - clk_disable_unprepare(scp->clk); + clk_disable(scp->clk); return ret; } diff --git a/drivers/reset/Kconfig b/drivers/reset/Kconfig index 78b7078478d46..b3b9e0f9d8c48 100644 --- a/drivers/reset/Kconfig +++ b/drivers/reset/Kconfig @@ -150,7 +150,7 @@ config RESET_K210 config RESET_K230 tristate "Reset controller driver for Canaan Kendryte K230 SoC" depends on ARCH_CANAAN || COMPILE_TEST - depends on OF + default ARCH_CANAAN help Support for the Canaan Kendryte K230 RISC-V SoC reset controller. Say Y if you want to control reset signals provided by this diff --git a/drivers/reset/reset-gpio.c b/drivers/reset/reset-gpio.c index 2290b25b67035..15353ba5758c3 100644 --- a/drivers/reset/reset-gpio.c +++ b/drivers/reset/reset-gpio.c @@ -110,6 +110,7 @@ static struct platform_driver reset_gpio_driver = { .id_table = reset_gpio_ids, .driver = { .name = "reset-gpio", + .suppress_bind_attrs = true, }, }; module_platform_driver(reset_gpio_driver); diff --git a/drivers/rpmsg/rpmsg_core.c b/drivers/rpmsg/rpmsg_core.c index 5d661681a9b6c..96964745065b1 100644 --- a/drivers/rpmsg/rpmsg_core.c +++ b/drivers/rpmsg/rpmsg_core.c @@ -352,50 +352,38 @@ field##_show(struct device *dev, \ } \ static DEVICE_ATTR_RO(field); -#define rpmsg_string_attr(field, member) \ -static ssize_t \ -field##_store(struct device *dev, struct device_attribute *attr, \ - const char *buf, size_t sz) \ -{ \ - struct rpmsg_device *rpdev = to_rpmsg_device(dev); \ - const char *old; \ - char *new; \ - \ - new = kstrndup(buf, sz, GFP_KERNEL); \ - if (!new) \ - return -ENOMEM; \ - new[strcspn(new, "\n")] = '\0'; \ - \ - device_lock(dev); \ - old = rpdev->member; \ - if (strlen(new)) { \ - rpdev->member = new; \ - } else { \ - kfree(new); \ - rpdev->member = NULL; \ - } \ - device_unlock(dev); \ - \ - kfree(old); \ - \ - return sz; \ -} \ -static ssize_t \ -field##_show(struct device *dev, \ - struct device_attribute *attr, char *buf) \ -{ \ - struct rpmsg_device *rpdev = to_rpmsg_device(dev); \ - \ - return sprintf(buf, "%s\n", rpdev->member); \ -} \ -static DEVICE_ATTR_RW(field) - /* for more info, see Documentation/ABI/testing/sysfs-bus-rpmsg */ rpmsg_show_attr(name, id.name, "%s\n"); rpmsg_show_attr(src, src, "0x%x\n"); rpmsg_show_attr(dst, dst, "0x%x\n"); rpmsg_show_attr(announce, announce ? "true" : "false", "%s\n"); -rpmsg_string_attr(driver_override, driver_override); + +static ssize_t driver_override_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct rpmsg_device *rpdev = to_rpmsg_device(dev); + int ret; + + ret = driver_set_override(dev, &rpdev->driver_override, buf, count); + if (ret) + return ret; + + return count; +} + +static ssize_t driver_override_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct rpmsg_device *rpdev = to_rpmsg_device(dev); + ssize_t len; + + device_lock(dev); + len = sysfs_emit(buf, "%s\n", rpdev->driver_override); + device_unlock(dev); + return len; +} +static DEVICE_ATTR_RW(driver_override); static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, char *buf) diff --git a/drivers/rtc/interface.c b/drivers/rtc/interface.c index b8b298efd9a9c..1906f4884a834 100644 --- a/drivers/rtc/interface.c +++ b/drivers/rtc/interface.c @@ -457,7 +457,7 @@ static int __rtc_set_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alarm) * are in, we can return -ETIME to signal that the timer has already * expired, which is true in both cases. */ - if ((scheduled - now) <= 1) { + if (!err && (scheduled - now) <= 1) { err = __rtc_read_time(rtc, &tm); if (err) return err; diff --git a/drivers/rtc/rtc-amlogic-a4.c b/drivers/rtc/rtc-amlogic-a4.c index a993d35e1d6b0..d766055d95848 100644 --- a/drivers/rtc/rtc-amlogic-a4.c +++ b/drivers/rtc/rtc-amlogic-a4.c @@ -371,7 +371,7 @@ static int aml_rtc_probe(struct platform_device *pdev) } ret = devm_request_irq(dev, rtc->irq, aml_rtc_handler, - IRQF_ONESHOT, "aml-rtc alarm", rtc); + 0, "aml-rtc alarm", rtc); if (ret) { dev_err_probe(dev, ret, "IRQ%d request failed, ret = %d\n", rtc->irq, ret); diff --git a/drivers/rtc/rtc-max31335.c b/drivers/rtc/rtc-max31335.c index 23b7bf16b4cd5..952b455071d68 100644 --- a/drivers/rtc/rtc-max31335.c +++ b/drivers/rtc/rtc-max31335.c @@ -591,7 +591,7 @@ static struct nvmem_config max31335_nvmem_cfg = { .size = MAX31335_RAM_SIZE, }; -#if IS_REACHABLE(HWMON) +#if IS_REACHABLE(CONFIG_HWMON) static int max31335_read_temp(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, long *val) { @@ -672,7 +672,7 @@ static int max31335_clkout_register(struct device *dev) static int max31335_probe(struct i2c_client *client) { struct max31335_data *max31335; -#if IS_REACHABLE(HWMON) +#if IS_REACHABLE(CONFIG_HWMON) struct device *hwmon; #endif const struct chip_desc *match; @@ -727,7 +727,7 @@ static int max31335_probe(struct i2c_client *client) return dev_err_probe(&client->dev, ret, "cannot register rtc nvmem\n"); -#if IS_REACHABLE(HWMON) +#if IS_REACHABLE(CONFIG_HWMON) if (max31335->chip->temp_reg) { hwmon = devm_hwmon_device_register_with_info(&client->dev, client->name, max31335, &max31335_chip_info, NULL); diff --git a/drivers/rtc/rtc-pcf8563.c b/drivers/rtc/rtc-pcf8563.c index 4e61011fb7a96..b281e9489df1d 100644 --- a/drivers/rtc/rtc-pcf8563.c +++ b/drivers/rtc/rtc-pcf8563.c @@ -424,7 +424,7 @@ static const struct clk_ops pcf8563_clkout_ops = { static struct clk *pcf8563_clkout_register_clk(struct pcf8563 *pcf8563) { - struct device_node *node = pcf8563->rtc->dev.of_node; + struct device_node *node = pcf8563->rtc->dev.parent->of_node; struct clk_init_data init; struct clk *clk; int ret; diff --git a/drivers/rtc/rtc-zynqmp.c b/drivers/rtc/rtc-zynqmp.c index 3baa2b481d9f2..856bc1678e7d3 100644 --- a/drivers/rtc/rtc-zynqmp.c +++ b/drivers/rtc/rtc-zynqmp.c @@ -345,7 +345,10 @@ static int xlnx_rtc_probe(struct platform_device *pdev) &xrtcdev->freq); if (ret) xrtcdev->freq = RTC_CALIB_DEF; + } else { + xrtcdev->freq--; } + ret = readl(xrtcdev->reg_base + RTC_CALIB_RD); if (!ret) writel(xrtcdev->freq, (xrtcdev->reg_base + RTC_CALIB_WR)); diff --git a/drivers/s390/cio/css.c b/drivers/s390/cio/css.c index be78a57f9bfde..8a70596a55447 100644 --- a/drivers/s390/cio/css.c +++ b/drivers/s390/cio/css.c @@ -236,7 +236,7 @@ struct subchannel *css_alloc_subchannel(struct subchannel_id schid, return sch; err: - kfree(sch); + put_device(&sch->dev); return ERR_PTR(ret); } diff --git a/drivers/scsi/BusLogic.c b/drivers/scsi/BusLogic.c index a86d780d1ba40..026c3e617cb1c 100644 --- a/drivers/scsi/BusLogic.c +++ b/drivers/scsi/BusLogic.c @@ -920,7 +920,8 @@ static int __init blogic_init_fp_probeinfo(struct blogic_adapter *adapter) a particular probe order. */ -static void __init blogic_init_probeinfo_list(struct blogic_adapter *adapter) +static noinline_for_stack void __init +blogic_init_probeinfo_list(struct blogic_adapter *adapter) { /* If a PCI BIOS is present, interrogate it for MultiMaster and @@ -1690,7 +1691,8 @@ static bool __init blogic_rdconfig(struct blogic_adapter *adapter) blogic_reportconfig reports the configuration of Host Adapter. */ -static bool __init blogic_reportconfig(struct blogic_adapter *adapter) +static noinline_for_stack bool __init +blogic_reportconfig(struct blogic_adapter *adapter) { unsigned short alltgt_mask = (1 << adapter->maxdev) - 1; unsigned short sync_ok, fast_ok; diff --git a/drivers/scsi/csiostor/csio_scsi.c b/drivers/scsi/csiostor/csio_scsi.c index 34bde6650fae0..356a7c577ec3e 100644 --- a/drivers/scsi/csiostor/csio_scsi.c +++ b/drivers/scsi/csiostor/csio_scsi.c @@ -2074,7 +2074,7 @@ csio_eh_lun_reset_handler(struct scsi_cmnd *cmnd) struct csio_scsi_level_data sld; if (!rn) - goto fail; + goto fail_ret; csio_dbg(hw, "Request to reset LUN:%llu (ssni:0x%x tgtid:%d)\n", cmnd->device->lun, rn->flowid, rn->scsi_id); @@ -2220,6 +2220,7 @@ csio_eh_lun_reset_handler(struct scsi_cmnd *cmnd) csio_put_scsi_ioreq_lock(hw, scsim, ioreq); fail: CSIO_INC_STATS(rn, n_lun_rst_fail); +fail_ret: return FAILED; } diff --git a/drivers/scsi/elx/efct/efct_driver.c b/drivers/scsi/elx/efct/efct_driver.c index 1bd42f7db1773..528399f725d42 100644 --- a/drivers/scsi/elx/efct/efct_driver.c +++ b/drivers/scsi/elx/efct/efct_driver.c @@ -415,12 +415,6 @@ efct_intr_thread(int irq, void *handle) return IRQ_HANDLED; } -static irqreturn_t -efct_intr_msix(int irq, void *handle) -{ - return IRQ_WAKE_THREAD; -} - static int efct_setup_msix(struct efct *efct, u32 num_intrs) { @@ -450,7 +444,7 @@ efct_setup_msix(struct efct *efct, u32 num_intrs) intr_ctx->index = i; rc = request_threaded_irq(pci_irq_vector(efct->pci, i), - efct_intr_msix, efct_intr_thread, 0, + NULL, efct_intr_thread, IRQF_ONESHOT, EFCT_DRIVER_NAME, intr_ctx); if (rc) { dev_err(&efct->pci->dev, diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index 065eb91de9c0f..adc0beaf5468f 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -12040,6 +12040,8 @@ lpfc_sli4_pci_mem_unset(struct lpfc_hba *phba) iounmap(phba->sli4_hba.conf_regs_memmap_p); if (phba->sli4_hba.dpp_regs_memmap_p) iounmap(phba->sli4_hba.dpp_regs_memmap_p); + if (phba->sli4_hba.dpp_regs_memmap_wc_p) + iounmap(phba->sli4_hba.dpp_regs_memmap_wc_p); break; case LPFC_SLI_INTF_IF_TYPE_1: break; diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c index 7ea7c4245c691..7b765719f4f6b 100644 --- a/drivers/scsi/lpfc/lpfc_sli.c +++ b/drivers/scsi/lpfc/lpfc_sli.c @@ -15911,6 +15911,32 @@ lpfc_dual_chute_pci_bar_map(struct lpfc_hba *phba, uint16_t pci_barset) return NULL; } +static __maybe_unused void __iomem * +lpfc_dpp_wc_map(struct lpfc_hba *phba, uint8_t dpp_barset) +{ + + /* DPP region is supposed to cover 64-bit BAR2 */ + if (dpp_barset != WQ_PCI_BAR_4_AND_5) { + lpfc_log_msg(phba, KERN_WARNING, LOG_INIT, + "3273 dpp_barset x%x != WQ_PCI_BAR_4_AND_5\n", + dpp_barset); + return NULL; + } + + if (!phba->sli4_hba.dpp_regs_memmap_wc_p) { + void __iomem *dpp_map; + + dpp_map = ioremap_wc(phba->pci_bar2_map, + pci_resource_len(phba->pcidev, + PCI_64BIT_BAR4)); + + if (dpp_map) + phba->sli4_hba.dpp_regs_memmap_wc_p = dpp_map; + } + + return phba->sli4_hba.dpp_regs_memmap_wc_p; +} + /** * lpfc_modify_hba_eq_delay - Modify Delay Multiplier on EQs * @phba: HBA structure that EQs are on. @@ -16874,9 +16900,6 @@ lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq, uint8_t dpp_barset; uint32_t dpp_offset; uint8_t wq_create_version; -#ifdef CONFIG_X86 - unsigned long pg_addr; -#endif /* sanity check on queue memory */ if (!wq || !cq) @@ -17062,14 +17085,15 @@ lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq, #ifdef CONFIG_X86 /* Enable combined writes for DPP aperture */ - pg_addr = (unsigned long)(wq->dpp_regaddr) & PAGE_MASK; - rc = set_memory_wc(pg_addr, 1); - if (rc) { + bar_memmap_p = lpfc_dpp_wc_map(phba, dpp_barset); + if (!bar_memmap_p) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "3272 Cannot setup Combined " "Write on WQ[%d] - disable DPP\n", wq->queue_id); phba->cfg_enable_dpp = 0; + } else { + wq->dpp_regaddr = bar_memmap_p + dpp_offset; } #else phba->cfg_enable_dpp = 0; diff --git a/drivers/scsi/lpfc/lpfc_sli4.h b/drivers/scsi/lpfc/lpfc_sli4.h index fd6dab1578872..40f313e2769fc 100644 --- a/drivers/scsi/lpfc/lpfc_sli4.h +++ b/drivers/scsi/lpfc/lpfc_sli4.h @@ -785,6 +785,9 @@ struct lpfc_sli4_hba { void __iomem *dpp_regs_memmap_p; /* Kernel memory mapped address for * dpp registers */ + void __iomem *dpp_regs_memmap_wc_p;/* Kernel memory mapped address for + * dpp registers with write combining + */ union { struct { /* IF Type 0, BAR 0 PCI cfg space reg mem map */ diff --git a/drivers/scsi/pm8001/pm8001_sas.c b/drivers/scsi/pm8001/pm8001_sas.c index 6a8d35aea93a5..645524f3fe2d0 100644 --- a/drivers/scsi/pm8001/pm8001_sas.c +++ b/drivers/scsi/pm8001/pm8001_sas.c @@ -525,8 +525,9 @@ int pm8001_queue_command(struct sas_task *task, gfp_t gfp_flags) } else { task->task_done(task); } - rc = -ENODEV; - goto err_out; + spin_unlock_irqrestore(&pm8001_ha->lock, flags); + pm8001_dbg(pm8001_ha, IO, "pm8001_task_exec device gone\n"); + return 0; } ccb = pm8001_ccb_alloc(pm8001_ha, pm8001_dev, task); diff --git a/drivers/scsi/qla2xxx/qla_bsg.c b/drivers/scsi/qla2xxx/qla_bsg.c index ccfc2d26dd372..0798bfd0372e4 100644 --- a/drivers/scsi/qla2xxx/qla_bsg.c +++ b/drivers/scsi/qla2xxx/qla_bsg.c @@ -1546,8 +1546,9 @@ qla2x00_update_optrom(struct bsg_job *bsg_job) ha->optrom_buffer = NULL; ha->optrom_state = QLA_SWAITING; mutex_unlock(&ha->optrom_mutex); - bsg_job_done(bsg_job, bsg_reply->result, - bsg_reply->reply_payload_rcv_len); + if (!rval) + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return rval; } @@ -2612,8 +2613,9 @@ qla2x00_manage_host_stats(struct bsg_job *bsg_job) sizeof(struct ql_vnd_mng_host_stats_resp)); bsg_reply->result = DID_OK; - bsg_job_done(bsg_job, bsg_reply->result, - bsg_reply->reply_payload_rcv_len); + if (!ret) + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return ret; } @@ -2702,8 +2704,9 @@ qla2x00_get_host_stats(struct bsg_job *bsg_job) bsg_job->reply_payload.sg_cnt, data, response_len); bsg_reply->result = DID_OK; - bsg_job_done(bsg_job, bsg_reply->result, - bsg_reply->reply_payload_rcv_len); + if (!ret) + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); kfree(data); host_stat_out: @@ -2802,8 +2805,9 @@ qla2x00_get_tgt_stats(struct bsg_job *bsg_job) bsg_job->reply_payload.sg_cnt, data, response_len); bsg_reply->result = DID_OK; - bsg_job_done(bsg_job, bsg_reply->result, - bsg_reply->reply_payload_rcv_len); + if (!ret) + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); tgt_stat_out: kfree(data); @@ -2864,8 +2868,9 @@ qla2x00_manage_host_port(struct bsg_job *bsg_job) bsg_job->reply_payload.sg_cnt, &rsp_data, sizeof(struct ql_vnd_mng_host_port_resp)); bsg_reply->result = DID_OK; - bsg_job_done(bsg_job, bsg_reply->result, - bsg_reply->reply_payload_rcv_len); + if (!ret) + bsg_job_done(bsg_job, bsg_reply->result, + bsg_reply->reply_payload_rcv_len); return ret; } @@ -3240,7 +3245,8 @@ int qla2x00_mailbox_passthru(struct bsg_job *bsg_job) bsg_job->reply_len = sizeof(*bsg_job->reply); bsg_reply->result = DID_OK << 16; - bsg_job_done(bsg_job, bsg_reply->result, bsg_reply->reply_payload_rcv_len); + if (!ret) + bsg_job_done(bsg_job, bsg_reply->result, bsg_reply->reply_payload_rcv_len); kfree(req_data); diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c index 3c6e089e80c38..f405ef9c0e1e8 100644 --- a/drivers/scsi/scsi_scan.c +++ b/drivers/scsi/scsi_scan.c @@ -356,6 +356,7 @@ static struct scsi_device *scsi_alloc_sdev(struct scsi_target *starget, * since we use this queue depth most of times. */ if (scsi_realloc_sdev_budget_map(sdev, depth)) { + kref_put(&sdev->host->tagset_refcnt, scsi_mq_free_tags); put_device(&starget->dev); kfree(sdev); goto out; diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c index 98e93900254cb..5a6e1bb57e7c8 100644 --- a/drivers/scsi/smartpqi/smartpqi_init.c +++ b/drivers/scsi/smartpqi/smartpqi_init.c @@ -1241,7 +1241,8 @@ static inline int pqi_report_phys_luns(struct pqi_ctrl_info *ctrl_info, void **b dev_err(&ctrl_info->pci_dev->dev, "RPL returned unsupported data format %u\n", rpl_response_format); - return -EINVAL; + rc = -EINVAL; + goto out_free_rpl_list; } else { dev_warn(&ctrl_info->pci_dev->dev, "RPL returned extended format 2 instead of 4\n"); @@ -1253,8 +1254,10 @@ static inline int pqi_report_phys_luns(struct pqi_ctrl_info *ctrl_info, void **b rpl_16byte_wwid_list = kmalloc(struct_size(rpl_16byte_wwid_list, lun_entries, num_physicals), GFP_KERNEL); - if (!rpl_16byte_wwid_list) - return -ENOMEM; + if (!rpl_16byte_wwid_list) { + rc = -ENOMEM; + goto out_free_rpl_list; + } put_unaligned_be32(num_physicals * sizeof(struct report_phys_lun_16byte_wwid), &rpl_16byte_wwid_list->header.list_length); @@ -1275,6 +1278,10 @@ static inline int pqi_report_phys_luns(struct pqi_ctrl_info *ctrl_info, void **b *buffer = rpl_16byte_wwid_list; return 0; + +out_free_rpl_list: + kfree(rpl_list); + return rc; } static inline int pqi_report_logical_luns(struct pqi_ctrl_info *ctrl_info, void **buffer) diff --git a/drivers/soc/imx/soc-imx8m.c b/drivers/soc/imx/soc-imx8m.c index 04a1b60f2f2b5..8e2322999f099 100644 --- a/drivers/soc/imx/soc-imx8m.c +++ b/drivers/soc/imx/soc-imx8m.c @@ -148,7 +148,11 @@ static int imx8m_soc_prepare(struct platform_device *pdev, const char *ocotp_com goto err_clk; } - return clk_prepare_enable(drvdata->clk); + ret = clk_prepare_enable(drvdata->clk); + if (ret) + goto err_clk; + + return 0; err_clk: iounmap(drvdata->ocotp_base); diff --git a/drivers/soc/mediatek/mtk-svs.c b/drivers/soc/mediatek/mtk-svs.c index f45537546553e..99edecb204f25 100644 --- a/drivers/soc/mediatek/mtk-svs.c +++ b/drivers/soc/mediatek/mtk-svs.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -789,7 +790,7 @@ static ssize_t svs_enable_debug_write(struct file *filp, struct svs_bank *svsb = file_inode(filp)->i_private; struct svs_platform *svsp = dev_get_drvdata(svsb->dev); int enabled, ret; - char *buf = NULL; + char *buf __free(kfree) = NULL; if (count >= PAGE_SIZE) return -EINVAL; @@ -807,8 +808,6 @@ static ssize_t svs_enable_debug_write(struct file *filp, svsb->mode_support = SVSB_MODE_ALL_DISABLE; } - kfree(buf); - return count; } diff --git a/drivers/soc/qcom/cmd-db.c b/drivers/soc/qcom/cmd-db.c index ae66c2623d250..84a75d8c4b702 100644 --- a/drivers/soc/qcom/cmd-db.c +++ b/drivers/soc/qcom/cmd-db.c @@ -349,15 +349,16 @@ static int cmd_db_dev_probe(struct platform_device *pdev) return -EINVAL; } - cmd_db_header = memremap(rmem->base, rmem->size, MEMREMAP_WC); - if (!cmd_db_header) { - ret = -ENOMEM; + cmd_db_header = devm_memremap(&pdev->dev, rmem->base, rmem->size, MEMREMAP_WC); + if (IS_ERR(cmd_db_header)) { + ret = PTR_ERR(cmd_db_header); cmd_db_header = NULL; return ret; } if (!cmd_db_magic_matches(cmd_db_header)) { dev_err(&pdev->dev, "Invalid Command DB Magic\n"); + cmd_db_header = NULL; return -EINVAL; } diff --git a/drivers/soc/qcom/smem.c b/drivers/soc/qcom/smem.c index f1d1b5aa5e4db..39177aa5793ad 100644 --- a/drivers/soc/qcom/smem.c +++ b/drivers/soc/qcom/smem.c @@ -1215,7 +1215,9 @@ static int qcom_smem_probe(struct platform_device *pdev) smem->item_count = qcom_smem_get_item_count(smem); break; case SMEM_GLOBAL_HEAP_VERSION: - qcom_smem_map_global(smem, size); + ret = qcom_smem_map_global(smem, size); + if (ret < 0) + return ret; smem->item_count = SMEM_ITEM_COUNT; break; default: diff --git a/drivers/soc/rockchip/grf.c b/drivers/soc/rockchip/grf.c index 344870da7675f..db407fa279850 100644 --- a/drivers/soc/rockchip/grf.c +++ b/drivers/soc/rockchip/grf.c @@ -134,7 +134,7 @@ static const struct rockchip_grf_info rk3576_sysgrf __initconst = { .num_values = ARRAY_SIZE(rk3576_defaults_sys_grf), }; -#define RK3576_IOCGRF_MISC_CON 0x04F0 +#define RK3576_IOCGRF_MISC_CON 0x40F0 static const struct rockchip_grf_value rk3576_defaults_ioc_grf[] __initconst = { { "jtag switching", RK3576_IOCGRF_MISC_CON, FIELD_PREP_WM16_CONST(BIT(1), 0) }, @@ -202,34 +202,33 @@ static int __init rockchip_grf_init(void) struct regmap *grf; int ret, i; - np = of_find_matching_node_and_match(NULL, rockchip_grf_dt_match, - &match); - if (!np) - return -ENODEV; - if (!match || !match->data) { - pr_err("%s: missing grf data\n", __func__); - of_node_put(np); - return -EINVAL; - } - - grf_info = match->data; - - grf = syscon_node_to_regmap(np); - of_node_put(np); - if (IS_ERR(grf)) { - pr_err("%s: could not get grf syscon\n", __func__); - return PTR_ERR(grf); - } - - for (i = 0; i < grf_info->num_values; i++) { - const struct rockchip_grf_value *val = &grf_info->values[i]; - - pr_debug("%s: adjusting %s in %#6x to %#10x\n", __func__, - val->desc, val->reg, val->val); - ret = regmap_write(grf, val->reg, val->val); - if (ret < 0) - pr_err("%s: write to %#6x failed with %d\n", - __func__, val->reg, ret); + for_each_matching_node_and_match(np, rockchip_grf_dt_match, &match) { + if (!of_device_is_available(np)) + continue; + if (!match || !match->data) { + pr_err("%s: missing grf data\n", __func__); + of_node_put(np); + return -EINVAL; + } + + grf_info = match->data; + + grf = syscon_node_to_regmap(np); + if (IS_ERR(grf)) { + pr_err("%s: could not get grf syscon\n", __func__); + return PTR_ERR(grf); + } + + for (i = 0; i < grf_info->num_values; i++) { + const struct rockchip_grf_value *val = &grf_info->values[i]; + + pr_debug("%s: adjusting %s in %#6x to %#10x\n", __func__, + val->desc, val->reg, val->val); + ret = regmap_write(grf, val->reg, val->val); + if (ret < 0) + pr_err("%s: write to %#6x failed with %d\n", + __func__, val->reg, ret); + } } return 0; diff --git a/drivers/soc/ti/k3-socinfo.c b/drivers/soc/ti/k3-socinfo.c index 50c170a995f90..42275cb5ba1c8 100644 --- a/drivers/soc/ti/k3-socinfo.c +++ b/drivers/soc/ti/k3-socinfo.c @@ -141,7 +141,7 @@ static int k3_chipinfo_probe(struct platform_device *pdev) if (IS_ERR(base)) return PTR_ERR(base); - regmap = regmap_init_mmio(dev, base, &k3_chipinfo_regmap_cfg); + regmap = devm_regmap_init_mmio(dev, base, &k3_chipinfo_regmap_cfg); if (IS_ERR(regmap)) return PTR_ERR(regmap); diff --git a/drivers/soc/ti/pruss.c b/drivers/soc/ti/pruss.c index 038576805bfa0..0fd59c73f585d 100644 --- a/drivers/soc/ti/pruss.c +++ b/drivers/soc/ti/pruss.c @@ -366,12 +366,10 @@ static int pruss_clk_mux_setup(struct pruss *pruss, struct clk *clk_mux, ret = devm_add_action_or_reset(dev, pruss_of_free_clk_provider, clk_mux_np); - if (ret) { + if (ret) dev_err(dev, "failed to add clkmux free action %d", ret); - goto put_clk_mux_np; - } - return 0; + return ret; put_clk_mux_np: of_node_put(clk_mux_np); diff --git a/drivers/soundwire/Kconfig b/drivers/soundwire/Kconfig index ad56393e4c93b..196a7daaabdb8 100644 --- a/drivers/soundwire/Kconfig +++ b/drivers/soundwire/Kconfig @@ -40,6 +40,7 @@ config SOUNDWIRE_INTEL select AUXILIARY_BUS depends on ACPI && SND_SOC depends on SND_SOC_SOF_HDA_MLINK || !SND_SOC_SOF_HDA_MLINK + depends on SND_HDA_CORE || !SND_HDA_ALIGNED_MMIO help SoundWire Intel Master driver. If you have an Intel platform which has a SoundWire Master then diff --git a/drivers/soundwire/dmi-quirks.c b/drivers/soundwire/dmi-quirks.c index 91ab97a456fa9..5854218e1a274 100644 --- a/drivers/soundwire/dmi-quirks.c +++ b/drivers/soundwire/dmi-quirks.c @@ -122,6 +122,17 @@ static const struct dmi_system_id adr_remap_quirk_table[] = { }, .driver_data = (void *)intel_tgl_bios, }, + { + /* + * quirk used for Avell B.ON (OEM rebrand of NUC15 'Bishop County' + * LAPBC510 and LAPBC710) + */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Avell High Performance"), + DMI_MATCH(DMI_PRODUCT_NAME, "B.ON"), + }, + .driver_data = (void *)intel_tgl_bios, + }, { /* quirk used for NUC15 'Rooks County' LAPRC510 and LAPRC710 skews */ .matches = { diff --git a/drivers/soundwire/intel_auxdevice.c b/drivers/soundwire/intel_auxdevice.c index 6df2601fff909..8752b0e3ce74c 100644 --- a/drivers/soundwire/intel_auxdevice.c +++ b/drivers/soundwire/intel_auxdevice.c @@ -52,6 +52,7 @@ struct wake_capable_part { static struct wake_capable_part wake_capable_list[] = { {0x01fa, 0x4243}, + {0x01fa, 0x4245}, {0x025d, 0x5682}, {0x025d, 0x700}, {0x025d, 0x711}, diff --git a/drivers/spi/spi-cadence-quadspi.c b/drivers/spi/spi-cadence-quadspi.c index 1cca9d87fbde4..b3e4dd72d8b8a 100644 --- a/drivers/spi/spi-cadence-quadspi.c +++ b/drivers/spi/spi-cadence-quadspi.c @@ -1853,6 +1853,12 @@ static int cqspi_probe(struct platform_device *pdev) return -ENODEV; } + ret = cqspi_setup_flash(cqspi); + if (ret) { + dev_err(dev, "failed to setup flash parameters %d\n", ret); + return ret; + } + /* Obtain QSPI clock. */ cqspi->clk = devm_clk_get(dev, NULL); if (IS_ERR(cqspi->clk)) { @@ -1996,12 +2002,6 @@ static int cqspi_probe(struct platform_device *pdev) pm_runtime_get_noresume(dev); } - ret = cqspi_setup_flash(cqspi); - if (ret) { - dev_err(dev, "failed to setup flash parameters %d\n", ret); - goto probe_setup_failed; - } - host->num_chipselect = cqspi->num_chipselect; if (ddata && (ddata->quirks & CQSPI_SUPPORT_DEVICE_RESET)) @@ -2044,6 +2044,7 @@ static void cqspi_remove(struct platform_device *pdev) const struct cqspi_driver_platdata *ddata; struct cqspi_st *cqspi = platform_get_drvdata(pdev); struct device *dev = &pdev->dev; + int ret = 0; ddata = of_device_get_match_data(dev); @@ -2059,8 +2060,10 @@ static void cqspi_remove(struct platform_device *pdev) dma_release_channel(cqspi->rx_chan); if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) - if (pm_runtime_get_sync(&pdev->dev) >= 0) - clk_disable(cqspi->clk); + ret = pm_runtime_get_sync(&pdev->dev); + + if (ret >= 0) + clk_disable(cqspi->clk); if (cqspi->is_jh7110) cqspi_jh7110_disable_clk(pdev, cqspi); diff --git a/drivers/spi/spi-geni-qcom.c b/drivers/spi/spi-geni-qcom.c index a0d8d3425c6c6..acfcf870efd84 100644 --- a/drivers/spi/spi-geni-qcom.c +++ b/drivers/spi/spi-geni-qcom.c @@ -160,24 +160,20 @@ static void handle_se_timeout(struct spi_controller *spi, xfer = mas->cur_xfer; mas->cur_xfer = NULL; - if (spi->target) { - /* - * skip CMD Cancel sequnece since spi target - * doesn`t support CMD Cancel sequnece - */ + /* The controller doesn't support the Cancel commnand in target mode */ + if (!spi->target) { + reinit_completion(&mas->cancel_done); + geni_se_cancel_m_cmd(se); + spin_unlock_irq(&mas->lock); - goto reset_if_dma; - } - reinit_completion(&mas->cancel_done); - geni_se_cancel_m_cmd(se); - spin_unlock_irq(&mas->lock); + time_left = wait_for_completion_timeout(&mas->cancel_done, HZ); + if (time_left) + goto reset_if_dma; - time_left = wait_for_completion_timeout(&mas->cancel_done, HZ); - if (time_left) - goto reset_if_dma; + spin_lock_irq(&mas->lock); + } - spin_lock_irq(&mas->lock); reinit_completion(&mas->abort_done); geni_se_abort_m_cmd(se); spin_unlock_irq(&mas->lock); @@ -548,10 +544,10 @@ static u32 get_xfer_len_in_words(struct spi_transfer *xfer, { u32 len; - if (!(mas->cur_bits_per_word % MIN_WORD_LEN)) - len = xfer->len * BITS_PER_BYTE / mas->cur_bits_per_word; + if (!(xfer->bits_per_word % MIN_WORD_LEN)) + len = xfer->len * BITS_PER_BYTE / xfer->bits_per_word; else - len = xfer->len / (mas->cur_bits_per_word / BITS_PER_BYTE + 1); + len = xfer->len / (xfer->bits_per_word / BITS_PER_BYTE + 1); len &= TRANS_LEN_MSK; return len; @@ -571,7 +567,7 @@ static bool geni_can_dma(struct spi_controller *ctlr, return true; len = get_xfer_len_in_words(xfer, mas); - fifo_size = mas->tx_fifo_depth * mas->fifo_width_bits / mas->cur_bits_per_word; + fifo_size = mas->tx_fifo_depth * mas->fifo_width_bits / xfer->bits_per_word; if (len > fifo_size) return true; @@ -724,6 +720,12 @@ static int spi_geni_init(struct spi_geni_master *mas) case 0: mas->cur_xfer_mode = GENI_SE_FIFO; geni_se_select_mode(se, GENI_SE_FIFO); + /* setup_fifo_params assumes that these registers start with a zero value */ + writel(0, se->base + SE_SPI_LOOPBACK); + writel(0, se->base + SE_SPI_DEMUX_SEL); + writel(0, se->base + SE_SPI_CPHA); + writel(0, se->base + SE_SPI_CPOL); + writel(0, se->base + SE_SPI_DEMUX_OUTPUT_INV); ret = 0; break; } diff --git a/drivers/spi/spi-mem.c b/drivers/spi/spi-mem.c index 064b99204d9ac..db9467535416d 100644 --- a/drivers/spi/spi-mem.c +++ b/drivers/spi/spi-mem.c @@ -175,8 +175,19 @@ bool spi_mem_default_supports_op(struct spi_mem *mem, if (op->data.swap16 && !spi_mem_controller_is_capable(ctlr, swap16)) return false; - if (op->cmd.nbytes != 2) - return false; + /* Extra 8D-8D-8D limitations */ + if (op->cmd.dtr && op->cmd.buswidth == 8) { + if (op->cmd.nbytes != 2) + return false; + + if ((op->addr.nbytes % 2) || + (op->dummy.nbytes % 2) || + (op->data.nbytes % 2)) { + dev_err(&ctlr->dev, + "Even byte numbers not allowed in octal DTR operations\n"); + return false; + } + } } else { if (op->cmd.nbytes != 1) return false; @@ -703,9 +714,18 @@ spi_mem_dirmap_create(struct spi_mem *mem, desc->mem = mem; desc->info = *info; - if (ctlr->mem_ops && ctlr->mem_ops->dirmap_create) + if (ctlr->mem_ops && ctlr->mem_ops->dirmap_create) { + ret = spi_mem_access_start(mem); + if (ret) { + kfree(desc); + return ERR_PTR(ret); + } + ret = ctlr->mem_ops->dirmap_create(desc); + spi_mem_access_end(mem); + } + if (ret) { desc->nodirmap = true; if (!spi_mem_supports_op(desc->mem, &desc->info.op_tmpl)) diff --git a/drivers/spi/spi-stm32.c b/drivers/spi/spi-stm32.c index 2c804c1aef989..7a6ee93be9bd4 100644 --- a/drivers/spi/spi-stm32.c +++ b/drivers/spi/spi-stm32.c @@ -1570,6 +1570,9 @@ static int stm32_spi_prepare_rx_dma_mdma_chaining(struct stm32_spi *spi, return -EINVAL; } + *rx_mdma_desc = _mdma_desc; + *rx_dma_desc = _dma_desc; + return 0; } @@ -1906,11 +1909,12 @@ static void stm32h7_spi_data_idleness(struct stm32_spi *spi, struct spi_transfer cfg2_clrb |= STM32H7_SPI_CFG2_MIDI; if ((len > 1) && (spi->cur_midi > 0)) { u32 sck_period_ns = DIV_ROUND_UP(NSEC_PER_SEC, spi->cur_speed); - u32 midi = min_t(u32, - DIV_ROUND_UP(spi->cur_midi, sck_period_ns), - FIELD_GET(STM32H7_SPI_CFG2_MIDI, - STM32H7_SPI_CFG2_MIDI)); + u32 midi = DIV_ROUND_UP(spi->cur_midi, sck_period_ns); + + if ((spi->cur_bpw + midi) < 8) + midi = 8 - spi->cur_bpw; + midi = min_t(u32, midi, FIELD_MAX(STM32H7_SPI_CFG2_MIDI)); dev_dbg(spi->dev, "period=%dns, midi=%d(=%dns)\n", sck_period_ns, midi, midi * sck_period_ns); diff --git a/drivers/spi/spi-wpcm-fiu.c b/drivers/spi/spi-wpcm-fiu.c index a9aee2a6c7dcb..c47b56f0933f1 100644 --- a/drivers/spi/spi-wpcm-fiu.c +++ b/drivers/spi/spi-wpcm-fiu.c @@ -459,11 +459,11 @@ static int wpcm_fiu_probe(struct platform_device *pdev) res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "memory"); fiu->memory = devm_ioremap_resource(dev, res); - fiu->memory_size = min_t(size_t, resource_size(res), MAX_MEMORY_SIZE_TOTAL); if (IS_ERR(fiu->memory)) return dev_err_probe(dev, PTR_ERR(fiu->memory), "Failed to map flash memory window\n"); + fiu->memory_size = min_t(size_t, resource_size(res), MAX_MEMORY_SIZE_TOTAL); fiu->shm_regmap = syscon_regmap_lookup_by_phandle_optional(dev->of_node, "nuvoton,shm"); wpcm_fiu_hw_init(fiu); diff --git a/drivers/spi/spidev.c b/drivers/spi/spidev.c index 5300c942a2a44..4d53c394101a4 100644 --- a/drivers/spi/spidev.c +++ b/drivers/spi/spidev.c @@ -74,7 +74,6 @@ struct spidev_data { struct list_head device_entry; /* TX/RX buffers are NULL unless this device is open (users > 0) */ - struct mutex buf_lock; unsigned users; u8 *tx_buffer; u8 *rx_buffer; @@ -102,24 +101,6 @@ spidev_sync_unlocked(struct spi_device *spi, struct spi_message *message) return status; } -static ssize_t -spidev_sync(struct spidev_data *spidev, struct spi_message *message) -{ - ssize_t status; - struct spi_device *spi; - - mutex_lock(&spidev->spi_lock); - spi = spidev->spi; - - if (spi == NULL) - status = -ESHUTDOWN; - else - status = spidev_sync_unlocked(spi, message); - - mutex_unlock(&spidev->spi_lock); - return status; -} - static inline ssize_t spidev_sync_write(struct spidev_data *spidev, size_t len) { @@ -132,7 +113,8 @@ spidev_sync_write(struct spidev_data *spidev, size_t len) spi_message_init(&m); spi_message_add_tail(&t, &m); - return spidev_sync(spidev, &m); + + return spidev_sync_unlocked(spidev->spi, &m); } static inline ssize_t @@ -147,7 +129,8 @@ spidev_sync_read(struct spidev_data *spidev, size_t len) spi_message_init(&m); spi_message_add_tail(&t, &m); - return spidev_sync(spidev, &m); + + return spidev_sync_unlocked(spidev->spi, &m); } /*-------------------------------------------------------------------------*/ @@ -157,7 +140,7 @@ static ssize_t spidev_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) { struct spidev_data *spidev; - ssize_t status; + ssize_t status = -ESHUTDOWN; /* chipselect only toggles at start or end of operation */ if (count > bufsiz) @@ -165,7 +148,11 @@ spidev_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) spidev = filp->private_data; - mutex_lock(&spidev->buf_lock); + mutex_lock(&spidev->spi_lock); + + if (spidev->spi == NULL) + goto err_spi_removed; + status = spidev_sync_read(spidev, count); if (status > 0) { unsigned long missing; @@ -176,7 +163,9 @@ spidev_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) else status = status - missing; } - mutex_unlock(&spidev->buf_lock); + +err_spi_removed: + mutex_unlock(&spidev->spi_lock); return status; } @@ -187,7 +176,7 @@ spidev_write(struct file *filp, const char __user *buf, size_t count, loff_t *f_pos) { struct spidev_data *spidev; - ssize_t status; + ssize_t status = -ESHUTDOWN; unsigned long missing; /* chipselect only toggles at start or end of operation */ @@ -196,13 +185,19 @@ spidev_write(struct file *filp, const char __user *buf, spidev = filp->private_data; - mutex_lock(&spidev->buf_lock); + mutex_lock(&spidev->spi_lock); + + if (spidev->spi == NULL) + goto err_spi_removed; + missing = copy_from_user(spidev->tx_buffer, buf, count); if (missing == 0) status = spidev_sync_write(spidev, count); else status = -EFAULT; - mutex_unlock(&spidev->buf_lock); + +err_spi_removed: + mutex_unlock(&spidev->spi_lock); return status; } @@ -379,14 +374,6 @@ spidev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) ctlr = spi->controller; - /* use the buffer lock here for triple duty: - * - prevent I/O (from us) so calling spi_setup() is safe; - * - prevent concurrent SPI_IOC_WR_* from morphing - * data fields while SPI_IOC_RD_* reads them; - * - SPI_IOC_MESSAGE needs the buffer locked "normally". - */ - mutex_lock(&spidev->buf_lock); - switch (cmd) { /* read requests */ case SPI_IOC_RD_MODE: @@ -510,7 +497,6 @@ spidev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) break; } - mutex_unlock(&spidev->buf_lock); spi_dev_put(spi); mutex_unlock(&spidev->spi_lock); return retval; @@ -541,9 +527,6 @@ spidev_compat_ioc_message(struct file *filp, unsigned int cmd, return -ESHUTDOWN; } - /* SPI_IOC_MESSAGE needs the buffer locked "normally" */ - mutex_lock(&spidev->buf_lock); - /* Check message and copy into scratch area */ ioc = spidev_get_ioc_message(cmd, u_ioc, &n_ioc); if (IS_ERR(ioc)) { @@ -564,7 +547,6 @@ spidev_compat_ioc_message(struct file *filp, unsigned int cmd, kfree(ioc); done: - mutex_unlock(&spidev->buf_lock); spi_dev_put(spi); mutex_unlock(&spidev->spi_lock); return retval; @@ -800,7 +782,6 @@ static int spidev_probe(struct spi_device *spi) /* Initialize the driver data */ spidev->spi = spi; mutex_init(&spidev->spi_lock); - mutex_init(&spidev->buf_lock); INIT_LIST_HEAD(&spidev->device_entry); diff --git a/drivers/spmi/spmi-apple-controller.c b/drivers/spmi/spmi-apple-controller.c index 697b3e8bb0235..87e3ee9d4f2aa 100644 --- a/drivers/spmi/spmi-apple-controller.c +++ b/drivers/spmi/spmi-apple-controller.c @@ -149,6 +149,7 @@ static int apple_spmi_probe(struct platform_device *pdev) } static const struct of_device_id apple_spmi_match_table[] = { + { .compatible = "apple,t8103-spmi", }, { .compatible = "apple,spmi", }, {} }; diff --git a/drivers/staging/gpib/common/iblib.c b/drivers/staging/gpib/common/iblib.c index 7cbb6a467177d..b672dd6aad25f 100644 --- a/drivers/staging/gpib/common/iblib.c +++ b/drivers/staging/gpib/common/iblib.c @@ -227,11 +227,10 @@ int ibonline(struct gpib_board *board) #ifndef CONFIG_NIOS2 board->autospoll_task = kthread_run(&autospoll_thread, board, "gpib%d_autospoll_kthread", board->minor); - retval = IS_ERR(board->autospoll_task); - if (retval) { + if (IS_ERR(board->autospoll_task)) { dev_err(board->gpib_dev, "failed to create autospoll thread\n"); board->interface->detach(board); - return retval; + return PTR_ERR(board->autospoll_task); } #endif board->online = 1; diff --git a/drivers/staging/gpib/ni_usb/ni_usb_gpib.c b/drivers/staging/gpib/ni_usb/ni_usb_gpib.c index 1f8412de9fa32..b6fddb437f552 100644 --- a/drivers/staging/gpib/ni_usb/ni_usb_gpib.c +++ b/drivers/staging/gpib/ni_usb/ni_usb_gpib.c @@ -566,7 +566,7 @@ static int ni_usb_write_registers(struct ni_usb_priv *ni_priv, retval, bytes_read); ni_usb_dump_raw_block(in_data, bytes_read); kfree(in_data); - return retval; + return retval ?: -EINVAL; } mutex_unlock(&ni_priv->addressed_transfer_lock); @@ -1780,7 +1780,7 @@ static int ni_usb_setup_init(struct gpib_board *board, struct ni_usb_register *w i++; if (i > NUM_INIT_WRITES) { dev_err(&usb_dev->dev, "bug!, buffer overrun, i=%i\n", i); - return 0; + return -EINVAL; } return i; } @@ -1799,10 +1799,12 @@ static int ni_usb_init(struct gpib_board *board) return -ENOMEM; writes_len = ni_usb_setup_init(board, writes); - if (writes_len) - retval = ni_usb_write_registers(ni_priv, writes, writes_len, &ibsta); - else - return -EFAULT; + if (writes_len < 0) { + kfree(writes); + return writes_len; + } + + retval = ni_usb_write_registers(ni_priv, writes, writes_len, &ibsta); kfree(writes); if (retval) { dev_err(&usb_dev->dev, "register write failed, retval=%i\n", retval); diff --git a/drivers/staging/greybus/light.c b/drivers/staging/greybus/light.c index e509fdc715dbb..38c233a706c48 100644 --- a/drivers/staging/greybus/light.c +++ b/drivers/staging/greybus/light.c @@ -1008,14 +1008,18 @@ static int gb_lights_light_config(struct gb_lights *glights, u8 id) if (!strlen(conf.name)) return -EINVAL; - light->channels_count = conf.channel_count; light->name = kstrndup(conf.name, NAMES_MAX, GFP_KERNEL); if (!light->name) return -ENOMEM; - light->channels = kcalloc(light->channels_count, + light->channels = kcalloc(conf.channel_count, sizeof(struct gb_channel), GFP_KERNEL); if (!light->channels) return -ENOMEM; + /* + * Publish channels_count only after channels allocation so cleanup + * doesn't walk a NULL channels pointer on allocation failure. + */ + light->channels_count = conf.channel_count; /* First we collect all the configurations for all channels */ for (i = 0; i < light->channels_count; i++) { diff --git a/drivers/staging/media/ipu7/ipu7-buttress.c b/drivers/staging/media/ipu7/ipu7-buttress.c index e5707f5e300ba..40c6c8473357c 100644 --- a/drivers/staging/media/ipu7/ipu7-buttress.c +++ b/drivers/staging/media/ipu7/ipu7-buttress.c @@ -342,14 +342,23 @@ irqreturn_t ipu_buttress_isr(int irq, void *isp_ptr) u32 disable_irqs = 0; u32 irq_status; unsigned int i; + int active; - pm_runtime_get_noresume(dev); + active = pm_runtime_get_if_active(dev); + if (active <= 0) + return IRQ_NONE; pb_irq = readl(isp->pb_base + INTERRUPT_STATUS); writel(pb_irq, isp->pb_base + INTERRUPT_STATUS); /* check btrs ATS, CFI and IMR errors, BIT(0) is unused for IPU */ pb_local_irq = readl(isp->pb_base + BTRS_LOCAL_INTERRUPT_MASK); + if (pb_local_irq == 0xffffffff) { + dev_warn_once(dev, "invalid PB irq status\n"); + pm_runtime_put_noidle(dev); + return IRQ_NONE; + } + if (pb_local_irq & ~BIT(0)) { dev_warn(dev, "PB interrupt status 0x%x local 0x%x\n", pb_irq, pb_local_irq); @@ -370,6 +379,12 @@ irqreturn_t ipu_buttress_isr(int irq, void *isp_ptr) return IRQ_NONE; } + if (irq_status == 0xffffffff) { + dev_warn_once(dev, "invalid irq status 0x%08x\n", irq_status); + pm_runtime_put_noidle(dev); + return IRQ_NONE; + } + do { writel(irq_status, isp->base + BUTTRESS_REG_IRQ_CLEAR); diff --git a/drivers/staging/media/ipu7/ipu7-isys-csi-phy.c b/drivers/staging/media/ipu7/ipu7-isys-csi-phy.c index b8c5db7ae3009..d8f3592ad6201 100644 --- a/drivers/staging/media/ipu7/ipu7-isys-csi-phy.c +++ b/drivers/staging/media/ipu7/ipu7-isys-csi-phy.c @@ -124,6 +124,7 @@ static const struct cdr_fbk_cap_prog_params table7[] = { { 1350, 1589, 4 }, { 1590, 1949, 5 }, { 1950, 2499, 6 }, + { 2500, 3500, 7 }, { } }; @@ -838,9 +839,10 @@ static void ipu7_isys_cphy_config(struct ipu7_isys *isys, u8 id, u8 lanes, dwc_phy_write_mask(isys, id, reg + 0x400 * i, reset_thresh, 9, 11); + /* Tuning ITMINRX to 2 for CPHY */ reg = CORE_DIG_CLANE_0_RW_LP_0; for (i = 0; i < trios; i++) - dwc_phy_write_mask(isys, id, reg + 0x400 * i, 1, 12, 15); + dwc_phy_write_mask(isys, id, reg + 0x400 * i, 2, 12, 15); reg = CORE_DIG_CLANE_0_RW_LP_2; for (i = 0; i < trios; i++) @@ -860,7 +862,11 @@ static void ipu7_isys_cphy_config(struct ipu7_isys *isys, u8 id, u8 lanes, for (i = 0; i < (lanes + 1); i++) { reg = CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_9 + 0x400 * i; dwc_phy_write_mask(isys, id, reg, 4U, 0, 2); - dwc_phy_write_mask(isys, id, reg, 0U, 3, 4); + /* Set GMODE to 2 when CPHY >= 1.5Gsps */ + if (mbps >= 1500) + dwc_phy_write_mask(isys, id, reg, 2U, 3, 4); + else + dwc_phy_write_mask(isys, id, reg, 0U, 3, 4); reg = CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_7 + 0x400 * i; dwc_phy_write_mask(isys, id, reg, cap_prog, 10, 12); @@ -930,8 +936,9 @@ static int ipu7_isys_phy_config(struct ipu7_isys *isys, u8 id, u8 lanes, 7, 12, 14); dwc_phy_write_mask(isys, id, CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_7, 0, 8, 10); + /* resistance tuning: 1 for 45ohm, 0 for 50ohm */ dwc_phy_write_mask(isys, id, CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_5, - 0, 8, 8); + 1, 8, 8); if (aggregation) phy_mode = isys->csi2[0].phy_mode; diff --git a/drivers/staging/media/ipu7/ipu7-mmu.c b/drivers/staging/media/ipu7/ipu7-mmu.c index ded1986eb8ba3..ea35cce4830ad 100644 --- a/drivers/staging/media/ipu7/ipu7-mmu.c +++ b/drivers/staging/media/ipu7/ipu7-mmu.c @@ -231,7 +231,7 @@ static u32 *alloc_l2_pt(struct ipu7_mmu_info *mmu_info) dev_dbg(mmu_info->dev, "alloc_l2: get_zeroed_page() = %p\n", pt); - for (i = 0; i < ISP_L1PT_PTES; i++) + for (i = 0; i < ISP_L2PT_PTES; i++) pt[i] = mmu_info->dummy_page_pteval; return pt; diff --git a/drivers/staging/media/ipu7/ipu7.c b/drivers/staging/media/ipu7/ipu7.c index 5cddc09c72bf2..fa5a1867626f8 100644 --- a/drivers/staging/media/ipu7/ipu7.c +++ b/drivers/staging/media/ipu7/ipu7.c @@ -2620,7 +2620,7 @@ static int ipu7_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (!IS_ERR_OR_NULL(isp->isys) && !IS_ERR_OR_NULL(isp->isys->mmu)) ipu7_mmu_cleanup(isp->isys->mmu); if (!IS_ERR_OR_NULL(isp->psys)) - pm_runtime_put(&isp->psys->auxdev.dev); + pm_runtime_put_sync(&isp->psys->auxdev.dev); ipu7_bus_del_devices(pdev); release_firmware(isp->cpd_fw); buttress_exit: @@ -2684,6 +2684,10 @@ static void ipu7_pci_reset_done(struct pci_dev *pdev) */ static int ipu7_suspend(struct device *dev) { + struct pci_dev *pdev = to_pci_dev(dev); + + synchronize_irq(pdev->irq); + return 0; } diff --git a/drivers/staging/media/tegra-video/vi.c b/drivers/staging/media/tegra-video/vi.c index c9276ff76157f..14b327afe045e 100644 --- a/drivers/staging/media/tegra-video/vi.c +++ b/drivers/staging/media/tegra-video/vi.c @@ -438,7 +438,7 @@ static int __tegra_channel_try_format(struct tegra_vi_channel *chan, .target = V4L2_SEL_TGT_CROP_BOUNDS, }; struct v4l2_rect *try_crop; - int ret; + int ret = 0; subdev = tegra_channel_get_remote_source_subdev(chan); if (!subdev) @@ -482,8 +482,10 @@ static int __tegra_channel_try_format(struct tegra_vi_channel *chan, } else { ret = v4l2_subdev_call(subdev, pad, get_selection, NULL, &sdsel); - if (ret) - return -EINVAL; + if (ret) { + ret = -EINVAL; + goto out_free; + } try_crop->width = sdsel.r.width; try_crop->height = sdsel.r.height; @@ -495,14 +497,15 @@ static int __tegra_channel_try_format(struct tegra_vi_channel *chan, ret = v4l2_subdev_call(subdev, pad, set_fmt, sd_state, &fmt); if (ret < 0) - return ret; + goto out_free; v4l2_fill_pix_format(pix, &fmt.format); chan->vi->ops->vi_fmt_align(pix, fmtinfo->bpp); +out_free: __v4l2_subdev_state_free(sd_state); - return 0; + return ret; } static int tegra_channel_try_format(struct file *file, void *fh, diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme.c b/drivers/staging/rtl8723bs/core/rtw_mlme.c index c06d990350e69..362c904adddc5 100644 --- a/drivers/staging/rtl8723bs/core/rtw_mlme.c +++ b/drivers/staging/rtl8723bs/core/rtw_mlme.c @@ -835,8 +835,10 @@ static void find_network(struct adapter *adapter) struct wlan_network *tgt_network = &pmlmepriv->cur_network; pwlan = rtw_find_network(&pmlmepriv->scanned_queue, tgt_network->network.mac_address); - if (pwlan) - pwlan->fixed = false; + if (!pwlan) + return; + + pwlan->fixed = false; if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) && (adapter->stapriv.asoc_sta_count == 1)) diff --git a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c index 315bab3737294..5d28dbf8b50ef 100644 --- a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c +++ b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c @@ -315,9 +315,10 @@ struct cfg80211_bss *rtw_cfg80211_inform_bss(struct adapter *padapter, struct wl len, notify_signal, GFP_ATOMIC); if (unlikely(!bss)) - goto exit; + goto free_buf; cfg80211_put_bss(wiphy, bss); +free_buf: kfree(buf); exit: diff --git a/drivers/staging/rtl8723bs/os_dep/sdio_intf.c b/drivers/staging/rtl8723bs/os_dep/sdio_intf.c index f3caaa857c864..139ace51486d2 100644 --- a/drivers/staging/rtl8723bs/os_dep/sdio_intf.c +++ b/drivers/staging/rtl8723bs/os_dep/sdio_intf.c @@ -377,7 +377,8 @@ static int rtw_drv_init( if (status != _SUCCESS) goto free_if1; - if (sdio_alloc_irq(dvobj) != _SUCCESS) + status = sdio_alloc_irq(dvobj); + if (status != _SUCCESS) goto free_if1; status = _SUCCESS; diff --git a/drivers/target/target_core_configfs.c b/drivers/target/target_core_configfs.c index 1bd28482e7cb3..a2bd2e81d2c68 100644 --- a/drivers/target/target_core_configfs.c +++ b/drivers/target/target_core_configfs.c @@ -108,8 +108,8 @@ static ssize_t target_core_item_dbroot_store(struct config_item *item, const char *page, size_t count) { ssize_t read_bytes; - struct file *fp; ssize_t r = -EINVAL; + struct path path = {}; mutex_lock(&target_devices_lock); if (target_devices) { @@ -131,17 +131,14 @@ static ssize_t target_core_item_dbroot_store(struct config_item *item, db_root_stage[read_bytes - 1] = '\0'; /* validate new db root before accepting it */ - fp = filp_open(db_root_stage, O_RDONLY, 0); - if (IS_ERR(fp)) { + r = kern_path(db_root_stage, LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &path); + if (r) { pr_err("db_root: cannot open: %s\n", db_root_stage); + if (r == -ENOTDIR) + pr_err("db_root: not a directory: %s\n", db_root_stage); goto unlock; } - if (!S_ISDIR(file_inode(fp)->i_mode)) { - filp_close(fp, NULL); - pr_err("db_root: not a directory: %s\n", db_root_stage); - goto unlock; - } - filp_close(fp, NULL); + path_put(&path); strscpy(db_root, db_root_stage); pr_debug("Target_Core_ConfigFS: db_root set to %s\n", db_root); diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c index 1f3d22b659dbe..635bc8fded1e9 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c @@ -451,8 +451,11 @@ int proc_thermal_rfim_add(struct pci_dev *pdev, struct proc_thermal_device *proc break; } ret = sysfs_create_group(&pdev->dev.kobj, &dlvr_attribute_group); - if (ret) + if (ret) { + if (proc_priv->mmio_feature_mask & PROC_THERMAL_FEATURE_FIVR) + sysfs_remove_group(&pdev->dev.kobj, &fivr_attribute_group); return ret; + } } if (proc_priv->mmio_feature_mask & PROC_THERMAL_FEATURE_DVFS) { diff --git a/drivers/thermal/intel/x86_pkg_temp_thermal.c b/drivers/thermal/intel/x86_pkg_temp_thermal.c index 3fc679b6f11b1..aab5f9fca9c33 100644 --- a/drivers/thermal/intel/x86_pkg_temp_thermal.c +++ b/drivers/thermal/intel/x86_pkg_temp_thermal.c @@ -128,6 +128,9 @@ sys_set_trip_temp(struct thermal_zone_device *tzd, u32 l, h, mask, shift, intr; int tj_max, val, ret; + if (temp == THERMAL_TEMP_INVALID) + temp = 0; + tj_max = intel_tcc_get_tjmax(zonedev->cpu); if (tj_max < 0) return tj_max; diff --git a/drivers/thermal/thermal_of.c b/drivers/thermal/thermal_of.c index 1a51a4d240ff6..b6d0c92f5522b 100644 --- a/drivers/thermal/thermal_of.c +++ b/drivers/thermal/thermal_of.c @@ -280,10 +280,10 @@ static bool thermal_of_cm_lookup(struct device_node *cm_np, struct cooling_spec *c) { for_each_child_of_node_scoped(cm_np, child) { - struct device_node *tr_np; int count, i; - tr_np = of_parse_phandle(child, "trip", 0); + struct device_node *tr_np __free(device_node) = + of_parse_phandle(child, "trip", 0); if (tr_np != trip->priv) continue; diff --git a/drivers/tty/serial/8250/8250_dw.c b/drivers/tty/serial/8250/8250_dw.c index 710ae4d40aec4..0ff500965c103 100644 --- a/drivers/tty/serial/8250/8250_dw.c +++ b/drivers/tty/serial/8250/8250_dw.c @@ -741,11 +741,18 @@ static int dw8250_runtime_suspend(struct device *dev) static int dw8250_runtime_resume(struct device *dev) { + int ret; struct dw8250_data *data = dev_get_drvdata(dev); - clk_prepare_enable(data->pclk); + ret = clk_prepare_enable(data->pclk); + if (ret) + return ret; - clk_prepare_enable(data->clk); + ret = clk_prepare_enable(data->clk); + if (ret) { + clk_disable_unprepare(data->pclk); + return ret; + } return 0; } diff --git a/drivers/tty/serial/8250/8250_omap.c b/drivers/tty/serial/8250/8250_omap.c index 9e49ef48b851b..272bc07c9a6b5 100644 --- a/drivers/tty/serial/8250/8250_omap.c +++ b/drivers/tty/serial/8250/8250_omap.c @@ -100,6 +100,9 @@ #define OMAP_UART_REV_52 0x0502 #define OMAP_UART_REV_63 0x0603 +/* Resume register */ +#define UART_OMAP_RESUME 0x0B + /* Interrupt Enable Register 2 */ #define UART_OMAP_IER2 0x1B #define UART_OMAP_IER2_RHR_IT_DIS BIT(2) @@ -119,7 +122,6 @@ /* Timeout low and High */ #define UART_OMAP_TO_L 0x26 #define UART_OMAP_TO_H 0x27 - struct omap8250_priv { void __iomem *membase; int line; @@ -929,7 +931,6 @@ static void __dma_rx_do_complete(struct uart_8250_port *p) goto out; cookie = dma->rx_cookie; - dma->rx_running = 0; /* Re-enable RX FIFO interrupt now that transfer is complete */ if (priv->habit & UART_HAS_RHR_IT_DIS) { @@ -963,6 +964,7 @@ static void __dma_rx_do_complete(struct uart_8250_port *p) goto out; ret = tty_insert_flip_string(tty_port, dma->rx_buf, count); + dma->rx_running = 0; p->port.icount.rx += ret; p->port.icount.buf_overrun += count - ret; out: @@ -1256,6 +1258,20 @@ static u16 omap_8250_handle_rx_dma(struct uart_8250_port *up, u8 iir, u16 status return status; } +static void am654_8250_handle_uart_errors(struct uart_8250_port *up, u8 iir, u16 status) +{ + if (status & UART_LSR_OE) { + serial8250_clear_and_reinit_fifos(up); + serial_in(up, UART_LSR); + serial_in(up, UART_OMAP_RESUME); + } else { + if (status & (UART_LSR_FE | UART_LSR_PE | UART_LSR_BI)) + serial_in(up, UART_RX); + if (iir & UART_IIR_XOFF) + serial_in(up, UART_IIR); + } +} + static void am654_8250_handle_rx_dma(struct uart_8250_port *up, u8 iir, u16 status) { @@ -1266,7 +1282,8 @@ static void am654_8250_handle_rx_dma(struct uart_8250_port *up, u8 iir, * Queue a new transfer if FIFO has data. */ if ((status & (UART_LSR_DR | UART_LSR_BI)) && - (up->ier & UART_IER_RDI)) { + (up->ier & UART_IER_RDI) && !(status & UART_LSR_OE)) { + am654_8250_handle_uart_errors(up, iir, status); omap_8250_rx_dma(up); serial_out(up, UART_OMAP_EFR2, UART_OMAP_EFR2_TIMEOUT_BEHAVE); } else if ((iir & 0x3f) == UART_IIR_RX_TIMEOUT) { @@ -1282,6 +1299,8 @@ static void am654_8250_handle_rx_dma(struct uart_8250_port *up, u8 iir, serial_out(up, UART_OMAP_EFR2, 0x0); up->ier |= UART_IER_RLSI | UART_IER_RDI; serial_out(up, UART_IER, up->ier); + } else { + am654_8250_handle_uart_errors(up, iir, status); } } diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig index 282116765e648..2b9c8b39d68f5 100644 --- a/drivers/tty/serial/Kconfig +++ b/drivers/tty/serial/Kconfig @@ -486,14 +486,14 @@ config SERIAL_IMX can enable its onboard serial port by enabling this option. config SERIAL_IMX_CONSOLE - tristate "Console on IMX serial port" + bool "Console on IMX serial port" depends on SERIAL_IMX select SERIAL_CORE_CONSOLE help If you have enabled the serial port on the Freescale IMX - CPU you can make it the console by answering Y/M to this option. + CPU you can make it the console by answering Y to this option. - Even if you say Y/M here, the currently visible virtual console + Even if you say Y here, the currently visible virtual console (/dev/tty0) will still be used as the system console by default, but you can alter that using a kernel command line option such as "console=ttymxc0". (Try "man bootparam" or see the documentation of @@ -671,7 +671,7 @@ config SERIAL_SH_SCI_EARLYCON default ARCH_RENESAS config SERIAL_SH_SCI_DMA - bool "DMA support" if EXPERT + bool "Support for DMA on SuperH SCI(F)" if EXPERT depends on SERIAL_SH_SCI && DMA_ENGINE default ARCH_RENESAS diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 2747f378ba5ed..30117a8387ebe 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -4401,14 +4401,6 @@ static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) spin_unlock_irqrestore(hba->host->host_lock, flags); mutex_unlock(&hba->uic_cmd_mutex); - /* - * If the h8 exit fails during the runtime resume process, it becomes - * stuck and cannot be recovered through the error handler. To fix - * this, use link recovery instead of the error handler. - */ - if (ret && hba->pm_op_in_progress) - ret = ufshcd_link_recovery(hba); - return ret; } @@ -9878,6 +9870,8 @@ static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE && req_link_state == UIC_LINK_ACTIVE_STATE) { + ufshcd_disable_auto_bkops(hba); + flush_work(&hba->eeh_work); goto vops_suspend; } @@ -10056,7 +10050,15 @@ static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) } else { dev_err(hba->dev, "%s: hibern8 exit failed %d\n", __func__, ret); - goto vendor_suspend; + /* + * If the h8 exit fails during the runtime resume + * process, it becomes stuck and cannot be recovered + * through the error handler. To fix this, use link + * recovery instead of the error handler. + */ + ret = ufshcd_link_recovery(hba); + if (ret) + goto vendor_suspend; } } else if (ufshcd_is_link_off(hba)) { /* diff --git a/drivers/ufs/host/Kconfig b/drivers/ufs/host/Kconfig index 191fbd799ec5b..48ee7e9b665ef 100644 --- a/drivers/ufs/host/Kconfig +++ b/drivers/ufs/host/Kconfig @@ -72,6 +72,7 @@ config SCSI_UFS_QCOM config SCSI_UFS_MEDIATEK tristate "Mediatek specific hooks to UFS controller platform driver" depends on SCSI_UFSHCD_PLATFORM && ARCH_MEDIATEK + depends on PM depends on RESET_CONTROLLER select PHY_MTK_UFS select RESET_TI_SYSCON diff --git a/drivers/ufs/host/ufs-mediatek-trace.h b/drivers/ufs/host/ufs-mediatek-trace.h index b5f2ec3140748..0df8ac843379a 100644 --- a/drivers/ufs/host/ufs-mediatek-trace.h +++ b/drivers/ufs/host/ufs-mediatek-trace.h @@ -33,19 +33,19 @@ TRACE_EVENT(ufs_mtk_clk_scale, TP_ARGS(name, scale_up, clk_rate), TP_STRUCT__entry( - __field(const char*, name) + __string(name, name) __field(bool, scale_up) __field(unsigned long, clk_rate) ), TP_fast_assign( - __entry->name = name; + __assign_str(name); __entry->scale_up = scale_up; __entry->clk_rate = clk_rate; ), TP_printk("ufs: clk (%s) scaled %s @ %lu", - __entry->name, + __get_str(name), __entry->scale_up ? "up" : "down", __entry->clk_rate) ); diff --git a/drivers/ufs/host/ufs-mediatek.c b/drivers/ufs/host/ufs-mediatek.c index d0cbd96ad29dc..3c63adca561d2 100644 --- a/drivers/ufs/host/ufs-mediatek.c +++ b/drivers/ufs/host/ufs-mediatek.c @@ -2366,7 +2366,6 @@ static void ufs_mtk_remove(struct platform_device *pdev) ufshcd_pltfrm_remove(pdev); } -#ifdef CONFIG_PM_SLEEP static int ufs_mtk_system_suspend(struct device *dev) { struct ufs_hba *hba = dev_get_drvdata(dev); @@ -2413,9 +2412,7 @@ static int ufs_mtk_system_resume(struct device *dev) return ret; } -#endif -#ifdef CONFIG_PM static int ufs_mtk_runtime_suspend(struct device *dev) { struct ufs_hba *hba = dev_get_drvdata(dev); @@ -2454,13 +2451,10 @@ static int ufs_mtk_runtime_resume(struct device *dev) return ufshcd_runtime_resume(dev); } -#endif static const struct dev_pm_ops ufs_mtk_pm_ops = { - SET_SYSTEM_SLEEP_PM_OPS(ufs_mtk_system_suspend, - ufs_mtk_system_resume) - SET_RUNTIME_PM_OPS(ufs_mtk_runtime_suspend, - ufs_mtk_runtime_resume, NULL) + SYSTEM_SLEEP_PM_OPS(ufs_mtk_system_suspend, ufs_mtk_system_resume) + RUNTIME_PM_OPS(ufs_mtk_runtime_suspend, ufs_mtk_runtime_resume, NULL) .prepare = ufshcd_suspend_prepare, .complete = ufshcd_resume_complete, }; @@ -2470,7 +2464,7 @@ static struct platform_driver ufs_mtk_pltform = { .remove = ufs_mtk_remove, .driver = { .name = "ufshcd-mtk", - .pm = &ufs_mtk_pm_ops, + .pm = pm_ptr(&ufs_mtk_pm_ops), .of_match_table = ufs_mtk_of_match, }, }; diff --git a/drivers/usb/cdns3/core.c b/drivers/usb/cdns3/core.c index 1243a5cea91b5..f0e32227c0b79 100644 --- a/drivers/usb/cdns3/core.c +++ b/drivers/usb/cdns3/core.c @@ -551,7 +551,7 @@ int cdns_resume(struct cdns *cdns) } } - if (cdns->roles[cdns->role]->resume) + if (!role_changed && cdns->roles[cdns->role]->resume) cdns->roles[cdns->role]->resume(cdns, power_lost); return 0; diff --git a/drivers/usb/chipidea/udc.c b/drivers/usb/chipidea/udc.c index 64a421ae0f05b..c8d931d9d4330 100644 --- a/drivers/usb/chipidea/udc.c +++ b/drivers/usb/chipidea/udc.c @@ -931,6 +931,13 @@ __acquires(hwep->lock) list_del_init(&hwreq->queue); hwreq->req.status = -ESHUTDOWN; + /* Unmap DMA and clean up bounce buffers before giving back */ + usb_gadget_unmap_request_by_dev(hwep->ci->dev->parent, + &hwreq->req, hwep->dir); + + if (hwreq->sgt.sgl) + sglist_do_debounce(hwreq, false); + if (hwreq->req.complete != NULL) { spin_unlock(hwep->lock); usb_gadget_giveback_request(&hwep->ep, &hwreq->req); diff --git a/drivers/usb/dwc2/core.c b/drivers/usb/dwc2/core.c index c3d24312db0fe..f375c5185bfe2 100644 --- a/drivers/usb/dwc2/core.c +++ b/drivers/usb/dwc2/core.c @@ -578,6 +578,7 @@ void dwc2_force_dr_mode(struct dwc2_hsotg *hsotg) { switch (hsotg->dr_mode) { case USB_DR_MODE_HOST: + dwc2_force_mode(hsotg, true); /* * NOTE: This is required for some rockchip soc based * platforms on their host-only dwc2. diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index eb10490a6d92c..c6b7df5682b4e 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -2149,6 +2149,20 @@ static int dwc3_get_num_ports(struct dwc3 *dwc) return 0; } +static void dwc3_vbus_draw_work(struct work_struct *work) +{ + struct dwc3 *dwc = container_of(work, struct dwc3, vbus_draw_work); + union power_supply_propval val = {0}; + int ret; + + val.intval = 1000 * (dwc->current_limit); + ret = power_supply_set_property(dwc->usb_psy, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val); + + if (ret < 0) + dev_dbg(dwc->dev, "Error (%d) setting vbus draw (%d mA)\n", + ret, dwc->current_limit); +} + static struct power_supply *dwc3_get_usb_power_supply(struct dwc3 *dwc) { struct power_supply *usb_psy; @@ -2163,6 +2177,7 @@ static struct power_supply *dwc3_get_usb_power_supply(struct dwc3 *dwc) if (!usb_psy) return ERR_PTR(-EPROBE_DEFER); + INIT_WORK(&dwc->vbus_draw_work, dwc3_vbus_draw_work); return usb_psy; } @@ -2386,8 +2401,10 @@ void dwc3_core_remove(struct dwc3 *dwc) dwc3_free_event_buffers(dwc); - if (dwc->usb_psy) + if (dwc->usb_psy) { + cancel_work_sync(&dwc->vbus_draw_work); power_supply_put(dwc->usb_psy); + } } EXPORT_SYMBOL_GPL(dwc3_core_remove); diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h index 45757169b672f..9cfc36d4bc259 100644 --- a/drivers/usb/dwc3/core.h +++ b/drivers/usb/dwc3/core.h @@ -1060,6 +1060,8 @@ struct dwc3_glue_ops { * @role_switch_default_mode: default operation mode of controller while * usb role is USB_ROLE_NONE. * @usb_psy: pointer to power supply interface. + * @vbus_draw_work: Work to set the vbus drawing limit + * @current_limit: How much current to draw from vbus, in milliAmperes. * @usb2_phy: pointer to USB2 PHY * @usb3_phy: pointer to USB3 PHY * @usb2_generic_phy: pointer to array of USB2 PHYs @@ -1246,6 +1248,8 @@ struct dwc3 { enum usb_dr_mode role_switch_default_mode; struct power_supply *usb_psy; + struct work_struct vbus_draw_work; + unsigned int current_limit; u32 fladj; u32 ref_clk_per; diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index 17eebb60900bf..db5e5b77b1eac 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -3123,8 +3123,6 @@ static void dwc3_gadget_set_ssp_rate(struct usb_gadget *g, static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA) { struct dwc3 *dwc = gadget_to_dwc(g); - union power_supply_propval val = {0}; - int ret; if (dwc->usb2_phy) return usb_phy_set_power(dwc->usb2_phy, mA); @@ -3132,10 +3130,10 @@ static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA) if (!dwc->usb_psy) return -EOPNOTSUPP; - val.intval = 1000 * mA; - ret = power_supply_set_property(dwc->usb_psy, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val); + dwc->current_limit = mA; + schedule_work(&dwc->vbus_draw_work); - return ret; + return 0; } /** diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c index 47cfbe41fdff8..497d20260b040 100644 --- a/drivers/usb/gadget/function/f_fs.c +++ b/drivers/usb/gadget/function/f_fs.c @@ -1489,7 +1489,7 @@ static int ffs_dmabuf_attach(struct file *file, int fd) goto err_dmabuf_detach; } - dir = epfile->in ? DMA_FROM_DEVICE : DMA_TO_DEVICE; + dir = epfile->in ? DMA_TO_DEVICE : DMA_FROM_DEVICE; err = ffs_dma_resv_lock(dmabuf, nonblock); if (err) @@ -1619,7 +1619,7 @@ static int ffs_dmabuf_transfer(struct file *file, /* Make sure we don't have writers */ timeout = nonblock ? 0 : msecs_to_jiffies(DMABUF_ENQUEUE_TIMEOUT_MS); retl = dma_resv_wait_timeout(dmabuf->resv, - dma_resv_usage_rw(epfile->in), + dma_resv_usage_rw(!epfile->in), true, timeout); if (retl == 0) retl = -EBUSY; @@ -1664,7 +1664,7 @@ static int ffs_dmabuf_transfer(struct file *file, dma_fence_init(&fence->base, &ffs_dmabuf_fence_ops, &priv->lock, priv->context, seqno); - resv_dir = epfile->in ? DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_READ; + resv_dir = epfile->in ? DMA_RESV_USAGE_READ : DMA_RESV_USAGE_WRITE; dma_resv_add_fence(dmabuf->resv, &fence->base, resv_dir); dma_resv_unlock(dmabuf->resv); @@ -1724,10 +1724,8 @@ static long ffs_epfile_ioctl(struct file *file, unsigned code, { int fd; - if (copy_from_user(&fd, (void __user *)value, sizeof(fd))) { - ret = -EFAULT; - break; - } + if (copy_from_user(&fd, (void __user *)value, sizeof(fd))) + return -EFAULT; return ffs_dmabuf_attach(file, fd); } @@ -1735,10 +1733,8 @@ static long ffs_epfile_ioctl(struct file *file, unsigned code, { int fd; - if (copy_from_user(&fd, (void __user *)value, sizeof(fd))) { - ret = -EFAULT; - break; - } + if (copy_from_user(&fd, (void __user *)value, sizeof(fd))) + return -EFAULT; return ffs_dmabuf_detach(file, fd); } @@ -1746,10 +1742,8 @@ static long ffs_epfile_ioctl(struct file *file, unsigned code, { struct usb_ffs_dmabuf_transfer_req req; - if (copy_from_user(&req, (void __user *)value, sizeof(req))) { - ret = -EFAULT; - break; - } + if (copy_from_user(&req, (void __user *)value, sizeof(req))) + return -EFAULT; return ffs_dmabuf_transfer(file, &req); } diff --git a/drivers/usb/gadget/function/f_ncm.c b/drivers/usb/gadget/function/f_ncm.c index 0e38330271d5a..e23adc132f886 100644 --- a/drivers/usb/gadget/function/f_ncm.c +++ b/drivers/usb/gadget/function/f_ncm.c @@ -83,6 +83,11 @@ static inline struct f_ncm *func_to_ncm(struct usb_function *f) return container_of(f, struct f_ncm, port.func); } +static inline struct f_ncm_opts *func_to_ncm_opts(struct usb_function *f) +{ + return container_of(f->fi, struct f_ncm_opts, func_inst); +} + /*-------------------------------------------------------------------------*/ /* @@ -859,6 +864,7 @@ static int ncm_setup(struct usb_function *f, const struct usb_ctrlrequest *ctrl) static int ncm_set_alt(struct usb_function *f, unsigned intf, unsigned alt) { struct f_ncm *ncm = func_to_ncm(f); + struct f_ncm_opts *opts = func_to_ncm_opts(f); struct usb_composite_dev *cdev = f->config->cdev; /* Control interface has only altsetting 0 */ @@ -881,12 +887,13 @@ static int ncm_set_alt(struct usb_function *f, unsigned intf, unsigned alt) if (alt > 1) goto fail; - if (ncm->netdev) { - DBG(cdev, "reset ncm\n"); - ncm->netdev = NULL; - gether_disconnect(&ncm->port); - ncm_reset_values(ncm); - } + scoped_guard(mutex, &opts->lock) + if (opts->net) { + DBG(cdev, "reset ncm\n"); + opts->net = NULL; + gether_disconnect(&ncm->port); + ncm_reset_values(ncm); + } /* * CDC Network only sends data in non-default altsettings. @@ -919,7 +926,8 @@ static int ncm_set_alt(struct usb_function *f, unsigned intf, unsigned alt) net = gether_connect(&ncm->port); if (IS_ERR(net)) return PTR_ERR(net); - ncm->netdev = net; + scoped_guard(mutex, &opts->lock) + opts->net = net; } spin_lock(&ncm->lock); @@ -1366,14 +1374,16 @@ static int ncm_unwrap_ntb(struct gether *port, static void ncm_disable(struct usb_function *f) { struct f_ncm *ncm = func_to_ncm(f); + struct f_ncm_opts *opts = func_to_ncm_opts(f); struct usb_composite_dev *cdev = f->config->cdev; DBG(cdev, "ncm deactivated\n"); - if (ncm->netdev) { - ncm->netdev = NULL; - gether_disconnect(&ncm->port); - } + scoped_guard(mutex, &opts->lock) + if (opts->net) { + opts->net = NULL; + gether_disconnect(&ncm->port); + } if (ncm->notify->enabled) { usb_ep_disable(ncm->notify); @@ -1433,39 +1443,44 @@ static int ncm_bind(struct usb_configuration *c, struct usb_function *f) { struct usb_composite_dev *cdev = c->cdev; struct f_ncm *ncm = func_to_ncm(f); + struct f_ncm_opts *ncm_opts = func_to_ncm_opts(f); struct usb_string *us; int status = 0; struct usb_ep *ep; - struct f_ncm_opts *ncm_opts; struct usb_os_desc_table *os_desc_table __free(kfree) = NULL; + struct net_device *netdev __free(free_gether_netdev) = NULL; struct usb_request *request __free(free_usb_request) = NULL; if (!can_support_ecm(cdev->gadget)) return -EINVAL; - ncm_opts = container_of(f->fi, struct f_ncm_opts, func_inst); - if (cdev->use_os_string) { os_desc_table = kzalloc(sizeof(*os_desc_table), GFP_KERNEL); if (!os_desc_table) return -ENOMEM; } - mutex_lock(&ncm_opts->lock); - gether_set_gadget(ncm_opts->net, cdev->gadget); - if (!ncm_opts->bound) { - ncm_opts->net->mtu = (ncm_opts->max_segment_size - ETH_HLEN); - status = gether_register_netdev(ncm_opts->net); + netdev = gether_setup_default(); + if (IS_ERR(netdev)) + return -ENOMEM; + + scoped_guard(mutex, &ncm_opts->lock) { + gether_apply_opts(netdev, &ncm_opts->net_opts); + netdev->mtu = ncm_opts->max_segment_size - ETH_HLEN; } - mutex_unlock(&ncm_opts->lock); + gether_set_gadget(netdev, cdev->gadget); + status = gether_register_netdev(netdev); if (status) return status; - ncm_opts->bound = true; - - ncm_string_defs[1].s = ncm->ethaddr; + /* export host's Ethernet address in CDC format */ + status = gether_get_host_addr_cdc(netdev, ncm->ethaddr, + sizeof(ncm->ethaddr)); + if (status < 12) + return -EINVAL; + ncm_string_defs[STRING_MAC_IDX].s = ncm->ethaddr; us = usb_gstrings_attach(cdev, ncm_strings, ARRAY_SIZE(ncm_string_defs)); @@ -1563,6 +1578,8 @@ static int ncm_bind(struct usb_configuration *c, struct usb_function *f) f->os_desc_n = 1; } ncm->notify_req = no_free_ptr(request); + ncm->netdev = no_free_ptr(netdev); + ncm->port.ioport = netdev_priv(ncm->netdev); DBG(cdev, "CDC Network: IN/%s OUT/%s NOTIFY/%s\n", ncm->port.in_ep->name, ncm->port.out_ep->name, @@ -1577,19 +1594,19 @@ static inline struct f_ncm_opts *to_f_ncm_opts(struct config_item *item) } /* f_ncm_item_ops */ -USB_ETHERNET_CONFIGFS_ITEM(ncm); +USB_ETHER_OPTS_ITEM(ncm); /* f_ncm_opts_dev_addr */ -USB_ETHERNET_CONFIGFS_ITEM_ATTR_DEV_ADDR(ncm); +USB_ETHER_OPTS_ATTR_DEV_ADDR(ncm); /* f_ncm_opts_host_addr */ -USB_ETHERNET_CONFIGFS_ITEM_ATTR_HOST_ADDR(ncm); +USB_ETHER_OPTS_ATTR_HOST_ADDR(ncm); /* f_ncm_opts_qmult */ -USB_ETHERNET_CONFIGFS_ITEM_ATTR_QMULT(ncm); +USB_ETHER_OPTS_ATTR_QMULT(ncm); /* f_ncm_opts_ifname */ -USB_ETHERNET_CONFIGFS_ITEM_ATTR_IFNAME(ncm); +USB_ETHER_OPTS_ATTR_IFNAME(ncm); static ssize_t ncm_opts_max_segment_size_show(struct config_item *item, char *page) @@ -1655,34 +1672,27 @@ static void ncm_free_inst(struct usb_function_instance *f) struct f_ncm_opts *opts; opts = container_of(f, struct f_ncm_opts, func_inst); - if (opts->bound) - gether_cleanup(netdev_priv(opts->net)); - else - free_netdev(opts->net); kfree(opts->ncm_interf_group); kfree(opts); } static struct usb_function_instance *ncm_alloc_inst(void) { - struct f_ncm_opts *opts; + struct usb_function_instance *ret; struct usb_os_desc *descs[1]; char *names[1]; struct config_group *ncm_interf_group; - opts = kzalloc(sizeof(*opts), GFP_KERNEL); + struct f_ncm_opts *opts __free(kfree) = kzalloc(sizeof(*opts), GFP_KERNEL); if (!opts) return ERR_PTR(-ENOMEM); + + opts->net = NULL; opts->ncm_os_desc.ext_compat_id = opts->ncm_ext_compat_id; + gether_setup_opts_default(&opts->net_opts, "usb"); mutex_init(&opts->lock); opts->func_inst.free_func_inst = ncm_free_inst; - opts->net = gether_setup_default(); - if (IS_ERR(opts->net)) { - struct net_device *net = opts->net; - kfree(opts); - return ERR_CAST(net); - } opts->max_segment_size = ETH_FRAME_LEN; INIT_LIST_HEAD(&opts->ncm_os_desc.ext_prop); @@ -1693,26 +1703,22 @@ static struct usb_function_instance *ncm_alloc_inst(void) ncm_interf_group = usb_os_desc_prepare_interf_dir(&opts->func_inst.group, 1, descs, names, THIS_MODULE); - if (IS_ERR(ncm_interf_group)) { - ncm_free_inst(&opts->func_inst); + if (IS_ERR(ncm_interf_group)) return ERR_CAST(ncm_interf_group); - } opts->ncm_interf_group = ncm_interf_group; - return &opts->func_inst; + ret = &opts->func_inst; + retain_and_null_ptr(opts); + return ret; } static void ncm_free(struct usb_function *f) { - struct f_ncm *ncm; - struct f_ncm_opts *opts; + struct f_ncm_opts *opts = func_to_ncm_opts(f); - ncm = func_to_ncm(f); - opts = container_of(f->fi, struct f_ncm_opts, func_inst); - kfree(ncm); - mutex_lock(&opts->lock); - opts->refcnt--; - mutex_unlock(&opts->lock); + scoped_guard(mutex, &opts->lock) + opts->refcnt--; + kfree(func_to_ncm(f)); } static void ncm_unbind(struct usb_configuration *c, struct usb_function *f) @@ -1736,13 +1742,15 @@ static void ncm_unbind(struct usb_configuration *c, struct usb_function *f) kfree(ncm->notify_req->buf); usb_ep_free_request(ncm->notify, ncm->notify_req); + + ncm->port.ioport = NULL; + gether_cleanup(netdev_priv(ncm->netdev)); } static struct usb_function *ncm_alloc(struct usb_function_instance *fi) { struct f_ncm *ncm; struct f_ncm_opts *opts; - int status; /* allocate and initialize one new instance */ ncm = kzalloc(sizeof(*ncm), GFP_KERNEL); @@ -1750,22 +1758,12 @@ static struct usb_function *ncm_alloc(struct usb_function_instance *fi) return ERR_PTR(-ENOMEM); opts = container_of(fi, struct f_ncm_opts, func_inst); - mutex_lock(&opts->lock); - opts->refcnt++; - /* export host's Ethernet address in CDC format */ - status = gether_get_host_addr_cdc(opts->net, ncm->ethaddr, - sizeof(ncm->ethaddr)); - if (status < 12) { /* strlen("01234567890a") */ - kfree(ncm); - mutex_unlock(&opts->lock); - return ERR_PTR(-EINVAL); - } + scoped_guard(mutex, &opts->lock) + opts->refcnt++; spin_lock_init(&ncm->lock); ncm_reset_values(ncm); - ncm->port.ioport = netdev_priv(opts->net); - mutex_unlock(&opts->lock); ncm->port.is_fixed = true; ncm->port.supports_multi_frame = true; diff --git a/drivers/usb/gadget/function/u_ether.c b/drivers/usb/gadget/function/u_ether.c index f58590bf5e02f..6c32665538cc0 100644 --- a/drivers/usb/gadget/function/u_ether.c +++ b/drivers/usb/gadget/function/u_ether.c @@ -1039,6 +1039,36 @@ int gether_set_ifname(struct net_device *net, const char *name, int len) } EXPORT_SYMBOL_GPL(gether_set_ifname); +void gether_setup_opts_default(struct gether_opts *opts, const char *name) +{ + opts->qmult = QMULT_DEFAULT; + snprintf(opts->name, sizeof(opts->name), "%s%%d", name); + eth_random_addr(opts->dev_mac); + opts->addr_assign_type = NET_ADDR_RANDOM; + eth_random_addr(opts->host_mac); +} +EXPORT_SYMBOL_GPL(gether_setup_opts_default); + +void gether_apply_opts(struct net_device *net, struct gether_opts *opts) +{ + struct eth_dev *dev = netdev_priv(net); + + dev->qmult = opts->qmult; + + if (opts->ifname_set) { + strscpy(net->name, opts->name, sizeof(net->name)); + dev->ifname_set = true; + } + + memcpy(dev->host_mac, opts->host_mac, sizeof(dev->host_mac)); + + if (opts->addr_assign_type == NET_ADDR_SET) { + memcpy(dev->dev_mac, opts->dev_mac, sizeof(dev->dev_mac)); + net->addr_assign_type = opts->addr_assign_type; + } +} +EXPORT_SYMBOL_GPL(gether_apply_opts); + void gether_suspend(struct gether *link) { struct eth_dev *dev = link->ioport; @@ -1095,6 +1125,21 @@ void gether_cleanup(struct eth_dev *dev) } EXPORT_SYMBOL_GPL(gether_cleanup); +void gether_unregister_free_netdev(struct net_device *net) +{ + if (!net) + return; + + struct eth_dev *dev = netdev_priv(net); + + if (net->reg_state == NETREG_REGISTERED) { + unregister_netdev(net); + flush_work(&dev->work); + } + free_netdev(net); +} +EXPORT_SYMBOL_GPL(gether_unregister_free_netdev); + /** * gether_connect - notify network layer that USB link is active * @link: the USB link, set up with endpoints, descriptors matching diff --git a/drivers/usb/gadget/function/u_ether.h b/drivers/usb/gadget/function/u_ether.h index 34be220cef77c..a212a8ec5eb1b 100644 --- a/drivers/usb/gadget/function/u_ether.h +++ b/drivers/usb/gadget/function/u_ether.h @@ -38,6 +38,31 @@ struct eth_dev; +/** + * struct gether_opts - Options for Ethernet gadget function instances + * @name: Pattern for the network interface name (e.g., "usb%d"). + * Used to generate the net device name. + * @qmult: Queue length multiplier for high/super speed. + * @host_mac: The MAC address to be used by the host side. + * @dev_mac: The MAC address to be used by the device side. + * @ifname_set: True if the interface name pattern has been set by userspace. + * @addr_assign_type: The method used for assigning the device MAC address + * (e.g., NET_ADDR_RANDOM, NET_ADDR_SET). + * + * This structure caches network-related settings provided through configfs + * before the net_device is fully instantiated. This allows for early + * configuration while deferring net_device allocation until the function + * is bound. + */ +struct gether_opts { + char name[IFNAMSIZ]; + unsigned int qmult; + u8 host_mac[ETH_ALEN]; + u8 dev_mac[ETH_ALEN]; + bool ifname_set; + unsigned char addr_assign_type; +}; + /* * This represents the USB side of an "ethernet" link, managed by a USB * function which provides control and (maybe) framing. Two functions @@ -258,6 +283,11 @@ int gether_get_ifname(struct net_device *net, char *name, int len); int gether_set_ifname(struct net_device *net, const char *name, int len); void gether_cleanup(struct eth_dev *dev); +void gether_unregister_free_netdev(struct net_device *net); +DEFINE_FREE(free_gether_netdev, struct net_device *, gether_unregister_free_netdev(_T)); + +void gether_setup_opts_default(struct gether_opts *opts, const char *name); +void gether_apply_opts(struct net_device *net, struct gether_opts *opts); void gether_suspend(struct gether *link); void gether_resume(struct gether *link); diff --git a/drivers/usb/gadget/function/u_ether_configfs.h b/drivers/usb/gadget/function/u_ether_configfs.h index f558c3139ebe5..a3696797e074a 100644 --- a/drivers/usb/gadget/function/u_ether_configfs.h +++ b/drivers/usb/gadget/function/u_ether_configfs.h @@ -13,6 +13,12 @@ #ifndef __U_ETHER_CONFIGFS_H #define __U_ETHER_CONFIGFS_H +#include +#include +#include +#include +#include + #define USB_ETHERNET_CONFIGFS_ITEM(_f_) \ static void _f_##_attr_release(struct config_item *item) \ { \ @@ -197,4 +203,174 @@ out: \ \ CONFIGFS_ATTR(_f_##_opts_, _n_) +#define USB_ETHER_OPTS_ITEM(_f_) \ + static void _f_##_attr_release(struct config_item *item) \ + { \ + struct f_##_f_##_opts *opts = to_f_##_f_##_opts(item); \ + \ + usb_put_function_instance(&opts->func_inst); \ + } \ + \ + static struct configfs_item_operations _f_##_item_ops = { \ + .release = _f_##_attr_release, \ + } + +#define USB_ETHER_OPTS_ATTR_DEV_ADDR(_f_) \ + static ssize_t _f_##_opts_dev_addr_show(struct config_item *item, \ + char *page) \ + { \ + struct f_##_f_##_opts *opts = to_f_##_f_##_opts(item); \ + \ + guard(mutex)(&opts->lock); \ + return sysfs_emit(page, "%pM\n", opts->net_opts.dev_mac); \ + } \ + \ + static ssize_t _f_##_opts_dev_addr_store(struct config_item *item, \ + const char *page, size_t len) \ + { \ + struct f_##_f_##_opts *opts = to_f_##_f_##_opts(item); \ + u8 new_addr[ETH_ALEN]; \ + const char *p = page; \ + \ + guard(mutex)(&opts->lock); \ + if (opts->refcnt) \ + return -EBUSY; \ + \ + for (int i = 0; i < ETH_ALEN; i++) { \ + unsigned char num; \ + if ((*p == '.') || (*p == ':')) \ + p++; \ + num = hex_to_bin(*p++) << 4; \ + num |= hex_to_bin(*p++); \ + new_addr[i] = num; \ + } \ + if (!is_valid_ether_addr(new_addr)) \ + return -EINVAL; \ + memcpy(opts->net_opts.dev_mac, new_addr, ETH_ALEN); \ + opts->net_opts.addr_assign_type = NET_ADDR_SET; \ + return len; \ + } \ + \ + CONFIGFS_ATTR(_f_##_opts_, dev_addr) + +#define USB_ETHER_OPTS_ATTR_HOST_ADDR(_f_) \ + static ssize_t _f_##_opts_host_addr_show(struct config_item *item, \ + char *page) \ + { \ + struct f_##_f_##_opts *opts = to_f_##_f_##_opts(item); \ + \ + guard(mutex)(&opts->lock); \ + return sysfs_emit(page, "%pM\n", opts->net_opts.host_mac); \ + } \ + \ + static ssize_t _f_##_opts_host_addr_store(struct config_item *item, \ + const char *page, size_t len) \ + { \ + struct f_##_f_##_opts *opts = to_f_##_f_##_opts(item); \ + u8 new_addr[ETH_ALEN]; \ + const char *p = page; \ + \ + guard(mutex)(&opts->lock); \ + if (opts->refcnt) \ + return -EBUSY; \ + \ + for (int i = 0; i < ETH_ALEN; i++) { \ + unsigned char num; \ + if ((*p == '.') || (*p == ':')) \ + p++; \ + num = hex_to_bin(*p++) << 4; \ + num |= hex_to_bin(*p++); \ + new_addr[i] = num; \ + } \ + if (!is_valid_ether_addr(new_addr)) \ + return -EINVAL; \ + memcpy(opts->net_opts.host_mac, new_addr, ETH_ALEN); \ + return len; \ + } \ + \ + CONFIGFS_ATTR(_f_##_opts_, host_addr) + +#define USB_ETHER_OPTS_ATTR_QMULT(_f_) \ + static ssize_t _f_##_opts_qmult_show(struct config_item *item, \ + char *page) \ + { \ + struct f_##_f_##_opts *opts = to_f_##_f_##_opts(item); \ + \ + guard(mutex)(&opts->lock); \ + return sysfs_emit(page, "%u\n", opts->net_opts.qmult); \ + } \ + \ + static ssize_t _f_##_opts_qmult_store(struct config_item *item, \ + const char *page, size_t len) \ + { \ + struct f_##_f_##_opts *opts = to_f_##_f_##_opts(item); \ + u32 val; \ + int ret; \ + \ + guard(mutex)(&opts->lock); \ + if (opts->refcnt) \ + return -EBUSY; \ + \ + ret = kstrtou32(page, 0, &val); \ + if (ret) \ + return ret; \ + \ + opts->net_opts.qmult = val; \ + return len; \ + } \ + \ + CONFIGFS_ATTR(_f_##_opts_, qmult) + +#define USB_ETHER_OPTS_ATTR_IFNAME(_f_) \ + static ssize_t _f_##_opts_ifname_show(struct config_item *item, \ + char *page) \ + { \ + struct f_##_f_##_opts *opts = to_f_##_f_##_opts(item); \ + const char *name; \ + \ + guard(mutex)(&opts->lock); \ + rtnl_lock(); \ + if (opts->net_opts.ifname_set) \ + name = opts->net_opts.name; \ + else if (opts->net) \ + name = netdev_name(opts->net); \ + else \ + name = "(inactive net_device)"; \ + rtnl_unlock(); \ + return sysfs_emit(page, "%s\n", name); \ + } \ + \ + static ssize_t _f_##_opts_ifname_store(struct config_item *item, \ + const char *page, size_t len) \ + { \ + struct f_##_f_##_opts *opts = to_f_##_f_##_opts(item); \ + char tmp[IFNAMSIZ]; \ + const char *p; \ + size_t c_len = len; \ + \ + if (c_len > 0 && page[c_len - 1] == '\n') \ + c_len--; \ + \ + if (c_len >= sizeof(tmp)) \ + return -E2BIG; \ + \ + strscpy(tmp, page, c_len + 1); \ + if (!dev_valid_name(tmp)) \ + return -EINVAL; \ + \ + /* Require exactly one %d */ \ + p = strchr(tmp, '%'); \ + if (!p || p[1] != 'd' || strchr(p + 2, '%')) \ + return -EINVAL; \ + \ + guard(mutex)(&opts->lock); \ + if (opts->refcnt) \ + return -EBUSY; \ + strscpy(opts->net_opts.name, tmp, sizeof(opts->net_opts.name)); \ + opts->net_opts.ifname_set = true; \ + return len; \ + } \ + \ + CONFIGFS_ATTR(_f_##_opts_, ifname) + #endif /* __U_ETHER_CONFIGFS_H */ diff --git a/drivers/usb/gadget/function/u_ncm.h b/drivers/usb/gadget/function/u_ncm.h index 49ec095cdb4b6..d99330fe31e88 100644 --- a/drivers/usb/gadget/function/u_ncm.h +++ b/drivers/usb/gadget/function/u_ncm.h @@ -15,11 +15,13 @@ #include +#include "u_ether.h" + struct f_ncm_opts { struct usb_function_instance func_inst; struct net_device *net; - bool bound; + struct gether_opts net_opts; struct config_group *ncm_interf_group; struct usb_os_desc ncm_os_desc; char ncm_ext_compat_id[16]; diff --git a/drivers/usb/gadget/udc/bdc/bdc_core.c b/drivers/usb/gadget/udc/bdc/bdc_core.c index 5c3d8b64c0e76..f47aac078f6be 100644 --- a/drivers/usb/gadget/udc/bdc/bdc_core.c +++ b/drivers/usb/gadget/udc/bdc/bdc_core.c @@ -35,8 +35,8 @@ static int poll_oip(struct bdc *bdc, u32 usec) u32 status; int ret; - ret = readl_poll_timeout(bdc->regs + BDC_BDCSC, status, - (BDC_CSTS(status) != BDC_OIP), 10, usec); + ret = readl_poll_timeout_atomic(bdc->regs + BDC_BDCSC, status, + (BDC_CSTS(status) != BDC_OIP), 10, usec); if (ret) dev_err(bdc->dev, "operation timedout BDCSC: 0x%08x\n", status); else diff --git a/drivers/usb/gadget/udc/tegra-xudc.c b/drivers/usb/gadget/udc/tegra-xudc.c index 9d2007f448c04..7f7251c10e952 100644 --- a/drivers/usb/gadget/udc/tegra-xudc.c +++ b/drivers/usb/gadget/udc/tegra-xudc.c @@ -3392,17 +3392,18 @@ static void tegra_xudc_device_params_init(struct tegra_xudc *xudc) { u32 val, imod; + val = xudc_readl(xudc, BLCG); if (xudc->soc->has_ipfs) { - val = xudc_readl(xudc, BLCG); val |= BLCG_ALL; val &= ~(BLCG_DFPCI | BLCG_UFPCI | BLCG_FE | BLCG_COREPLL_PWRDN); val |= BLCG_IOPLL_0_PWRDN; val |= BLCG_IOPLL_1_PWRDN; val |= BLCG_IOPLL_2_PWRDN; - - xudc_writel(xudc, val, BLCG); + } else { + val &= ~BLCG_COREPLL_PWRDN; } + xudc_writel(xudc, val, BLCG); if (xudc->soc->port_speed_quirk) tegra_xudc_limit_port_speed(xudc); @@ -3953,6 +3954,7 @@ static void tegra_xudc_remove(struct platform_device *pdev) static int __maybe_unused tegra_xudc_powergate(struct tegra_xudc *xudc) { unsigned long flags; + u32 val; dev_dbg(xudc->dev, "entering ELPG\n"); @@ -3965,6 +3967,10 @@ static int __maybe_unused tegra_xudc_powergate(struct tegra_xudc *xudc) spin_unlock_irqrestore(&xudc->lock, flags); + val = xudc_readl(xudc, BLCG); + val |= BLCG_COREPLL_PWRDN; + xudc_writel(xudc, val, BLCG); + clk_bulk_disable_unprepare(xudc->soc->num_clks, xudc->clks); regulator_bulk_disable(xudc->soc->num_supplies, xudc->supplies); diff --git a/drivers/usb/host/xhci-tegra.c b/drivers/usb/host/xhci-tegra.c index c78bed0aa844e..83b1766ff1521 100644 --- a/drivers/usb/host/xhci-tegra.c +++ b/drivers/usb/host/xhci-tegra.c @@ -1571,7 +1571,6 @@ static int tegra_xusb_setup_wakeup(struct platform_device *pdev, struct tegra_xu data = irq_get_irq_data(tegra->wake_irqs[i]); if (!data) { dev_warn(tegra->dev, "get wake event %d irq data fail\n", i); - irq_dispose_mapping(tegra->wake_irqs[i]); break; } @@ -1584,16 +1583,6 @@ static int tegra_xusb_setup_wakeup(struct platform_device *pdev, struct tegra_xu return 0; } -static void tegra_xusb_dispose_wake(struct tegra_xusb *tegra) -{ - unsigned int i; - - for (i = 0; i < tegra->num_wakes; i++) - irq_dispose_mapping(tegra->wake_irqs[i]); - - tegra->num_wakes = 0; -} - static int tegra_xusb_probe(struct platform_device *pdev) { struct tegra_xusb *tegra; @@ -1649,10 +1638,8 @@ static int tegra_xusb_probe(struct platform_device *pdev) return err; tegra->padctl = tegra_xusb_padctl_get(&pdev->dev); - if (IS_ERR(tegra->padctl)) { - err = PTR_ERR(tegra->padctl); - goto dispose_wake; - } + if (IS_ERR(tegra->padctl)) + return PTR_ERR(tegra->padctl); np = of_parse_phandle(pdev->dev.of_node, "nvidia,xusb-padctl", 0); if (!np) { @@ -1976,8 +1963,6 @@ static int tegra_xusb_probe(struct platform_device *pdev) put_padctl: of_node_put(np); tegra_xusb_padctl_put(tegra->padctl); -dispose_wake: - tegra_xusb_dispose_wake(tegra); return err; } @@ -2010,8 +1995,6 @@ static void tegra_xusb_remove(struct platform_device *pdev) if (tegra->padctl_irq) pm_runtime_disable(&pdev->dev); - tegra_xusb_dispose_wake(tegra); - pm_runtime_put(&pdev->dev); tegra_xusb_disable(tegra); diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c index 9f2cc5fb9f456..d4505a4264460 100644 --- a/drivers/usb/serial/option.c +++ b/drivers/usb/serial/option.c @@ -1401,12 +1401,16 @@ static const struct usb_device_id option_ids[] = { .driver_info = NCTRL(0) | RSVD(1) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a0, 0xff), /* Telit FN20C04 (rmnet) */ .driver_info = RSVD(0) | NCTRL(3) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a1, 0xff), /* Telit FN20C04 (RNDIS) */ + .driver_info = NCTRL(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a2, 0xff), /* Telit FN920C04 (MBIM) */ .driver_info = NCTRL(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a3, 0xff), /* Telit FN920C04 (ECM) */ .driver_info = NCTRL(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a4, 0xff), /* Telit FN20C04 (rmnet) */ .driver_info = RSVD(0) | NCTRL(3) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a6, 0xff), /* Telit FN920C04 (RNDIS) */ + .driver_info = NCTRL(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a7, 0xff), /* Telit FN920C04 (MBIM) */ .driver_info = NCTRL(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10a8, 0xff), /* Telit FN920C04 (ECM) */ @@ -1415,6 +1419,8 @@ static const struct usb_device_id option_ids[] = { .driver_info = RSVD(0) | NCTRL(2) | RSVD(3) | RSVD(4) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10aa, 0xff), /* Telit FN920C04 (MBIM) */ .driver_info = NCTRL(3) | RSVD(4) | RSVD(5) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x10ab, 0xff), /* Telit FN920C04 (RNDIS) */ + .driver_info = NCTRL(3) | RSVD(4) | RSVD(5) }, { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10b0, 0xff, 0xff, 0x30), /* Telit FE990B (rmnet) */ .driver_info = NCTRL(5) }, { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x10b0, 0xff, 0xff, 0x40) }, diff --git a/drivers/usb/typec/tcpm/fusb302.c b/drivers/usb/typec/tcpm/fusb302.c index 870a71f953f6c..19ff8217818e7 100644 --- a/drivers/usb/typec/tcpm/fusb302.c +++ b/drivers/usb/typec/tcpm/fusb302.c @@ -1756,8 +1756,7 @@ static int fusb302_probe(struct i2c_client *client) } ret = request_irq(chip->gpio_int_n_irq, fusb302_irq_intn, - IRQF_ONESHOT | IRQF_TRIGGER_LOW, - "fsc_interrupt_int_n", chip); + IRQF_TRIGGER_LOW, "fsc_interrupt_int_n", chip); if (ret < 0) { dev_err(dev, "cannot request IRQ for GPIO Int_N, ret=%d", ret); goto tcpm_unregister_port; diff --git a/drivers/usb/typec/ucsi/Kconfig b/drivers/usb/typec/ucsi/Kconfig index b812be4d0e674..87dd992a4b9e9 100644 --- a/drivers/usb/typec/ucsi/Kconfig +++ b/drivers/usb/typec/ucsi/Kconfig @@ -73,7 +73,6 @@ config CROS_EC_UCSI tristate "UCSI Driver for ChromeOS EC" depends on MFD_CROS_EC_DEV depends on CROS_USBPD_NOTIFY - depends on !EXTCON_TCSS_CROS_EC default MFD_CROS_EC_DEV help This driver enables UCSI support for a ChromeOS EC. The EC is diff --git a/drivers/usb/typec/ucsi/psy.c b/drivers/usb/typec/ucsi/psy.c index 8ae900c8c1321..525c6fc2217da 100644 --- a/drivers/usb/typec/ucsi/psy.c +++ b/drivers/usb/typec/ucsi/psy.c @@ -88,15 +88,20 @@ static int ucsi_psy_get_voltage_max(struct ucsi_connector *con, union power_supply_propval *val) { u32 pdo; + int max_voltage = 0; switch (UCSI_CONSTAT(con, PWR_OPMODE)) { case UCSI_CONSTAT_PWR_OPMODE_PD: - if (con->num_pdos > 0) { - pdo = con->src_pdos[con->num_pdos - 1]; - val->intval = pdo_fixed_voltage(pdo) * 1000; - } else { - val->intval = 0; + for (int i = 0; i < con->num_pdos; i++) { + int pdo_voltage = 0; + + pdo = con->src_pdos[i]; + if (pdo_type(pdo) == PDO_TYPE_FIXED) + pdo_voltage = pdo_fixed_voltage(pdo) * 1000; + max_voltage = (pdo_voltage > max_voltage) ? pdo_voltage + : max_voltage; } + val->intval = max_voltage; break; case UCSI_CONSTAT_PWR_OPMODE_TYPEC3_0: case UCSI_CONSTAT_PWR_OPMODE_TYPEC1_5: @@ -144,6 +149,7 @@ static int ucsi_psy_get_current_max(struct ucsi_connector *con, union power_supply_propval *val) { u32 pdo; + int max_current = 0; if (!UCSI_CONSTAT(con, CONNECTED)) { val->intval = 0; @@ -152,12 +158,16 @@ static int ucsi_psy_get_current_max(struct ucsi_connector *con, switch (UCSI_CONSTAT(con, PWR_OPMODE)) { case UCSI_CONSTAT_PWR_OPMODE_PD: - if (con->num_pdos > 0) { - pdo = con->src_pdos[con->num_pdos - 1]; - val->intval = pdo_max_current(pdo) * 1000; - } else { - val->intval = 0; + for (int i = 0; i < con->num_pdos; i++) { + int pdo_current = 0; + + pdo = con->src_pdos[i]; + if (pdo_type(pdo) == PDO_TYPE_FIXED) + pdo_current = pdo_max_current(pdo) * 1000; + max_current = (pdo_current > max_current) ? pdo_current + : max_current; } + val->intval = max_current; break; case UCSI_CONSTAT_PWR_OPMODE_TYPEC1_5: val->intval = UCSI_TYPEC_1_5_CURRENT * 1000; diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c index ddaa1366704bb..44062e9d68f00 100644 --- a/drivers/vdpa/mlx5/net/mlx5_vnet.c +++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c @@ -3640,9 +3640,6 @@ static int mlx5_set_group_asid(struct vdpa_device *vdev, u32 group, struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); int err = 0; - if (group >= MLX5_VDPA_NUMVQ_GROUPS) - return -EINVAL; - mvdev->mres.group2asid[group] = asid; mutex_lock(&mvdev->mres.lock); diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c index c1c6431950e1b..df9c7ddc5d782 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c @@ -606,12 +606,6 @@ static int vdpasim_set_group_asid(struct vdpa_device *vdpa, unsigned int group, struct vhost_iotlb *iommu; int i; - if (group > vdpasim->dev_attr.ngroups) - return -EINVAL; - - if (asid >= vdpasim->dev_attr.nas) - return -EINVAL; - iommu = &vdpasim->iommu[asid]; mutex_lock(&vdpasim->mutex); diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c index d07093d7cc3f5..f141f21566c52 100644 --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c @@ -407,7 +407,7 @@ static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev, ret = qm_get_vft(vf_qm, &vf_qm->qp_base); if (ret <= 0) { dev_err(dev, "failed to get vft qp nums\n"); - return ret; + return ret < 0 ? ret : -EINVAL; } if (ret != vf_data->qp_num) { @@ -1169,12 +1169,34 @@ hisi_acc_vfio_pci_get_device_state(struct vfio_device *vdev, return 0; } +static void hisi_acc_vf_pci_reset_prepare(struct pci_dev *pdev) +{ + struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev); + struct hisi_qm *qm = hisi_acc_vdev->pf_qm; + struct device *dev = &qm->pdev->dev; + u32 delay = 0; + + /* All reset requests need to be queued for processing */ + while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { + msleep(1); + if (++delay > QM_RESET_WAIT_TIMEOUT) { + dev_err(dev, "reset prepare failed\n"); + return; + } + } + + hisi_acc_vdev->set_reset_flag = true; +} + static void hisi_acc_vf_pci_aer_reset_done(struct pci_dev *pdev) { struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev); + struct hisi_qm *qm = hisi_acc_vdev->pf_qm; + + if (hisi_acc_vdev->set_reset_flag) + clear_bit(QM_RESETTING, &qm->misc_ctl); - if (hisi_acc_vdev->core_device.vdev.migration_flags != - VFIO_MIGRATION_STOP_COPY) + if (!hisi_acc_vdev->core_device.vdev.mig_ops) return; mutex_lock(&hisi_acc_vdev->state_mutex); @@ -1506,6 +1528,7 @@ static int hisi_acc_vfio_pci_open_device(struct vfio_device *core_vdev) } hisi_acc_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING; hisi_acc_vdev->dev_opened = true; + hisi_acc_vdev->match_done = 0; mutex_unlock(&hisi_acc_vdev->open_mutex); } @@ -1690,6 +1713,7 @@ static const struct pci_device_id hisi_acc_vfio_pci_table[] = { MODULE_DEVICE_TABLE(pci, hisi_acc_vfio_pci_table); static const struct pci_error_handlers hisi_acc_vf_err_handlers = { + .reset_prepare = hisi_acc_vf_pci_reset_prepare, .reset_done = hisi_acc_vf_pci_aer_reset_done, .error_detected = vfio_pci_core_aer_err_detected, }; diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h index 91002ceeebc18..6253fa074003e 100644 --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h @@ -27,6 +27,7 @@ #define ERROR_CHECK_TIMEOUT 100 #define CHECK_DELAY_TIME 100 +#define QM_RESET_WAIT_TIMEOUT 60000 #define QM_SQC_VFT_BASE_SHIFT_V2 28 #define QM_SQC_VFT_BASE_MASK_V2 GENMASK(15, 0) @@ -110,6 +111,7 @@ struct hisi_acc_vf_migration_file { struct hisi_acc_vf_core_device { struct vfio_pci_core_device core_device; u8 match_done; + bool set_reset_flag; /* * io_base is only valid when dev_opened is true, * which is protected by open_mutex. diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 5efe7535f41ed..085373d71e9c2 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -589,6 +589,7 @@ EXPORT_SYMBOL_GPL(vfio_pci_core_enable); void vfio_pci_core_disable(struct vfio_pci_core_device *vdev) { + struct pci_dev *bridge; struct pci_dev *pdev = vdev->pdev; struct vfio_pci_dummy_resource *dummy_res, *tmp; struct vfio_pci_ioeventfd *ioeventfd, *ioeventfd_tmp; @@ -695,12 +696,20 @@ void vfio_pci_core_disable(struct vfio_pci_core_device *vdev) * We can not use the "try" reset interface here, which will * overwrite the previously restored configuration information. */ - if (vdev->reset_works && pci_dev_trylock(pdev)) { - if (!__pci_reset_function_locked(pdev)) - vdev->needs_reset = false; - pci_dev_unlock(pdev); + if (vdev->reset_works) { + bridge = pci_upstream_bridge(pdev); + if (bridge && !pci_dev_trylock(bridge)) + goto out_restore_state; + if (pci_dev_trylock(pdev)) { + if (!__pci_reset_function_locked(pdev)) + vdev->needs_reset = false; + pci_dev_unlock(pdev); + } + if (bridge) + pci_dev_unlock(bridge); } +out_restore_state: pci_restore_state(pdev); out: pci_disable_device(pdev); diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index 05a481e4c385a..7e51eec842b8c 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -680,7 +680,7 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, case VHOST_VDPA_SET_GROUP_ASID: if (copy_from_user(&s, argp, sizeof(s))) return -EFAULT; - if (s.num >= vdpa->nas) + if (idx >= vdpa->ngroups || s.num >= vdpa->nas) return -EINVAL; if (!ops->set_group_asid) return -EOPNOTSUPP; @@ -1527,6 +1527,7 @@ static int vhost_vdpa_mmap(struct file *file, struct vm_area_struct *vma) if (vma->vm_end - vma->vm_start != notify.size) return -ENOTSUPP; + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); vma->vm_ops = &vhost_vdpa_vm_ops; return 0; diff --git a/drivers/video/backlight/qcom-wled.c b/drivers/video/backlight/qcom-wled.c index a63bb42c8f8b0..8054e4787725e 100644 --- a/drivers/video/backlight/qcom-wled.c +++ b/drivers/video/backlight/qcom-wled.c @@ -1244,6 +1244,15 @@ static const struct wled_var_cfg wled4_ovp_cfg = { .size = ARRAY_SIZE(wled4_ovp_values), }; +static const u32 pmi8994_wled_ovp_values[] = { + 31000, 29500, 19400, 17800, +}; + +static const struct wled_var_cfg pmi8994_wled_ovp_cfg = { + .values = pmi8994_wled_ovp_values, + .size = ARRAY_SIZE(pmi8994_wled_ovp_values), +}; + static inline u32 wled5_ovp_values_fn(u32 idx) { /* @@ -1357,6 +1366,29 @@ static int wled_configure(struct wled *wled) }, }; + const struct wled_u32_opts pmi8994_wled_opts[] = { + { + .name = "qcom,current-boost-limit", + .val_ptr = &cfg->boost_i_limit, + .cfg = &wled4_boost_i_limit_cfg, + }, + { + .name = "qcom,current-limit-microamp", + .val_ptr = &cfg->string_i_limit, + .cfg = &wled4_string_i_limit_cfg, + }, + { + .name = "qcom,ovp-millivolt", + .val_ptr = &cfg->ovp, + .cfg = &pmi8994_wled_ovp_cfg, + }, + { + .name = "qcom,switching-freq", + .val_ptr = &cfg->switch_freq, + .cfg = &wled3_switch_freq_cfg, + }, + }; + const struct wled_u32_opts wled5_opts[] = { { .name = "qcom,current-boost-limit", @@ -1423,8 +1455,14 @@ static int wled_configure(struct wled *wled) break; case 4: - u32_opts = wled4_opts; - size = ARRAY_SIZE(wled4_opts); + if (of_device_is_compatible(dev->of_node, "qcom,pmi8950-wled") || + of_device_is_compatible(dev->of_node, "qcom,pmi8994-wled")) { + u32_opts = pmi8994_wled_opts; + size = ARRAY_SIZE(pmi8994_wled_opts); + } else { + u32_opts = wled4_opts; + size = ARRAY_SIZE(wled4_opts); + } *cfg = wled4_config_defaults; wled->wled_set_brightness = wled4_set_brightness; wled->wled_sync_toggle = wled3_sync_toggle; diff --git a/drivers/video/fbdev/au1200fb.c b/drivers/video/fbdev/au1200fb.c index ed770222660b5..685e629e7e164 100644 --- a/drivers/video/fbdev/au1200fb.c +++ b/drivers/video/fbdev/au1200fb.c @@ -1724,8 +1724,10 @@ static int au1200fb_drv_probe(struct platform_device *dev) /* Now hook interrupt too */ irq = platform_get_irq(dev, 0); - if (irq < 0) - return irq; + if (irq < 0) { + ret = irq; + goto failed; + } ret = request_irq(irq, au1200fb_handle_irq, IRQF_SHARED, "lcd", (void *)dev); diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index e7e07eb2142eb..7453377f34336 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -1047,7 +1047,8 @@ static void fbcon_init(struct vc_data *vc, bool init) return; if (!info->fbcon_par) - con2fb_acquire_newinfo(vc, info, vc->vc_num); + if (con2fb_acquire_newinfo(vc, info, vc->vc_num)) + return; /* If we are not the first console on this fb, copy the font from that console */ diff --git a/drivers/video/fbdev/core/fbcon.h b/drivers/video/fbdev/core/fbcon.h index 4d97e6d8a16a2..7e21c8b336692 100644 --- a/drivers/video/fbdev/core/fbcon.h +++ b/drivers/video/fbdev/core/fbcon.h @@ -30,7 +30,6 @@ struct fbcon_display { #ifdef CONFIG_FRAMEBUFFER_CONSOLE_LEGACY_ACCELERATION u_short scrollmode; /* Scroll Method, use fb_scrollmode() */ #endif - u_short inverse; /* != 0 text black on white as default */ short yscroll; /* Hardware scrolling */ int vrows; /* number of virtual rows */ int cursor_shape; diff --git a/drivers/video/fbdev/core/fbsysfs.c b/drivers/video/fbdev/core/fbsysfs.c index b8344c40073b4..baa2bae0fb5b3 100644 --- a/drivers/video/fbdev/core/fbsysfs.c +++ b/drivers/video/fbdev/core/fbsysfs.c @@ -12,8 +12,6 @@ #include "fb_internal.h" -#define FB_SYSFS_FLAG_ATTR 1 - static int activate(struct fb_info *fb_info, struct fb_var_screeninfo *var) { int err; @@ -451,33 +449,7 @@ static struct attribute *fb_device_attrs[] = { NULL, }; -static const struct attribute_group fb_device_attr_group = { - .attrs = fb_device_attrs, -}; - -static int fb_init_device(struct fb_info *fb_info) -{ - int ret; - - dev_set_drvdata(fb_info->dev, fb_info); - - fb_info->class_flag |= FB_SYSFS_FLAG_ATTR; - - ret = device_add_group(fb_info->dev, &fb_device_attr_group); - if (ret) - fb_info->class_flag &= ~FB_SYSFS_FLAG_ATTR; - - return 0; -} - -static void fb_cleanup_device(struct fb_info *fb_info) -{ - if (fb_info->class_flag & FB_SYSFS_FLAG_ATTR) { - device_remove_group(fb_info->dev, &fb_device_attr_group); - - fb_info->class_flag &= ~FB_SYSFS_FLAG_ATTR; - } -} +ATTRIBUTE_GROUPS(fb_device); int fb_device_create(struct fb_info *fb_info) { @@ -485,14 +457,13 @@ int fb_device_create(struct fb_info *fb_info) dev_t devt = MKDEV(FB_MAJOR, node); int ret; - fb_info->dev = device_create(fb_class, fb_info->device, devt, NULL, "fb%d", node); + fb_info->dev = device_create_with_groups(fb_class, fb_info->device, devt, fb_info, + fb_device_groups, "fb%d", node); if (IS_ERR(fb_info->dev)) { /* Not fatal */ ret = PTR_ERR(fb_info->dev); pr_warn("Unable to create device for framebuffer %d; error %d\n", node, ret); fb_info->dev = NULL; - } else { - fb_init_device(fb_info); } return 0; @@ -505,7 +476,6 @@ void fb_device_destroy(struct fb_info *fb_info) if (!fb_info->dev) return; - fb_cleanup_device(fb_info); device_destroy(fb_class, devt); fb_info->dev = NULL; } diff --git a/drivers/video/fbdev/ffb.c b/drivers/video/fbdev/ffb.c index 34b6abff9493e..da531b4cb4513 100644 --- a/drivers/video/fbdev/ffb.c +++ b/drivers/video/fbdev/ffb.c @@ -335,6 +335,9 @@ struct ffb_dac { }; #define FFB_DAC_UCTRL 0x1001 /* User Control */ +#define FFB_DAC_UCTRL_OVENAB 0x00000008 /* Overlay Enable */ +#define FFB_DAC_UCTRL_WMODE 0x00000030 /* Window Mode */ +#define FFB_DAC_UCTRL_WM_COMB 0x00000000 /* Window Mode = Combined */ #define FFB_DAC_UCTRL_MANREV 0x00000f00 /* 4-bit Manufacturing Revision */ #define FFB_DAC_UCTRL_MANREV_SHIFT 8 #define FFB_DAC_TGEN 0x6000 /* Timing Generator */ @@ -425,7 +428,7 @@ static void ffb_switch_from_graph(struct ffb_par *par) { struct ffb_fbc __iomem *fbc = par->fbc; struct ffb_dac __iomem *dac = par->dac; - unsigned long flags; + unsigned long flags, uctrl; spin_lock_irqsave(&par->lock, flags); FFBWait(par); @@ -450,6 +453,15 @@ static void ffb_switch_from_graph(struct ffb_par *par) upa_writel((FFB_DAC_CUR_CTRL_P0 | FFB_DAC_CUR_CTRL_P1), &dac->value2); + /* Disable overlay and window modes. */ + upa_writel(FFB_DAC_UCTRL, &dac->type); + uctrl = upa_readl(&dac->value); + uctrl &= ~FFB_DAC_UCTRL_WMODE; + uctrl |= FFB_DAC_UCTRL_WM_COMB; + uctrl &= ~FFB_DAC_UCTRL_OVENAB; + upa_writel(FFB_DAC_UCTRL, &dac->type); + upa_writel(uctrl, &dac->value); + spin_unlock_irqrestore(&par->lock, flags); } diff --git a/drivers/video/fbdev/riva/riva_hw.c b/drivers/video/fbdev/riva/riva_hw.c index 8b829b7200642..f292079566cfc 100644 --- a/drivers/video/fbdev/riva/riva_hw.c +++ b/drivers/video/fbdev/riva/riva_hw.c @@ -436,6 +436,9 @@ static char nv3_arb(nv3_fifo_info * res_info, nv3_sim_state * state, nv3_arb_in vmisses = 2; eburst_size = state->memory_width * 1; mburst_size = 32; + if (!state->mclk_khz) + return (0); + gns = 1000000 * (gmisses*state->mem_page_miss + state->mem_latency)/state->mclk_khz; ainfo->by_gfacc = gns*ainfo->gdrain_rate/1000000; ainfo->wcmocc = 0; diff --git a/drivers/video/fbdev/smscufx.c b/drivers/video/fbdev/smscufx.c index 5f0dd01fd8349..891ce7b76d637 100644 --- a/drivers/video/fbdev/smscufx.c +++ b/drivers/video/fbdev/smscufx.c @@ -932,7 +932,6 @@ static int ufx_ops_ioctl(struct fb_info *info, unsigned int cmd, unsigned long arg) { struct ufx_data *dev = info->par; - struct dloarea *area = NULL; if (!atomic_read(&dev->usb_active)) return 0; @@ -947,6 +946,10 @@ static int ufx_ops_ioctl(struct fb_info *info, unsigned int cmd, /* TODO: Help propose a standard fb.h ioctl to report mmap damage */ if (cmd == UFX_IOCTL_REPORT_DAMAGE) { + struct dloarea *area __free(kfree) = kmalloc(sizeof(*area), GFP_KERNEL); + if (!area) + return -ENOMEM; + /* If we have a damage-aware client, turn fb_defio "off" * To avoid perf imact of unnecessary page fault handling. * Done by resetting the delay for this fb_info to a very @@ -956,7 +959,8 @@ static int ufx_ops_ioctl(struct fb_info *info, unsigned int cmd, if (info->fbdefio) info->fbdefio->delay = UFX_DEFIO_WRITE_DISABLE; - area = (struct dloarea *)arg; + if (copy_from_user(area, (u8 __user *)arg, sizeof(*area))) + return -EFAULT; if (area->x < 0) area->x = 0; diff --git a/drivers/video/fbdev/vt8500lcdfb.c b/drivers/video/fbdev/vt8500lcdfb.c index b08a6fdc53fd2..85c7a99a7d648 100644 --- a/drivers/video/fbdev/vt8500lcdfb.c +++ b/drivers/video/fbdev/vt8500lcdfb.c @@ -369,7 +369,7 @@ static int vt8500lcd_probe(struct platform_device *pdev) if (fbi->palette_cpu == NULL) { dev_err(&pdev->dev, "Failed to allocate palette buffer\n"); ret = -ENOMEM; - goto failed_free_io; + goto failed_free_mem_virt; } irq = platform_get_irq(pdev, 0); @@ -432,6 +432,9 @@ static int vt8500lcd_probe(struct platform_device *pdev) failed_free_palette: dma_free_coherent(&pdev->dev, fbi->palette_size, fbi->palette_cpu, fbi->palette_phys); +failed_free_mem_virt: + dma_free_coherent(&pdev->dev, fbi->fb.fix.smem_len, + fbi->fb.screen_buffer, fbi->fb.fix.smem_start); failed_free_io: iounmap(fbi->regbase); failed_free_res: diff --git a/drivers/video/of_display_timing.c b/drivers/video/of_display_timing.c index bebd371c6b93e..a6ec392253c3e 100644 --- a/drivers/video/of_display_timing.c +++ b/drivers/video/of_display_timing.c @@ -181,7 +181,7 @@ struct display_timings *of_get_display_timings(const struct device_node *np) if (disp->num_timings == 0) { /* should never happen, as entry was already found above */ pr_err("%pOF: no timings specified\n", np); - goto entryfail; + goto timingfail; } disp->timings = kcalloc(disp->num_timings, @@ -189,13 +189,13 @@ struct display_timings *of_get_display_timings(const struct device_node *np) GFP_KERNEL); if (!disp->timings) { pr_err("%pOF: could not allocate timings array\n", np); - goto entryfail; + goto timingfail; } disp->num_timings = 0; disp->native_mode = 0; - for_each_child_of_node(timings_np, entry) { + for_each_child_of_node_scoped(timings_np, child) { struct display_timing *dt; int r; @@ -206,7 +206,7 @@ struct display_timings *of_get_display_timings(const struct device_node *np) goto timingfail; } - r = of_parse_display_timing(entry, dt); + r = of_parse_display_timing(child, dt); if (r) { /* * to not encourage wrong devicetrees, fail in case of @@ -218,7 +218,7 @@ struct display_timings *of_get_display_timings(const struct device_node *np) goto timingfail; } - if (native_mode == entry) + if (native_mode == child) disp->native_mode = disp->num_timings; disp->timings[disp->num_timings] = dt; diff --git a/drivers/watchdog/imx7ulp_wdt.c b/drivers/watchdog/imx7ulp_wdt.c index 0f13a30533574..03479110453ce 100644 --- a/drivers/watchdog/imx7ulp_wdt.c +++ b/drivers/watchdog/imx7ulp_wdt.c @@ -346,6 +346,7 @@ static int imx7ulp_wdt_probe(struct platform_device *pdev) watchdog_stop_on_reboot(wdog); watchdog_stop_on_unregister(wdog); watchdog_set_drvdata(wdog, imx7ulp_wdt); + watchdog_set_nowayout(wdog, nowayout); imx7ulp_wdt->hw = of_device_get_match_data(dev); ret = imx7ulp_wdt_init(imx7ulp_wdt, wdog->timeout * imx7ulp_wdt->hw->wdog_clock_rate); diff --git a/drivers/watchdog/it87_wdt.c b/drivers/watchdog/it87_wdt.c index 3b8488c86a2f3..1d9f8591f38d8 100644 --- a/drivers/watchdog/it87_wdt.c +++ b/drivers/watchdog/it87_wdt.c @@ -188,6 +188,12 @@ static void _wdt_update_timeout(unsigned int t) superio_outb(t >> 8, WDTVALMSB); } +/* Internal function, should be called after superio_select(GPIO) */ +static bool _wdt_running(void) +{ + return superio_inb(WDTVALLSB) || (max_units > 255 && superio_inb(WDTVALMSB)); +} + static int wdt_update_timeout(unsigned int t) { int ret; @@ -374,6 +380,12 @@ static int __init it87_wdt_init(void) } } + /* wdt already left running by firmware? */ + if (_wdt_running()) { + pr_info("Left running by firmware.\n"); + set_bit(WDOG_HW_RUNNING, &wdt_dev.status); + } + superio_exit(); if (timeout < 1 || timeout > max_units * 60) { diff --git a/drivers/watchdog/rzv2h_wdt.c b/drivers/watchdog/rzv2h_wdt.c index a694786837e11..f9bb4ef3d327b 100644 --- a/drivers/watchdog/rzv2h_wdt.c +++ b/drivers/watchdog/rzv2h_wdt.c @@ -270,9 +270,7 @@ static int rzt2h_wdt_wdtdcr_init(struct platform_device *pdev, rzt2h_wdt_wdtdcr_count_stop(priv); - ret = pm_runtime_put(&pdev->dev); - if (ret < 0) - return ret; + pm_runtime_put(&pdev->dev); return 0; } diff --git a/drivers/watchdog/starfive-wdt.c b/drivers/watchdog/starfive-wdt.c index ed71d3960a0f2..af55adc4a3c69 100644 --- a/drivers/watchdog/starfive-wdt.c +++ b/drivers/watchdog/starfive-wdt.c @@ -446,7 +446,7 @@ static int starfive_wdt_probe(struct platform_device *pdev) platform_set_drvdata(pdev, wdt); pm_runtime_enable(&pdev->dev); if (pm_runtime_enabled(&pdev->dev)) { - ret = pm_runtime_get_sync(&pdev->dev); + ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) return ret; } else { diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c index 49c3f99263943..8c44a25a7d2b9 100644 --- a/drivers/xen/balloon.c +++ b/drivers/xen/balloon.c @@ -724,6 +724,7 @@ static int __init balloon_add_regions(void) static int __init balloon_init(void) { struct task_struct *task; + unsigned long current_pages; int rc; if (!xen_domain()) @@ -731,12 +732,18 @@ static int __init balloon_init(void) pr_info("Initialising balloon driver\n"); - if (xen_released_pages >= get_num_physpages()) { - WARN(1, "Released pages underflow current target"); - return -ERANGE; + if (xen_pv_domain()) { + if (xen_released_pages >= xen_start_info->nr_pages) + goto underflow; + current_pages = min(xen_start_info->nr_pages - + xen_released_pages, max_pfn); + } else { + if (xen_unpopulated_pages >= get_num_physpages()) + goto underflow; + current_pages = get_num_physpages() - xen_unpopulated_pages; } - balloon_stats.current_pages = get_num_physpages() - xen_released_pages; + balloon_stats.current_pages = current_pages; balloon_stats.target_pages = balloon_stats.current_pages; balloon_stats.balloon_low = 0; balloon_stats.balloon_high = 0; @@ -767,6 +774,10 @@ static int __init balloon_init(void) xen_balloon_init(); return 0; + + underflow: + WARN(1, "Released pages underflow current target"); + return -ERANGE; } subsys_initcall(balloon_init); diff --git a/drivers/xen/grant-dma-ops.c b/drivers/xen/grant-dma-ops.c index 29257d2639dbf..43a918c498c6c 100644 --- a/drivers/xen/grant-dma-ops.c +++ b/drivers/xen/grant-dma-ops.c @@ -362,7 +362,8 @@ static int xen_grant_init_backend_domid(struct device *dev, if (np) { ret = xen_dt_grant_init_backend_domid(dev, np, backend_domid); of_node_put(np); - } else if (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT) || xen_pv_domain()) { + } else if (!xen_initial_domain() && + (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT) || xen_pv_domain())) { dev_info(dev, "Using dom0 as backend\n"); *backend_domid = 0; ret = 0; diff --git a/drivers/xen/unpopulated-alloc.c b/drivers/xen/unpopulated-alloc.c index d6fc2aefe2646..1dc0b495c8e59 100644 --- a/drivers/xen/unpopulated-alloc.c +++ b/drivers/xen/unpopulated-alloc.c @@ -18,6 +18,9 @@ static unsigned int list_count; static struct resource *target_resource; +/* Pages to subtract from the memory count when setting balloon target. */ +unsigned long xen_unpopulated_pages __initdata; + /* * If arch is not happy with system "iomem_resource" being used for * the region allocation it can provide it's own view by creating specific diff --git a/drivers/xen/xen-acpi-processor.c b/drivers/xen/xen-acpi-processor.c index 2967039398463..520756159d3d3 100644 --- a/drivers/xen/xen-acpi-processor.c +++ b/drivers/xen/xen-acpi-processor.c @@ -379,11 +379,8 @@ read_acpi_id(acpi_handle handle, u32 lvl, void *context, void **rv) acpi_psd[acpi_id].domain); } - status = acpi_evaluate_object(handle, "_CST", NULL, &buffer); - if (ACPI_FAILURE(status)) { - if (!pblk) - return AE_OK; - } + if (!pblk && !acpi_has_method(handle, "_CST")) + return AE_OK; /* .. and it has a C-state */ __set_bit(acpi_id, acpi_id_cst_present); diff --git a/drivers/xen/xenbus/xenbus_probe_frontend.c b/drivers/xen/xenbus/xenbus_probe_frontend.c index 6d1819269cbe5..199917b6f77ca 100644 --- a/drivers/xen/xenbus/xenbus_probe_frontend.c +++ b/drivers/xen/xenbus/xenbus_probe_frontend.c @@ -148,11 +148,9 @@ static void xenbus_frontend_dev_shutdown(struct device *_dev) } static const struct dev_pm_ops xenbus_pm_ops = { - .suspend = xenbus_dev_suspend, - .resume = xenbus_frontend_dev_resume, .freeze = xenbus_dev_suspend, .thaw = xenbus_dev_cancel, - .restore = xenbus_dev_resume, + .restore = xenbus_frontend_dev_resume, }; static struct xen_bus_type xenbus_frontend = { diff --git a/fs/btrfs/accessors.h b/fs/btrfs/accessors.h index 99b3ced12805b..78721412951c5 100644 --- a/fs/btrfs/accessors.h +++ b/fs/btrfs/accessors.h @@ -12,6 +12,7 @@ #include #include #include +#include "fs.h" #include "extent_io.h" struct extent_buffer; diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c index 21df48e6c4fa2..7d8aaa019c8ce 100644 --- a/fs/btrfs/bio.c +++ b/fs/btrfs/bio.c @@ -41,13 +41,17 @@ static bool bbio_has_ordered_extent(const struct btrfs_bio *bbio) * Initialize a btrfs_bio structure. This skips the embedded bio itself as it * is already initialized by the block layer. */ -void btrfs_bio_init(struct btrfs_bio *bbio, struct btrfs_fs_info *fs_info, +void btrfs_bio_init(struct btrfs_bio *bbio, struct btrfs_inode *inode, u64 file_offset, btrfs_bio_end_io_t end_io, void *private) { + /* @inode parameter is mandatory. */ + ASSERT(inode); + memset(bbio, 0, offsetof(struct btrfs_bio, bio)); - bbio->fs_info = fs_info; + bbio->inode = inode; bbio->end_io = end_io; bbio->private = private; + bbio->file_offset = file_offset; atomic_set(&bbio->pending_ios, 1); WRITE_ONCE(bbio->status, BLK_STS_OK); } @@ -60,7 +64,7 @@ void btrfs_bio_init(struct btrfs_bio *bbio, struct btrfs_fs_info *fs_info, * a mempool. */ struct btrfs_bio *btrfs_bio_alloc(unsigned int nr_vecs, blk_opf_t opf, - struct btrfs_fs_info *fs_info, + struct btrfs_inode *inode, u64 file_offset, btrfs_bio_end_io_t end_io, void *private) { struct btrfs_bio *bbio; @@ -68,7 +72,7 @@ struct btrfs_bio *btrfs_bio_alloc(unsigned int nr_vecs, blk_opf_t opf, bio = bio_alloc_bioset(NULL, nr_vecs, opf, GFP_NOFS, &btrfs_bioset); bbio = btrfs_bio(bio); - btrfs_bio_init(bbio, fs_info, end_io, private); + btrfs_bio_init(bbio, inode, file_offset, end_io, private); return bbio; } @@ -85,9 +89,7 @@ static struct btrfs_bio *btrfs_split_bio(struct btrfs_fs_info *fs_info, return ERR_CAST(bio); bbio = btrfs_bio(bio); - btrfs_bio_init(bbio, fs_info, NULL, orig_bbio); - bbio->inode = orig_bbio->inode; - bbio->file_offset = orig_bbio->file_offset; + btrfs_bio_init(bbio, orig_bbio->inode, orig_bbio->file_offset, NULL, orig_bbio); orig_bbio->file_offset += map_length; if (bbio_has_ordered_extent(bbio)) { refcount_inc(&orig_bbio->ordered->refs); @@ -100,6 +102,12 @@ static struct btrfs_bio *btrfs_split_bio(struct btrfs_fs_info *fs_info, void btrfs_bio_end_io(struct btrfs_bio *bbio, blk_status_t status) { + /* Make sure we're already in task context. */ + ASSERT(in_task()); + + if (bbio->async_csum) + wait_for_completion(&bbio->csum_done); + bbio->bio.bi_status = status; if (bbio->bio.bi_pool == &btrfs_clone_bioset) { struct btrfs_bio *orig_bbio = bbio->private; @@ -244,9 +252,8 @@ static struct btrfs_failed_bio *repair_one_sector(struct btrfs_bio *failed_bbio, bio_add_folio_nofail(repair_bio, folio, sectorsize, foff); repair_bbio = btrfs_bio(repair_bio); - btrfs_bio_init(repair_bbio, fs_info, NULL, fbio); - repair_bbio->inode = failed_bbio->inode; - repair_bbio->file_offset = failed_bbio->file_offset + bio_offset; + btrfs_bio_init(repair_bbio, failed_bbio->inode, failed_bbio->file_offset + bio_offset, + NULL, fbio); mirror = next_repair_mirror(fbio, failed_bbio->mirror_num); btrfs_debug(fs_info, "submitting repair read to mirror %d", mirror); @@ -317,36 +324,35 @@ static struct workqueue_struct *btrfs_end_io_wq(const struct btrfs_fs_info *fs_i return fs_info->endio_workers; } -static void btrfs_end_bio_work(struct work_struct *work) +static void simple_end_io_work(struct work_struct *work) { struct btrfs_bio *bbio = container_of(work, struct btrfs_bio, end_io_work); + struct bio *bio = &bbio->bio; - /* Metadata reads are checked and repaired by the submitter. */ - if (is_data_bbio(bbio)) - btrfs_check_read_bio(bbio, bbio->bio.bi_private); - else - btrfs_bio_end_io(bbio, bbio->bio.bi_status); + if (bio_op(bio) == REQ_OP_READ) { + /* Metadata reads are checked and repaired by the submitter. */ + if (is_data_bbio(bbio)) + return btrfs_check_read_bio(bbio, bbio->bio.bi_private); + return btrfs_bio_end_io(bbio, bbio->bio.bi_status); + } + if (bio_is_zone_append(bio) && !bio->bi_status) + btrfs_record_physical_zoned(bbio); + btrfs_bio_end_io(bbio, bbio->bio.bi_status); } static void btrfs_simple_end_io(struct bio *bio) { struct btrfs_bio *bbio = btrfs_bio(bio); struct btrfs_device *dev = bio->bi_private; - struct btrfs_fs_info *fs_info = bbio->fs_info; + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; btrfs_bio_counter_dec(fs_info); if (bio->bi_status) btrfs_log_dev_io_error(bio, dev); - if (bio_op(bio) == REQ_OP_READ) { - INIT_WORK(&bbio->end_io_work, btrfs_end_bio_work); - queue_work(btrfs_end_io_wq(fs_info, bio), &bbio->end_io_work); - } else { - if (bio_is_zone_append(bio) && !bio->bi_status) - btrfs_record_physical_zoned(bbio); - btrfs_bio_end_io(bbio, bbio->bio.bi_status); - } + INIT_WORK(&bbio->end_io_work, simple_end_io_work); + queue_work(btrfs_end_io_wq(fs_info, bio), &bbio->end_io_work); } static void btrfs_raid56_end_io(struct bio *bio) @@ -354,6 +360,9 @@ static void btrfs_raid56_end_io(struct bio *bio) struct btrfs_io_context *bioc = bio->bi_private; struct btrfs_bio *bbio = btrfs_bio(bio); + /* RAID56 endio is always handled in workqueue. */ + ASSERT(in_task()); + btrfs_bio_counter_dec(bioc->fs_info); bbio->mirror_num = bioc->mirror_num; if (bio_op(bio) == REQ_OP_READ && is_data_bbio(bbio)) @@ -364,11 +373,12 @@ static void btrfs_raid56_end_io(struct bio *bio) btrfs_put_bioc(bioc); } -static void btrfs_orig_write_end_io(struct bio *bio) +static void orig_write_end_io_work(struct work_struct *work) { + struct btrfs_bio *bbio = container_of(work, struct btrfs_bio, end_io_work); + struct bio *bio = &bbio->bio; struct btrfs_io_stripe *stripe = bio->bi_private; struct btrfs_io_context *bioc = stripe->bioc; - struct btrfs_bio *bbio = btrfs_bio(bio); btrfs_bio_counter_dec(bioc->fs_info); @@ -393,8 +403,18 @@ static void btrfs_orig_write_end_io(struct bio *bio) btrfs_put_bioc(bioc); } -static void btrfs_clone_write_end_io(struct bio *bio) +static void btrfs_orig_write_end_io(struct bio *bio) +{ + struct btrfs_bio *bbio = btrfs_bio(bio); + + INIT_WORK(&bbio->end_io_work, orig_write_end_io_work); + queue_work(btrfs_end_io_wq(bbio->inode->root->fs_info, bio), &bbio->end_io_work); +} + +static void clone_write_end_io_work(struct work_struct *work) { + struct btrfs_bio *bbio = container_of(work, struct btrfs_bio, end_io_work); + struct bio *bio = &bbio->bio; struct btrfs_io_stripe *stripe = bio->bi_private; if (bio->bi_status) { @@ -409,8 +429,18 @@ static void btrfs_clone_write_end_io(struct bio *bio) bio_put(bio); } +static void btrfs_clone_write_end_io(struct bio *bio) +{ + struct btrfs_bio *bbio = btrfs_bio(bio); + + INIT_WORK(&bbio->end_io_work, clone_write_end_io_work); + queue_work(btrfs_end_io_wq(bbio->inode->root->fs_info, bio), &bbio->end_io_work); +} + static void btrfs_submit_dev_bio(struct btrfs_device *dev, struct bio *bio) { + u64 physical = bio->bi_iter.bi_sector << SECTOR_SHIFT; + if (!dev || !dev->bdev || test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) || (btrfs_op(bio) == BTRFS_MAP_WRITE && @@ -425,12 +455,13 @@ static void btrfs_submit_dev_bio(struct btrfs_device *dev, struct bio *bio) * For zone append writing, bi_sector must point the beginning of the * zone */ - if (bio_op(bio) == REQ_OP_ZONE_APPEND) { - u64 physical = bio->bi_iter.bi_sector << SECTOR_SHIFT; + if (btrfs_bio(bio)->can_use_append && btrfs_dev_is_sequential(dev, physical)) { u64 zone_start = round_down(physical, dev->fs_info->zone_size); ASSERT(btrfs_dev_is_sequential(dev, physical)); bio->bi_iter.bi_sector = zone_start >> SECTOR_SHIFT; + bio->bi_opf &= ~REQ_OP_WRITE; + bio->bi_opf |= REQ_OP_ZONE_APPEND; } btrfs_debug(dev->fs_info, "%s: rw %d 0x%x, sector=%llu, dev=%lu (%s id %llu), size=%u", @@ -455,6 +486,7 @@ static void btrfs_submit_dev_bio(struct btrfs_device *dev, struct bio *bio) static void btrfs_submit_mirrored_bio(struct btrfs_io_context *bioc, int dev_nr) { struct bio *orig_bio = bioc->orig_bio, *bio; + struct btrfs_bio *orig_bbio = btrfs_bio(orig_bio); ASSERT(bio_op(orig_bio) != REQ_OP_READ); @@ -463,8 +495,11 @@ static void btrfs_submit_mirrored_bio(struct btrfs_io_context *bioc, int dev_nr) bio = orig_bio; bio->bi_end_io = btrfs_orig_write_end_io; } else { - bio = bio_alloc_clone(NULL, orig_bio, GFP_NOFS, &fs_bio_set); + /* We need to use endio_work to run end_io in task context. */ + bio = bio_alloc_clone(NULL, orig_bio, GFP_NOFS, &btrfs_bioset); bio_inc_remaining(orig_bio); + btrfs_bio_init(btrfs_bio(bio), orig_bbio->inode, + orig_bbio->file_offset, NULL, NULL); bio->bi_end_io = btrfs_clone_write_end_io; } @@ -509,7 +544,11 @@ static int btrfs_bio_csum(struct btrfs_bio *bbio) { if (bbio->bio.bi_opf & REQ_META) return btree_csum_one_bio(bbio); - return btrfs_csum_one_bio(bbio); +#ifdef CONFIG_BTRFS_EXPERIMENTAL + return btrfs_csum_one_bio(bbio, true); +#else + return btrfs_csum_one_bio(bbio, false); +#endif } /* @@ -581,20 +620,25 @@ static void run_one_async_done(struct btrfs_work *work, bool do_free) static bool should_async_write(struct btrfs_bio *bbio) { + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; bool auto_csum_mode = true; #ifdef CONFIG_BTRFS_EXPERIMENTAL - struct btrfs_fs_devices *fs_devices = bbio->fs_info->fs_devices; + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; enum btrfs_offload_csum_mode csum_mode = READ_ONCE(fs_devices->offload_csum_mode); - if (csum_mode == BTRFS_OFFLOAD_CSUM_FORCE_OFF) - return false; - - auto_csum_mode = (csum_mode == BTRFS_OFFLOAD_CSUM_AUTO); + if (csum_mode == BTRFS_OFFLOAD_CSUM_FORCE_ON) + return true; + /* + * Write bios will calculate checksum and submit bio at the same time. + * Unless explicitly required don't offload serial csum calculate and bio + * submit into a workqueue. + */ + return false; #endif /* Submit synchronously if the checksum implementation is fast. */ - if (auto_csum_mode && test_bit(BTRFS_FS_CSUM_IMPL_FAST, &bbio->fs_info->flags)) + if (auto_csum_mode && test_bit(BTRFS_FS_CSUM_IMPL_FAST, &fs_info->flags)) return false; /* @@ -605,7 +649,7 @@ static bool should_async_write(struct btrfs_bio *bbio) return false; /* Zoned devices require I/O to be submitted in order. */ - if ((bbio->bio.bi_opf & REQ_META) && btrfs_is_zoned(bbio->fs_info)) + if ((bbio->bio.bi_opf & REQ_META) && btrfs_is_zoned(fs_info)) return false; return true; @@ -620,7 +664,7 @@ static bool btrfs_wq_submit_bio(struct btrfs_bio *bbio, struct btrfs_io_context *bioc, struct btrfs_io_stripe *smap, int mirror_num) { - struct btrfs_fs_info *fs_info = bbio->fs_info; + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; struct async_submit_bio *async; async = kmalloc(sizeof(*async), GFP_NOFS); @@ -639,11 +683,12 @@ static bool btrfs_wq_submit_bio(struct btrfs_bio *bbio, static u64 btrfs_append_map_length(struct btrfs_bio *bbio, u64 map_length) { + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; unsigned int nr_segs; int sector_offset; - map_length = min(map_length, bbio->fs_info->max_zone_append_size); - sector_offset = bio_split_rw_at(&bbio->bio, &bbio->fs_info->limits, + map_length = min(map_length, fs_info->max_zone_append_size); + sector_offset = bio_split_rw_at(&bbio->bio, &fs_info->limits, &nr_segs, map_length); if (sector_offset) { /* @@ -651,7 +696,7 @@ static u64 btrfs_append_map_length(struct btrfs_bio *bbio, u64 map_length) * sectorsize and thus cause unaligned I/Os. Fix that by * always rounding down to the nearest boundary. */ - return ALIGN_DOWN(sector_offset << SECTOR_SHIFT, bbio->fs_info->sectorsize); + return ALIGN_DOWN(sector_offset << SECTOR_SHIFT, fs_info->sectorsize); } return map_length; } @@ -659,18 +704,17 @@ static u64 btrfs_append_map_length(struct btrfs_bio *bbio, u64 map_length) static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num) { struct btrfs_inode *inode = bbio->inode; - struct btrfs_fs_info *fs_info = bbio->fs_info; + struct btrfs_fs_info *fs_info = inode->root->fs_info; struct bio *bio = &bbio->bio; u64 logical = bio->bi_iter.bi_sector << SECTOR_SHIFT; u64 length = bio->bi_iter.bi_size; u64 map_length = length; - bool use_append = btrfs_use_zone_append(bbio); struct btrfs_io_context *bioc = NULL; struct btrfs_io_stripe smap; blk_status_t status; int ret; - if (!bbio->inode || btrfs_is_data_reloc_root(inode->root)) + if (bbio->is_scrub || btrfs_is_data_reloc_root(inode->root)) smap.rst_search_commit_root = true; else smap.rst_search_commit_root = false; @@ -684,8 +728,10 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num) goto end_bbio; } + bbio->can_use_append = btrfs_use_zone_append(bbio); + map_length = min(map_length, length); - if (use_append) + if (bbio->can_use_append) map_length = btrfs_append_map_length(bbio, map_length); if (map_length < length) { @@ -714,11 +760,6 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num) } if (btrfs_op(bio) == BTRFS_MAP_WRITE) { - if (use_append) { - bio->bi_opf &= ~REQ_OP_WRITE; - bio->bi_opf |= REQ_OP_ZONE_APPEND; - } - if (is_data_bbio(bbio) && bioc && bioc->use_rst) { /* * No locking for the list update, as we only add to @@ -734,7 +775,7 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num) * Csum items for reloc roots have already been cloned at this * point, so they are handled as part of the no-checksum case. */ - if (inode && !(inode->flags & BTRFS_INODE_NODATASUM) && + if (!(inode->flags & BTRFS_INODE_NODATASUM) && !test_bit(BTRFS_FS_STATE_NO_DATA_CSUMS, &fs_info->fs_state) && !btrfs_is_data_reloc_root(inode->root)) { if (should_async_write(bbio) && @@ -745,7 +786,7 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num) status = errno_to_blk_status(ret); if (status) goto fail; - } else if (use_append || + } else if (bbio->can_use_append || (btrfs_is_zoned(fs_info) && inode && inode->flags & BTRFS_INODE_NODATASUM)) { ret = btrfs_alloc_dummy_sum(bbio); @@ -782,7 +823,7 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num) static void assert_bbio_alignment(struct btrfs_bio *bbio) { #ifdef CONFIG_BTRFS_ASSERT - struct btrfs_fs_info *fs_info = bbio->fs_info; + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; struct bio_vec bvec; struct bvec_iter iter; const u32 blocksize = fs_info->sectorsize; @@ -885,16 +926,16 @@ int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start, */ void btrfs_submit_repair_write(struct btrfs_bio *bbio, int mirror_num, bool dev_replace) { - struct btrfs_fs_info *fs_info = bbio->fs_info; + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; u64 logical = bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT; u64 length = bbio->bio.bi_iter.bi_size; struct btrfs_io_stripe smap = { 0 }; int ret; - ASSERT(fs_info); ASSERT(mirror_num > 0); ASSERT(btrfs_op(&bbio->bio) == BTRFS_MAP_WRITE); - ASSERT(!bbio->inode); + ASSERT(!is_data_inode(bbio->inode)); + ASSERT(bbio->is_scrub); btrfs_bio_counter_inc_blocked(fs_info); ret = btrfs_map_repair_block(fs_info, &smap, logical, length, mirror_num); diff --git a/fs/btrfs/bio.h b/fs/btrfs/bio.h index 00883aea55d70..69fe54f564fcb 100644 --- a/fs/btrfs/bio.h +++ b/fs/btrfs/bio.h @@ -34,7 +34,10 @@ typedef void (*btrfs_bio_end_io_t)(struct btrfs_bio *bbio); struct btrfs_bio { /* * Inode and offset into it that this I/O operates on. - * Only set for data I/O. + * + * If the inode is a data one, csum verification and read-repair + * will be done automatically. + * If the inode is a metadata one, everything is handled by the caller. */ struct btrfs_inode *inode; u64 file_offset; @@ -60,6 +63,9 @@ struct btrfs_bio { struct { struct btrfs_ordered_extent *ordered; struct btrfs_ordered_sum *sums; + struct work_struct csum_work; + struct completion csum_done; + struct bvec_iter csum_saved_iter; u64 orig_physical; }; @@ -76,14 +82,24 @@ struct btrfs_bio { atomic_t pending_ios; struct work_struct end_io_work; - /* File system that this I/O operates on. */ - struct btrfs_fs_info *fs_info; - /* Save the first error status of split bio. */ blk_status_t status; /* Use the commit root to look up csums (data read bio only). */ bool csum_search_commit_root; + + /* + * Since scrub will reuse btree inode, we need this flag to distinguish + * scrub bios. + */ + bool is_scrub; + + /* Whether the csum generation for data write is async. */ + bool async_csum; + + /* Whether the bio is written using zone append. */ + bool can_use_append; + /* * This member must come last, bio_alloc_bioset will allocate enough * bytes for entire btrfs_bio but relies on bio being last. @@ -99,10 +115,10 @@ static inline struct btrfs_bio *btrfs_bio(struct bio *bio) int __init btrfs_bioset_init(void); void __cold btrfs_bioset_exit(void); -void btrfs_bio_init(struct btrfs_bio *bbio, struct btrfs_fs_info *fs_info, +void btrfs_bio_init(struct btrfs_bio *bbio, struct btrfs_inode *inode, u64 file_offset, btrfs_bio_end_io_t end_io, void *private); struct btrfs_bio *btrfs_bio_alloc(unsigned int nr_vecs, blk_opf_t opf, - struct btrfs_fs_info *fs_info, + struct btrfs_inode *inode, u64 file_offset, btrfs_bio_end_io_t end_io, void *private); void btrfs_bio_end_io(struct btrfs_bio *bbio, blk_status_t status); diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 8bf501fbcc0be..4689ef206d0ee 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -1877,6 +1877,7 @@ void btrfs_reclaim_bgs_work(struct work_struct *work) while (!list_empty(&fs_info->reclaim_bgs)) { u64 used; u64 reserved; + u64 old_total; int ret = 0; bg = list_first_entry(&fs_info->reclaim_bgs, @@ -1942,6 +1943,7 @@ void btrfs_reclaim_bgs_work(struct work_struct *work) } spin_unlock(&bg->lock); + old_total = space_info->total_bytes; spin_unlock(&space_info->lock); /* @@ -1994,14 +1996,14 @@ void btrfs_reclaim_bgs_work(struct work_struct *work) reserved = 0; spin_lock(&space_info->lock); space_info->reclaim_errors++; - if (READ_ONCE(space_info->periodic_reclaim)) - space_info->periodic_reclaim_ready = false; spin_unlock(&space_info->lock); } spin_lock(&space_info->lock); space_info->reclaim_count++; space_info->reclaim_bytes += used; space_info->reclaim_bytes += reserved; + if (space_info->total_bytes < old_total) + btrfs_set_periodic_reclaim_ready(space_info, true); spin_unlock(&space_info->lock); next: @@ -3681,6 +3683,14 @@ int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans) return ret; } +static void btrfs_maybe_reset_size_class(struct btrfs_block_group *bg) +{ + lockdep_assert_held(&bg->lock); + if (btrfs_block_group_should_use_size_class(bg) && + bg->used == 0 && bg->reserved == 0) + bg->size_class = BTRFS_BG_SZ_NONE; +} + int btrfs_update_block_group(struct btrfs_trans_handle *trans, u64 bytenr, u64 num_bytes, bool alloc) { @@ -3745,6 +3755,7 @@ int btrfs_update_block_group(struct btrfs_trans_handle *trans, old_val -= num_bytes; cache->used = old_val; cache->pinned += num_bytes; + btrfs_maybe_reset_size_class(cache); btrfs_space_info_update_bytes_pinned(space_info, num_bytes); space_info->bytes_used -= num_bytes; space_info->disk_used -= num_bytes * factor; @@ -3836,7 +3847,7 @@ int btrfs_add_reserved_bytes(struct btrfs_block_group *cache, * that happens. */ if (num_bytes < ram_bytes) - btrfs_try_granting_tickets(cache->fs_info, space_info); + btrfs_try_granting_tickets(space_info); out: spin_unlock(&cache->lock); spin_unlock(&space_info->lock); @@ -3859,22 +3870,26 @@ void btrfs_free_reserved_bytes(struct btrfs_block_group *cache, u64 num_bytes, bool is_delalloc) { struct btrfs_space_info *space_info = cache->space_info; + bool bg_ro; spin_lock(&space_info->lock); spin_lock(&cache->lock); - if (cache->ro) + bg_ro = cache->ro; + cache->reserved -= num_bytes; + btrfs_maybe_reset_size_class(cache); + if (is_delalloc) + cache->delalloc_bytes -= num_bytes; + spin_unlock(&cache->lock); + + if (bg_ro) space_info->bytes_readonly += num_bytes; else if (btrfs_is_zoned(cache->fs_info)) space_info->bytes_zone_unusable += num_bytes; - cache->reserved -= num_bytes; + space_info->bytes_reserved -= num_bytes; space_info->max_extent_size = 0; - if (is_delalloc) - cache->delalloc_bytes -= num_bytes; - spin_unlock(&cache->lock); - - btrfs_try_granting_tickets(cache->fs_info, space_info); + btrfs_try_granting_tickets(space_info); spin_unlock(&space_info->lock); } diff --git a/fs/btrfs/block-rsv.c b/fs/btrfs/block-rsv.c index 5ad6de738aee0..fc378d2038a2d 100644 --- a/fs/btrfs/block-rsv.c +++ b/fs/btrfs/block-rsv.c @@ -278,10 +278,11 @@ u64 btrfs_block_rsv_release(struct btrfs_fs_info *fs_info, struct btrfs_block_rsv *target = NULL; /* - * If we are a delayed block reserve then push to the global rsv, - * otherwise dump into the global delayed reserve if it is not full. + * If we are a delayed refs block reserve then push to the global + * reserve, otherwise dump into the global delayed refs reserve if it is + * not full. */ - if (block_rsv->type == BTRFS_BLOCK_RSV_DELOPS) + if (block_rsv->type == BTRFS_BLOCK_RSV_DELREFS) target = global_rsv; else if (block_rsv != global_rsv && !btrfs_block_rsv_full(delayed_rsv)) target = delayed_rsv; @@ -387,7 +388,7 @@ void btrfs_update_global_block_rsv(struct btrfs_fs_info *fs_info) num_bytes = block_rsv->reserved - block_rsv->size; btrfs_space_info_update_bytes_may_use(sinfo, -num_bytes); block_rsv->reserved = block_rsv->size; - btrfs_try_granting_tickets(fs_info, sinfo); + btrfs_try_granting_tickets(sinfo); } block_rsv->full = (block_rsv->reserved == block_rsv->size); diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index af373d50a901f..a66ca5531b5c5 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -18,20 +18,20 @@ #include #include #include +#include "ctree.h" #include "block-rsv.h" #include "extent_map.h" -#include "extent_io.h" #include "extent-io-tree.h" -#include "ordered-data.h" -#include "delayed-inode.h" -struct extent_state; struct posix_acl; struct iov_iter; struct writeback_control; struct btrfs_root; struct btrfs_fs_info; struct btrfs_trans_handle; +struct btrfs_bio; +struct btrfs_file_extent; +struct btrfs_delayed_node; /* * Since we search a directory based on f_pos (struct dir_context::pos) we have diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index bacad18357b33..8c3899832a1aa 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -67,9 +67,7 @@ static struct compressed_bio *alloc_compressed_bio(struct btrfs_inode *inode, bbio = btrfs_bio(bio_alloc_bioset(NULL, BTRFS_MAX_COMPRESSED_PAGES, op, GFP_NOFS, &btrfs_compressed_bioset)); - btrfs_bio_init(bbio, inode->root->fs_info, end_io, NULL); - bbio->inode = inode; - bbio->file_offset = start; + btrfs_bio_init(bbio, inode, start, end_io, NULL); return to_compressed_bio(bbio); } @@ -354,7 +352,7 @@ static void end_bbio_compressed_write(struct btrfs_bio *bbio) static void btrfs_add_compressed_bio_folios(struct compressed_bio *cb) { - struct btrfs_fs_info *fs_info = cb->bbio.fs_info; + struct btrfs_fs_info *fs_info = cb->bbio.inode->root->fs_info; struct bio *bio = &cb->bbio.bio; u32 offset = 0; diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h index eba188a9e3bb5..062ebd9c2d32d 100644 --- a/fs/btrfs/compression.h +++ b/fs/btrfs/compression.h @@ -14,14 +14,12 @@ #include #include "bio.h" #include "fs.h" -#include "messages.h" +#include "btrfs_inode.h" struct address_space; -struct page; struct inode; struct btrfs_inode; struct btrfs_ordered_extent; -struct btrfs_bio; /* * We want to make sure that amount of RAM required to uncompress an extent is @@ -77,7 +75,7 @@ struct compressed_bio { static inline struct btrfs_fs_info *cb_to_fs_info(const struct compressed_bio *cb) { - return cb->bbio.fs_info; + return cb->bbio.inode->root->fs_info; } /* @range_end must be exclusive. */ diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index fe70b593c7cd9..16dd11c485313 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -17,9 +17,7 @@ #include #include #include "locking.h" -#include "fs.h" #include "accessors.h" -#include "extent-io-tree.h" struct extent_buffer; struct btrfs_block_rsv; diff --git a/fs/btrfs/defrag.c b/fs/btrfs/defrag.c index 7b277934f66f9..a4cc1bc635622 100644 --- a/fs/btrfs/defrag.c +++ b/fs/btrfs/defrag.c @@ -15,6 +15,7 @@ #include "defrag.h" #include "file-item.h" #include "super.h" +#include "compression.h" static struct kmem_cache *btrfs_inode_defrag_cachep; diff --git a/fs/btrfs/delayed-inode.c b/fs/btrfs/delayed-inode.c index 59b489d7e4b58..ea48706a3d810 100644 --- a/fs/btrfs/delayed-inode.c +++ b/fs/btrfs/delayed-inode.c @@ -1676,7 +1676,7 @@ int btrfs_delete_delayed_dir_index(struct btrfs_trans_handle *trans, if (unlikely(ret)) { btrfs_err(trans->fs_info, "failed to add delayed dir index item, root: %llu, inode: %llu, index: %llu, error: %d", - index, btrfs_root_id(node->root), node->inode_id, ret); + btrfs_root_id(node->root), node->inode_id, index, ret); btrfs_delayed_item_release_metadata(dir->root, item); btrfs_release_delayed_item(item); } diff --git a/fs/btrfs/dir-item.c b/fs/btrfs/dir-item.c index 69863e398e223..77e1bcb2a74bf 100644 --- a/fs/btrfs/dir-item.c +++ b/fs/btrfs/dir-item.c @@ -9,6 +9,7 @@ #include "transaction.h" #include "accessors.h" #include "dir-item.h" +#include "delayed-inode.h" /* * insert a name into a directory, doing overflow properly if there is a hash diff --git a/fs/btrfs/direct-io.c b/fs/btrfs/direct-io.c index 802d4dbe5b381..3836414cbe371 100644 --- a/fs/btrfs/direct-io.c +++ b/fs/btrfs/direct-io.c @@ -10,6 +10,8 @@ #include "fs.h" #include "transaction.h" #include "volumes.h" +#include "bio.h" +#include "ordered-data.h" struct btrfs_dio_data { ssize_t submitted; @@ -713,10 +715,8 @@ static void btrfs_dio_submit_io(const struct iomap_iter *iter, struct bio *bio, container_of(bbio, struct btrfs_dio_private, bbio); struct btrfs_dio_data *dio_data = iter->private; - btrfs_bio_init(bbio, BTRFS_I(iter->inode)->root->fs_info, + btrfs_bio_init(bbio, BTRFS_I(iter->inode), file_offset, btrfs_dio_end_io, bio->bi_private); - bbio->inode = BTRFS_I(iter->inode); - bbio->file_offset = file_offset; dip->file_offset = file_offset; dip->bytes = bio->bi_iter.bi_size; @@ -814,6 +814,8 @@ ssize_t btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from) ssize_t ret; unsigned int ilock_flags = 0; struct iomap_dio *dio; + const u64 data_profile = btrfs_data_alloc_profile(fs_info) & + BTRFS_BLOCK_GROUP_PROFILE_MASK; if (iocb->ki_flags & IOCB_NOWAIT) ilock_flags |= BTRFS_ILOCK_TRY; @@ -827,6 +829,16 @@ ssize_t btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from) if (iocb->ki_pos + iov_iter_count(from) <= i_size_read(inode) && IS_NOSEC(inode)) ilock_flags |= BTRFS_ILOCK_SHARED; + /* + * If our data profile has duplication (either extra mirrors or RAID56), + * we can not trust the direct IO buffer, the content may change during + * writeback and cause different contents written to different mirrors. + * + * Thus only RAID0 and SINGLE can go true zero-copy direct IO. + */ + if (data_profile != BTRFS_BLOCK_GROUP_RAID0 && data_profile != 0) + goto buffered; + relock: ret = btrfs_inode_lock(BTRFS_I(inode), ilock_flags); if (ret < 0) diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 745ae698bbc8a..9c3a944cbc24a 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -50,6 +50,7 @@ #include "relocation.h" #include "scrub.h" #include "super.h" +#include "delayed-inode.h" #define BTRFS_SUPER_FLAG_SUPP (BTRFS_HEADER_FLAG_WRITTEN |\ BTRFS_HEADER_FLAG_RELOC |\ @@ -3159,7 +3160,7 @@ int btrfs_check_features(struct btrfs_fs_info *fs_info, bool is_rw_mount) if (incompat & ~BTRFS_FEATURE_INCOMPAT_SUPP) { btrfs_err(fs_info, "cannot mount because of unknown incompat features (0x%llx)", - incompat); + incompat & ~BTRFS_FEATURE_INCOMPAT_SUPP); return -EINVAL; } @@ -3191,7 +3192,7 @@ int btrfs_check_features(struct btrfs_fs_info *fs_info, bool is_rw_mount) if (compat_ro_unsupp && is_rw_mount) { btrfs_err(fs_info, "cannot mount read-write because of unknown compat_ro features (0x%llx)", - compat_ro); + compat_ro_unsupp); return -EINVAL; } @@ -3204,7 +3205,7 @@ int btrfs_check_features(struct btrfs_fs_info *fs_info, bool is_rw_mount) !btrfs_test_opt(fs_info, NOLOGREPLAY)) { btrfs_err(fs_info, "cannot replay dirty log with unsupported compat_ro features (0x%llx), try rescue=nologreplay", - compat_ro); + compat_ro_unsupp); return -EINVAL; } diff --git a/fs/btrfs/disk-io.h b/fs/btrfs/disk-io.h index 57920f2c6fe4e..5320da83d0cf8 100644 --- a/fs/btrfs/disk-io.h +++ b/fs/btrfs/disk-io.h @@ -9,7 +9,8 @@ #include #include #include "ctree.h" -#include "fs.h" +#include "bio.h" +#include "ordered-data.h" struct block_device; struct super_block; diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index dc4ca98c37800..08b7109299472 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -40,6 +40,7 @@ #include "orphan.h" #include "tree-checker.h" #include "raid-stripe-tree.h" +#include "delayed-inode.h" #undef SCRAMBLE_DELAYED_REFS @@ -6567,6 +6568,10 @@ int btrfs_trim_fs(struct btrfs_fs_info *fs_info, struct fstrim_range *range) range->minlen); trimmed += group_trimmed; + if (ret == -ERESTARTSYS || ret == -EINTR) { + btrfs_put_block_group(cache); + break; + } if (ret) { bg_failed++; bg_ret = ret; @@ -6580,6 +6585,9 @@ int btrfs_trim_fs(struct btrfs_fs_info *fs_info, struct fstrim_range *range) "failed to trim %llu block group(s), last error %d", bg_failed, bg_ret); + if (ret == -ERESTARTSYS || ret == -EINTR) + return ret; + mutex_lock(&fs_devices->device_list_mutex); list_for_each_entry(device, &fs_devices->devices, dev_list) { if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) @@ -6588,10 +6596,12 @@ int btrfs_trim_fs(struct btrfs_fs_info *fs_info, struct fstrim_range *range) ret = btrfs_trim_free_extents(device, &group_trimmed); trimmed += group_trimmed; + if (ret == -ERESTARTSYS || ret == -EINTR) + break; if (ret) { dev_failed++; dev_ret = ret; - break; + continue; } } mutex_unlock(&fs_devices->device_list_mutex); @@ -6601,6 +6611,8 @@ int btrfs_trim_fs(struct btrfs_fs_info *fs_info, struct fstrim_range *range) "failed to trim %llu device(s), last error %d", dev_failed, dev_ret); range->len = trimmed; + if (ret == -ERESTARTSYS || ret == -EINTR) + return ret; if (bg_ret) return bg_ret; return dev_ret; diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index e6ffa12f57535..c3524401ff03e 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -518,7 +518,7 @@ static void end_folio_read(struct folio *folio, bool uptodate, u64 start, u32 le */ static void end_bbio_data_write(struct btrfs_bio *bbio) { - struct btrfs_fs_info *fs_info = bbio->fs_info; + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; struct bio *bio = &bbio->bio; int error = blk_status_to_errno(bio->bi_status); struct folio_iter fi; @@ -574,7 +574,7 @@ static void begin_folio_read(struct btrfs_fs_info *fs_info, struct folio *folio) */ static void end_bbio_data_read(struct btrfs_bio *bbio) { - struct btrfs_fs_info *fs_info = bbio->fs_info; + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; struct bio *bio = &bbio->bio; struct folio_iter fi; @@ -739,12 +739,10 @@ static void alloc_new_bio(struct btrfs_inode *inode, struct btrfs_fs_info *fs_info = inode->root->fs_info; struct btrfs_bio *bbio; - bbio = btrfs_bio_alloc(BIO_MAX_VECS, bio_ctrl->opf, fs_info, - bio_ctrl->end_io_func, NULL); + bbio = btrfs_bio_alloc(BIO_MAX_VECS, bio_ctrl->opf, inode, + file_offset, bio_ctrl->end_io_func, NULL); bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT; bbio->bio.bi_write_hint = inode->vfs_inode.i_write_hint; - bbio->inode = inode; - bbio->file_offset = file_offset; bio_ctrl->bbio = bbio; bio_ctrl->len_to_oe_boundary = U32_MAX; bio_ctrl->next_file_offset = file_offset; @@ -2223,12 +2221,11 @@ static noinline_for_stack void write_one_eb(struct extent_buffer *eb, bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES, REQ_OP_WRITE | REQ_META | wbc_to_write_flags(wbc), - eb->fs_info, end_bbio_meta_write, eb); + BTRFS_I(fs_info->btree_inode), eb->start, + end_bbio_meta_write, eb); bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT; bio_set_dev(&bbio->bio, fs_info->fs_devices->latest_dev->bdev); wbc_init_bio(wbc, &bbio->bio); - bbio->inode = BTRFS_I(eb->fs_info->btree_inode); - bbio->file_offset = eb->start; for (int i = 0; i < num_extent_folios(eb); i++) { struct folio *folio = eb->folios[i]; u64 range_start = max_t(u64, eb->start, folio_pos(folio)); @@ -3842,6 +3839,7 @@ static void end_bbio_meta_read(struct btrfs_bio *bbio) int read_extent_buffer_pages_nowait(struct extent_buffer *eb, int mirror_num, const struct btrfs_tree_parent_check *check) { + struct btrfs_fs_info *fs_info = eb->fs_info; struct btrfs_bio *bbio; if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags)) @@ -3875,11 +3873,9 @@ int read_extent_buffer_pages_nowait(struct extent_buffer *eb, int mirror_num, refcount_inc(&eb->refs); bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES, - REQ_OP_READ | REQ_META, eb->fs_info, - end_bbio_meta_read, eb); + REQ_OP_READ | REQ_META, BTRFS_I(fs_info->btree_inode), + eb->start, end_bbio_meta_read, eb); bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT; - bbio->inode = BTRFS_I(eb->fs_info->btree_inode); - bbio->file_offset = eb->start; memcpy(&bbio->parent_check, check, sizeof(*check)); for (int i = 0; i < num_extent_folios(eb); i++) { struct folio *folio = eb->folios[i]; diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h index 559bec44a7a8e..73571d5d3d5ad 100644 --- a/fs/btrfs/extent_io.h +++ b/fs/btrfs/extent_io.h @@ -12,7 +12,6 @@ #include #include #include -#include "compression.h" #include "messages.h" #include "ulist.h" #include "misc.h" diff --git a/fs/btrfs/extent_map.h b/fs/btrfs/extent_map.h index d4b81ee4d97bd..6f685f3c93272 100644 --- a/fs/btrfs/extent_map.h +++ b/fs/btrfs/extent_map.h @@ -8,8 +8,7 @@ #include #include #include -#include "misc.h" -#include "compression.h" +#include "fs.h" struct btrfs_inode; struct btrfs_fs_info; diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c index a42e6d54e7cd7..4b7c40f05e8f9 100644 --- a/fs/btrfs/file-item.c +++ b/fs/btrfs/file-item.c @@ -18,6 +18,7 @@ #include "fs.h" #include "accessors.h" #include "file-item.h" +#include "volumes.h" #define __MAX_CSUM_ITEMS(r, size) ((unsigned long)(((BTRFS_LEAF_DATA_SIZE(r) - \ sizeof(struct btrfs_item) * 2) / \ @@ -764,21 +765,46 @@ int btrfs_lookup_csums_bitmap(struct btrfs_root *root, struct btrfs_path *path, return ret; } +static void csum_one_bio(struct btrfs_bio *bbio, struct bvec_iter *src) +{ + struct btrfs_inode *inode = bbio->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); + struct bio *bio = &bbio->bio; + struct btrfs_ordered_sum *sums = bbio->sums; + struct bvec_iter iter = *src; + phys_addr_t paddr; + const u32 blocksize = fs_info->sectorsize; + int index = 0; + + shash->tfm = fs_info->csum_shash; + + btrfs_bio_for_each_block(paddr, bio, &iter, blocksize) { + btrfs_calculate_block_csum(fs_info, paddr, sums->sums + index); + index += fs_info->csum_size; + } +} + +static void csum_one_bio_work(struct work_struct *work) +{ + struct btrfs_bio *bbio = container_of(work, struct btrfs_bio, csum_work); + + ASSERT(btrfs_op(&bbio->bio) == BTRFS_MAP_WRITE); + ASSERT(bbio->async_csum == true); + csum_one_bio(bbio, &bbio->csum_saved_iter); + complete(&bbio->csum_done); +} + /* * Calculate checksums of the data contained inside a bio. */ -int btrfs_csum_one_bio(struct btrfs_bio *bbio) +int btrfs_csum_one_bio(struct btrfs_bio *bbio, bool async) { struct btrfs_ordered_extent *ordered = bbio->ordered; struct btrfs_inode *inode = bbio->inode; struct btrfs_fs_info *fs_info = inode->root->fs_info; - SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); struct bio *bio = &bbio->bio; struct btrfs_ordered_sum *sums; - struct bvec_iter iter = bio->bi_iter; - phys_addr_t paddr; - const u32 blocksize = fs_info->sectorsize; - int index; unsigned nofs_flag; nofs_flag = memalloc_nofs_save(); @@ -789,21 +815,21 @@ int btrfs_csum_one_bio(struct btrfs_bio *bbio) if (!sums) return -ENOMEM; + sums->logical = bio->bi_iter.bi_sector << SECTOR_SHIFT; sums->len = bio->bi_iter.bi_size; INIT_LIST_HEAD(&sums->list); - - sums->logical = bio->bi_iter.bi_sector << SECTOR_SHIFT; - index = 0; - - shash->tfm = fs_info->csum_shash; - - btrfs_bio_for_each_block(paddr, bio, &iter, blocksize) { - btrfs_calculate_block_csum(fs_info, paddr, sums->sums + index); - index += fs_info->csum_size; - } - bbio->sums = sums; btrfs_add_ordered_sum(ordered, sums); + + if (!async) { + csum_one_bio(bbio, &bbio->bio.bi_iter); + return 0; + } + init_completion(&bbio->csum_done); + bbio->async_csum = true; + bbio->csum_saved_iter = bbio->bio.bi_iter; + INIT_WORK(&bbio->csum_work, csum_one_bio_work); + schedule_work(&bbio->csum_work); return 0; } diff --git a/fs/btrfs/file-item.h b/fs/btrfs/file-item.h index 63216c43676de..5645c5e3abdb7 100644 --- a/fs/btrfs/file-item.h +++ b/fs/btrfs/file-item.h @@ -7,7 +7,7 @@ #include #include #include "ctree.h" -#include "accessors.h" +#include "ordered-data.h" struct extent_map; struct btrfs_file_extent_item; @@ -64,7 +64,7 @@ int btrfs_lookup_file_extent(struct btrfs_trans_handle *trans, int btrfs_csum_file_blocks(struct btrfs_trans_handle *trans, struct btrfs_root *root, struct btrfs_ordered_sum *sums); -int btrfs_csum_one_bio(struct btrfs_bio *bbio); +int btrfs_csum_one_bio(struct btrfs_bio *bbio, bool async); int btrfs_alloc_dummy_sum(struct btrfs_bio *bbio); int btrfs_lookup_csums_range(struct btrfs_root *root, u64 start, u64 end, struct list_head *list, int search_commit, diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 76a66c74249a2..2799b10592d5a 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -71,6 +71,7 @@ #include "backref.h" #include "raid-stripe-tree.h" #include "fiemap.h" +#include "delayed-inode.h" #define COW_FILE_RANGE_KEEP_LOCKED (1UL << 0) #define COW_FILE_RANGE_NO_INLINE (1UL << 1) @@ -4657,7 +4658,7 @@ int btrfs_delete_subvolume(struct btrfs_inode *dir, struct dentry *dentry) spin_unlock(&dest->root_item_lock); btrfs_warn(fs_info, "attempt to delete subvolume %llu with active swapfile", - btrfs_root_id(root)); + btrfs_root_id(dest)); ret = -EPERM; goto out_up_write; } @@ -9422,7 +9423,6 @@ int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, u64 disk_bytenr, u64 disk_io_size, struct page **pages, void *uring_ctx) { - struct btrfs_fs_info *fs_info = inode->root->fs_info; struct btrfs_encoded_read_private *priv, sync_priv; struct completion sync_reads; unsigned long i = 0; @@ -9447,10 +9447,9 @@ int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, priv->status = 0; priv->uring_ctx = uring_ctx; - bbio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, fs_info, + bbio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, inode, 0, btrfs_encoded_read_endio, priv); bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT; - bbio->inode = inode; do { size_t bytes = min_t(u64, disk_io_size, PAGE_SIZE); @@ -9459,10 +9458,9 @@ int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, refcount_inc(&priv->pending_refs); btrfs_submit_bbio(bbio, 0); - bbio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, fs_info, + bbio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, inode, 0, btrfs_encoded_read_endio, priv); bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT; - bbio->inode = inode; continue; } diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index 9a34d6530658e..736a1b3170700 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -4725,7 +4725,7 @@ static int btrfs_uring_read_extent(struct kiocb *iocb, struct iov_iter *iter, { struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp)); struct extent_io_tree *io_tree = &inode->io_tree; - struct page **pages; + struct page **pages = NULL; struct btrfs_uring_priv *priv = NULL; unsigned long nr_pages; int ret; @@ -4783,6 +4783,11 @@ static int btrfs_uring_read_extent(struct kiocb *iocb, struct iov_iter *iter, btrfs_unlock_extent(io_tree, start, lockend, &cached_state); btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); kfree(priv); + for (int i = 0; i < nr_pages; i++) { + if (pages[i]) + __free_page(pages[i]); + } + kfree(pages); return ret; } diff --git a/fs/btrfs/misc.h b/fs/btrfs/misc.h index 60f9b000d644b..a82032c66ccd3 100644 --- a/fs/btrfs/misc.h +++ b/fs/btrfs/misc.h @@ -13,6 +13,13 @@ #include #include +/* + * Convenience macros to define a pointer with the __free(kfree) and + * __free(kvfree) cleanup attributes and initialized to NULL. + */ +#define AUTO_KFREE(name) *name __free(kfree) = NULL +#define AUTO_KVFREE(name) *name __free(kvfree) = NULL + /* * Enumerate bits using enum autoincrement. Define the @name as the n-th bit. */ diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c index febc22d1b6487..302bb3ecf39a3 100644 --- a/fs/btrfs/qgroup.c +++ b/fs/btrfs/qgroup.c @@ -1157,11 +1157,14 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info, } if (ret > 0) { /* - * Shouldn't happen, but in case it does we - * don't need to do the btrfs_next_item, just - * continue. + * Shouldn't happen because the key should still + * be there (return 0), but in case it does it + * means we have reached the end of the tree - + * there are no more leaves with items that have + * a key greater than or equals to @found_key, + * so just stop the search loop. */ - continue; + break; } } ret = btrfs_next_item(tree_root, path); @@ -1627,8 +1630,10 @@ static int __del_qgroup_relation(struct btrfs_trans_handle *trans, u64 src, if (ret < 0 && ret != -ENOENT) goto out; ret2 = del_qgroup_relation_item(trans, dst, src); - if (ret2 < 0 && ret2 != -ENOENT) + if (ret2 < 0 && ret2 != -ENOENT) { + ret = ret2; goto out; + } /* At least one deletion succeeded, return 0 */ if (!ret || !ret2) diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index b6a7ea105eb13..16936d17166ee 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -747,7 +747,7 @@ static void scrub_verify_one_metadata(struct scrub_stripe *stripe, int sector_nr btrfs_warn_rl(fs_info, "scrub: tree block %llu mirror %u has bad fsid, has %pU want %pU", logical, stripe->mirror_num, - header->fsid, fs_info->fs_devices->fsid); + header->fsid, fs_info->fs_devices->metadata_uuid); return; } if (memcmp(header->chunk_tree_uuid, fs_info->chunk_tree_uuid, @@ -929,10 +929,11 @@ static int calc_next_mirror(int mirror, int num_copies) static void scrub_bio_add_sector(struct btrfs_bio *bbio, struct scrub_stripe *stripe, int sector_nr) { + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; void *kaddr = scrub_stripe_get_kaddr(stripe, sector_nr); int ret; - ret = bio_add_page(&bbio->bio, virt_to_page(kaddr), bbio->fs_info->sectorsize, + ret = bio_add_page(&bbio->bio, virt_to_page(kaddr), fs_info->sectorsize, offset_in_page(kaddr)); /* * Caller should ensure the bbio has enough size. @@ -942,7 +943,21 @@ static void scrub_bio_add_sector(struct btrfs_bio *bbio, struct scrub_stripe *st * to create the minimal amount of bio vectors, for fs block size < page * size cases. */ - ASSERT(ret == bbio->fs_info->sectorsize); + ASSERT(ret == fs_info->sectorsize); +} + +static struct btrfs_bio *alloc_scrub_bbio(struct btrfs_fs_info *fs_info, + unsigned int nr_vecs, blk_opf_t opf, + u64 logical, + btrfs_bio_end_io_t end_io, void *private) +{ + struct btrfs_bio *bbio; + + bbio = btrfs_bio_alloc(nr_vecs, opf, BTRFS_I(fs_info->btree_inode), + logical, end_io, private); + bbio->is_scrub = true; + bbio->bio.bi_iter.bi_sector = logical >> SECTOR_SHIFT; + return bbio; } static void scrub_stripe_submit_repair_read(struct scrub_stripe *stripe, @@ -968,12 +983,10 @@ static void scrub_stripe_submit_repair_read(struct scrub_stripe *stripe, bbio = NULL; } - if (!bbio) { - bbio = btrfs_bio_alloc(stripe->nr_sectors, REQ_OP_READ, - fs_info, scrub_repair_read_endio, stripe); - bbio->bio.bi_iter.bi_sector = (stripe->logical + - (i << fs_info->sectorsize_bits)) >> SECTOR_SHIFT; - } + if (!bbio) + bbio = alloc_scrub_bbio(fs_info, stripe->nr_sectors, REQ_OP_READ, + stripe->logical + (i << fs_info->sectorsize_bits), + scrub_repair_read_endio, stripe); scrub_bio_add_sector(bbio, stripe, i); } @@ -1352,13 +1365,10 @@ static void scrub_write_sectors(struct scrub_ctx *sctx, struct scrub_stripe *str scrub_submit_write_bio(sctx, stripe, bbio, dev_replace); bbio = NULL; } - if (!bbio) { - bbio = btrfs_bio_alloc(stripe->nr_sectors, REQ_OP_WRITE, - fs_info, scrub_write_endio, stripe); - bbio->bio.bi_iter.bi_sector = (stripe->logical + - (sector_nr << fs_info->sectorsize_bits)) >> - SECTOR_SHIFT; - } + if (!bbio) + bbio = alloc_scrub_bbio(fs_info, stripe->nr_sectors, REQ_OP_WRITE, + stripe->logical + (sector_nr << fs_info->sectorsize_bits), + scrub_write_endio, stripe); scrub_bio_add_sector(bbio, stripe, sector_nr); } if (bbio) @@ -1849,9 +1859,8 @@ static void scrub_submit_extent_sector_read(struct scrub_stripe *stripe) continue; } - bbio = btrfs_bio_alloc(stripe->nr_sectors, REQ_OP_READ, - fs_info, scrub_read_endio, stripe); - bbio->bio.bi_iter.bi_sector = logical >> SECTOR_SHIFT; + bbio = alloc_scrub_bbio(fs_info, stripe->nr_sectors, REQ_OP_READ, + logical, scrub_read_endio, stripe); } scrub_bio_add_sector(bbio, stripe, i); @@ -1888,10 +1897,8 @@ static void scrub_submit_initial_read(struct scrub_ctx *sctx, return; } - bbio = btrfs_bio_alloc(BTRFS_STRIPE_LEN >> min_folio_shift, REQ_OP_READ, fs_info, - scrub_read_endio, stripe); - - bbio->bio.bi_iter.bi_sector = stripe->logical >> SECTOR_SHIFT; + bbio = alloc_scrub_bbio(fs_info, BTRFS_STRIPE_LEN >> min_folio_shift, REQ_OP_READ, + stripe->logical, scrub_read_endio, stripe); /* Read the whole range inside the chunk boundary. */ for (unsigned int cur = 0; cur < nr_sectors; cur++) scrub_bio_add_sector(bbio, stripe, cur); diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index a6f94e9f55915..6b64691034de4 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -15,6 +15,7 @@ #include "accessors.h" #include "extent-tree.h" #include "zoned.h" +#include "delayed-inode.h" /* * HOW DOES SPACE RESERVATION WORK @@ -377,7 +378,7 @@ void btrfs_add_bg_to_space_info(struct btrfs_fs_info *info, btrfs_space_info_update_bytes_zone_unusable(space_info, block_group->zone_unusable); if (block_group->length > 0) space_info->full = false; - btrfs_try_granting_tickets(info, space_info); + btrfs_try_granting_tickets(space_info); spin_unlock(&space_info->lock); block_group->space_info = space_info; @@ -527,9 +528,9 @@ static void remove_ticket(struct btrfs_space_info *space_info, * This is for space we already have accounted in space_info->bytes_may_use, so * basically when we're returning space from block_rsv's. */ -void btrfs_try_granting_tickets(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info) +void btrfs_try_granting_tickets(struct btrfs_space_info *space_info) { + struct btrfs_fs_info *fs_info = space_info->fs_info; struct list_head *head; enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_NO_FLUSH; @@ -1128,7 +1129,7 @@ static bool maybe_fail_all_tickets(struct btrfs_fs_info *fs_info, * the list. */ if (!aborted) - btrfs_try_granting_tickets(fs_info, space_info); + btrfs_try_granting_tickets(space_info); } return (tickets_id != space_info->tickets_id); } @@ -1548,7 +1549,7 @@ static void priority_reclaim_metadata_space(struct btrfs_fs_info *fs_info, * ticket in front of a smaller ticket that can now be satisfied with * the available space. */ - btrfs_try_granting_tickets(fs_info, space_info); + btrfs_try_granting_tickets(space_info); spin_unlock(&space_info->lock); } @@ -1576,7 +1577,7 @@ static void priority_reclaim_data_space(struct btrfs_fs_info *fs_info, ticket->error = -ENOSPC; remove_ticket(space_info, ticket); - btrfs_try_granting_tickets(fs_info, space_info); + btrfs_try_granting_tickets(space_info); spin_unlock(&space_info->lock); } @@ -2073,11 +2074,11 @@ static bool is_reclaim_urgent(struct btrfs_space_info *space_info) return unalloc < data_chunk_size; } -static void do_reclaim_sweep(struct btrfs_space_info *space_info, int raid) +static bool do_reclaim_sweep(struct btrfs_space_info *space_info, int raid) { struct btrfs_block_group *bg; int thresh_pct; - bool try_again = true; + bool will_reclaim = false; bool urgent; spin_lock(&space_info->lock); @@ -2095,7 +2096,7 @@ static void do_reclaim_sweep(struct btrfs_space_info *space_info, int raid) spin_lock(&bg->lock); thresh = mult_perc(bg->length, thresh_pct); if (bg->used < thresh && bg->reclaim_mark) { - try_again = false; + will_reclaim = true; reclaim = true; } bg->reclaim_mark++; @@ -2112,12 +2113,13 @@ static void do_reclaim_sweep(struct btrfs_space_info *space_info, int raid) * If we have any staler groups, we don't touch the fresher ones, but if we * really need a block group, do take a fresh one. */ - if (try_again && urgent) { - try_again = false; + if (!will_reclaim && urgent) { + urgent = false; goto again; } up_read(&space_info->groups_sem); + return will_reclaim; } void btrfs_space_info_update_reclaimable(struct btrfs_space_info *space_info, s64 bytes) @@ -2127,7 +2129,8 @@ void btrfs_space_info_update_reclaimable(struct btrfs_space_info *space_info, s6 lockdep_assert_held(&space_info->lock); space_info->reclaimable_bytes += bytes; - if (space_info->reclaimable_bytes >= chunk_sz) + if (space_info->reclaimable_bytes > 0 && + space_info->reclaimable_bytes >= chunk_sz) btrfs_set_periodic_reclaim_ready(space_info, true); } @@ -2154,7 +2157,6 @@ static bool btrfs_should_periodic_reclaim(struct btrfs_space_info *space_info) spin_lock(&space_info->lock); ret = space_info->periodic_reclaim_ready; - btrfs_set_periodic_reclaim_ready(space_info, false); spin_unlock(&space_info->lock); return ret; @@ -2168,8 +2170,10 @@ void btrfs_reclaim_sweep(const struct btrfs_fs_info *fs_info) list_for_each_entry(space_info, &fs_info->space_info, list) { if (!btrfs_should_periodic_reclaim(space_info)) continue; - for (raid = 0; raid < BTRFS_NR_RAID_TYPES; raid++) - do_reclaim_sweep(space_info, raid); + for (raid = 0; raid < BTRFS_NR_RAID_TYPES; raid++) { + if (do_reclaim_sweep(space_info, raid)) + btrfs_set_periodic_reclaim_ready(space_info, false); + } } } @@ -2199,5 +2203,5 @@ void btrfs_return_free_space(struct btrfs_space_info *space_info, u64 len) grant: /* Add to any tickets we may have. */ if (len) - btrfs_try_granting_tickets(fs_info, space_info); + btrfs_try_granting_tickets(space_info); } diff --git a/fs/btrfs/space-info.h b/fs/btrfs/space-info.h index a846f63585c95..596a1e923ddfe 100644 --- a/fs/btrfs/space-info.h +++ b/fs/btrfs/space-info.h @@ -283,8 +283,7 @@ int btrfs_reserve_metadata_bytes(struct btrfs_fs_info *fs_info, struct btrfs_space_info *space_info, u64 orig_bytes, enum btrfs_reserve_flush_enum flush); -void btrfs_try_granting_tickets(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info); +void btrfs_try_granting_tickets(struct btrfs_space_info *space_info); int btrfs_can_overcommit(struct btrfs_fs_info *fs_info, const struct btrfs_space_info *space_info, u64 bytes, enum btrfs_reserve_flush_enum flush); @@ -295,7 +294,7 @@ static inline void btrfs_space_info_free_bytes_may_use( { spin_lock(&space_info->lock); btrfs_space_info_update_bytes_may_use(space_info, -num_bytes); - btrfs_try_granting_tickets(space_info->fs_info, space_info); + btrfs_try_granting_tickets(space_info); spin_unlock(&space_info->lock); } int btrfs_reserve_data_bytes(struct btrfs_space_info *space_info, u64 bytes, diff --git a/fs/btrfs/subpage.h b/fs/btrfs/subpage.h index ad0552db7c7dc..d81a0ade559fd 100644 --- a/fs/btrfs/subpage.h +++ b/fs/btrfs/subpage.h @@ -7,7 +7,6 @@ #include #include #include "btrfs_inode.h" -#include "fs.h" struct address_space; struct folio; diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index c457316c2788b..089712b15d603 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -32,6 +32,8 @@ #include "ioctl.h" #include "relocation.h" #include "scrub.h" +#include "ordered-data.h" +#include "delayed-inode.h" static struct kmem_cache *btrfs_trans_handle_cachep; @@ -724,7 +726,7 @@ start_transaction(struct btrfs_root *root, unsigned int num_items, h->type = type; INIT_LIST_HEAD(&h->new_bgs); - btrfs_init_metadata_block_rsv(fs_info, &h->delayed_rsv, BTRFS_BLOCK_RSV_DELOPS); + btrfs_init_metadata_block_rsv(fs_info, &h->delayed_rsv, BTRFS_BLOCK_RSV_DELREFS); smp_mb(); if (cur_trans->state >= TRANS_STATE_COMMIT_START && @@ -2482,13 +2484,6 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) list_add_tail(&fs_info->chunk_root->dirty_list, &cur_trans->switch_commits); - if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) { - btrfs_set_root_node(&fs_info->block_group_root->root_item, - fs_info->block_group_root->node); - list_add_tail(&fs_info->block_group_root->dirty_list, - &cur_trans->switch_commits); - } - switch_commit_roots(trans); ASSERT(list_empty(&cur_trans->dirty_bgs)); diff --git a/fs/btrfs/transaction.h b/fs/btrfs/transaction.h index 9f7c777af6356..18ef069197e5b 100644 --- a/fs/btrfs/transaction.h +++ b/fs/btrfs/transaction.h @@ -14,10 +14,6 @@ #include #include "btrfs_inode.h" #include "delayed-ref.h" -#include "extent-io-tree.h" -#include "block-rsv.h" -#include "messages.h" -#include "misc.h" struct dentry; struct inode; diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c index c10b4c242acfc..420c0f0e17c85 100644 --- a/fs/btrfs/tree-checker.c +++ b/fs/btrfs/tree-checker.c @@ -1713,7 +1713,7 @@ static int check_extent_data_ref(struct extent_buffer *leaf, objectid > BTRFS_LAST_FREE_OBJECTID)) { extent_err(leaf, slot, "invalid extent data backref objectid value %llu", - root); + objectid); return -EUCLEAN; } if (unlikely(!IS_ALIGNED(offset, leaf->fs_info->sectorsize))) { @@ -1894,7 +1894,7 @@ static int check_dev_extent_item(const struct extent_buffer *leaf, if (unlikely(prev_key->offset + prev_len > key->offset)) { generic_err(leaf, slot, "dev extent overlap, prev offset %llu len %llu current offset %llu", - prev_key->objectid, prev_len, key->offset); + prev_key->offset, prev_len, key->offset); return -EUCLEAN; } } diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index ae2e035d013e2..6c5db73c3e85f 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -29,6 +29,7 @@ #include "orphan.h" #include "print-tree.h" #include "tree-checker.h" +#include "delayed-inode.h" #define MAX_CONFLICT_INODES 10 diff --git a/fs/btrfs/tree-log.h b/fs/btrfs/tree-log.h index dc313e6bb2faa..4f149d7d4fdee 100644 --- a/fs/btrfs/tree-log.h +++ b/fs/btrfs/tree-log.h @@ -8,8 +8,7 @@ #include #include -#include "messages.h" -#include "ctree.h" +#include #include "transaction.h" struct inode; diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 8e7dcb12af4c4..4a1dc4720a0ba 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -1506,30 +1506,158 @@ struct btrfs_device *btrfs_scan_one_device(const char *path, } /* - * Try to find a chunk that intersects [start, start + len] range and when one - * such is found, record the end of it in *start + * Find the first pending extent intersecting a range. + * + * @device: the device to search + * @start: start of the range to check + * @len: length of the range to check + * @pending_start: output pointer for the start of the found pending extent + * @pending_end: output pointer for the end of the found pending extent (inclusive) + * + * Search for a pending chunk allocation that intersects the half-open range + * [start, start + len). + * + * Return: true if a pending extent was found, false otherwise. + * If the return value is true, store the first pending extent in + * [*pending_start, *pending_end]. Otherwise, the two output variables + * may still be modified, to something outside the range and should not + * be used. */ -static bool contains_pending_extent(struct btrfs_device *device, u64 *start, - u64 len) +static bool first_pending_extent(struct btrfs_device *device, u64 start, u64 len, + u64 *pending_start, u64 *pending_end) { - u64 physical_start, physical_end; - lockdep_assert_held(&device->fs_info->chunk_mutex); - if (btrfs_find_first_extent_bit(&device->alloc_state, *start, - &physical_start, &physical_end, + if (btrfs_find_first_extent_bit(&device->alloc_state, start, + pending_start, pending_end, CHUNK_ALLOCATED, NULL)) { - if (in_range(physical_start, *start, len) || - in_range(*start, physical_start, - physical_end + 1 - physical_start)) { - *start = physical_end + 1; + if (in_range(*pending_start, start, len) || + in_range(start, *pending_start, *pending_end + 1 - *pending_start)) { return true; } } return false; } +/* + * Find the first real hole accounting for pending extents. + * + * @device: the device containing the candidate hole + * @start: input/output pointer for the hole start position + * @len: input/output pointer for the hole length + * @min_hole_size: the size of hole we are looking for + * + * Given a potential hole specified by [*start, *start + *len), check for pending + * chunk allocations within that range. If pending extents are found, the hole is + * adjusted to represent the first true free space that is large enough when + * accounting for pending chunks. + * + * Note that this function must handle various cases involving non consecutive + * pending extents. + * + * Returns: true if a suitable hole was found and false otherwise. + * If the return value is true, then *start and *len are set to represent the hole. + * If the return value is false, then *start is set to the largest hole we + * found and *len is set to its length. + * If there are no holes at all, then *start is set to the end of the range and + * *len is set to 0. + */ +static bool find_hole_in_pending_extents(struct btrfs_device *device, u64 *start, + u64 *len, u64 min_hole_size) +{ + u64 pending_start, pending_end; + u64 end; + u64 max_hole_start = 0; + u64 max_hole_len = 0; + + lockdep_assert_held(&device->fs_info->chunk_mutex); + + if (*len == 0) + return false; + + end = *start + *len - 1; + + /* + * Loop until we either see a large enough hole or check every pending + * extent overlapping the candidate hole. + * At every hole that we observe, record it if it is the new max. + * At the end of the iteration, set the output variables to the max hole. + */ + while (true) { + if (first_pending_extent(device, *start, *len, &pending_start, &pending_end)) { + /* + * Case 1: the pending extent overlaps the start of + * candidate hole. That means the true hole is after the + * pending extent, but we need to find the next pending + * extent to properly size the hole. In the next loop, + * we will reduce to case 2 or 3. + * e.g., + * + * |----pending A----| real hole |----pending B----| + * | candidate hole | + * *start end + */ + if (pending_start <= *start) { + *start = pending_end + 1; + goto next; + } + /* + * Case 2: The pending extent starts after *start (and overlaps + * [*start, end), so the first hole just goes up to the start + * of the pending extent. + * e.g., + * + * | real hole |----pending A----| + * | candidate hole | + * *start end + */ + *len = pending_start - *start; + if (*len > max_hole_len) { + max_hole_start = *start; + max_hole_len = *len; + } + if (*len >= min_hole_size) + break; + /* + * If the hole wasn't big enough, then we advance past + * the pending extent and keep looking. + */ + *start = pending_end + 1; + goto next; + } else { + /* + * Case 3: There is no pending extent overlapping the + * range [*start, *start + *len - 1], so the only remaining + * hole is the remaining range. + * e.g., + * + * | candidate hole | + * | real hole | + * *start end + */ + + if (*len > max_hole_len) { + max_hole_start = *start; + max_hole_len = *len; + } + break; + } +next: + if (*start > end) + break; + *len = end - *start + 1; + } + if (max_hole_len) { + *start = max_hole_start; + *len = max_hole_len; + } else { + *start = end + 1; + *len = 0; + } + return max_hole_len >= min_hole_size; +} + static u64 dev_extent_search_start(struct btrfs_device *device) { switch (device->fs_devices->chunk_alloc_policy) { @@ -1594,59 +1722,57 @@ static bool dev_extent_hole_check_zoned(struct btrfs_device *device, } /* - * Check if specified hole is suitable for allocation. + * Validate and adjust a hole for chunk allocation * - * @device: the device which we have the hole - * @hole_start: starting position of the hole - * @hole_size: the size of the hole - * @num_bytes: the size of the free space that we need + * @device: the device containing the candidate hole + * @hole_start: input/output pointer for the hole start position + * @hole_size: input/output pointer for the hole size + * @num_bytes: minimum allocation size required * - * This function may modify @hole_start and @hole_size to reflect the suitable - * position for allocation. Returns 1 if hole position is updated, 0 otherwise. + * Check if the specified hole is suitable for allocation and adjust it if + * necessary. The hole may be modified to skip over pending chunk allocations + * and to satisfy stricter zoned requirements on zoned filesystems. + * + * For regular (non-zoned) allocation, if the hole after adjustment is smaller + * than @num_bytes, the search continues past additional pending extents until + * either a sufficiently large hole is found or no more pending extents exist. + * + * Return: true if a suitable hole was found and false otherwise. + * If the return value is true, then *hole_start and *hole_size are set to + * represent the hole we found. + * If the return value is false, then *hole_start is set to the largest + * hole we found and *hole_size is set to its length. + * If there are no holes at all, then *hole_start is set to the end of the range + * and *hole_size is set to 0. */ static bool dev_extent_hole_check(struct btrfs_device *device, u64 *hole_start, u64 *hole_size, u64 num_bytes) { - bool changed = false; - u64 hole_end = *hole_start + *hole_size; + bool found = false; + const u64 hole_end = *hole_start + *hole_size - 1; - for (;;) { - /* - * Check before we set max_hole_start, otherwise we could end up - * sending back this offset anyway. - */ - if (contains_pending_extent(device, hole_start, *hole_size)) { - if (hole_end >= *hole_start) - *hole_size = hole_end - *hole_start; - else - *hole_size = 0; - changed = true; - } + ASSERT(*hole_size > 0); - switch (device->fs_devices->chunk_alloc_policy) { - default: - btrfs_warn_unknown_chunk_allocation(device->fs_devices->chunk_alloc_policy); - fallthrough; - case BTRFS_CHUNK_ALLOC_REGULAR: - /* No extra check */ - break; - case BTRFS_CHUNK_ALLOC_ZONED: - if (dev_extent_hole_check_zoned(device, hole_start, - hole_size, num_bytes)) { - changed = true; - /* - * The changed hole can contain pending extent. - * Loop again to check that. - */ - continue; - } - break; - } +again: + *hole_size = hole_end - *hole_start + 1; + found = find_hole_in_pending_extents(device, hole_start, hole_size, num_bytes); + if (!found) + return found; + ASSERT(*hole_size >= num_bytes); + switch (device->fs_devices->chunk_alloc_policy) { + default: + btrfs_warn_unknown_chunk_allocation(device->fs_devices->chunk_alloc_policy); + fallthrough; + case BTRFS_CHUNK_ALLOC_REGULAR: + return found; + case BTRFS_CHUNK_ALLOC_ZONED: + if (dev_extent_hole_check_zoned(device, hole_start, hole_size, num_bytes)) + goto again; break; } - return changed; + return found; } /* @@ -1705,7 +1831,7 @@ static int find_free_dev_extent(struct btrfs_device *device, u64 num_bytes, ret = -ENOMEM; goto out; } -again: + if (search_start >= search_end || test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) { ret = -ENOSPC; @@ -1792,11 +1918,7 @@ static int find_free_dev_extent(struct btrfs_device *device, u64 num_bytes, */ if (search_end > search_start) { hole_size = search_end - search_start; - if (dev_extent_hole_check(device, &search_start, &hole_size, - num_bytes)) { - btrfs_release_path(path); - goto again; - } + dev_extent_hole_check(device, &search_start, &hole_size, num_bytes); if (hole_size > max_hole_size) { max_hole_start = search_start; @@ -4142,8 +4264,14 @@ static int __btrfs_balance(struct btrfs_fs_info *fs_info) * this shouldn't happen, it means the last relocate * failed */ - if (ret == 0) - BUG(); /* FIXME break ? */ + if (unlikely(ret == 0)) { + btrfs_err(fs_info, + "unexpected exact match of CHUNK_ITEM in chunk tree, offset 0x%llx", + key.offset); + mutex_unlock(&fs_info->reclaim_bgs_lock); + ret = -EUCLEAN; + goto error; + } ret = btrfs_previous_item(chunk_root, path, 0, BTRFS_CHUNK_ITEM_KEY); @@ -4882,6 +5010,7 @@ int btrfs_shrink_device(struct btrfs_device *device, u64 new_size) u64 diff; u64 start; u64 free_diff = 0; + u64 pending_start, pending_end; new_size = round_down(new_size, fs_info->sectorsize); start = new_size; @@ -4927,7 +5056,7 @@ int btrfs_shrink_device(struct btrfs_device *device, u64 new_size) * in-memory chunks are synced to disk so that the loop below sees them * and relocates them accordingly. */ - if (contains_pending_extent(device, &start, diff)) { + if (first_pending_extent(device, start, diff, &pending_start, &pending_end)) { mutex_unlock(&fs_info->chunk_mutex); ret = btrfs_commit_transaction(trans); if (ret) diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index d1db7fa1fe583..e14a4234954ba 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -1442,6 +1442,20 @@ static int btrfs_load_block_group_dup(struct btrfs_block_group *bg, return -EIO; } + /* + * When the last extent is removed, last_alloc can be smaller than the other write + * pointer. In that case, last_alloc should be moved to the corresponding write + * pointer position. + */ + for (int i = 0; i < map->num_stripes; i++) { + if (zone_info[i].alloc_offset == WP_CONVENTIONAL) + continue; + if (last_alloc <= zone_info[i].alloc_offset) { + last_alloc = zone_info[i].alloc_offset; + break; + } + } + if (zone_info[0].alloc_offset == WP_CONVENTIONAL) zone_info[0].alloc_offset = last_alloc; @@ -1483,6 +1497,21 @@ static int btrfs_load_block_group_raid1(struct btrfs_block_group *bg, /* In case a device is missing we have a cap of 0, so don't use it. */ bg->zone_capacity = min_not_zero(zone_info[0].capacity, zone_info[1].capacity); + /* + * When the last extent is removed, last_alloc can be smaller than the other write + * pointer. In that case, last_alloc should be moved to the corresponding write + * pointer position. + */ + for (i = 0; i < map->num_stripes; i++) { + if (zone_info[i].alloc_offset == WP_MISSING_DEV || + zone_info[i].alloc_offset == WP_CONVENTIONAL) + continue; + if (last_alloc <= zone_info[i].alloc_offset) { + last_alloc = zone_info[i].alloc_offset; + break; + } + } + for (i = 0; i < map->num_stripes; i++) { if (zone_info[i].alloc_offset == WP_MISSING_DEV) continue; @@ -1524,7 +1553,9 @@ static int btrfs_load_block_group_raid0(struct btrfs_block_group *bg, { struct btrfs_fs_info *fs_info = bg->fs_info; u64 stripe_nr = 0, stripe_offset = 0; + u64 prev_offset = 0; u32 stripe_index = 0; + bool has_partial = false, has_conventional = false; if ((map->type & BTRFS_BLOCK_GROUP_DATA) && !fs_info->stripe_root) { btrfs_err(fs_info, "zoned: data %s needs raid-stripe-tree", @@ -1532,6 +1563,35 @@ static int btrfs_load_block_group_raid0(struct btrfs_block_group *bg, return -EINVAL; } + /* + * When the last extent is removed, last_alloc can be smaller than the other write + * pointer. In that case, last_alloc should be moved to the corresponding write + * pointer position. + */ + for (int i = 0; i < map->num_stripes; i++) { + u64 alloc; + + if (zone_info[i].alloc_offset == WP_MISSING_DEV || + zone_info[i].alloc_offset == WP_CONVENTIONAL) + continue; + + stripe_nr = zone_info[i].alloc_offset >> BTRFS_STRIPE_LEN_SHIFT; + stripe_offset = zone_info[i].alloc_offset & BTRFS_STRIPE_LEN_MASK; + if (stripe_offset == 0 && stripe_nr > 0) { + stripe_nr--; + stripe_offset = BTRFS_STRIPE_LEN; + } + alloc = ((stripe_nr * map->num_stripes + i) << BTRFS_STRIPE_LEN_SHIFT) + + stripe_offset; + last_alloc = max(last_alloc, alloc); + + /* Partially written stripe found. It should be last. */ + if (zone_info[i].alloc_offset & BTRFS_STRIPE_LEN_MASK) + break; + } + stripe_nr = 0; + stripe_offset = 0; + if (last_alloc) { u32 factor = map->num_stripes; @@ -1545,7 +1605,7 @@ static int btrfs_load_block_group_raid0(struct btrfs_block_group *bg, continue; if (zone_info[i].alloc_offset == WP_CONVENTIONAL) { - + has_conventional = true; zone_info[i].alloc_offset = btrfs_stripe_nr_to_offset(stripe_nr); if (stripe_index > i) @@ -1554,6 +1614,28 @@ static int btrfs_load_block_group_raid0(struct btrfs_block_group *bg, zone_info[i].alloc_offset += stripe_offset; } + /* Verification */ + if (i != 0) { + if (unlikely(prev_offset < zone_info[i].alloc_offset)) { + btrfs_err(fs_info, + "zoned: stripe position disorder found in block group %llu", + bg->start); + return -EIO; + } + + if (unlikely(has_partial && + (zone_info[i].alloc_offset & BTRFS_STRIPE_LEN_MASK))) { + btrfs_err(fs_info, + "zoned: multiple partial written stripe found in block group %llu", + bg->start); + return -EIO; + } + } + prev_offset = zone_info[i].alloc_offset; + + if ((zone_info[i].alloc_offset & BTRFS_STRIPE_LEN_MASK) != 0) + has_partial = true; + if (test_bit(0, active) != test_bit(i, active)) { if (unlikely(!btrfs_zone_activate(bg))) return -EIO; @@ -1565,6 +1647,19 @@ static int btrfs_load_block_group_raid0(struct btrfs_block_group *bg, bg->alloc_offset += zone_info[i].alloc_offset; } + /* Check if all devices stay in the same stripe row. */ + if (unlikely(zone_info[0].alloc_offset - + zone_info[map->num_stripes - 1].alloc_offset > BTRFS_STRIPE_LEN)) { + btrfs_err(fs_info, "zoned: stripe gap too large in block group %llu", bg->start); + return -EIO; + } + + if (unlikely(has_conventional && bg->alloc_offset < last_alloc)) { + btrfs_err(fs_info, "zoned: allocated extent stays beyond write pointers %llu %llu", + bg->alloc_offset, last_alloc); + return -EIO; + } + return 0; } @@ -1575,8 +1670,11 @@ static int btrfs_load_block_group_raid10(struct btrfs_block_group *bg, u64 last_alloc) { struct btrfs_fs_info *fs_info = bg->fs_info; + u64 AUTO_KFREE(raid0_allocs); u64 stripe_nr = 0, stripe_offset = 0; u32 stripe_index = 0; + bool has_partial = false, has_conventional = false; + u64 prev_offset = 0; if ((map->type & BTRFS_BLOCK_GROUP_DATA) && !fs_info->stripe_root) { btrfs_err(fs_info, "zoned: data %s needs raid-stripe-tree", @@ -1584,6 +1682,60 @@ static int btrfs_load_block_group_raid10(struct btrfs_block_group *bg, return -EINVAL; } + raid0_allocs = kcalloc(map->num_stripes / map->sub_stripes, sizeof(*raid0_allocs), + GFP_NOFS); + if (!raid0_allocs) + return -ENOMEM; + + /* + * When the last extent is removed, last_alloc can be smaller than the other write + * pointer. In that case, last_alloc should be moved to the corresponding write + * pointer position. + */ + for (int i = 0; i < map->num_stripes; i += map->sub_stripes) { + u64 alloc = zone_info[i].alloc_offset; + + for (int j = 1; j < map->sub_stripes; j++) { + int idx = i + j; + + if (zone_info[idx].alloc_offset == WP_MISSING_DEV || + zone_info[idx].alloc_offset == WP_CONVENTIONAL) + continue; + if (alloc == WP_MISSING_DEV || alloc == WP_CONVENTIONAL) { + alloc = zone_info[idx].alloc_offset; + } else if (unlikely(zone_info[idx].alloc_offset != alloc)) { + btrfs_err(fs_info, + "zoned: write pointer mismatch found in block group %llu", + bg->start); + return -EIO; + } + } + + raid0_allocs[i / map->sub_stripes] = alloc; + if (alloc == WP_CONVENTIONAL) + continue; + if (unlikely(alloc == WP_MISSING_DEV)) { + btrfs_err(fs_info, + "zoned: cannot recover write pointer of block group %llu due to missing device", + bg->start); + return -EIO; + } + + stripe_nr = alloc >> BTRFS_STRIPE_LEN_SHIFT; + stripe_offset = alloc & BTRFS_STRIPE_LEN_MASK; + if (stripe_offset == 0 && stripe_nr > 0) { + stripe_nr--; + stripe_offset = BTRFS_STRIPE_LEN; + } + + alloc = ((stripe_nr * (map->num_stripes / map->sub_stripes) + + (i / map->sub_stripes)) << + BTRFS_STRIPE_LEN_SHIFT) + stripe_offset; + last_alloc = max(last_alloc, alloc); + } + stripe_nr = 0; + stripe_offset = 0; + if (last_alloc) { u32 factor = map->num_stripes / map->sub_stripes; @@ -1593,24 +1745,51 @@ static int btrfs_load_block_group_raid10(struct btrfs_block_group *bg, } for (int i = 0; i < map->num_stripes; i++) { - if (zone_info[i].alloc_offset == WP_MISSING_DEV) - continue; + int idx = i / map->sub_stripes; - if (test_bit(0, active) != test_bit(i, active)) { - if (unlikely(!btrfs_zone_activate(bg))) - return -EIO; - } else { - if (test_bit(0, active)) - set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &bg->runtime_flags); + if (raid0_allocs[idx] == WP_CONVENTIONAL) { + has_conventional = true; + raid0_allocs[idx] = btrfs_stripe_nr_to_offset(stripe_nr); + + if (stripe_index > idx) + raid0_allocs[idx] += BTRFS_STRIPE_LEN; + else if (stripe_index == idx) + raid0_allocs[idx] += stripe_offset; } - if (zone_info[i].alloc_offset == WP_CONVENTIONAL) { - zone_info[i].alloc_offset = btrfs_stripe_nr_to_offset(stripe_nr); + if ((i % map->sub_stripes) == 0) { + /* Verification */ + if (i != 0) { + if (unlikely(prev_offset < raid0_allocs[idx])) { + btrfs_err(fs_info, + "zoned: stripe position disorder found in block group %llu", + bg->start); + return -EIO; + } - if (stripe_index > (i / map->sub_stripes)) - zone_info[i].alloc_offset += BTRFS_STRIPE_LEN; - else if (stripe_index == (i / map->sub_stripes)) - zone_info[i].alloc_offset += stripe_offset; + if (unlikely(has_partial && + (raid0_allocs[idx] & BTRFS_STRIPE_LEN_MASK))) { + btrfs_err(fs_info, + "zoned: multiple partial written stripe found in block group %llu", + bg->start); + return -EIO; + } + } + prev_offset = raid0_allocs[idx]; + + if ((raid0_allocs[idx] & BTRFS_STRIPE_LEN_MASK) != 0) + has_partial = true; + } + + if (zone_info[i].alloc_offset == WP_MISSING_DEV || + zone_info[i].alloc_offset == WP_CONVENTIONAL) + zone_info[i].alloc_offset = raid0_allocs[idx]; + + if (test_bit(0, active) != test_bit(i, active)) { + if (unlikely(!btrfs_zone_activate(bg))) + return -EIO; + } else if (test_bit(0, active)) { + set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &bg->runtime_flags); } if ((i % map->sub_stripes) == 0) { @@ -1619,6 +1798,20 @@ static int btrfs_load_block_group_raid10(struct btrfs_block_group *bg, } } + /* Check if all devices stay in the same stripe row. */ + if (unlikely(zone_info[0].alloc_offset - + zone_info[map->num_stripes - 1].alloc_offset > BTRFS_STRIPE_LEN)) { + btrfs_err(fs_info, "zoned: stripe gap too large in block group %llu", + bg->start); + return -EIO; + } + + if (unlikely(has_conventional && bg->alloc_offset < last_alloc)) { + btrfs_err(fs_info, "zoned: allocated extent stays beyond write pointers %llu %llu", + bg->alloc_offset, last_alloc); + return -EIO; + } + return 0; } @@ -1809,14 +2002,14 @@ bool btrfs_use_zone_append(struct btrfs_bio *bbio) { u64 start = (bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT); struct btrfs_inode *inode = bbio->inode; - struct btrfs_fs_info *fs_info = bbio->fs_info; + struct btrfs_fs_info *fs_info = inode->root->fs_info; struct btrfs_block_group *cache; bool ret = false; if (!btrfs_is_zoned(fs_info)) return false; - if (!inode || !is_data_inode(inode)) + if (!is_data_inode(inode)) return false; if (btrfs_op(&bbio->bio) != BTRFS_MAP_WRITE) diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index 17c5656580dd9..2b807a02d1a8a 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -15,7 +15,6 @@ #include "disk-io.h" #include "block-group.h" #include "btrfs_inode.h" -#include "fs.h" struct block_device; struct extent_buffer; diff --git a/fs/buffer.c b/fs/buffer.c index 6a8752f7bbedb..b6b477ff7b75d 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -2948,6 +2948,10 @@ bool try_to_free_buffers(struct folio *folio) if (folio_test_writeback(folio)) return false; + /* Misconfigured folio check */ + if (WARN_ON_ONCE(!folio_buffers(folio))) + return true; + if (mapping == NULL) { /* can this still happen? */ ret = drop_buffers(folio, &buffers_to_free); goto out; diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index 322ed268f14aa..261f8996abc0f 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -1000,7 +1000,8 @@ unsigned int ceph_define_write_size(struct address_space *mapping) { struct inode *inode = mapping->host; struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); - unsigned int wsize = i_blocksize(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + unsigned int wsize = ci->i_layout.stripe_unit; if (fsc->mount_options->wsize < wsize) wsize = fsc->mount_options->wsize; @@ -1373,6 +1374,7 @@ int ceph_process_folio_batch(struct address_space *mapping, rc = move_dirty_folio_in_page_array(mapping, wbc, ceph_wbc, folio); if (rc) { + rc = 0; folio_redirty_for_writepage(wbc, folio); folio_unlock(folio); break; @@ -2203,6 +2205,7 @@ int ceph_uninline_data(struct file *file) struct ceph_osd_request *req = NULL; struct ceph_cap_flush *prealloc_cf = NULL; struct folio *folio = NULL; + struct ceph_snap_context *snapc = NULL; u64 inline_version = CEPH_INLINE_NONE; struct page *pages[1]; int err = 0; @@ -2230,6 +2233,24 @@ int ceph_uninline_data(struct file *file) if (inline_version == 1) /* initial version, no data */ goto out_uninline; + down_read(&fsc->mdsc->snap_rwsem); + spin_lock(&ci->i_ceph_lock); + if (__ceph_have_pending_cap_snap(ci)) { + struct ceph_cap_snap *capsnap = + list_last_entry(&ci->i_cap_snaps, + struct ceph_cap_snap, + ci_item); + snapc = ceph_get_snap_context(capsnap->context); + } else { + if (!ci->i_head_snapc) { + ci->i_head_snapc = ceph_get_snap_context( + ci->i_snap_realm->cached_context); + } + snapc = ceph_get_snap_context(ci->i_head_snapc); + } + spin_unlock(&ci->i_ceph_lock); + up_read(&fsc->mdsc->snap_rwsem); + folio = read_mapping_folio(inode->i_mapping, 0, file); if (IS_ERR(folio)) { err = PTR_ERR(folio); @@ -2245,7 +2266,7 @@ int ceph_uninline_data(struct file *file) req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, ceph_vino(inode), 0, &len, 0, 1, CEPH_OSD_OP_CREATE, CEPH_OSD_FLAG_WRITE, - NULL, 0, 0, false); + snapc, 0, 0, false); if (IS_ERR(req)) { err = PTR_ERR(req); goto out_unlock; @@ -2261,7 +2282,7 @@ int ceph_uninline_data(struct file *file) req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, ceph_vino(inode), 0, &len, 1, 3, CEPH_OSD_OP_WRITE, CEPH_OSD_FLAG_WRITE, - NULL, ci->i_truncate_seq, + snapc, ci->i_truncate_seq, ci->i_truncate_size, false); if (IS_ERR(req)) { err = PTR_ERR(req); @@ -2324,6 +2345,7 @@ int ceph_uninline_data(struct file *file) folio_put(folio); } out: + ceph_put_snap_context(snapc); ceph_free_cap_flush(prealloc_cf); doutc(cl, "%llx.%llx inline_version %llu = %d\n", ceph_vinop(inode), inline_version, err); diff --git a/fs/ceph/file.c b/fs/ceph/file.c index 99b30f784ee24..f43a42909e7cf 100644 --- a/fs/ceph/file.c +++ b/fs/ceph/file.c @@ -2568,6 +2568,7 @@ static int ceph_zero_partial_object(struct inode *inode, struct ceph_inode_info *ci = ceph_inode(inode); struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); struct ceph_osd_request *req; + struct ceph_snap_context *snapc; int ret = 0; loff_t zero = 0; int op; @@ -2582,12 +2583,25 @@ static int ceph_zero_partial_object(struct inode *inode, op = CEPH_OSD_OP_ZERO; } + spin_lock(&ci->i_ceph_lock); + if (__ceph_have_pending_cap_snap(ci)) { + struct ceph_cap_snap *capsnap = + list_last_entry(&ci->i_cap_snaps, + struct ceph_cap_snap, + ci_item); + snapc = ceph_get_snap_context(capsnap->context); + } else { + BUG_ON(!ci->i_head_snapc); + snapc = ceph_get_snap_context(ci->i_head_snapc); + } + spin_unlock(&ci->i_ceph_lock); + req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, ceph_vino(inode), offset, length, 0, 1, op, CEPH_OSD_FLAG_WRITE, - NULL, 0, 0, false); + snapc, 0, 0, false); if (IS_ERR(req)) { ret = PTR_ERR(req); goto out; @@ -2601,6 +2615,7 @@ static int ceph_zero_partial_object(struct inode *inode, ceph_osdc_put_request(req); out: + ceph_put_snap_context(snapc); return ret; } diff --git a/fs/dlm/lock.c b/fs/dlm/lock.c index be938fdf17d96..a393ecaf3442a 100644 --- a/fs/dlm/lock.c +++ b/fs/dlm/lock.c @@ -626,7 +626,8 @@ int dlm_search_rsb_tree(struct rhashtable *rhash, const void *name, int len, struct dlm_rsb **r_ret) { char key[DLM_RESNAME_MAXLEN] = {}; - + if (len > DLM_RESNAME_MAXLEN) + return -EINVAL; memcpy(key, name, len); *r_ret = rhashtable_lookup_fast(rhash, &key, dlm_rhash_rsb_params); if (*r_ret) @@ -5014,25 +5015,8 @@ void dlm_receive_buffer(const union dlm_packet *p, int nodeid) static void recover_convert_waiter(struct dlm_ls *ls, struct dlm_lkb *lkb, struct dlm_message *ms_local) { - if (middle_conversion(lkb)) { - log_rinfo(ls, "%s %x middle convert in progress", __func__, - lkb->lkb_id); - - /* We sent this lock to the new master. The new master will - * tell us when it's granted. We no longer need a reply, so - * use a fake reply to put the lkb into the right state. - */ - hold_lkb(lkb); - memset(ms_local, 0, sizeof(struct dlm_message)); - ms_local->m_type = cpu_to_le32(DLM_MSG_CONVERT_REPLY); - ms_local->m_result = cpu_to_le32(to_dlm_errno(-EINPROGRESS)); - ms_local->m_header.h_nodeid = cpu_to_le32(lkb->lkb_nodeid); - _receive_convert_reply(lkb, ms_local, true); - unhold_lkb(lkb); - - } else if (lkb->lkb_rqmode >= lkb->lkb_grmode) { + if (middle_conversion(lkb) || lkb->lkb_rqmode >= lkb->lkb_grmode) set_bit(DLM_IFL_RESEND_BIT, &lkb->lkb_iflags); - } /* lkb->lkb_rqmode < lkb->lkb_grmode shouldn't happen since down conversions are async; there's no reply from the remote master */ diff --git a/fs/erofs/fileio.c b/fs/erofs/fileio.c index a47c6bab98ff9..5b77ee8cc99f4 100644 --- a/fs/erofs/fileio.c +++ b/fs/erofs/fileio.c @@ -25,23 +25,19 @@ static void erofs_fileio_ki_complete(struct kiocb *iocb, long ret) container_of(iocb, struct erofs_fileio_rq, iocb); struct folio_iter fi; - if (ret > 0) { - if (ret != rq->bio.bi_iter.bi_size) { - bio_advance(&rq->bio, ret); - zero_fill_bio(&rq->bio); - } - ret = 0; + if (ret >= 0 && ret != rq->bio.bi_iter.bi_size) { + bio_advance(&rq->bio, ret); + zero_fill_bio(&rq->bio); } - if (rq->bio.bi_end_io) { - if (ret < 0 && !rq->bio.bi_status) - rq->bio.bi_status = errno_to_blk_status(ret); - rq->bio.bi_end_io(&rq->bio); - } else { + if (!rq->bio.bi_end_io) { bio_for_each_folio_all(fi, &rq->bio) { DBG_BUGON(folio_test_uptodate(fi.folio)); - erofs_onlinefolio_end(fi.folio, ret, false); + erofs_onlinefolio_end(fi.folio, ret < 0, false); } + } else if (ret < 0 && !rq->bio.bi_status) { + rq->bio.bi_status = errno_to_blk_status(ret); } + bio_endio(&rq->bio); bio_uninit(&rq->bio); if (refcount_dec_and_test(&rq->ref)) kfree(rq); @@ -51,7 +47,7 @@ static void erofs_fileio_rq_submit(struct erofs_fileio_rq *rq) { const struct cred *old_cred; struct iov_iter iter; - int ret; + ssize_t ret; if (!rq) return; diff --git a/fs/erofs/fscache.c b/fs/erofs/fscache.c index 362acf828279f..7a346e20f7b7a 100644 --- a/fs/erofs/fscache.c +++ b/fs/erofs/fscache.c @@ -185,7 +185,7 @@ static void erofs_fscache_bio_endio(void *priv, ssize_t transferred_or_error) if (IS_ERR_VALUE(transferred_or_error)) io->bio.bi_status = errno_to_blk_status(transferred_or_error); - io->bio.bi_end_io(&io->bio); + bio_endio(&io->bio); BUILD_BUG_ON(offsetof(struct erofs_fscache_bio, io) != 0); erofs_fscache_io_put(&io->io); } @@ -216,7 +216,7 @@ void erofs_fscache_submit_bio(struct bio *bio) if (!ret) return; bio->bi_status = errno_to_blk_status(ret); - bio->bi_end_io(bio); + bio_endio(bio); } static int erofs_fscache_meta_read_folio(struct file *data, struct folio *folio) diff --git a/fs/erofs/super.c b/fs/erofs/super.c index 5136cda5972a9..ee37628ec99fb 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -330,12 +330,13 @@ static int erofs_read_superblock(struct super_block *sb) } sbi->packed_nid = le64_to_cpu(dsb->packed_nid); if (erofs_sb_has_metabox(sbi)) { + ret = -EFSCORRUPTED; if (sbi->sb_size <= offsetof(struct erofs_super_block, metabox_nid)) - return -EFSCORRUPTED; + goto out; sbi->metabox_nid = le64_to_cpu(dsb->metabox_nid); if (sbi->metabox_nid & BIT_ULL(EROFS_DIRENT_NID_METABOX_BIT)) - return -EFSCORRUPTED; /* self-loop detection */ + goto out; /* self-loop detection */ } sbi->inos = le64_to_cpu(dsb->inos); @@ -346,8 +347,10 @@ static int erofs_read_superblock(struct super_block *sb) if (dsb->volume_name[0]) { sbi->volume_name = kstrndup(dsb->volume_name, sizeof(dsb->volume_name), GFP_KERNEL); - if (!sbi->volume_name) - return -ENOMEM; + if (!sbi->volume_name) { + ret = -ENOMEM; + goto out; + } } /* parse on-disk compression configurations */ diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index 683703aee5ef2..98e44570841a3 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -805,14 +805,26 @@ static int z_erofs_pcluster_begin(struct z_erofs_frontend *fe) struct erofs_map_blocks *map = &fe->map; struct super_block *sb = fe->inode->i_sb; struct z_erofs_pcluster *pcl = NULL; - void *ptr; + void *ptr = NULL; int ret; DBG_BUGON(fe->pcl); /* must be Z_EROFS_PCLUSTER_TAIL or pointed to previous pcluster */ DBG_BUGON(!fe->head); - if (!(map->m_flags & EROFS_MAP_META)) { + if (map->m_flags & EROFS_MAP_META) { + ret = erofs_init_metabuf(&map->buf, sb, + erofs_inode_in_metabox(fe->inode)); + if (ret) + return ret; + ptr = erofs_bread(&map->buf, map->m_pa, false); + if (IS_ERR(ptr)) { + erofs_err(sb, "failed to read inline data %pe @ pa %llu of nid %llu", + ptr, map->m_pa, EROFS_I(fe->inode)->nid); + return PTR_ERR(ptr); + } + ptr = map->buf.page; + } else { while (1) { rcu_read_lock(); pcl = xa_load(&EROFS_SB(sb)->managed_pslots, map->m_pa); @@ -852,18 +864,8 @@ static int z_erofs_pcluster_begin(struct z_erofs_frontend *fe) /* bind cache first when cached decompression is preferred */ z_erofs_bind_cache(fe); } else { - ret = erofs_init_metabuf(&map->buf, sb, - erofs_inode_in_metabox(fe->inode)); - if (ret) - return ret; - ptr = erofs_bread(&map->buf, map->m_pa, false); - if (IS_ERR(ptr)) { - ret = PTR_ERR(ptr); - erofs_err(sb, "failed to get inline folio %d", ret); - return ret; - } - folio_get(page_folio(map->buf.page)); - WRITE_ONCE(fe->pcl->compressed_bvecs[0].page, map->buf.page); + folio_get(page_folio((struct page *)ptr)); + WRITE_ONCE(fe->pcl->compressed_bvecs[0].page, ptr); fe->pcl->pageofs_in = map->m_pa & ~PAGE_MASK; fe->mode = Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE; } diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c index c8d8e129eb4ba..30775502b56da 100644 --- a/fs/erofs/zmap.c +++ b/fs/erofs/zmap.c @@ -513,6 +513,7 @@ static int z_erofs_map_blocks_ext(struct inode *inode, unsigned int recsz = z_erofs_extent_recsize(vi->z_advise); erofs_off_t pos = round_up(Z_EROFS_MAP_HEADER_END(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize), recsz); + unsigned int bmask = sb->s_blocksize - 1; bool in_mbox = erofs_inode_in_metabox(inode); erofs_off_t lend = inode->i_size; erofs_off_t l, r, mid, pa, la, lstart; @@ -596,17 +597,17 @@ static int z_erofs_map_blocks_ext(struct inode *inode, map->m_flags |= EROFS_MAP_MAPPED | EROFS_MAP_FULL_MAPPED | EROFS_MAP_ENCODED; fmt = map->m_plen >> Z_EROFS_EXTENT_PLEN_FMT_BIT; + if (map->m_plen & Z_EROFS_EXTENT_PLEN_PARTIAL) + map->m_flags |= EROFS_MAP_PARTIAL_REF; + map->m_plen &= Z_EROFS_EXTENT_PLEN_MASK; if (fmt) map->m_algorithmformat = fmt - 1; - else if (interlaced && !erofs_blkoff(sb, map->m_pa)) + else if (interlaced && !((map->m_pa | map->m_plen) & bmask)) map->m_algorithmformat = Z_EROFS_COMPRESSION_INTERLACED; else map->m_algorithmformat = Z_EROFS_COMPRESSION_SHIFTED; - if (map->m_plen & Z_EROFS_EXTENT_PLEN_PARTIAL) - map->m_flags |= EROFS_MAP_PARTIAL_REF; - map->m_plen &= Z_EROFS_EXTENT_PLEN_MASK; } } map->m_llen = lend - map->m_la; diff --git a/fs/eventpoll.c b/fs/eventpoll.c index ee7c4b683ec3d..bcc7dcbefc419 100644 --- a/fs/eventpoll.c +++ b/fs/eventpoll.c @@ -2061,7 +2061,8 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, * @ep: the &struct eventpoll to be currently checked. * @depth: Current depth of the path being checked. * - * Return: depth of the subtree, or INT_MAX if we found a loop or went too deep. + * Return: depth of the subtree, or a value bigger than EP_MAX_NESTS if we found + * a loop or went too deep. */ static int ep_loop_check_proc(struct eventpoll *ep, int depth) { @@ -2080,7 +2081,7 @@ static int ep_loop_check_proc(struct eventpoll *ep, int depth) struct eventpoll *ep_tovisit; ep_tovisit = epi->ffd.file->private_data; if (ep_tovisit == inserting_into || depth > EP_MAX_NESTS) - result = INT_MAX; + result = EP_MAX_NESTS+1; else result = max(result, ep_loop_check_proc(ep_tovisit, depth + 1) + 1); if (result > EP_MAX_NESTS) diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 57087da6c7bee..933297251f66a 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -1771,6 +1771,10 @@ struct ext4_sb_info { * Main fast commit lock. This lock protects accesses to the * following fields: * ei->i_fc_list, s_fc_dentry_q, s_fc_q, s_fc_bytes, s_fc_bh. + * + * s_fc_lock can be taken from reclaim context (inode eviction) and is + * thus reclaim unsafe. Use ext4_fc_lock()/ext4_fc_unlock() helpers + * when acquiring / releasing the lock. */ struct mutex s_fc_lock; struct buffer_head *s_fc_bh; @@ -1815,6 +1819,18 @@ static inline void ext4_writepages_up_write(struct super_block *sb, int ctx) percpu_up_write(&EXT4_SB(sb)->s_writepages_rwsem); } +static inline int ext4_fc_lock(struct super_block *sb) +{ + mutex_lock(&EXT4_SB(sb)->s_fc_lock); + return memalloc_nofs_save(); +} + +static inline void ext4_fc_unlock(struct super_block *sb, int ctx) +{ + memalloc_nofs_restore(ctx); + mutex_unlock(&EXT4_SB(sb)->s_fc_lock); +} + static inline int ext4_valid_inum(struct super_block *sb, unsigned long ino) { return ino == EXT4_ROOT_INO || diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c index ca5499e9412b0..3ff8dcdd80ce9 100644 --- a/fs/ext4/extents.c +++ b/fs/ext4/extents.c @@ -43,8 +43,13 @@ #define EXT4_EXT_MARK_UNWRIT1 0x2 /* mark first half unwritten */ #define EXT4_EXT_MARK_UNWRIT2 0x4 /* mark second half unwritten */ -#define EXT4_EXT_DATA_VALID1 0x8 /* first half contains valid data */ -#define EXT4_EXT_DATA_VALID2 0x10 /* second half contains valid data */ +/* first half contains valid data */ +#define EXT4_EXT_DATA_ENTIRE_VALID1 0x8 /* has entirely valid data */ +#define EXT4_EXT_DATA_PARTIAL_VALID1 0x10 /* has partially valid data */ +#define EXT4_EXT_DATA_VALID1 (EXT4_EXT_DATA_ENTIRE_VALID1 | \ + EXT4_EXT_DATA_PARTIAL_VALID1) + +#define EXT4_EXT_DATA_VALID2 0x20 /* second half contains valid data */ static __le32 ext4_extent_block_csum(struct inode *inode, struct ext4_extent_header *eh) @@ -3190,8 +3195,12 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, unsigned int ee_len, depth; int err = 0; - BUG_ON((split_flag & (EXT4_EXT_DATA_VALID1 | EXT4_EXT_DATA_VALID2)) == - (EXT4_EXT_DATA_VALID1 | EXT4_EXT_DATA_VALID2)); + BUG_ON((split_flag & EXT4_EXT_DATA_VALID1) == EXT4_EXT_DATA_VALID1); + BUG_ON((split_flag & EXT4_EXT_DATA_VALID1) && + (split_flag & EXT4_EXT_DATA_VALID2)); + + /* Do not cache extents that are in the process of being modified. */ + flags |= EXT4_EX_NOCACHE; ext_debug(inode, "logical block %llu\n", (unsigned long long)split); @@ -3258,7 +3267,7 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, err = PTR_ERR(path); if (err != -ENOSPC && err != -EDQUOT && err != -ENOMEM) - return path; + goto out_path; /* * Get a new path to try to zeroout or fix the extent length. @@ -3272,7 +3281,7 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, if (IS_ERR(path)) { EXT4_ERROR_INODE(inode, "Failed split extent on %u, err %ld", split, PTR_ERR(path)); - return path; + goto out_path; } depth = ext_depth(inode); ex = path[depth].p_ext; @@ -3304,6 +3313,23 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, } if (!err) { + /* + * The first half contains partially valid data, the + * splitting of this extent has not been completed, fix + * extent length and ext4_split_extent() split will the + * first half again. + */ + if (split_flag & EXT4_EXT_DATA_PARTIAL_VALID1) { + /* + * Drop extent cache to prevent stale unwritten + * extents remaining after zeroing out. + */ + ext4_es_remove_extent(inode, + le32_to_cpu(zero_ex.ee_block), + ext4_ext_get_actual_len(&zero_ex)); + goto fix_extent_len; + } + /* update the extent length and mark as initialized */ ex->ee_len = cpu_to_le16(ee_len); ext4_ext_try_to_merge(handle, inode, path, ex); @@ -3332,6 +3358,10 @@ static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, ext4_free_ext_path(path); path = ERR_PTR(err); } +out_path: + if (IS_ERR(path)) + /* Remove all remaining potentially stale extents. */ + ext4_es_remove_extent(inode, ee_block, ee_len); ext4_ext_show_leaf(inode, path); return path; } @@ -3366,6 +3396,9 @@ static struct ext4_ext_path *ext4_split_extent(handle_t *handle, ee_len = ext4_ext_get_actual_len(ex); unwritten = ext4_ext_is_unwritten(ex); + /* Do not cache extents that are in the process of being modified. */ + flags |= EXT4_EX_NOCACHE; + if (map->m_lblk + map->m_len < ee_block + ee_len) { split_flag1 = split_flag & EXT4_EXT_MAY_ZEROOUT; flags1 = flags | EXT4_GET_BLOCKS_PRE_IO; @@ -3373,7 +3406,9 @@ static struct ext4_ext_path *ext4_split_extent(handle_t *handle, split_flag1 |= EXT4_EXT_MARK_UNWRIT1 | EXT4_EXT_MARK_UNWRIT2; if (split_flag & EXT4_EXT_DATA_VALID2) - split_flag1 |= EXT4_EXT_DATA_VALID1; + split_flag1 |= map->m_lblk > ee_block ? + EXT4_EXT_DATA_PARTIAL_VALID1 : + EXT4_EXT_DATA_ENTIRE_VALID1; path = ext4_split_extent_at(handle, inode, path, map->m_lblk + map->m_len, split_flag1, flags1); if (IS_ERR(path)) @@ -3721,10 +3756,6 @@ static struct ext4_ext_path *ext4_split_convert_extents(handle_t *handle, >> inode->i_sb->s_blocksize_bits; if (eof_block < map->m_lblk + map->m_len) eof_block = map->m_lblk + map->m_len; - /* - * It is safe to convert extent to initialized via explicit - * zeroout only if extent is fully inside i_size or new_size. - */ depth = ext_depth(inode); ex = path[depth].p_ext; ee_block = le32_to_cpu(ex->ee_block); @@ -3732,12 +3763,20 @@ static struct ext4_ext_path *ext4_split_convert_extents(handle_t *handle, /* Convert to unwritten */ if (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN) { - split_flag |= EXT4_EXT_DATA_VALID1; - /* Convert to initialized */ - } else if (flags & EXT4_GET_BLOCKS_CONVERT) { + split_flag |= EXT4_EXT_DATA_ENTIRE_VALID1; + /* Split the existing unwritten extent */ + } else if (flags & (EXT4_GET_BLOCKS_UNWRIT_EXT | + EXT4_GET_BLOCKS_CONVERT)) { + /* + * It is safe to convert extent to initialized via explicit + * zeroout only if extent is fully inside i_size or new_size. + */ split_flag |= ee_block + ee_len <= eof_block ? EXT4_EXT_MAY_ZEROOUT : 0; - split_flag |= (EXT4_EXT_MARK_UNWRIT2 | EXT4_EXT_DATA_VALID2); + split_flag |= EXT4_EXT_MARK_UNWRIT2; + /* Convert to initialized */ + if (flags & EXT4_GET_BLOCKS_CONVERT) + split_flag |= EXT4_EXT_DATA_VALID2; } flags |= EXT4_GET_BLOCKS_PRE_IO; return ext4_split_extent(handle, inode, path, map, split_flag, flags, @@ -3770,6 +3809,8 @@ ext4_convert_unwritten_extents_endio(handle_t *handle, struct inode *inode, * illegal. */ if (ee_block != map->m_lblk || ee_len > map->m_len) { + int flags = EXT4_GET_BLOCKS_CONVERT | + EXT4_GET_BLOCKS_METADATA_NOFAIL; #ifdef CONFIG_EXT4_DEBUG ext4_warning(inode->i_sb, "Inode (%ld) finished: extent logical block %llu," " len %u; IO logical block %llu, len %u", @@ -3777,7 +3818,7 @@ ext4_convert_unwritten_extents_endio(handle_t *handle, struct inode *inode, (unsigned long long)map->m_lblk, map->m_len); #endif path = ext4_split_convert_extents(handle, inode, map, path, - EXT4_GET_BLOCKS_CONVERT, NULL); + flags, NULL); if (IS_ERR(path)) return path; @@ -3816,6 +3857,7 @@ static struct ext4_ext_path * convert_initialized_extent(handle_t *handle, struct inode *inode, struct ext4_map_blocks *map, struct ext4_ext_path *path, + int flags, unsigned int *allocated) { struct ext4_extent *ex; @@ -3841,11 +3883,11 @@ convert_initialized_extent(handle_t *handle, struct inode *inode, if (ee_block != map->m_lblk || ee_len > map->m_len) { path = ext4_split_convert_extents(handle, inode, map, path, - EXT4_GET_BLOCKS_CONVERT_UNWRITTEN, NULL); + flags, NULL); if (IS_ERR(path)) return path; - path = ext4_find_extent(inode, map->m_lblk, path, 0); + path = ext4_find_extent(inode, map->m_lblk, path, flags); if (IS_ERR(path)) return path; depth = ext_depth(inode); @@ -3913,7 +3955,7 @@ ext4_ext_handle_unwritten_extents(handle_t *handle, struct inode *inode, /* get_block() before submitting IO, split the extent */ if (flags & EXT4_GET_BLOCKS_PRE_IO) { path = ext4_split_convert_extents(handle, inode, map, path, - flags | EXT4_GET_BLOCKS_CONVERT, allocated); + flags, allocated); if (IS_ERR(path)) return path; /* @@ -4257,7 +4299,7 @@ int ext4_ext_map_blocks(handle_t *handle, struct inode *inode, if ((!ext4_ext_is_unwritten(ex)) && (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN)) { path = convert_initialized_extent(handle, - inode, map, path, &allocated); + inode, map, path, flags, &allocated); if (IS_ERR(path)) err = PTR_ERR(path); goto out; @@ -5375,7 +5417,8 @@ ext4_ext_shift_extents(struct inode *inode, handle_t *handle, if (!extent) { EXT4_ERROR_INODE(inode, "unexpected hole at %lu", (unsigned long) *iterator); - return -EFSCORRUPTED; + ret = -EFSCORRUPTED; + goto out; } if (SHIFT == SHIFT_LEFT && *iterator > le32_to_cpu(extent->ee_block)) { diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c index fa66b08de9994..5bd57d7f921b9 100644 --- a/fs/ext4/fast_commit.c +++ b/fs/ext4/fast_commit.c @@ -231,16 +231,16 @@ static bool ext4_fc_disabled(struct super_block *sb) void ext4_fc_del(struct inode *inode) { struct ext4_inode_info *ei = EXT4_I(inode); - struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); struct ext4_fc_dentry_update *fc_dentry; wait_queue_head_t *wq; + int alloc_ctx; if (ext4_fc_disabled(inode->i_sb)) return; - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(inode->i_sb); if (list_empty(&ei->i_fc_list) && list_empty(&ei->i_fc_dilist)) { - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(inode->i_sb, alloc_ctx); return; } @@ -275,9 +275,9 @@ void ext4_fc_del(struct inode *inode) #endif prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE); if (ext4_test_inode_state(inode, EXT4_STATE_FC_FLUSHING_DATA)) { - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(inode->i_sb, alloc_ctx); schedule(); - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(inode->i_sb); } finish_wait(wq, &wait.wq_entry); } @@ -288,7 +288,7 @@ void ext4_fc_del(struct inode *inode) * dentry create references, since it is not needed to log it anyways. */ if (list_empty(&ei->i_fc_dilist)) { - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(inode->i_sb, alloc_ctx); return; } @@ -298,7 +298,7 @@ void ext4_fc_del(struct inode *inode) list_del_init(&fc_dentry->fcd_dilist); WARN_ON(!list_empty(&ei->i_fc_dilist)); - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(inode->i_sb, alloc_ctx); release_dentry_name_snapshot(&fc_dentry->fcd_name); kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry); @@ -315,6 +315,7 @@ void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handl tid_t tid; bool has_transaction = true; bool is_ineligible; + int alloc_ctx; if (ext4_fc_disabled(sb)) return; @@ -329,12 +330,12 @@ void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handl has_transaction = false; read_unlock(&sbi->s_journal->j_state_lock); } - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); is_ineligible = ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); if (has_transaction && (!is_ineligible || tid_gt(tid, sbi->s_fc_ineligible_tid))) sbi->s_fc_ineligible_tid = tid; ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); WARN_ON(reason >= EXT4_FC_REASON_MAX); sbi->s_fc_stats.fc_ineligible_reason_count[reason]++; } @@ -358,6 +359,7 @@ static int ext4_fc_track_template( struct ext4_inode_info *ei = EXT4_I(inode); struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); tid_t tid = 0; + int alloc_ctx; int ret; tid = handle->h_transaction->t_tid; @@ -373,14 +375,14 @@ static int ext4_fc_track_template( if (!enqueue) return ret; - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(inode->i_sb); if (list_empty(&EXT4_I(inode)->i_fc_list)) list_add_tail(&EXT4_I(inode)->i_fc_list, (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING || sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) ? &sbi->s_fc_q[FC_Q_STAGING] : &sbi->s_fc_q[FC_Q_MAIN]); - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(inode->i_sb, alloc_ctx); return ret; } @@ -402,6 +404,7 @@ static int __track_dentry_update(handle_t *handle, struct inode *inode, struct inode *dir = dentry->d_parent->d_inode; struct super_block *sb = inode->i_sb; struct ext4_sb_info *sbi = EXT4_SB(sb); + int alloc_ctx; spin_unlock(&ei->i_fc_lock); @@ -425,7 +428,7 @@ static int __track_dentry_update(handle_t *handle, struct inode *inode, take_dentry_name_snapshot(&node->fcd_name, dentry); INIT_LIST_HEAD(&node->fcd_dilist); INIT_LIST_HEAD(&node->fcd_list); - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); if (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING || sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) list_add_tail(&node->fcd_list, @@ -446,7 +449,7 @@ static int __track_dentry_update(handle_t *handle, struct inode *inode, WARN_ON(!list_empty(&ei->i_fc_dilist)); list_add_tail(&node->fcd_dilist, &ei->i_fc_dilist); } - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); spin_lock(&ei->i_fc_lock); return 0; @@ -1046,18 +1049,19 @@ static int ext4_fc_perform_commit(journal_t *journal) struct blk_plug plug; int ret = 0; u32 crc = 0; + int alloc_ctx; /* * Step 1: Mark all inodes on s_fc_q[MAIN] with * EXT4_STATE_FC_FLUSHING_DATA. This prevents these inodes from being * freed until the data flush is over. */ - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { ext4_set_inode_state(&iter->vfs_inode, EXT4_STATE_FC_FLUSHING_DATA); } - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); /* Step 2: Flush data for all the eligible inodes. */ ret = ext4_fc_flush_data(journal); @@ -1067,7 +1071,7 @@ static int ext4_fc_perform_commit(journal_t *journal) * any error from step 2. This ensures that waiters waiting on * EXT4_STATE_FC_FLUSHING_DATA can resume. */ - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { ext4_clear_inode_state(&iter->vfs_inode, EXT4_STATE_FC_FLUSHING_DATA); @@ -1084,7 +1088,7 @@ static int ext4_fc_perform_commit(journal_t *journal) * prepare_to_wait() in ext4_fc_del(). */ smp_mb(); - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); /* * If we encountered error in Step 2, return it now after clearing @@ -1101,12 +1105,12 @@ static int ext4_fc_perform_commit(journal_t *journal) * previous handles are now drained. We now mark the inodes on the * commit queue as being committed. */ - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { ext4_set_inode_state(&iter->vfs_inode, EXT4_STATE_FC_COMMITTING); } - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); jbd2_journal_unlock_updates(journal); /* @@ -1117,6 +1121,7 @@ static int ext4_fc_perform_commit(journal_t *journal) blkdev_issue_flush(journal->j_fs_dev); blk_start_plug(&plug); + alloc_ctx = ext4_fc_lock(sb); /* Step 6: Write fast commit blocks to disk. */ if (sbi->s_fc_bytes == 0) { /* @@ -1134,7 +1139,6 @@ static int ext4_fc_perform_commit(journal_t *journal) } /* Step 6.2: Now write all the dentry updates. */ - mutex_lock(&sbi->s_fc_lock); ret = ext4_fc_commit_dentry_updates(journal, &crc); if (ret) goto out; @@ -1156,7 +1160,7 @@ static int ext4_fc_perform_commit(journal_t *journal) ret = ext4_fc_write_tail(sb, crc); out: - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); blk_finish_plug(&plug); return ret; } @@ -1290,6 +1294,7 @@ static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid) struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_inode_info *ei; struct ext4_fc_dentry_update *fc_dentry; + int alloc_ctx; if (full && sbi->s_fc_bh) sbi->s_fc_bh = NULL; @@ -1297,7 +1302,7 @@ static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid) trace_ext4_fc_cleanup(journal, full, tid); jbd2_fc_release_bufs(journal); - mutex_lock(&sbi->s_fc_lock); + alloc_ctx = ext4_fc_lock(sb); while (!list_empty(&sbi->s_fc_q[FC_Q_MAIN])) { ei = list_first_entry(&sbi->s_fc_q[FC_Q_MAIN], struct ext4_inode_info, @@ -1356,7 +1361,7 @@ static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid) if (full) sbi->s_fc_bytes = 0; - mutex_unlock(&sbi->s_fc_lock); + ext4_fc_unlock(sb, alloc_ctx); trace_ext4_fc_stats(sb); } diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c index 3dec26c939fde..acc28aa5744b1 100644 --- a/fs/ext4/ioctl.c +++ b/fs/ext4/ioctl.c @@ -966,6 +966,7 @@ static long ext4_ioctl_group_add(struct file *file, err = ext4_group_add(sb, input); if (EXT4_SB(sb)->s_journal) { + ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_RESIZE, NULL); jbd2_journal_lock_updates(EXT4_SB(sb)->s_journal); err2 = jbd2_journal_flush(EXT4_SB(sb)->s_journal, 0); jbd2_journal_unlock_updates(EXT4_SB(sb)->s_journal); @@ -1611,6 +1612,8 @@ static long __ext4_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) err = ext4_group_extend(sb, EXT4_SB(sb)->s_es, n_blocks_count); if (EXT4_SB(sb)->s_journal) { + ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_RESIZE, + NULL); jbd2_journal_lock_updates(EXT4_SB(sb)->s_journal); err2 = jbd2_journal_flush(EXT4_SB(sb)->s_journal, 0); jbd2_journal_unlock_updates(EXT4_SB(sb)->s_journal); diff --git a/fs/ext4/mballoc-test.c b/fs/ext4/mballoc-test.c index a9416b20ff64c..4abb40d4561ce 100644 --- a/fs/ext4/mballoc-test.c +++ b/fs/ext4/mballoc-test.c @@ -567,7 +567,7 @@ test_mark_diskspace_used_range(struct kunit *test, bitmap = mbt_ctx_bitmap(sb, TEST_GOAL_GROUP); memset(bitmap, 0, sb->s_blocksize); - ret = ext4_mb_mark_diskspace_used(ac, NULL, 0); + ret = ext4_mb_mark_diskspace_used(ac, NULL); KUNIT_ASSERT_EQ(test, ret, 0); max = EXT4_CLUSTERS_PER_GROUP(sb); diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index 65335248825ce..412289e5c0afe 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -892,6 +892,21 @@ mb_update_avg_fragment_size(struct super_block *sb, struct ext4_group_info *grp) } } +static ext4_group_t ext4_get_allocation_groups_count( + struct ext4_allocation_context *ac) +{ + ext4_group_t ngroups = ext4_get_groups_count(ac->ac_sb); + + /* non-extent files are limited to low blocks/groups */ + if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))) + ngroups = EXT4_SB(ac->ac_sb)->s_blockfile_groups; + + /* Pairs with smp_wmb() in ext4_update_super() */ + smp_rmb(); + + return ngroups; +} + static int ext4_mb_scan_groups_xa_range(struct ext4_allocation_context *ac, struct xarray *xa, ext4_group_t start, ext4_group_t end) @@ -899,7 +914,7 @@ static int ext4_mb_scan_groups_xa_range(struct ext4_allocation_context *ac, struct super_block *sb = ac->ac_sb; struct ext4_sb_info *sbi = EXT4_SB(sb); enum criteria cr = ac->ac_criteria; - ext4_group_t ngroups = ext4_get_groups_count(sb); + ext4_group_t ngroups = ext4_get_allocation_groups_count(ac); unsigned long group = start; struct ext4_group_info *grp; @@ -951,7 +966,7 @@ static int ext4_mb_scan_groups_p2_aligned(struct ext4_allocation_context *ac, ext4_group_t start, end; start = group; - end = ext4_get_groups_count(ac->ac_sb); + end = ext4_get_allocation_groups_count(ac); wrap_around: for (i = ac->ac_2order; i < MB_NUM_ORDERS(ac->ac_sb); i++) { ret = ext4_mb_scan_groups_largest_free_order_range(ac, i, @@ -1001,7 +1016,7 @@ static int ext4_mb_scan_groups_goal_fast(struct ext4_allocation_context *ac, ext4_group_t start, end; start = group; - end = ext4_get_groups_count(ac->ac_sb); + end = ext4_get_allocation_groups_count(ac); wrap_around: i = mb_avg_fragment_size_order(ac->ac_sb, ac->ac_g_ex.fe_len); for (; i < MB_NUM_ORDERS(ac->ac_sb); i++) { @@ -1083,7 +1098,7 @@ static int ext4_mb_scan_groups_best_avail(struct ext4_allocation_context *ac, min_order = fls(ac->ac_o_ex.fe_len); start = group; - end = ext4_get_groups_count(ac->ac_sb); + end = ext4_get_allocation_groups_count(ac); wrap_around: for (i = order; i >= min_order; i--) { int frag_order; @@ -1133,8 +1148,6 @@ static inline int should_optimize_scan(struct ext4_allocation_context *ac) return 0; if (ac->ac_criteria >= CR_GOAL_LEN_SLOW) return 0; - if (!ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) - return 0; return 1; } @@ -1182,11 +1195,7 @@ static int ext4_mb_scan_groups(struct ext4_allocation_context *ac) int ret = 0; ext4_group_t start; struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); - ext4_group_t ngroups = ext4_get_groups_count(ac->ac_sb); - - /* non-extent files are limited to low blocks/groups */ - if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))) - ngroups = sbi->s_blockfile_groups; + ext4_group_t ngroups = ext4_get_allocation_groups_count(ac); /* searching for the right group start from the goal value specified */ start = ac->ac_g_ex.fe_group; @@ -1712,16 +1721,17 @@ ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, /* Avoid locking the folio in the fast path ... */ folio = __filemap_get_folio(inode->i_mapping, pnum, FGP_ACCESSED, 0); - if (IS_ERR(folio) || !folio_test_uptodate(folio)) { + if (IS_ERR(folio) || !folio_test_uptodate(folio) || folio_test_locked(folio)) { + /* + * folio_test_locked is employed to detect ongoing folio + * migrations, since concurrent migrations can lead to + * bitmap inconsistency. And if we are not uptodate that + * implies somebody just created the folio but is yet to + * initialize it. We can drop the folio reference and + * try to get the folio with lock in both cases to avoid + * concurrency. + */ if (!IS_ERR(folio)) - /* - * drop the folio reference and try - * to get the folio with lock. If we - * are not uptodate that implies - * somebody just created the folio but - * is yet to initialize it. So - * wait for it to initialize. - */ folio_put(folio); folio = __filemap_get_folio(inode->i_mapping, pnum, FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp); @@ -1763,7 +1773,7 @@ ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, poff = block % blocks_per_page; folio = __filemap_get_folio(inode->i_mapping, pnum, FGP_ACCESSED, 0); - if (IS_ERR(folio) || !folio_test_uptodate(folio)) { + if (IS_ERR(folio) || !folio_test_uptodate(folio) || folio_test_locked(folio)) { if (!IS_ERR(folio)) folio_put(folio); folio = __filemap_get_folio(inode->i_mapping, pnum, @@ -4180,8 +4190,7 @@ ext4_mb_mark_context(handle_t *handle, struct super_block *sb, bool state, * Returns 0 if success or error code */ static noinline_for_stack int -ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, - handle_t *handle, unsigned int reserv_clstrs) +ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, handle_t *handle) { struct ext4_group_desc *gdp; struct ext4_sb_info *sbi; @@ -4236,13 +4245,6 @@ ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, BUG_ON(changed != ac->ac_b_ex.fe_len); #endif percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len); - /* - * Now reduce the dirty block count also. Should not go negative - */ - if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED)) - /* release all the reserved blocks if non delalloc */ - percpu_counter_sub(&sbi->s_dirtyclusters_counter, - reserv_clstrs); return err; } @@ -6327,7 +6329,7 @@ ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle, ext4_mb_pa_put_free(ac); } if (likely(ac->ac_status == AC_STATUS_FOUND)) { - *errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs); + *errp = ext4_mb_mark_diskspace_used(ac, handle); if (*errp) { ext4_discard_allocated_blocks(ac); goto errout; @@ -6358,12 +6360,9 @@ ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle, out: if (inquota && ar->len < inquota) dquot_free_block(ar->inode, EXT4_C2B(sbi, inquota - ar->len)); - if (!ar->len) { - if ((ar->flags & EXT4_MB_DELALLOC_RESERVED) == 0) - /* release all the reserved blocks if non delalloc */ - percpu_counter_sub(&sbi->s_dirtyclusters_counter, - reserv_clstrs); - } + /* release any reserved blocks */ + if (reserv_clstrs) + percpu_counter_sub(&sbi->s_dirtyclusters_counter, reserv_clstrs); trace_ext4_allocate_blocks(ar, (unsigned long long)block); diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 4ef5590c73fda..b5774f4101045 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -5565,6 +5565,10 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) clear_opt2(sb, MB_OPTIMIZE_SCAN); } + err = ext4_percpu_param_init(sbi); + if (err) + goto failed_mount5; + err = ext4_mb_init(sb); if (err) { ext4_msg(sb, KERN_ERR, "failed to initialize mballoc (%d)", @@ -5580,10 +5584,6 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) sbi->s_journal->j_commit_callback = ext4_journal_commit_callback; - err = ext4_percpu_param_init(sbi); - if (err) - goto failed_mount6; - if (ext4_has_feature_flex_bg(sb)) if (!ext4_fill_flex_info(sb)) { ext4_msg(sb, KERN_ERR, @@ -5665,8 +5665,8 @@ failed_mount8: __maybe_unused failed_mount6: ext4_mb_release(sb); ext4_flex_groups_free(sbi); - ext4_percpu_param_destroy(sbi); failed_mount5: + ext4_percpu_param_destroy(sbi); ext4_ext_release(sb); ext4_release_system_zone(sb); failed_mount4a: diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 0f9446143c8e1..3ac0ecbf3ced3 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -151,6 +151,12 @@ static void f2fs_finish_read_bio(struct bio *bio, bool in_task) } dec_page_count(F2FS_F_SB(folio), __read_io_type(folio)); + + if (F2FS_F_SB(folio)->node_inode && is_node_folio(folio) && + f2fs_sanity_check_node_footer(F2FS_F_SB(folio), + folio, folio->index, NODE_TYPE_REGULAR, true)) + bio->bi_status = BLK_STS_IOERR; + folio_end_read(folio, bio->bi_status == BLK_STS_OK); } @@ -352,18 +358,27 @@ static void f2fs_write_end_io(struct bio *bio) STOP_CP_REASON_WRITE_FAIL); } - f2fs_bug_on(sbi, is_node_folio(folio) && - folio->index != nid_of_node(folio)); + if (is_node_folio(folio)) { + f2fs_sanity_check_node_footer(sbi, folio, + folio->index, NODE_TYPE_REGULAR, true); + f2fs_bug_on(sbi, folio->index != nid_of_node(folio)); + } dec_page_count(sbi, type); + + /* + * we should access sbi before folio_end_writeback() to + * avoid racing w/ kill_f2fs_super() + */ + if (type == F2FS_WB_CP_DATA && !get_pages(sbi, type) && + wq_has_sleeper(&sbi->cp_wait)) + wake_up(&sbi->cp_wait); + if (f2fs_in_warm_node_list(sbi, folio)) f2fs_del_fsync_node_entry(sbi, folio); folio_clear_f2fs_gcing(folio); folio_end_writeback(folio); } - if (!get_pages(sbi, F2FS_WB_CP_DATA) && - wq_has_sleeper(&sbi->cp_wait)) - wake_up(&sbi->cp_wait); bio_put(bio); } @@ -1793,7 +1808,8 @@ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag) return err; } -bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len) +static bool __f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len, + bool check_first) { struct f2fs_map_blocks map; block_t last_lblk; @@ -1815,10 +1831,17 @@ bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len) if (err || map.m_len == 0) return false; map.m_lblk += map.m_len; + if (check_first) + break; } return true; } +bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len) +{ + return __f2fs_overwrite_io(inode, pos, len, false); +} + static int f2fs_xattr_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo) { @@ -3933,6 +3956,7 @@ static int check_swap_activate(struct swap_info_struct *sis, while (cur_lblock < last_lblock && cur_lblock < sis->max) { struct f2fs_map_blocks map; + bool last_extent = false; retry: cond_resched(); @@ -3958,11 +3982,10 @@ static int check_swap_activate(struct swap_info_struct *sis, pblock = map.m_pblk; nr_pblocks = map.m_len; - if ((pblock - SM_I(sbi)->main_blkaddr) % blks_per_sec || - nr_pblocks % blks_per_sec || - f2fs_is_sequential_zone_area(sbi, pblock)) { - bool last_extent = false; - + if (!last_extent && + ((pblock - SM_I(sbi)->main_blkaddr) % blks_per_sec || + nr_pblocks % blks_per_sec || + f2fs_is_sequential_zone_area(sbi, pblock))) { not_aligned++; nr_pblocks = roundup(nr_pblocks, blks_per_sec); @@ -3983,8 +4006,8 @@ static int check_swap_activate(struct swap_info_struct *sis, goto out; } - if (!last_extent) - goto retry; + /* lookup block mapping info after block migration */ + goto retry; } if (cur_lblock + nr_pblocks >= sis->max) @@ -4185,7 +4208,7 @@ static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, * f2fs_map_lock and f2fs_balance_fs are not necessary. */ if ((flags & IOMAP_WRITE) && - !f2fs_overwrite_io(inode, offset, length)) + !__f2fs_overwrite_io(inode, offset, length, true)) map.m_may_create = true; err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_DIO); diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 37ad0c27c5b48..123c50f6619ae 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -508,13 +508,25 @@ struct fsync_inode_entry { #define nats_in_cursum(jnl) (le16_to_cpu((jnl)->n_nats)) #define sits_in_cursum(jnl) (le16_to_cpu((jnl)->n_sits)) -#define nat_in_journal(jnl, i) ((jnl)->nat_j.entries[i].ne) -#define nid_in_journal(jnl, i) ((jnl)->nat_j.entries[i].nid) -#define sit_in_journal(jnl, i) ((jnl)->sit_j.entries[i].se) -#define segno_in_journal(jnl, i) ((jnl)->sit_j.entries[i].segno) - -#define MAX_NAT_JENTRIES(jnl) (NAT_JOURNAL_ENTRIES - nats_in_cursum(jnl)) -#define MAX_SIT_JENTRIES(jnl) (SIT_JOURNAL_ENTRIES - sits_in_cursum(jnl)) +#define nat_in_journal(jnl, i) \ + (((struct nat_journal_entry *)(jnl)->nat_j.entries)[i].ne) +#define nid_in_journal(jnl, i) \ + (((struct nat_journal_entry *)(jnl)->nat_j.entries)[i].nid) +#define sit_in_journal(jnl, i) \ + (((struct sit_journal_entry *)(jnl)->sit_j.entries)[i].se) +#define segno_in_journal(jnl, i) \ + (((struct sit_journal_entry *)(jnl)->sit_j.entries)[i].segno) + +#define sum_entries(sum) ((struct f2fs_summary *)(sum)) +#define sum_journal(sbi, sum) \ + ((struct f2fs_journal *)((char *)(sum) + \ + ((sbi)->entries_in_sum * sizeof(struct f2fs_summary)))) +#define sum_footer(sbi, sum) \ + ((struct summary_footer *)((char *)(sum) + (sbi)->sum_blocksize - \ + sizeof(struct summary_footer))) + +#define MAX_NAT_JENTRIES(sbi, jnl) ((sbi)->nat_journal_entries - nats_in_cursum(jnl)) +#define MAX_SIT_JENTRIES(sbi, jnl) ((sbi)->sit_journal_entries - sits_in_cursum(jnl)) static inline int update_nats_in_cursum(struct f2fs_journal *journal, int i) { @@ -532,14 +544,6 @@ static inline int update_sits_in_cursum(struct f2fs_journal *journal, int i) return before; } -static inline bool __has_cursum_space(struct f2fs_journal *journal, - int size, int type) -{ - if (type == NAT_JOURNAL) - return size <= MAX_NAT_JENTRIES(journal); - return size <= MAX_SIT_JENTRIES(journal); -} - /* for inline stuff */ #define DEF_INLINE_RESERVED_SIZE 1 static inline int get_extra_isize(struct inode *inode); @@ -1512,6 +1516,15 @@ enum f2fs_lookup_mode { LOOKUP_AUTO, }; +/* For node type in __get_node_folio() */ +enum node_type { + NODE_TYPE_REGULAR, + NODE_TYPE_INODE, + NODE_TYPE_XATTR, + NODE_TYPE_NON_INODE, +}; + + static inline int f2fs_test_bit(unsigned int nr, char *addr); static inline void f2fs_set_bit(unsigned int nr, char *addr); static inline void f2fs_clear_bit(unsigned int nr, char *addr); @@ -1750,6 +1763,15 @@ struct f2fs_sb_info { bool readdir_ra; /* readahead inode in readdir */ u64 max_io_bytes; /* max io bytes to merge IOs */ + /* variable summary block units */ + unsigned int sum_blocksize; /* sum block size */ + unsigned int sums_per_block; /* sum block count per block */ + unsigned int entries_in_sum; /* entry count in sum block */ + unsigned int sum_entry_size; /* total entry size in sum block */ + unsigned int sum_journal_size; /* journal size in sum block */ + unsigned int nat_journal_entries; /* nat journal entry count in the journal */ + unsigned int sit_journal_entries; /* sit journal entry count in the journal */ + block_t user_block_count; /* # of user blocks */ block_t total_valid_block_count; /* # of valid blocks */ block_t discard_blks; /* discard command candidats */ @@ -2799,6 +2821,14 @@ static inline block_t __start_sum_addr(struct f2fs_sb_info *sbi) return le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_start_sum); } +static inline bool __has_cursum_space(struct f2fs_sb_info *sbi, + struct f2fs_journal *journal, int size, int type) +{ + if (type == NAT_JOURNAL) + return size <= MAX_NAT_JENTRIES(sbi, journal); + return size <= MAX_SIT_JENTRIES(sbi, journal); +} + extern void f2fs_mark_inode_dirty_sync(struct inode *inode, bool sync); static inline int inc_valid_node_count(struct f2fs_sb_info *sbi, struct inode *inode, bool is_inode) @@ -3853,6 +3883,9 @@ struct folio *f2fs_new_node_folio(struct dnode_of_data *dn, unsigned int ofs); void f2fs_ra_node_page(struct f2fs_sb_info *sbi, nid_t nid); struct folio *f2fs_get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid, enum node_type node_type); +int f2fs_sanity_check_node_footer(struct f2fs_sb_info *sbi, + struct folio *folio, pgoff_t nid, + enum node_type ntype, bool in_irq); struct folio *f2fs_get_inode_folio(struct f2fs_sb_info *sbi, pgoff_t ino); struct folio *f2fs_get_xnode_folio(struct f2fs_sb_info *sbi, pgoff_t xnid); int f2fs_move_node_folio(struct folio *node_folio, int gc_type); @@ -3952,7 +3985,8 @@ void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr, block_t len); void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk); void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk); -int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type, +int f2fs_lookup_journal_in_cursum(struct f2fs_sb_info *sbi, + struct f2fs_journal *journal, int type, unsigned int val, int alloc); void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc); int f2fs_check_and_fix_write_pointer(struct f2fs_sb_info *sbi); diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index 5f90cca64c7aa..9d2f4f22fd396 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -1766,8 +1766,8 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, sanity_check_seg_type(sbi, get_seg_entry(sbi, segno)->type); - segno = rounddown(segno, SUMS_PER_BLOCK); - sum_blk_cnt = DIV_ROUND_UP(end_segno - segno, SUMS_PER_BLOCK); + segno = rounddown(segno, sbi->sums_per_block); + sum_blk_cnt = DIV_ROUND_UP(end_segno - segno, sbi->sums_per_block); /* readahead multi ssa blocks those have contiguous address */ if (__is_large_section(sbi)) f2fs_ra_meta_pages(sbi, GET_SUM_BLOCK(sbi, segno), @@ -1777,17 +1777,17 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, while (segno < end_segno) { struct folio *sum_folio = f2fs_get_sum_folio(sbi, segno); - segno += SUMS_PER_BLOCK; + segno += sbi->sums_per_block; if (IS_ERR(sum_folio)) { int err = PTR_ERR(sum_folio); - end_segno = segno - SUMS_PER_BLOCK; - segno = rounddown(start_segno, SUMS_PER_BLOCK); + end_segno = segno - sbi->sums_per_block; + segno = rounddown(start_segno, sbi->sums_per_block); while (segno < end_segno) { sum_folio = filemap_get_folio(META_MAPPING(sbi), GET_SUM_BLOCK(sbi, segno)); folio_put_refs(sum_folio, 2); - segno += SUMS_PER_BLOCK; + segno += sbi->sums_per_block; } return err; } @@ -1803,8 +1803,8 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, /* find segment summary of victim */ struct folio *sum_folio = filemap_get_folio(META_MAPPING(sbi), GET_SUM_BLOCK(sbi, segno)); - unsigned int block_end_segno = rounddown(segno, SUMS_PER_BLOCK) - + SUMS_PER_BLOCK; + unsigned int block_end_segno = rounddown(segno, sbi->sums_per_block) + + sbi->sums_per_block; if (block_end_segno > end_segno) block_end_segno = end_segno; @@ -1830,12 +1830,13 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, migrated >= sbi->migration_granularity) continue; - sum = SUM_BLK_PAGE_ADDR(sum_folio, cur_segno); - if (type != GET_SUM_TYPE((&sum->footer))) { + sum = SUM_BLK_PAGE_ADDR(sbi, sum_folio, cur_segno); + if (type != GET_SUM_TYPE(sum_footer(sbi, sum))) { f2fs_err(sbi, "Inconsistent segment (%u) type " "[%d, %d] in SSA and SIT", cur_segno, type, - GET_SUM_TYPE((&sum->footer))); + GET_SUM_TYPE( + sum_footer(sbi, sum))); f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_CORRUPTED_SUMMARY); continue; @@ -2093,6 +2094,7 @@ int f2fs_gc_range(struct f2fs_sb_info *sbi, if (unlikely(f2fs_cp_error(sbi))) return -EIO; + stat_inc_gc_call_count(sbi, FOREGROUND); for (segno = start_seg; segno <= end_seg; segno += SEGS_PER_SEC(sbi)) { struct gc_inode_list gc_list = { .ilist = LIST_HEAD_INIT(gc_list.ilist), diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 482a362f26254..591fcdf3ba77b 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -606,7 +606,7 @@ int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid, goto retry; } - i = f2fs_lookup_journal_in_cursum(journal, NAT_JOURNAL, nid, 0); + i = f2fs_lookup_journal_in_cursum(sbi, journal, NAT_JOURNAL, nid, 0); if (i >= 0) { ne = nat_in_journal(journal, i); node_info_from_raw_nat(ni, &ne); @@ -1500,9 +1500,9 @@ void f2fs_ra_node_page(struct f2fs_sb_info *sbi, nid_t nid) f2fs_folio_put(afolio, err ? true : false); } -static int sanity_check_node_footer(struct f2fs_sb_info *sbi, +int f2fs_sanity_check_node_footer(struct f2fs_sb_info *sbi, struct folio *folio, pgoff_t nid, - enum node_type ntype) + enum node_type ntype, bool in_irq) { if (unlikely(nid != nid_of_node(folio))) goto out_err; @@ -1527,12 +1527,13 @@ static int sanity_check_node_footer(struct f2fs_sb_info *sbi, goto out_err; return 0; out_err: - f2fs_warn(sbi, "inconsistent node block, node_type:%d, nid:%lu, " - "node_footer[nid:%u,ino:%u,ofs:%u,cpver:%llu,blkaddr:%u]", - ntype, nid, nid_of_node(folio), ino_of_node(folio), - ofs_of_node(folio), cpver_of_node(folio), - next_blkaddr_of_node(folio)); set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_warn_ratelimited(sbi, "inconsistent node block, node_type:%d, nid:%lu, " + "node_footer[nid:%u,ino:%u,ofs:%u,cpver:%llu,blkaddr:%u]", + ntype, nid, nid_of_node(folio), ino_of_node(folio), + ofs_of_node(folio), cpver_of_node(folio), + next_blkaddr_of_node(folio)); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_FOOTER); return -EFSCORRUPTED; } @@ -1578,7 +1579,7 @@ static struct folio *__get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid, goto out_err; } page_hit: - err = sanity_check_node_footer(sbi, folio, nid, ntype); + err = f2fs_sanity_check_node_footer(sbi, folio, nid, ntype, false); if (!err) return folio; out_err: @@ -1751,7 +1752,12 @@ static bool __write_node_folio(struct folio *folio, bool atomic, bool *submitted /* get old block addr of this node page */ nid = nid_of_node(folio); - f2fs_bug_on(sbi, folio->index != nid); + + if (f2fs_sanity_check_node_footer(sbi, folio, nid, + NODE_TYPE_REGULAR, false)) { + f2fs_handle_critical_error(sbi, STOP_CP_REASON_CORRUPTED_NID); + goto redirty_out; + } if (f2fs_get_node_info(sbi, nid, &ni, !do_balance)) goto redirty_out; @@ -1774,8 +1780,13 @@ static bool __write_node_folio(struct folio *folio, bool atomic, bool *submitted goto redirty_out; } - if (atomic && !test_opt(sbi, NOBARRIER)) - fio.op_flags |= REQ_PREFLUSH | REQ_FUA; + if (atomic) { + if (!test_opt(sbi, NOBARRIER)) + fio.op_flags |= REQ_PREFLUSH | REQ_FUA; + if (IS_INODE(folio)) + set_dentry_mark(folio, + f2fs_need_dentry_mark(sbi, ino_of_node(folio))); + } /* should add to global list before clearing PAGECACHE status */ if (f2fs_in_warm_node_list(sbi, folio)) { @@ -1916,8 +1927,9 @@ int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, if (is_inode_flag_set(inode, FI_DIRTY_INODE)) f2fs_update_inode(inode, folio); - set_dentry_mark(folio, - f2fs_need_dentry_mark(sbi, ino)); + if (!atomic) + set_dentry_mark(folio, + f2fs_need_dentry_mark(sbi, ino)); } /* may be written by other thread */ if (!folio_test_dirty(folio)) @@ -2937,7 +2949,7 @@ int f2fs_restore_node_summary(struct f2fs_sb_info *sbi, /* scan the node segment */ last_offset = BLKS_PER_SEG(sbi); addr = START_BLOCK(sbi, segno); - sum_entry = &sum->entries[0]; + sum_entry = sum_entries(sum); for (i = 0; i < last_offset; i += nrpages, addr += nrpages) { nrpages = bio_max_segs(last_offset - i); @@ -3078,7 +3090,7 @@ static int __flush_nat_entry_set(struct f2fs_sb_info *sbi, * #2, flush nat entries to nat page. */ if (enabled_nat_bits(sbi, cpc) || - !__has_cursum_space(journal, set->entry_cnt, NAT_JOURNAL)) + !__has_cursum_space(sbi, journal, set->entry_cnt, NAT_JOURNAL)) to_journal = false; if (to_journal) { @@ -3101,7 +3113,7 @@ static int __flush_nat_entry_set(struct f2fs_sb_info *sbi, f2fs_bug_on(sbi, nat_get_blkaddr(ne) == NEW_ADDR); if (to_journal) { - offset = f2fs_lookup_journal_in_cursum(journal, + offset = f2fs_lookup_journal_in_cursum(sbi, journal, NAT_JOURNAL, nid, 1); f2fs_bug_on(sbi, offset < 0); raw_ne = &nat_in_journal(journal, offset); @@ -3172,7 +3184,7 @@ int f2fs_flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) * into nat entry set. */ if (enabled_nat_bits(sbi, cpc) || - !__has_cursum_space(journal, + !__has_cursum_space(sbi, journal, nm_i->nat_cnt[DIRTY_NAT], NAT_JOURNAL)) remove_nats_in_journal(sbi); @@ -3183,7 +3195,7 @@ int f2fs_flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) set_idx = setvec[found - 1]->set + 1; for (idx = 0; idx < found; idx++) __adjust_nat_entry_set(setvec[idx], &sets, - MAX_NAT_JENTRIES(journal)); + MAX_NAT_JENTRIES(sbi, journal)); } /* flush dirty nats in nat entry set */ diff --git a/fs/f2fs/node.h b/fs/f2fs/node.h index 9cb8dcf8d4176..824ac9f0e6e42 100644 --- a/fs/f2fs/node.h +++ b/fs/f2fs/node.h @@ -52,14 +52,6 @@ enum { IS_PREALLOC, /* nat entry is preallocated */ }; -/* For node type in __get_node_folio() */ -enum node_type { - NODE_TYPE_REGULAR, - NODE_TYPE_INODE, - NODE_TYPE_XATTR, - NODE_TYPE_NON_INODE, -}; - /* * For node information */ diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c index 62a0c71b5b75d..06674e694b271 100644 --- a/fs/f2fs/recovery.c +++ b/fs/f2fs/recovery.c @@ -514,7 +514,7 @@ static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi, struct curseg_info *curseg = CURSEG_I(sbi, i); if (curseg->segno == segno) { - sum = curseg->sum_blk->entries[blkoff]; + sum = sum_entries(curseg->sum_blk)[blkoff]; goto got_it; } } @@ -522,8 +522,8 @@ static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi, sum_folio = f2fs_get_sum_folio(sbi, segno); if (IS_ERR(sum_folio)) return PTR_ERR(sum_folio); - sum_node = SUM_BLK_PAGE_ADDR(sum_folio, segno); - sum = sum_node->entries[blkoff]; + sum_node = SUM_BLK_PAGE_ADDR(sbi, sum_folio, segno); + sum = sum_entries(sum_node)[blkoff]; f2fs_folio_put(sum_folio, true); got_it: /* Use the locked dnode page and inode */ diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 10d873d1b328c..23b94a8fd843d 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -2689,12 +2689,12 @@ int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra) valid_sum_count += f2fs_curseg_valid_blocks(sbi, i); } - sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE - + sum_in_page = (sbi->blocksize - 2 * sbi->sum_journal_size - SUM_FOOTER_SIZE) / SUMMARY_SIZE; if (valid_sum_count <= sum_in_page) return 1; else if ((valid_sum_count - sum_in_page) <= - (PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE) + (sbi->blocksize - SUM_FOOTER_SIZE) / SUMMARY_SIZE) return 2; return 3; } @@ -2714,7 +2714,7 @@ void f2fs_update_meta_page(struct f2fs_sb_info *sbi, { struct folio *folio; - if (SUMS_PER_BLOCK == 1) + if (!f2fs_sb_has_packed_ssa(sbi)) folio = f2fs_grab_meta_folio(sbi, blk_addr); else folio = f2fs_get_meta_folio_retry(sbi, blk_addr); @@ -2732,7 +2732,7 @@ static void write_sum_page(struct f2fs_sb_info *sbi, { struct folio *folio; - if (SUMS_PER_BLOCK == 1) + if (!f2fs_sb_has_packed_ssa(sbi)) return f2fs_update_meta_page(sbi, (void *)sum_blk, GET_SUM_BLOCK(sbi, segno)); @@ -2740,7 +2740,8 @@ static void write_sum_page(struct f2fs_sb_info *sbi, if (IS_ERR(folio)) return; - memcpy(SUM_BLK_PAGE_ADDR(folio, segno), sum_blk, sizeof(*sum_blk)); + memcpy(SUM_BLK_PAGE_ADDR(sbi, folio, segno), sum_blk, + sbi->sum_blocksize); folio_mark_dirty(folio); f2fs_folio_put(folio, true); } @@ -2759,11 +2760,11 @@ static void write_current_sum_page(struct f2fs_sb_info *sbi, mutex_lock(&curseg->curseg_mutex); down_read(&curseg->journal_rwsem); - memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE); + memcpy(sum_journal(sbi, dst), curseg->journal, sbi->sum_journal_size); up_read(&curseg->journal_rwsem); - memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE); - memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE); + memcpy(sum_entries(dst), sum_entries(src), sbi->sum_entry_size); + memcpy(sum_footer(sbi, dst), sum_footer(sbi, src), SUM_FOOTER_SIZE); mutex_unlock(&curseg->curseg_mutex); @@ -2936,7 +2937,7 @@ static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified) curseg->next_blkoff = 0; curseg->next_segno = NULL_SEGNO; - sum_footer = &(curseg->sum_blk->footer); + sum_footer = sum_footer(sbi, curseg->sum_blk); memset(sum_footer, 0, sizeof(struct summary_footer)); sanity_check_seg_type(sbi, seg_type); @@ -3082,11 +3083,11 @@ static int change_curseg(struct f2fs_sb_info *sbi, int type) sum_folio = f2fs_get_sum_folio(sbi, new_segno); if (IS_ERR(sum_folio)) { /* GC won't be able to use stale summary pages by cp_error */ - memset(curseg->sum_blk, 0, SUM_ENTRY_SIZE); + memset(curseg->sum_blk, 0, sbi->sum_entry_size); return PTR_ERR(sum_folio); } - sum_node = SUM_BLK_PAGE_ADDR(sum_folio, new_segno); - memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE); + sum_node = SUM_BLK_PAGE_ADDR(sbi, sum_folio, new_segno); + memcpy(curseg->sum_blk, sum_node, sbi->sum_entry_size); f2fs_folio_put(sum_folio, true); return 0; } @@ -3818,7 +3819,7 @@ int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct folio *folio, f2fs_wait_discard_bio(sbi, *new_blkaddr); - curseg->sum_blk->entries[curseg->next_blkoff] = *sum; + sum_entries(curseg->sum_blk)[curseg->next_blkoff] = *sum; if (curseg->alloc_type == SSR) { curseg->next_blkoff = f2fs_find_next_ssr_block(sbi, curseg); } else { @@ -4187,7 +4188,7 @@ void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, } curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr); - curseg->sum_blk->entries[curseg->next_blkoff] = *sum; + sum_entries(curseg->sum_blk)[curseg->next_blkoff] = *sum; if (!recover_curseg || recover_newaddr) { if (!from_gc) @@ -4307,12 +4308,12 @@ static int read_compacted_summaries(struct f2fs_sb_info *sbi) /* Step 1: restore nat cache */ seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); - memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE); + memcpy(seg_i->journal, kaddr, sbi->sum_journal_size); /* Step 2: restore sit cache */ seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); - memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE); - offset = 2 * SUM_JOURNAL_SIZE; + memcpy(seg_i->journal, kaddr + sbi->sum_journal_size, sbi->sum_journal_size); + offset = 2 * sbi->sum_journal_size; /* Step 3: restore summary entries */ for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { @@ -4334,9 +4335,9 @@ static int read_compacted_summaries(struct f2fs_sb_info *sbi) struct f2fs_summary *s; s = (struct f2fs_summary *)(kaddr + offset); - seg_i->sum_blk->entries[j] = *s; + sum_entries(seg_i->sum_blk)[j] = *s; offset += SUMMARY_SIZE; - if (offset + SUMMARY_SIZE <= PAGE_SIZE - + if (offset + SUMMARY_SIZE <= sbi->blocksize - SUM_FOOTER_SIZE) continue; @@ -4392,7 +4393,7 @@ static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) if (IS_NODESEG(type)) { if (__exist_node_summaries(sbi)) { - struct f2fs_summary *ns = &sum->entries[0]; + struct f2fs_summary *ns = sum_entries(sum); int i; for (i = 0; i < BLKS_PER_SEG(sbi); i++, ns++) { @@ -4412,11 +4413,13 @@ static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) /* update journal info */ down_write(&curseg->journal_rwsem); - memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE); + memcpy(curseg->journal, sum_journal(sbi, sum), sbi->sum_journal_size); up_write(&curseg->journal_rwsem); - memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE); - memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE); + memcpy(sum_entries(curseg->sum_blk), sum_entries(sum), + sbi->sum_entry_size); + memcpy(sum_footer(sbi, curseg->sum_blk), sum_footer(sbi, sum), + SUM_FOOTER_SIZE); curseg->next_segno = segno; reset_curseg(sbi, type, 0); curseg->alloc_type = ckpt->alloc_type[type]; @@ -4460,8 +4463,8 @@ static int restore_curseg_summaries(struct f2fs_sb_info *sbi) } /* sanity check for summary blocks */ - if (nats_in_cursum(nat_j) > NAT_JOURNAL_ENTRIES || - sits_in_cursum(sit_j) > SIT_JOURNAL_ENTRIES) { + if (nats_in_cursum(nat_j) > sbi->nat_journal_entries || + sits_in_cursum(sit_j) > sbi->sit_journal_entries) { f2fs_err(sbi, "invalid journal entries nats %u sits %u", nats_in_cursum(nat_j), sits_in_cursum(sit_j)); return -EINVAL; @@ -4485,13 +4488,13 @@ static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) /* Step 1: write nat cache */ seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); - memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE); - written_size += SUM_JOURNAL_SIZE; + memcpy(kaddr, seg_i->journal, sbi->sum_journal_size); + written_size += sbi->sum_journal_size; /* Step 2: write sit cache */ seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); - memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE); - written_size += SUM_JOURNAL_SIZE; + memcpy(kaddr + written_size, seg_i->journal, sbi->sum_journal_size); + written_size += sbi->sum_journal_size; /* Step 3: write summary entries */ for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { @@ -4504,10 +4507,10 @@ static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) written_size = 0; } summary = (struct f2fs_summary *)(kaddr + written_size); - *summary = seg_i->sum_blk->entries[j]; + *summary = sum_entries(seg_i->sum_blk)[j]; written_size += SUMMARY_SIZE; - if (written_size + SUMMARY_SIZE <= PAGE_SIZE - + if (written_size + SUMMARY_SIZE <= sbi->blocksize - SUM_FOOTER_SIZE) continue; @@ -4549,8 +4552,9 @@ void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk) write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE); } -int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type, - unsigned int val, int alloc) +int f2fs_lookup_journal_in_cursum(struct f2fs_sb_info *sbi, + struct f2fs_journal *journal, int type, + unsigned int val, int alloc) { int i; @@ -4559,13 +4563,13 @@ int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type, if (le32_to_cpu(nid_in_journal(journal, i)) == val) return i; } - if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL)) + if (alloc && __has_cursum_space(sbi, journal, 1, NAT_JOURNAL)) return update_nats_in_cursum(journal, 1); } else if (type == SIT_JOURNAL) { for (i = 0; i < sits_in_cursum(journal); i++) if (le32_to_cpu(segno_in_journal(journal, i)) == val) return i; - if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL)) + if (alloc && __has_cursum_space(sbi, journal, 1, SIT_JOURNAL)) return update_sits_in_cursum(journal, 1); } return -1; @@ -4713,8 +4717,8 @@ void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) * entries, remove all entries from journal and add and account * them in sit entry set. */ - if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL) || - !to_journal) + if (!__has_cursum_space(sbi, journal, + sit_i->dirty_sentries, SIT_JOURNAL) || !to_journal) remove_sits_in_journal(sbi); /* @@ -4731,7 +4735,8 @@ void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) unsigned int segno = start_segno; if (to_journal && - !__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL)) + !__has_cursum_space(sbi, journal, ses->entry_cnt, + SIT_JOURNAL)) to_journal = false; if (to_journal) { @@ -4759,7 +4764,7 @@ void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) } if (to_journal) { - offset = f2fs_lookup_journal_in_cursum(journal, + offset = f2fs_lookup_journal_in_cursum(sbi, journal, SIT_JOURNAL, segno, 1); f2fs_bug_on(sbi, offset < 0); segno_in_journal(journal, offset) = @@ -4966,12 +4971,13 @@ static int build_curseg(struct f2fs_sb_info *sbi) for (i = 0; i < NO_CHECK_TYPE; i++) { mutex_init(&array[i].curseg_mutex); - array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL); + array[i].sum_blk = f2fs_kzalloc(sbi, sbi->sum_blocksize, + GFP_KERNEL); if (!array[i].sum_blk) return -ENOMEM; init_rwsem(&array[i].journal_rwsem); array[i].journal = f2fs_kzalloc(sbi, - sizeof(struct f2fs_journal), GFP_KERNEL); + sbi->sum_journal_size, GFP_KERNEL); if (!array[i].journal) return -ENOMEM; array[i].seg_type = log_type_to_seg_type(i); diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h index f3e2fff45cf53..d05d133c89aff 100644 --- a/fs/f2fs/segment.h +++ b/fs/f2fs/segment.h @@ -85,12 +85,11 @@ static inline void sanity_check_seg_type(struct f2fs_sb_info *sbi, #define GET_ZONE_FROM_SEG(sbi, segno) \ GET_ZONE_FROM_SEC(sbi, GET_SEC_FROM_SEG(sbi, segno)) -#define SUMS_PER_BLOCK (F2FS_BLKSIZE / F2FS_SUM_BLKSIZE) #define GET_SUM_BLOCK(sbi, segno) \ - (SM_I(sbi)->ssa_blkaddr + (segno / SUMS_PER_BLOCK)) -#define GET_SUM_BLKOFF(segno) (segno % SUMS_PER_BLOCK) -#define SUM_BLK_PAGE_ADDR(folio, segno) \ - (folio_address(folio) + GET_SUM_BLKOFF(segno) * F2FS_SUM_BLKSIZE) + (SM_I(sbi)->ssa_blkaddr + (segno / (sbi)->sums_per_block)) +#define GET_SUM_BLKOFF(sbi, segno) (segno % (sbi)->sums_per_block) +#define SUM_BLK_PAGE_ADDR(sbi, folio, segno) \ + (folio_address(folio) + GET_SUM_BLKOFF(sbi, segno) * (sbi)->sum_blocksize) #define GET_SUM_TYPE(footer) ((footer)->entry_type) #define SET_SUM_TYPE(footer, type) ((footer)->entry_type = (type)) diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index eb466a11d9d7f..907f632193abf 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -4059,20 +4059,6 @@ static int sanity_check_raw_super(struct f2fs_sb_info *sbi, if (sanity_check_area_boundary(sbi, folio, index)) return -EFSCORRUPTED; - /* - * Check for legacy summary layout on 16KB+ block devices. - * Modern f2fs-tools packs multiple 4KB summary areas into one block, - * whereas legacy versions used one block per summary, leading - * to a much larger SSA. - */ - if (SUMS_PER_BLOCK > 1 && - !(__F2FS_HAS_FEATURE(raw_super, F2FS_FEATURE_PACKED_SSA))) { - f2fs_info(sbi, "Error: Device formatted with a legacy version. " - "Please reformat with a tool supporting the packed ssa " - "feature for block sizes larger than 4kb."); - return -EOPNOTSUPP; - } - return 0; } @@ -4283,6 +4269,18 @@ static void init_sb_info(struct f2fs_sb_info *sbi) spin_lock_init(&sbi->gc_remaining_trials_lock); atomic64_set(&sbi->current_atomic_write, 0); + sbi->sum_blocksize = f2fs_sb_has_packed_ssa(sbi) ? + 4096 : sbi->blocksize; + sbi->sums_per_block = sbi->blocksize / sbi->sum_blocksize; + sbi->entries_in_sum = sbi->sum_blocksize / 8; + sbi->sum_entry_size = SUMMARY_SIZE * sbi->entries_in_sum; + sbi->sum_journal_size = sbi->sum_blocksize - SUM_FOOTER_SIZE - + sbi->sum_entry_size; + sbi->nat_journal_entries = (sbi->sum_journal_size - 2) / + sizeof(struct nat_journal_entry); + sbi->sit_journal_entries = (sbi->sum_journal_size - 2) / + sizeof(struct sit_journal_entry); + sbi->dir_level = DEF_DIR_LEVEL; sbi->interval_time[CP_TIME] = DEF_CP_INTERVAL; sbi->interval_time[REQ_TIME] = DEF_IDLE_INTERVAL; diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c index 5685b454bfd12..c6fa8f74b91c2 100644 --- a/fs/f2fs/sysfs.c +++ b/fs/f2fs/sysfs.c @@ -58,6 +58,7 @@ struct f2fs_attr { const char *buf, size_t len); int struct_type; int offset; + int size; int id; }; @@ -344,11 +345,30 @@ static ssize_t main_blkaddr_show(struct f2fs_attr *a, (unsigned long long)MAIN_BLKADDR(sbi)); } +static ssize_t __sbi_show_value(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf, + unsigned char *value) +{ + switch (a->size) { + case 1: + return sysfs_emit(buf, "%u\n", *(u8 *)value); + case 2: + return sysfs_emit(buf, "%u\n", *(u16 *)value); + case 4: + return sysfs_emit(buf, "%u\n", *(u32 *)value); + case 8: + return sysfs_emit(buf, "%llu\n", *(u64 *)value); + default: + f2fs_bug_on(sbi, 1); + return sysfs_emit(buf, + "show sysfs node value with wrong type\n"); + } +} + static ssize_t f2fs_sbi_show(struct f2fs_attr *a, struct f2fs_sb_info *sbi, char *buf) { unsigned char *ptr = NULL; - unsigned int *ui; ptr = __struct_ptr(sbi, a->struct_type); if (!ptr) @@ -428,9 +448,30 @@ static ssize_t f2fs_sbi_show(struct f2fs_attr *a, atomic_read(&sbi->cp_call_count[BACKGROUND])); #endif - ui = (unsigned int *)(ptr + a->offset); + return __sbi_show_value(a, sbi, buf, ptr + a->offset); +} - return sysfs_emit(buf, "%u\n", *ui); +static void __sbi_store_value(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, + unsigned char *ui, unsigned long value) +{ + switch (a->size) { + case 1: + *(u8 *)ui = value; + break; + case 2: + *(u16 *)ui = value; + break; + case 4: + *(u32 *)ui = value; + break; + case 8: + *(u64 *)ui = value; + break; + default: + f2fs_bug_on(sbi, 1); + f2fs_err(sbi, "store sysfs node value with wrong type"); + } } static ssize_t __sbi_store(struct f2fs_attr *a, @@ -749,7 +790,7 @@ static ssize_t __sbi_store(struct f2fs_attr *a, return count; } - if (!strcmp(a->attr.name, "gc_pin_file_threshold")) { + if (!strcmp(a->attr.name, "gc_pin_file_thresh")) { if (t > MAX_GC_FAILED_PINNED_FILES) return -EINVAL; sbi->gc_pin_file_threshold = t; @@ -906,7 +947,7 @@ static ssize_t __sbi_store(struct f2fs_attr *a, return count; } - *ui = (unsigned int)t; + __sbi_store_value(a, sbi, ptr + a->offset, t); return count; } @@ -1053,24 +1094,27 @@ static struct f2fs_attr f2fs_attr_sb_##_name = { \ .id = F2FS_FEATURE_##_feat, \ } -#define F2FS_ATTR_OFFSET(_struct_type, _name, _mode, _show, _store, _offset) \ +#define F2FS_ATTR_OFFSET(_struct_type, _name, _mode, _show, _store, _offset, _size) \ static struct f2fs_attr f2fs_attr_##_name = { \ .attr = {.name = __stringify(_name), .mode = _mode }, \ .show = _show, \ .store = _store, \ .struct_type = _struct_type, \ - .offset = _offset \ + .offset = _offset, \ + .size = _size \ } #define F2FS_RO_ATTR(struct_type, struct_name, name, elname) \ F2FS_ATTR_OFFSET(struct_type, name, 0444, \ f2fs_sbi_show, NULL, \ - offsetof(struct struct_name, elname)) + offsetof(struct struct_name, elname), \ + sizeof_field(struct struct_name, elname)) #define F2FS_RW_ATTR(struct_type, struct_name, name, elname) \ F2FS_ATTR_OFFSET(struct_type, name, 0644, \ f2fs_sbi_show, f2fs_sbi_store, \ - offsetof(struct struct_name, elname)) + offsetof(struct struct_name, elname), \ + sizeof_field(struct struct_name, elname)) #define F2FS_GENERAL_RO_ATTR(name) \ static struct f2fs_attr f2fs_attr_##name = __ATTR(name, 0444, name##_show, NULL) diff --git a/fs/fat/namei_msdos.c b/fs/fat/namei_msdos.c index 0b920ee40a7f9..262ec1b790b56 100644 --- a/fs/fat/namei_msdos.c +++ b/fs/fat/namei_msdos.c @@ -325,7 +325,12 @@ static int msdos_rmdir(struct inode *dir, struct dentry *dentry) err = fat_remove_entries(dir, &sinfo); /* and releases bh */ if (err) goto out; - drop_nlink(dir); + if (dir->i_nlink >= 3) + drop_nlink(dir); + else { + fat_fs_error(sb, "parent dir link count too low (%u)", + dir->i_nlink); + } clear_nlink(inode); fat_truncate_time(inode, NULL, S_CTIME); diff --git a/fs/fat/namei_vfat.c b/fs/fat/namei_vfat.c index 5dbc4cbb8fce3..47ff083cfc7e6 100644 --- a/fs/fat/namei_vfat.c +++ b/fs/fat/namei_vfat.c @@ -803,7 +803,12 @@ static int vfat_rmdir(struct inode *dir, struct dentry *dentry) err = fat_remove_entries(dir, &sinfo); /* and releases bh */ if (err) goto out; - drop_nlink(dir); + if (dir->i_nlink >= 3) + drop_nlink(dir); + else { + fat_fs_error(sb, "parent dir link count too low (%u)", + dir->i_nlink); + } clear_nlink(inode); fat_truncate_time(inode, NULL, S_ATIME|S_MTIME); diff --git a/fs/file_table.c b/fs/file_table.c index cd4a3db4659ac..34244fccf2edf 100644 --- a/fs/file_table.c +++ b/fs/file_table.c @@ -176,6 +176,11 @@ static int init_file(struct file *f, int flags, const struct cred *cred) f->f_flags = flags; f->f_mode = OPEN_FMODE(flags); + /* + * Disable permission and pre-content events for all files by default. + * They may be enabled later by fsnotify_open_perm_and_set_mode(). + */ + file_set_fsnotify_mode(f, FMODE_NONOTIFY_PERM); f->f_op = NULL; f->f_mapping = NULL; @@ -197,11 +202,6 @@ static int init_file(struct file *f, int flags, const struct cred *cred) * refcount bumps we should reinitialize the reused file first. */ file_ref_init(&f->f_ref, 1); - /* - * Disable permission and pre-content events for all files by default. - * They may be enabled later by fsnotify_open_perm_and_set_mode(). - */ - file_set_fsnotify_mode(f, FMODE_NONOTIFY_PERM); return 0; } diff --git a/fs/fs_struct.c b/fs/fs_struct.c index 28be762ac1c63..a0b40ad5e7423 100644 --- a/fs/fs_struct.c +++ b/fs/fs_struct.c @@ -6,6 +6,7 @@ #include #include #include +#include #include "internal.h" /* diff --git a/fs/gfs2/bmap.c b/fs/gfs2/bmap.c index 131091520de6b..fdcac8e3f2ba2 100644 --- a/fs/gfs2/bmap.c +++ b/fs/gfs2/bmap.c @@ -1127,10 +1127,18 @@ static int gfs2_iomap_begin(struct inode *inode, loff_t pos, loff_t length, goto out_unlock; break; default: - goto out_unlock; + goto out; } ret = gfs2_iomap_begin_write(inode, pos, length, flags, iomap, &mp); + if (ret) + goto out_unlock; + +out: + if (iomap->type == IOMAP_INLINE) { + iomap->private = metapath_dibh(&mp); + get_bh(iomap->private); + } out_unlock: release_metapath(&mp); @@ -1144,6 +1152,9 @@ static int gfs2_iomap_end(struct inode *inode, loff_t pos, loff_t length, struct gfs2_inode *ip = GFS2_I(inode); struct gfs2_sbd *sdp = GFS2_SB(inode); + if (iomap->private) + brelse(iomap->private); + switch (flags & (IOMAP_WRITE | IOMAP_ZERO)) { case IOMAP_WRITE: if (flags & IOMAP_DIRECT) diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c index 9f2eb7e385695..7aec6cfdfd91d 100644 --- a/fs/gfs2/glock.c +++ b/fs/gfs2/glock.c @@ -1354,31 +1354,45 @@ static int glocks_pending(unsigned int num_gh, struct gfs2_holder *ghs) * gfs2_glock_async_wait - wait on multiple asynchronous glock acquisitions * @num_gh: the number of holders in the array * @ghs: the glock holder array + * @retries: number of retries attempted so far * * Returns: 0 on success, meaning all glocks have been granted and are held. * -ESTALE if the request timed out, meaning all glocks were released, * and the caller should retry the operation. */ -int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs) +int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs, + unsigned int retries) { struct gfs2_sbd *sdp = ghs[0].gh_gl->gl_name.ln_sbd; - int i, ret = 0, timeout = 0; unsigned long start_time = jiffies; + int i, ret = 0; + long timeout; might_sleep(); - /* - * Total up the (minimum hold time * 2) of all glocks and use that to - * determine the max amount of time we should wait. - */ - for (i = 0; i < num_gh; i++) - timeout += ghs[i].gh_gl->gl_hold_time << 1; - if (!wait_event_timeout(sdp->sd_async_glock_wait, + timeout = GL_GLOCK_MIN_HOLD; + if (retries) { + unsigned int max_shift; + long incr; + + /* Add a random delay and increase the timeout exponentially. */ + max_shift = BITS_PER_LONG - 2 - __fls(GL_GLOCK_HOLD_INCR); + incr = min(GL_GLOCK_HOLD_INCR << min(retries - 1, max_shift), + 10 * HZ - GL_GLOCK_MIN_HOLD); + schedule_timeout_interruptible(get_random_long() % (incr / 3)); + if (signal_pending(current)) + goto interrupted; + timeout += (incr / 3) + get_random_long() % (incr / 3); + } + + if (!wait_event_interruptible_timeout(sdp->sd_async_glock_wait, !glocks_pending(num_gh, ghs), timeout)) { ret = -ESTALE; /* request timed out. */ goto out; } + if (signal_pending(current)) + goto interrupted; for (i = 0; i < num_gh; i++) { struct gfs2_holder *gh = &ghs[i]; @@ -1402,6 +1416,10 @@ int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs) } } return ret; + +interrupted: + ret = -EINTR; + goto out; } /** diff --git a/fs/gfs2/glock.h b/fs/gfs2/glock.h index d041b922b45e3..2d4fd1a2bbbb8 100644 --- a/fs/gfs2/glock.h +++ b/fs/gfs2/glock.h @@ -204,7 +204,8 @@ int gfs2_glock_poll(struct gfs2_holder *gh); int gfs2_instantiate(struct gfs2_holder *gh); int gfs2_glock_holder_ready(struct gfs2_holder *gh); int gfs2_glock_wait(struct gfs2_holder *gh); -int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs); +int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs, + unsigned int retries); void gfs2_glock_dq(struct gfs2_holder *gh); void gfs2_glock_dq_wait(struct gfs2_holder *gh); void gfs2_glock_dq_uninit(struct gfs2_holder *gh); diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c index d7e35a05c1610..03dd54fb7e8c8 100644 --- a/fs/gfs2/inode.c +++ b/fs/gfs2/inode.c @@ -1495,7 +1495,7 @@ static int gfs2_rename(struct inode *odir, struct dentry *odentry, unsigned int num_gh; int dir_rename = 0; struct gfs2_diradd da = { .nr_blocks = 0, .save_loc = 0, }; - unsigned int x; + unsigned int retries = 0, x; int error; gfs2_holder_mark_uninitialized(&r_gh); @@ -1545,12 +1545,17 @@ static int gfs2_rename(struct inode *odir, struct dentry *odentry, num_gh++; } +again: for (x = 0; x < num_gh; x++) { error = gfs2_glock_nq(ghs + x); if (error) goto out_gunlock; } - error = gfs2_glock_async_wait(num_gh, ghs); + error = gfs2_glock_async_wait(num_gh, ghs, retries); + if (error == -ESTALE) { + retries++; + goto again; + } if (error) goto out_gunlock; @@ -1739,7 +1744,7 @@ static int gfs2_exchange(struct inode *odir, struct dentry *odentry, struct gfs2_sbd *sdp = GFS2_SB(odir); struct gfs2_holder ghs[4], r_gh; unsigned int num_gh; - unsigned int x; + unsigned int retries = 0, x; umode_t old_mode = oip->i_inode.i_mode; umode_t new_mode = nip->i_inode.i_mode; int error; @@ -1783,13 +1788,18 @@ static int gfs2_exchange(struct inode *odir, struct dentry *odentry, gfs2_holder_init(nip->i_gl, LM_ST_EXCLUSIVE, GL_ASYNC, ghs + num_gh); num_gh++; +again: for (x = 0; x < num_gh; x++) { error = gfs2_glock_nq(ghs + x); if (error) goto out_gunlock; } - error = gfs2_glock_async_wait(num_gh, ghs); + error = gfs2_glock_async_wait(num_gh, ghs, retries); + if (error == -ESTALE) { + retries++; + goto again; + } if (error) goto out_gunlock; @@ -2182,6 +2192,14 @@ static int gfs2_getattr(struct mnt_idmap *idmap, return 0; } +static bool fault_in_fiemap(struct fiemap_extent_info *fi) +{ + struct fiemap_extent __user *dest = fi->fi_extents_start; + size_t size = sizeof(*dest) * fi->fi_extents_max; + + return fault_in_safe_writeable((char __user *)dest, size) == 0; +} + static int gfs2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len) { @@ -2191,14 +2209,22 @@ static int gfs2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, inode_lock_shared(inode); +retry: ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, 0, &gh); if (ret) goto out; + pagefault_disable(); ret = iomap_fiemap(inode, fieinfo, start, len, &gfs2_iomap_ops); + pagefault_enable(); gfs2_glock_dq_uninit(&gh); + if (ret == -EFAULT && fault_in_fiemap(fieinfo)) { + fieinfo->fi_extents_mapped = 0; + goto retry; + } + out: inode_unlock_shared(inode); return ret; diff --git a/fs/gfs2/quota.c b/fs/gfs2/quota.c index f2df01f801b81..898fc3937b449 100644 --- a/fs/gfs2/quota.c +++ b/fs/gfs2/quota.c @@ -334,6 +334,7 @@ static void qd_put(struct gfs2_quota_data *qd) lockref_mark_dead(&qd->qd_lockref); spin_unlock(&qd->qd_lockref.lock); + list_lru_del_obj(&gfs2_qd_lru, &qd->qd_lru); gfs2_qd_dispose(qd); return; } diff --git a/fs/hfsplus/bnode.c b/fs/hfsplus/bnode.c index 482a6c5faa197..8e60e04c427bd 100644 --- a/fs/hfsplus/bnode.c +++ b/fs/hfsplus/bnode.c @@ -629,7 +629,7 @@ struct hfs_bnode *hfs_bnode_create(struct hfs_btree *tree, u32 num) if (node) { pr_crit("new node %u already hashed?\n", num); WARN_ON(1); - return node; + return ERR_PTR(-EEXIST); } node = __hfs_bnode_create(tree, num); if (!node) diff --git a/fs/hfsplus/dir.c b/fs/hfsplus/dir.c index cadf0b5f93422..ca5f74a140ec1 100644 --- a/fs/hfsplus/dir.c +++ b/fs/hfsplus/dir.c @@ -313,6 +313,9 @@ static int hfsplus_link(struct dentry *src_dentry, struct inode *dst_dir, if (!S_ISREG(inode->i_mode)) return -EPERM; + hfs_dbg("src_dir->i_ino %lu, dst_dir->i_ino %lu, inode->i_ino %lu\n", + src_dir->i_ino, dst_dir->i_ino, inode->i_ino); + mutex_lock(&sbi->vh_mutex); if (inode->i_ino == (u32)(unsigned long)src_dentry->d_fsdata) { for (;;) { @@ -332,7 +335,7 @@ static int hfsplus_link(struct dentry *src_dentry, struct inode *dst_dir, cnid = sbi->next_cnid++; src_dentry->d_fsdata = (void *)(unsigned long)cnid; res = hfsplus_create_cat(cnid, src_dir, - &src_dentry->d_name, inode); + &src_dentry->d_name, inode); if (res) /* panic? */ goto out; @@ -350,6 +353,21 @@ static int hfsplus_link(struct dentry *src_dentry, struct inode *dst_dir, mark_inode_dirty(inode); sbi->file_count++; hfsplus_mark_mdb_dirty(dst_dir->i_sb); + + res = hfsplus_cat_write_inode(src_dir); + if (res) + goto out; + + res = hfsplus_cat_write_inode(dst_dir); + if (res) + goto out; + + res = hfsplus_cat_write_inode(sbi->hidden_dir); + if (res) + goto out; + + res = hfsplus_cat_write_inode(inode); + out: mutex_unlock(&sbi->vh_mutex); return res; @@ -367,6 +385,9 @@ static int hfsplus_unlink(struct inode *dir, struct dentry *dentry) if (HFSPLUS_IS_RSRC(inode)) return -EPERM; + hfs_dbg("dir->i_ino %lu, inode->i_ino %lu\n", + dir->i_ino, inode->i_ino); + mutex_lock(&sbi->vh_mutex); cnid = (u32)(unsigned long)dentry->d_fsdata; if (inode->i_ino == cnid && @@ -408,6 +429,15 @@ static int hfsplus_unlink(struct inode *dir, struct dentry *dentry) inode_set_ctime_current(inode); mark_inode_dirty(inode); out: + if (!res) { + res = hfsplus_cat_write_inode(dir); + if (!res) { + res = hfsplus_cat_write_inode(sbi->hidden_dir); + if (!res) + res = hfsplus_cat_write_inode(inode); + } + } + mutex_unlock(&sbi->vh_mutex); return res; } @@ -429,6 +459,8 @@ static int hfsplus_rmdir(struct inode *dir, struct dentry *dentry) inode_set_ctime_current(inode); hfsplus_delete_inode(inode); mark_inode_dirty(inode); + + res = hfsplus_cat_write_inode(dir); out: mutex_unlock(&sbi->vh_mutex); return res; @@ -465,6 +497,12 @@ static int hfsplus_symlink(struct mnt_idmap *idmap, struct inode *dir, hfsplus_instantiate(dentry, inode, inode->i_ino); mark_inode_dirty(inode); + + res = hfsplus_cat_write_inode(dir); + if (res) + goto out; + + res = hfsplus_cat_write_inode(inode); goto out; out_err: @@ -506,6 +544,12 @@ static int hfsplus_mknod(struct mnt_idmap *idmap, struct inode *dir, hfsplus_instantiate(dentry, inode, inode->i_ino); mark_inode_dirty(inode); + + res = hfsplus_cat_write_inode(dir); + if (res) + goto out; + + res = hfsplus_cat_write_inode(inode); goto out; failed_mknod: diff --git a/fs/hfsplus/inode.c b/fs/hfsplus/inode.c index 7ae6745ca7ae1..6153e5cc6eb65 100644 --- a/fs/hfsplus/inode.c +++ b/fs/hfsplus/inode.c @@ -328,6 +328,9 @@ int hfsplus_file_fsync(struct file *file, loff_t start, loff_t end, struct hfsplus_vh *vhdr = sbi->s_vhdr; int error = 0, error2; + hfs_dbg("inode->i_ino %lu, start %llu, end %llu\n", + inode->i_ino, start, end); + error = file_write_and_wait_range(file, start, end); if (error) return error; @@ -612,17 +615,20 @@ int hfsplus_cat_read_inode(struct inode *inode, struct hfs_find_data *fd) int hfsplus_cat_write_inode(struct inode *inode) { struct inode *main_inode = inode; + struct hfs_btree *tree = HFSPLUS_SB(inode->i_sb)->cat_tree; struct hfs_find_data fd; hfsplus_cat_entry entry; int res = 0; + hfs_dbg("inode->i_ino %lu\n", inode->i_ino); + if (HFSPLUS_IS_RSRC(inode)) main_inode = HFSPLUS_I(inode)->rsrc_inode; if (!main_inode->i_nlink) return 0; - if (hfs_find_init(HFSPLUS_SB(main_inode->i_sb)->cat_tree, &fd)) + if (hfs_find_init(tree, &fd)) /* panic? */ return -EIO; @@ -687,6 +693,15 @@ int hfsplus_cat_write_inode(struct inode *inode) set_bit(HFSPLUS_I_CAT_DIRTY, &HFSPLUS_I(inode)->flags); out: hfs_find_exit(&fd); + + if (!res) { + res = hfs_btree_write(tree); + if (res) { + pr_err("b-tree write err: %d, ino %lu\n", + res, inode->i_ino); + } + } + return res; } diff --git a/fs/hfsplus/super.c b/fs/hfsplus/super.c index 67a7a2a093476..5230d368bd4f2 100644 --- a/fs/hfsplus/super.c +++ b/fs/hfsplus/super.c @@ -53,6 +53,12 @@ static int hfsplus_system_read_inode(struct inode *inode) return -EIO; } + /* + * Assign a dummy file type, for may_open() requires that + * an inode has a valid file type. + */ + inode->i_mode = S_IFREG; + return 0; } @@ -344,8 +350,6 @@ static void hfsplus_put_super(struct super_block *sb) hfs_btree_close(sbi->ext_tree); kfree(sbi->s_vhdr_buf); kfree(sbi->s_backup_vhdr_buf); - call_rcu(&sbi->rcu, delayed_free); - hfs_dbg("finished\n"); } @@ -650,7 +654,6 @@ static int hfsplus_fill_super(struct super_block *sb, struct fs_context *fc) out_unload_nls: unload_nls(sbi->nls); unload_nls(nls); - kfree(sbi); return err; } @@ -709,10 +712,18 @@ static int hfsplus_init_fs_context(struct fs_context *fc) return 0; } +static void hfsplus_kill_super(struct super_block *sb) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); + + kill_block_super(sb); + call_rcu(&sbi->rcu, delayed_free); +} + static struct file_system_type hfsplus_fs_type = { .owner = THIS_MODULE, .name = "hfsplus", - .kill_sb = kill_block_super, + .kill_sb = hfsplus_kill_super, .fs_flags = FS_REQUIRES_DEV, .init_fs_context = hfsplus_init_fs_context, }; diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index 6317e4cd42517..e73c71f39bd45 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -429,9 +429,13 @@ static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio) nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_VECS); do { size_t n; - if (dio->error) { - iov_iter_revert(dio->submit.iter, copied); - copied = ret = 0; + + /* + * If completions already occurred and reported errors, give up now and + * don't bother submitting more bios. + */ + if (unlikely(data_race(dio->error))) { + ret = 0; goto out; } diff --git a/fs/jfs/jfs_dtree.c b/fs/jfs/jfs_dtree.c index 0ab83bb7bbdf9..9ab3f2fc61d17 100644 --- a/fs/jfs/jfs_dtree.c +++ b/fs/jfs/jfs_dtree.c @@ -2903,7 +2903,7 @@ int jfs_readdir(struct file *file, struct dir_context *ctx) stbl = DT_GETSTBL(p); for (i = index; i < p->header.nextindex; i++) { - if (stbl[i] < 0 || stbl[i] >= DTPAGEMAXSLOT) { + if (stbl[i] < 0) { jfs_err("JFS: Invalid stbl[%d] = %d for inode %ld, block = %lld", i, stbl[i], (long)ip->i_ino, (long long)bn); free_page(dirent_buf); @@ -3108,7 +3108,7 @@ static int dtReadFirst(struct inode *ip, struct btstack * btstack) /* get the leftmost entry */ stbl = DT_GETSTBL(p); - if (stbl[0] < 0 || stbl[0] >= DTPAGEMAXSLOT) { + if (stbl[0] < 0) { DT_PUTPAGE(mp); jfs_error(ip->i_sb, "stbl[0] out of bound\n"); return -EIO; diff --git a/fs/jfs/jfs_logmgr.c b/fs/jfs/jfs_logmgr.c index b343c5ea11592..5b1c5da041630 100644 --- a/fs/jfs/jfs_logmgr.c +++ b/fs/jfs/jfs_logmgr.c @@ -2311,6 +2311,7 @@ int jfsIOWait(void *arg) { struct lbuf *bp; + set_freezable(); do { spin_lock_irq(&log_redrive_lock); while ((bp = log_redrive_list)) { diff --git a/fs/jfs/namei.c b/fs/jfs/namei.c index 65a218eba8faf..7879c049632b3 100644 --- a/fs/jfs/namei.c +++ b/fs/jfs/namei.c @@ -1228,7 +1228,7 @@ static int jfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, jfs_err("jfs_rename: dtInsert returned -EIO"); goto out_tx; } - if (S_ISDIR(old_ip->i_mode)) + if (S_ISDIR(old_ip->i_mode) && old_dir != new_dir) inc_nlink(new_dir); } /* @@ -1244,7 +1244,9 @@ static int jfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, goto out_tx; } if (S_ISDIR(old_ip->i_mode)) { - drop_nlink(old_dir); + if (new_ip || old_dir != new_dir) + drop_nlink(old_dir); + if (old_dir != new_dir) { /* * Change inode number of parent for moved directory diff --git a/fs/minix/inode.c b/fs/minix/inode.c index 32db676127a9e..7bdd240ea1584 100644 --- a/fs/minix/inode.c +++ b/fs/minix/inode.c @@ -154,10 +154,38 @@ static int minix_reconfigure(struct fs_context *fc) static bool minix_check_superblock(struct super_block *sb) { struct minix_sb_info *sbi = minix_sb(sb); + unsigned long block; - if (sbi->s_imap_blocks == 0 || sbi->s_zmap_blocks == 0) + if (sbi->s_log_zone_size != 0) { + printk("minix-fs error: zone size must equal block size. " + "s_log_zone_size > 0 is not supported.\n"); + return false; + } + + if (sbi->s_ninodes < 1 || sbi->s_firstdatazone <= 4 || + sbi->s_firstdatazone >= sbi->s_nzones) return false; + /* Apparently minix can create filesystems that allocate more blocks for + * the bitmaps than needed. We simply ignore that, but verify it didn't + * create one with not enough blocks and bail out if so. + */ + block = minix_blocks_needed(sbi->s_ninodes, sb->s_blocksize); + if (sbi->s_imap_blocks < block) { + printk("MINIX-fs: file system does not have enough " + "imap blocks allocated. Refusing to mount.\n"); + return false; + } + + block = minix_blocks_needed( + (sbi->s_nzones - sbi->s_firstdatazone + 1), + sb->s_blocksize); + if (sbi->s_zmap_blocks < block) { + printk("MINIX-fs: file system does not have enough " + "zmap blocks allocated. Refusing to mount.\n"); + return false; + } + /* * s_max_size must not exceed the block mapping limitation. This check * is only needed for V1 filesystems, since V2/V3 support an extra level @@ -277,26 +305,6 @@ static int minix_fill_super(struct super_block *s, struct fs_context *fc) minix_set_bit(0,sbi->s_imap[0]->b_data); minix_set_bit(0,sbi->s_zmap[0]->b_data); - /* Apparently minix can create filesystems that allocate more blocks for - * the bitmaps than needed. We simply ignore that, but verify it didn't - * create one with not enough blocks and bail out if so. - */ - block = minix_blocks_needed(sbi->s_ninodes, s->s_blocksize); - if (sbi->s_imap_blocks < block) { - printk("MINIX-fs: file system does not have enough " - "imap blocks allocated. Refusing to mount.\n"); - goto out_no_bitmap; - } - - block = minix_blocks_needed( - (sbi->s_nzones - sbi->s_firstdatazone + 1), - s->s_blocksize); - if (sbi->s_zmap_blocks < block) { - printk("MINIX-fs: file system does not have enough " - "zmap blocks allocated. Refusing to mount.\n"); - goto out_no_bitmap; - } - /* set up enough so that it can read an inode */ s->s_op = &minix_sops; s->s_time_min = 0; diff --git a/fs/namespace.c b/fs/namespace.c index 4272349650b14..8531b8deee416 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -221,7 +221,7 @@ static int mnt_alloc_id(struct mount *mnt) int res; xa_lock(&mnt_id_xa); - res = __xa_alloc(&mnt_id_xa, &mnt->mnt_id, mnt, XA_LIMIT(1, INT_MAX), GFP_KERNEL); + res = __xa_alloc(&mnt_id_xa, &mnt->mnt_id, mnt, xa_limit_31b, GFP_KERNEL); if (!res) mnt->mnt_id_unique = ++mnt_id_ctr; xa_unlock(&mnt_id_xa); @@ -1531,23 +1531,33 @@ static struct mount *mnt_find_id_at_reverse(struct mnt_namespace *ns, u64 mnt_id static void *m_start(struct seq_file *m, loff_t *pos) { struct proc_mounts *p = m->private; + struct mount *mnt; down_read(&namespace_sem); - return mnt_find_id_at(p->ns, *pos); + mnt = mnt_find_id_at(p->ns, *pos); + if (mnt) + *pos = mnt->mnt_id_unique; + return mnt; } static void *m_next(struct seq_file *m, void *v, loff_t *pos) { - struct mount *next = NULL, *mnt = v; + struct mount *mnt = v; struct rb_node *node = rb_next(&mnt->mnt_node); - ++*pos; if (node) { - next = node_to_mount(node); + struct mount *next = node_to_mount(node); *pos = next->mnt_id_unique; + return next; } - return next; + + /* + * No more mounts. Set pos past current mount's ID so that if + * iteration restarts, mnt_find_id_at() returns NULL. + */ + *pos = mnt->mnt_id_unique + 1; + return NULL; } static void m_stop(struct seq_file *m, void *v) @@ -5796,7 +5806,7 @@ SYSCALL_DEFINE4(statmount, const struct mnt_id_req __user *, req, if (kreq.mnt_ns_id && (ns != current->nsproxy->mnt_ns) && !ns_capable_noaudit(ns->user_ns, CAP_SYS_ADMIN)) - return -ENOENT; + return -EPERM; ks = kmalloc(sizeof(*ks), GFP_KERNEL_ACCOUNT); if (!ks) diff --git a/fs/netfs/direct_write.c b/fs/netfs/direct_write.c index a9d1c3b2c0842..dd1451bf7543d 100644 --- a/fs/netfs/direct_write.c +++ b/fs/netfs/direct_write.c @@ -9,6 +9,202 @@ #include #include "internal.h" +/* + * Perform the cleanup rituals after an unbuffered write is complete. + */ +static void netfs_unbuffered_write_done(struct netfs_io_request *wreq) +{ + struct netfs_inode *ictx = netfs_inode(wreq->inode); + + _enter("R=%x", wreq->debug_id); + + /* Okay, declare that all I/O is complete. */ + trace_netfs_rreq(wreq, netfs_rreq_trace_write_done); + + if (!wreq->error) + netfs_update_i_size(ictx, &ictx->inode, wreq->start, wreq->transferred); + + if (wreq->origin == NETFS_DIO_WRITE && + wreq->mapping->nrpages) { + /* mmap may have got underfoot and we may now have folios + * locally covering the region we just wrote. Attempt to + * discard the folios, but leave in place any modified locally. + * ->write_iter() is prevented from interfering by the DIO + * counter. + */ + pgoff_t first = wreq->start >> PAGE_SHIFT; + pgoff_t last = (wreq->start + wreq->transferred - 1) >> PAGE_SHIFT; + + invalidate_inode_pages2_range(wreq->mapping, first, last); + } + + if (wreq->origin == NETFS_DIO_WRITE) + inode_dio_end(wreq->inode); + + _debug("finished"); + netfs_wake_rreq_flag(wreq, NETFS_RREQ_IN_PROGRESS, netfs_rreq_trace_wake_ip); + /* As we cleared NETFS_RREQ_IN_PROGRESS, we acquired its ref. */ + + if (wreq->iocb) { + size_t written = umin(wreq->transferred, wreq->len); + + wreq->iocb->ki_pos += written; + if (wreq->iocb->ki_complete) { + trace_netfs_rreq(wreq, netfs_rreq_trace_ki_complete); + wreq->iocb->ki_complete(wreq->iocb, wreq->error ?: written); + } + wreq->iocb = VFS_PTR_POISON; + } + + netfs_clear_subrequests(wreq); +} + +/* + * Collect the subrequest results of unbuffered write subrequests. + */ +static void netfs_unbuffered_write_collect(struct netfs_io_request *wreq, + struct netfs_io_stream *stream, + struct netfs_io_subrequest *subreq) +{ + trace_netfs_collect_sreq(wreq, subreq); + + spin_lock(&wreq->lock); + list_del_init(&subreq->rreq_link); + spin_unlock(&wreq->lock); + + wreq->transferred += subreq->transferred; + iov_iter_advance(&wreq->buffer.iter, subreq->transferred); + + stream->collected_to = subreq->start + subreq->transferred; + wreq->collected_to = stream->collected_to; + netfs_put_subrequest(subreq, netfs_sreq_trace_put_done); + + trace_netfs_collect_stream(wreq, stream); + trace_netfs_collect_state(wreq, wreq->collected_to, 0); +} + +/* + * Write data to the server without going through the pagecache and without + * writing it to the local cache. We dispatch the subrequests serially and + * wait for each to complete before dispatching the next, lest we leave a gap + * in the data written due to a failure such as ENOSPC. We could, however + * attempt to do preparation such as content encryption for the next subreq + * whilst the current is in progress. + */ +static int netfs_unbuffered_write(struct netfs_io_request *wreq) +{ + struct netfs_io_subrequest *subreq = NULL; + struct netfs_io_stream *stream = &wreq->io_streams[0]; + int ret; + + _enter("%llx", wreq->len); + + if (wreq->origin == NETFS_DIO_WRITE) + inode_dio_begin(wreq->inode); + + stream->collected_to = wreq->start; + + for (;;) { + bool retry = false; + + if (!subreq) { + netfs_prepare_write(wreq, stream, wreq->start + wreq->transferred); + subreq = stream->construct; + stream->construct = NULL; + stream->front = NULL; + } + + /* Check if (re-)preparation failed. */ + if (unlikely(test_bit(NETFS_SREQ_FAILED, &subreq->flags))) { + netfs_write_subrequest_terminated(subreq, subreq->error); + wreq->error = subreq->error; + break; + } + + iov_iter_truncate(&subreq->io_iter, wreq->len - wreq->transferred); + if (!iov_iter_count(&subreq->io_iter)) + break; + + subreq->len = netfs_limit_iter(&subreq->io_iter, 0, + stream->sreq_max_len, + stream->sreq_max_segs); + iov_iter_truncate(&subreq->io_iter, subreq->len); + stream->submit_extendable_to = subreq->len; + + trace_netfs_sreq(subreq, netfs_sreq_trace_submit); + stream->issue_write(subreq); + + /* Async, need to wait. */ + netfs_wait_for_in_progress_stream(wreq, stream); + + if (test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { + retry = true; + } else if (test_bit(NETFS_SREQ_FAILED, &subreq->flags)) { + ret = subreq->error; + wreq->error = ret; + netfs_see_subrequest(subreq, netfs_sreq_trace_see_failed); + subreq = NULL; + break; + } + ret = 0; + + if (!retry) { + netfs_unbuffered_write_collect(wreq, stream, subreq); + subreq = NULL; + if (wreq->transferred >= wreq->len) + break; + if (!wreq->iocb && signal_pending(current)) { + ret = wreq->transferred ? -EINTR : -ERESTARTSYS; + trace_netfs_rreq(wreq, netfs_rreq_trace_intr); + break; + } + continue; + } + + /* We need to retry the last subrequest, so first reset the + * iterator, taking into account what, if anything, we managed + * to transfer. + */ + subreq->error = -EAGAIN; + trace_netfs_sreq(subreq, netfs_sreq_trace_retry); + if (subreq->transferred > 0) + iov_iter_advance(&wreq->buffer.iter, subreq->transferred); + + if (stream->source == NETFS_UPLOAD_TO_SERVER && + wreq->netfs_ops->retry_request) + wreq->netfs_ops->retry_request(wreq, stream); + + __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); + __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); + __clear_bit(NETFS_SREQ_FAILED, &subreq->flags); + subreq->io_iter = wreq->buffer.iter; + subreq->start = wreq->start + wreq->transferred; + subreq->len = wreq->len - wreq->transferred; + subreq->transferred = 0; + subreq->retry_count += 1; + stream->sreq_max_len = UINT_MAX; + stream->sreq_max_segs = INT_MAX; + + netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); + stream->prepare_write(subreq); + + __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); + netfs_stat(&netfs_n_wh_retry_write_subreq); + } + + netfs_unbuffered_write_done(wreq); + _leave(" = %d", ret); + return ret; +} + +static void netfs_unbuffered_write_async(struct work_struct *work) +{ + struct netfs_io_request *wreq = container_of(work, struct netfs_io_request, work); + + netfs_unbuffered_write(wreq); + netfs_put_request(wreq, netfs_rreq_trace_put_complete); +} + /* * Perform an unbuffered write where we may have to do an RMW operation on an * encrypted file. This can also be used for direct I/O writes. @@ -70,35 +266,35 @@ ssize_t netfs_unbuffered_write_iter_locked(struct kiocb *iocb, struct iov_iter * */ wreq->buffer.iter = *iter; } + + wreq->len = iov_iter_count(&wreq->buffer.iter); } __set_bit(NETFS_RREQ_USE_IO_ITER, &wreq->flags); - if (async) - __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &wreq->flags); /* Copy the data into the bounce buffer and encrypt it. */ // TODO /* Dispatch the write. */ __set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags); - if (async) - wreq->iocb = iocb; - wreq->len = iov_iter_count(&wreq->buffer.iter); - ret = netfs_unbuffered_write(wreq, is_sync_kiocb(iocb), wreq->len); - if (ret < 0) { - _debug("begin = %zd", ret); - goto out; - } - if (!async) { - ret = netfs_wait_for_write(wreq); - if (ret > 0) - iocb->ki_pos += ret; - } else { + if (async) { + INIT_WORK(&wreq->work, netfs_unbuffered_write_async); + wreq->iocb = iocb; + queue_work(system_dfl_wq, &wreq->work); ret = -EIOCBQUEUED; + } else { + ret = netfs_unbuffered_write(wreq); + if (ret < 0) { + _debug("begin = %zd", ret); + } else { + iocb->ki_pos += wreq->transferred; + ret = wreq->transferred ?: wreq->error; + } + + netfs_put_request(wreq, netfs_rreq_trace_put_complete); } -out: netfs_put_request(wreq, netfs_rreq_trace_put_return); return ret; diff --git a/fs/netfs/internal.h b/fs/netfs/internal.h index 4319611f53544..d436e20d34185 100644 --- a/fs/netfs/internal.h +++ b/fs/netfs/internal.h @@ -198,6 +198,9 @@ struct netfs_io_request *netfs_create_write_req(struct address_space *mapping, struct file *file, loff_t start, enum netfs_io_origin origin); +void netfs_prepare_write(struct netfs_io_request *wreq, + struct netfs_io_stream *stream, + loff_t start); void netfs_reissue_write(struct netfs_io_stream *stream, struct netfs_io_subrequest *subreq, struct iov_iter *source); @@ -212,7 +215,6 @@ int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_c struct folio **writethrough_cache); ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc, struct folio *writethrough_cache); -int netfs_unbuffered_write(struct netfs_io_request *wreq, bool may_wait, size_t len); /* * write_retry.c diff --git a/fs/netfs/read_collect.c b/fs/netfs/read_collect.c index 7a0ffa675fb17..137f0e28a44c5 100644 --- a/fs/netfs/read_collect.c +++ b/fs/netfs/read_collect.c @@ -546,6 +546,15 @@ void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq) } } + /* If need retry is set, error should not matter unless we hit too many + * retries. Pause the generation of new subreqs + */ + if (test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { + trace_netfs_rreq(rreq, netfs_rreq_trace_set_pause); + set_bit(NETFS_RREQ_PAUSE, &rreq->flags); + goto skip_error_checks; + } + if (unlikely(subreq->error < 0)) { trace_netfs_failure(rreq, subreq, subreq->error, netfs_fail_read); if (subreq->source == NETFS_READ_FROM_CACHE) { @@ -559,6 +568,7 @@ void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq) set_bit(NETFS_RREQ_PAUSE, &rreq->flags); } +skip_error_checks: trace_netfs_sreq(subreq, netfs_sreq_trace_terminated); netfs_subreq_clear_in_progress(subreq); netfs_put_subrequest(subreq, netfs_sreq_trace_put_terminated); diff --git a/fs/netfs/read_retry.c b/fs/netfs/read_retry.c index b99e84a8170af..7793ba5e3e8fc 100644 --- a/fs/netfs/read_retry.c +++ b/fs/netfs/read_retry.c @@ -12,6 +12,7 @@ static void netfs_reissue_read(struct netfs_io_request *rreq, struct netfs_io_subrequest *subreq) { + subreq->error = 0; __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); netfs_stat(&netfs_n_rh_retry_read_subreq); @@ -242,8 +243,7 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) subreq = list_next_entry(subreq, rreq_link); abandon: list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) { - if (!subreq->error && - !test_bit(NETFS_SREQ_FAILED, &subreq->flags) && + if (!test_bit(NETFS_SREQ_FAILED, &subreq->flags) && !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) continue; subreq->error = -ENOMEM; diff --git a/fs/netfs/write_collect.c b/fs/netfs/write_collect.c index cbf3d9194c7bf..83eb3dc1adf8a 100644 --- a/fs/netfs/write_collect.c +++ b/fs/netfs/write_collect.c @@ -399,27 +399,6 @@ bool netfs_write_collection(struct netfs_io_request *wreq) ictx->ops->invalidate_cache(wreq); } - if ((wreq->origin == NETFS_UNBUFFERED_WRITE || - wreq->origin == NETFS_DIO_WRITE) && - !wreq->error) - netfs_update_i_size(ictx, &ictx->inode, wreq->start, wreq->transferred); - - if (wreq->origin == NETFS_DIO_WRITE && - wreq->mapping->nrpages) { - /* mmap may have got underfoot and we may now have folios - * locally covering the region we just wrote. Attempt to - * discard the folios, but leave in place any modified locally. - * ->write_iter() is prevented from interfering by the DIO - * counter. - */ - pgoff_t first = wreq->start >> PAGE_SHIFT; - pgoff_t last = (wreq->start + wreq->transferred - 1) >> PAGE_SHIFT; - invalidate_inode_pages2_range(wreq->mapping, first, last); - } - - if (wreq->origin == NETFS_DIO_WRITE) - inode_dio_end(wreq->inode); - _debug("finished"); netfs_wake_rreq_flag(wreq, NETFS_RREQ_IN_PROGRESS, netfs_rreq_trace_wake_ip); /* As we cleared NETFS_RREQ_IN_PROGRESS, we acquired its ref. */ @@ -492,11 +471,11 @@ void netfs_write_subrequest_terminated(void *_op, ssize_t transferred_or_error) if (IS_ERR_VALUE(transferred_or_error)) { subreq->error = transferred_or_error; - if (subreq->error == -EAGAIN) - set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); - else + /* if need retry is set, error should not matter */ + if (!test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { set_bit(NETFS_SREQ_FAILED, &subreq->flags); - trace_netfs_failure(wreq, subreq, transferred_or_error, netfs_fail_write); + trace_netfs_failure(wreq, subreq, transferred_or_error, netfs_fail_write); + } switch (subreq->source) { case NETFS_WRITE_TO_CACHE: diff --git a/fs/netfs/write_issue.c b/fs/netfs/write_issue.c index dd8743bc8d7fe..437268f656409 100644 --- a/fs/netfs/write_issue.c +++ b/fs/netfs/write_issue.c @@ -154,9 +154,9 @@ EXPORT_SYMBOL(netfs_prepare_write_failed); * Prepare a write subrequest. We need to allocate a new subrequest * if we don't have one. */ -static void netfs_prepare_write(struct netfs_io_request *wreq, - struct netfs_io_stream *stream, - loff_t start) +void netfs_prepare_write(struct netfs_io_request *wreq, + struct netfs_io_stream *stream, + loff_t start) { struct netfs_io_subrequest *subreq; struct iov_iter *wreq_iter = &wreq->buffer.iter; @@ -250,6 +250,7 @@ void netfs_reissue_write(struct netfs_io_stream *stream, iov_iter_truncate(&subreq->io_iter, size); subreq->retry_count++; + subreq->error = 0; __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); netfs_stat(&netfs_n_wh_retry_write_subreq); @@ -697,41 +698,6 @@ ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_c return ret; } -/* - * Write data to the server without going through the pagecache and without - * writing it to the local cache. - */ -int netfs_unbuffered_write(struct netfs_io_request *wreq, bool may_wait, size_t len) -{ - struct netfs_io_stream *upload = &wreq->io_streams[0]; - ssize_t part; - loff_t start = wreq->start; - int error = 0; - - _enter("%zx", len); - - if (wreq->origin == NETFS_DIO_WRITE) - inode_dio_begin(wreq->inode); - - while (len) { - // TODO: Prepare content encryption - - _debug("unbuffered %zx", len); - part = netfs_advance_write(wreq, upload, start, len, false); - start += part; - len -= part; - rolling_buffer_advance(&wreq->buffer, part); - if (test_bit(NETFS_RREQ_PAUSE, &wreq->flags)) - netfs_wait_for_paused_write(wreq); - if (test_bit(NETFS_RREQ_FAILED, &wreq->flags)) - break; - } - - netfs_end_issue_write(wreq); - _leave(" = %d", error); - return error; -} - /* * Write some of a pending folio data back to the server and/or the cache. */ diff --git a/fs/netfs/write_retry.c b/fs/netfs/write_retry.c index fc9c3e0d34d81..29489a23a2209 100644 --- a/fs/netfs/write_retry.c +++ b/fs/netfs/write_retry.c @@ -98,7 +98,6 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq, subreq->start = start; subreq->len = len; __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); - subreq->retry_count++; trace_netfs_sreq(subreq, netfs_sreq_trace_retry); /* Renegotiate max_len (wsize) */ diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index 3b8250ee01412..a653a401b7970 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -72,7 +72,7 @@ const struct address_space_operations nfs_dir_aops = { .free_folio = nfs_readdir_clear_array, }; -#define NFS_INIT_DTSIZE PAGE_SIZE +#define NFS_INIT_DTSIZE SZ_64K static struct nfs_open_dir_context * alloc_nfs_open_dir_context(struct inode *dir) @@ -83,7 +83,7 @@ alloc_nfs_open_dir_context(struct inode *dir) ctx = kzalloc(sizeof(*ctx), GFP_KERNEL_ACCOUNT); if (ctx != NULL) { ctx->attr_gencount = nfsi->attr_gencount; - ctx->dtsize = NFS_INIT_DTSIZE; + ctx->dtsize = min(NFS_SERVER(dir)->dtsize, NFS_INIT_DTSIZE); spin_lock(&dir->i_lock); if (list_empty(&nfsi->open_files) && (nfsi->cache_validity & NFS_INO_DATA_INVAL_DEFER)) diff --git a/fs/nfs/localio.c b/fs/nfs/localio.c index f537bc3386bf2..358d686d2ae33 100644 --- a/fs/nfs/localio.c +++ b/fs/nfs/localio.c @@ -58,6 +58,11 @@ struct nfs_local_fsync_ctx { static bool localio_enabled __read_mostly = true; module_param(localio_enabled, bool, 0644); +static int nfs_local_do_read(struct nfs_local_kiocb *iocb, + const struct rpc_call_ops *call_ops); +static int nfs_local_do_write(struct nfs_local_kiocb *iocb, + const struct rpc_call_ops *call_ops); + static inline bool nfs_client_is_local(const struct nfs_client *clp) { return !!rcu_access_pointer(clp->cl_uuid.net); @@ -286,6 +291,18 @@ nfs_local_open_fh(struct nfs_client *clp, const struct cred *cred, } EXPORT_SYMBOL_GPL(nfs_local_open_fh); +/* + * Ensure all page cache allocations are done from GFP_NOFS context to + * prevent direct reclaim recursion back into NFS via nfs_writepages. + */ +static void +nfs_local_mapping_set_gfp_nofs_context(struct address_space *m) +{ + gfp_t gfp_mask = mapping_gfp_mask(m); + + mapping_set_gfp_mask(m, (gfp_mask & ~(__GFP_FS))); +} + static void nfs_local_iocb_free(struct nfs_local_kiocb *iocb) { @@ -310,6 +327,7 @@ nfs_local_iocb_alloc(struct nfs_pgio_header *hdr, return NULL; } + nfs_local_mapping_set_gfp_nofs_context(file->f_mapping); init_sync_kiocb(&iocb->kiocb, file); iocb->hdr = hdr; @@ -542,13 +560,50 @@ nfs_local_iocb_release(struct nfs_local_kiocb *iocb) nfs_local_iocb_free(iocb); } -static void -nfs_local_pgio_release(struct nfs_local_kiocb *iocb) +static void nfs_local_pgio_restart(struct nfs_local_kiocb *iocb, + struct nfs_pgio_header *hdr) +{ + int status = 0; + + iocb->kiocb.ki_pos = hdr->args.offset; + iocb->kiocb.ki_flags &= ~(IOCB_DSYNC | IOCB_SYNC | IOCB_DIRECT); + iocb->kiocb.ki_complete = NULL; + iocb->aio_complete_work = NULL; + iocb->end_iter_index = -1; + + switch (hdr->rw_mode) { + case FMODE_READ: + nfs_local_iters_init(iocb, ITER_DEST); + status = nfs_local_do_read(iocb, hdr->task.tk_ops); + break; + case FMODE_WRITE: + nfs_local_iters_init(iocb, ITER_SOURCE); + status = nfs_local_do_write(iocb, hdr->task.tk_ops); + break; + default: + status = -EOPNOTSUPP; + } + + if (status != 0) { + nfs_local_iocb_release(iocb); + hdr->task.tk_status = status; + nfs_local_hdr_release(hdr, hdr->task.tk_ops); + } +} + +static void nfs_local_pgio_release(struct nfs_local_kiocb *iocb) { struct nfs_pgio_header *hdr = iocb->hdr; + struct rpc_task *task = &hdr->task; + + task->tk_action = NULL; + task->tk_ops->rpc_call_done(task, hdr); - nfs_local_iocb_release(iocb); - nfs_local_hdr_release(hdr, hdr->task.tk_ops); + if (task->tk_action == NULL) { + nfs_local_iocb_release(iocb); + task->tk_ops->rpc_release(hdr); + } else + nfs_local_pgio_restart(iocb, hdr); } /* @@ -776,19 +831,7 @@ static void nfs_local_write_done(struct nfs_local_kiocb *iocb) pr_info_ratelimited("nfs: Unexpected direct I/O write alignment failure\n"); } - /* Handle short writes as if they are ENOSPC */ - status = hdr->res.count; - if (status > 0 && status < hdr->args.count) { - hdr->mds_offset += status; - hdr->args.offset += status; - hdr->args.pgbase += status; - hdr->args.count -= status; - nfs_set_pgio_error(hdr, -ENOSPC, hdr->args.offset); - status = -ENOSPC; - /* record -ENOSPC in terms of nfs_local_pgio_done */ - (void) nfs_local_pgio_done(iocb, status, true); - } - if (hdr->task.tk_status < 0) + if (status < 0) nfs_reset_boot_verifier(hdr->inode); } @@ -952,8 +995,6 @@ int nfs_local_doio(struct nfs_client *clp, struct nfsd_file *localio, } if (status != 0) { - if (status == -EAGAIN) - nfs_localio_disable_client(clp); nfs_local_iocb_release(iocb); hdr->task.tk_status = status; nfs_local_hdr_release(hdr, call_ops); @@ -980,6 +1021,8 @@ nfs_local_run_commit(struct file *filp, struct nfs_commit_data *data) end = LLONG_MAX; } + nfs_local_mapping_set_gfp_nofs_context(filp->f_mapping); + dprintk("%s: commit %llu - %llu\n", __func__, start, end); return vfs_fsync_range(filp, start, end, 0); } @@ -1021,17 +1064,22 @@ nfs_local_fsync_ctx_free(struct nfs_local_fsync_ctx *ctx) static void nfs_local_fsync_work(struct work_struct *work) { + unsigned long old_flags = current->flags; struct nfs_local_fsync_ctx *ctx; int status; ctx = container_of(work, struct nfs_local_fsync_ctx, work); + current->flags |= PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO; + status = nfs_local_run_commit(nfs_to->nfsd_file_file(ctx->localio), ctx->data); nfs_local_commit_done(ctx->data, status); if (ctx->done != NULL) complete(ctx->done); nfs_local_fsync_ctx_free(ctx); + + current->flags = old_flags; } static struct nfs_local_fsync_ctx * @@ -1055,7 +1103,7 @@ int nfs_local_commit(struct nfsd_file *localio, { struct nfs_local_fsync_ctx *ctx; - ctx = nfs_local_fsync_ctx_alloc(data, localio, GFP_KERNEL); + ctx = nfs_local_fsync_ctx_alloc(data, localio, GFP_NOIO); if (!ctx) { nfs_local_commit_done(data, -ENOMEM); nfs_local_release_commit_data(localio, data, call_ops); @@ -1067,10 +1115,10 @@ int nfs_local_commit(struct nfsd_file *localio, if (how & FLUSH_SYNC) { DECLARE_COMPLETION_ONSTACK(done); ctx->done = &done; - queue_work(nfsiod_workqueue, &ctx->work); + queue_work(nfslocaliod_workqueue, &ctx->work); wait_for_completion(&done); } else - queue_work(nfsiod_workqueue, &ctx->work); + queue_work(nfslocaliod_workqueue, &ctx->work); return 0; } diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index 33bc6db0dc92f..b3cb5ee9d821e 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -463,7 +463,8 @@ pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo, }; struct pnfs_layout_segment *lseg, *next; - set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); + if (test_and_set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) + return !list_empty(&lo->plh_segs); clear_bit(NFS_INO_LAYOUTCOMMIT, &NFS_I(lo->plh_inode)->flags); list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) pnfs_clear_lseg_state(lseg, lseg_list); diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c index 5fb202acb0fd0..0ac538c761800 100644 --- a/fs/nfsd/nfs2acl.c +++ b/fs/nfsd/nfs2acl.c @@ -45,7 +45,7 @@ static __be32 nfsacld_proc_getacl(struct svc_rqst *rqstp) inode = d_inode(fh->fh_dentry); if (argp->mask & ~NFS_ACL_MASK) { - resp->status = nfserr_inval; + resp->status = nfserr_io; goto out; } resp->mask = argp->mask; diff --git a/fs/nfsd/nfs4idmap.c b/fs/nfsd/nfs4idmap.c index 8cca1329f3485..c319c31b0f647 100644 --- a/fs/nfsd/nfs4idmap.c +++ b/fs/nfsd/nfs4idmap.c @@ -643,34 +643,74 @@ static __be32 encode_name_from_id(struct xdr_stream *xdr, return idmap_id_to_name(xdr, rqstp, type, id); } -__be32 -nfsd_map_name_to_uid(struct svc_rqst *rqstp, const char *name, size_t namelen, - kuid_t *uid) +/** + * nfsd_map_name_to_uid - Map user@domain to local UID + * @rqstp: RPC execution context + * @name: user@domain name to be mapped + * @namelen: length of name, in bytes + * @uid: OUT: mapped local UID value + * + * Returns nfs_ok on success or an NFSv4 status code on failure. + */ +__be32 nfsd_map_name_to_uid(struct svc_rqst *rqstp, const char *name, + size_t namelen, kuid_t *uid) { __be32 status; u32 id = -1; + /* + * The idmap lookup below triggers an upcall that invokes + * cache_check(). RQ_USEDEFERRAL must be clear to prevent + * cache_check() from setting RQ_DROPME via svc_defer(). + * NFSv4 servers are not permitted to drop requests. Also + * RQ_DROPME will force NFSv4.1 session slot processing to + * be skipped. + */ + WARN_ON_ONCE(test_bit(RQ_USEDEFERRAL, &rqstp->rq_flags)); + if (name == NULL || namelen == 0) return nfserr_inval; status = do_name_to_id(rqstp, IDMAP_TYPE_USER, name, namelen, &id); + if (status) + return status; *uid = make_kuid(nfsd_user_namespace(rqstp), id); if (!uid_valid(*uid)) status = nfserr_badowner; return status; } -__be32 -nfsd_map_name_to_gid(struct svc_rqst *rqstp, const char *name, size_t namelen, - kgid_t *gid) +/** + * nfsd_map_name_to_gid - Map user@domain to local GID + * @rqstp: RPC execution context + * @name: user@domain name to be mapped + * @namelen: length of name, in bytes + * @gid: OUT: mapped local GID value + * + * Returns nfs_ok on success or an NFSv4 status code on failure. + */ +__be32 nfsd_map_name_to_gid(struct svc_rqst *rqstp, const char *name, + size_t namelen, kgid_t *gid) { __be32 status; u32 id = -1; + /* + * The idmap lookup below triggers an upcall that invokes + * cache_check(). RQ_USEDEFERRAL must be clear to prevent + * cache_check() from setting RQ_DROPME via svc_defer(). + * NFSv4 servers are not permitted to drop requests. Also + * RQ_DROPME will force NFSv4.1 session slot processing to + * be skipped. + */ + WARN_ON_ONCE(test_bit(RQ_USEDEFERRAL, &rqstp->rq_flags)); + if (name == NULL || namelen == 0) return nfserr_inval; status = do_name_to_id(rqstp, IDMAP_TYPE_GROUP, name, namelen, &id); + if (status) + return status; *gid = make_kgid(nfsd_user_namespace(rqstp), id); if (!gid_valid(*gid)) status = nfserr_badowner; diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index 42a6b914c0fe6..8dada7ef97cb1 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -2995,8 +2995,6 @@ nfsd4_proc_compound(struct svc_rqst *rqstp) BUG_ON(cstate->replay_owner); out: cstate->status = status; - /* Reset deferral mechanism for RPC deferrals */ - set_bit(RQ_USEDEFERRAL, &rqstp->rq_flags); return rpc_success; } diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 740c40eb5b366..c5dba49c90356 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -6356,7 +6356,8 @@ nfs4_open_delegation(struct svc_rqst *rqstp, struct nfsd4_open *open, dp->dl_ctime = stat.ctime; dp->dl_mtime = stat.mtime; spin_lock(&f->f_lock); - f->f_mode |= FMODE_NOCMTIME; + if (deleg_ts) + f->f_mode |= FMODE_NOCMTIME; spin_unlock(&f->f_lock); trace_nfsd_deleg_write(&dp->dl_stid.sc_stateid); } else { diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c index 4a403ce4fd468..5f046d5be4a6e 100644 --- a/fs/nfsd/nfs4xdr.c +++ b/fs/nfsd/nfs4xdr.c @@ -6001,6 +6001,22 @@ nfs4svc_decode_compoundargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) args->ops = args->iops; args->rqstp = rqstp; + /* + * NFSv4 operation decoders can invoke svc cache lookups + * that trigger svc_defer() when RQ_USEDEFERRAL is set, + * setting RQ_DROPME. This creates two problems: + * + * 1. Non-idempotency: Compounds make it too hard to avoid + * problems if a request is deferred and replayed. + * + * 2. Session slot leakage (NFSv4.1+): If RQ_DROPME is set + * during decode but SEQUENCE executes successfully, the + * session slot will be marked INUSE. The request is then + * dropped before encoding, so the slot is never released, + * rendering it permanently unusable by the client. + */ + clear_bit(RQ_USEDEFERRAL, &rqstp->rq_flags); + return nfsd4_decode_compound(args); } diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index 8cbfb9dc3abb7..5b00b7a863b94 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -1642,7 +1642,7 @@ int nfsd_nl_threads_set_doit(struct sk_buff *skb, struct genl_info *info) scope = nla_data(attr); } - ret = nfsd_svc(nrpools, nthreads, net, get_current_cred(), scope); + ret = nfsd_svc(nrpools, nthreads, net, current_cred(), scope); if (ret > 0) ret = 0; out_unlock: diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c index 8f71f5748c75b..906a672578900 100644 --- a/fs/nfsd/nfsproc.c +++ b/fs/nfsd/nfsproc.c @@ -33,7 +33,7 @@ static __be32 nfsd_map_status(__be32 status) break; case nfserr_symlink: case nfserr_wrong_type: - status = nfserr_inval; + status = nfserr_io; break; } return status; diff --git a/fs/ntfs3/attrib.c b/fs/ntfs3/attrib.c index eced9013a8818..5a7675112e7b8 100644 --- a/fs/ntfs3/attrib.c +++ b/fs/ntfs3/attrib.c @@ -448,8 +448,10 @@ int attr_set_size(struct ntfs_inode *ni, enum ATTR_TYPE type, is_ext = is_attr_ext(attr_b); align = sbi->cluster_size; - if (is_ext) + if (is_ext) { align <<= attr_b->nres.c_unit; + keep_prealloc = false; + } old_valid = le64_to_cpu(attr_b->nres.valid_size); old_size = le64_to_cpu(attr_b->nres.data_size); @@ -1354,19 +1356,28 @@ int attr_load_runs_range(struct ntfs_inode *ni, enum ATTR_TYPE type, CLST vcn; CLST vcn_last = (to - 1) >> cluster_bits; CLST lcn, clen; - int err; + int err = 0; + int retry = 0; for (vcn = from >> cluster_bits; vcn <= vcn_last; vcn += clen) { if (!run_lookup_entry(run, vcn, &lcn, &clen, NULL)) { + if (retry != 0) { /* Next run_lookup_entry(vcn) also failed. */ + err = -EINVAL; + break; + } err = attr_load_runs_vcn(ni, type, name, name_len, run, vcn); if (err) - return err; + break; + clen = 0; /* Next run_lookup_entry(vcn) must be success. */ + retry++; } + else + retry = 0; } - return 0; + return err; } #ifdef CONFIG_NTFS3_LZX_XPRESS diff --git a/fs/ntfs3/attrlist.c b/fs/ntfs3/attrlist.c index a4d74bed74fab..098bd7e8c3d64 100644 --- a/fs/ntfs3/attrlist.c +++ b/fs/ntfs3/attrlist.c @@ -52,6 +52,11 @@ int ntfs_load_attr_list(struct ntfs_inode *ni, struct ATTRIB *attr) if (!attr->non_res) { lsize = le32_to_cpu(attr->res.data_size); + if (!lsize) { + err = -EINVAL; + goto out; + } + /* attr is resident: lsize < record_size (1K or 4K) */ le = kvmalloc(al_aligned(lsize), GFP_KERNEL); if (!le) { @@ -66,6 +71,10 @@ int ntfs_load_attr_list(struct ntfs_inode *ni, struct ATTRIB *attr) u16 run_off = le16_to_cpu(attr->nres.run_off); lsize = le64_to_cpu(attr->nres.data_size); + if (!lsize) { + err = -EINVAL; + goto out; + } run_init(&ni->attr_list.run); diff --git a/fs/ntfs3/dir.c b/fs/ntfs3/dir.c index 1b5c865a0339a..460df046482c7 100644 --- a/fs/ntfs3/dir.c +++ b/fs/ntfs3/dir.c @@ -424,8 +424,7 @@ static int ntfs_readdir(struct file *file, struct dir_context *ctx) if (!dir_emit_dots(file, ctx)) return 0; - /* Allocate PATH_MAX bytes. */ - name = __getname(); + name = kmalloc(PATH_MAX, GFP_KERNEL); if (!name) return -ENOMEM; @@ -503,7 +502,7 @@ static int ntfs_readdir(struct file *file, struct dir_context *ctx) out: - __putname(name); + kfree(name); put_indx_node(node); if (err == 1) { diff --git a/fs/ntfs3/file.c b/fs/ntfs3/file.c index 83f0072f0896c..cd7aaeef45fe9 100644 --- a/fs/ntfs3/file.c +++ b/fs/ntfs3/file.c @@ -930,7 +930,7 @@ static int ntfs_get_frame_pages(struct address_space *mapping, pgoff_t index, folio = __filemap_get_folio(mapping, index, FGP_LOCK | FGP_ACCESSED | FGP_CREAT, - gfp_mask); + gfp_mask | __GFP_ZERO); if (IS_ERR(folio)) { while (npages--) { folio = page_folio(pages[npages]); @@ -1012,8 +1012,12 @@ static ssize_t ntfs_compress_write(struct kiocb *iocb, struct iov_iter *from) goto out; if (lcn == SPARSE_LCN) { - ni->i_valid = valid = - frame_vbo + ((u64)clen << sbi->cluster_bits); + valid = frame_vbo + ((u64)clen << sbi->cluster_bits); + if (ni->i_valid == valid) { + err = -EINVAL; + goto out; + } + ni->i_valid = valid; continue; } diff --git a/fs/ntfs3/fslog.c b/fs/ntfs3/fslog.c index 38934e6978ece..28bd611f580d9 100644 --- a/fs/ntfs3/fslog.c +++ b/fs/ntfs3/fslog.c @@ -3429,6 +3429,9 @@ static int do_action(struct ntfs_log *log, struct OPEN_ATTR_ENRTY *oe, e1 = Add2Ptr(attr, le16_to_cpu(lrh->attr_off)); esize = le16_to_cpu(e1->size); + if (PtrOffset(e1, Add2Ptr(hdr, used)) < esize) + goto dirty_vol; + e2 = Add2Ptr(e1, esize); memmove(e1, e2, PtrOffset(e2, Add2Ptr(hdr, used))); diff --git a/fs/ntfs3/fsntfs.c b/fs/ntfs3/fsntfs.c index 5ae910e9ecbda..83df92df1ee0c 100644 --- a/fs/ntfs3/fsntfs.c +++ b/fs/ntfs3/fsntfs.c @@ -1252,6 +1252,12 @@ int ntfs_read_run_nb(struct ntfs_sb_info *sbi, const struct runs_tree *run, } while (len32); + if (!run) { + err = -EINVAL; + goto out; + } + + /* Get next fragment to read. */ vcn_next = vcn + clen; if (!run_get_entry(run, ++idx, &vcn, &lcn, &clen) || vcn != vcn_next) { @@ -2640,7 +2646,7 @@ int ntfs_set_label(struct ntfs_sb_info *sbi, u8 *label, int len) u32 uni_bytes; struct ntfs_inode *ni = sbi->volume.ni; /* Allocate PATH_MAX bytes. */ - struct cpu_str *uni = __getname(); + struct cpu_str *uni = kmalloc(PATH_MAX, GFP_KERNEL); if (!uni) return -ENOMEM; @@ -2684,6 +2690,6 @@ int ntfs_set_label(struct ntfs_sb_info *sbi, u8 *label, int len) err = _ni_write_inode(&ni->vfs_inode, 0); out: - __putname(uni); + kfree(uni); return err; } diff --git a/fs/ntfs3/index.c b/fs/ntfs3/index.c index 6d1bf890929d9..050b3709e0204 100644 --- a/fs/ntfs3/index.c +++ b/fs/ntfs3/index.c @@ -1190,7 +1190,12 @@ int indx_find(struct ntfs_index *indx, struct ntfs_inode *ni, return -EINVAL; } - fnd_push(fnd, node, e); + err = fnd_push(fnd, node, e); + + if (err) { + put_indx_node(node); + return err; + } } *entry = e; diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c index 164fd63dff40d..205baa791f998 100644 --- a/fs/ntfs3/inode.c +++ b/fs/ntfs3/inode.c @@ -1279,7 +1279,7 @@ int ntfs_create_inode(struct mnt_idmap *idmap, struct inode *dir, fa |= FILE_ATTRIBUTE_READONLY; /* Allocate PATH_MAX bytes. */ - new_de = kmem_cache_zalloc(names_cachep, GFP_KERNEL); + new_de = kzalloc(PATH_MAX, GFP_KERNEL); if (!new_de) { err = -ENOMEM; goto out1; @@ -1699,7 +1699,7 @@ int ntfs_create_inode(struct mnt_idmap *idmap, struct inode *dir, ntfs_mark_rec_free(sbi, ino, false); out2: - __putname(new_de); + kfree(new_de); kfree(rp); out1: @@ -1720,7 +1720,7 @@ int ntfs_link_inode(struct inode *inode, struct dentry *dentry) struct NTFS_DE *de; /* Allocate PATH_MAX bytes. */ - de = kmem_cache_zalloc(names_cachep, GFP_KERNEL); + de = kzalloc(PATH_MAX, GFP_KERNEL); if (!de) return -ENOMEM; @@ -1734,7 +1734,7 @@ int ntfs_link_inode(struct inode *inode, struct dentry *dentry) err = ni_add_name(ntfs_i(d_inode(dentry->d_parent)), ni, de); out: - __putname(de); + kfree(de); return err; } @@ -1757,8 +1757,7 @@ int ntfs_unlink_inode(struct inode *dir, const struct dentry *dentry) if (ntfs_is_meta_file(sbi, ni->mi.rno)) return -EINVAL; - /* Allocate PATH_MAX bytes. */ - de = kmem_cache_zalloc(names_cachep, GFP_KERNEL); + de = kzalloc(PATH_MAX, GFP_KERNEL); if (!de) return -ENOMEM; @@ -1794,7 +1793,7 @@ int ntfs_unlink_inode(struct inode *dir, const struct dentry *dentry) out: ni_unlock(ni); - __putname(de); + kfree(de); return err; } diff --git a/fs/ntfs3/namei.c b/fs/ntfs3/namei.c index 82c8ae56beee6..612923d93f4dc 100644 --- a/fs/ntfs3/namei.c +++ b/fs/ntfs3/namei.c @@ -68,7 +68,7 @@ static struct dentry *ntfs_lookup(struct inode *dir, struct dentry *dentry, u32 flags) { struct ntfs_inode *ni = ntfs_i(dir); - struct cpu_str *uni = __getname(); + struct cpu_str *uni = kmalloc(PATH_MAX, GFP_KERNEL); struct inode *inode; int err; @@ -85,7 +85,7 @@ static struct dentry *ntfs_lookup(struct inode *dir, struct dentry *dentry, inode = dir_search_u(dir, uni, NULL); ni_unlock(ni); } - __putname(uni); + kfree(uni); } /* @@ -303,8 +303,7 @@ static int ntfs_rename(struct mnt_idmap *idmap, struct inode *dir, return err; } - /* Allocate PATH_MAX bytes. */ - de = __getname(); + de = kmalloc(PATH_MAX, GFP_KERNEL); if (!de) return -ENOMEM; @@ -349,7 +348,7 @@ static int ntfs_rename(struct mnt_idmap *idmap, struct inode *dir, ni_unlock(ni); ni_unlock(dir_ni); out: - __putname(de); + kfree(de); return err; } @@ -407,7 +406,7 @@ static int ntfs_d_hash(const struct dentry *dentry, struct qstr *name) /* * Try slow way with current upcase table */ - uni = kmem_cache_alloc(names_cachep, GFP_NOWAIT); + uni = kmalloc(PATH_MAX, GFP_NOWAIT); if (!uni) return -ENOMEM; @@ -429,7 +428,7 @@ static int ntfs_d_hash(const struct dentry *dentry, struct qstr *name) err = 0; out: - kmem_cache_free(names_cachep, uni); + kfree(uni); return err; } @@ -468,7 +467,7 @@ static int ntfs_d_compare(const struct dentry *dentry, unsigned int len1, * Try slow way with current upcase table */ sbi = dentry->d_sb->s_fs_info; - uni1 = __getname(); + uni1 = kmalloc(PATH_MAX, GFP_NOWAIT); if (!uni1) return -ENOMEM; @@ -498,7 +497,7 @@ static int ntfs_d_compare(const struct dentry *dentry, unsigned int len1, ret = !ntfs_cmp_names_cpu(uni1, uni2, sbi->upcase, false) ? 0 : 1; out: - __putname(uni1); + kfree(uni1); return ret; } diff --git a/fs/ntfs3/run.c b/fs/ntfs3/run.c index 5df55e4adbb11..21decc2922823 100644 --- a/fs/ntfs3/run.c +++ b/fs/ntfs3/run.c @@ -1124,11 +1124,14 @@ int run_unpack_ex(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, struct rw_semaphore *lock = is_mounted(sbi) ? &sbi->mft.ni->file.run_lock : NULL; - if (lock) - down_read(lock); - ntfs_refresh_zone(sbi); - if (lock) - up_read(lock); + if (lock) { + if (down_read_trylock(lock)) { + ntfs_refresh_zone(sbi); + up_read(lock); + } + } else { + ntfs_refresh_zone(sbi); + } } up_write(&wnd->rw_lock); if (err) diff --git a/fs/ntfs3/xattr.c b/fs/ntfs3/xattr.c index e519e21596a70..142ecb3847e59 100644 --- a/fs/ntfs3/xattr.c +++ b/fs/ntfs3/xattr.c @@ -556,8 +556,7 @@ struct posix_acl *ntfs_get_acl(struct mnt_idmap *idmap, struct dentry *dentry, if (unlikely(is_bad_ni(ni))) return ERR_PTR(-EINVAL); - /* Allocate PATH_MAX bytes. */ - buf = __getname(); + buf = kmalloc(PATH_MAX, GFP_KERNEL); if (!buf) return ERR_PTR(-ENOMEM); @@ -588,7 +587,7 @@ struct posix_acl *ntfs_get_acl(struct mnt_idmap *idmap, struct dentry *dentry, if (!IS_ERR(acl)) set_cached_acl(inode, type, acl); - __putname(buf); + kfree(buf); return acl; } diff --git a/fs/ocfs2/xattr.c b/fs/ocfs2/xattr.c index d70a20d29e3e9..64ba3946f8408 100644 --- a/fs/ocfs2/xattr.c +++ b/fs/ocfs2/xattr.c @@ -6364,6 +6364,10 @@ static int ocfs2_reflink_xattr_header(handle_t *handle, (void *)last - (void *)xe); memset(last, 0, sizeof(struct ocfs2_xattr_entry)); + last = &new_xh->xh_entries[le16_to_cpu(new_xh->xh_count)] - 1; + } else { + memset(xe, 0, sizeof(struct ocfs2_xattr_entry)); + last = NULL; } /* diff --git a/fs/overlayfs/readdir.c b/fs/overlayfs/readdir.c index 1e9792cc557b8..3c27e7a16f94b 100644 --- a/fs/overlayfs/readdir.c +++ b/fs/overlayfs/readdir.c @@ -761,7 +761,7 @@ static bool ovl_fill_real(struct dir_context *ctx, const char *name, struct dir_context *orig_ctx = rdt->orig_ctx; bool res; - if (rdt->parent_ino && strcmp(name, "..") == 0) { + if (rdt->parent_ino && namelen == 2 && !strncmp(name, "..", 2)) { ino = rdt->parent_ino; } else if (rdt->cache) { struct ovl_cache_entry *p; diff --git a/fs/pidfs.c b/fs/pidfs.c index f4d7dac1b4494..34987fcdd9a87 100644 --- a/fs/pidfs.c +++ b/fs/pidfs.c @@ -321,7 +321,7 @@ static long pidfd_info(struct file *file, unsigned int cmd, unsigned long arg) * namespace hierarchy. */ if (!pid_in_current_pidns(pid)) - return -ESRCH; + return -EREMOTE; attr = READ_ONCE(pid->attr); if (mask & PIDFD_INFO_EXIT) { diff --git a/fs/proc/array.c b/fs/proc/array.c index 2ae63189091e0..038d4b57127fe 100644 --- a/fs/proc/array.c +++ b/fs/proc/array.c @@ -529,7 +529,7 @@ static int do_task_stat(struct seq_file *m, struct pid_namespace *ns, } sid = task_session_nr_ns(task, ns); - ppid = task_tgid_nr_ns(task->real_parent, ns); + ppid = task_ppid_nr_ns(task, ns); pgid = task_pgrp_nr_ns(task, ns); unlock_task_sighand(task, &flags); diff --git a/fs/proc/base.c b/fs/proc/base.c index 6299878e3d97e..a314fd9af17c0 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -2128,6 +2128,9 @@ bool proc_fill_cache(struct file *file, struct dir_context *ctx, ino_t ino = 1; child = try_lookup_noperm(&qname, dir); + if (IS_ERR(child)) + goto end_instantiate; + if (!child) { DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq); child = d_alloc_parallel(dir, &qname, &wq); diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 2ee152a318f50..b490245ff9be7 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -780,7 +780,7 @@ static int do_procmap_query(struct mm_struct *mm, void __user *uarg) } else { if (karg.build_id_size < build_id_sz) { err = -ENAMETOOLONG; - goto out; + goto out_file; } karg.build_id_size = build_id_sz; } @@ -808,6 +808,7 @@ static int do_procmap_query(struct mm_struct *mm, void __user *uarg) out: query_vma_teardown(&lock_ctx); mmput(mm); +out_file: if (vm_file) fput(vm_file); kfree(name_buf); diff --git a/fs/pstore/ram_core.c b/fs/pstore/ram_core.c index f1848cdd6d348..7b6d6378a3b87 100644 --- a/fs/pstore/ram_core.c +++ b/fs/pstore/ram_core.c @@ -298,6 +298,17 @@ void persistent_ram_save_old(struct persistent_ram_zone *prz) if (!size) return; + /* + * If the existing buffer is differently sized, free it so a new + * one is allocated. This can happen when persistent_ram_save_old() + * is called early in boot and later for a timer-triggered + * survivable crash when the crash dumps don't match in size + * (which would be extremely unlikely given kmsg buffers usually + * exceed prz buffer sizes). + */ + if (prz->old_log && prz->old_log_size != size) + persistent_ram_free_old(prz); + if (!prz->old_log) { persistent_ram_ecc_old(prz); prz->old_log = kvzalloc(size, GFP_KERNEL); @@ -446,6 +457,13 @@ static void *persistent_ram_vmap(phys_addr_t start, size_t size, vaddr = vmap(pages, page_count, VM_MAP | VM_IOREMAP, prot); kfree(pages); + /* + * vmap() may fail and return NULL. Do not add the offset in this + * case, otherwise a NULL mapping would appear successful. + */ + if (!vaddr) + return NULL; + /* * Since vmap() uses page granularity, we must add the offset * into the page here, to get the byte granularity address diff --git a/fs/quota/quota.c b/fs/quota/quota.c index 7c2b75a444852..de4379a9c7920 100644 --- a/fs/quota/quota.c +++ b/fs/quota/quota.c @@ -899,6 +899,7 @@ static struct super_block *quotactl_block(const char __user *special, int cmd) sb_start_write(sb); sb_end_write(sb); put_super(sb); + cond_resched(); goto retry; } return sb; diff --git a/fs/romfs/super.c b/fs/romfs/super.c index 0addcc849ff2c..e83f9b78d7a16 100644 --- a/fs/romfs/super.c +++ b/fs/romfs/super.c @@ -458,7 +458,10 @@ static int romfs_fill_super(struct super_block *sb, struct fs_context *fc) #ifdef CONFIG_BLOCK if (!sb->s_mtd) { - sb_set_blocksize(sb, ROMBSIZE); + if (!sb_set_blocksize(sb, ROMBSIZE)) { + errorf(fc, "romfs: unable to set blocksize\n"); + return -EINVAL; + } } else { sb->s_blocksize = ROMBSIZE; sb->s_blocksize_bits = blksize_bits(ROMBSIZE); diff --git a/fs/smb/client/cached_dir.c b/fs/smb/client/cached_dir.c index e3ea6fe7edb47..09939fe9666ea 100644 --- a/fs/smb/client/cached_dir.c +++ b/fs/smb/client/cached_dir.c @@ -788,11 +788,11 @@ static void cfids_laundromat_worker(struct work_struct *work) cfid->dentry = NULL; if (cfid->is_open) { - spin_lock(&cifs_tcp_ses_lock); + spin_lock(&cfid->tcon->tc_lock); ++cfid->tcon->tc_count; trace_smb3_tcon_ref(cfid->tcon->debug_id, cfid->tcon->tc_count, netfs_trace_tcon_ref_get_cached_laundromat); - spin_unlock(&cifs_tcp_ses_lock); + spin_unlock(&cfid->tcon->tc_lock); queue_work(serverclose_wq, &cfid->close_work); } else /* diff --git a/fs/smb/client/connect.c b/fs/smb/client/connect.c index 2f94d93b95e91..dd7f48f530979 100644 --- a/fs/smb/client/connect.c +++ b/fs/smb/client/connect.c @@ -2235,7 +2235,6 @@ cifs_set_cifscreds(struct smb3_fs_context *ctx, struct cifs_ses *ses) /* find first : in payload */ payload = upayload->data; delim = strnchr(payload, upayload->datalen, ':'); - cifs_dbg(FYI, "payload=%s\n", payload); if (!delim) { cifs_dbg(FYI, "Unable to find ':' in payload (datalen=%d)\n", upayload->datalen); @@ -4267,7 +4266,9 @@ cifs_setup_session(const unsigned int xid, struct cifs_ses *ses, ses->ses_status = SES_IN_SETUP; /* force iface_list refresh */ + spin_lock(&ses->iface_lock); ses->iface_last_update = 0; + spin_unlock(&ses->iface_lock); } spin_unlock(&ses->ses_lock); diff --git a/fs/smb/client/smb2file.c b/fs/smb/client/smb2file.c index 03f90553d8319..e6cdf2efc7f4f 100644 --- a/fs/smb/client/smb2file.c +++ b/fs/smb/client/smb2file.c @@ -178,6 +178,8 @@ int smb2_open_file(const unsigned int xid, struct cifs_open_parms *oparms, __u32 &err_buftype); if (rc == -EACCES && retry_without_read_attributes) { free_rsp_buf(err_buftype, err_iov.iov_base); + memset(&err_iov, 0, sizeof(err_iov)); + err_buftype = CIFS_NO_BUFFER; oparms->desired_access &= ~FILE_READ_ATTRIBUTES; rc = SMB2_open(xid, oparms, smb2_path, &smb2_oplock, smb2_data, NULL, &err_iov, &err_buftype); diff --git a/fs/smb/client/smb2inode.c b/fs/smb/client/smb2inode.c index 69cb81fa0d3ab..a5f9f73ac91b9 100644 --- a/fs/smb/client/smb2inode.c +++ b/fs/smb/client/smb2inode.c @@ -322,7 +322,7 @@ static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, cfile->fid.volatile_fid, SMB_FIND_FILE_POSIX_INFO, SMB2_O_INFO_FILE, 0, - sizeof(struct smb311_posix_qinfo *) + + sizeof(struct smb311_posix_qinfo) + (PATH_MAX * 2) + (sizeof(struct smb_sid) * 2), 0, NULL); } else { @@ -332,7 +332,7 @@ static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, COMPOUND_FID, SMB_FIND_FILE_POSIX_INFO, SMB2_O_INFO_FILE, 0, - sizeof(struct smb311_posix_qinfo *) + + sizeof(struct smb311_posix_qinfo) + (PATH_MAX * 2) + (sizeof(struct smb_sid) * 2), 0, NULL); } @@ -1205,6 +1205,7 @@ smb2_unlink(const unsigned int xid, struct cifs_tcon *tcon, const char *name, memset(resp_buftype, 0, sizeof(resp_buftype)); memset(rsp_iov, 0, sizeof(rsp_iov)); + memset(open_iov, 0, sizeof(open_iov)); rqst[0].rq_iov = open_iov; rqst[0].rq_nvec = ARRAY_SIZE(open_iov); @@ -1229,14 +1230,15 @@ smb2_unlink(const unsigned int xid, struct cifs_tcon *tcon, const char *name, creq = rqst[0].rq_iov[0].iov_base; creq->ShareAccess = FILE_SHARE_DELETE_LE; + memset(&close_iov, 0, sizeof(close_iov)); rqst[1].rq_iov = &close_iov; rqst[1].rq_nvec = 1; rc = SMB2_close_init(tcon, server, &rqst[1], COMPOUND_FID, COMPOUND_FID, false); - smb2_set_related(&rqst[1]); if (rc) goto err_free; + smb2_set_related(&rqst[1]); if (retries) { for (int i = 0; i < ARRAY_SIZE(rqst); i++) diff --git a/fs/smb/client/smb2misc.c b/fs/smb/client/smb2misc.c index 96bfe4c63ccf9..a6fc8ff398ebf 100644 --- a/fs/smb/client/smb2misc.c +++ b/fs/smb/client/smb2misc.c @@ -819,14 +819,14 @@ smb2_handle_cancelled_close(struct cifs_tcon *tcon, __u64 persistent_fid, int rc; cifs_dbg(FYI, "%s: tc_count=%d\n", __func__, tcon->tc_count); - spin_lock(&cifs_tcp_ses_lock); + spin_lock(&tcon->tc_lock); if (tcon->tc_count <= 0) { struct TCP_Server_Info *server = NULL; trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, netfs_trace_tcon_ref_see_cancelled_close); WARN_ONCE(tcon->tc_count < 0, "tcon refcount is negative"); - spin_unlock(&cifs_tcp_ses_lock); + spin_unlock(&tcon->tc_lock); if (tcon->ses) { server = tcon->ses->server; @@ -840,7 +840,7 @@ smb2_handle_cancelled_close(struct cifs_tcon *tcon, __u64 persistent_fid, tcon->tc_count++; trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, netfs_trace_tcon_ref_get_cancelled_close); - spin_unlock(&cifs_tcp_ses_lock); + spin_unlock(&tcon->tc_lock); rc = __smb2_handle_cancelled_cmd(tcon, SMB2_CLOSE_HE, 0, persistent_fid, volatile_fid); diff --git a/fs/smb/client/smb2ops.c b/fs/smb/client/smb2ops.c index 1e39f2165e427..9c22daff24970 100644 --- a/fs/smb/client/smb2ops.c +++ b/fs/smb/client/smb2ops.c @@ -637,13 +637,6 @@ parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, p = buf; spin_lock(&ses->iface_lock); - /* do not query too frequently, this time with lock held */ - if (ses->iface_last_update && - time_before(jiffies, ses->iface_last_update + - (SMB_INTERFACE_POLL_INTERVAL * HZ))) { - spin_unlock(&ses->iface_lock); - return 0; - } /* * Go through iface_list and mark them as inactive @@ -666,7 +659,6 @@ parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, "Empty network interface list returned by server %s\n", ses->server->hostname); rc = -EOPNOTSUPP; - ses->iface_last_update = jiffies; goto out; } @@ -795,8 +787,6 @@ parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, + sizeof(p->Next) && p->Next)) cifs_dbg(VFS, "%s: incomplete interface info\n", __func__); - ses->iface_last_update = jiffies; - out: /* * Go through the list again and put the inactive entries @@ -825,10 +815,17 @@ SMB3_request_interfaces(const unsigned int xid, struct cifs_tcon *tcon, bool in_ struct TCP_Server_Info *pserver; /* do not query too frequently */ + spin_lock(&ses->iface_lock); if (ses->iface_last_update && time_before(jiffies, ses->iface_last_update + - (SMB_INTERFACE_POLL_INTERVAL * HZ))) + (SMB_INTERFACE_POLL_INTERVAL * HZ))) { + spin_unlock(&ses->iface_lock); return 0; + } + + ses->iface_last_update = jiffies; + + spin_unlock(&ses->iface_lock); rc = SMB2_ioctl(xid, tcon, NO_FILE_ID, NO_FILE_ID, FSCTL_QUERY_NETWORK_INTERFACE_INFO, @@ -1187,6 +1184,7 @@ smb2_set_ea(const unsigned int xid, struct cifs_tcon *tcon, replay_again: /* reinitialize for possible replay */ + used_len = 0; flags = CIFS_CP_CREATE_CLOSE_OP; oplock = SMB2_OPLOCK_LEVEL_NONE; server = cifs_pick_channel(ses); @@ -1585,6 +1583,7 @@ smb2_ioctl_query_info(const unsigned int xid, replay_again: /* reinitialize for possible replay */ + buffer = NULL; flags = CIFS_CP_CREATE_CLOSE_OP; oplock = SMB2_OPLOCK_LEVEL_NONE; server = cifs_pick_channel(ses); @@ -3077,7 +3076,9 @@ smb2_get_dfs_refer(const unsigned int xid, struct cifs_ses *ses, struct cifs_tcon, tcon_list); if (tcon) { + spin_lock(&tcon->tc_lock); tcon->tc_count++; + spin_unlock(&tcon->tc_lock); trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, netfs_trace_tcon_ref_get_dfs_refer); } @@ -3146,13 +3147,9 @@ smb2_get_dfs_refer(const unsigned int xid, struct cifs_ses *ses, out: if (tcon && !tcon->ipc) { /* ipc tcons are not refcounted */ - spin_lock(&cifs_tcp_ses_lock); - tcon->tc_count--; + cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_dfs_refer); trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, netfs_trace_tcon_ref_dec_dfs_refer); - /* tc_count can never go negative */ - WARN_ON(tcon->tc_count < 0); - spin_unlock(&cifs_tcp_ses_lock); } kfree(utf16_path); kfree(dfs_req); diff --git a/fs/smb/client/smb2pdu.c b/fs/smb/client/smb2pdu.c index e661d40213eab..309e2fcabc087 100644 --- a/fs/smb/client/smb2pdu.c +++ b/fs/smb/client/smb2pdu.c @@ -1650,19 +1650,17 @@ SMB2_auth_kerberos(struct SMB2_sess_data *sess_data) is_binding = (ses->ses_status == SES_GOOD); spin_unlock(&ses->ses_lock); - /* keep session key if binding */ - if (!is_binding) { - kfree_sensitive(ses->auth_key.response); - ses->auth_key.response = kmemdup(msg->data, msg->sesskey_len, - GFP_KERNEL); - if (!ses->auth_key.response) { - cifs_dbg(VFS, "Kerberos can't allocate (%u bytes) memory\n", - msg->sesskey_len); - rc = -ENOMEM; - goto out_put_spnego_key; - } - ses->auth_key.len = msg->sesskey_len; + kfree_sensitive(ses->auth_key.response); + ses->auth_key.response = kmemdup(msg->data, + msg->sesskey_len, + GFP_KERNEL); + if (!ses->auth_key.response) { + cifs_dbg(VFS, "%s: can't allocate (%u bytes) memory\n", + __func__, msg->sesskey_len); + rc = -ENOMEM; + goto out_put_spnego_key; } + ses->auth_key.len = msg->sesskey_len; sess_data->iov[1].iov_base = msg->data + msg->sesskey_len; sess_data->iov[1].iov_len = msg->secblob_len; @@ -2845,6 +2843,7 @@ int smb311_posix_mkdir(const unsigned int xid, struct inode *inode, replay_again: /* reinitialize for possible replay */ + pc_buf = NULL; flags = 0; n_iov = 2; server = cifs_pick_channel(ses); @@ -3916,7 +3915,7 @@ int SMB311_posix_query_info(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid, u64 volatile_fid, struct smb311_posix_qinfo *data, u32 *plen) { - size_t output_len = sizeof(struct smb311_posix_qinfo *) + + size_t output_len = sizeof(struct smb311_posix_qinfo) + (sizeof(struct smb_sid) * 2) + (PATH_MAX * 2); *plen = 0; @@ -4174,7 +4173,9 @@ void smb2_reconnect_server(struct work_struct *work) list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { if (tcon->need_reconnect || tcon->need_reopen_files) { + spin_lock(&tcon->tc_lock); tcon->tc_count++; + spin_unlock(&tcon->tc_lock); trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, netfs_trace_tcon_ref_get_reconnect_server); list_add_tail(&tcon->rlist, &tmp_list); diff --git a/fs/smb/client/smbdirect.c b/fs/smb/client/smbdirect.c index 01d55bcc6d0f9..c8cef098d4806 100644 --- a/fs/smb/client/smbdirect.c +++ b/fs/smb/client/smbdirect.c @@ -101,8 +101,23 @@ int smbd_send_credit_target = 255; /* The maximum single message size can be sent to remote peer */ int smbd_max_send_size = 1364; -/* The maximum fragmented upper-layer payload receive size supported */ -int smbd_max_fragmented_recv_size = 1024 * 1024; +/* + * The maximum fragmented upper-layer payload receive size supported + * + * Assume max_payload_per_credit is + * smbd_max_receive_size - 24 = 1340 + * + * The maximum number would be + * smbd_receive_credit_max * max_payload_per_credit + * + * 1340 * 255 = 341700 (0x536C4) + * + * The minimum value from the spec is 131072 (0x20000) + * + * For now we use the logic we used in ksmbd before: + * (1364 * 255) / 2 = 173910 (0x2A756) + */ +int smbd_max_fragmented_recv_size = (1364 * 255) / 2; /* The maximum single-message size which can be received */ int smbd_max_receive_size = 1364; diff --git a/fs/smb/client/trace.h b/fs/smb/client/trace.h index 28e00c34df1cb..e592b2627119f 100644 --- a/fs/smb/client/trace.h +++ b/fs/smb/client/trace.h @@ -59,6 +59,7 @@ EM(netfs_trace_tcon_ref_put_cancelled_close_fid, "PUT Cn-Fid") \ EM(netfs_trace_tcon_ref_put_cancelled_mid, "PUT Cn-Mid") \ EM(netfs_trace_tcon_ref_put_mnt_ctx, "PUT MntCtx") \ + EM(netfs_trace_tcon_ref_put_dfs_refer, "PUT DfsRfr") \ EM(netfs_trace_tcon_ref_put_reconnect_server, "PUT Reconn") \ EM(netfs_trace_tcon_ref_put_tlink, "PUT Tlink ") \ EM(netfs_trace_tcon_ref_see_cancelled_close, "SEE Cn-Cls") \ diff --git a/fs/smb/client/transport.c b/fs/smb/client/transport.c index 915cedde5d664..2d7140767dc43 100644 --- a/fs/smb/client/transport.c +++ b/fs/smb/client/transport.c @@ -806,16 +806,21 @@ cifs_cancelled_callback(struct mid_q_entry *mid) } /* - * Return a channel (master if none) of @ses that can be used to send - * regular requests. + * cifs_pick_channel - pick an eligible channel for network operations * - * If we are currently binding a new channel (negprot/sess.setup), - * return the new incomplete channel. + * @ses: session reference + * + * Select an eligible channel (not terminating and not marked as needing + * reconnect), preferring the least loaded one. If no eligible channel is + * found, fall back to the primary channel (index 0). + * + * Return: TCP_Server_Info pointer for the chosen channel, or NULL if @ses is + * NULL. */ struct TCP_Server_Info *cifs_pick_channel(struct cifs_ses *ses) { uint index = 0; - unsigned int min_in_flight = UINT_MAX, max_in_flight = 0; + unsigned int min_in_flight = UINT_MAX; struct TCP_Server_Info *server = NULL; int i, start, cur; @@ -845,14 +850,8 @@ struct TCP_Server_Info *cifs_pick_channel(struct cifs_ses *ses) min_in_flight = server->in_flight; index = cur; } - if (server->in_flight > max_in_flight) - max_in_flight = server->in_flight; } - /* if all channels are equally loaded, fall back to round-robin */ - if (min_in_flight == max_in_flight) - index = (uint)start % ses->chan_count; - server = ses->chans[index].server; spin_unlock(&ses->chan_lock); diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c index bf8c480594367..e52b9136abbff 100644 --- a/fs/smb/server/smb2pdu.c +++ b/fs/smb/server/smb2pdu.c @@ -6117,14 +6117,14 @@ static int smb2_create_link(struct ksmbd_work *work, rc = -EINVAL; ksmbd_debug(SMB, "cannot delete %s\n", link_name); - goto out; } } else { rc = -EEXIST; ksmbd_debug(SMB, "link already exists\n"); - goto out; } ksmbd_vfs_kern_path_unlock(&path); + if (rc) + goto out; } rc = ksmbd_vfs_link(work, target_name, link_name); if (rc) diff --git a/fs/smb/server/transport_rdma.c b/fs/smb/server/transport_rdma.c index 4e74934e1f277..f00bb28a4aa88 100644 --- a/fs/smb/server/transport_rdma.c +++ b/fs/smb/server/transport_rdma.c @@ -2414,9 +2414,9 @@ static int smb_direct_prepare(struct ksmbd_transport *t) goto put; req = (struct smbdirect_negotiate_req *)recvmsg->packet; - sp->max_recv_size = min_t(int, sp->max_recv_size, + sp->max_recv_size = min_t(u32, sp->max_recv_size, le32_to_cpu(req->preferred_send_size)); - sp->max_send_size = min_t(int, sp->max_send_size, + sp->max_send_size = min_t(u32, sp->max_send_size, le32_to_cpu(req->max_receive_size)); sp->max_fragmented_send_size = le32_to_cpu(req->max_fragmented_size); diff --git a/fs/squashfs/cache.c b/fs/squashfs/cache.c index 181260e72680c..92fb857d2c761 100644 --- a/fs/squashfs/cache.c +++ b/fs/squashfs/cache.c @@ -344,6 +344,9 @@ int squashfs_read_metadata(struct super_block *sb, void *buffer, if (unlikely(length < 0)) return -EIO; + if (unlikely(*offset < 0 || *offset >= SQUASHFS_METADATA_SIZE)) + return -EIO; + while (length) { entry = squashfs_cache_get(sb, msblk->block_cache, *block, 0); if (entry->error) { diff --git a/fs/tests/exec_kunit.c b/fs/tests/exec_kunit.c index 7c77d039680bb..f412d1a0f6bba 100644 --- a/fs/tests/exec_kunit.c +++ b/fs/tests/exec_kunit.c @@ -87,9 +87,6 @@ static const struct bprm_stack_limits_result bprm_stack_limits_results[] = { .argc = 0, .envc = ARG_MAX / sizeof(void *) - 1 }, .expected_argmin = ULONG_MAX - sizeof(void *) }, /* Raising rlim_stack / 4 to _STK_LIM / 4 * 3 will see more space. */ - { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * (_STK_LIM / 4 * 3), - .argc = 0, .envc = 0 }, - .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * (_STK_LIM / 4 * 3), .argc = 0, .envc = 0 }, .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, @@ -103,9 +100,6 @@ static const struct bprm_stack_limits_result bprm_stack_limits_results[] = { { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * _STK_LIM, .argc = 0, .envc = 0 }, .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, - { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * _STK_LIM, - .argc = 0, .envc = 0 }, - .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, }; static void exec_test_bprm_stack_limits(struct kunit *test) diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index 8c04acd30d489..b88e65c7e45de 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -50,7 +50,6 @@ STATIC int xfs_attr_shortform_addname(xfs_da_args_t *args); */ STATIC int xfs_attr_leaf_get(xfs_da_args_t *args); STATIC int xfs_attr_leaf_removename(xfs_da_args_t *args); -STATIC int xfs_attr_leaf_hasname(struct xfs_da_args *args, struct xfs_buf **bp); /* * Internal routines when attribute list is more than one block. @@ -979,11 +978,12 @@ xfs_attr_lookup( return error; if (xfs_attr_is_leaf(dp)) { - error = xfs_attr_leaf_hasname(args, &bp); - - if (bp) - xfs_trans_brelse(args->trans, bp); - + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, + 0, &bp); + if (error) + return error; + error = xfs_attr3_leaf_lookup_int(bp, args); + xfs_trans_brelse(args->trans, bp); return error; } @@ -1222,27 +1222,6 @@ xfs_attr_shortform_addname( * External routines when attribute list is one block *========================================================================*/ -/* - * Return EEXIST if attr is found, or ENOATTR if not - */ -STATIC int -xfs_attr_leaf_hasname( - struct xfs_da_args *args, - struct xfs_buf **bp) -{ - int error = 0; - - error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, 0, bp); - if (error) - return error; - - error = xfs_attr3_leaf_lookup_int(*bp, args); - if (error != -ENOATTR && error != -EEXIST) - xfs_trans_brelse(args->trans, *bp); - - return error; -} - /* * Remove a name from the leaf attribute list structure * @@ -1253,25 +1232,22 @@ STATIC int xfs_attr_leaf_removename( struct xfs_da_args *args) { - struct xfs_inode *dp; - struct xfs_buf *bp; + struct xfs_inode *dp = args->dp; int error, forkoff; + struct xfs_buf *bp; trace_xfs_attr_leaf_removename(args); - /* - * Remove the attribute. - */ - dp = args->dp; - - error = xfs_attr_leaf_hasname(args, &bp); - if (error == -ENOATTR) { + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, 0, &bp); + if (error) + return error; + error = xfs_attr3_leaf_lookup_int(bp, args); + if (error != -EEXIST) { xfs_trans_brelse(args->trans, bp); - if (args->op_flags & XFS_DA_OP_RECOVERY) + if (error == -ENOATTR && (args->op_flags & XFS_DA_OP_RECOVERY)) return 0; return error; - } else if (error != -EEXIST) - return error; + } xfs_attr3_leaf_remove(bp, args); @@ -1295,23 +1271,20 @@ xfs_attr_leaf_removename( * Returns 0 on successful retrieval, otherwise an error. */ STATIC int -xfs_attr_leaf_get(xfs_da_args_t *args) +xfs_attr_leaf_get( + struct xfs_da_args *args) { - struct xfs_buf *bp; - int error; + struct xfs_buf *bp; + int error; trace_xfs_attr_leaf_get(args); - error = xfs_attr_leaf_hasname(args, &bp); - - if (error == -ENOATTR) { - xfs_trans_brelse(args->trans, bp); - return error; - } else if (error != -EEXIST) + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, 0, &bp); + if (error) return error; - - - error = xfs_attr3_leaf_getvalue(bp, args); + error = xfs_attr3_leaf_lookup_int(bp, args); + if (error == -EEXIST) + error = xfs_attr3_leaf_getvalue(bp, args); xfs_trans_brelse(args->trans, bp); return error; } diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index 91c1b30ebaab3..b858e3c2ad50a 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -1476,6 +1476,7 @@ xfs_attr3_leaf_add_work( struct xfs_attr_leaf_name_local *name_loc; struct xfs_attr_leaf_name_remote *name_rmt; struct xfs_mount *mp; + int old_end, new_end; int tmp; int i; @@ -1568,17 +1569,49 @@ xfs_attr3_leaf_add_work( if (be16_to_cpu(entry->nameidx) < ichdr->firstused) ichdr->firstused = be16_to_cpu(entry->nameidx); - ASSERT(ichdr->firstused >= ichdr->count * sizeof(xfs_attr_leaf_entry_t) - + xfs_attr3_leaf_hdr_size(leaf)); - tmp = (ichdr->count - 1) * sizeof(xfs_attr_leaf_entry_t) - + xfs_attr3_leaf_hdr_size(leaf); + new_end = ichdr->count * sizeof(struct xfs_attr_leaf_entry) + + xfs_attr3_leaf_hdr_size(leaf); + old_end = new_end - sizeof(struct xfs_attr_leaf_entry); + + ASSERT(ichdr->firstused >= new_end); for (i = 0; i < XFS_ATTR_LEAF_MAPSIZE; i++) { - if (ichdr->freemap[i].base == tmp) { - ichdr->freemap[i].base += sizeof(xfs_attr_leaf_entry_t); + int diff = 0; + + if (ichdr->freemap[i].base == old_end) { + /* + * This freemap entry starts at the old end of the + * leaf entry array, so we need to adjust its base + * upward to accomodate the larger array. + */ + diff = sizeof(struct xfs_attr_leaf_entry); + } else if (ichdr->freemap[i].size > 0 && + ichdr->freemap[i].base < new_end) { + /* + * This freemap entry starts in the space claimed by + * the new leaf entry. Adjust its base upward to + * reflect that. + */ + diff = new_end - ichdr->freemap[i].base; + } + + if (diff) { + ichdr->freemap[i].base += diff; ichdr->freemap[i].size -= - min_t(uint16_t, ichdr->freemap[i].size, - sizeof(xfs_attr_leaf_entry_t)); + min_t(uint16_t, ichdr->freemap[i].size, diff); + } + + /* + * Don't leave zero-length freemaps with nonzero base lying + * around, because we don't want the code in _remove that + * matches on base address to get confused and create + * overlapping freemaps. If we end up with no freemap entries + * then the next _add will compact the leaf block and + * regenerate the freemaps. + */ + if (ichdr->freemap[i].size == 0 && ichdr->freemap[i].base > 0) { + ichdr->freemap[i].base = 0; + ichdr->holes = 1; } } ichdr->usedbytes += xfs_attr_leaf_entsize(leaf, args->index); diff --git a/fs/xfs/scrub/agheader_repair.c b/fs/xfs/scrub/agheader_repair.c index cd6f0223879f4..6e3fef36d6614 100644 --- a/fs/xfs/scrub/agheader_repair.c +++ b/fs/xfs/scrub/agheader_repair.c @@ -837,8 +837,12 @@ xrep_agi_buf_cleanup( { struct xrep_agi *ragi = buf; - xfarray_destroy(ragi->iunlink_prev); - xfarray_destroy(ragi->iunlink_next); + if (ragi->iunlink_prev) + xfarray_destroy(ragi->iunlink_prev); + ragi->iunlink_prev = NULL; + if (ragi->iunlink_next) + xfarray_destroy(ragi->iunlink_next); + ragi->iunlink_next = NULL; xagino_bitmap_destroy(&ragi->iunlink_bmp); } @@ -1708,7 +1712,6 @@ xrep_agi( { struct xrep_agi *ragi; struct xfs_mount *mp = sc->mp; - char *descr; unsigned int i; int error; @@ -1742,17 +1745,13 @@ xrep_agi( xagino_bitmap_init(&ragi->iunlink_bmp); sc->buf_cleanup = xrep_agi_buf_cleanup; - descr = xchk_xfile_ag_descr(sc, "iunlinked next pointers"); - error = xfarray_create(descr, 0, sizeof(xfs_agino_t), - &ragi->iunlink_next); - kfree(descr); + error = xfarray_create("iunlinked next pointers", 0, + sizeof(xfs_agino_t), &ragi->iunlink_next); if (error) return error; - descr = xchk_xfile_ag_descr(sc, "iunlinked prev pointers"); - error = xfarray_create(descr, 0, sizeof(xfs_agino_t), - &ragi->iunlink_prev); - kfree(descr); + error = xfarray_create("iunlinked prev pointers", 0, + sizeof(xfs_agino_t), &ragi->iunlink_prev); if (error) return error; diff --git a/fs/xfs/scrub/alloc_repair.c b/fs/xfs/scrub/alloc_repair.c index bed6a09aa7911..35035d02a2316 100644 --- a/fs/xfs/scrub/alloc_repair.c +++ b/fs/xfs/scrub/alloc_repair.c @@ -850,7 +850,6 @@ xrep_allocbt( struct xrep_abt *ra; struct xfs_mount *mp = sc->mp; unsigned int busy_gen; - char *descr; int error; /* We require the rmapbt to rebuild anything. */ @@ -876,11 +875,9 @@ xrep_allocbt( } /* Set up enough storage to handle maximally fragmented free space. */ - descr = xchk_xfile_ag_descr(sc, "free space records"); - error = xfarray_create(descr, mp->m_sb.sb_agblocks / 2, + error = xfarray_create("free space records", mp->m_sb.sb_agblocks / 2, sizeof(struct xfs_alloc_rec_incore), &ra->free_records); - kfree(descr); if (error) goto out_ra; @@ -926,7 +923,22 @@ xrep_revalidate_allocbt( if (error) goto out; + /* + * If the bnobt is still corrupt, we've failed to repair the filesystem + * and should just bail out. + * + * If the bnobt fails cross-examination with the cntbt, the scan will + * free the cntbt cursor, so we need to mark the repair incomplete + * and avoid walking off the end of the NULL cntbt cursor. + */ + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out; + sc->sm->sm_type = XFS_SCRUB_TYPE_CNTBT; + if (!sc->sa.cnt_cur) { + xchk_set_incomplete(sc); + goto out; + } error = xchk_allocbt(sc); out: sc->sm->sm_type = old_type; diff --git a/fs/xfs/scrub/attr.c b/fs/xfs/scrub/attr.c index 708334f9b2bd1..a0878fdbcf386 100644 --- a/fs/xfs/scrub/attr.c +++ b/fs/xfs/scrub/attr.c @@ -287,32 +287,6 @@ xchk_xattr_set_map( return ret; } -/* - * Check the leaf freemap from the usage bitmap. Returns false if the - * attr freemap has problems or points to used space. - */ -STATIC bool -xchk_xattr_check_freemap( - struct xfs_scrub *sc, - struct xfs_attr3_icleaf_hdr *leafhdr) -{ - struct xchk_xattr_buf *ab = sc->buf; - unsigned int mapsize = sc->mp->m_attr_geo->blksize; - int i; - - /* Construct bitmap of freemap contents. */ - bitmap_zero(ab->freemap, mapsize); - for (i = 0; i < XFS_ATTR_LEAF_MAPSIZE; i++) { - if (!xchk_xattr_set_map(sc, ab->freemap, - leafhdr->freemap[i].base, - leafhdr->freemap[i].size)) - return false; - } - - /* Look for bits that are set in freemap and are marked in use. */ - return !bitmap_intersects(ab->freemap, ab->usedmap, mapsize); -} - /* * Check this leaf entry's relations to everything else. * Returns the number of bytes used for the name/value data. @@ -364,7 +338,10 @@ xchk_xattr_entry( rentry = xfs_attr3_leaf_name_remote(leaf, idx); namesize = xfs_attr_leaf_entsize_remote(rentry->namelen); name_end = (char *)rentry + namesize; - if (rentry->namelen == 0 || rentry->valueblk == 0) + if (rentry->namelen == 0) + xchk_da_set_corrupt(ds, level); + if (rentry->valueblk == 0 && + !(ent->flags & XFS_ATTR_INCOMPLETE)) xchk_da_set_corrupt(ds, level); } if (name_end > buf_end) @@ -403,6 +380,7 @@ xchk_xattr_block( *last_checked = blk->blkno; bitmap_zero(ab->usedmap, mp->m_attr_geo->blksize); + bitmap_zero(ab->freemap, mp->m_attr_geo->blksize); /* Check all the padding. */ if (xfs_has_crc(ds->sc->mp)) { @@ -449,6 +427,9 @@ xchk_xattr_block( if ((char *)&entries[leafhdr.count] > (char *)leaf + leafhdr.firstused) xchk_da_set_corrupt(ds, level); + if (ds->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out; + buf_end = (char *)bp->b_addr + mp->m_attr_geo->blksize; for (i = 0, ent = entries; i < leafhdr.count; ent++, i++) { /* Mark the leaf entry itself. */ @@ -467,7 +448,29 @@ xchk_xattr_block( goto out; } - if (!xchk_xattr_check_freemap(ds->sc, &leafhdr)) + /* Construct bitmap of freemap contents. */ + for (i = 0; i < XFS_ATTR_LEAF_MAPSIZE; i++) { + if (!xchk_xattr_set_map(ds->sc, ab->freemap, + leafhdr.freemap[i].base, + leafhdr.freemap[i].size)) + xchk_da_set_corrupt(ds, level); + + /* + * freemap entries with zero length and nonzero base can cause + * problems with older kernels, so we mark these for preening + * even though there's no inconsistency. + */ + if (leafhdr.freemap[i].size == 0 && + leafhdr.freemap[i].base > 0) + xchk_da_set_preen(ds, level); + + if (ds->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out; + } + + /* Look for bits that are set in freemap and are marked in use. */ + if (bitmap_intersects(ab->freemap, ab->usedmap, + mp->m_attr_geo->blksize)) xchk_da_set_corrupt(ds, level); if (leafhdr.usedbytes != usedbytes) diff --git a/fs/xfs/scrub/attr_repair.c b/fs/xfs/scrub/attr_repair.c index 09d63aa10314b..dd24044c44efd 100644 --- a/fs/xfs/scrub/attr_repair.c +++ b/fs/xfs/scrub/attr_repair.c @@ -1516,8 +1516,10 @@ xrep_xattr_teardown( xfblob_destroy(rx->pptr_names); if (rx->pptr_recs) xfarray_destroy(rx->pptr_recs); - xfblob_destroy(rx->xattr_blobs); - xfarray_destroy(rx->xattr_records); + if (rx->xattr_blobs) + xfblob_destroy(rx->xattr_blobs); + if (rx->xattr_records) + xfarray_destroy(rx->xattr_records); mutex_destroy(&rx->lock); kfree(rx); } @@ -1529,7 +1531,6 @@ xrep_xattr_setup_scan( struct xrep_xattr **rxp) { struct xrep_xattr *rx; - char *descr; int max_len; int error; @@ -1555,35 +1556,26 @@ xrep_xattr_setup_scan( goto out_rx; /* Set up some staging for salvaged attribute keys and values */ - descr = xchk_xfile_ino_descr(sc, "xattr keys"); - error = xfarray_create(descr, 0, sizeof(struct xrep_xattr_key), + error = xfarray_create("xattr keys", 0, sizeof(struct xrep_xattr_key), &rx->xattr_records); - kfree(descr); if (error) goto out_rx; - descr = xchk_xfile_ino_descr(sc, "xattr names"); - error = xfblob_create(descr, &rx->xattr_blobs); - kfree(descr); + error = xfblob_create("xattr names", &rx->xattr_blobs); if (error) goto out_keys; if (xfs_has_parent(sc->mp)) { ASSERT(sc->flags & XCHK_FSGATES_DIRENTS); - descr = xchk_xfile_ino_descr(sc, - "xattr retained parent pointer entries"); - error = xfarray_create(descr, 0, + error = xfarray_create("xattr parent pointer entries", 0, sizeof(struct xrep_xattr_pptr), &rx->pptr_recs); - kfree(descr); if (error) goto out_values; - descr = xchk_xfile_ino_descr(sc, - "xattr retained parent pointer names"); - error = xfblob_create(descr, &rx->pptr_names); - kfree(descr); + error = xfblob_create("xattr parent pointer names", + &rx->pptr_names); if (error) goto out_pprecs; diff --git a/fs/xfs/scrub/bmap_repair.c b/fs/xfs/scrub/bmap_repair.c index 1084213b8e9b8..747cd9389b491 100644 --- a/fs/xfs/scrub/bmap_repair.c +++ b/fs/xfs/scrub/bmap_repair.c @@ -923,7 +923,6 @@ xrep_bmap( bool allow_unwritten) { struct xrep_bmap *rb; - char *descr; xfs_extnum_t max_bmbt_recs; bool large_extcount; int error = 0; @@ -945,11 +944,8 @@ xrep_bmap( /* Set up enough storage to handle the max records for this fork. */ large_extcount = xfs_has_large_extent_counts(sc->mp); max_bmbt_recs = xfs_iext_max_nextents(large_extcount, whichfork); - descr = xchk_xfile_ino_descr(sc, "%s fork mapping records", - whichfork == XFS_DATA_FORK ? "data" : "attr"); - error = xfarray_create(descr, max_bmbt_recs, + error = xfarray_create("fork mapping records", max_bmbt_recs, sizeof(struct xfs_bmbt_rec), &rb->bmap_records); - kfree(descr); if (error) goto out_rb; diff --git a/fs/xfs/scrub/btree.c b/fs/xfs/scrub/btree.c index acade92c5fce1..b497f6a474c77 100644 --- a/fs/xfs/scrub/btree.c +++ b/fs/xfs/scrub/btree.c @@ -42,6 +42,8 @@ __xchk_btree_process_error( break; case -EFSBADCRC: case -EFSCORRUPTED: + case -EIO: + case -ENODATA: /* Note the badness but don't abort. */ sc->sm->sm_flags |= errflag; *error = 0; diff --git a/fs/xfs/scrub/common.c b/fs/xfs/scrub/common.c index 2ef7742be7d3d..975c879c8d7f5 100644 --- a/fs/xfs/scrub/common.c +++ b/fs/xfs/scrub/common.c @@ -103,6 +103,8 @@ __xchk_process_error( break; case -EFSBADCRC: case -EFSCORRUPTED: + case -EIO: + case -ENODATA: /* Note the badness but don't abort. */ sc->sm->sm_flags |= errflag; *error = 0; @@ -177,6 +179,8 @@ __xchk_fblock_process_error( break; case -EFSBADCRC: case -EFSCORRUPTED: + case -EIO: + case -ENODATA: /* Note the badness but don't abort. */ sc->sm->sm_flags |= errflag; *error = 0; @@ -1395,6 +1399,9 @@ xchk_metadata_inode_subtype( int error; sub = xchk_scrub_create_subord(sc, scrub_type); + if (!sub) + return -ENOMEM; + error = sub->sc.ops->scrub(&sub->sc); xchk_scrub_free_subord(sub); return error; diff --git a/fs/xfs/scrub/common.h b/fs/xfs/scrub/common.h index ddbc065c798cd..f2ecc68538f0c 100644 --- a/fs/xfs/scrub/common.h +++ b/fs/xfs/scrub/common.h @@ -246,31 +246,6 @@ static inline bool xchk_could_repair(const struct xfs_scrub *sc) int xchk_metadata_inode_forks(struct xfs_scrub *sc); -/* - * Helper macros to allocate and format xfile description strings. - * Callers must kfree the pointer returned. - */ -#define xchk_xfile_descr(sc, fmt, ...) \ - kasprintf(XCHK_GFP_FLAGS, "XFS (%s): " fmt, \ - (sc)->mp->m_super->s_id, ##__VA_ARGS__) -#define xchk_xfile_ag_descr(sc, fmt, ...) \ - kasprintf(XCHK_GFP_FLAGS, "XFS (%s): AG 0x%x " fmt, \ - (sc)->mp->m_super->s_id, \ - (sc)->sa.pag ? \ - pag_agno((sc)->sa.pag) : (sc)->sm->sm_agno, \ - ##__VA_ARGS__) -#define xchk_xfile_ino_descr(sc, fmt, ...) \ - kasprintf(XCHK_GFP_FLAGS, "XFS (%s): inode 0x%llx " fmt, \ - (sc)->mp->m_super->s_id, \ - (sc)->ip ? (sc)->ip->i_ino : (sc)->sm->sm_ino, \ - ##__VA_ARGS__) -#define xchk_xfile_rtgroup_descr(sc, fmt, ...) \ - kasprintf(XCHK_GFP_FLAGS, "XFS (%s): rtgroup 0x%x " fmt, \ - (sc)->mp->m_super->s_id, \ - (sc)->sa.pag ? \ - rtg_rgno((sc)->sr.rtg) : (sc)->sm->sm_agno, \ - ##__VA_ARGS__) - /* * Setting up a hook to wait for intents to drain is costly -- we have to take * the CPU hotplug lock and force an i-cache flush on all CPUs once to set it diff --git a/fs/xfs/scrub/dabtree.c b/fs/xfs/scrub/dabtree.c index 056de4819f866..a6a5d3a75d994 100644 --- a/fs/xfs/scrub/dabtree.c +++ b/fs/xfs/scrub/dabtree.c @@ -45,6 +45,8 @@ xchk_da_process_error( break; case -EFSBADCRC: case -EFSCORRUPTED: + case -EIO: + case -ENODATA: /* Note the badness but don't abort. */ sc->sm->sm_flags |= XFS_SCRUB_OFLAG_CORRUPT; *error = 0; diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c index c877bde71e628..4f849d98cbdd2 100644 --- a/fs/xfs/scrub/dir.c +++ b/fs/xfs/scrub/dir.c @@ -1102,22 +1102,17 @@ xchk_directory( sd->xname.name = sd->namebuf; if (xfs_has_parent(sc->mp)) { - char *descr; - /* * Set up some staging memory for dirents that we can't check * due to locking contention. */ - descr = xchk_xfile_ino_descr(sc, "slow directory entries"); - error = xfarray_create(descr, 0, sizeof(struct xchk_dirent), - &sd->dir_entries); - kfree(descr); + error = xfarray_create("slow directory entries", 0, + sizeof(struct xchk_dirent), &sd->dir_entries); if (error) goto out_sd; - descr = xchk_xfile_ino_descr(sc, "slow directory entry names"); - error = xfblob_create(descr, &sd->dir_names); - kfree(descr); + error = xfblob_create("slow directory entry names", + &sd->dir_names); if (error) goto out_entries; } diff --git a/fs/xfs/scrub/dir_repair.c b/fs/xfs/scrub/dir_repair.c index 8d3b550990b58..e0fa5e6ca1fe8 100644 --- a/fs/xfs/scrub/dir_repair.c +++ b/fs/xfs/scrub/dir_repair.c @@ -172,8 +172,12 @@ xrep_dir_teardown( struct xrep_dir *rd = sc->buf; xrep_findparent_scan_teardown(&rd->pscan); - xfblob_destroy(rd->dir_names); - xfarray_destroy(rd->dir_entries); + if (rd->dir_names) + xfblob_destroy(rd->dir_names); + rd->dir_names = NULL; + if (rd->dir_entries) + xfarray_destroy(rd->dir_entries); + rd->dir_entries = NULL; } /* Set up for a directory repair. */ @@ -1784,20 +1788,15 @@ xrep_dir_setup_scan( struct xrep_dir *rd) { struct xfs_scrub *sc = rd->sc; - char *descr; int error; /* Set up some staging memory for salvaging dirents. */ - descr = xchk_xfile_ino_descr(sc, "directory entries"); - error = xfarray_create(descr, 0, sizeof(struct xrep_dirent), - &rd->dir_entries); - kfree(descr); + error = xfarray_create("directory entries", 0, + sizeof(struct xrep_dirent), &rd->dir_entries); if (error) return error; - descr = xchk_xfile_ino_descr(sc, "directory entry names"); - error = xfblob_create(descr, &rd->dir_names); - kfree(descr); + error = xfblob_create("directory entry names", &rd->dir_names); if (error) goto out_xfarray; diff --git a/fs/xfs/scrub/dirtree.c b/fs/xfs/scrub/dirtree.c index 3a9cdf8738b6d..3e0bbe75c44cf 100644 --- a/fs/xfs/scrub/dirtree.c +++ b/fs/xfs/scrub/dirtree.c @@ -81,8 +81,12 @@ xchk_dirtree_buf_cleanup( kfree(path); } - xfblob_destroy(dl->path_names); - xfarray_destroy(dl->path_steps); + if (dl->path_names) + xfblob_destroy(dl->path_names); + dl->path_names = NULL; + if (dl->path_steps) + xfarray_destroy(dl->path_steps); + dl->path_steps = NULL; mutex_destroy(&dl->lock); } @@ -92,7 +96,6 @@ xchk_setup_dirtree( struct xfs_scrub *sc) { struct xchk_dirtree *dl; - char *descr; int error; xchk_fsgates_enable(sc, XCHK_FSGATES_DIRENTS); @@ -116,16 +119,12 @@ xchk_setup_dirtree( mutex_init(&dl->lock); - descr = xchk_xfile_ino_descr(sc, "dirtree path steps"); - error = xfarray_create(descr, 0, sizeof(struct xchk_dirpath_step), - &dl->path_steps); - kfree(descr); + error = xfarray_create("dirtree path steps", 0, + sizeof(struct xchk_dirpath_step), &dl->path_steps); if (error) goto out_dl; - descr = xchk_xfile_ino_descr(sc, "dirtree path names"); - error = xfblob_create(descr, &dl->path_names); - kfree(descr); + error = xfblob_create("dirtree path names", &dl->path_names); if (error) goto out_steps; diff --git a/fs/xfs/scrub/ialloc_repair.c b/fs/xfs/scrub/ialloc_repair.c index 14e48d3f1912b..f28459f58832f 100644 --- a/fs/xfs/scrub/ialloc_repair.c +++ b/fs/xfs/scrub/ialloc_repair.c @@ -797,7 +797,6 @@ xrep_iallocbt( { struct xrep_ibt *ri; struct xfs_mount *mp = sc->mp; - char *descr; xfs_agino_t first_agino, last_agino; int error = 0; @@ -816,11 +815,9 @@ xrep_iallocbt( /* Set up enough storage to handle an AG with nothing but inodes. */ xfs_agino_range(mp, pag_agno(sc->sa.pag), &first_agino, &last_agino); last_agino /= XFS_INODES_PER_CHUNK; - descr = xchk_xfile_ag_descr(sc, "inode index records"); - error = xfarray_create(descr, last_agino, + error = xfarray_create("inode index records", last_agino, sizeof(struct xfs_inobt_rec_incore), &ri->inode_records); - kfree(descr); if (error) goto out_ri; @@ -866,10 +863,24 @@ xrep_revalidate_iallocbt( if (error) goto out; - if (xfs_has_finobt(sc->mp)) { - sc->sm->sm_type = XFS_SCRUB_TYPE_FINOBT; - error = xchk_iallocbt(sc); + /* + * If the inobt is still corrupt, we've failed to repair the filesystem + * and should just bail out. + * + * If the inobt fails cross-examination with the finobt, the scan will + * free the finobt cursor, so we need to mark the repair incomplete + * and avoid walking off the end of the NULL finobt cursor. + */ + if (!xfs_has_finobt(sc->mp) || + (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) + goto out; + + sc->sm->sm_type = XFS_SCRUB_TYPE_FINOBT; + if (!sc->sa.fino_cur) { + xchk_set_incomplete(sc); + goto out; } + error = xchk_iallocbt(sc); out: sc->sm->sm_type = old_type; diff --git a/fs/xfs/scrub/nlinks.c b/fs/xfs/scrub/nlinks.c index 091c79e432e59..dec3b9b47453e 100644 --- a/fs/xfs/scrub/nlinks.c +++ b/fs/xfs/scrub/nlinks.c @@ -971,7 +971,8 @@ xchk_nlinks_teardown_scan( xfs_dir_hook_del(xnc->sc->mp, &xnc->dhook); - xfarray_destroy(xnc->nlinks); + if (xnc->nlinks) + xfarray_destroy(xnc->nlinks); xnc->nlinks = NULL; xchk_iscan_teardown(&xnc->collect_iscan); @@ -990,7 +991,6 @@ xchk_nlinks_setup_scan( struct xchk_nlink_ctrs *xnc) { struct xfs_mount *mp = sc->mp; - char *descr; unsigned long long max_inos; xfs_agnumber_t last_agno = mp->m_sb.sb_agcount - 1; xfs_agino_t first_agino, last_agino; @@ -1007,10 +1007,9 @@ xchk_nlinks_setup_scan( */ xfs_agino_range(mp, last_agno, &first_agino, &last_agino); max_inos = XFS_AGINO_TO_INO(mp, last_agno, last_agino) + 1; - descr = xchk_xfile_descr(sc, "file link counts"); - error = xfarray_create(descr, min(XFS_MAXINUMBER + 1, max_inos), + error = xfarray_create("file link counts", + min(XFS_MAXINUMBER + 1, max_inos), sizeof(struct xchk_nlink), &xnc->nlinks); - kfree(descr); if (error) goto out_teardown; diff --git a/fs/xfs/scrub/orphanage.c b/fs/xfs/scrub/orphanage.c index 9c12cb8442311..3c4f3b542c20f 100644 --- a/fs/xfs/scrub/orphanage.c +++ b/fs/xfs/scrub/orphanage.c @@ -445,6 +445,11 @@ xrep_adoption_check_dcache( return 0; d_child = try_lookup_noperm(&qname, d_orphanage); + if (IS_ERR(d_child)) { + dput(d_orphanage); + return PTR_ERR(d_child); + } + if (d_child) { trace_xrep_adoption_check_child(sc->mp, d_child); @@ -482,7 +487,7 @@ xrep_adoption_zap_dcache( return; d_child = try_lookup_noperm(&qname, d_orphanage); - while (d_child != NULL) { + while (!IS_ERR_OR_NULL(d_child)) { trace_xrep_adoption_invalidate_child(sc->mp, d_child); ASSERT(d_is_negative(d_child)); diff --git a/fs/xfs/scrub/parent.c b/fs/xfs/scrub/parent.c index 3b692c4acc1e6..f2ee520cc9429 100644 --- a/fs/xfs/scrub/parent.c +++ b/fs/xfs/scrub/parent.c @@ -755,7 +755,6 @@ xchk_parent_pptr( struct xfs_scrub *sc) { struct xchk_pptrs *pp; - char *descr; int error; pp = kvzalloc(sizeof(struct xchk_pptrs), XCHK_GFP_FLAGS); @@ -768,16 +767,12 @@ xchk_parent_pptr( * Set up some staging memory for parent pointers that we can't check * due to locking contention. */ - descr = xchk_xfile_ino_descr(sc, "slow parent pointer entries"); - error = xfarray_create(descr, 0, sizeof(struct xchk_pptr), - &pp->pptr_entries); - kfree(descr); + error = xfarray_create("slow parent pointer entries", 0, + sizeof(struct xchk_pptr), &pp->pptr_entries); if (error) goto out_pp; - descr = xchk_xfile_ino_descr(sc, "slow parent pointer names"); - error = xfblob_create(descr, &pp->pptr_names); - kfree(descr); + error = xfblob_create("slow parent pointer names", &pp->pptr_names); if (error) goto out_entries; diff --git a/fs/xfs/scrub/parent_repair.c b/fs/xfs/scrub/parent_repair.c index 2949feda62717..897902c54178d 100644 --- a/fs/xfs/scrub/parent_repair.c +++ b/fs/xfs/scrub/parent_repair.c @@ -1497,7 +1497,6 @@ xrep_parent_setup_scan( struct xrep_parent *rp) { struct xfs_scrub *sc = rp->sc; - char *descr; struct xfs_da_geometry *geo = sc->mp->m_attr_geo; int max_len; int error; @@ -1525,32 +1524,22 @@ xrep_parent_setup_scan( goto out_xattr_name; /* Set up some staging memory for logging parent pointer updates. */ - descr = xchk_xfile_ino_descr(sc, "parent pointer entries"); - error = xfarray_create(descr, 0, sizeof(struct xrep_pptr), - &rp->pptr_recs); - kfree(descr); + error = xfarray_create("parent pointer entries", 0, + sizeof(struct xrep_pptr), &rp->pptr_recs); if (error) goto out_xattr_value; - descr = xchk_xfile_ino_descr(sc, "parent pointer names"); - error = xfblob_create(descr, &rp->pptr_names); - kfree(descr); + error = xfblob_create("parent pointer names", &rp->pptr_names); if (error) goto out_recs; /* Set up some storage for copying attrs before the mapping exchange */ - descr = xchk_xfile_ino_descr(sc, - "parent pointer retained xattr entries"); - error = xfarray_create(descr, 0, sizeof(struct xrep_parent_xattr), - &rp->xattr_records); - kfree(descr); + error = xfarray_create("parent pointer xattr entries", 0, + sizeof(struct xrep_parent_xattr), &rp->xattr_records); if (error) goto out_names; - descr = xchk_xfile_ino_descr(sc, - "parent pointer retained xattr values"); - error = xfblob_create(descr, &rp->xattr_blobs); - kfree(descr); + error = xfblob_create("parent pointer xattr values", &rp->xattr_blobs); if (error) goto out_attr_keys; diff --git a/fs/xfs/scrub/quotacheck.c b/fs/xfs/scrub/quotacheck.c index e4105aaafe845..38e855fe0feeb 100644 --- a/fs/xfs/scrub/quotacheck.c +++ b/fs/xfs/scrub/quotacheck.c @@ -742,7 +742,6 @@ xqcheck_setup_scan( struct xfs_scrub *sc, struct xqcheck *xqc) { - char *descr; struct xfs_quotainfo *qi = sc->mp->m_quotainfo; unsigned long long max_dquots = XFS_DQ_ID_MAX + 1ULL; int error; @@ -757,28 +756,22 @@ xqcheck_setup_scan( error = -ENOMEM; if (xfs_this_quota_on(sc->mp, XFS_DQTYPE_USER)) { - descr = xchk_xfile_descr(sc, "user dquot records"); - error = xfarray_create(descr, max_dquots, + error = xfarray_create("user dquot records", max_dquots, sizeof(struct xqcheck_dquot), &xqc->ucounts); - kfree(descr); if (error) goto out_teardown; } if (xfs_this_quota_on(sc->mp, XFS_DQTYPE_GROUP)) { - descr = xchk_xfile_descr(sc, "group dquot records"); - error = xfarray_create(descr, max_dquots, + error = xfarray_create("group dquot records", max_dquots, sizeof(struct xqcheck_dquot), &xqc->gcounts); - kfree(descr); if (error) goto out_teardown; } if (xfs_this_quota_on(sc->mp, XFS_DQTYPE_PROJ)) { - descr = xchk_xfile_descr(sc, "project dquot records"); - error = xfarray_create(descr, max_dquots, + error = xfarray_create("project dquot records", max_dquots, sizeof(struct xqcheck_dquot), &xqc->pcounts); - kfree(descr); if (error) goto out_teardown; } diff --git a/fs/xfs/scrub/refcount_repair.c b/fs/xfs/scrub/refcount_repair.c index 9c8cb5332da04..360fd7354880a 100644 --- a/fs/xfs/scrub/refcount_repair.c +++ b/fs/xfs/scrub/refcount_repair.c @@ -123,13 +123,7 @@ int xrep_setup_ag_refcountbt( struct xfs_scrub *sc) { - char *descr; - int error; - - descr = xchk_xfile_ag_descr(sc, "rmap record bag"); - error = xrep_setup_xfbtree(sc, descr); - kfree(descr); - return error; + return xrep_setup_xfbtree(sc, "rmap record bag"); } /* Check for any obvious conflicts with this shared/CoW staging extent. */ @@ -704,7 +698,6 @@ xrep_refcountbt( { struct xrep_refc *rr; struct xfs_mount *mp = sc->mp; - char *descr; int error; /* We require the rmapbt to rebuild anything. */ @@ -717,11 +710,9 @@ xrep_refcountbt( rr->sc = sc; /* Set up enough storage to handle one refcount record per block. */ - descr = xchk_xfile_ag_descr(sc, "reference count records"); - error = xfarray_create(descr, mp->m_sb.sb_agblocks, + error = xfarray_create("reference count records", mp->m_sb.sb_agblocks, sizeof(struct xfs_refcount_irec), &rr->refcount_records); - kfree(descr); if (error) goto out_rr; diff --git a/fs/xfs/scrub/repair.c b/fs/xfs/scrub/repair.c index efd5a7ccdf624..4d45d39e67f11 100644 --- a/fs/xfs/scrub/repair.c +++ b/fs/xfs/scrub/repair.c @@ -1136,6 +1136,9 @@ xrep_metadata_inode_subtype( * setup/teardown routines. */ sub = xchk_scrub_create_subord(sc, scrub_type); + if (!sub) + return -ENOMEM; + error = sub->sc.ops->scrub(&sub->sc); if (error) goto out; diff --git a/fs/xfs/scrub/rmap_repair.c b/fs/xfs/scrub/rmap_repair.c index 17d4a38d735cb..cfd1cf403b37e 100644 --- a/fs/xfs/scrub/rmap_repair.c +++ b/fs/xfs/scrub/rmap_repair.c @@ -164,14 +164,11 @@ xrep_setup_ag_rmapbt( struct xfs_scrub *sc) { struct xrep_rmap *rr; - char *descr; int error; xchk_fsgates_enable(sc, XCHK_FSGATES_RMAP); - descr = xchk_xfile_ag_descr(sc, "reverse mapping records"); - error = xrep_setup_xfbtree(sc, descr); - kfree(descr); + error = xrep_setup_xfbtree(sc, "reverse mapping records"); if (error) return error; diff --git a/fs/xfs/scrub/rtbitmap_repair.c b/fs/xfs/scrub/rtbitmap_repair.c index 203a1a97c5026..41d6736a529d0 100644 --- a/fs/xfs/scrub/rtbitmap_repair.c +++ b/fs/xfs/scrub/rtbitmap_repair.c @@ -43,7 +43,6 @@ xrep_setup_rtbitmap( struct xchk_rtbitmap *rtb) { struct xfs_mount *mp = sc->mp; - char *descr; unsigned long long blocks = mp->m_sb.sb_rbmblocks; int error; @@ -52,9 +51,8 @@ xrep_setup_rtbitmap( return error; /* Create an xfile to hold our reconstructed bitmap. */ - descr = xchk_xfile_rtgroup_descr(sc, "bitmap file"); - error = xfile_create(descr, blocks * mp->m_sb.sb_blocksize, &sc->xfile); - kfree(descr); + error = xfile_create("realtime bitmap file", + blocks * mp->m_sb.sb_blocksize, &sc->xfile); if (error) return error; diff --git a/fs/xfs/scrub/rtrefcount_repair.c b/fs/xfs/scrub/rtrefcount_repair.c index 983362447826d..b35e39cce7ad5 100644 --- a/fs/xfs/scrub/rtrefcount_repair.c +++ b/fs/xfs/scrub/rtrefcount_repair.c @@ -128,13 +128,7 @@ int xrep_setup_rtrefcountbt( struct xfs_scrub *sc) { - char *descr; - int error; - - descr = xchk_xfile_ag_descr(sc, "rmap record bag"); - error = xrep_setup_xfbtree(sc, descr); - kfree(descr); - return error; + return xrep_setup_xfbtree(sc, "realtime rmap record bag"); } /* Check for any obvious conflicts with this shared/CoW staging extent. */ @@ -704,7 +698,6 @@ xrep_rtrefcountbt( { struct xrep_rtrefc *rr; struct xfs_mount *mp = sc->mp; - char *descr; int error; /* We require the rmapbt to rebuild anything. */ @@ -722,11 +715,9 @@ xrep_rtrefcountbt( rr->sc = sc; /* Set up enough storage to handle one refcount record per rt extent. */ - descr = xchk_xfile_ag_descr(sc, "reference count records"); - error = xfarray_create(descr, mp->m_sb.sb_rextents, - sizeof(struct xfs_refcount_irec), + error = xfarray_create("realtime reference count records", + mp->m_sb.sb_rextents, sizeof(struct xfs_refcount_irec), &rr->refcount_records); - kfree(descr); if (error) goto out_rr; diff --git a/fs/xfs/scrub/rtrmap_repair.c b/fs/xfs/scrub/rtrmap_repair.c index 7561941a337a1..749977a66e40f 100644 --- a/fs/xfs/scrub/rtrmap_repair.c +++ b/fs/xfs/scrub/rtrmap_repair.c @@ -103,14 +103,11 @@ xrep_setup_rtrmapbt( struct xfs_scrub *sc) { struct xrep_rtrmap *rr; - char *descr; int error; xchk_fsgates_enable(sc, XCHK_FSGATES_RMAP); - descr = xchk_xfile_rtgroup_descr(sc, "reverse mapping records"); - error = xrep_setup_xfbtree(sc, descr); - kfree(descr); + error = xrep_setup_xfbtree(sc, "realtime reverse mapping records"); if (error) return error; diff --git a/fs/xfs/scrub/rtsummary.c b/fs/xfs/scrub/rtsummary.c index 4ac679c1bd29c..fb78cff2ac3a1 100644 --- a/fs/xfs/scrub/rtsummary.c +++ b/fs/xfs/scrub/rtsummary.c @@ -43,7 +43,6 @@ xchk_setup_rtsummary( struct xfs_scrub *sc) { struct xfs_mount *mp = sc->mp; - char *descr; struct xchk_rtsummary *rts; int error; @@ -70,10 +69,8 @@ xchk_setup_rtsummary( * Create an xfile to construct a new rtsummary file. The xfile allows * us to avoid pinning kernel memory for this purpose. */ - descr = xchk_xfile_descr(sc, "realtime summary file"); - error = xfile_create(descr, XFS_FSB_TO_B(mp, mp->m_rsumblocks), - &sc->xfile); - kfree(descr); + error = xfile_create("realtime summary file", + XFS_FSB_TO_B(mp, mp->m_rsumblocks), &sc->xfile); if (error) return error; diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index 3c3b0d25006ff..c312f0a672e65 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -634,7 +634,7 @@ xchk_scrub_create_subord( sub = kzalloc(sizeof(*sub), XCHK_GFP_FLAGS); if (!sub) - return ERR_PTR(-ENOMEM); + return NULL; sub->old_smtype = sc->sm->sm_type; sub->old_smflags = sc->sm->sm_flags; diff --git a/fs/xfs/xfs_notify_failure.c b/fs/xfs/xfs_notify_failure.c index b176728899420..0700a723f38e7 100644 --- a/fs/xfs/xfs_notify_failure.c +++ b/fs/xfs/xfs_notify_failure.c @@ -293,7 +293,7 @@ xfs_dax_notify_dev_failure( error = xfs_alloc_read_agf(pag, tp, 0, &agf_bp); if (error) { - xfs_perag_put(pag); + xfs_perag_rele(pag); break; } @@ -329,7 +329,7 @@ xfs_dax_notify_dev_failure( if (rtg) xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_RMAP); if (error) { - xfs_group_put(xg); + xfs_group_rele(xg); break; } } diff --git a/include/acpi/ghes.h b/include/acpi/ghes.h index ebd21b05fe6ed..93db60da5934e 100644 --- a/include/acpi/ghes.h +++ b/include/acpi/ghes.h @@ -21,6 +21,7 @@ struct ghes { struct acpi_hest_generic_v2 *generic_v2; }; struct acpi_hest_generic_status *estatus; + unsigned int estatus_length; unsigned long flags; union { struct list_head list; diff --git a/include/acpi/pcc.h b/include/acpi/pcc.h index 9af3b502f8395..840bfc95bae33 100644 --- a/include/acpi/pcc.h +++ b/include/acpi/pcc.h @@ -17,35 +17,6 @@ struct pcc_mbox_chan { u32 latency; u32 max_access_rate; u16 min_turnaround_time; - - /* Set to true to indicate that the mailbox should manage - * writing the dat to the shared buffer. This differs from - * the case where the drivesr are writing to the buffer and - * using send_data only to ring the doorbell. If this flag - * is set, then the void * data parameter of send_data must - * point to a kernel-memory buffer formatted in accordance with - * the PCC specification. - * - * The active buffer management will include reading the - * notify_on_completion flag, and will then - * call mbox_chan_txdone when the acknowledgment interrupt is - * received. - */ - bool manage_writes; - - /* Optional callback that allows the driver - * to allocate the memory used for receiving - * messages. The return value is the location - * inside the buffer where the mailbox should write the data. - */ - void *(*rx_alloc)(struct mbox_client *cl, int size); -}; - -struct pcc_header { - u32 signature; - u32 flags; - u32 length; - u32 command; }; /* Generic Communications Channel Shared Memory Region */ diff --git a/include/asm-generic/audit_change_attr.h b/include/asm-generic/audit_change_attr.h index cc840537885fb..ddd90bbe40dfc 100644 --- a/include/asm-generic/audit_change_attr.h +++ b/include/asm-generic/audit_change_attr.h @@ -26,6 +26,9 @@ __NR_fremovexattr, __NR_fchownat, __NR_fchmodat, #endif +#ifdef __NR_fchmodat2 +__NR_fchmodat2, +#endif #ifdef __NR_chown32 __NR_chown32, __NR_fchown32, diff --git a/include/asm-generic/audit_read.h b/include/asm-generic/audit_read.h index 7bb7b5a83ae2e..fb9991f53fb6f 100644 --- a/include/asm-generic/audit_read.h +++ b/include/asm-generic/audit_read.h @@ -4,9 +4,15 @@ __NR_readlink, #endif __NR_quotactl, __NR_listxattr, +#ifdef __NR_listxattrat +__NR_listxattrat, +#endif __NR_llistxattr, __NR_flistxattr, __NR_getxattr, +#ifdef __NR_getxattrat +__NR_getxattrat, +#endif __NR_lgetxattr, __NR_fgetxattr, #ifdef __NR_readlinkat diff --git a/include/asm-generic/rqspinlock.h b/include/asm-generic/rqspinlock.h index 0f2dcbbfee2f0..5c5cf2f7fc395 100644 --- a/include/asm-generic/rqspinlock.h +++ b/include/asm-generic/rqspinlock.h @@ -191,7 +191,7 @@ static __always_inline int res_spin_lock(rqspinlock_t *lock) #else -#define res_spin_lock(lock) resilient_tas_spin_lock(lock) +#define res_spin_lock(lock) ({ grab_held_lock_entry(lock); resilient_tas_spin_lock(lock); }) #endif /* CONFIG_QUEUED_SPINLOCKS */ diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index 1fff717cae510..4d679d2a206b4 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -46,7 +46,8 @@ * * The mmu_gather API consists of: * - * - tlb_gather_mmu() / tlb_gather_mmu_fullmm() / tlb_finish_mmu() + * - tlb_gather_mmu() / tlb_gather_mmu_fullmm() / tlb_gather_mmu_vma() / + * tlb_finish_mmu() * * start and finish a mmu_gather * @@ -364,6 +365,20 @@ struct mmu_gather { unsigned int vma_huge : 1; unsigned int vma_pfn : 1; + /* + * Did we unshare (unmap) any shared page tables? For now only + * used for hugetlb PMD table sharing. + */ + unsigned int unshared_tables : 1; + + /* + * Did we unshare any page tables such that they are now exclusive + * and could get reused+modified by the new owner? When setting this + * flag, "unshared_tables" will be set as well. For now only used + * for hugetlb PMD table sharing. + */ + unsigned int fully_unshared_tables : 1; + unsigned int batch_count; #ifndef CONFIG_MMU_GATHER_NO_GATHER @@ -400,6 +415,7 @@ static inline void __tlb_reset_range(struct mmu_gather *tlb) tlb->cleared_pmds = 0; tlb->cleared_puds = 0; tlb->cleared_p4ds = 0; + tlb->unshared_tables = 0; /* * Do not reset mmu_gather::vma_* fields here, we do not * call into tlb_start_vma() again to set them if there is an @@ -484,7 +500,7 @@ static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb) * these bits. */ if (!(tlb->freed_tables || tlb->cleared_ptes || tlb->cleared_pmds || - tlb->cleared_puds || tlb->cleared_p4ds)) + tlb->cleared_puds || tlb->cleared_p4ds || tlb->unshared_tables)) return; tlb_flush(tlb); @@ -773,6 +789,63 @@ static inline bool huge_pmd_needs_flush(pmd_t oldpmd, pmd_t newpmd) } #endif +#ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING +static inline void tlb_unshare_pmd_ptdesc(struct mmu_gather *tlb, struct ptdesc *pt, + unsigned long addr) +{ + /* + * The caller must make sure that concurrent unsharing + exclusive + * reuse is impossible until tlb_flush_unshared_tables() was called. + */ + VM_WARN_ON_ONCE(!ptdesc_pmd_is_shared(pt)); + ptdesc_pmd_pts_dec(pt); + + /* Clearing a PUD pointing at a PMD table with PMD leaves. */ + tlb_flush_pmd_range(tlb, addr & PUD_MASK, PUD_SIZE); + + /* + * If the page table is now exclusively owned, we fully unshared + * a page table. + */ + if (!ptdesc_pmd_is_shared(pt)) + tlb->fully_unshared_tables = true; + tlb->unshared_tables = true; +} + +static inline void tlb_flush_unshared_tables(struct mmu_gather *tlb) +{ + /* + * As soon as the caller drops locks to allow for reuse of + * previously-shared tables, these tables could get modified and + * even reused outside of hugetlb context, so we have to make sure that + * any page table walkers (incl. TLB, GUP-fast) are aware of that + * change. + * + * Even if we are not fully unsharing a PMD table, we must + * flush the TLB for the unsharer now. + */ + if (tlb->unshared_tables) + tlb_flush_mmu_tlbonly(tlb); + + /* + * Similarly, we must make sure that concurrent GUP-fast will not + * walk previously-shared page tables that are getting modified+reused + * elsewhere. So broadcast an IPI to wait for any concurrent GUP-fast. + * + * We only perform this when we are the last sharer of a page table, + * as the IPI will reach all CPUs: any GUP-fast. + * + * Note that on configs where tlb_remove_table_sync_one() is a NOP, + * the expectation is that the tlb_flush_mmu_tlbonly() would have issued + * required IPIs already for us. + */ + if (tlb->fully_unshared_tables) { + tlb_remove_table_sync_one(); + tlb->fully_unshared_tables = false; + } +} +#endif /* CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING */ + #endif /* CONFIG_MMU */ #endif /* _ASM_GENERIC__TLB_H */ diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index e04d56a5332e6..6640e06ef2f13 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -832,12 +832,14 @@ defined(CONFIG_AUTOFDO_CLANG) || defined(CONFIG_PROPELLER_CLANG) /* Required sections not related to debugging. */ #define ELF_DETAILS \ - .modinfo : { *(.modinfo) . = ALIGN(8); } \ .comment 0 : { *(.comment) } \ .symtab 0 : { *(.symtab) } \ .strtab 0 : { *(.strtab) } \ .shstrtab 0 : { *(.shstrtab) } +#define MODINFO \ + .modinfo : { *(.modinfo) . = ALIGN(8); } + #ifdef CONFIG_GENERIC_BUG #define BUG_TABLE \ . = ALIGN(8); \ diff --git a/include/cxl/event.h b/include/cxl/event.h index 6fd90f9cc2034..4d7d1036ea9cb 100644 --- a/include/cxl/event.h +++ b/include/cxl/event.h @@ -320,4 +320,14 @@ static inline int cxl_cper_prot_err_kfifo_get(struct cxl_cper_prot_err_work_data } #endif +#ifdef CONFIG_ACPI_APEI_PCIEAER +int cxl_cper_sec_prot_err_valid(struct cxl_cper_sec_prot_err *prot_err); +#else +static inline int +cxl_cper_sec_prot_err_valid(struct cxl_cper_sec_prot_err *prot_err) +{ + return -EOPNOTSUPP; +} +#endif + #endif /* _LINUX_CXL_EVENT_H */ diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h index 92f5db84b9c22..6802896e30c7c 100644 --- a/include/drm/drm_gem_shmem_helper.h +++ b/include/drm/drm_gem_shmem_helper.h @@ -301,4 +301,15 @@ struct drm_gem_object *drm_gem_shmem_prime_import_no_map(struct drm_device *dev, .gem_prime_import = drm_gem_shmem_prime_import_no_map, \ .dumb_create = drm_gem_shmem_dumb_create +/* + * Kunit helpers + */ + +#if IS_ENABLED(CONFIG_KUNIT) +int drm_gem_shmem_vmap(struct drm_gem_shmem_object *shmem, struct iosys_map *map); +void drm_gem_shmem_vunmap(struct drm_gem_shmem_object *shmem, struct iosys_map *map); +int drm_gem_shmem_madvise(struct drm_gem_shmem_object *shmem, int madv); +int drm_gem_shmem_purge(struct drm_gem_shmem_object *shmem); +#endif + #endif /* __DRM_GEM_SHMEM_HELPER_H__ */ diff --git a/include/drm/drm_of.h b/include/drm/drm_of.h index 7f0256dae3f13..f3e55ea2174c0 100644 --- a/include/drm/drm_of.h +++ b/include/drm/drm_of.h @@ -5,6 +5,7 @@ #include #include #if IS_ENABLED(CONFIG_OF) && IS_ENABLED(CONFIG_DRM_PANEL_BRIDGE) +#include #include #endif @@ -173,6 +174,8 @@ static inline int drm_of_panel_bridge_remove(const struct device_node *np, bridge = of_drm_find_bridge(remote); drm_panel_bridge_remove(bridge); + of_node_put(remote); + return 0; #else return -EINVAL; diff --git a/include/drm/intel/intel_lb_mei_interface.h b/include/drm/intel/intel_lb_mei_interface.h index d65be2cba2ab9..0850738a30fc7 100644 --- a/include/drm/intel/intel_lb_mei_interface.h +++ b/include/drm/intel/intel_lb_mei_interface.h @@ -53,7 +53,8 @@ enum intel_lb_status { */ struct intel_lb_component_ops { /** - * push_payload - Sends a payload to the authentication firmware + * @push_payload: Sends a payload to the authentication firmware + * * @dev: Device struct corresponding to the mei device * @type: Payload type (see &enum intel_lb_type) * @flags: Payload flags bitmap (e.g. %INTEL_LB_FLAGS_IS_PERSISTENT) diff --git a/include/linux/audit.h b/include/linux/audit.h index 536f8ee8da818..b8d8029c6c480 100644 --- a/include/linux/audit.h +++ b/include/linux/audit.h @@ -128,12 +128,6 @@ enum audit_nfcfgop { extern int __init audit_register_class(int class, unsigned *list); extern int audit_classify_syscall(int abi, unsigned syscall); extern int audit_classify_arch(int arch); -/* only for compat system calls */ -extern unsigned compat_write_class[]; -extern unsigned compat_read_class[]; -extern unsigned compat_dir_class[]; -extern unsigned compat_chattr_class[]; -extern unsigned compat_signal_class[]; /* audit_names->type values */ #define AUDIT_TYPE_UNKNOWN 0 /* we don't know yet */ diff --git a/include/linux/audit_arch.h b/include/linux/audit_arch.h index 0e34d673ef171..2b8153791e6a5 100644 --- a/include/linux/audit_arch.h +++ b/include/linux/audit_arch.h @@ -23,4 +23,11 @@ enum auditsc_class_t { extern int audit_classify_compat_syscall(int abi, unsigned syscall); +/* only for compat system calls */ +extern unsigned compat_write_class[]; +extern unsigned compat_read_class[]; +extern unsigned compat_dir_class[]; +extern unsigned compat_chattr_class[]; +extern unsigned compat_signal_class[]; + #endif diff --git a/include/linux/bpf.h b/include/linux/bpf.h index d808253f2e945..e2dd3a6d495af 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -3200,6 +3200,11 @@ static inline void bpf_prog_report_arena_violation(bool write, unsigned long add } #endif /* CONFIG_BPF_SYSCALL */ +static inline bool bpf_net_capable(void) +{ + return capable(CAP_NET_ADMIN) || capable(CAP_SYS_ADMIN); +} + static __always_inline int bpf_probe_read_kernel_common(void *dst, u32 size, const void *unsafe_ptr) { diff --git a/include/linux/bpf_mprog.h b/include/linux/bpf_mprog.h index 929225f7b0959..0b9f4caeeb0a3 100644 --- a/include/linux/bpf_mprog.h +++ b/include/linux/bpf_mprog.h @@ -340,4 +340,14 @@ static inline bool bpf_mprog_supported(enum bpf_prog_type type) return false; } } + +static inline bool bpf_mprog_detach_empty(enum bpf_prog_type type) +{ + switch (type) { + case BPF_PROG_TYPE_SCHED_CLS: + return bpf_net_capable(); + default: + return false; + } +} #endif /* __BPF_MPROG_H */ diff --git a/include/linux/capability.h b/include/linux/capability.h index 1fb08922552c7..37db92b3d6f89 100644 --- a/include/linux/capability.h +++ b/include/linux/capability.h @@ -203,6 +203,12 @@ static inline bool checkpoint_restore_ns_capable(struct user_namespace *ns) ns_capable(ns, CAP_SYS_ADMIN); } +static inline bool checkpoint_restore_ns_capable_noaudit(struct user_namespace *ns) +{ + return ns_capable_noaudit(ns, CAP_CHECKPOINT_RESTORE) || + ns_capable_noaudit(ns, CAP_SYS_ADMIN); +} + /* audit system wants to get cap info from files as well */ int get_vfs_caps_from_disk(struct mnt_idmap *idmap, const struct dentry *dentry, diff --git a/include/linux/clk.h b/include/linux/clk.h index b607482ca77e9..64ff118ffb1a1 100644 --- a/include/linux/clk.h +++ b/include/linux/clk.h @@ -228,6 +228,23 @@ int devm_clk_rate_exclusive_get(struct device *dev, struct clk *clk); */ void clk_rate_exclusive_put(struct clk *clk); +/** + * clk_save_context - save clock context for poweroff + * + * Saves the context of the clock register for powerstates in which the + * contents of the registers will be lost. Occurs deep within the suspend + * code so locking is not necessary. + */ +int clk_save_context(void); + +/** + * clk_restore_context - restore clock context after poweroff + * + * This occurs with all clocks enabled. Occurs deep within the resume code + * so locking is not necessary. + */ +void clk_restore_context(void); + #else static inline int clk_notifier_register(struct clk *clk, @@ -293,6 +310,13 @@ static inline int devm_clk_rate_exclusive_get(struct device *dev, struct clk *cl static inline void clk_rate_exclusive_put(struct clk *clk) {} +static inline int clk_save_context(void) +{ + return 0; +} + +static inline void clk_restore_context(void) {} + #endif #ifdef CONFIG_HAVE_CLK_PREPARE @@ -933,23 +957,6 @@ struct clk *clk_get_parent(struct clk *clk); */ struct clk *clk_get_sys(const char *dev_id, const char *con_id); -/** - * clk_save_context - save clock context for poweroff - * - * Saves the context of the clock register for powerstates in which the - * contents of the registers will be lost. Occurs deep within the suspend - * code so locking is not necessary. - */ -int clk_save_context(void); - -/** - * clk_restore_context - restore clock context after poweroff - * - * This occurs with all clocks enabled. Occurs deep within the resume code - * so locking is not necessary. - */ -void clk_restore_context(void); - #else /* !CONFIG_HAVE_CLK */ static inline struct clk *clk_get(struct device *dev, const char *id) @@ -1129,13 +1136,6 @@ static inline struct clk *clk_get_sys(const char *dev_id, const char *con_id) return NULL; } -static inline int clk_save_context(void) -{ - return 0; -} - -static inline void clk_restore_context(void) {} - #endif /* clk_prepare_enable helps cases using clk_enable in non-atomic context. */ diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h index 107ce05bd16eb..cccc72fd336b8 100644 --- a/include/linux/compiler-clang.h +++ b/include/linux/compiler-clang.h @@ -152,4 +152,4 @@ * Bindgen uses LLVM even if our C compiler is GCC, so we cannot * rely on the auto-detected CONFIG_CC_HAS_TYPEOF_UNQUAL. */ -#define CC_HAS_TYPEOF_UNQUAL (__clang_major__ >= 19) +#define CC_HAS_TYPEOF_UNQUAL (__clang_major__ > 19 || (__clang_major__ == 19 && __clang_minor__ > 0)) diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index 8128a445f0480..2998af80cbef3 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -303,6 +303,22 @@ struct ftrace_likely_data { # define __no_kasan_or_inline __always_inline #endif +#ifdef CONFIG_KCSAN +/* + * Type qualifier to mark variables where all data-racy accesses should be + * ignored by KCSAN. Note, the implementation simply marks these variables as + * volatile, since KCSAN will treat such accesses as "marked". + * + * Defined here because defining __data_racy as volatile for KCSAN objects only + * causes problems in BPF Type Format (BTF) generation since struct members + * of core kernel data structs will be volatile in some objects and not in + * others. Instead define it globally for KCSAN kernels. + */ +# define __data_racy volatile +#else +# define __data_racy +#endif + #ifdef __SANITIZE_THREAD__ /* * Clang still emits instrumentation for __tsan_func_{entry,exit}() and builtin @@ -314,16 +330,9 @@ struct ftrace_likely_data { * disable all instrumentation. See Kconfig.kcsan where this is mandatory. */ # define __no_kcsan __no_sanitize_thread __disable_sanitizer_instrumentation -/* - * Type qualifier to mark variables where all data-racy accesses should be - * ignored by KCSAN. Note, the implementation simply marks these variables as - * volatile, since KCSAN will treat such accesses as "marked". - */ -# define __data_racy volatile # define __no_sanitize_or_inline __no_kcsan notrace __maybe_unused #else # define __no_kcsan -# define __data_racy #endif #ifdef __SANITIZE_MEMORY__ diff --git a/include/linux/cper.h b/include/linux/cper.h index 5b1236d8c65bb..440b35e459e53 100644 --- a/include/linux/cper.h +++ b/include/linux/cper.h @@ -595,7 +595,8 @@ void cper_mem_err_pack(const struct cper_sec_mem_err *, const char *cper_mem_err_unpack(struct trace_seq *, struct cper_mem_err_compact *); void cper_print_proc_arm(const char *pfx, - const struct cper_sec_proc_arm *proc); + const struct cper_sec_proc_arm *proc, + u32 length); void cper_print_proc_ia(const char *pfx, const struct cper_sec_proc_ia *proc); int cper_mem_err_location(struct cper_mem_err_compact *mem, char *msg); diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h index 2ddb256187b51..75283991e77ab 100644 --- a/include/linux/cpuset.h +++ b/include/linux/cpuset.h @@ -173,7 +173,7 @@ static inline void set_mems_allowed(nodemask_t nodemask) task_unlock(current); } -extern bool cpuset_node_allowed(struct cgroup *cgroup, int nid); +extern void cpuset_nodes_allowed(struct cgroup *cgroup, nodemask_t *mask); #else /* !CONFIG_CPUSETS */ static inline bool cpusets_enabled(void) { return false; } @@ -294,9 +294,9 @@ static inline bool read_mems_allowed_retry(unsigned int seq) return false; } -static inline bool cpuset_node_allowed(struct cgroup *cgroup, int nid) +static inline void cpuset_nodes_allowed(struct cgroup *cgroup, nodemask_t *mask) { - return true; + nodes_copy(*mask, node_states[N_MEMORY]); } #endif /* !CONFIG_CPUSETS */ diff --git a/include/linux/device_cgroup.h b/include/linux/device_cgroup.h index 0864773a57e8f..822085bc2d202 100644 --- a/include/linux/device_cgroup.h +++ b/include/linux/device_cgroup.h @@ -21,7 +21,7 @@ static inline int devcgroup_inode_permission(struct inode *inode, int mask) if (likely(!S_ISBLK(inode->i_mode) && !S_ISCHR(inode->i_mode))) return 0; - if (likely(!inode->i_rdev)) + if (!inode->i_rdev) return 0; if (S_ISBLK(inode->i_mode)) diff --git a/include/linux/dpll.h b/include/linux/dpll.h index 25be745bf41f1..562f520b23c27 100644 --- a/include/linux/dpll.h +++ b/include/linux/dpll.h @@ -163,6 +163,7 @@ struct dpll_pin_properties { u32 freq_supported_num; struct dpll_pin_frequency *freq_supported; struct dpll_pin_phase_adjust_range phase_range; + u32 phase_gran; }; #if IS_ENABLED(CONFIG_DPLL) diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h index a7880787cad36..dc41722fcc9dd 100644 --- a/include/linux/f2fs_fs.h +++ b/include/linux/f2fs_fs.h @@ -17,7 +17,6 @@ #define F2FS_LOG_SECTORS_PER_BLOCK (PAGE_SHIFT - 9) /* log number for sector/blk */ #define F2FS_BLKSIZE PAGE_SIZE /* support only block == page */ #define F2FS_BLKSIZE_BITS PAGE_SHIFT /* bits for F2FS_BLKSIZE */ -#define F2FS_SUM_BLKSIZE 4096 /* only support 4096 byte sum block */ #define F2FS_MAX_EXTENSION 64 /* # of extension entries */ #define F2FS_EXTENSION_LEN 8 /* max size of extension */ @@ -442,10 +441,8 @@ struct f2fs_sit_block { * from node's page's beginning to get a data block address. * ex) data_blkaddr = (block_t)(nodepage_start_address + ofs_in_node) */ -#define ENTRIES_IN_SUM (F2FS_SUM_BLKSIZE / 8) #define SUMMARY_SIZE (7) /* sizeof(struct f2fs_summary) */ #define SUM_FOOTER_SIZE (5) /* sizeof(struct summary_footer) */ -#define SUM_ENTRY_SIZE (SUMMARY_SIZE * ENTRIES_IN_SUM) /* a summary entry for a block in a segment */ struct f2fs_summary { @@ -468,22 +465,6 @@ struct summary_footer { __le32 check_sum; /* summary checksum */ } __packed; -#define SUM_JOURNAL_SIZE (F2FS_SUM_BLKSIZE - SUM_FOOTER_SIZE -\ - SUM_ENTRY_SIZE) -#define NAT_JOURNAL_ENTRIES ((SUM_JOURNAL_SIZE - 2) /\ - sizeof(struct nat_journal_entry)) -#define NAT_JOURNAL_RESERVED ((SUM_JOURNAL_SIZE - 2) %\ - sizeof(struct nat_journal_entry)) -#define SIT_JOURNAL_ENTRIES ((SUM_JOURNAL_SIZE - 2) /\ - sizeof(struct sit_journal_entry)) -#define SIT_JOURNAL_RESERVED ((SUM_JOURNAL_SIZE - 2) %\ - sizeof(struct sit_journal_entry)) - -/* Reserved area should make size of f2fs_extra_info equals to - * that of nat_journal and sit_journal. - */ -#define EXTRA_INFO_RESERVED (SUM_JOURNAL_SIZE - 2 - 8) - /* * frequently updated NAT/SIT entries can be stored in the spare area in * summary blocks @@ -498,9 +479,16 @@ struct nat_journal_entry { struct f2fs_nat_entry ne; } __packed; +/* + * The nat_journal structure is a placeholder whose actual size varies depending + * on the use of packed_ssa. Therefore, it must always be accessed only through + * specific sets of macros and fields, and size calculations should use + * size-related macros instead of sizeof(). + * Relevant macros: sbi->nat_journal_entries, nat_in_journal(), + * nid_in_journal(), MAX_NAT_JENTRIES(). + */ struct nat_journal { - struct nat_journal_entry entries[NAT_JOURNAL_ENTRIES]; - __u8 reserved[NAT_JOURNAL_RESERVED]; + struct nat_journal_entry entries[0]; } __packed; struct sit_journal_entry { @@ -508,14 +496,21 @@ struct sit_journal_entry { struct f2fs_sit_entry se; } __packed; +/* + * The sit_journal structure is a placeholder whose actual size varies depending + * on the use of packed_ssa. Therefore, it must always be accessed only through + * specific sets of macros and fields, and size calculations should use + * size-related macros instead of sizeof(). + * Relevant macros: sbi->sit_journal_entries, sit_in_journal(), + * segno_in_journal(), MAX_SIT_JENTRIES(). + */ struct sit_journal { - struct sit_journal_entry entries[SIT_JOURNAL_ENTRIES]; - __u8 reserved[SIT_JOURNAL_RESERVED]; + struct sit_journal_entry entries[0]; } __packed; struct f2fs_extra_info { __le64 kbytes_written; - __u8 reserved[EXTRA_INFO_RESERVED]; + __u8 reserved[]; } __packed; struct f2fs_journal { @@ -531,11 +526,33 @@ struct f2fs_journal { }; } __packed; -/* Block-sized summary block structure */ +/* + * Block-sized summary block structure + * + * The f2fs_summary_block structure is a placeholder whose actual size varies + * depending on the use of packed_ssa. Therefore, it must always be accessed + * only through specific sets of macros and fields, and size calculations should + * use size-related macros instead of sizeof(). + * Relevant macros: sbi->sum_blocksize, sbi->entries_in_sum, + * sbi->sum_entry_size, sum_entries(), sum_journal(), sum_footer(). + * + * Summary Block Layout + * + * +-----------------------+ <--- Block Start + * | struct f2fs_summary | + * | entries[0] | + * | ... | + * | entries[N-1] | + * +-----------------------+ + * | struct f2fs_journal | + * +-----------------------+ + * | struct summary_footer | + * +-----------------------+ <--- Block End + */ struct f2fs_summary_block { - struct f2fs_summary entries[ENTRIES_IN_SUM]; - struct f2fs_journal journal; - struct summary_footer footer; + struct f2fs_summary entries[0]; + // struct f2fs_journal journal; + // struct summary_footer footer; } __packed; /* diff --git a/include/linux/fb.h b/include/linux/fb.h index 05cc251035da9..c3302d5135466 100644 --- a/include/linux/fb.h +++ b/include/linux/fb.h @@ -497,7 +497,6 @@ struct fb_info { #if defined(CONFIG_FB_DEVICE) struct device *dev; /* This is this fb device */ #endif - int class_flag; /* private sysfs flags */ #ifdef CONFIG_FB_TILEBLITTING struct fb_tile_ops *tileops; /* Tile Blitting */ #endif diff --git a/include/linux/filter.h b/include/linux/filter.h index 569de3b14279a..cf7a0bce1bb6d 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1375,24 +1375,13 @@ static inline bool bpf_jit_kallsyms_enabled(void) return false; } -int __bpf_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char *sym); +int bpf_address_lookup(unsigned long addr, unsigned long *size, + unsigned long *off, char *sym); bool is_bpf_text_address(unsigned long addr); int bpf_get_kallsym(unsigned int symnum, unsigned long *value, char *type, char *sym); struct bpf_prog *bpf_prog_ksym_find(unsigned long addr); -static inline int -bpf_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char **modname, char *sym) -{ - int ret = __bpf_address_lookup(addr, size, off, sym); - - if (ret && modname) - *modname = NULL; - return ret; -} - void bpf_prog_kallsyms_add(struct bpf_prog *fp); void bpf_prog_kallsyms_del(struct bpf_prog *fp); @@ -1431,8 +1420,8 @@ static inline bool bpf_jit_kallsyms_enabled(void) } static inline int -__bpf_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char *sym) +bpf_address_lookup(unsigned long addr, unsigned long *size, + unsigned long *off, char *sym) { return 0; } @@ -1453,13 +1442,6 @@ static inline struct bpf_prog *bpf_prog_ksym_find(unsigned long addr) return NULL; } -static inline int -bpf_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char **modname, char *sym) -{ - return 0; -} - static inline void bpf_prog_kallsyms_add(struct bpf_prog *fp) { } diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 07f8c309e4327..c3b1c74bdd7a4 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -87,11 +87,13 @@ struct ftrace_hash; defined(CONFIG_DYNAMIC_FTRACE) int ftrace_mod_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char **modname, char *sym); + unsigned long *off, char **modname, + const unsigned char **modbuildid, char *sym); #else static inline int ftrace_mod_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char **modname, char *sym) + unsigned long *off, char **modname, + const unsigned char **modbuildid, char *sym) { return 0; } @@ -1030,10 +1032,17 @@ static inline bool is_ftrace_trampoline(unsigned long addr) #ifdef CONFIG_FUNCTION_GRAPH_TRACER #ifndef ftrace_graph_func -#define ftrace_graph_func ftrace_stub -#define FTRACE_OPS_GRAPH_STUB FTRACE_OPS_FL_STUB +# define ftrace_graph_func ftrace_stub +# define FTRACE_OPS_GRAPH_STUB FTRACE_OPS_FL_STUB +/* + * The function graph is called every time the function tracer is called. + * It must always test the ops hash and cannot just directly call + * the handler. + */ +# define FGRAPH_NO_DIRECT 1 #else -#define FTRACE_OPS_GRAPH_STUB 0 +# define FTRACE_OPS_GRAPH_STUB 0 +# define FGRAPH_NO_DIRECT 0 #endif #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/include/linux/ftrace_regs.h b/include/linux/ftrace_regs.h index 15627ceea9bcc..386fa48c4a957 100644 --- a/include/linux/ftrace_regs.h +++ b/include/linux/ftrace_regs.h @@ -33,6 +33,31 @@ struct ftrace_regs; #define ftrace_regs_get_frame_pointer(fregs) \ frame_pointer(&arch_ftrace_regs(fregs)->regs) +static __always_inline void +ftrace_partial_regs_update(struct ftrace_regs *fregs, struct pt_regs *regs) { } + +#else + +/* + * ftrace_partial_regs_update - update the original ftrace_regs from regs + * @fregs: The ftrace_regs to update from @regs + * @regs: The partial regs from ftrace_partial_regs() that was updated + * + * Some architectures have the partial regs living in the ftrace_regs + * structure, whereas other architectures need to make a different copy + * of the @regs. If a partial @regs is retrieved by ftrace_partial_regs() and + * if the code using @regs updates a field (like the instruction pointer or + * stack pointer) it may need to propagate that change to the original @fregs + * it retrieved the partial @regs from. Use this function to guarantee that + * update happens. + */ +static __always_inline void +ftrace_partial_regs_update(struct ftrace_regs *fregs, struct pt_regs *regs) +{ + ftrace_regs_set_instruction_pointer(fregs, instruction_pointer(regs)); + ftrace_regs_set_return_value(fregs, regs_return_value(regs)); +} + #endif /* HAVE_ARCH_FTRACE_REGS */ /* This can be overridden by the architectures */ diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index c4690e365ade9..4cf418a41fe43 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -444,12 +444,16 @@ struct hisi_qp_ops { int (*fill_sqe)(void *sqe, void *q_parm, void *d_parm); }; +struct instance_backlog { + struct list_head list; + spinlock_t lock; +}; + struct hisi_qp { u32 qp_id; u16 sq_depth; u16 cq_depth; u8 alg_type; - u8 req_type; struct qm_dma qdma; void *sqe; @@ -459,7 +463,6 @@ struct hisi_qp { struct hisi_qp_status qp_status; struct hisi_qp_ops *hw_ops; - void *qp_ctx; void (*req_cb)(struct hisi_qp *qp, void *data); void (*event_cb)(struct hisi_qp *qp); @@ -468,6 +471,10 @@ struct hisi_qp { bool is_in_kernel; u16 pasid; struct uacce_queue *uacce_q; + + spinlock_t qp_lock; + struct instance_backlog backlog; + const void **msg; }; static inline int vfs_num_set(const char *val, const struct kernel_param *kp) @@ -572,7 +579,7 @@ struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev, void hisi_acc_free_sgl_pool(struct device *dev, struct hisi_acc_sgl_pool *pool); int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, - u8 alg_type, int node, struct hisi_qp **qps); + u8 *alg_type, int node, struct hisi_qp **qps); void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num); void hisi_qm_dev_shutdown(struct pci_dev *pdev); void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list); diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 89054f714992f..6fc7934eafa11 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -241,8 +241,9 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, unsigned long sz); unsigned long hugetlb_mask_last_page(struct hstate *h); -int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, - unsigned long addr, pte_t *ptep); +int huge_pmd_unshare(struct mmu_gather *tlb, struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep); +void huge_pmd_unshare_flush(struct mmu_gather *tlb, struct vm_area_struct *vma); void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma, unsigned long *start, unsigned long *end); @@ -302,13 +303,17 @@ static inline struct address_space *hugetlb_folio_mapping_lock_write( return NULL; } -static inline int huge_pmd_unshare(struct mm_struct *mm, - struct vm_area_struct *vma, - unsigned long addr, pte_t *ptep) +static inline int huge_pmd_unshare(struct mmu_gather *tlb, + struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) { return 0; } +static inline void huge_pmd_unshare_flush(struct mmu_gather *tlb, + struct vm_area_struct *vma) +{ +} + static inline void adjust_range_if_pmd_sharing_possible( struct vm_area_struct *vma, unsigned long *start, unsigned long *end) diff --git a/include/linux/hw_random.h b/include/linux/hw_random.h index b424555753b11..b77bc55a4cf35 100644 --- a/include/linux/hw_random.h +++ b/include/linux/hw_random.h @@ -15,6 +15,7 @@ #include #include #include +#include /** * struct hwrng - Hardware Random Number Generator driver @@ -48,6 +49,7 @@ struct hwrng { /* internal. */ struct list_head list; struct kref ref; + struct work_struct cleanup_work; struct completion cleanup_done; struct completion dying; }; diff --git a/include/linux/ima.h b/include/linux/ima.h index 8e29cb4e6a01d..abf8923f8fc51 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -69,6 +69,7 @@ static inline int ima_measure_critical_data(const char *event_label, #ifdef CONFIG_HAVE_IMA_KEXEC int __init ima_free_kexec_buffer(void); int __init ima_get_kexec_buffer(void **addr, size_t *size); +int ima_validate_range(phys_addr_t phys, size_t size); #endif #ifdef CONFIG_IMA_SECURE_AND_OR_TRUSTED_BOOT diff --git a/include/linux/indirect_call_wrapper.h b/include/linux/indirect_call_wrapper.h index 35227d47cfc98..dc272b514a01b 100644 --- a/include/linux/indirect_call_wrapper.h +++ b/include/linux/indirect_call_wrapper.h @@ -16,22 +16,26 @@ */ #define INDIRECT_CALL_1(f, f1, ...) \ ({ \ - likely(f == f1) ? f1(__VA_ARGS__) : f(__VA_ARGS__); \ + typeof(f) __f1 = (f); \ + likely(__f1 == f1) ? f1(__VA_ARGS__) : __f1(__VA_ARGS__); \ }) #define INDIRECT_CALL_2(f, f2, f1, ...) \ ({ \ - likely(f == f2) ? f2(__VA_ARGS__) : \ - INDIRECT_CALL_1(f, f1, __VA_ARGS__); \ + typeof(f) __f2 = (f); \ + likely(__f2 == f2) ? f2(__VA_ARGS__) : \ + INDIRECT_CALL_1(__f2, f1, __VA_ARGS__); \ }) #define INDIRECT_CALL_3(f, f3, f2, f1, ...) \ ({ \ - likely(f == f3) ? f3(__VA_ARGS__) : \ - INDIRECT_CALL_2(f, f2, f1, __VA_ARGS__); \ + typeof(f) __f3 = (f); \ + likely(__f3 == f3) ? f3(__VA_ARGS__) : \ + INDIRECT_CALL_2(__f3, f2, f1, __VA_ARGS__); \ }) #define INDIRECT_CALL_4(f, f4, f3, f2, f1, ...) \ ({ \ - likely(f == f4) ? f4(__VA_ARGS__) : \ - INDIRECT_CALL_3(f, f3, f2, f1, __VA_ARGS__); \ + typeof(f) __f4 = (f); \ + likely(__f4 == f4) ? f4(__VA_ARGS__) : \ + INDIRECT_CALL_3(__f4, f3, f2, f1, __VA_ARGS__); \ }) #define INDIRECT_CALLABLE_DECLARE(f) f diff --git a/include/linux/inetdevice.h b/include/linux/inetdevice.h index 5730ba6b1cfaf..dccbeb25f7014 100644 --- a/include/linux/inetdevice.h +++ b/include/linux/inetdevice.h @@ -38,11 +38,11 @@ struct in_device { struct ip_mc_list *mc_tomb; unsigned long mr_v1_seen; unsigned long mr_v2_seen; - unsigned long mr_maxdelay; unsigned long mr_qi; /* Query Interval */ unsigned long mr_qri; /* Query Response Interval */ unsigned char mr_qrv; /* Query Robustness Variable */ unsigned char mr_gq_running; + u32 mr_maxdelay; u32 mr_ifc_count; struct timer_list mr_gq_timer; /* general query timer */ struct timer_list mr_ifc_timer; /* interface change timer */ diff --git a/include/linux/input/adp5589.h b/include/linux/input/adp5589.h deleted file mode 100644 index 0e4742c8c81e3..0000000000000 --- a/include/linux/input/adp5589.h +++ /dev/null @@ -1,180 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Analog Devices ADP5589/ADP5585 I/O Expander and QWERTY Keypad Controller - * - * Copyright 2010-2011 Analog Devices Inc. - */ - -#ifndef _ADP5589_H -#define _ADP5589_H - -/* - * ADP5589 specific GPI and Keymap defines - */ - -#define ADP5589_KEYMAPSIZE 88 - -#define ADP5589_GPI_PIN_ROW0 97 -#define ADP5589_GPI_PIN_ROW1 98 -#define ADP5589_GPI_PIN_ROW2 99 -#define ADP5589_GPI_PIN_ROW3 100 -#define ADP5589_GPI_PIN_ROW4 101 -#define ADP5589_GPI_PIN_ROW5 102 -#define ADP5589_GPI_PIN_ROW6 103 -#define ADP5589_GPI_PIN_ROW7 104 -#define ADP5589_GPI_PIN_COL0 105 -#define ADP5589_GPI_PIN_COL1 106 -#define ADP5589_GPI_PIN_COL2 107 -#define ADP5589_GPI_PIN_COL3 108 -#define ADP5589_GPI_PIN_COL4 109 -#define ADP5589_GPI_PIN_COL5 110 -#define ADP5589_GPI_PIN_COL6 111 -#define ADP5589_GPI_PIN_COL7 112 -#define ADP5589_GPI_PIN_COL8 113 -#define ADP5589_GPI_PIN_COL9 114 -#define ADP5589_GPI_PIN_COL10 115 -#define GPI_LOGIC1 116 -#define GPI_LOGIC2 117 - -#define ADP5589_GPI_PIN_ROW_BASE ADP5589_GPI_PIN_ROW0 -#define ADP5589_GPI_PIN_ROW_END ADP5589_GPI_PIN_ROW7 -#define ADP5589_GPI_PIN_COL_BASE ADP5589_GPI_PIN_COL0 -#define ADP5589_GPI_PIN_COL_END ADP5589_GPI_PIN_COL10 - -#define ADP5589_GPI_PIN_BASE ADP5589_GPI_PIN_ROW_BASE -#define ADP5589_GPI_PIN_END ADP5589_GPI_PIN_COL_END - -#define ADP5589_GPIMAPSIZE_MAX (ADP5589_GPI_PIN_END - ADP5589_GPI_PIN_BASE + 1) - -/* - * ADP5585 specific GPI and Keymap defines - */ - -#define ADP5585_KEYMAPSIZE 30 - -#define ADP5585_GPI_PIN_ROW0 37 -#define ADP5585_GPI_PIN_ROW1 38 -#define ADP5585_GPI_PIN_ROW2 39 -#define ADP5585_GPI_PIN_ROW3 40 -#define ADP5585_GPI_PIN_ROW4 41 -#define ADP5585_GPI_PIN_ROW5 42 -#define ADP5585_GPI_PIN_COL0 43 -#define ADP5585_GPI_PIN_COL1 44 -#define ADP5585_GPI_PIN_COL2 45 -#define ADP5585_GPI_PIN_COL3 46 -#define ADP5585_GPI_PIN_COL4 47 -#define GPI_LOGIC 48 - -#define ADP5585_GPI_PIN_ROW_BASE ADP5585_GPI_PIN_ROW0 -#define ADP5585_GPI_PIN_ROW_END ADP5585_GPI_PIN_ROW5 -#define ADP5585_GPI_PIN_COL_BASE ADP5585_GPI_PIN_COL0 -#define ADP5585_GPI_PIN_COL_END ADP5585_GPI_PIN_COL4 - -#define ADP5585_GPI_PIN_BASE ADP5585_GPI_PIN_ROW_BASE -#define ADP5585_GPI_PIN_END ADP5585_GPI_PIN_COL_END - -#define ADP5585_GPIMAPSIZE_MAX (ADP5585_GPI_PIN_END - ADP5585_GPI_PIN_BASE + 1) - -struct adp5589_gpi_map { - unsigned short pin; - unsigned short sw_evt; -}; - -/* scan_cycle_time */ -#define ADP5589_SCAN_CYCLE_10ms 0 -#define ADP5589_SCAN_CYCLE_20ms 1 -#define ADP5589_SCAN_CYCLE_30ms 2 -#define ADP5589_SCAN_CYCLE_40ms 3 - -/* RESET_CFG */ -#define RESET_PULSE_WIDTH_500us 0 -#define RESET_PULSE_WIDTH_1ms 1 -#define RESET_PULSE_WIDTH_2ms 2 -#define RESET_PULSE_WIDTH_10ms 3 - -#define RESET_TRIG_TIME_0ms (0 << 2) -#define RESET_TRIG_TIME_1000ms (1 << 2) -#define RESET_TRIG_TIME_1500ms (2 << 2) -#define RESET_TRIG_TIME_2000ms (3 << 2) -#define RESET_TRIG_TIME_2500ms (4 << 2) -#define RESET_TRIG_TIME_3000ms (5 << 2) -#define RESET_TRIG_TIME_3500ms (6 << 2) -#define RESET_TRIG_TIME_4000ms (7 << 2) - -#define RESET_PASSTHRU_EN (1 << 5) -#define RESET1_POL_HIGH (1 << 6) -#define RESET1_POL_LOW (0 << 6) -#define RESET2_POL_HIGH (1 << 7) -#define RESET2_POL_LOW (0 << 7) - -/* ADP5589 Mask Bits: - * C C C C C C C C C C C | R R R R R R R R - * 1 9 8 7 6 5 4 3 2 1 0 | 7 6 5 4 3 2 1 0 - * 0 - * ---------------- BIT ------------------ - * 1 1 1 1 1 1 1 1 1 0 0 | 0 0 0 0 0 0 0 0 - * 8 7 6 5 4 3 2 1 0 9 8 | 7 6 5 4 3 2 1 0 - */ - -#define ADP_ROW(x) (1 << (x)) -#define ADP_COL(x) (1 << (x + 8)) -#define ADP5589_ROW_MASK 0xFF -#define ADP5589_COL_MASK 0xFF -#define ADP5589_COL_SHIFT 8 -#define ADP5589_MAX_ROW_NUM 7 -#define ADP5589_MAX_COL_NUM 10 - -/* ADP5585 Mask Bits: - * C C C C C | R R R R R R - * 4 3 2 1 0 | 5 4 3 2 1 0 - * - * ---- BIT -- ----------- - * 1 0 0 0 0 | 0 0 0 0 0 0 - * 0 9 8 7 6 | 5 4 3 2 1 0 - */ - -#define ADP5585_ROW_MASK 0x3F -#define ADP5585_COL_MASK 0x1F -#define ADP5585_ROW_SHIFT 0 -#define ADP5585_COL_SHIFT 6 -#define ADP5585_MAX_ROW_NUM 5 -#define ADP5585_MAX_COL_NUM 4 - -#define ADP5585_ROW(x) (1 << ((x) & ADP5585_ROW_MASK)) -#define ADP5585_COL(x) (1 << (((x) & ADP5585_COL_MASK) + ADP5585_COL_SHIFT)) - -/* Put one of these structures in i2c_board_info platform_data */ - -struct adp5589_kpad_platform_data { - unsigned keypad_en_mask; /* Keypad (Rows/Columns) enable mask */ - const unsigned short *keymap; /* Pointer to keymap */ - unsigned short keymapsize; /* Keymap size */ - bool repeat; /* Enable key repeat */ - bool en_keylock; /* Enable key lock feature (ADP5589 only)*/ - unsigned char unlock_key1; /* Unlock Key 1 (ADP5589 only) */ - unsigned char unlock_key2; /* Unlock Key 2 (ADP5589 only) */ - unsigned char unlock_timer; /* Time in seconds [0..7] between the two unlock keys 0=disable (ADP5589 only) */ - unsigned char scan_cycle_time; /* Time between consecutive scan cycles */ - unsigned char reset_cfg; /* Reset config */ - unsigned short reset1_key_1; /* Reset Key 1 */ - unsigned short reset1_key_2; /* Reset Key 2 */ - unsigned short reset1_key_3; /* Reset Key 3 */ - unsigned short reset2_key_1; /* Reset Key 1 */ - unsigned short reset2_key_2; /* Reset Key 2 */ - unsigned debounce_dis_mask; /* Disable debounce mask */ - unsigned pull_dis_mask; /* Disable all pull resistors mask */ - unsigned pullup_en_100k; /* Pull-Up 100k Enable Mask */ - unsigned pullup_en_300k; /* Pull-Up 300k Enable Mask */ - unsigned pulldown_en_300k; /* Pull-Down 300k Enable Mask */ - const struct adp5589_gpi_map *gpimap; - unsigned short gpimapsize; - const struct adp5589_gpio_platform_data *gpio_data; -}; - -struct i2c_client; /* forward declaration */ - -struct adp5589_gpio_platform_data { - int gpio_start; /* GPIO Chip base # */ -}; - -#endif diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index 51b6484c04934..8f1166bc3b1ce 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -215,7 +215,7 @@ static inline int __must_check devm_request_irq(struct device *dev, unsigned int irq, irq_handler_t handler, unsigned long irqflags, const char *devname, void *dev_id) { - return devm_request_threaded_irq(dev, irq, handler, NULL, irqflags, + return devm_request_threaded_irq(dev, irq, handler, NULL, irqflags | IRQF_COND_ONESHOT, devname, dev_id); } diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index c2ea6280901dc..b4d8aca3e7860 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -439,6 +439,9 @@ struct io_ring_ctx { struct list_head defer_list; unsigned nr_drained; + /* protected by ->completion_lock */ + unsigned nr_req_allocated; + #ifdef CONFIG_NET_RX_BUSY_POLL struct list_head napi_list; /* track busy poll napi_id */ spinlock_t napi_lock; /* napi_list lock */ @@ -451,10 +454,6 @@ struct io_ring_ctx { DECLARE_HASHTABLE(napi_ht, 4); #endif - /* protected by ->completion_lock */ - unsigned evfd_last_cq_tail; - unsigned nr_req_allocated; - /* * Protection for resize vs mmap races - both the mmap and resize * side will need to grab this lock, to prevent either side from diff --git a/include/linux/ipv6.h b/include/linux/ipv6.h index 43b7bb8287388..7294e4e89b797 100644 --- a/include/linux/ipv6.h +++ b/include/linux/ipv6.h @@ -271,7 +271,6 @@ struct ipv6_pinfo { struct ipv6_mc_socklist __rcu *ipv6_mc_list; struct ipv6_ac_socklist *ipv6_ac_list; - struct ipv6_fl_socklist __rcu *ipv6_fl_list; }; /* We currently use available bits from inet_sk(sk)->inet_flags, diff --git a/include/linux/leds-expresswire.h b/include/linux/leds-expresswire.h index a422921f4159f..7f8c4795f69fa 100644 --- a/include/linux/leds-expresswire.h +++ b/include/linux/leds-expresswire.h @@ -30,9 +30,6 @@ struct expresswire_common_props { void expresswire_power_off(struct expresswire_common_props *props); void expresswire_enable(struct expresswire_common_props *props); -void expresswire_start(struct expresswire_common_props *props); -void expresswire_end(struct expresswire_common_props *props); -void expresswire_set_bit(struct expresswire_common_props *props, bool bit); void expresswire_write_u8(struct expresswire_common_props *props, u8 val); #endif /* _LEDS_EXPRESSWIRE_H */ diff --git a/include/linux/libata.h b/include/linux/libata.h index 7a98de1cc995c..3b8bdea8516d0 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -899,6 +899,9 @@ struct ata_port { u64 qc_active; int nr_active_links; /* #links with active qcs */ + struct work_struct deferred_qc_work; + struct ata_queued_cmd *deferred_qc; + struct ata_link link; /* host default link */ struct ata_link *slave_link; /* see ata_slave_link_init() */ diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 873e510d6f8d9..133591199933a 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1789,7 +1789,7 @@ static inline void count_objcg_events(struct obj_cgroup *objcg, rcu_read_unlock(); } -bool mem_cgroup_node_allowed(struct mem_cgroup *memcg, int nid); +void mem_cgroup_node_filter_allowed(struct mem_cgroup *memcg, nodemask_t *mask); #else static inline bool mem_cgroup_kmem_disabled(void) @@ -1853,9 +1853,9 @@ static inline ino_t page_cgroup_ino(struct page *page) return 0; } -static inline bool mem_cgroup_node_allowed(struct mem_cgroup *memcg, int nid) +static inline void mem_cgroup_node_filter_allowed(struct mem_cgroup *memcg, + nodemask_t *mask) { - return true; } #endif /* CONFIG_MEMCG */ diff --git a/include/linux/mfd/tps65219.h b/include/linux/mfd/tps65219.h index 55234e771ba73..3abf937191d0c 100644 --- a/include/linux/mfd/tps65219.h +++ b/include/linux/mfd/tps65219.h @@ -149,6 +149,8 @@ enum pmic_id { #define TPS65215_ENABLE_LDO2_EN_MASK BIT(5) #define TPS65214_ENABLE_LDO1_EN_MASK BIT(5) #define TPS65219_ENABLE_LDO4_EN_MASK BIT(6) +/* Register Unlock */ +#define TPS65214_LOCK_ACCESS_CMD 0x5a /* power ON-OFF sequence slot */ #define TPS65219_BUCKS_LDOS_SEQUENCE_OFF_SLOT_MASK GENMASK(3, 0) #define TPS65219_BUCKS_LDOS_SEQUENCE_ON_SLOT_MASK GENMASK(7, 4) diff --git a/include/linux/mfd/wm8350/core.h b/include/linux/mfd/wm8350/core.h index 5f70d3b5d1b1a..097ef4dfcdac8 100644 --- a/include/linux/mfd/wm8350/core.h +++ b/include/linux/mfd/wm8350/core.h @@ -667,7 +667,7 @@ static inline int wm8350_register_irq(struct wm8350 *wm8350, int irq, return -ENODEV; return request_threaded_irq(irq + wm8350->irq_base, NULL, - handler, flags, name, data); + handler, flags | IRQF_ONESHOT, name, data); } static inline void wm8350_free_irq(struct wm8350 *wm8350, int irq, void *data) diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 5405ca1038f9e..85c2b3d358ec1 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -1274,12 +1274,12 @@ static inline bool mlx5_rl_is_supported(struct mlx5_core_dev *dev) static inline int mlx5_core_is_mp_slave(struct mlx5_core_dev *dev) { return MLX5_CAP_GEN(dev, affiliate_nic_vport_criteria) && - MLX5_CAP_GEN(dev, num_vhca_ports) <= 1; + MLX5_CAP_GEN_MAX(dev, num_vhca_ports) <= 1; } static inline int mlx5_core_is_mp_master(struct mlx5_core_dev *dev) { - return MLX5_CAP_GEN(dev, num_vhca_ports) > 1; + return MLX5_CAP_GEN_MAX(dev, num_vhca_ports) > 1; } static inline int mlx5_core_mp_enabled(struct mlx5_core_dev *dev) diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 90e5790c318f0..8b1045c51e0a6 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -1490,6 +1490,7 @@ static inline void mm_set_cpus_allowed(struct mm_struct *mm, const struct cpumas struct mmu_gather; extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm); extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm); +void tlb_gather_mmu_vma(struct mmu_gather *tlb, struct vm_area_struct *vma); extern void tlb_finish_mmu(struct mmu_gather *tlb); struct vm_fault; diff --git a/include/linux/module.h b/include/linux/module.h index e135cc79aceea..4decae2b16755 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -747,6 +747,15 @@ static inline void __module_get(struct module *module) __mod ? __mod->name : "kernel"; \ }) +static inline const unsigned char *module_buildid(struct module *mod) +{ +#ifdef CONFIG_STACKTRACE_BUILD_ID + return mod->build_id; +#else + return NULL; +#endif +} + /* Dereference module function descriptor */ void *dereference_module_function_descriptor(struct module *mod, void *ptr); diff --git a/include/linux/mtd/spinand.h b/include/linux/mtd/spinand.h index 927c10d787695..1c741145e4971 100644 --- a/include/linux/mtd/spinand.h +++ b/include/linux/mtd/spinand.h @@ -287,7 +287,7 @@ struct spinand_device; /** * struct spinand_id - SPI NAND id structure - * @data: buffer containing the id bytes. Currently 4 bytes large, but can + * @data: buffer containing the id bytes. Currently 5 bytes large, but can * be extended if required * @len: ID length */ diff --git a/include/linux/netdev_features.h b/include/linux/netdev_features.h index 7a01c518e5730..93e4da7046a10 100644 --- a/include/linux/netdev_features.h +++ b/include/linux/netdev_features.h @@ -255,6 +255,24 @@ static inline int find_next_netdev_feature(u64 feature, unsigned long start) NETIF_F_GSO_UDP_TUNNEL | \ NETIF_F_GSO_UDP_TUNNEL_CSUM) +/* virtual device features */ +#define MASTER_UPPER_DEV_VLAN_FEATURES (NETIF_F_HW_CSUM | NETIF_F_SG | \ + NETIF_F_FRAGLIST | NETIF_F_GSO_SOFTWARE | \ + NETIF_F_GSO_ENCAP_ALL | \ + NETIF_F_HIGHDMA | NETIF_F_LRO) + +#define MASTER_UPPER_DEV_ENC_FEATURES (NETIF_F_HW_CSUM | NETIF_F_SG | \ + NETIF_F_RXCSUM | NETIF_F_GSO_SOFTWARE | \ + NETIF_F_GSO_PARTIAL) + +#define MASTER_UPPER_DEV_MPLS_FEATURES (NETIF_F_HW_CSUM | NETIF_F_SG | \ + NETIF_F_GSO_SOFTWARE) + +#define MASTER_UPPER_DEV_XFRM_FEATURES (NETIF_F_HW_ESP | NETIF_F_HW_ESP_TX_CSUM | \ + NETIF_F_GSO_ESP) + +#define MASTER_UPPER_DEV_GSO_PARTIAL_FEATURES (NETIF_F_GSO_ESP) + static inline netdev_features_t netdev_base_features(netdev_features_t features) { features &= ~NETIF_F_ONE_FOR_ALL; diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index c6c04cd0a6816..8bb7b0e2c5438 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -4684,7 +4684,7 @@ static inline u32 netif_msg_init(int debug_value, int default_msg_enable_bits) static inline void __netif_tx_lock(struct netdev_queue *txq, int cpu) { spin_lock(&txq->_xmit_lock); - /* Pairs with READ_ONCE() in __dev_queue_xmit() */ + /* Pairs with READ_ONCE() in netif_tx_owned() */ WRITE_ONCE(txq->xmit_lock_owner, cpu); } @@ -4702,7 +4702,7 @@ static inline void __netif_tx_release(struct netdev_queue *txq) static inline void __netif_tx_lock_bh(struct netdev_queue *txq) { spin_lock_bh(&txq->_xmit_lock); - /* Pairs with READ_ONCE() in __dev_queue_xmit() */ + /* Pairs with READ_ONCE() in netif_tx_owned() */ WRITE_ONCE(txq->xmit_lock_owner, smp_processor_id()); } @@ -4711,7 +4711,7 @@ static inline bool __netif_tx_trylock(struct netdev_queue *txq) bool ok = spin_trylock(&txq->_xmit_lock); if (likely(ok)) { - /* Pairs with READ_ONCE() in __dev_queue_xmit() */ + /* Pairs with READ_ONCE() in netif_tx_owned() */ WRITE_ONCE(txq->xmit_lock_owner, smp_processor_id()); } return ok; @@ -4719,14 +4719,14 @@ static inline bool __netif_tx_trylock(struct netdev_queue *txq) static inline void __netif_tx_unlock(struct netdev_queue *txq) { - /* Pairs with READ_ONCE() in __dev_queue_xmit() */ + /* Pairs with READ_ONCE() in netif_tx_owned() */ WRITE_ONCE(txq->xmit_lock_owner, -1); spin_unlock(&txq->_xmit_lock); } static inline void __netif_tx_unlock_bh(struct netdev_queue *txq) { - /* Pairs with READ_ONCE() in __dev_queue_xmit() */ + /* Pairs with READ_ONCE() in netif_tx_owned() */ WRITE_ONCE(txq->xmit_lock_owner, -1); spin_unlock_bh(&txq->_xmit_lock); } @@ -4819,6 +4819,23 @@ static inline void netif_tx_disable(struct net_device *dev) local_bh_enable(); } +#ifndef CONFIG_PREEMPT_RT +static inline bool netif_tx_owned(struct netdev_queue *txq, unsigned int cpu) +{ + /* Other cpus might concurrently change txq->xmit_lock_owner + * to -1 or to their cpu id, but not to our id. + */ + return READ_ONCE(txq->xmit_lock_owner) == cpu; +} + +#else +static inline bool netif_tx_owned(struct netdev_queue *txq, unsigned int cpu) +{ + return rt_mutex_owner(&txq->_xmit_lock.lock) == current; +} + +#endif + static inline void netif_addr_lock(struct net_device *dev) { unsigned char nest_level = 0; @@ -5306,6 +5323,7 @@ static inline netdev_features_t netdev_add_tso_features(netdev_features_t featur int __netdev_update_features(struct net_device *dev); void netdev_update_features(struct net_device *dev); void netdev_change_features(struct net_device *dev); +void netdev_compute_master_upper_features(struct net_device *dev, bool update_header); void netif_stacked_transfer_operstate(const struct net_device *rootdev, struct net_device *dev); diff --git a/include/linux/pci.h b/include/linux/pci.h index bf97d49c23cf5..05aeee8c8844a 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -406,6 +406,7 @@ struct pci_dev { user sysfs */ unsigned int clear_retrain_link:1; /* Need to clear Retrain Link bit manually */ + unsigned int no_bw_notif:1; /* BW notifications may cause issues */ unsigned int d3hot_delay; /* D3hot->D0 transition time in ms */ unsigned int d3cold_delay; /* D3cold->D0 transition time in ms */ diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index 92ffc4373f6de..03b7c0380f71c 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -3142,6 +3142,7 @@ #define PCI_DEVICE_ID_INTEL_HDA_CML_S 0xa3f0 #define PCI_DEVICE_ID_INTEL_HDA_LNL_P 0xa828 #define PCI_DEVICE_ID_INTEL_S21152BB 0xb152 +#define PCI_DEVICE_ID_INTEL_HDA_NVL 0xd328 #define PCI_DEVICE_ID_INTEL_HDA_BMG 0xe2f7 #define PCI_DEVICE_ID_INTEL_HDA_PTL_H 0xe328 #define PCI_DEVICE_ID_INTEL_HDA_PTL 0xe428 diff --git a/include/linux/pinctrl/pinconf-generic.h b/include/linux/pinctrl/pinconf-generic.h index d9245ecec71dc..4e22aa44bcd4c 100644 --- a/include/linux/pinctrl/pinconf-generic.h +++ b/include/linux/pinctrl/pinconf-generic.h @@ -235,9 +235,4 @@ static inline int pinconf_generic_dt_node_to_map_all(struct pinctrl_dev *pctldev return pinconf_generic_dt_node_to_map(pctldev, np_config, map, num_maps, PIN_MAP_TYPE_INVALID); } - -int pinconf_generic_dt_node_to_map_pinmux(struct pinctrl_dev *pctldev, - struct device_node *np, - struct pinctrl_map **map, - unsigned int *num_maps); #endif /* __LINUX_PINCTRL_PINCONF_GENERIC_H */ diff --git a/include/linux/pm.h b/include/linux/pm.h index cc7b2dc28574c..12782f775a17e 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -677,10 +677,10 @@ struct dev_pm_info { struct list_head entry; struct completion completion; struct wakeup_source *wakeup; + bool work_in_progress; /* Owned by the PM core */ bool wakeup_path:1; bool syscore:1; bool no_pm_callbacks:1; /* Owned by the PM core */ - bool work_in_progress:1; /* Owned by the PM core */ bool smart_suspend:1; /* Owned by the PM core */ bool must_resume:1; /* Owned by the PM core */ bool may_skip_resume:1; /* Set by subsystems */ diff --git a/include/linux/psp.h b/include/linux/psp.h index 92e60aeef21e1..b337dcce1e991 100644 --- a/include/linux/psp.h +++ b/include/linux/psp.h @@ -18,6 +18,7 @@ * and should include an appropriate local definition in their source file. */ #define PSP_CMDRESP_STS GENMASK(15, 0) +#define PSP_TEE_STS_RING_BUSY 0x0000000d /* Ring already initialized */ #define PSP_CMDRESP_CMD GENMASK(23, 16) #define PSP_CMDRESP_RESERVED GENMASK(29, 24) #define PSP_CMDRESP_RECOVERY BIT(30) diff --git a/include/linux/ring_buffer.h b/include/linux/ring_buffer.h index 876358cfe1b12..d862fa610270b 100644 --- a/include/linux/ring_buffer.h +++ b/include/linux/ring_buffer.h @@ -248,6 +248,7 @@ int trace_rb_cpu_prepare(unsigned int cpu, struct hlist_node *node); int ring_buffer_map(struct trace_buffer *buffer, int cpu, struct vm_area_struct *vma); +void ring_buffer_map_dup(struct trace_buffer *buffer, int cpu); int ring_buffer_unmap(struct trace_buffer *buffer, int cpu); int ring_buffer_map_get_reader(struct trace_buffer *buffer, int cpu); #endif /* _LINUX_RING_BUFFER_H */ diff --git a/include/linux/sched.h b/include/linux/sched.h index 6ad294330c0b6..3e2005e9e2f0b 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -574,6 +574,7 @@ struct sched_entity { u64 deadline; u64 min_vruntime; u64 min_slice; + u64 max_slice; struct list_head group_node; unsigned char on_rq; diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h index 49847888c287a..829b281d6c9c2 100644 --- a/include/linux/skmsg.h +++ b/include/linux/skmsg.h @@ -97,6 +97,8 @@ struct sk_psock { struct sk_buff_head ingress_skb; struct list_head ingress_msg; spinlock_t ingress_lock; + /** @msg_tot_len: Total bytes queued in ingress_msg list. */ + u32 msg_tot_len; unsigned long state; struct list_head link; spinlock_t link_lock; @@ -141,6 +143,8 @@ int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from, struct sk_msg *msg, u32 bytes); int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, int len, int flags); +int __sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, + int len, int flags, int *copied_from_self); bool sk_msg_is_readable(struct sock *sk); static inline void sk_msg_check_to_free(struct sk_msg *msg, u32 i, u32 bytes) @@ -319,6 +323,27 @@ static inline void sock_drop(struct sock *sk, struct sk_buff *skb) kfree_skb(skb); } +static inline u32 sk_psock_get_msg_len_nolock(struct sk_psock *psock) +{ + /* Used by ioctl to read msg_tot_len only; lock-free for performance */ + return READ_ONCE(psock->msg_tot_len); +} + +static inline void sk_psock_msg_len_add_locked(struct sk_psock *psock, int diff) +{ + /* Use WRITE_ONCE to ensure correct read in sk_psock_get_msg_len_nolock(). + * ingress_lock should be held to prevent concurrent updates to msg_tot_len + */ + WRITE_ONCE(psock->msg_tot_len, psock->msg_tot_len + diff); +} + +static inline void sk_psock_msg_len_add(struct sk_psock *psock, int diff) +{ + spin_lock_bh(&psock->ingress_lock); + sk_psock_msg_len_add_locked(psock, diff); + spin_unlock_bh(&psock->ingress_lock); +} + static inline bool sk_psock_queue_msg(struct sk_psock *psock, struct sk_msg *msg) { @@ -327,6 +352,7 @@ static inline bool sk_psock_queue_msg(struct sk_psock *psock, spin_lock_bh(&psock->ingress_lock); if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) { list_add_tail(&msg->list, &psock->ingress_msg); + sk_psock_msg_len_add_locked(psock, msg->sg.size); ret = true; } else { sk_msg_free(psock->sk, msg); @@ -343,18 +369,25 @@ static inline struct sk_msg *sk_psock_dequeue_msg(struct sk_psock *psock) spin_lock_bh(&psock->ingress_lock); msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list); - if (msg) + if (msg) { list_del(&msg->list); + sk_psock_msg_len_add_locked(psock, -msg->sg.size); + } spin_unlock_bh(&psock->ingress_lock); return msg; } +static inline struct sk_msg *sk_psock_peek_msg_locked(struct sk_psock *psock) +{ + return list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list); +} + static inline struct sk_msg *sk_psock_peek_msg(struct sk_psock *psock) { struct sk_msg *msg; spin_lock_bh(&psock->ingress_lock); - msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list); + msg = sk_psock_peek_msg_locked(psock); spin_unlock_bh(&psock->ingress_lock); return msg; } @@ -521,6 +554,39 @@ static inline bool sk_psock_strp_enabled(struct sk_psock *psock) return !!psock->saved_data_ready; } +/* for tcp only, sk is locked */ +static inline ssize_t sk_psock_msg_inq(struct sock *sk) +{ + struct sk_psock *psock; + ssize_t inq = 0; + + psock = sk_psock_get(sk); + if (likely(psock)) { + inq = sk_psock_get_msg_len_nolock(psock); + sk_psock_put(sk, psock); + } + return inq; +} + +/* for udp only, sk is not locked */ +static inline ssize_t sk_msg_first_len(struct sock *sk) +{ + struct sk_psock *psock; + struct sk_msg *msg; + ssize_t inq = 0; + + psock = sk_psock_get(sk); + if (likely(psock)) { + spin_lock_bh(&psock->ingress_lock); + msg = sk_psock_peek_msg_locked(psock); + if (msg) + inq = msg->sg.size; + spin_unlock_bh(&psock->ingress_lock); + sk_psock_put(sk, psock); + } + return inq; +} + #if IS_ENABLED(CONFIG_NET_SOCK_MSG) #define BPF_F_STRPARSER (1UL << 1) diff --git a/include/linux/soc/qcom/ubwc.h b/include/linux/soc/qcom/ubwc.h index 1ed8b1b16bc90..d9dfc9edc1b2f 100644 --- a/include/linux/soc/qcom/ubwc.h +++ b/include/linux/soc/qcom/ubwc.h @@ -8,6 +8,7 @@ #define __QCOM_UBWC_H__ #include +#include #include struct qcom_ubwc_cfg_data { diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h index fa1318bac06c4..7e989d0edead2 100644 --- a/include/linux/stmmac.h +++ b/include/linux/stmmac.h @@ -171,7 +171,13 @@ struct dwmac4_addrs { u32 mtl_low_cred_offset; }; -#define STMMAC_FLAG_HAS_INTEGRATED_PCS BIT(0) +enum dwmac_core_type { + DWMAC_CORE_MAC100, + DWMAC_CORE_GMAC, + DWMAC_CORE_GMAC4, + DWMAC_CORE_XGMAC, +}; + #define STMMAC_FLAG_SPH_DISABLE BIT(1) #define STMMAC_FLAG_USE_PHY_WOL BIT(2) #define STMMAC_FLAG_HAS_SUN8I BIT(3) @@ -187,6 +193,7 @@ struct dwmac4_addrs { #define STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY BIT(13) struct plat_stmmacenet_data { + enum dwmac_core_type core_type; int bus_id; int phy_addr; /* MAC ----- optional PCS ----- SerDes ----- optional PHY ----- Media @@ -220,7 +227,6 @@ struct plat_stmmacenet_data { struct stmmac_dma_cfg *dma_cfg; struct stmmac_safety_feature_cfg *safety_feat_cfg; int clk_csr; - int has_gmac; int enh_desc; int tx_coe; int rx_coe; @@ -283,16 +289,13 @@ struct plat_stmmacenet_data { struct reset_control *stmmac_rst; struct reset_control *stmmac_ahb_rst; struct stmmac_axi *axi; - int has_gmac4; int rss_en; int mac_port_sel_speed; - int has_xgmac; u8 vlan_fail_q; struct pci_dev *pdev; int int_snapshot_num; int msi_mac_vec; int msi_wol_vec; - int msi_lpi_vec; int msi_sfty_ce_vec; int msi_sfty_ue_vec; int msi_rx_base_vec; diff --git a/include/linux/sunrpc/xdrgen/_builtins.h b/include/linux/sunrpc/xdrgen/_builtins.h index 66ca3ece951ab..a5ab75d2db044 100644 --- a/include/linux/sunrpc/xdrgen/_builtins.h +++ b/include/linux/sunrpc/xdrgen/_builtins.h @@ -188,12 +188,10 @@ xdrgen_decode_string(struct xdr_stream *xdr, string *ptr, u32 maxlen) return false; if (unlikely(maxlen && len > maxlen)) return false; - if (len != 0) { - p = xdr_inline_decode(xdr, len); - if (unlikely(!p)) - return false; - ptr->data = (unsigned char *)p; - } + p = xdr_inline_decode(xdr, len); + if (unlikely(!p)) + return false; + ptr->data = (unsigned char *)p; ptr->len = len; return true; } @@ -219,12 +217,10 @@ xdrgen_decode_opaque(struct xdr_stream *xdr, opaque *ptr, u32 maxlen) return false; if (unlikely(maxlen && len > maxlen)) return false; - if (len != 0) { - p = xdr_inline_decode(xdr, len); - if (unlikely(!p)) - return false; - ptr->data = (u8 *)p; - } + p = xdr_inline_decode(xdr, len); + if (unlikely(!p)) + return false; + ptr->data = (u8 *)p; ptr->len = len; return true; } diff --git a/include/linux/tnum.h b/include/linux/tnum.h index c52b862dad45b..ca2cfec8de08a 100644 --- a/include/linux/tnum.h +++ b/include/linux/tnum.h @@ -63,6 +63,11 @@ struct tnum tnum_union(struct tnum t1, struct tnum t2); /* Return @a with all but the lowest @size bytes cleared */ struct tnum tnum_cast(struct tnum a, u8 size); +/* Swap the bytes of a tnum */ +struct tnum tnum_bswap16(struct tnum a); +struct tnum tnum_bswap32(struct tnum a); +struct tnum tnum_bswap64(struct tnum a); + /* Returns true if @a is a known constant */ static inline bool tnum_is_const(struct tnum a) { @@ -126,4 +131,7 @@ static inline bool tnum_subreg_is_const(struct tnum a) return !(tnum_subreg(a)).mask; } +/* Returns the smallest member of t larger than z */ +u64 tnum_step(struct tnum t, u64 z); + #endif /* _LINUX_TNUM_H */ diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index 04307a19cde30..64ada9cc38864 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h @@ -682,6 +682,11 @@ static inline void hist_poll_wakeup(void) #define hist_poll_wait(file, wait) \ poll_wait(file, &hist_poll_wq, wait) + +#else +static inline void hist_poll_wakeup(void) +{ +} #endif #define __TRACE_EVENT_FLAGS(name, value) \ diff --git a/include/linux/u64_stats_sync.h b/include/linux/u64_stats_sync.h index 457879938fc19..3366090a86bd2 100644 --- a/include/linux/u64_stats_sync.h +++ b/include/linux/u64_stats_sync.h @@ -89,6 +89,11 @@ static inline void u64_stats_add(u64_stats_t *p, unsigned long val) local64_add(val, &p->v); } +static inline void u64_stats_sub(u64_stats_t *p, s64 val) +{ + local64_sub(val, &p->v); +} + static inline void u64_stats_inc(u64_stats_t *p) { local64_inc(&p->v); @@ -130,6 +135,11 @@ static inline void u64_stats_add(u64_stats_t *p, unsigned long val) p->v += val; } +static inline void u64_stats_sub(u64_stats_t *p, s64 val) +{ + p->v -= val; +} + static inline void u64_stats_inc(u64_stats_t *p) { p->v++; diff --git a/include/linux/unwind_user_types.h b/include/linux/unwind_user_types.h index a449f15be8902..412729a269bcc 100644 --- a/include/linux/unwind_user_types.h +++ b/include/linux/unwind_user_types.h @@ -36,8 +36,10 @@ struct unwind_user_state { unsigned long ip; unsigned long sp; unsigned long fp; + unsigned int ws; enum unwind_user_type current_type; unsigned int available_types; + bool topmost; bool done; }; diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h index ee3d36eda45dd..f548fea2adec8 100644 --- a/include/linux/uprobes.h +++ b/include/linux/uprobes.h @@ -242,6 +242,7 @@ extern void arch_uprobe_clear_state(struct mm_struct *mm); extern void arch_uprobe_init_state(struct mm_struct *mm); extern void handle_syscall_uprobe(struct pt_regs *regs, unsigned long bp_vaddr); extern void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr); +extern unsigned long arch_uprobe_get_xol_area(void); #else /* !CONFIG_UPROBES */ struct uprobes_state { }; diff --git a/include/media/dvb_vb2.h b/include/media/dvb_vb2.h index 8cb88452cd6c2..0fbbfc65157e6 100644 --- a/include/media/dvb_vb2.h +++ b/include/media/dvb_vb2.h @@ -124,7 +124,7 @@ static inline int dvb_vb2_release(struct dvb_vb2_ctx *ctx) return 0; }; #define dvb_vb2_is_streaming(ctx) (0) -#define dvb_vb2_fill_buffer(ctx, file, wait, flags) (0) +#define dvb_vb2_fill_buffer(ctx, file, wait, flags, flush) (0) static inline __poll_t dvb_vb2_poll(struct dvb_vb2_ctx *ctx, struct file *file, @@ -166,10 +166,12 @@ int dvb_vb2_is_streaming(struct dvb_vb2_ctx *ctx); * @buffer_flags: * pointer to buffer flags as defined by &enum dmx_buffer_flags. * can be NULL. + * @flush: flush the buffer, even if it isn't full. */ int dvb_vb2_fill_buffer(struct dvb_vb2_ctx *ctx, const unsigned char *src, int len, - enum dmx_buffer_flags *buffer_flags); + enum dmx_buffer_flags *buffer_flags, + bool flush); /** * dvb_vb2_poll - Wrapper to vb2_core_streamon() for Digital TV diff --git a/include/net/act_api.h b/include/net/act_api.h index 91a24b5e0b93e..2ba40eb45aad2 100644 --- a/include/net/act_api.h +++ b/include/net/act_api.h @@ -70,6 +70,7 @@ struct tc_action { #define TCA_ACT_FLAGS_REPLACE (1U << (TCA_ACT_FLAGS_USER_BITS + 2)) #define TCA_ACT_FLAGS_NO_RTNL (1U << (TCA_ACT_FLAGS_USER_BITS + 3)) #define TCA_ACT_FLAGS_AT_INGRESS (1U << (TCA_ACT_FLAGS_USER_BITS + 4)) +#define TCA_ACT_FLAGS_AT_INGRESS_OR_CLSACT (1U << (TCA_ACT_FLAGS_USER_BITS + 5)) /* Update lastuse only if needed, to avoid dirtying a cache line. * We use a temp variable to avoid fetching jiffies twice. diff --git a/include/net/bluetooth/l2cap.h b/include/net/bluetooth/l2cap.h index 00e182a22720a..f08ed93bb6fa3 100644 --- a/include/net/bluetooth/l2cap.h +++ b/include/net/bluetooth/l2cap.h @@ -284,9 +284,9 @@ struct l2cap_conn_rsp { #define L2CAP_CR_LE_BAD_KEY_SIZE 0x0007 #define L2CAP_CR_LE_ENCRYPTION 0x0008 #define L2CAP_CR_LE_INVALID_SCID 0x0009 -#define L2CAP_CR_LE_SCID_IN_USE 0X000A -#define L2CAP_CR_LE_UNACCEPT_PARAMS 0X000B -#define L2CAP_CR_LE_INVALID_PARAMS 0X000C +#define L2CAP_CR_LE_SCID_IN_USE 0x000A +#define L2CAP_CR_LE_UNACCEPT_PARAMS 0x000B +#define L2CAP_CR_LE_INVALID_PARAMS 0x000C /* connect/create channel status */ #define L2CAP_CS_NO_INFO 0x0000 @@ -493,6 +493,8 @@ struct l2cap_ecred_reconf_req { #define L2CAP_RECONF_SUCCESS 0x0000 #define L2CAP_RECONF_INVALID_MTU 0x0001 #define L2CAP_RECONF_INVALID_MPS 0x0002 +#define L2CAP_RECONF_INVALID_CID 0x0003 +#define L2CAP_RECONF_INVALID_PARAMS 0x0004 struct l2cap_ecred_reconf_rsp { __le16 result; diff --git a/include/net/bonding.h b/include/net/bonding.h index 4620784035570..99c1bdadcd11a 100644 --- a/include/net/bonding.h +++ b/include/net/bonding.h @@ -698,6 +698,7 @@ void bond_debug_register(struct bonding *bond); void bond_debug_unregister(struct bonding *bond); void bond_debug_reregister(struct bonding *bond); const char *bond_mode_name(int mode); +bool __bond_xdp_check(int mode, int xmit_policy); bool bond_xdp_check(struct bonding *bond, int mode); void bond_setup(struct net_device *bond_dev); unsigned int bond_get_num_tx_queues(void); diff --git a/include/net/inet6_hashtables.h b/include/net/inet6_hashtables.h index 282e29237d936..c16de5b7963fd 100644 --- a/include/net/inet6_hashtables.h +++ b/include/net/inet6_hashtables.h @@ -175,7 +175,7 @@ static inline bool inet6_match(const struct net *net, const struct sock *sk, { if (!net_eq(sock_net(sk), net) || sk->sk_family != AF_INET6 || - sk->sk_portpair != ports || + READ_ONCE(sk->sk_portpair) != ports || !ipv6_addr_equal(&sk->sk_v6_daddr, saddr) || !ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr)) return false; diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h index b4b8866476075..afec146b8bafd 100644 --- a/include/net/inet_connection_sock.h +++ b/include/net/inet_connection_sock.h @@ -42,7 +42,9 @@ struct inet_connection_sock_af_ops { struct request_sock *req, struct dst_entry *dst, struct request_sock *req_unhash, - bool *own_req); + bool *own_req, + void (*opt_child_init)(struct sock *newsk, + const struct sock *sk)); u16 net_header_len; int (*setsockopt)(struct sock *sk, int level, int optname, sockptr_t optval, unsigned int optlen); diff --git a/include/net/inet_ecn.h b/include/net/inet_ecn.h index ea32393464a29..827b87a95dab3 100644 --- a/include/net/inet_ecn.h +++ b/include/net/inet_ecn.h @@ -51,11 +51,25 @@ static inline __u8 INET_ECN_encapsulate(__u8 outer, __u8 inner) return outer; } +/* Apply either ECT(0) or ECT(1) */ +static inline void __INET_ECN_xmit(struct sock *sk, bool use_ect_1) +{ + __u8 ect = use_ect_1 ? INET_ECN_ECT_1 : INET_ECN_ECT_0; + + /* Mask the complete byte in case the connection alternates between + * ECT(0) and ECT(1). + */ + inet_sk(sk)->tos &= ~INET_ECN_MASK; + inet_sk(sk)->tos |= ect; + if (inet6_sk(sk)) { + inet6_sk(sk)->tclass &= ~INET_ECN_MASK; + inet6_sk(sk)->tclass |= ect; + } +} + static inline void INET_ECN_xmit(struct sock *sk) { - inet_sk(sk)->tos |= INET_ECN_ECT_0; - if (inet6_sk(sk) != NULL) - inet6_sk(sk)->tclass |= INET_ECN_ECT_0; + __INET_ECN_xmit(sk, false); } static inline void INET_ECN_dontxmit(struct sock *sk) diff --git a/include/net/inet_hashtables.h b/include/net/inet_hashtables.h index ac05a52d9e138..5a979dcab5383 100644 --- a/include/net/inet_hashtables.h +++ b/include/net/inet_hashtables.h @@ -345,7 +345,7 @@ static inline bool inet_match(const struct net *net, const struct sock *sk, int dif, int sdif) { if (!net_eq(sock_net(sk), net) || - sk->sk_portpair != ports || + READ_ONCE(sk->sk_portpair) != ports || sk->sk_addrpair != cookie) return false; diff --git a/include/net/inet_sock.h b/include/net/inet_sock.h index 1086256549faa..b6ec08072533a 100644 --- a/include/net/inet_sock.h +++ b/include/net/inet_sock.h @@ -214,6 +214,7 @@ struct inet_sock { struct sock sk; #if IS_ENABLED(CONFIG_IPV6) struct ipv6_pinfo *pinet6; + struct ipv6_fl_socklist __rcu *ipv6_fl_list; #endif /* Socket demultiplex comparisons on incoming packets. */ #define inet_daddr sk.__sk_common.skc_daddr diff --git a/include/net/ioam6.h b/include/net/ioam6.h index 2cbbee6e806aa..a75912fe247e6 100644 --- a/include/net/ioam6.h +++ b/include/net/ioam6.h @@ -60,6 +60,8 @@ void ioam6_fill_trace_data(struct sk_buff *skb, struct ioam6_trace_hdr *trace, bool is_input); +u8 ioam6_trace_compute_nodelen(u32 trace_type); + int ioam6_init(void); void ioam6_exit(void); diff --git a/include/net/ip.h b/include/net/ip.h index 380afb691c419..1ce79e62a76fb 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -101,7 +101,7 @@ static inline void ipcm_init_sk(struct ipcm_cookie *ipcm, ipcm->oif = READ_ONCE(inet->sk.sk_bound_dev_if); ipcm->addr = inet->inet_saddr; - ipcm->protocol = inet->inet_num; + ipcm->protocol = READ_ONCE(inet->inet_num); } #define IPCB(skb) ((struct inet_skb_parm*)((skb)->cb)) diff --git a/include/net/ip_fib.h b/include/net/ip_fib.h index b4495c38e0a01..318593743b6e1 100644 --- a/include/net/ip_fib.h +++ b/include/net/ip_fib.h @@ -559,7 +559,7 @@ static inline u32 fib_multipath_hash_from_keys(const struct net *net, siphash_aligned_key_t hash_key; u32 mp_seed; - mp_seed = READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_seed).mp_seed; + mp_seed = READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_seed.mp_seed); fib_multipath_hash_construct_key(&hash_key, mp_seed); return flow_hash_from_keys_seed(keys, &hash_key); diff --git a/include/net/ipv6.h b/include/net/ipv6.h index 2ccdf85f34f16..7e984e75f3345 100644 --- a/include/net/ipv6.h +++ b/include/net/ipv6.h @@ -1010,11 +1010,11 @@ static inline int ip6_default_np_autolabel(struct net *net) #if IS_ENABLED(CONFIG_IPV6) static inline int ip6_multipath_hash_policy(const struct net *net) { - return net->ipv6.sysctl.multipath_hash_policy; + return READ_ONCE(net->ipv6.sysctl.multipath_hash_policy); } static inline u32 ip6_multipath_hash_fields(const struct net *net) { - return net->ipv6.sysctl.multipath_hash_fields; + return READ_ONCE(net->ipv6.sysctl.multipath_hash_fields); } #else static inline int ip6_multipath_hash_policy(const struct net *net) @@ -1280,12 +1280,15 @@ int ipv6_sock_mc_drop(struct sock *sk, int ifindex, static inline int ip6_sock_set_v6only(struct sock *sk) { - if (inet_sk(sk)->inet_num) - return -EINVAL; + int ret = 0; + lock_sock(sk); - sk->sk_ipv6only = true; + if (inet_sk(sk)->inet_num) + ret = -EINVAL; + else + sk->sk_ipv6only = true; release_sock(sk); - return 0; + return ret; } static inline void ip6_sock_set_recverr(struct sock *sk) diff --git a/include/net/netfilter/nf_conntrack_count.h b/include/net/netfilter/nf_conntrack_count.h index 52a06de41aa0f..cf0166520cf33 100644 --- a/include/net/netfilter/nf_conntrack_count.h +++ b/include/net/netfilter/nf_conntrack_count.h @@ -13,6 +13,7 @@ struct nf_conncount_list { u32 last_gc; /* jiffies at most recent gc */ struct list_head head; /* connections with the same filtering key */ unsigned int count; /* length of list */ + unsigned int last_gc_count; /* length of list at most recent gc */ }; struct nf_conncount_data *nf_conncount_init(struct net *net, unsigned int keylen); diff --git a/include/net/netfilter/nf_queue.h b/include/net/netfilter/nf_queue.h index 4aeffddb75861..45eb26b2e95b3 100644 --- a/include/net/netfilter/nf_queue.h +++ b/include/net/netfilter/nf_queue.h @@ -6,11 +6,13 @@ #include #include #include +#include #include /* Each queued (to userspace) skbuff has one of these. */ struct nf_queue_entry { struct list_head list; + struct rhash_head hash_node; struct sk_buff *skb; unsigned int id; unsigned int hook_index; /* index in hook_entries->hook[] */ @@ -19,7 +21,9 @@ struct nf_queue_entry { struct net_device *physout; #endif struct nf_hook_state state; + bool nf_ct_is_unconfirmed; u16 size; /* sizeof(entry) + saved route keys */ + u16 queue_num; /* extra space to store route keys */ }; diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h index 0e266c2d0e7f0..c18cffafc9696 100644 --- a/include/net/netfilter/nf_tables.h +++ b/include/net/netfilter/nf_tables.h @@ -317,11 +317,13 @@ static inline void *nft_elem_priv_cast(const struct nft_elem_priv *priv) * @NFT_ITER_UNSPEC: unspecified, to catch errors * @NFT_ITER_READ: read-only iteration over set elements * @NFT_ITER_UPDATE: iteration under mutex to update set element state + * @NFT_ITER_UPDATE_CLONE: clone set before iteration under mutex to update element */ enum nft_iter_type { NFT_ITER_UNSPEC, NFT_ITER_READ, NFT_ITER_UPDATE, + NFT_ITER_UPDATE_CLONE, }; struct nft_set; @@ -452,6 +454,7 @@ struct nft_set_ext; * @init: initialize private data of new set instance * @destroy: destroy private data of set instance * @gc_init: initialize garbage collection + * @abort_skip_removal: skip removal of elements from abort path * @elemsize: element private size * * Operations lookup, update and delete have simpler interfaces, are faster @@ -509,6 +512,7 @@ struct nft_set_ops { const struct nft_set *set); void (*gc_init)(const struct nft_set *set); + bool abort_skip_removal; unsigned int elemsize; }; @@ -1856,6 +1860,11 @@ struct nft_trans_gc { struct rcu_head rcu; }; +static inline int nft_trans_gc_space(const struct nft_trans_gc *trans) +{ + return NFT_TRANS_GC_BATCHCOUNT - trans->count; +} + static inline void nft_ctx_update(struct nft_ctx *ctx, const struct nft_trans *trans) { diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h index 34eb3aecb3f21..62166da045548 100644 --- a/include/net/netns/ipv4.h +++ b/include/net/netns/ipv4.h @@ -87,6 +87,12 @@ struct netns_ipv4 { int sysctl_tcp_rmem[3]; __cacheline_group_end(netns_ipv4_read_rx); + /* ICMP rate limiter hot cache line. */ + __cacheline_group_begin_aligned(icmp); + atomic_t icmp_global_credit; + u32 icmp_global_stamp; + __cacheline_group_end_aligned(icmp); + struct inet_timewait_death_row tcp_death_row; struct udp_table *udp_table; @@ -139,8 +145,7 @@ struct netns_ipv4 { int sysctl_icmp_ratemask; int sysctl_icmp_msgs_per_sec; int sysctl_icmp_msgs_burst; - atomic_t icmp_global_credit; - u32 icmp_global_stamp; + u32 ip_rt_min_pmtu; int ip_rt_mtu_expires; int ip_rt_min_advmss; diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index 738cd5b13c62f..1518454c906e1 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -758,13 +758,23 @@ static inline bool skb_skip_tc_classify(struct sk_buff *skb) static inline void qdisc_reset_all_tx_gt(struct net_device *dev, unsigned int i) { struct Qdisc *qdisc; + bool nolock; for (; i < dev->num_tx_queues; i++) { qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc); if (qdisc) { + nolock = qdisc->flags & TCQ_F_NOLOCK; + + if (nolock) + spin_lock_bh(&qdisc->seqlock); spin_lock_bh(qdisc_lock(qdisc)); qdisc_reset(qdisc); spin_unlock_bh(qdisc_lock(qdisc)); + if (nolock) { + clear_bit(__QDISC_STATE_MISSED, &qdisc->state); + clear_bit(__QDISC_STATE_DRAINING, &qdisc->state); + spin_unlock_bh(&qdisc->seqlock); + } } } } diff --git a/include/net/secure_seq.h b/include/net/secure_seq.h index cddebafb9f779..6f996229167b3 100644 --- a/include/net/secure_seq.h +++ b/include/net/secure_seq.h @@ -5,16 +5,47 @@ #include struct net; +extern struct net init_net; + +union tcp_seq_and_ts_off { + struct { + u32 seq; + u32 ts_off; + }; + u64 hash64; +}; u64 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport); u64 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr, __be16 dport); -u32 secure_tcp_seq(__be32 saddr, __be32 daddr, - __be16 sport, __be16 dport); -u32 secure_tcp_ts_off(const struct net *net, __be32 saddr, __be32 daddr); -u32 secure_tcpv6_seq(const __be32 *saddr, const __be32 *daddr, - __be16 sport, __be16 dport); -u32 secure_tcpv6_ts_off(const struct net *net, - const __be32 *saddr, const __be32 *daddr); +union tcp_seq_and_ts_off +secure_tcp_seq_and_ts_off(const struct net *net, __be32 saddr, __be32 daddr, + __be16 sport, __be16 dport); + +static inline u32 secure_tcp_seq(__be32 saddr, __be32 daddr, + __be16 sport, __be16 dport) +{ + union tcp_seq_and_ts_off ts; + + ts = secure_tcp_seq_and_ts_off(&init_net, saddr, daddr, + sport, dport); + + return ts.seq; +} + +union tcp_seq_and_ts_off +secure_tcpv6_seq_and_ts_off(const struct net *net, const __be32 *saddr, + const __be32 *daddr, + __be16 sport, __be16 dport); + +static inline u32 secure_tcpv6_seq(const __be32 *saddr, const __be32 *daddr, + __be16 sport, __be16 dport) +{ + union tcp_seq_and_ts_off ts; + + ts = secure_tcpv6_seq_and_ts_off(&init_net, saddr, daddr, + sport, dport); + return ts.seq; +} #endif /* _NET_SECURE_SEQ */ diff --git a/include/net/sock.h b/include/net/sock.h index 60bcb13f045c3..9540dcc5a0c01 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -2067,7 +2067,7 @@ static inline int sk_rx_queue_get(const struct sock *sk) static inline void sk_set_socket(struct sock *sk, struct socket *sock) { - sk->sk_socket = sock; + WRITE_ONCE(sk->sk_socket, sock); if (sock) { WRITE_ONCE(sk->sk_uid, SOCK_INODE(sock)->i_uid); WRITE_ONCE(sk->sk_ino, SOCK_INODE(sock)->i_ino); diff --git a/include/net/tc_act/tc_gate.h b/include/net/tc_act/tc_gate.h index c1a67149c6b62..5223c00279d5a 100644 --- a/include/net/tc_act/tc_gate.h +++ b/include/net/tc_act/tc_gate.h @@ -32,6 +32,7 @@ struct tcf_gate_params { s32 tcfg_clockid; size_t num_entries; struct list_head entries; + struct rcu_head rcu; }; #define GATE_ACT_GATE_OPEN BIT(0) @@ -39,7 +40,7 @@ struct tcf_gate_params { struct tcf_gate { struct tc_action common; - struct tcf_gate_params param; + struct tcf_gate_params __rcu *param; u8 current_gate_status; ktime_t current_close_time; u32 current_entry_octets; @@ -51,47 +52,65 @@ struct tcf_gate { #define to_gate(a) ((struct tcf_gate *)a) +static inline struct tcf_gate_params *tcf_gate_params_locked(const struct tc_action *a) +{ + struct tcf_gate *gact = to_gate(a); + + return rcu_dereference_protected(gact->param, + lockdep_is_held(&gact->tcf_lock)); +} + static inline s32 tcf_gate_prio(const struct tc_action *a) { + struct tcf_gate_params *p; s32 tcfg_prio; - tcfg_prio = to_gate(a)->param.tcfg_priority; + p = tcf_gate_params_locked(a); + tcfg_prio = p->tcfg_priority; return tcfg_prio; } static inline u64 tcf_gate_basetime(const struct tc_action *a) { + struct tcf_gate_params *p; u64 tcfg_basetime; - tcfg_basetime = to_gate(a)->param.tcfg_basetime; + p = tcf_gate_params_locked(a); + tcfg_basetime = p->tcfg_basetime; return tcfg_basetime; } static inline u64 tcf_gate_cycletime(const struct tc_action *a) { + struct tcf_gate_params *p; u64 tcfg_cycletime; - tcfg_cycletime = to_gate(a)->param.tcfg_cycletime; + p = tcf_gate_params_locked(a); + tcfg_cycletime = p->tcfg_cycletime; return tcfg_cycletime; } static inline u64 tcf_gate_cycletimeext(const struct tc_action *a) { + struct tcf_gate_params *p; u64 tcfg_cycletimeext; - tcfg_cycletimeext = to_gate(a)->param.tcfg_cycletime_ext; + p = tcf_gate_params_locked(a); + tcfg_cycletimeext = p->tcfg_cycletime_ext; return tcfg_cycletimeext; } static inline u32 tcf_gate_num_entries(const struct tc_action *a) { + struct tcf_gate_params *p; u32 num_entries; - num_entries = to_gate(a)->param.num_entries; + p = tcf_gate_params_locked(a); + num_entries = p->num_entries; return num_entries; } @@ -105,7 +124,7 @@ static inline struct action_gate_entry u32 num_entries; int i = 0; - p = &to_gate(a)->param; + p = tcf_gate_params_locked(a); num_entries = p->num_entries; list_for_each_entry(entry, &p->entries, list) diff --git a/include/net/tc_act/tc_ife.h b/include/net/tc_act/tc_ife.h index c7f24a2da1cad..24d4d5a62b3c2 100644 --- a/include/net/tc_act/tc_ife.h +++ b/include/net/tc_act/tc_ife.h @@ -13,15 +13,13 @@ struct tcf_ife_params { u8 eth_src[ETH_ALEN]; u16 eth_type; u16 flags; - + struct list_head metalist; struct rcu_head rcu; }; struct tcf_ife_info { struct tc_action common; struct tcf_ife_params __rcu *params; - /* list of metaids allowed */ - struct list_head metalist; }; #define to_ife(a) ((struct tcf_ife_info *)a) diff --git a/include/net/tcp.h b/include/net/tcp.h index ab20f549b8f91..7647ed5c732c1 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -43,6 +43,7 @@ #include #include #include +#include #include #include @@ -528,7 +529,9 @@ struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, struct request_sock *req, struct dst_entry *dst, struct request_sock *req_unhash, - bool *own_req); + bool *own_req, + void (*opt_child_init)(struct sock *newsk, + const struct sock *sk)); int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb); int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len); int tcp_connect(struct sock *sk); @@ -1190,7 +1193,15 @@ enum tcp_ca_ack_event_flags { #define TCP_CONG_NON_RESTRICTED BIT(0) /* Requires ECN/ECT set on all packets */ #define TCP_CONG_NEEDS_ECN BIT(1) -#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN) +/* Require successfully negotiated AccECN capability */ +#define TCP_CONG_NEEDS_ACCECN BIT(2) +/* Use ECT(1) instead of ECT(0) while the CA is uninitialized */ +#define TCP_CONG_ECT_1_NEGOTIATION BIT(3) +/* Cannot fallback to RFC3168 during AccECN negotiation */ +#define TCP_CONG_NO_FALLBACK_RFC3168 BIT(4) +#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN | \ + TCP_CONG_NEEDS_ACCECN | TCP_CONG_ECT_1_NEGOTIATION | \ + TCP_CONG_NO_FALLBACK_RFC3168) union tcp_cc_info; @@ -1322,6 +1333,27 @@ static inline bool tcp_ca_needs_ecn(const struct sock *sk) return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN; } +static inline bool tcp_ca_needs_accecn(const struct sock *sk) +{ + const struct inet_connection_sock *icsk = inet_csk(sk); + + return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ACCECN; +} + +static inline bool tcp_ca_ect_1_negotiation(const struct sock *sk) +{ + const struct inet_connection_sock *icsk = inet_csk(sk); + + return icsk->icsk_ca_ops->flags & TCP_CONG_ECT_1_NEGOTIATION; +} + +static inline bool tcp_ca_no_fallback_rfc3168(const struct sock *sk) +{ + const struct inet_connection_sock *icsk = inet_csk(sk); + + return icsk->icsk_ca_ops->flags & TCP_CONG_NO_FALLBACK_RFC3168; +} + static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) { const struct inet_connection_sock *icsk = inet_csk(sk); @@ -2404,8 +2436,9 @@ struct tcp_request_sock_ops { struct flowi *fl, struct request_sock *req, u32 tw_isn); - u32 (*init_seq)(const struct sk_buff *skb); - u32 (*init_ts_off)(const struct net *net, const struct sk_buff *skb); + union tcp_seq_and_ts_off (*init_seq_and_ts_off)( + const struct net *net, + const struct sk_buff *skb); int (*send_synack)(const struct sock *sk, struct dst_entry *dst, struct flowi *fl, struct request_sock *req, struct tcp_fastopen_cookie *foc, diff --git a/include/net/tcp_ecn.h b/include/net/tcp_ecn.h index f13e5cd2b1ac3..a709fb1756eb7 100644 --- a/include/net/tcp_ecn.h +++ b/include/net/tcp_ecn.h @@ -31,6 +31,12 @@ enum tcp_accecn_option { TCP_ACCECN_OPTION_FULL = 2, }; +/* Apply either ECT(0) or ECT(1) based on TCP_CONG_ECT_1_NEGOTIATION flag */ +static inline void INET_ECN_xmit_ect_1_negotiation(struct sock *sk) +{ + __INET_ECN_xmit(sk, tcp_ca_ect_1_negotiation(sk)); +} + static inline void tcp_ecn_queue_cwr(struct tcp_sock *tp) { /* Do not set CWR if in AccECN mode! */ @@ -467,6 +473,26 @@ static inline u8 tcp_accecn_option_init(const struct sk_buff *skb, return TCP_ACCECN_OPT_COUNTER_SEEN; } +static inline void tcp_ecn_rcv_synack_accecn(struct sock *sk, + const struct sk_buff *skb, u8 dsf) +{ + struct tcp_sock *tp = tcp_sk(sk); + + tcp_ecn_mode_set(tp, TCP_ECN_MODE_ACCECN); + tp->syn_ect_rcv = dsf & INET_ECN_MASK; + /* Demand Accurate ECN option in response to the SYN on the SYN/ACK + * and the TCP server will try to send one more packet with an AccECN + * Option at a later point during the connection. + */ + if (tp->rx_opt.accecn && + tp->saw_accecn_opt < TCP_ACCECN_OPT_COUNTER_SEEN) { + u8 saw_opt = tcp_accecn_option_init(skb, tp->rx_opt.accecn); + + tcp_accecn_saw_opt_fail_recv(tp, saw_opt); + tp->accecn_opt_demand = 2; + } +} + /* See Table 2 of the AccECN draft */ static inline void tcp_ecn_rcv_synack(struct sock *sk, const struct sk_buff *skb, const struct tcphdr *th, u8 ip_dsfield) @@ -489,32 +515,32 @@ static inline void tcp_ecn_rcv_synack(struct sock *sk, const struct sk_buff *skb tcp_ecn_mode_set(tp, TCP_ECN_DISABLED); break; case 0x1: - case 0x5: /* +========+========+============+=============+ * | A | B | SYN/ACK | Feedback | * | | | B->A | Mode of A | * | | | AE CWR ECE | | * +========+========+============+=============+ - * | AccECN | Nonce | 1 0 1 | (Reserved) | * | AccECN | ECN | 0 0 1 | Classic ECN | * | Nonce | AccECN | 0 0 1 | Classic ECN | * | ECN | AccECN | 0 0 1 | Classic ECN | * +========+========+============+=============+ */ - if (tcp_ecn_mode_pending(tp)) - /* Downgrade from AccECN, or requested initially */ + if (tcp_ca_no_fallback_rfc3168(sk)) + tcp_ecn_mode_set(tp, TCP_ECN_DISABLED); + else tcp_ecn_mode_set(tp, TCP_ECN_MODE_RFC3168); break; - default: - tcp_ecn_mode_set(tp, TCP_ECN_MODE_ACCECN); - tp->syn_ect_rcv = ip_dsfield & INET_ECN_MASK; - if (tp->rx_opt.accecn && - tp->saw_accecn_opt < TCP_ACCECN_OPT_COUNTER_SEEN) { - u8 saw_opt = tcp_accecn_option_init(skb, tp->rx_opt.accecn); - - tcp_accecn_saw_opt_fail_recv(tp, saw_opt); - tp->accecn_opt_demand = 2; + case 0x5: + if (tcp_ecn_mode_pending(tp)) { + tcp_ecn_rcv_synack_accecn(sk, skb, ip_dsfield); + if (INET_ECN_is_ce(ip_dsfield)) { + tp->received_ce++; + tp->received_ce_pending++; + } } + break; + default: + tcp_ecn_rcv_synack_accecn(sk, skb, ip_dsfield); if (INET_ECN_is_ce(ip_dsfield) && tcp_accecn_validate_syn_feedback(sk, ace, tp->syn_ect_snt)) { @@ -525,9 +551,11 @@ static inline void tcp_ecn_rcv_synack(struct sock *sk, const struct sk_buff *skb } } -static inline void tcp_ecn_rcv_syn(struct tcp_sock *tp, const struct tcphdr *th, +static inline void tcp_ecn_rcv_syn(struct sock *sk, const struct tcphdr *th, const struct sk_buff *skb) { + struct tcp_sock *tp = tcp_sk(sk); + if (tcp_ecn_mode_pending(tp)) { if (!tcp_accecn_syn_requested(th)) { /* Downgrade to classic ECN feedback */ @@ -539,7 +567,8 @@ static inline void tcp_ecn_rcv_syn(struct tcp_sock *tp, const struct tcphdr *th, tcp_ecn_mode_set(tp, TCP_ECN_MODE_ACCECN); } } - if (tcp_ecn_mode_rfc3168(tp) && (!th->ece || !th->cwr)) + if (tcp_ecn_mode_rfc3168(tp) && + (!th->ece || !th->cwr || tcp_ca_no_fallback_rfc3168(sk))) tcp_ecn_mode_set(tp, TCP_ECN_DISABLED); } @@ -561,7 +590,7 @@ static inline void tcp_ecn_send_synack(struct sock *sk, struct sk_buff *skb) TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ECE; else if (tcp_ca_needs_ecn(sk) || tcp_bpf_ca_needs_ecn(sk)) - INET_ECN_xmit(sk); + INET_ECN_xmit_ect_1_negotiation(sk); if (tp->ecn_flags & TCP_ECN_MODE_ACCECN) { TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ACE; @@ -579,7 +608,8 @@ static inline void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb) bool use_ecn, use_accecn; u8 tcp_ecn = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn); - use_accecn = tcp_ecn == TCP_ECN_IN_ACCECN_OUT_ACCECN; + use_accecn = tcp_ecn == TCP_ECN_IN_ACCECN_OUT_ACCECN || + tcp_ca_needs_accecn(sk); use_ecn = tcp_ecn == TCP_ECN_IN_ECN_OUT_ECN || tcp_ecn == TCP_ECN_IN_ACCECN_OUT_ECN || tcp_ca_needs_ecn(sk) || bpf_needs_ecn || use_accecn; @@ -595,7 +625,7 @@ static inline void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb) if (use_ecn) { if (tcp_ca_needs_ecn(sk) || bpf_needs_ecn) - INET_ECN_xmit(sk); + INET_ECN_xmit_ect_1_negotiation(sk); TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ECE | TCPHDR_CWR; if (use_accecn) { diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h index 4f2d3268a6769..33e072768de9d 100644 --- a/include/net/xdp_sock_drv.h +++ b/include/net/xdp_sock_drv.h @@ -47,6 +47,11 @@ static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool) return xsk_pool_get_chunk_size(pool) - xsk_pool_get_headroom(pool); } +static inline u32 xsk_pool_get_rx_frag_step(struct xsk_buff_pool *pool) +{ + return pool->unaligned ? 0 : xsk_pool_get_chunk_size(pool); +} + static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq) { @@ -118,7 +123,7 @@ static inline void xsk_buff_free(struct xdp_buff *xdp) goto out; list_for_each_entry_safe(pos, tmp, xskb_list, list_node) { - list_del(&pos->list_node); + list_del_init(&pos->list_node); xp_free(pos); } @@ -153,7 +158,7 @@ static inline struct xdp_buff *xsk_buff_get_frag(const struct xdp_buff *first) frag = list_first_entry_or_null(&xskb->pool->xskb_list, struct xdp_buff_xsk, list_node); if (frag) { - list_del(&frag->list_node); + list_del_init(&frag->list_node); ret = &frag->xdp; } @@ -164,7 +169,7 @@ static inline void xsk_buff_del_frag(struct xdp_buff *xdp) { struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); - list_del(&xskb->list_node); + list_del_init(&xskb->list_node); } static inline struct xdp_buff *xsk_buff_get_head(struct xdp_buff *first) @@ -333,6 +338,11 @@ static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool) return 0; } +static inline u32 xsk_pool_get_rx_frag_step(struct xsk_buff_pool *pool) +{ + return 0; +} + static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq) { diff --git a/include/rdma/rw.h b/include/rdma/rw.h index d606cac482338..9a8f4b76ce588 100644 --- a/include/rdma/rw.h +++ b/include/rdma/rw.h @@ -66,6 +66,8 @@ int rdma_rw_ctx_post(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u32 port_num, unsigned int rdma_rw_mr_factor(struct ib_device *device, u32 port_num, unsigned int maxpages); +unsigned int rdma_rw_max_send_wr(struct ib_device *dev, u32 port_num, + unsigned int max_rdma_ctxs, u32 create_flags); void rdma_rw_init_qp(struct ib_device *dev, struct ib_qp_init_attr *attr); int rdma_rw_init_mrs(struct ib_qp *qp, struct ib_qp_init_attr *attr); void rdma_rw_cleanup_mrs(struct ib_qp *qp); diff --git a/include/sound/sdca_function.h b/include/sound/sdca_function.h index ea68856e4c8c4..86fd74146c33d 100644 --- a/include/sound/sdca_function.h +++ b/include/sound/sdca_function.h @@ -771,6 +771,7 @@ struct sdca_control { u8 layers; bool deferrable; + bool is_volatile; bool has_default; bool has_fixed; }; diff --git a/include/trace/events/dma.h b/include/trace/events/dma.h index b3fef140ae155..33e99e792f1aa 100644 --- a/include/trace/events/dma.h +++ b/include/trace/events/dma.h @@ -275,6 +275,8 @@ TRACE_EVENT(dma_free_sgt, sizeof(u64), sizeof(u64))) ); +#define DMA_TRACE_MAX_ENTRIES 128 + TRACE_EVENT(dma_map_sg, TP_PROTO(struct device *dev, struct scatterlist *sgl, int nents, int ents, enum dma_data_direction dir, unsigned long attrs), @@ -282,9 +284,12 @@ TRACE_EVENT(dma_map_sg, TP_STRUCT__entry( __string(device, dev_name(dev)) - __dynamic_array(u64, phys_addrs, nents) - __dynamic_array(u64, dma_addrs, ents) - __dynamic_array(unsigned int, lengths, ents) + __field(int, full_nents) + __field(int, full_ents) + __field(bool, truncated) + __dynamic_array(u64, phys_addrs, min(nents, DMA_TRACE_MAX_ENTRIES)) + __dynamic_array(u64, dma_addrs, min(ents, DMA_TRACE_MAX_ENTRIES)) + __dynamic_array(unsigned int, lengths, min(ents, DMA_TRACE_MAX_ENTRIES)) __field(enum dma_data_direction, dir) __field(unsigned long, attrs) ), @@ -292,11 +297,16 @@ TRACE_EVENT(dma_map_sg, TP_fast_assign( struct scatterlist *sg; int i; + int traced_nents = min_t(int, nents, DMA_TRACE_MAX_ENTRIES); + int traced_ents = min_t(int, ents, DMA_TRACE_MAX_ENTRIES); __assign_str(device); - for_each_sg(sgl, sg, nents, i) + __entry->full_nents = nents; + __entry->full_ents = ents; + __entry->truncated = (nents > DMA_TRACE_MAX_ENTRIES) || (ents > DMA_TRACE_MAX_ENTRIES); + for_each_sg(sgl, sg, traced_nents, i) ((u64 *)__get_dynamic_array(phys_addrs))[i] = sg_phys(sg); - for_each_sg(sgl, sg, ents, i) { + for_each_sg(sgl, sg, traced_ents, i) { ((u64 *)__get_dynamic_array(dma_addrs))[i] = sg_dma_address(sg); ((unsigned int *)__get_dynamic_array(lengths))[i] = @@ -306,9 +316,12 @@ TRACE_EVENT(dma_map_sg, __entry->attrs = attrs; ), - TP_printk("%s dir=%s dma_addrs=%s sizes=%s phys_addrs=%s attrs=%s", + TP_printk("%s dir=%s nents=%d/%d ents=%d/%d%s dma_addrs=%s sizes=%s phys_addrs=%s attrs=%s", __get_str(device), decode_dma_data_direction(__entry->dir), + min_t(int, __entry->full_nents, DMA_TRACE_MAX_ENTRIES), __entry->full_nents, + min_t(int, __entry->full_ents, DMA_TRACE_MAX_ENTRIES), __entry->full_ents, + __entry->truncated ? " [TRUNCATED]" : "", __print_array(__get_dynamic_array(dma_addrs), __get_dynamic_array_len(dma_addrs) / sizeof(u64), sizeof(u64)), diff --git a/include/trace/events/netfs.h b/include/trace/events/netfs.h index 64a382fbc31a8..2d366be46a1c3 100644 --- a/include/trace/events/netfs.h +++ b/include/trace/events/netfs.h @@ -57,6 +57,7 @@ EM(netfs_rreq_trace_done, "DONE ") \ EM(netfs_rreq_trace_end_copy_to_cache, "END-C2C") \ EM(netfs_rreq_trace_free, "FREE ") \ + EM(netfs_rreq_trace_intr, "INTR ") \ EM(netfs_rreq_trace_ki_complete, "KI-CMPL") \ EM(netfs_rreq_trace_recollect, "RECLLCT") \ EM(netfs_rreq_trace_redirty, "REDIRTY") \ @@ -169,7 +170,8 @@ EM(netfs_sreq_trace_put_oom, "PUT OOM ") \ EM(netfs_sreq_trace_put_wip, "PUT WIP ") \ EM(netfs_sreq_trace_put_work, "PUT WORK ") \ - E_(netfs_sreq_trace_put_terminated, "PUT TERM ") + EM(netfs_sreq_trace_put_terminated, "PUT TERM ") \ + E_(netfs_sreq_trace_see_failed, "SEE FAILED ") #define netfs_folio_traces \ EM(netfs_folio_is_uptodate, "mod-uptodate") \ diff --git a/include/uapi/drm/drm_fourcc.h b/include/uapi/drm/drm_fourcc.h index e527b24bd824b..c89aede3cb120 100644 --- a/include/uapi/drm/drm_fourcc.h +++ b/include/uapi/drm/drm_fourcc.h @@ -401,8 +401,8 @@ extern "C" { * implementation can multiply the values by 2^6=64. For that reason the padding * must only contain zeros. * index 0 = Y plane, [15:0] z:Y [6:10] little endian - * index 1 = Cr plane, [15:0] z:Cr [6:10] little endian - * index 2 = Cb plane, [15:0] z:Cb [6:10] little endian + * index 1 = Cb plane, [15:0] z:Cb [6:10] little endian + * index 2 = Cr plane, [15:0] z:Cr [6:10] little endian */ #define DRM_FORMAT_S010 fourcc_code('S', '0', '1', '0') /* 2x2 subsampled Cb (1) and Cr (2) planes 10 bits per channel */ #define DRM_FORMAT_S210 fourcc_code('S', '2', '1', '0') /* 2x1 subsampled Cb (1) and Cr (2) planes 10 bits per channel */ @@ -414,8 +414,8 @@ extern "C" { * implementation can multiply the values by 2^4=16. For that reason the padding * must only contain zeros. * index 0 = Y plane, [15:0] z:Y [4:12] little endian - * index 1 = Cr plane, [15:0] z:Cr [4:12] little endian - * index 2 = Cb plane, [15:0] z:Cb [4:12] little endian + * index 1 = Cb plane, [15:0] z:Cb [4:12] little endian + * index 2 = Cr plane, [15:0] z:Cr [4:12] little endian */ #define DRM_FORMAT_S012 fourcc_code('S', '0', '1', '2') /* 2x2 subsampled Cb (1) and Cr (2) planes 12 bits per channel */ #define DRM_FORMAT_S212 fourcc_code('S', '2', '1', '2') /* 2x1 subsampled Cb (1) and Cr (2) planes 12 bits per channel */ @@ -424,8 +424,8 @@ extern "C" { /* * 3 plane YCbCr * index 0 = Y plane, [15:0] Y little endian - * index 1 = Cr plane, [15:0] Cr little endian - * index 2 = Cb plane, [15:0] Cb little endian + * index 1 = Cb plane, [15:0] Cb little endian + * index 2 = Cr plane, [15:0] Cr little endian */ #define DRM_FORMAT_S016 fourcc_code('S', '0', '1', '6') /* 2x2 subsampled Cb (1) and Cr (2) planes 16 bits per channel */ #define DRM_FORMAT_S216 fourcc_code('S', '2', '1', '6') /* 2x1 subsampled Cb (1) and Cr (2) planes 16 bits per channel */ diff --git a/include/uapi/linux/dpll.h b/include/uapi/linux/dpll.h index ab1725a954d74..69d35570ac4f1 100644 --- a/include/uapi/linux/dpll.h +++ b/include/uapi/linux/dpll.h @@ -251,6 +251,7 @@ enum dpll_a_pin { DPLL_A_PIN_ESYNC_FREQUENCY_SUPPORTED, DPLL_A_PIN_ESYNC_PULSE, DPLL_A_PIN_REFERENCE_SYNC, + DPLL_A_PIN_PHASE_ADJUST_GRAN, __DPLL_A_PIN_MAX, DPLL_A_PIN_MAX = (__DPLL_A_PIN_MAX - 1) diff --git a/include/uapi/linux/hyperv.h b/include/uapi/linux/hyperv.h index aaa502a7bff46..1749b35ab2c21 100644 --- a/include/uapi/linux/hyperv.h +++ b/include/uapi/linux/hyperv.h @@ -362,7 +362,7 @@ struct hv_kvp_exchg_msg_value { __u8 value[HV_KVP_EXCHANGE_MAX_VALUE_SIZE]; __u32 value_u32; __u64 value_u64; - }; + } __attribute__((packed)); } __attribute__((packed)); struct hv_kvp_msg_enumerate { diff --git a/include/uapi/linux/netfilter_bridge.h b/include/uapi/linux/netfilter_bridge.h index 1610fdbab98df..ad520d3e9df8f 100644 --- a/include/uapi/linux/netfilter_bridge.h +++ b/include/uapi/linux/netfilter_bridge.h @@ -5,6 +5,10 @@ /* bridge-specific defines for netfilter. */ +#ifndef __KERNEL__ +#include /* for __UAPI_DEF_ETHHDR if defined */ +#endif + #include #include #include diff --git a/include/uapi/linux/nfs.h b/include/uapi/linux/nfs.h index 71c7196d32817..e629c49535345 100644 --- a/include/uapi/linux/nfs.h +++ b/include/uapi/linux/nfs.h @@ -55,7 +55,7 @@ NFSERR_NODEV = 19, /* v2 v3 v4 */ NFSERR_NOTDIR = 20, /* v2 v3 v4 */ NFSERR_ISDIR = 21, /* v2 v3 v4 */ - NFSERR_INVAL = 22, /* v2 v3 v4 */ + NFSERR_INVAL = 22, /* v3 v4 */ NFSERR_FBIG = 27, /* v2 v3 v4 */ NFSERR_NOSPC = 28, /* v2 v3 v4 */ NFSERR_ROFS = 30, /* v2 v3 v4 */ diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h index 07e06aafec502..1172bda87abff 100644 --- a/include/uapi/linux/pci_regs.h +++ b/include/uapi/linux/pci_regs.h @@ -706,7 +706,7 @@ #define PCI_EXP_LNKCTL2_HASD 0x0020 /* HW Autonomous Speed Disable */ #define PCI_EXP_LNKSTA2 0x32 /* Link Status 2 */ #define PCI_EXP_LNKSTA2_FLIT 0x0400 /* Flit Mode Status */ -#define PCI_CAP_EXP_ENDPOINT_SIZEOF_V2 0x32 /* end of v2 EPs w/ link */ +#define PCI_CAP_EXP_ENDPOINT_SIZEOF_V2 0x34 /* end of v2 EPs w/ link */ #define PCI_EXP_SLTCAP2 0x34 /* Slot Capabilities 2 */ #define PCI_EXP_SLTCAP2_IBPD 0x00000001 /* In-band PD Disable Supported */ #define PCI_EXP_SLTCTL2 0x38 /* Slot Control 2 */ diff --git a/include/uapi/linux/vbox_vmmdev_types.h b/include/uapi/linux/vbox_vmmdev_types.h index 6073858d52a2e..11f3627c3729b 100644 --- a/include/uapi/linux/vbox_vmmdev_types.h +++ b/include/uapi/linux/vbox_vmmdev_types.h @@ -236,7 +236,7 @@ struct vmmdev_hgcm_function_parameter32 { /** Relative to the request header. */ __u32 offset; } page_list; - } u; + } __packed u; } __packed; VMMDEV_ASSERT_SIZE(vmmdev_hgcm_function_parameter32, 4 + 8); @@ -251,7 +251,7 @@ struct vmmdev_hgcm_function_parameter64 { union { __u64 phys_addr; __u64 linear_addr; - } u; + } __packed u; } __packed pointer; struct { /** Size of the buffer described by the page list. */ diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index d949db3a46759..17fe07dac6a7e 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -1350,17 +1350,13 @@ static inline void *ufshcd_get_variant(struct ufs_hba *hba) return hba->priv; } -#ifdef CONFIG_PM extern int ufshcd_runtime_suspend(struct device *dev); extern int ufshcd_runtime_resume(struct device *dev); -#endif -#ifdef CONFIG_PM_SLEEP extern int ufshcd_system_suspend(struct device *dev); extern int ufshcd_system_resume(struct device *dev); extern int ufshcd_system_freeze(struct device *dev); extern int ufshcd_system_thaw(struct device *dev); extern int ufshcd_system_restore(struct device *dev); -#endif extern int ufshcd_dme_reset(struct ufs_hba *hba); extern int ufshcd_dme_enable(struct ufs_hba *hba); diff --git a/include/xen/xen.h b/include/xen/xen.h index 61854e3f28377..f280c5dcf9236 100644 --- a/include/xen/xen.h +++ b/include/xen/xen.h @@ -69,11 +69,13 @@ extern u64 xen_saved_max_mem_size; #endif #ifdef CONFIG_XEN_UNPOPULATED_ALLOC +extern unsigned long xen_unpopulated_pages; int xen_alloc_unpopulated_pages(unsigned int nr_pages, struct page **pages); void xen_free_unpopulated_pages(unsigned int nr_pages, struct page **pages); #include int arch_xen_unpopulated_init(struct resource **res); #else +#define xen_unpopulated_pages 0UL #include static inline int xen_alloc_unpopulated_pages(unsigned int nr_pages, struct page **pages) diff --git a/io_uring/cancel.h b/io_uring/cancel.h index 43e9bb74e9d19..eaa4069e258c9 100644 --- a/io_uring/cancel.h +++ b/io_uring/cancel.h @@ -6,10 +6,8 @@ struct io_cancel_data { struct io_ring_ctx *ctx; - union { - u64 data; - struct file *file; - }; + u64 data; + struct file *file; u8 opcode; u32 flags; int seq; diff --git a/io_uring/cmd_net.c b/io_uring/cmd_net.c index 3b75931bd5695..54d05a205e623 100644 --- a/io_uring/cmd_net.c +++ b/io_uring/cmd_net.c @@ -139,16 +139,19 @@ int io_uring_cmd_sock(struct io_uring_cmd *cmd, unsigned int issue_flags) struct proto *prot = READ_ONCE(sk->sk_prot); int ret, arg = 0; - if (!prot || !prot->ioctl) - return -EOPNOTSUPP; - switch (cmd->cmd_op) { case SOCKET_URING_OP_SIOCINQ: + if (!prot || !prot->ioctl) + return -EOPNOTSUPP; + ret = prot->ioctl(sk, SIOCINQ, &arg); if (ret) return ret; return arg; case SOCKET_URING_OP_SIOCOUTQ: + if (!prot || !prot->ioctl) + return -EOPNOTSUPP; + ret = prot->ioctl(sk, SIOCOUTQ, &arg); if (ret) return ret; diff --git a/io_uring/fdinfo.c b/io_uring/fdinfo.c index 294c75a8a3bdb..3585ad8308504 100644 --- a/io_uring/fdinfo.c +++ b/io_uring/fdinfo.c @@ -65,7 +65,7 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) unsigned int cq_head = READ_ONCE(r->cq.head); unsigned int cq_tail = READ_ONCE(r->cq.tail); unsigned int sq_shift = 0; - unsigned int sq_entries; + unsigned int cq_entries, sq_entries; int sq_pid = -1, sq_cpu = -1; u64 sq_total_time = 0, sq_work_time = 0; unsigned int i; @@ -119,9 +119,11 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) } } seq_printf(m, "\n"); + cond_resched(); } seq_printf(m, "CQEs:\t%u\n", cq_tail - cq_head); - while (cq_head < cq_tail) { + cq_entries = min(cq_tail - cq_head, ctx->cq_entries); + for (i = 0; i < cq_entries; i++) { struct io_uring_cqe *cqe; bool cqe32 = false; @@ -136,8 +138,11 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) cqe->big_cqe[0], cqe->big_cqe[1]); seq_printf(m, "\n"); cq_head++; - if (cqe32) + if (cqe32) { cq_head++; + i++; + } + cond_resched(); } if (ctx->flags & IORING_SETUP_SQPOLL) { diff --git a/io_uring/filetable.c b/io_uring/filetable.c index 794ef95df293c..cb1838c9fc377 100644 --- a/io_uring/filetable.c +++ b/io_uring/filetable.c @@ -22,6 +22,10 @@ static int io_file_bitmap_get(struct io_ring_ctx *ctx) if (!table->bitmap) return -ENFILE; + if (table->alloc_hint < ctx->file_alloc_start || + table->alloc_hint >= ctx->file_alloc_end) + table->alloc_hint = ctx->file_alloc_start; + do { ret = find_next_zero_bit(table->bitmap, nr, table->alloc_hint); if (ret != nr) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 104192bcc8e4b..65af47b9135b8 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -148,7 +148,7 @@ static bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, static void io_queue_sqe(struct io_kiocb *req, unsigned int extra_flags); static void __io_req_caches_free(struct io_ring_ctx *ctx); -static __read_mostly DEFINE_STATIC_KEY_FALSE(io_key_has_sqarray); +static __read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(io_key_has_sqarray, HZ); struct kmem_cache *req_cachep; static struct workqueue_struct *iou_wq __ro_after_init; @@ -2390,7 +2390,7 @@ static bool io_get_sqe(struct io_ring_ctx *ctx, const struct io_uring_sqe **sqe) unsigned mask = ctx->sq_entries - 1; unsigned head = ctx->cached_sq_head++ & mask; - if (static_branch_unlikely(&io_key_has_sqarray) && + if (static_branch_unlikely(&io_key_has_sqarray.key) && (!(ctx->flags & IORING_SETUP_NO_SQARRAY))) { head = READ_ONCE(ctx->sq_array[head]); if (unlikely(head >= ctx->sq_entries)) { @@ -2869,7 +2869,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) io_rings_free(ctx); if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) - static_branch_dec(&io_key_has_sqarray); + static_branch_slow_dec_deferred(&io_key_has_sqarray); percpu_ref_exit(&ctx->refs); free_uid(ctx->user); @@ -3490,7 +3490,11 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, ctx = file->private_data; ret = -EBADFD; - if (unlikely(ctx->flags & IORING_SETUP_R_DISABLED)) + /* + * Keep IORING_SETUP_R_DISABLED check before submitter_task load + * in io_uring_add_tctx_node() -> __io_uring_add_tctx_node_from_submit() + */ + if (unlikely(smp_load_acquire(&ctx->flags) & IORING_SETUP_R_DISABLED)) goto out; /* @@ -3813,7 +3817,7 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, ctx->clock_offset = 0; if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) - static_branch_inc(&io_key_has_sqarray); + static_branch_deferred_inc(&io_key_has_sqarray); if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) && !(ctx->flags & IORING_SETUP_IOPOLL) && diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index d974381d93ff7..308ef71bcb286 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -669,8 +669,9 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg) bl->buf_ring = br; if (reg.flags & IOU_PBUF_RING_INC) bl->flags |= IOBL_INC; - io_buffer_add_list(ctx, bl, reg.bgid); - return 0; + ret = io_buffer_add_list(ctx, bl, reg.bgid); + if (!ret) + return 0; fail: io_free_region(ctx, &bl->region); kfree(bl); diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c index 5e5b94236d720..bce74a8b64c6e 100644 --- a/io_uring/msg_ring.c +++ b/io_uring/msg_ring.c @@ -124,7 +124,11 @@ static int __io_msg_ring_data(struct io_ring_ctx *target_ctx, return -EINVAL; if (!(msg->flags & IORING_MSG_RING_FLAGS_PASS) && msg->dst_fd) return -EINVAL; - if (target_ctx->flags & IORING_SETUP_R_DISABLED) + /* + * Keep IORING_SETUP_R_DISABLED check before submitter_task load + * in io_msg_data_remote() -> io_msg_remote_post() + */ + if (smp_load_acquire(&target_ctx->flags) & IORING_SETUP_R_DISABLED) return -EBADFD; if (io_msg_need_remote(target_ctx)) @@ -244,7 +248,11 @@ static int io_msg_send_fd(struct io_kiocb *req, unsigned int issue_flags) return -EINVAL; if (target_ctx == ctx) return -EINVAL; - if (target_ctx->flags & IORING_SETUP_R_DISABLED) + /* + * Keep IORING_SETUP_R_DISABLED check before submitter_task load + * in io_msg_fd_remote() + */ + if (smp_load_acquire(&target_ctx->flags) & IORING_SETUP_R_DISABLED) return -EBADFD; if (!msg->src_file) { int ret = io_msg_grab_file(req, issue_flags); diff --git a/io_uring/net.c b/io_uring/net.c index 43d77f95db51d..2e21a4294407b 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -516,7 +516,11 @@ static inline bool io_send_finish(struct io_kiocb *req, cflags = io_put_kbufs(req, sel->val, sel->buf_list, io_bundle_nbufs(kmsg, sel->val)); - if (bundle_finished || req->flags & REQ_F_BL_EMPTY) + /* + * Don't start new bundles if the buffer list is empty, or if the + * current operation needed to go through polling to complete. + */ + if (bundle_finished || req->flags & (REQ_F_BL_EMPTY | REQ_F_POLLED)) goto finish; /* diff --git a/io_uring/openclose.c b/io_uring/openclose.c index 15dde9bd6ff67..606ce0664e6a4 100644 --- a/io_uring/openclose.c +++ b/io_uring/openclose.c @@ -336,31 +336,34 @@ static int io_pipe_fixed(struct io_kiocb *req, struct file **files, { struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe); struct io_ring_ctx *ctx = req->ctx; + bool alloc_slot; int ret, fds[2] = { -1, -1 }; int slot = p->file_slot; if (p->flags & O_CLOEXEC) return -EINVAL; + alloc_slot = slot == IORING_FILE_INDEX_ALLOC; + io_ring_submit_lock(ctx, issue_flags); ret = __io_fixed_fd_install(ctx, files[0], slot); if (ret < 0) goto err; - fds[0] = ret; + fds[0] = alloc_slot ? ret : slot - 1; files[0] = NULL; /* * If a specific slot is given, next one will be used for * the write side. */ - if (slot != IORING_FILE_INDEX_ALLOC) + if (!alloc_slot) slot++; ret = __io_fixed_fd_install(ctx, files[1], slot); if (ret < 0) goto err; - fds[1] = ret; + fds[1] = alloc_slot ? ret : slot - 1; files[1] = NULL; io_ring_submit_unlock(ctx, issue_flags); diff --git a/io_uring/register.c b/io_uring/register.c index d189b266b8cce..db53e664348d7 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -193,7 +193,8 @@ static int io_register_enable_rings(struct io_ring_ctx *ctx) if (ctx->restrictions.registered) ctx->restricted = 1; - ctx->flags &= ~IORING_SETUP_R_DISABLED; + /* Keep submitter_task store before clearing IORING_SETUP_R_DISABLED */ + smp_store_release(&ctx->flags, ctx->flags & ~IORING_SETUP_R_DISABLED); if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait)) wake_up(&ctx->sq_data->wait); return 0; diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 160b4de2d00d3..44442bf4827e4 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -1185,12 +1185,16 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx return -EBUSY; nbufs = src_ctx->buf_table.nr; + if (!nbufs) + return -ENXIO; if (!arg->nr) arg->nr = nbufs; else if (arg->nr > nbufs) return -EINVAL; else if (arg->nr > IORING_MAX_REG_BUFFERS) return -EINVAL; + if (check_add_overflow(arg->nr, arg->src_off, &off) || off > nbufs) + return -EOVERFLOW; if (check_add_overflow(arg->nr, arg->dst_off, &nbufs)) return -EOVERFLOW; if (nbufs > IORING_MAX_REG_BUFFERS) @@ -1210,21 +1214,6 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx } } - ret = -ENXIO; - nbufs = src_ctx->buf_table.nr; - if (!nbufs) - goto out_free; - ret = -EINVAL; - if (!arg->nr) - arg->nr = nbufs; - else if (arg->nr > nbufs) - goto out_free; - ret = -EOVERFLOW; - if (check_add_overflow(arg->nr, arg->src_off, &off)) - goto out_free; - if (off > nbufs) - goto out_free; - off = arg->dst_off; i = arg->src_off; nr = arg->nr; @@ -1237,8 +1226,8 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx } else { dst_node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); if (!dst_node) { - ret = -ENOMEM; - goto out_free; + io_rsrc_data_free(ctx, &data); + return -ENOMEM; } refcount_inc(&src_node->buf->refs); @@ -1274,10 +1263,6 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx WARN_ON_ONCE(ctx->buf_table.nr); ctx->buf_table = data; return 0; - -out_free: - io_rsrc_data_free(ctx, &data); - return ret; } /* diff --git a/io_uring/sync.c b/io_uring/sync.c index cea2d381ffd2a..ab7fa1cd7dd63 100644 --- a/io_uring/sync.c +++ b/io_uring/sync.c @@ -62,6 +62,8 @@ int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EINVAL; sync->off = READ_ONCE(sqe->off); + if (sync->off < 0) + return -EINVAL; sync->len = READ_ONCE(sqe->len); req->flags |= REQ_F_FORCE_ASYNC; return 0; diff --git a/io_uring/timeout.c b/io_uring/timeout.c index 17e3aab0af367..cd42316d0f3c0 100644 --- a/io_uring/timeout.c +++ b/io_uring/timeout.c @@ -129,7 +129,7 @@ __cold void io_flush_timeouts(struct io_ring_ctx *ctx) u32 seq; raw_spin_lock_irq(&ctx->timeout_lock); - seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); + seq = READ_ONCE(ctx->cached_cq_tail) - atomic_read(&ctx->cq_timeouts); list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) { struct io_kiocb *req = cmd_to_io_kiocb(timeout); diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 03396769c775d..c524be7109c29 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -287,6 +287,9 @@ static int io_zcrx_map_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) } ret = io_populate_area_dma(ifq, area); + if (ret && !area->mem.is_dmabuf) + dma_unmap_sgtable(ifq->dev, &area->mem.page_sg_table, + DMA_FROM_DEVICE, IO_DMA_ATTR); if (ret == 0) area->is_mapped = true; return ret; @@ -333,10 +336,14 @@ static inline atomic_t *io_get_user_counter(struct net_iov *niov) static bool io_zcrx_put_niov_uref(struct net_iov *niov) { atomic_t *uref = io_get_user_counter(niov); + int old; + + old = atomic_read(uref); + do { + if (unlikely(old == 0)) + return false; + } while (!atomic_try_cmpxchg(uref, &old, old - 1)); - if (unlikely(!atomic_read(uref))) - return false; - atomic_dec(uref); return true; } diff --git a/ipc/ipc_sysctl.c b/ipc/ipc_sysctl.c index 15b17e86e198c..9b087ebeb643b 100644 --- a/ipc/ipc_sysctl.c +++ b/ipc/ipc_sysctl.c @@ -214,7 +214,7 @@ static int ipc_permissions(struct ctl_table_header *head, const struct ctl_table if (((table->data == &ns->ids[IPC_SEM_IDS].next_id) || (table->data == &ns->ids[IPC_MSG_IDS].next_id) || (table->data == &ns->ids[IPC_SHM_IDS].next_id)) && - checkpoint_restore_ns_capable(ns->user_ns)) + checkpoint_restore_ns_capable_noaudit(ns->user_ns)) mode = 0666; else #endif diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index d595fe512498c..c2278f392e932 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -713,8 +713,8 @@ static struct bpf_ksym *bpf_ksym_find(unsigned long addr) return n ? container_of(n, struct bpf_ksym, tnode) : NULL; } -int __bpf_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char *sym) +int bpf_address_lookup(unsigned long addr, unsigned long *size, + unsigned long *off, char *sym) { struct bpf_ksym *ksym; int ret = 0; diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c index 703e5df1f4ef9..306bf98378041 100644 --- a/kernel/bpf/cpumap.c +++ b/kernel/bpf/cpumap.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -52,6 +53,7 @@ struct xdp_bulk_queue { struct list_head flush_node; struct bpf_cpu_map_entry *obj; unsigned int count; + local_lock_t bq_lock; }; /* Struct for every remote "destination" CPU in map */ @@ -451,6 +453,7 @@ __cpu_map_entry_alloc(struct bpf_map *map, struct bpf_cpumap_val *value, for_each_possible_cpu(i) { bq = per_cpu_ptr(rcpu->bulkq, i); bq->obj = rcpu; + local_lock_init(&bq->bq_lock); } /* Alloc queue */ @@ -717,6 +720,8 @@ static void bq_flush_to_queue(struct xdp_bulk_queue *bq) struct ptr_ring *q; int i; + lockdep_assert_held(&bq->bq_lock); + if (unlikely(!bq->count)) return; @@ -744,11 +749,15 @@ static void bq_flush_to_queue(struct xdp_bulk_queue *bq) } /* Runs under RCU-read-side, plus in softirq under NAPI protection. - * Thus, safe percpu variable access. + * Thus, safe percpu variable access. PREEMPT_RT relies on + * local_lock_nested_bh() to serialise access to the per-CPU bq. */ static void bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf) { - struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq); + struct xdp_bulk_queue *bq; + + local_lock_nested_bh(&rcpu->bulkq->bq_lock); + bq = this_cpu_ptr(rcpu->bulkq); if (unlikely(bq->count == CPU_MAP_BULK_SIZE)) bq_flush_to_queue(bq); @@ -769,6 +778,8 @@ static void bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf) list_add(&bq->flush_node, flush_list); } + + local_unlock_nested_bh(&rcpu->bulkq->bq_lock); } int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf, @@ -805,7 +816,9 @@ void __cpu_map_flush(struct list_head *flush_list) struct xdp_bulk_queue *bq, *tmp; list_for_each_entry_safe(bq, tmp, flush_list, flush_node) { + local_lock_nested_bh(&bq->obj->bulkq->bq_lock); bq_flush_to_queue(bq); + local_unlock_nested_bh(&bq->obj->bulkq->bq_lock); /* If already running, costs spin_lock_irqsave + smb_mb */ wake_up_process(bq->obj->kthread); diff --git a/kernel/bpf/crypto.c b/kernel/bpf/crypto.c index 83c4d9943084b..1d024fe7248ac 100644 --- a/kernel/bpf/crypto.c +++ b/kernel/bpf/crypto.c @@ -261,6 +261,12 @@ __bpf_kfunc void bpf_crypto_ctx_release(struct bpf_crypto_ctx *ctx) call_rcu(&ctx->rcu, crypto_free_cb); } +__bpf_kfunc void bpf_crypto_ctx_release_dtor(void *ctx) +{ + bpf_crypto_ctx_release(ctx); +} +CFI_NOSEAL(bpf_crypto_ctx_release_dtor); + static int bpf_crypto_crypt(const struct bpf_crypto_ctx *ctx, const struct bpf_dynptr_kern *src, const struct bpf_dynptr_kern *dst, @@ -368,7 +374,7 @@ static const struct btf_kfunc_id_set crypt_kfunc_set = { BTF_ID_LIST(bpf_crypto_dtor_ids) BTF_ID(struct, bpf_crypto_ctx) -BTF_ID(func, bpf_crypto_ctx_release) +BTF_ID(func, bpf_crypto_ctx_release_dtor) static int __init crypto_kfunc_init(void) { diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c index 2625601de76e9..3d619d01088e3 100644 --- a/kernel/bpf/devmap.c +++ b/kernel/bpf/devmap.c @@ -45,6 +45,7 @@ * types of devmap; only the lookup and insertion is different. */ #include +#include #include #include #include @@ -60,6 +61,7 @@ struct xdp_dev_bulk_queue { struct net_device *dev_rx; struct bpf_prog *xdp_prog; unsigned int count; + local_lock_t bq_lock; }; struct bpf_dtab_netdev { @@ -381,6 +383,8 @@ static void bq_xmit_all(struct xdp_dev_bulk_queue *bq, u32 flags) int to_send = cnt; int i; + lockdep_assert_held(&bq->bq_lock); + if (unlikely(!cnt)) return; @@ -425,10 +429,12 @@ void __dev_flush(struct list_head *flush_list) struct xdp_dev_bulk_queue *bq, *tmp; list_for_each_entry_safe(bq, tmp, flush_list, flush_node) { + local_lock_nested_bh(&bq->dev->xdp_bulkq->bq_lock); bq_xmit_all(bq, XDP_XMIT_FLUSH); bq->dev_rx = NULL; bq->xdp_prog = NULL; __list_del_clearprev(&bq->flush_node); + local_unlock_nested_bh(&bq->dev->xdp_bulkq->bq_lock); } } @@ -451,12 +457,16 @@ static void *__dev_map_lookup_elem(struct bpf_map *map, u32 key) /* Runs in NAPI, i.e., softirq under local_bh_disable(). Thus, safe percpu * variable access, and map elements stick around. See comment above - * xdp_do_flush() in filter.c. + * xdp_do_flush() in filter.c. PREEMPT_RT relies on local_lock_nested_bh() + * to serialise access to the per-CPU bq. */ static void bq_enqueue(struct net_device *dev, struct xdp_frame *xdpf, struct net_device *dev_rx, struct bpf_prog *xdp_prog) { - struct xdp_dev_bulk_queue *bq = this_cpu_ptr(dev->xdp_bulkq); + struct xdp_dev_bulk_queue *bq; + + local_lock_nested_bh(&dev->xdp_bulkq->bq_lock); + bq = this_cpu_ptr(dev->xdp_bulkq); if (unlikely(bq->count == DEV_MAP_BULK_SIZE)) bq_xmit_all(bq, 0); @@ -477,6 +487,8 @@ static void bq_enqueue(struct net_device *dev, struct xdp_frame *xdpf, } bq->q[bq->count++] = xdpf; + + local_unlock_nested_bh(&dev->xdp_bulkq->bq_lock); } static inline int __xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf, @@ -588,18 +600,22 @@ static inline bool is_ifindex_excluded(int *excluded, int num_excluded, int ifin } /* Get ifindex of each upper device. 'indexes' must be able to hold at - * least MAX_NEST_DEV elements. - * Returns the number of ifindexes added. + * least 'max' elements. + * Returns the number of ifindexes added, or -EOVERFLOW if there are too + * many upper devices. */ -static int get_upper_ifindexes(struct net_device *dev, int *indexes) +static int get_upper_ifindexes(struct net_device *dev, int *indexes, int max) { struct net_device *upper; struct list_head *iter; int n = 0; netdev_for_each_upper_dev_rcu(dev, upper, iter) { + if (n >= max) + return -EOVERFLOW; indexes[n++] = upper->ifindex; } + return n; } @@ -615,7 +631,11 @@ int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx, int err; if (exclude_ingress) { - num_excluded = get_upper_ifindexes(dev_rx, excluded_devices); + num_excluded = get_upper_ifindexes(dev_rx, excluded_devices, + ARRAY_SIZE(excluded_devices) - 1); + if (num_excluded < 0) + return num_excluded; + excluded_devices[num_excluded++] = dev_rx->ifindex; } @@ -733,7 +753,11 @@ int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, int err; if (exclude_ingress) { - num_excluded = get_upper_ifindexes(dev, excluded_devices); + num_excluded = get_upper_ifindexes(dev, excluded_devices, + ARRAY_SIZE(excluded_devices) - 1); + if (num_excluded < 0) + return num_excluded; + excluded_devices[num_excluded++] = dev->ifindex; } @@ -1115,8 +1139,13 @@ static int dev_map_notification(struct notifier_block *notifier, if (!netdev->xdp_bulkq) return NOTIFY_BAD; - for_each_possible_cpu(cpu) - per_cpu_ptr(netdev->xdp_bulkq, cpu)->dev = netdev; + for_each_possible_cpu(cpu) { + struct xdp_dev_bulk_queue *bq; + + bq = per_cpu_ptr(netdev->xdp_bulkq, cpu); + bq->dev = netdev; + local_lock_init(&bq->bq_lock); + } break; case NETDEV_UNREGISTER: /* This rcu_read_lock/unlock pair is needed because diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 81ef159ef89bd..68da6dcfb4bb7 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -1080,7 +1080,7 @@ const struct bpf_func_proto bpf_snprintf_proto = { .func = bpf_snprintf, .gpl_only = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM_OR_NULL, + .arg1_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, .arg2_type = ARG_CONST_SIZE_OR_ZERO, .arg3_type = ARG_PTR_TO_CONST_STR, .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, diff --git a/kernel/bpf/rqspinlock.c b/kernel/bpf/rqspinlock.c index 3faf9cbd6c753..c0c93a0f5af63 100644 --- a/kernel/bpf/rqspinlock.c +++ b/kernel/bpf/rqspinlock.c @@ -276,10 +276,11 @@ int __lockfunc resilient_tas_spin_lock(rqspinlock_t *lock) RES_INIT_TIMEOUT(ts); /* - * The fast path is not invoked for the TAS fallback, so we must grab - * the deadlock detection entry here. + * We are either called directly from res_spin_lock after grabbing the + * deadlock detection entry when queued spinlocks are disabled, or from + * resilient_queued_spin_lock_slowpath after grabbing the deadlock + * detection entry. No need to obtain it here. */ - grab_held_lock_entry(lock); /* * Since the waiting loop's time is dependent on the amount of diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index df219e7259099..586ece78f783a 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1365,11 +1365,6 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, return ret; } -static bool bpf_net_capable(void) -{ - return capable(CAP_NET_ADMIN) || capable(CAP_SYS_ADMIN); -} - #define BPF_MAP_CREATE_LAST_FIELD excl_prog_hash_size /* called via syscall */ static int map_create(union bpf_attr *attr, bpfptr_t uattr) @@ -2830,6 +2825,13 @@ static int bpf_prog_verify_signature(struct bpf_prog *prog, union bpf_attr *attr void *sig; int err = 0; + /* + * Don't attempt to use kmalloc_large or vmalloc for signatures. + * Practical signature for BPF program should be below this limit. + */ + if (attr->signature_size > KMALLOC_MAX_CACHE_SIZE) + return -EINVAL; + if (system_keyring_id_check(attr->keyring_id) == 0) key = bpf_lookup_system_key(attr->keyring_id); else @@ -4554,6 +4556,8 @@ static int bpf_prog_detach(const union bpf_attr *attr) prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); if (IS_ERR(prog)) return PTR_ERR(prog); + } else if (!bpf_mprog_detach_empty(ptype)) { + return -EPERM; } } else if (is_cgroup_prog_type(ptype, 0, false)) { if (attr->attach_flags || attr->relative_fd) @@ -5299,6 +5303,9 @@ static int bpf_map_get_info_by_fd(struct file *file, if (info.hash_size != SHA256_DIGEST_SIZE) return -EINVAL; + if (!READ_ONCE(map->frozen)) + return -EPERM; + err = map->ops->map_get_hash(map, SHA256_DIGEST_SIZE, map->sha); if (err != 0) return err; @@ -6396,7 +6403,7 @@ static const struct bpf_func_proto bpf_kallsyms_lookup_name_proto = { .func = bpf_kallsyms_lookup_name, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM, + .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg2_type = ARG_CONST_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED, diff --git a/kernel/bpf/tnum.c b/kernel/bpf/tnum.c index f8e70e9c3998d..4abc359b3db01 100644 --- a/kernel/bpf/tnum.c +++ b/kernel/bpf/tnum.c @@ -8,6 +8,7 @@ */ #include #include +#include #define TNUM(_v, _m) (struct tnum){.value = _v, .mask = _m} /* A completely unknown value */ @@ -253,3 +254,74 @@ struct tnum tnum_const_subreg(struct tnum a, u32 value) { return tnum_with_subreg(a, tnum_const(value)); } + +struct tnum tnum_bswap16(struct tnum a) +{ + return TNUM(swab16(a.value & 0xFFFF), swab16(a.mask & 0xFFFF)); +} + +struct tnum tnum_bswap32(struct tnum a) +{ + return TNUM(swab32(a.value & 0xFFFFFFFF), swab32(a.mask & 0xFFFFFFFF)); +} + +struct tnum tnum_bswap64(struct tnum a) +{ + return TNUM(swab64(a.value), swab64(a.mask)); +} + +/* Given tnum t, and a number z such that tmin <= z < tmax, where tmin + * is the smallest member of the t (= t.value) and tmax is the largest + * member of t (= t.value | t.mask), returns the smallest member of t + * larger than z. + * + * For example, + * t = x11100x0 + * z = 11110001 (241) + * result = 11110010 (242) + * + * Note: if this function is called with z >= tmax, it just returns + * early with tmax; if this function is called with z < tmin, the + * algorithm already returns tmin. + */ +u64 tnum_step(struct tnum t, u64 z) +{ + u64 tmax, j, p, q, r, s, v, u, w, res; + u8 k; + + tmax = t.value | t.mask; + + /* if z >= largest member of t, return largest member of t */ + if (z >= tmax) + return tmax; + + /* if z < smallest member of t, return smallest member of t */ + if (z < t.value) + return t.value; + + /* keep t's known bits, and match all unknown bits to z */ + j = t.value | (z & t.mask); + + if (j > z) { + p = ~z & t.value & ~t.mask; + k = fls64(p); /* k is the most-significant 0-to-1 flip */ + q = U64_MAX << k; + r = q & z; /* positions > k matched to z */ + s = ~q & t.value; /* positions <= k matched to t.value */ + v = r | s; + res = v; + } else { + p = z & ~t.value & ~t.mask; + k = fls64(p); /* k is the most-significant 1-to-0 flip */ + q = U64_MAX << k; + r = q & t.mask & z; /* unknown positions > k, matched to z */ + s = q & ~t.mask; /* known positions > k, set to 1 */ + v = r | s; + /* add 1 to unknown positions > k to make value greater than z */ + u = v + (1ULL << k); + /* extract bits in unknown positions > k from u, rest from t.value */ + w = (u & t.mask) | t.value; + res = w; + } + return res; +} diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 04104397c432e..40f19147f227e 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -751,10 +751,8 @@ int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, mutex_lock(&tr->mutex); shim_link = cgroup_shim_find(tr, bpf_func); - if (shim_link) { + if (shim_link && !IS_ERR(bpf_link_inc_not_zero(&shim_link->link.link))) { /* Reusing existing shim attached by the other program. */ - bpf_link_inc(&shim_link->link.link); - mutex_unlock(&tr->mutex); bpf_trampoline_put(tr); /* bpf_trampoline_get above */ return 0; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 89560e455ce7b..74d645add518d 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2347,6 +2347,9 @@ static void __update_reg32_bounds(struct bpf_reg_state *reg) static void __update_reg64_bounds(struct bpf_reg_state *reg) { + u64 tnum_next, tmax; + bool umin_in_tnum; + /* min signed is max(sign bit) | min(other bits) */ reg->smin_value = max_t(s64, reg->smin_value, reg->var_off.value | (reg->var_off.mask & S64_MIN)); @@ -2356,6 +2359,33 @@ static void __update_reg64_bounds(struct bpf_reg_state *reg) reg->umin_value = max(reg->umin_value, reg->var_off.value); reg->umax_value = min(reg->umax_value, reg->var_off.value | reg->var_off.mask); + + /* Check if u64 and tnum overlap in a single value */ + tnum_next = tnum_step(reg->var_off, reg->umin_value); + umin_in_tnum = (reg->umin_value & ~reg->var_off.mask) == reg->var_off.value; + tmax = reg->var_off.value | reg->var_off.mask; + if (umin_in_tnum && tnum_next > reg->umax_value) { + /* The u64 range and the tnum only overlap in umin. + * u64: ---[xxxxxx]----- + * tnum: --xx----------x- + */ + ___mark_reg_known(reg, reg->umin_value); + } else if (!umin_in_tnum && tnum_next == tmax) { + /* The u64 range and the tnum only overlap in the maximum value + * represented by the tnum, called tmax. + * u64: ---[xxxxxx]----- + * tnum: xx-----x-------- + */ + ___mark_reg_known(reg, tmax); + } else if (!umin_in_tnum && tnum_next <= reg->umax_value && + tnum_step(reg->var_off, tnum_next) > reg->umax_value) { + /* The u64 range and the tnum only overlap in between umin + * (excluded) and umax. + * u64: ---[xxxxxx]----- + * tnum: xx----x-------x- + */ + ___mark_reg_known(reg, tnum_next); + } } static void __update_reg_bounds(struct bpf_reg_state *reg) @@ -15224,21 +15254,17 @@ static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg, u64 umin_val, u64 umax_val) { /* Special case <<32 because it is a common compiler pattern to sign - * extend subreg by doing <<32 s>>32. In this case if 32bit bounds are - * positive we know this shift will also be positive so we can track - * bounds correctly. Otherwise we lose all sign bit information except - * what we can pick up from var_off. Perhaps we can generalize this - * later to shifts of any length. + * extend subreg by doing <<32 s>>32. smin/smax assignments are correct + * because s32 bounds don't flip sign when shifting to the left by + * 32bits. */ - if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0) + if (umin_val == 32 && umax_val == 32) { dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32; - else - dst_reg->smax_value = S64_MAX; - - if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0) dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32; - else + } else { + dst_reg->smax_value = S64_MAX; dst_reg->smin_value = S64_MIN; + } /* If we might shift our top bit out, then we know nothing */ if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { @@ -15381,6 +15407,48 @@ static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg, __update_reg_bounds(dst_reg); } +static void scalar_byte_swap(struct bpf_reg_state *dst_reg, struct bpf_insn *insn) +{ + /* + * Byte swap operation - update var_off using tnum_bswap. + * Three cases: + * 1. bswap(16|32|64): opcode=0xd7 (BPF_END | BPF_ALU64 | BPF_TO_LE) + * unconditional swap + * 2. to_le(16|32|64): opcode=0xd4 (BPF_END | BPF_ALU | BPF_TO_LE) + * swap on big-endian, truncation or no-op on little-endian + * 3. to_be(16|32|64): opcode=0xdc (BPF_END | BPF_ALU | BPF_TO_BE) + * swap on little-endian, truncation or no-op on big-endian + */ + + bool alu64 = BPF_CLASS(insn->code) == BPF_ALU64; + bool to_le = BPF_SRC(insn->code) == BPF_TO_LE; + bool is_big_endian; +#ifdef CONFIG_CPU_BIG_ENDIAN + is_big_endian = true; +#else + is_big_endian = false; +#endif + /* Apply bswap if alu64 or switch between big-endian and little-endian machines */ + bool need_bswap = alu64 || (to_le == is_big_endian); + + if (need_bswap) { + if (insn->imm == 16) + dst_reg->var_off = tnum_bswap16(dst_reg->var_off); + else if (insn->imm == 32) + dst_reg->var_off = tnum_bswap32(dst_reg->var_off); + else if (insn->imm == 64) + dst_reg->var_off = tnum_bswap64(dst_reg->var_off); + /* + * Byteswap scrambles the range, so we must reset bounds. + * Bounds will be re-derived from the new tnum later. + */ + __mark_reg_unbounded(dst_reg); + } + /* For bswap16/32, truncate dst register to match the swapped size */ + if (insn->imm == 16 || insn->imm == 32) + coerce_reg_to_size(dst_reg, insn->imm / 8); +} + static bool is_safe_to_compute_dst_reg_range(struct bpf_insn *insn, const struct bpf_reg_state *src_reg) { @@ -15407,6 +15475,7 @@ static bool is_safe_to_compute_dst_reg_range(struct bpf_insn *insn, case BPF_XOR: case BPF_OR: case BPF_MUL: + case BPF_END: return true; /* Shift operators range is only computable if shift dimension operand @@ -15422,6 +15491,35 @@ static bool is_safe_to_compute_dst_reg_range(struct bpf_insn *insn, } } +static int maybe_fork_scalars(struct bpf_verifier_env *env, struct bpf_insn *insn, + struct bpf_reg_state *dst_reg) +{ + struct bpf_verifier_state *branch; + struct bpf_reg_state *regs; + bool alu32; + + if (dst_reg->smin_value == -1 && dst_reg->smax_value == 0) + alu32 = false; + else if (dst_reg->s32_min_value == -1 && dst_reg->s32_max_value == 0) + alu32 = true; + else + return 0; + + branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); + if (IS_ERR(branch)) + return PTR_ERR(branch); + + regs = branch->frame[branch->curframe]->regs; + if (alu32) { + __mark_reg32_known(®s[insn->dst_reg], 0); + __mark_reg32_known(dst_reg, -1ull); + } else { + __mark_reg_known(®s[insn->dst_reg], 0); + __mark_reg_known(dst_reg, -1ull); + } + return 0; +} + /* WARNING: This function does calculations on 64-bit values, but the actual * execution may occur on 32-bit values. Therefore, things like bitshifts * need extra checks in the 32-bit case. @@ -15484,11 +15582,21 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, scalar_min_max_mul(dst_reg, &src_reg); break; case BPF_AND: + if (tnum_is_const(src_reg.var_off)) { + ret = maybe_fork_scalars(env, insn, dst_reg); + if (ret) + return ret; + } dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); scalar32_min_max_and(dst_reg, &src_reg); scalar_min_max_and(dst_reg, &src_reg); break; case BPF_OR: + if (tnum_is_const(src_reg.var_off)) { + ret = maybe_fork_scalars(env, insn, dst_reg); + if (ret) + return ret; + } dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); scalar32_min_max_or(dst_reg, &src_reg); scalar_min_max_or(dst_reg, &src_reg); @@ -15516,12 +15624,23 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, else scalar_min_max_arsh(dst_reg, &src_reg); break; + case BPF_END: + scalar_byte_swap(dst_reg, insn); + break; default: break; } - /* ALU32 ops are zero extended into 64bit register */ - if (alu32) + /* + * ALU32 ops are zero extended into 64bit register. + * + * BPF_END is already handled inside the helper (truncation), + * so skip zext here to avoid unexpected zero extension. + * e.g., le64: opcode=(BPF_END|BPF_ALU|BPF_TO_LE), imm=0x40 + * This is a 64bit byte swap operation with alu32==true, + * but we should not zero extend the result. + */ + if (alu32 && opcode != BPF_END) zext_32_to_64(dst_reg); reg_bounds_sync(dst_reg); return 0; @@ -15701,7 +15820,7 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) } /* check dest operand */ - if (opcode == BPF_NEG && + if ((opcode == BPF_NEG || opcode == BPF_END) && regs[insn->dst_reg].type == SCALAR_VALUE) { err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); err = err ?: adjust_scalar_min_max_vals(env, insn, @@ -16664,17 +16783,24 @@ static void __collect_linked_regs(struct linked_regs *reg_set, struct bpf_reg_st * in verifier state, save R in linked_regs if R->id == id. * If there are too many Rs sharing same id, reset id for leftover Rs. */ -static void collect_linked_regs(struct bpf_verifier_state *vstate, u32 id, +static void collect_linked_regs(struct bpf_verifier_env *env, + struct bpf_verifier_state *vstate, + u32 id, struct linked_regs *linked_regs) { + struct bpf_insn_aux_data *aux = env->insn_aux_data; struct bpf_func_state *func; struct bpf_reg_state *reg; + u16 live_regs; int i, j; id = id & ~BPF_ADD_CONST; for (i = vstate->curframe; i >= 0; i--) { + live_regs = aux[frame_insn_idx(vstate, i)].live_regs_before; func = vstate->frame[i]; for (j = 0; j < BPF_REG_FP; j++) { + if (!(live_regs & BIT(j))) + continue; reg = &func->regs[j]; __collect_linked_regs(linked_regs, reg, id, i, j, true); } @@ -16715,6 +16841,7 @@ static void sync_linked_regs(struct bpf_verifier_state *vstate, struct bpf_reg_s } else { s32 saved_subreg_def = reg->subreg_def; s32 saved_off = reg->off; + u32 saved_id = reg->id; fake_reg.type = SCALAR_VALUE; __mark_reg_known(&fake_reg, (s32)reg->off - (s32)known_reg->off); @@ -16722,10 +16849,11 @@ static void sync_linked_regs(struct bpf_verifier_state *vstate, struct bpf_reg_s /* reg = known_reg; reg += delta */ copy_register_state(reg, known_reg); /* - * Must preserve off, id and add_const flag, + * Must preserve off, id and subreg_def flag, * otherwise another sync_linked_regs() will be incorrect. */ reg->off = saved_off; + reg->id = saved_id; reg->subreg_def = saved_subreg_def; scalar32_min_max_add(reg, &fake_reg); @@ -16878,9 +17006,9 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, * if parent state is created. */ if (BPF_SRC(insn->code) == BPF_X && src_reg->type == SCALAR_VALUE && src_reg->id) - collect_linked_regs(this_branch, src_reg->id, &linked_regs); + collect_linked_regs(env, this_branch, src_reg->id, &linked_regs); if (dst_reg->type == SCALAR_VALUE && dst_reg->id) - collect_linked_regs(this_branch, dst_reg->id, &linked_regs); + collect_linked_regs(env, this_branch, dst_reg->id, &linked_regs); if (linked_regs.cnt > 1) { err = push_jmp_history(env, this_branch, 0, linked_regs_pack(&linked_regs)); if (err) @@ -20147,17 +20275,19 @@ static int do_check(struct bpf_verifier_env *env) * may skip a nospec patched-in after the jump. This can * currently never happen because nospec_result is only * used for the write-ops - * `*(size*)(dst_reg+off)=src_reg|imm32` which must - * never skip the following insn. Still, add a warning - * to document this in case nospec_result is used - * elsewhere in the future. + * `*(size*)(dst_reg+off)=src_reg|imm32` and helper + * calls. These must never skip the following insn + * (i.e., bpf_insn_successors()'s opcode_info.can_jump + * is false). Still, add a warning to document this in + * case nospec_result is used elsewhere in the future. * * All non-branch instructions have a single * fall-through edge. For these, nospec_result should * already work. */ - if (verifier_bug_if(BPF_CLASS(insn->code) == BPF_JMP || - BPF_CLASS(insn->code) == BPF_JMP32, env, + if (verifier_bug_if((BPF_CLASS(insn->code) == BPF_JMP || + BPF_CLASS(insn->code) == BPF_JMP32) && + BPF_OP(insn->code) != BPF_CALL, env, "speculation barrier after jump instruction may not have the desired effect")) return -EFAULT; process_bpf_exit: @@ -20215,29 +20345,29 @@ static int find_btf_percpu_datasec(struct btf *btf) } /* - * Add btf to the used_btfs array and return the index. (If the btf was - * already added, then just return the index.) Upon successful insertion - * increase btf refcnt, and, if present, also refcount the corresponding - * kernel module. + * Add btf to the env->used_btfs array. If needed, refcount the + * corresponding kernel module. To simplify caller's logic + * in case of error or if btf was added before the function + * decreases the btf refcount. */ static int __add_used_btf(struct bpf_verifier_env *env, struct btf *btf) { struct btf_mod_pair *btf_mod; + int ret = 0; int i; /* check whether we recorded this BTF (and maybe module) already */ for (i = 0; i < env->used_btf_cnt; i++) if (env->used_btfs[i].btf == btf) - return i; + goto ret_put; if (env->used_btf_cnt >= MAX_USED_BTFS) { verbose(env, "The total number of btfs per program has reached the limit of %u\n", MAX_USED_BTFS); - return -E2BIG; + ret = -E2BIG; + goto ret_put; } - btf_get(btf); - btf_mod = &env->used_btfs[env->used_btf_cnt]; btf_mod->btf = btf; btf_mod->module = NULL; @@ -20246,12 +20376,18 @@ static int __add_used_btf(struct bpf_verifier_env *env, struct btf *btf) if (btf_is_module(btf)) { btf_mod->module = btf_try_get_module(btf); if (!btf_mod->module) { - btf_put(btf); - return -ENXIO; + ret = -ENXIO; + goto ret_put; } } - return env->used_btf_cnt++; + env->used_btf_cnt++; + return 0; + +ret_put: + /* Either error or this BTF was already added */ + btf_put(btf); + return ret; } /* replace pseudo btf_id with kernel symbol address */ @@ -20348,9 +20484,7 @@ static int check_pseudo_btf_id(struct bpf_verifier_env *env, btf_fd = insn[1].imm; if (btf_fd) { - CLASS(fd, f)(btf_fd); - - btf = __btf_get_by_fd(f); + btf = btf_get_by_fd(btf_fd); if (IS_ERR(btf)) { verbose(env, "invalid module BTF object FD specified.\n"); return -EINVAL; @@ -20360,17 +20494,17 @@ static int check_pseudo_btf_id(struct bpf_verifier_env *env, verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"); return -EINVAL; } + btf_get(btf_vmlinux); btf = btf_vmlinux; } err = __check_pseudo_btf_id(env, insn, aux, btf); - if (err) + if (err) { + btf_put(btf); return err; + } - err = __add_used_btf(env, btf); - if (err < 0) - return err; - return 0; + return __add_used_btf(env, btf); } static bool is_tracing_prog_type(enum bpf_prog_type type) @@ -24092,10 +24226,8 @@ static int add_fd_from_fd_array(struct bpf_verifier_env *env, int fd) btf = __btf_get_by_fd(f); if (!IS_ERR(btf)) { - err = __add_used_btf(env, btf); - if (err < 0) - return err; - return 0; + btf_get(btf); + return __add_used_btf(env, btf); } verbose(env, "fd %d is not pointing to valid bpf_map or btf\n", fd); @@ -24251,6 +24383,12 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, case BPF_JMP32: switch (code) { case BPF_JA: + def = 0; + if (BPF_SRC(insn->code) == BPF_X) + use = dst; + else + use = 0; + break; case BPF_JCOND: def = 0; use = 0; diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 61b56b6ca66a7..08b0c264bd268 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -385,7 +385,7 @@ static inline bool partition_is_populated(struct cpuset *cs, cs->attach_in_progress) return true; if (!excluded_child && !cs->nr_subparts) - return cgroup_is_populated(cs->css.cgroup); + return cpuset_is_populated(cs); rcu_read_lock(); cpuset_for_each_descendant_pre(cp, pos_css, cs) { @@ -599,33 +599,31 @@ static inline bool cpusets_are_exclusive(struct cpuset *cs1, struct cpuset *cs2) /** * cpus_excl_conflict - Check if two cpusets have exclusive CPU conflicts - * @cs1: first cpuset to check - * @cs2: second cpuset to check + * @trial: the trial cpuset to be checked + * @sibling: a sibling cpuset to be checked against + * @xcpus_changed: set if exclusive_cpus has been set * * Returns: true if CPU exclusivity conflict exists, false otherwise * * Conflict detection rules: * 1. If either cpuset is CPU exclusive, they must be mutually exclusive * 2. exclusive_cpus masks cannot intersect between cpusets - * 3. The allowed CPUs of one cpuset cannot be a subset of another's exclusive CPUs + * 3. The allowed CPUs of a sibling cpuset cannot be a subset of the new exclusive CPUs */ -static inline bool cpus_excl_conflict(struct cpuset *cs1, struct cpuset *cs2) +static inline bool cpus_excl_conflict(struct cpuset *trial, struct cpuset *sibling, + bool xcpus_changed) { /* If either cpuset is exclusive, check if they are mutually exclusive */ - if (is_cpu_exclusive(cs1) || is_cpu_exclusive(cs2)) - return !cpusets_are_exclusive(cs1, cs2); + if (is_cpu_exclusive(trial) || is_cpu_exclusive(sibling)) + return !cpusets_are_exclusive(trial, sibling); /* Exclusive_cpus cannot intersect */ - if (cpumask_intersects(cs1->exclusive_cpus, cs2->exclusive_cpus)) + if (cpumask_intersects(trial->exclusive_cpus, sibling->exclusive_cpus)) return true; - /* The cpus_allowed of one cpuset cannot be a subset of another cpuset's exclusive_cpus */ - if (!cpumask_empty(cs1->cpus_allowed) && - cpumask_subset(cs1->cpus_allowed, cs2->exclusive_cpus)) - return true; - - if (!cpumask_empty(cs2->cpus_allowed) && - cpumask_subset(cs2->cpus_allowed, cs1->exclusive_cpus)) + /* The cpus_allowed of a sibling cpuset cannot be a subset of the new exclusive_cpus */ + if (xcpus_changed && !cpumask_empty(sibling->cpus_allowed) && + cpumask_subset(sibling->cpus_allowed, trial->exclusive_cpus)) return true; return false; @@ -662,6 +660,7 @@ static int validate_change(struct cpuset *cur, struct cpuset *trial) { struct cgroup_subsys_state *css; struct cpuset *c, *par; + bool xcpus_changed; int ret = 0; rcu_read_lock(); @@ -718,10 +717,11 @@ static int validate_change(struct cpuset *cur, struct cpuset *trial) * overlap. exclusive_cpus cannot overlap with each other if set. */ ret = -EINVAL; + xcpus_changed = !cpumask_equal(cur->exclusive_cpus, trial->exclusive_cpus); cpuset_for_each_child(c, css, par) { if (c == cur) continue; - if (cpus_excl_conflict(trial, c)) + if (cpus_excl_conflict(trial, c, xcpus_changed)) goto out; if (mems_excl_conflict(trial, c)) goto out; @@ -2301,7 +2301,7 @@ static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp, WARN_ON(!is_in_v2_mode() && !cpumask_equal(cp->cpus_allowed, cp->effective_cpus)); - cpuset_update_tasks_cpumask(cp, cp->effective_cpus); + cpuset_update_tasks_cpumask(cp, tmp->new_cpus); /* * On default hierarchy, inherit the CS_SCHED_LOAD_BALANCE @@ -4358,40 +4358,58 @@ bool cpuset_current_node_allowed(int node, gfp_t gfp_mask) return allowed; } -bool cpuset_node_allowed(struct cgroup *cgroup, int nid) +/** + * cpuset_nodes_allowed - return effective_mems mask from a cgroup cpuset. + * @cgroup: pointer to struct cgroup. + * @mask: pointer to struct nodemask_t to be returned. + * + * Returns effective_mems mask from a cgroup cpuset if it is cgroup v2 and + * has cpuset subsys. Otherwise, returns node_states[N_MEMORY]. + * + * This function intentionally avoids taking the cpuset_mutex or callback_lock + * when accessing effective_mems. This is because the obtained effective_mems + * is stale immediately after the query anyway (e.g., effective_mems is updated + * immediately after releasing the lock but before returning). + * + * As a result, returned @mask may be empty because cs->effective_mems can be + * rebound during this call. Besides, nodes in @mask are not guaranteed to be + * online due to hot plugins. Callers should check the mask for validity on + * return based on its subsequent use. + **/ +void cpuset_nodes_allowed(struct cgroup *cgroup, nodemask_t *mask) { struct cgroup_subsys_state *css; struct cpuset *cs; - bool allowed; /* * In v1, mem_cgroup and cpuset are unlikely in the same hierarchy * and mems_allowed is likely to be empty even if we could get to it, - * so return true to avoid taking a global lock on the empty check. + * so return directly to avoid taking a global lock on the empty check. */ - if (!cpuset_v2()) - return true; + if (!cgroup || !cpuset_v2()) { + nodes_copy(*mask, node_states[N_MEMORY]); + return; + } css = cgroup_get_e_css(cgroup, &cpuset_cgrp_subsys); - if (!css) - return true; + if (!css) { + nodes_copy(*mask, node_states[N_MEMORY]); + return; + } /* - * Normally, accessing effective_mems would require the cpuset_mutex - * or callback_lock - but node_isset is atomic and the reference - * taken via cgroup_get_e_css is sufficient to protect css. - * - * Since this interface is intended for use by migration paths, we - * relax locking here to avoid taking global locks - while accepting - * there may be rare scenarios where the result may be innaccurate. + * The reference taken via cgroup_get_e_css is sufficient to + * protect css, but it does not imply safe accesses to effective_mems. * - * Reclaim and migration are subject to these same race conditions, and - * cannot make strong isolation guarantees, so this is acceptable. + * Normally, accessing effective_mems would require the cpuset_mutex + * or callback_lock - but the correctness of this information is stale + * immediately after the query anyway. We do not acquire the lock + * during this process to save lock contention in exchange for racing + * against mems_allowed rebinds. */ cs = container_of(css, struct cpuset, css); - allowed = node_isset(nid, cs->effective_mems); + nodes_copy(*mask, cs->effective_mems); css_put(css); - return allowed; } /** diff --git a/kernel/configs/debug.config b/kernel/configs/debug.config index 8da4b25b1f4db..4d71215a94dd7 100644 --- a/kernel/configs/debug.config +++ b/kernel/configs/debug.config @@ -29,7 +29,6 @@ CONFIG_SECTION_MISMATCH_WARN_ONLY=y # CONFIG_UBSAN_ALIGNMENT is not set # CONFIG_UBSAN_DIV_ZERO is not set # CONFIG_UBSAN_TRAP is not set -# CONFIG_WARN_ALL_UNSEEDED_RANDOM is not set CONFIG_DEBUG_FS=y CONFIG_DEBUG_FS_ALLOW_ALL=y CONFIG_DEBUG_IRQFLAGS=y diff --git a/kernel/crash_dump_dm_crypt.c b/kernel/crash_dump_dm_crypt.c index 401423ba477da..abb307a23de33 100644 --- a/kernel/crash_dump_dm_crypt.c +++ b/kernel/crash_dump_dm_crypt.c @@ -143,6 +143,7 @@ static int read_key_from_user_keying(struct dm_crypt_key *dm_key) { const struct user_key_payload *ukp; struct key *key; + int ret = 0; kexec_dprintk("Requesting logon key %s", dm_key->key_desc); key = request_key(&key_type_logon, dm_key->key_desc, NULL); @@ -152,20 +153,28 @@ static int read_key_from_user_keying(struct dm_crypt_key *dm_key) return PTR_ERR(key); } + down_read(&key->sem); ukp = user_key_payload_locked(key); - if (!ukp) - return -EKEYREVOKED; + if (!ukp) { + ret = -EKEYREVOKED; + goto out; + } if (ukp->datalen > KEY_SIZE_MAX) { pr_err("Key size %u exceeds maximum (%u)\n", ukp->datalen, KEY_SIZE_MAX); - return -EINVAL; + ret = -EINVAL; + goto out; } memcpy(dm_key->data, ukp->data, ukp->datalen); dm_key->key_size = ukp->datalen; kexec_dprintk("Get dm crypt key (size=%u) %s: %8ph\n", dm_key->key_size, dm_key->key_desc, dm_key->data); - return 0; + +out: + up_read(&key->sem); + key_put(key); + return ret; } struct config_key { diff --git a/kernel/dma/direct.h b/kernel/dma/direct.h index da2fadf45bcd6..62f0d9d0ba02e 100644 --- a/kernel/dma/direct.h +++ b/kernel/dma/direct.h @@ -88,7 +88,7 @@ static inline dma_addr_t dma_direct_map_phys(struct device *dev, if (is_swiotlb_force_bounce(dev)) { if (attrs & DMA_ATTR_MMIO) - goto err_overflow; + return DMA_MAPPING_ERROR; return swiotlb_map(dev, phys, size, dir, attrs); } diff --git a/kernel/events/core.c b/kernel/events/core.c index 1d8ca8e34f5c4..b7e73ac3e512f 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -4016,7 +4016,8 @@ static int merge_sched_in(struct perf_event *event, void *data) if (*perf_event_fasync(event)) event->pending_kill = POLL_ERR; - perf_event_wakeup(event); + event->pending_wakeup = 1; + irq_work_queue(&event->pending_irq); } else { struct perf_cpu_pmu_context *cpc = this_cpc(event->pmu_ctx->pmu); @@ -5279,9 +5280,20 @@ attach_task_ctx_data(struct task_struct *task, struct kmem_cache *ctx_cache, return -ENOMEM; for (;;) { - if (try_cmpxchg((struct perf_ctx_data **)&task->perf_ctx_data, &old, cd)) { + if (try_cmpxchg(&task->perf_ctx_data, &old, cd)) { if (old) perf_free_ctx_data_rcu(old); + /* + * Above try_cmpxchg() pairs with try_cmpxchg() from + * detach_task_ctx_data() such that + * if we race with perf_event_exit_task(), we must + * observe PF_EXITING. + */ + if (task->flags & PF_EXITING) { + /* detach_task_ctx_data() may free it already */ + if (try_cmpxchg(&task->perf_ctx_data, &cd, NULL)) + perf_free_ctx_data_rcu(cd); + } return 0; } @@ -5327,6 +5339,8 @@ attach_global_ctx_data(struct kmem_cache *ctx_cache) /* Allocate everything */ scoped_guard (rcu) { for_each_process_thread(g, p) { + if (p->flags & PF_EXITING) + continue; cd = rcu_dereference(p->perf_ctx_data); if (cd && !cd->global) { cd->global = 1; @@ -7174,28 +7188,28 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) ret = perf_mmap_aux(vma, event, nr_pages); if (ret) return ret; - } - /* - * Since pinned accounting is per vm we cannot allow fork() to copy our - * vma. - */ - vm_flags_set(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP); - vma->vm_ops = &perf_mmap_vmops; + /* + * Since pinned accounting is per vm we cannot allow fork() to copy our + * vma. + */ + vm_flags_set(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP); + vma->vm_ops = &perf_mmap_vmops; - mapped = get_mapped(event, event_mapped); - if (mapped) - mapped(event, vma->vm_mm); + mapped = get_mapped(event, event_mapped); + if (mapped) + mapped(event, vma->vm_mm); - /* - * Try to map it into the page table. On fail, invoke - * perf_mmap_close() to undo the above, as the callsite expects - * full cleanup in this case and therefore does not invoke - * vmops::close(). - */ - ret = map_range(event->rb, vma); - if (ret) - perf_mmap_close(vma); + /* + * Try to map it into the page table. On fail, invoke + * perf_mmap_close() to undo the above, as the callsite expects + * full cleanup in this case and therefore does not invoke + * vmops::close(). + */ + ret = map_range(event->rb, vma); + if (ret) + perf_mmap_close(vma); + } return ret; } @@ -10413,6 +10427,13 @@ int perf_event_overflow(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs) { + /* + * Entry point from hardware PMI, interrupts should be disabled here. + * This serializes us against perf_event_remove_from_context() in + * things like perf_event_release_kernel(). + */ + lockdep_assert_irqs_disabled(); + return __perf_event_overflow(event, 1, data, regs); } @@ -10489,6 +10510,19 @@ static void perf_swevent_event(struct perf_event *event, u64 nr, { struct hw_perf_event *hwc = &event->hw; + /* + * This is: + * - software preempt + * - tracepoint preempt + * - tp_target_task irq (ctx->lock) + * - uprobes preempt/irq + * - kprobes preempt/irq + * - hw_breakpoint irq + * + * Any of these are sufficient to hold off RCU and thus ensure @event + * exists. + */ + lockdep_assert_preemption_disabled(); local64_add(nr, &event->count); if (!regs) @@ -10497,6 +10531,16 @@ static void perf_swevent_event(struct perf_event *event, u64 nr, if (!is_sampling_event(event)) return; + /* + * Serialize against event_function_call() IPIs like normal overflow + * event handling. Specifically, must not allow + * perf_event_release_kernel() -> perf_remove_from_context() to make + * progress and 'release' the event from under us. + */ + guard(irqsave)(); + if (event->state != PERF_EVENT_STATE_ACTIVE) + return; + if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) { data->period = nr; return perf_swevent_overflow(event, 1, data, regs); @@ -10995,6 +11039,11 @@ void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size, struct perf_sample_data data; struct perf_event *event; + /* + * Per being a tracepoint, this runs with preemption disabled. + */ + lockdep_assert_preemption_disabled(); + struct perf_raw_record raw = { .frag = { .size = entry_size, @@ -11327,6 +11376,11 @@ void perf_bp_event(struct perf_event *bp, void *data) struct perf_sample_data sample; struct pt_regs *regs = data; + /* + * Exception context, will have interrupts disabled. + */ + lockdep_assert_irqs_disabled(); + perf_sample_data_init(&sample, bp->attr.bp_addr, 0); if (!bp->hw.state && !perf_exclude_event(bp, regs)) @@ -11791,7 +11845,7 @@ static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer) if (regs && !perf_exclude_event(event, regs)) { if (!(event->attr.exclude_idle && is_idle_task(current))) - if (__perf_event_overflow(event, 1, &data, regs)) + if (perf_event_overflow(event, &data, regs)) ret = HRTIMER_NORESTART; } @@ -14223,8 +14277,11 @@ void perf_event_exit_task(struct task_struct *task) /* * Detach the perf_ctx_data for the system-wide event. + * + * Done without holding global_ctx_data_rwsem; typically + * attach_global_ctx_data() will skip over this task, but otherwise + * attach_task_ctx_data() will observe PF_EXITING. */ - guard(percpu_read)(&global_ctx_data_rwsem); detach_task_ctx_data(task); } diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index f11ceb8be8c41..725c5772429dc 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -1138,7 +1138,7 @@ static bool filter_chain(struct uprobe *uprobe, struct mm_struct *mm) bool ret = false; down_read(&uprobe->consumer_rwsem); - list_for_each_entry_rcu(uc, &uprobe->consumers, cons_node, rcu_read_lock_trace_held()) { + list_for_each_entry(uc, &uprobe->consumers, cons_node) { ret = consumer_filter(uc, mm); if (ret) break; @@ -1694,6 +1694,12 @@ static const struct vm_special_mapping xol_mapping = { .mremap = xol_mremap, }; +unsigned long __weak arch_uprobe_get_xol_area(void) +{ + /* Try to map as high as possible, this is only a hint. */ + return get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, PAGE_SIZE, 0, 0); +} + /* Slot allocation for XOL */ static int xol_add_vma(struct mm_struct *mm, struct xol_area *area) { @@ -1709,9 +1715,7 @@ static int xol_add_vma(struct mm_struct *mm, struct xol_area *area) } if (!area->vaddr) { - /* Try to map as high as possible, this is only a hint. */ - area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, - PAGE_SIZE, 0, 0); + area->vaddr = arch_uprobe_get_xol_area(); if (IS_ERR_VALUE(area->vaddr)) { ret = area->vaddr; goto fail; diff --git a/kernel/irq/cpuhotplug.c b/kernel/irq/cpuhotplug.c index 755346ea98196..cd5689e383b00 100644 --- a/kernel/irq/cpuhotplug.c +++ b/kernel/irq/cpuhotplug.c @@ -177,9 +177,11 @@ void irq_migrate_all_off_this_cpu(void) bool affinity_broken; desc = irq_to_desc(irq); - scoped_guard(raw_spinlock, &desc->lock) + scoped_guard(raw_spinlock, &desc->lock) { affinity_broken = migrate_one_irq(desc); - + if (affinity_broken && desc->affinity_notify) + irq_affinity_schedule_notify_work(desc); + } if (affinity_broken) { pr_debug_ratelimited("IRQ %u: no longer affine to CPU%u\n", irq, smp_processor_id()); diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index 0164ca48da59e..5568ed3a8b852 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h @@ -135,6 +135,7 @@ extern bool irq_can_set_affinity_usr(unsigned int irq); extern int irq_do_set_affinity(struct irq_data *data, const struct cpumask *dest, bool force); +extern void irq_affinity_schedule_notify_work(struct irq_desc *desc); #ifdef CONFIG_SMP extern int irq_setup_affinity(struct irq_desc *desc); @@ -142,7 +143,6 @@ extern int irq_setup_affinity(struct irq_desc *desc); static inline int irq_setup_affinity(struct irq_desc *desc) { return 0; } #endif - #define for_each_action_of_desc(desc, act) \ for (act = desc->action; act; act = act->next) diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 400856abf6721..c09751b7a0c49 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -347,6 +347,21 @@ static bool irq_set_affinity_deactivated(struct irq_data *data, return true; } +/** + * irq_affinity_schedule_notify_work - Schedule work to notify about affinity change + * @desc: Interrupt descriptor whose affinity changed + */ +void irq_affinity_schedule_notify_work(struct irq_desc *desc) +{ + lockdep_assert_held(&desc->lock); + + kref_get(&desc->affinity_notify->kref); + if (!schedule_work(&desc->affinity_notify->work)) { + /* Work was already scheduled, drop our extra ref */ + kref_put(&desc->affinity_notify->kref, desc->affinity_notify->release); + } +} + int irq_set_affinity_locked(struct irq_data *data, const struct cpumask *mask, bool force) { @@ -367,14 +382,9 @@ int irq_set_affinity_locked(struct irq_data *data, const struct cpumask *mask, irq_copy_pending(desc, mask); } - if (desc->affinity_notify) { - kref_get(&desc->affinity_notify->kref); - if (!schedule_work(&desc->affinity_notify->work)) { - /* Work was already scheduled, drop our extra ref */ - kref_put(&desc->affinity_notify->kref, - desc->affinity_notify->release); - } - } + if (desc->affinity_notify) + irq_affinity_schedule_notify_work(desc); + irqd_set(data, IRQD_AFFINITY_SET); return ret; diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index 049e296f586cc..cdd6e025935d3 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c @@ -345,7 +345,7 @@ int kallsyms_lookup_size_offset(unsigned long addr, unsigned long *symbolsize, return 1; } return !!module_address_lookup(addr, symbolsize, offset, NULL, NULL, namebuf) || - !!__bpf_address_lookup(addr, symbolsize, offset, namebuf); + !!bpf_address_lookup(addr, symbolsize, offset, namebuf); } static int kallsyms_lookup_buildid(unsigned long addr, @@ -377,12 +377,11 @@ static int kallsyms_lookup_buildid(unsigned long addr, ret = module_address_lookup(addr, symbolsize, offset, modname, modbuildid, namebuf); if (!ret) - ret = bpf_address_lookup(addr, symbolsize, - offset, modname, namebuf); + ret = bpf_address_lookup(addr, symbolsize, offset, namebuf); if (!ret) - ret = ftrace_mod_address_lookup(addr, symbolsize, - offset, modname, namebuf); + ret = ftrace_mod_address_lookup(addr, symbolsize, offset, + modname, modbuildid, namebuf); return ret; } diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index eb62a97942428..2bfbb2d144e69 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -882,6 +882,60 @@ static int kexec_calculate_store_digests(struct kimage *image) } #ifdef CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY +/* + * kexec_purgatory_find_symbol - find a symbol in the purgatory + * @pi: Purgatory to search in. + * @name: Name of the symbol. + * + * Return: pointer to symbol in read-only symtab on success, NULL on error. + */ +static const Elf_Sym *kexec_purgatory_find_symbol(struct purgatory_info *pi, + const char *name) +{ + const Elf_Shdr *sechdrs; + const Elf_Ehdr *ehdr; + const Elf_Sym *syms; + const char *strtab; + int i, k; + + if (!pi->ehdr) + return NULL; + + ehdr = pi->ehdr; + sechdrs = (void *)ehdr + ehdr->e_shoff; + + for (i = 0; i < ehdr->e_shnum; i++) { + if (sechdrs[i].sh_type != SHT_SYMTAB) + continue; + + if (sechdrs[i].sh_link >= ehdr->e_shnum) + /* Invalid strtab section number */ + continue; + strtab = (void *)ehdr + sechdrs[sechdrs[i].sh_link].sh_offset; + syms = (void *)ehdr + sechdrs[i].sh_offset; + + /* Go through symbols for a match */ + for (k = 0; k < sechdrs[i].sh_size/sizeof(Elf_Sym); k++) { + if (ELF_ST_BIND(syms[k].st_info) != STB_GLOBAL) + continue; + + if (strcmp(strtab + syms[k].st_name, name) != 0) + continue; + + if (syms[k].st_shndx == SHN_UNDEF || + syms[k].st_shndx >= ehdr->e_shnum) { + pr_debug("Symbol: %s has bad section index %d.\n", + name, syms[k].st_shndx); + return NULL; + } + + /* Found the symbol we are looking for */ + return &syms[k]; + } + } + + return NULL; +} /* * kexec_purgatory_setup_kbuf - prepare buffer to load purgatory. * @pi: Purgatory to be loaded. @@ -960,6 +1014,10 @@ static int kexec_purgatory_setup_sechdrs(struct purgatory_info *pi, unsigned long offset; size_t sechdrs_size; Elf_Shdr *sechdrs; + const Elf_Sym *entry_sym; + u16 entry_shndx = 0; + unsigned long entry_off = 0; + bool start_fixed = false; int i; /* @@ -977,6 +1035,12 @@ static int kexec_purgatory_setup_sechdrs(struct purgatory_info *pi, bss_addr = kbuf->mem + kbuf->bufsz; kbuf->image->start = pi->ehdr->e_entry; + entry_sym = kexec_purgatory_find_symbol(pi, "purgatory_start"); + if (entry_sym) { + entry_shndx = entry_sym->st_shndx; + entry_off = entry_sym->st_value; + } + for (i = 0; i < pi->ehdr->e_shnum; i++) { unsigned long align; void *src, *dst; @@ -994,6 +1058,13 @@ static int kexec_purgatory_setup_sechdrs(struct purgatory_info *pi, offset = ALIGN(offset, align); + if (!start_fixed && entry_sym && i == entry_shndx && + (sechdrs[i].sh_flags & SHF_EXECINSTR) && + entry_off < sechdrs[i].sh_size) { + kbuf->image->start = kbuf->mem + offset + entry_off; + start_fixed = true; + } + /* * Check if the segment contains the entry point, if so, * calculate the value of image->start based on it. @@ -1004,13 +1075,14 @@ static int kexec_purgatory_setup_sechdrs(struct purgatory_info *pi, * is not set to the initial value, and warn the user so they * have a chance to fix their purgatory's linker script. */ - if (sechdrs[i].sh_flags & SHF_EXECINSTR && + if (!start_fixed && sechdrs[i].sh_flags & SHF_EXECINSTR && pi->ehdr->e_entry >= sechdrs[i].sh_addr && pi->ehdr->e_entry < (sechdrs[i].sh_addr + sechdrs[i].sh_size) && - !WARN_ON(kbuf->image->start != pi->ehdr->e_entry)) { + kbuf->image->start == pi->ehdr->e_entry) { kbuf->image->start -= sechdrs[i].sh_addr; kbuf->image->start += kbuf->mem + offset; + start_fixed = true; } src = (void *)pi->ehdr + sechdrs[i].sh_offset; @@ -1128,61 +1200,6 @@ int kexec_load_purgatory(struct kimage *image, struct kexec_buf *kbuf) return ret; } -/* - * kexec_purgatory_find_symbol - find a symbol in the purgatory - * @pi: Purgatory to search in. - * @name: Name of the symbol. - * - * Return: pointer to symbol in read-only symtab on success, NULL on error. - */ -static const Elf_Sym *kexec_purgatory_find_symbol(struct purgatory_info *pi, - const char *name) -{ - const Elf_Shdr *sechdrs; - const Elf_Ehdr *ehdr; - const Elf_Sym *syms; - const char *strtab; - int i, k; - - if (!pi->ehdr) - return NULL; - - ehdr = pi->ehdr; - sechdrs = (void *)ehdr + ehdr->e_shoff; - - for (i = 0; i < ehdr->e_shnum; i++) { - if (sechdrs[i].sh_type != SHT_SYMTAB) - continue; - - if (sechdrs[i].sh_link >= ehdr->e_shnum) - /* Invalid strtab section number */ - continue; - strtab = (void *)ehdr + sechdrs[sechdrs[i].sh_link].sh_offset; - syms = (void *)ehdr + sechdrs[i].sh_offset; - - /* Go through symbols for a match */ - for (k = 0; k < sechdrs[i].sh_size/sizeof(Elf_Sym); k++) { - if (ELF_ST_BIND(syms[k].st_info) != STB_GLOBAL) - continue; - - if (strcmp(strtab + syms[k].st_name, name) != 0) - continue; - - if (syms[k].st_shndx == SHN_UNDEF || - syms[k].st_shndx >= ehdr->e_shnum) { - pr_debug("Symbol: %s has bad section index %d.\n", - name, syms[k].st_shndx); - return NULL; - } - - /* Found the symbol we are looking for */ - return &syms[k]; - } - } - - return NULL; -} - void *kexec_purgatory_get_symbol_addr(struct kimage *image, const char *name) { struct purgatory_info *pi = &image->purgatory_info; diff --git a/kernel/kexec_handover.c b/kernel/kexec_handover.c index 03d12e27189fc..2da4bd994322f 100644 --- a/kernel/kexec_handover.c +++ b/kernel/kexec_handover.c @@ -619,7 +619,7 @@ static void __init kho_reserve_scratch(void) scratch_size_update(); /* FIXME: deal with node hot-plug/remove */ - kho_scratch_cnt = num_online_nodes() + 2; + kho_scratch_cnt = nodes_weight(node_states[N_MEMORY]) + 2; size = kho_scratch_cnt * sizeof(*kho_scratch); kho_scratch = memblock_alloc(size, PAGE_SIZE); if (!kho_scratch) @@ -649,7 +649,11 @@ static void __init kho_reserve_scratch(void) kho_scratch[i].size = size; i++; - for_each_online_node(nid) { + /* + * Loop over nodes that have both memory and are online. Skip + * memoryless nodes, as we can not allocate scratch areas there. + */ + for_each_node_state(nid, N_MEMORY) { size = scratch_size_node(nid); addr = memblock_alloc_range_nid(size, CMA_MIN_ALIGNMENT_BYTES, 0, MEMBLOCK_ALLOC_ACCESSIBLE, diff --git a/kernel/module/kallsyms.c b/kernel/module/kallsyms.c index 00a60796327c0..0fc11e45df9b9 100644 --- a/kernel/module/kallsyms.c +++ b/kernel/module/kallsyms.c @@ -334,13 +334,8 @@ int module_address_lookup(unsigned long addr, if (mod) { if (modname) *modname = mod->name; - if (modbuildid) { -#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID) - *modbuildid = mod->build_id; -#else - *modbuildid = NULL; -#endif - } + if (modbuildid) + *modbuildid = module_buildid(mod); sym = find_kallsyms_symbol(mod, addr, size, offset); diff --git a/kernel/module/main.c b/kernel/module/main.c index c66b261849362..a2c798d06e3f5 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -3544,12 +3544,6 @@ static int load_module(struct load_info *info, const char __user *uargs, mutex_unlock(&module_mutex); free_module: mod_stat_bump_invalid(info, flags); - /* Free lock-classes; relies on the preceding sync_rcu() */ - for_class_mod_mem_type(type, core_data) { - lockdep_free_key_range(mod->mem[type].base, - mod->mem[type].size); - } - module_memory_restore_rox(mod); module_deallocate(mod, info); free_copy: diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index b8bbe7960cda7..2265b9c2906e1 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -203,7 +203,7 @@ struct rcu_data { /* during and after the last grace */ /* period it is aware of. */ struct irq_work defer_qs_iw; /* Obtain later scheduler attention. */ - int defer_qs_iw_pending; /* Scheduler attention pending? */ + int defer_qs_pending; /* irqwork or softirq pending? */ struct work_struct strict_work; /* Schedule readers for strict GPs. */ /* 2) batch handling */ diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index d85763336b3c0..cafb1cc8eff84 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -487,8 +487,8 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags) union rcu_special special; rdp = this_cpu_ptr(&rcu_data); - if (rdp->defer_qs_iw_pending == DEFER_QS_PENDING) - rdp->defer_qs_iw_pending = DEFER_QS_IDLE; + if (rdp->defer_qs_pending == DEFER_QS_PENDING) + rdp->defer_qs_pending = DEFER_QS_IDLE; /* * If RCU core is waiting for this CPU to exit its critical section, @@ -645,7 +645,7 @@ static void rcu_preempt_deferred_qs_handler(struct irq_work *iwp) * 5. Deferred QS reporting does not happen. */ if (rcu_preempt_depth() > 0) - WRITE_ONCE(rdp->defer_qs_iw_pending, DEFER_QS_IDLE); + WRITE_ONCE(rdp->defer_qs_pending, DEFER_QS_IDLE); } /* @@ -747,7 +747,10 @@ static void rcu_read_unlock_special(struct task_struct *t) // Using softirq, safe to awaken, and either the // wakeup is free or there is either an expedited // GP in flight or a potential need to deboost. - raise_softirq_irqoff(RCU_SOFTIRQ); + if (rdp->defer_qs_pending != DEFER_QS_PENDING) { + rdp->defer_qs_pending = DEFER_QS_PENDING; + raise_softirq_irqoff(RCU_SOFTIRQ); + } } else { // Enabling BH or preempt does reschedule, so... // Also if no expediting and no possible deboosting, @@ -756,11 +759,11 @@ static void rcu_read_unlock_special(struct task_struct *t) set_tsk_need_resched(current); set_preempt_need_resched(); if (IS_ENABLED(CONFIG_IRQ_WORK) && irqs_were_disabled && - needs_exp && rdp->defer_qs_iw_pending != DEFER_QS_PENDING && + needs_exp && rdp->defer_qs_pending != DEFER_QS_PENDING && cpu_online(rdp->cpu)) { // Get scheduler to re-evaluate and call hooks. // If !IRQ_WORK, FQS scan will eventually IPI. - rdp->defer_qs_iw_pending = DEFER_QS_PENDING; + rdp->defer_qs_pending = DEFER_QS_PENDING; irq_work_queue_on(&rdp->defer_qs_iw, rdp->cpu); } } diff --git a/kernel/rseq.c b/kernel/rseq.c index 2452b7366b00e..07b0b46ec640f 100644 --- a/kernel/rseq.c +++ b/kernel/rseq.c @@ -519,8 +519,9 @@ SYSCALL_DEFINE4(rseq, struct rseq __user *, rseq, u32, rseq_len, * auxiliary vector AT_RSEQ_ALIGN. If rseq_len is the original rseq * size, the required alignment is the original struct rseq alignment. * - * In order to be valid, rseq_len is either the original rseq size, or - * large enough to contain all supported fields, as communicated to + * The rseq_len is required to be greater or equal to the original rseq + * size. In order to be valid, rseq_len is either the original rseq size, + * or large enough to contain all supported fields, as communicated to * user-space through the ELF auxiliary vector AT_RSEQ_FEATURE_SIZE. */ if (rseq_len < ORIG_RSEQ_SIZE || diff --git a/kernel/sched/core.c b/kernel/sched/core.c index e460c22de8ad4..582c3847f483a 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -119,6 +119,9 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_cfs_tp); EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_se_tp); EXPORT_TRACEPOINT_SYMBOL_GPL(sched_update_nr_running_tp); EXPORT_TRACEPOINT_SYMBOL_GPL(sched_compute_energy_tp); +EXPORT_TRACEPOINT_SYMBOL_GPL(sched_entry_tp); +EXPORT_TRACEPOINT_SYMBOL_GPL(sched_exit_tp); +EXPORT_TRACEPOINT_SYMBOL_GPL(sched_set_need_resched_tp); DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); DEFINE_PER_CPU(struct rnd_state, sched_rnd_state); @@ -1136,6 +1139,7 @@ void __trace_set_need_resched(struct task_struct *curr, int tif) { trace_sched_set_need_resched_tp(curr, smp_processor_id(), tif); } +EXPORT_SYMBOL_GPL(__trace_set_need_resched); void resched_curr(struct rq *rq) { diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index c7a8717e837dd..72499cf2a1db5 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -3591,6 +3591,9 @@ static void __dl_clear_params(struct sched_dl_entity *dl_se) dl_se->dl_non_contending = 0; dl_se->dl_overrun = 0; dl_se->dl_server = 0; + dl_se->dl_defer = 0; + dl_se->dl_defer_running = 0; + dl_se->dl_defer_armed = 0; #ifdef CONFIG_RT_MUTEXES dl_se->pi_se = dl_se; diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 41caa22e0680a..93f009e1076d8 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -345,8 +345,8 @@ static ssize_t sched_fair_server_write(struct file *filp, const char __user *ubu long cpu = (long) ((struct seq_file *) filp->private_data)->private; struct rq *rq = cpu_rq(cpu); u64 runtime, period; + int retval = 0; size_t err; - int retval; u64 value; err = kstrtoull_from_user(ubuf, cnt, 10, &value); @@ -380,8 +380,6 @@ static ssize_t sched_fair_server_write(struct file *filp, const char __user *ubu dl_server_stop(&rq->fair_server); retval = dl_server_apply_params(&rq->fair_server, runtime, period, 0); - if (retval) - cnt = retval; if (!runtime) printk_deferred("Fair server disabled in CPU %d, system may crash due to starvation.\n", @@ -389,6 +387,9 @@ static ssize_t sched_fair_server_write(struct file *filp, const char __user *ubu if (rq->cfs.h_nr_queued) dl_server_start(&rq->fair_server); + + if (retval < 0) + return retval; } *ppos += cnt; diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h index 601cfae8cc765..8039a750490f8 100644 --- a/kernel/sched/ext_internal.h +++ b/kernel/sched/ext_internal.h @@ -69,7 +69,7 @@ enum scx_exit_flags { * info communication. The following flag indicates whether ops.init() * finished successfully. */ - SCX_EFLAG_INITIALIZED, + SCX_EFLAG_INITIALIZED = 1LLU << 0, }; /* diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 82038166d7b0c..292141f4aaa54 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -524,10 +524,48 @@ void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec); * Scheduling class tree data structure manipulation methods: */ +extern void __BUILD_BUG_vruntime_cmp(void); + +/* Use __builtin_strcmp() because of __HAVE_ARCH_STRCMP: */ + +#define vruntime_cmp(A, CMP_STR, B) ({ \ + int __res = 0; \ + \ + if (!__builtin_strcmp(CMP_STR, "<")) { \ + __res = ((s64)((A)-(B)) < 0); \ + } else if (!__builtin_strcmp(CMP_STR, "<=")) { \ + __res = ((s64)((A)-(B)) <= 0); \ + } else if (!__builtin_strcmp(CMP_STR, ">")) { \ + __res = ((s64)((A)-(B)) > 0); \ + } else if (!__builtin_strcmp(CMP_STR, ">=")) { \ + __res = ((s64)((A)-(B)) >= 0); \ + } else { \ + /* Unknown operator throws linker error: */ \ + __BUILD_BUG_vruntime_cmp(); \ + } \ + \ + __res; \ +}) + +extern void __BUILD_BUG_vruntime_op(void); + +#define vruntime_op(A, OP_STR, B) ({ \ + s64 __res = 0; \ + \ + if (!__builtin_strcmp(OP_STR, "-")) { \ + __res = (s64)((A)-(B)); \ + } else { \ + /* Unknown operator throws linker error: */ \ + __BUILD_BUG_vruntime_op(); \ + } \ + \ + __res; \ +}) + + static inline __maybe_unused u64 max_vruntime(u64 max_vruntime, u64 vruntime) { - s64 delta = (s64)(vruntime - max_vruntime); - if (delta > 0) + if (vruntime_cmp(vruntime, ">", max_vruntime)) max_vruntime = vruntime; return max_vruntime; @@ -535,8 +573,7 @@ static inline __maybe_unused u64 max_vruntime(u64 max_vruntime, u64 vruntime) static inline __maybe_unused u64 min_vruntime(u64 min_vruntime, u64 vruntime) { - s64 delta = (s64)(vruntime - min_vruntime); - if (delta < 0) + if (vruntime_cmp(vruntime, "<", min_vruntime)) min_vruntime = vruntime; return min_vruntime; @@ -549,12 +586,27 @@ static inline bool entity_before(const struct sched_entity *a, * Tiebreak on vruntime seems unnecessary since it can * hardly happen. */ - return (s64)(a->deadline - b->deadline) < 0; + return vruntime_cmp(a->deadline, "<", b->deadline); } +/* + * Per avg_vruntime() below, cfs_rq::zero_vruntime is only slightly stale + * and this value should be no more than two lag bounds. Which puts it in the + * general order of: + * + * (slice + TICK_NSEC) << NICE_0_LOAD_SHIFT + * + * which is around 44 bits in size (on 64bit); that is 20 for + * NICE_0_LOAD_SHIFT, another 20 for NSEC_PER_MSEC and then a handful for + * however many msec the actual slice+tick ends up begin. + * + * (disregarding the actual divide-by-weight part makes for the worst case + * weight of 2, which nicely cancels vs the fuzz in zero_vruntime not actually + * being the zero-lag point). + */ static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se) { - return (s64)(se->vruntime - cfs_rq->zero_vruntime); + return vruntime_op(se->vruntime, "-", cfs_rq->zero_vruntime); } #define __node_2_se(node) \ @@ -607,8 +659,8 @@ static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se) * Which we track using: * * v0 := cfs_rq->zero_vruntime - * \Sum (v_i - v0) * w_i := cfs_rq->avg_vruntime - * \Sum w_i := cfs_rq->avg_load + * \Sum (v_i - v0) * w_i := cfs_rq->sum_w_vruntime + * \Sum w_i := cfs_rq->sum_weight * * Since zero_vruntime closely tracks the per-task service, these * deltas: (v_i - v), will be in the order of the maximal (virtual) lag @@ -619,61 +671,85 @@ static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se) * As measured, the max (key * weight) value was ~44 bits for a kernel build. */ static void -avg_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) +sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) { unsigned long weight = scale_load_down(se->load.weight); s64 key = entity_key(cfs_rq, se); - cfs_rq->avg_vruntime += key * weight; - cfs_rq->avg_load += weight; + cfs_rq->sum_w_vruntime += key * weight; + cfs_rq->sum_weight += weight; } static void -avg_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se) +sum_w_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se) { unsigned long weight = scale_load_down(se->load.weight); s64 key = entity_key(cfs_rq, se); - cfs_rq->avg_vruntime -= key * weight; - cfs_rq->avg_load -= weight; + cfs_rq->sum_w_vruntime -= key * weight; + cfs_rq->sum_weight -= weight; } static inline -void avg_vruntime_update(struct cfs_rq *cfs_rq, s64 delta) +void update_zero_vruntime(struct cfs_rq *cfs_rq, s64 delta) { /* - * v' = v + d ==> avg_vruntime' = avg_runtime - d*avg_load + * v' = v + d ==> sum_w_vruntime' = sum_w_vruntime - d*sum_weight */ - cfs_rq->avg_vruntime -= cfs_rq->avg_load * delta; + cfs_rq->sum_w_vruntime -= cfs_rq->sum_weight * delta; + cfs_rq->zero_vruntime += delta; } /* - * Specifically: avg_runtime() + 0 must result in entity_eligible() := true + * Specifically: avg_vruntime() + 0 must result in entity_eligible() := true * For this to be so, the result of this function must have a left bias. + * + * Called in: + * - place_entity() -- before enqueue + * - update_entity_lag() -- before dequeue + * - entity_tick() + * + * This means it is one entry 'behind' but that puts it close enough to where + * the bound on entity_key() is at most two lag bounds. */ u64 avg_vruntime(struct cfs_rq *cfs_rq) { struct sched_entity *curr = cfs_rq->curr; - s64 avg = cfs_rq->avg_vruntime; - long load = cfs_rq->avg_load; + long weight = cfs_rq->sum_weight; + s64 delta = 0; - if (curr && curr->on_rq) { - unsigned long weight = scale_load_down(curr->load.weight); + if (curr && !curr->on_rq) + curr = NULL; - avg += entity_key(cfs_rq, curr) * weight; - load += weight; - } + if (weight) { + s64 runtime = cfs_rq->sum_w_vruntime; + + if (curr) { + unsigned long w = scale_load_down(curr->load.weight); + + runtime += entity_key(cfs_rq, curr) * w; + weight += w; + } - if (load) { /* sign flips effective floor / ceiling */ - if (avg < 0) - avg -= (load - 1); - avg = div_s64(avg, load); + if (runtime < 0) + runtime -= (weight - 1); + + delta = div_s64(runtime, weight); + } else if (curr) { + /* + * When there is but one element, it is the average. + */ + delta = curr->vruntime - cfs_rq->zero_vruntime; } - return cfs_rq->zero_vruntime + avg; + update_zero_vruntime(cfs_rq, delta); + + return cfs_rq->zero_vruntime; } +static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq); + /* * lag_i = S - s_i = w_i * (V - v_i) * @@ -687,17 +763,16 @@ u64 avg_vruntime(struct cfs_rq *cfs_rq) * EEVDF gives the following limit for a steady state system: * * -r_max < lag < max(r_max, q) - * - * XXX could add max_slice to the augmented data to track this. */ static void update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se) { + u64 max_slice = cfs_rq_max_slice(cfs_rq) + TICK_NSEC; s64 vlag, limit; WARN_ON_ONCE(!se->on_rq); vlag = avg_vruntime(cfs_rq) - se->vruntime; - limit = calc_delta_fair(max_t(u64, 2*se->slice, TICK_NSEC), se); + limit = calc_delta_fair(max_slice, se); se->vlag = clamp(vlag, -limit, limit); } @@ -722,8 +797,8 @@ static void update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se) static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime) { struct sched_entity *curr = cfs_rq->curr; - s64 avg = cfs_rq->avg_vruntime; - long load = cfs_rq->avg_load; + s64 avg = cfs_rq->sum_w_vruntime; + long load = cfs_rq->sum_weight; if (curr && curr->on_rq) { unsigned long weight = scale_load_down(curr->load.weight); @@ -732,7 +807,7 @@ static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime) load += weight; } - return avg >= (s64)(vruntime - cfs_rq->zero_vruntime) * load; + return avg >= vruntime_op(vruntime, "-", cfs_rq->zero_vruntime) * load; } int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se) @@ -740,16 +815,6 @@ int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se) return vruntime_eligible(cfs_rq, se->vruntime); } -static void update_zero_vruntime(struct cfs_rq *cfs_rq) -{ - u64 vruntime = avg_vruntime(cfs_rq); - s64 delta = (s64)(vruntime - cfs_rq->zero_vruntime); - - avg_vruntime_update(cfs_rq, delta); - - cfs_rq->zero_vruntime = vruntime; -} - static inline u64 cfs_rq_min_slice(struct cfs_rq *cfs_rq) { struct sched_entity *root = __pick_root_entity(cfs_rq); @@ -765,18 +830,32 @@ static inline u64 cfs_rq_min_slice(struct cfs_rq *cfs_rq) return min_slice; } +static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq) +{ + struct sched_entity *root = __pick_root_entity(cfs_rq); + struct sched_entity *curr = cfs_rq->curr; + u64 max_slice = 0ULL; + + if (curr && curr->on_rq) + max_slice = curr->slice; + + if (root) + max_slice = max(max_slice, root->max_slice); + + return max_slice; +} + static inline bool __entity_less(struct rb_node *a, const struct rb_node *b) { return entity_before(__node_2_se(a), __node_2_se(b)); } -#define vruntime_gt(field, lse, rse) ({ (s64)((lse)->field - (rse)->field) > 0; }) - static inline void __min_vruntime_update(struct sched_entity *se, struct rb_node *node) { if (node) { struct sched_entity *rse = __node_2_se(node); - if (vruntime_gt(min_vruntime, se, rse)) + + if (vruntime_cmp(se->min_vruntime, ">", rse->min_vruntime)) se->min_vruntime = rse->min_vruntime; } } @@ -790,6 +869,15 @@ static inline void __min_slice_update(struct sched_entity *se, struct rb_node *n } } +static inline void __max_slice_update(struct sched_entity *se, struct rb_node *node) +{ + if (node) { + struct sched_entity *rse = __node_2_se(node); + if (rse->max_slice > se->max_slice) + se->max_slice = rse->max_slice; + } +} + /* * se->min_vruntime = min(se->vruntime, {left,right}->min_vruntime) */ @@ -797,6 +885,7 @@ static inline bool min_vruntime_update(struct sched_entity *se, bool exit) { u64 old_min_vruntime = se->min_vruntime; u64 old_min_slice = se->min_slice; + u64 old_max_slice = se->max_slice; struct rb_node *node = &se->run_node; se->min_vruntime = se->vruntime; @@ -807,8 +896,13 @@ static inline bool min_vruntime_update(struct sched_entity *se, bool exit) __min_slice_update(se, node->rb_right); __min_slice_update(se, node->rb_left); + se->max_slice = se->slice; + __max_slice_update(se, node->rb_right); + __max_slice_update(se, node->rb_left); + return se->min_vruntime == old_min_vruntime && - se->min_slice == old_min_slice; + se->min_slice == old_min_slice && + se->max_slice == old_max_slice; } RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity, @@ -819,8 +913,7 @@ RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity, */ static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { - avg_vruntime_add(cfs_rq, se); - update_zero_vruntime(cfs_rq); + sum_w_vruntime_add(cfs_rq, se); se->min_vruntime = se->vruntime; se->min_slice = se->slice; rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, @@ -831,8 +924,7 @@ static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, &min_vruntime_cb); - avg_vruntime_sub(cfs_rq, se); - update_zero_vruntime(cfs_rq); + sum_w_vruntime_sub(cfs_rq, se); } struct sched_entity *__pick_root_entity(struct cfs_rq *cfs_rq) @@ -887,7 +979,7 @@ static inline void update_protect_slice(struct cfs_rq *cfs_rq, struct sched_enti static inline bool protect_slice(struct sched_entity *se) { - return ((s64)(se->vprot - se->vruntime) > 0); + return vruntime_cmp(se->vruntime, "<", se->vprot); } static inline void cancel_protect_slice(struct sched_entity *se) @@ -1014,7 +1106,7 @@ static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se); */ static bool update_deadline(struct cfs_rq *cfs_rq, struct sched_entity *se) { - if ((s64)(se->vruntime - se->deadline) < 0) + if (vruntime_cmp(se->vruntime, "<", se->deadline)) return false; /* @@ -3744,6 +3836,8 @@ static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, unsigned long weight) { bool curr = cfs_rq->curr == se; + bool rel_vprot = false; + u64 vprot; if (se->on_rq) { /* commit outstanding execution time */ @@ -3751,6 +3845,11 @@ static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, update_entity_lag(cfs_rq, se); se->deadline -= se->vruntime; se->rel_deadline = 1; + if (curr && protect_slice(se)) { + vprot = se->vprot - se->vruntime; + rel_vprot = true; + } + cfs_rq->nr_queued--; if (!curr) __dequeue_entity(cfs_rq, se); @@ -3766,6 +3865,9 @@ static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, if (se->rel_deadline) se->deadline = div_s64(se->deadline * se->load.weight, weight); + if (rel_vprot) + vprot = div_s64(vprot * se->load.weight, weight); + update_load_set(&se->load, weight); do { @@ -3777,6 +3879,8 @@ static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, enqueue_load_avg(cfs_rq, se); if (se->on_rq) { place_entity(cfs_rq, se, 0); + if (rel_vprot) + se->vprot = se->vruntime + vprot; update_load_add(&cfs_rq->load, se->load.weight); if (!curr) __enqueue_entity(cfs_rq, se); @@ -5164,7 +5268,7 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) * * vl_i = (W + w_i)*vl'_i / W */ - load = cfs_rq->avg_load; + load = cfs_rq->sum_weight; if (curr && curr->on_rq) load += scale_load_down(curr->load.weight); @@ -5416,7 +5520,7 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) } static void -set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) +set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, bool first) { clear_buddies(cfs_rq, se); @@ -5431,7 +5535,8 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) __dequeue_entity(cfs_rq, se); update_load_avg(cfs_rq, se, UPDATE_TG); - set_protect_slice(cfs_rq, se); + if (first) + set_protect_slice(cfs_rq, se); } update_stats_curr_start(cfs_rq, se); @@ -5530,6 +5635,11 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) update_load_avg(cfs_rq, curr, UPDATE_TG); update_cfs_group(curr); + /* + * Pulls along cfs_rq::zero_vruntime. + */ + avg_vruntime(cfs_rq); + #ifdef CONFIG_SCHED_HRTICK /* * queued ticks are scheduled to match the slice, so don't bother @@ -8866,13 +8976,13 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf pse = parent_entity(pse); } if (se_depth >= pse_depth) { - set_next_entity(cfs_rq_of(se), se); + set_next_entity(cfs_rq_of(se), se, true); se = parent_entity(se); } } put_prev_entity(cfs_rq, pse); - set_next_entity(cfs_rq, se); + set_next_entity(cfs_rq, se, true); __set_next_task_fair(rq, p, true); } @@ -13238,8 +13348,8 @@ bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b, * zero_vruntime_fi, which would have been updated in prior calls * to se_fi_update(). */ - delta = (s64)(sea->vruntime - seb->vruntime) + - (s64)(cfs_rqb->zero_vruntime_fi - cfs_rqa->zero_vruntime_fi); + delta = vruntime_op(sea->vruntime, "-", seb->vruntime) + + vruntime_op(cfs_rqb->zero_vruntime_fi, "-", cfs_rqa->zero_vruntime_fi); return delta > 0; } @@ -13464,7 +13574,7 @@ static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) for_each_sched_entity(se) { struct cfs_rq *cfs_rq = cfs_rq_of(se); - set_next_entity(cfs_rq, se); + set_next_entity(cfs_rq, se, first); /* ensure bandwidth has been allocated on our new cfs_rq */ account_cfs_rq_runtime(cfs_rq, 0); } diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index fb07dcfc60a24..d4d994fb8999a 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -2100,6 +2100,7 @@ static void push_rt_tasks(struct rq *rq) */ static int rto_next_cpu(struct root_domain *rd) { + int this_cpu = smp_processor_id(); int next; int cpu; @@ -2123,6 +2124,10 @@ static int rto_next_cpu(struct root_domain *rd) rd->rto_cpu = cpu; + /* Do not send IPI to self */ + if (cpu == this_cpu) + continue; + if (cpu < nr_cpu_ids) return cpu; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 2f8b06b12a98f..ed37ab9209e59 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -680,8 +680,8 @@ struct cfs_rq { unsigned int h_nr_runnable; /* SCHED_{NORMAL,BATCH,IDLE} */ unsigned int h_nr_idle; /* SCHED_IDLE */ - s64 avg_vruntime; - u64 avg_load; + s64 sum_w_vruntime; + u64 sum_weight; u64 zero_vruntime; #ifdef CONFIG_SCHED_CORE diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index e618addb58641..21b6d93401480 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1742,7 +1742,7 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, lockdep_assert_held(&cpu_base->lock); - debug_deactivate(timer); + debug_hrtimer_deactivate(timer); base->running = timer; /* diff --git a/kernel/time/sched_clock.c b/kernel/time/sched_clock.c index cc1afec306b3f..425d429906d0d 100644 --- a/kernel/time/sched_clock.c +++ b/kernel/time/sched_clock.c @@ -215,7 +215,7 @@ void sched_clock_register(u64 (*read)(void), int bits, unsigned long rate) update_clock_read_data(&rd); - if (sched_clock_timer.function != NULL) { + if (ACCESS_PRIVATE(&sched_clock_timer, function) != NULL) { /* update timeout for clock wrap */ hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL_HARD); diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 340fef20bdcd0..c7dcccc5f3d6b 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -2639,7 +2639,8 @@ static int timekeeping_validate_timex(const struct __kernel_timex *txc, bool aux if (aux_clock) { /* Auxiliary clocks are similar to TAI and do not have leap seconds */ - if (txc->status & (STA_INS | STA_DEL)) + if (txc->modes & ADJ_STATUS && + txc->status & (STA_INS | STA_DEL)) return -EINVAL; /* No TAI offset setting */ @@ -2647,7 +2648,8 @@ static int timekeeping_validate_timex(const struct __kernel_timex *txc, bool aux return -EINVAL; /* No PPS support either */ - if (txc->status & (STA_PPSFREQ | STA_PPSTIME)) + if (txc->modes & ADJ_STATUS && + txc->status & (STA_PPSFREQ | STA_PPSTIME)) return -EINVAL; } diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 49e0bdaa7a1bf..ae64b261de8e3 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -1022,7 +1022,7 @@ const struct bpf_func_proto bpf_snprintf_btf_proto = { .func = bpf_snprintf_btf, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM, + .arg1_type = ARG_PTR_TO_MEM | MEM_WRITE, .arg2_type = ARG_CONST_SIZE, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg4_type = ARG_CONST_SIZE, @@ -1526,7 +1526,7 @@ static const struct bpf_func_proto bpf_read_branch_records_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM_OR_NULL, + .arg2_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, .arg3_type = ARG_CONST_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -1661,7 +1661,7 @@ static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, + .arg2_type = ARG_PTR_TO_UNINIT_MEM, .arg3_type = ARG_CONST_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -2564,6 +2564,7 @@ kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, old_run_ctx = bpf_set_run_ctx(&run_ctx.session_ctx.run_ctx); err = bpf_prog_run(link->link.prog, regs); bpf_reset_run_ctx(old_run_ctx); + ftrace_partial_regs_update(fregs, bpf_kprobe_multi_pt_regs_ptr()); rcu_read_unlock(); out: diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index 599f2939cd940..234fd9d45b56e 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -542,7 +542,11 @@ static struct fgraph_ops fgraph_stub = { static struct fgraph_ops *fgraph_direct_gops = &fgraph_stub; DEFINE_STATIC_CALL(fgraph_func, ftrace_graph_entry_stub); DEFINE_STATIC_CALL(fgraph_retfunc, ftrace_graph_ret_stub); +#if FGRAPH_NO_DIRECT +static DEFINE_STATIC_KEY_FALSE(fgraph_do_direct); +#else static DEFINE_STATIC_KEY_TRUE(fgraph_do_direct); +#endif /** * ftrace_graph_stop - set to permanently disable function graph tracing @@ -846,7 +850,7 @@ __ftrace_return_to_handler(struct ftrace_regs *fregs, unsigned long frame_pointe bitmap = get_bitmap_bits(current, offset); #ifdef CONFIG_HAVE_STATIC_CALL - if (static_branch_likely(&fgraph_do_direct)) { + if (!FGRAPH_NO_DIRECT && static_branch_likely(&fgraph_do_direct)) { if (test_bit(fgraph_direct_gops->idx, &bitmap)) static_call(fgraph_retfunc)(&trace, fgraph_direct_gops, fregs); } else @@ -1293,6 +1297,9 @@ static void ftrace_graph_enable_direct(bool enable_branch, struct fgraph_ops *go trace_func_graph_ret_t retfunc = NULL; int i; + if (FGRAPH_NO_DIRECT) + return; + if (gops) { func = gops->entryfunc; retfunc = gops->retfunc; @@ -1311,11 +1318,14 @@ static void ftrace_graph_enable_direct(bool enable_branch, struct fgraph_ops *go static_call_update(fgraph_func, func); static_call_update(fgraph_retfunc, retfunc); if (enable_branch) - static_branch_disable(&fgraph_do_direct); + static_branch_enable(&fgraph_do_direct); } static void ftrace_graph_disable_direct(bool disable_branch) { + if (FGRAPH_NO_DIRECT) + return; + if (disable_branch) static_branch_disable(&fgraph_do_direct); static_call_update(fgraph_func, ftrace_graph_entry_stub); diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index e95408a47c1d0..905f4d1679559 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -7709,7 +7709,8 @@ ftrace_func_address_lookup(struct ftrace_mod_map *mod_map, int ftrace_mod_address_lookup(unsigned long addr, unsigned long *size, - unsigned long *off, char **modname, char *sym) + unsigned long *off, char **modname, + const unsigned char **modbuildid, char *sym) { struct ftrace_mod_map *mod_map; int ret = 0; @@ -7721,6 +7722,8 @@ ftrace_mod_address_lookup(unsigned long addr, unsigned long *size, if (ret) { if (modname) *modname = mod_map->mod->name; + if (modbuildid) + *modbuildid = module_buildid(mod_map->mod); break; } } diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 3ba08fc1b7d05..98ca4beabf023 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -1813,6 +1813,7 @@ static int rb_read_data_buffer(struct buffer_data_page *dpage, int tail, int cpu struct ring_buffer_event *event; u64 ts, delta; int events = 0; + int len; int e; *delta_ptr = 0; @@ -1820,9 +1821,12 @@ static int rb_read_data_buffer(struct buffer_data_page *dpage, int tail, int cpu ts = dpage->time_stamp; - for (e = 0; e < tail; e += rb_event_length(event)) { + for (e = 0; e < tail; e += len) { event = (struct ring_buffer_event *)(dpage->data + e); + len = rb_event_length(event); + if (len <= 0 || len > tail - e) + return -1; switch (event->type_len) { @@ -1883,6 +1887,8 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) if (!meta || !meta->head_buffer) return; + orig_head = head_page = cpu_buffer->head_page; + /* Do the reader page first */ ret = rb_validate_buffer(cpu_buffer->reader_page->page, cpu_buffer->cpu); if (ret < 0) { @@ -1893,7 +1899,6 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) entry_bytes += local_read(&cpu_buffer->reader_page->page->commit); local_set(&cpu_buffer->reader_page->entries, ret); - orig_head = head_page = cpu_buffer->head_page; ts = head_page->page->time_stamp; /* @@ -7278,6 +7283,27 @@ int ring_buffer_map(struct trace_buffer *buffer, int cpu, return err; } +/* + * This is called when a VMA is duplicated (e.g., on fork()) to increment + * the user_mapped counter without remapping pages. + */ +void ring_buffer_map_dup(struct trace_buffer *buffer, int cpu) +{ + struct ring_buffer_per_cpu *cpu_buffer; + + if (WARN_ON(!cpumask_test_cpu(cpu, buffer->cpumask))) + return; + + cpu_buffer = buffer->buffers[cpu]; + + guard(mutex)(&cpu_buffer->mapping_lock); + + if (cpu_buffer->user_mapped) + __rb_inc_dec_mapped(cpu_buffer, true); + else + WARN(1, "Unexpected buffer stat, it should be mapped"); +} + int ring_buffer_unmap(struct trace_buffer *buffer, int cpu) { struct ring_buffer_per_cpu *cpu_buffer; diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 907923d5f8bbb..38fab063c3687 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -4852,6 +4852,8 @@ static int tracing_single_release_tr(struct inode *inode, struct file *file) return single_release(inode, file); } +static bool update_last_data_if_empty(struct trace_array *tr); + static int tracing_open(struct inode *inode, struct file *file) { struct trace_array *tr = inode->i_private; @@ -4876,6 +4878,8 @@ static int tracing_open(struct inode *inode, struct file *file) tracing_reset_online_cpus(trace_buf); else tracing_reset_cpu(trace_buf, cpu); + + update_last_data_if_empty(tr); } if (file->f_mode & FMODE_READ) { @@ -5917,6 +5921,7 @@ tracing_set_trace_read(struct file *filp, char __user *ubuf, int tracer_init(struct tracer *t, struct trace_array *tr) { tracing_reset_online_cpus(&tr->array_buffer); + update_last_data_if_empty(tr); return t->init(tr); } @@ -7582,6 +7587,7 @@ int tracing_set_clock(struct trace_array *tr, const char *clockstr) ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func); tracing_reset_online_cpus(&tr->max_buffer); #endif + update_last_data_if_empty(tr); if (tr->scratch && !(tr->flags & TRACE_ARRAY_FL_LAST_BOOT)) { struct trace_scratch *tscratch = tr->scratch; @@ -8778,6 +8784,18 @@ static inline int get_snapshot_map(struct trace_array *tr) { return 0; } static inline void put_snapshot_map(struct trace_array *tr) { } #endif +/* + * This is called when a VMA is duplicated (e.g., on fork()) to increment + * the user_mapped counter without remapping pages. + */ +static void tracing_buffers_mmap_open(struct vm_area_struct *vma) +{ + struct ftrace_buffer_info *info = vma->vm_file->private_data; + struct trace_iterator *iter = &info->iter; + + ring_buffer_map_dup(iter->array_buffer->buffer, iter->cpu_file); +} + static void tracing_buffers_mmap_close(struct vm_area_struct *vma) { struct ftrace_buffer_info *info = vma->vm_file->private_data; @@ -8797,6 +8815,7 @@ static int tracing_buffers_may_split(struct vm_area_struct *vma, unsigned long a } static const struct vm_operations_struct tracing_buffers_vmops = { + .open = tracing_buffers_mmap_open, .close = tracing_buffers_mmap_close, .may_split = tracing_buffers_may_split, }; @@ -9183,7 +9202,7 @@ tracing_init_tracefs_percpu(struct trace_array *tr, long cpu) trace_create_cpu_file("stats", TRACE_MODE_READ, d_cpu, tr, cpu, &tracing_stats_fops); - trace_create_cpu_file("buffer_size_kb", TRACE_MODE_READ, d_cpu, + trace_create_cpu_file("buffer_size_kb", TRACE_MODE_WRITE, d_cpu, tr, cpu, &tracing_entries_fops); if (tr->range_addr_start) diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 099f081329021..e4ce7f856f630 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -1296,6 +1296,9 @@ static void remove_event_file_dir(struct trace_event_file *file) free_event_filter(file->filter); file->flags |= EVENT_FILE_FL_FREED; event_file_put(file); + + /* Wake up hist poll waiters to notice the EVENT_FILE_FL_FREED flag. */ + hist_poll_wakeup(); } /* @@ -3964,11 +3967,6 @@ void trace_put_event_file(struct trace_event_file *file) EXPORT_SYMBOL_GPL(trace_put_event_file); #ifdef CONFIG_DYNAMIC_FTRACE - -/* Avoid typos */ -#define ENABLE_EVENT_STR "enable_event" -#define DISABLE_EVENT_STR "disable_event" - struct event_probe_data { struct trace_event_file *file; unsigned long count; diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 425ae26064bab..2a0726e1bc97f 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -5778,7 +5778,7 @@ static __poll_t event_hist_poll(struct file *file, struct poll_table_struct *wai guard(mutex)(&event_mutex); - event_file = event_file_data(file); + event_file = event_file_file(file); if (!event_file) return EPOLLERR; @@ -5816,7 +5816,7 @@ static int event_hist_open(struct inode *inode, struct file *file) guard(mutex)(&event_mutex); - event_file = event_file_data(file); + event_file = event_file_file(file); if (!event_file) { ret = -ENODEV; goto err; @@ -6909,7 +6909,7 @@ static int event_hist_trigger_parse(struct event_command *cmd_ops, remove_hist_vars(hist_data); - kfree(trigger_data); + trigger_data_free(trigger_data); destroy_hist_data(hist_data); goto out; diff --git a/kernel/trace/trace_events_trigger.c b/kernel/trace/trace_events_trigger.c index cbfc306c0159a..98b8d5df15c79 100644 --- a/kernel/trace/trace_events_trigger.c +++ b/kernel/trace/trace_events_trigger.c @@ -19,6 +19,9 @@ static DEFINE_MUTEX(trigger_cmd_mutex); void trigger_data_free(struct event_trigger_data *data) { + if (!data) + return; + if (data->cmd_ops->set_filter) data->cmd_ops->set_filter(NULL, data, NULL); diff --git a/kernel/trace/trace_hwlat.c b/kernel/trace/trace_hwlat.c index 2f7b94e98317c..3fe274b84f1c2 100644 --- a/kernel/trace/trace_hwlat.c +++ b/kernel/trace/trace_hwlat.c @@ -102,9 +102,9 @@ struct hwlat_sample { /* keep the global state somewhere. */ static struct hwlat_data { - struct mutex lock; /* protect changes */ + struct mutex lock; /* protect changes */ - u64 count; /* total since reset */ + atomic64_t count; /* total since reset */ u64 sample_window; /* total sampling window (on+off) */ u64 sample_width; /* active sampling portion of window */ @@ -193,8 +193,7 @@ void trace_hwlat_callback(bool enter) * get_sample - sample the CPU TSC and look for likely hardware latencies * * Used to repeatedly capture the CPU TSC (or similar), looking for potential - * hardware-induced latency. Called with interrupts disabled and with - * hwlat_data.lock held. + * hardware-induced latency. Called with interrupts disabled. */ static int get_sample(void) { @@ -204,6 +203,7 @@ static int get_sample(void) time_type start, t1, t2, last_t2; s64 diff, outer_diff, total, last_total = 0; u64 sample = 0; + u64 sample_width = READ_ONCE(hwlat_data.sample_width); u64 thresh = tracing_thresh; u64 outer_sample = 0; int ret = -1; @@ -267,7 +267,7 @@ static int get_sample(void) if (diff > sample) sample = diff; /* only want highest value */ - } while (total <= hwlat_data.sample_width); + } while (total <= sample_width); barrier(); /* finish the above in the view for NMIs */ trace_hwlat_callback_enabled = false; @@ -285,8 +285,7 @@ static int get_sample(void) if (kdata->nmi_total_ts) do_div(kdata->nmi_total_ts, NSEC_PER_USEC); - hwlat_data.count++; - s.seqnum = hwlat_data.count; + s.seqnum = atomic64_inc_return(&hwlat_data.count); s.duration = sample; s.outer_duration = outer_sample; s.nmi_total_ts = kdata->nmi_total_ts; @@ -832,7 +831,7 @@ static int hwlat_tracer_init(struct trace_array *tr) hwlat_trace = tr; - hwlat_data.count = 0; + atomic64_set(&hwlat_data.count, 0); tr->max_latency = 0; save_tracing_thresh = tracing_thresh; diff --git a/kernel/ucount.c b/kernel/ucount.c index 586af49fc03e4..fc4a8f2d30965 100644 --- a/kernel/ucount.c +++ b/kernel/ucount.c @@ -47,7 +47,7 @@ static int set_permissions(struct ctl_table_header *head, int mode; /* Allow users with CAP_SYS_RESOURCE unrestrained access */ - if (ns_capable(user_ns, CAP_SYS_RESOURCE)) + if (ns_capable_noaudit(user_ns, CAP_SYS_RESOURCE)) mode = (table->mode & S_IRWXU) >> 6; else /* Allow all others at most read-only access */ diff --git a/kernel/unwind/user.c b/kernel/unwind/user.c index 97a8415e3216f..39e2707894447 100644 --- a/kernel/unwind/user.c +++ b/kernel/unwind/user.c @@ -8,18 +8,28 @@ #include #include -static const struct unwind_user_frame fp_frame = { - ARCH_INIT_USER_FP_FRAME -}; - #define for_each_user_frame(state) \ for (unwind_user_start(state); !(state)->done; unwind_user_next(state)) -static int unwind_user_next_fp(struct unwind_user_state *state) +static inline int +get_user_word(unsigned long *word, unsigned long base, int off, unsigned int ws) +{ + unsigned long __user *addr = (void __user *)base + off; +#ifdef CONFIG_COMPAT + if (ws == sizeof(int)) { + unsigned int data; + int ret = get_user(data, (unsigned int __user *)addr); + *word = data; + return ret; + } +#endif + return get_user(*word, addr); +} + +static int unwind_user_next_common(struct unwind_user_state *state, + const struct unwind_user_frame *frame) { - const struct unwind_user_frame *frame = &fp_frame; unsigned long cfa, fp, ra; - unsigned int shift; if (frame->use_fp) { if (state->fp < state->sp) @@ -37,24 +47,45 @@ static int unwind_user_next_fp(struct unwind_user_state *state) return -EINVAL; /* Make sure that the address is word aligned */ - shift = sizeof(long) == 4 ? 2 : 3; - if (cfa & ((1 << shift) - 1)) + if (cfa & (state->ws - 1)) return -EINVAL; /* Find the Return Address (RA) */ - if (get_user(ra, (unsigned long *)(cfa + frame->ra_off))) + if (get_user_word(&ra, cfa, frame->ra_off, state->ws)) return -EINVAL; - if (frame->fp_off && get_user(fp, (unsigned long __user *)(cfa + frame->fp_off))) + if (frame->fp_off && get_user_word(&fp, cfa, frame->fp_off, state->ws)) return -EINVAL; state->ip = ra; state->sp = cfa; if (frame->fp_off) state->fp = fp; + state->topmost = false; return 0; } +static int unwind_user_next_fp(struct unwind_user_state *state) +{ +#ifdef CONFIG_HAVE_UNWIND_USER_FP + struct pt_regs *regs = task_pt_regs(current); + + if (state->topmost && unwind_user_at_function_start(regs)) { + const struct unwind_user_frame fp_entry_frame = { + ARCH_INIT_USER_FP_ENTRY_FRAME(state->ws) + }; + return unwind_user_next_common(state, &fp_entry_frame); + } + + const struct unwind_user_frame fp_frame = { + ARCH_INIT_USER_FP_FRAME(state->ws) + }; + return unwind_user_next_common(state, &fp_frame); +#else + return -EINVAL; +#endif +} + static int unwind_user_next(struct unwind_user_state *state) { unsigned long iter_mask = state->available_types; @@ -102,6 +133,12 @@ static int unwind_user_start(struct unwind_user_state *state) state->ip = instruction_pointer(regs); state->sp = user_stack_pointer(regs); state->fp = frame_pointer(regs); + state->ws = unwind_user_word_size(regs); + if (!state->ws) { + state->done = true; + return -EINVAL; + } + state->topmost = true; return 0; } diff --git a/kernel/watchdog.c b/kernel/watchdog.c index 5b62d10027836..25d134306e146 100644 --- a/kernel/watchdog.c +++ b/kernel/watchdog.c @@ -521,7 +521,7 @@ static bool need_counting_irqs(void) u8 util; int tail = __this_cpu_read(cpustat_tail); - tail = (tail + NUM_HARDIRQ_REPORT - 1) % NUM_HARDIRQ_REPORT; + tail = (tail + NUM_SAMPLE_PERIODS - 1) % NUM_SAMPLE_PERIODS; util = __this_cpu_read(cpustat_util[tail][STATS_HARDIRQ]); return util > HARDIRQ_PERCENT_THRESH; } diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 45320e27a16c4..885a8b31f855b 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -117,6 +117,8 @@ enum wq_internal_consts { MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */ CREATE_COOLDOWN = HZ, /* time to breath after fail */ + RESCUER_BATCH = 16, /* process items per turn */ + /* * Rescue workers are used only on emergencies and shared by * all cpus. Give MIN_NICE. @@ -286,6 +288,7 @@ struct pool_workqueue { struct list_head pending_node; /* LN: node on wq_node_nr_active->pending_pwqs */ struct list_head pwqs_node; /* WR: node on wq->pwqs */ struct list_head mayday_node; /* MD: node on wq->maydays */ + struct work_struct mayday_cursor; /* L: cursor on pool->worklist */ u64 stats[PWQ_NR_STATS]; @@ -1126,6 +1129,12 @@ static struct worker *find_worker_executing_work(struct worker_pool *pool, return NULL; } +static void mayday_cursor_func(struct work_struct *work) +{ + /* should not be processed, only for marking position */ + BUG(); +} + /** * move_linked_works - move linked works to a list * @work: start of series of works to be scheduled @@ -1188,6 +1197,16 @@ static bool assign_work(struct work_struct *work, struct worker *worker, lockdep_assert_held(&pool->lock); + /* The cursor work should not be processed */ + if (unlikely(work->func == mayday_cursor_func)) { + /* only worker_thread() can possibly take this branch */ + WARN_ON_ONCE(worker->rescue_wq); + if (nextp) + *nextp = list_next_entry(work, entry); + list_del_init(&work->entry); + return false; + } + /* * A single work shouldn't be executed concurrently by multiple workers. * __queue_work() ensures that @work doesn't jump to a different pool @@ -3443,6 +3462,35 @@ static int worker_thread(void *__worker) goto woke_up; } +static bool assign_rescuer_work(struct pool_workqueue *pwq, struct worker *rescuer) +{ + struct worker_pool *pool = pwq->pool; + struct work_struct *cursor = &pwq->mayday_cursor; + struct work_struct *work, *n; + + /* need rescue? */ + if (!pwq->nr_active || !need_to_create_worker(pool)) + return false; + + /* search from the start or cursor if available */ + if (list_empty(&cursor->entry)) + work = list_first_entry(&pool->worklist, struct work_struct, entry); + else + work = list_next_entry(cursor, entry); + + /* find the next work item to rescue */ + list_for_each_entry_safe_from(work, n, &pool->worklist, entry) { + if (get_work_pwq(work) == pwq && assign_work(work, rescuer, &n)) { + pwq->stats[PWQ_STAT_RESCUED]++; + /* put the cursor for next search */ + list_move_tail(&cursor->entry, &n->entry); + return true; + } + } + + return false; +} + /** * rescuer_thread - the rescuer thread function * @__rescuer: self @@ -3497,7 +3545,7 @@ static int rescuer_thread(void *__rescuer) struct pool_workqueue *pwq = list_first_entry(&wq->maydays, struct pool_workqueue, mayday_node); struct worker_pool *pool = pwq->pool; - struct work_struct *work, *n; + unsigned int count = 0; __set_current_state(TASK_RUNNING); list_del_init(&pwq->mayday_node); @@ -3508,30 +3556,18 @@ static int rescuer_thread(void *__rescuer) raw_spin_lock_irq(&pool->lock); - /* - * Slurp in all works issued via this workqueue and - * process'em. - */ WARN_ON_ONCE(!list_empty(&rescuer->scheduled)); - list_for_each_entry_safe(work, n, &pool->worklist, entry) { - if (get_work_pwq(work) == pwq && - assign_work(work, rescuer, &n)) - pwq->stats[PWQ_STAT_RESCUED]++; - } - if (!list_empty(&rescuer->scheduled)) { + while (assign_rescuer_work(pwq, rescuer)) { process_scheduled_works(rescuer); /* - * The above execution of rescued work items could - * have created more to rescue through - * pwq_activate_first_inactive() or chained - * queueing. Let's put @pwq back on mayday list so - * that such back-to-back work items, which may be - * being used to relieve memory pressure, don't - * incur MAYDAY_INTERVAL delay inbetween. + * If the per-turn work item limit is reached and other + * PWQs are in mayday, requeue mayday for this PWQ and + * let the rescuer handle the other PWQs first. */ - if (pwq->nr_active && need_to_create_worker(pool)) { + if (++count > RESCUER_BATCH && !list_empty(&pwq->wq->maydays) && + pwq->nr_active && need_to_create_worker(pool)) { raw_spin_lock(&wq_mayday_lock); /* * Queue iff we aren't racing destruction @@ -3542,9 +3578,14 @@ static int rescuer_thread(void *__rescuer) list_add_tail(&pwq->mayday_node, &wq->maydays); } raw_spin_unlock(&wq_mayday_lock); + break; } } + /* The cursor can not be left behind without the rescuer watching it. */ + if (!list_empty(&pwq->mayday_cursor.entry) && list_empty(&pwq->mayday_node)) + list_del_init(&pwq->mayday_cursor.entry); + /* * Leave this pool. Notify regular workers; otherwise, we end up * with 0 concurrency and stalling the execution. @@ -5163,6 +5204,19 @@ static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq, INIT_LIST_HEAD(&pwq->pwqs_node); INIT_LIST_HEAD(&pwq->mayday_node); kthread_init_work(&pwq->release_work, pwq_release_workfn); + + /* + * Set the dummy cursor work with valid function and get_work_pwq(). + * + * The cursor work should only be in the pwq->pool->worklist, and + * should not be treated as a processable work item. + * + * WORK_STRUCT_PENDING and WORK_STRUCT_INACTIVE just make it less + * surprise for kernel debugging tools and reviewers. + */ + INIT_WORK(&pwq->mayday_cursor, mayday_cursor_func); + atomic_long_set(&pwq->mayday_cursor.data, (unsigned long)pwq | + WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | WORK_STRUCT_INACTIVE); } /* sync @pwq with the current state of its associated wq and link it */ diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 713cc94caa02e..21cd68084e468 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -721,6 +721,7 @@ source "mm/Kconfig.debug" config DEBUG_OBJECTS bool "Debug object operations" + depends on PREEMPT_COUNT || !DEFERRED_STRUCT_PAGE_INIT depends on DEBUG_KERNEL help If you say Y here, additional code will be inserted into the @@ -1684,33 +1685,6 @@ config STACKTRACE It is also used by various kernel debugging features that require stack trace generation. -config WARN_ALL_UNSEEDED_RANDOM - bool "Warn for all uses of unseeded randomness" - default n - help - Some parts of the kernel contain bugs relating to their use of - cryptographically secure random numbers before it's actually possible - to generate those numbers securely. This setting ensures that these - flaws don't go unnoticed, by enabling a message, should this ever - occur. This will allow people with obscure setups to know when things - are going wrong, so that they might contact developers about fixing - it. - - Unfortunately, on some models of some architectures getting - a fully seeded CRNG is extremely difficult, and so this can - result in dmesg getting spammed for a surprisingly long - time. This is really bad from a security perspective, and - so architecture maintainers really need to do what they can - to get the CRNG seeded sooner after the system is booted. - However, since users cannot do anything actionable to - address this, by default this option is disabled. - - Say Y here if you want to receive warnings for all uses of - unseeded randomness. This will be of use primarily for - those developers interested in improving the security of - Linux kernels running on their architecture (or - subarchitecture). - config DEBUG_KOBJECT bool "kobject debugging" depends on DEBUG_KERNEL diff --git a/lib/debugobjects.c b/lib/debugobjects.c index 7f50c4480a4e3..b3151679d0d30 100644 --- a/lib/debugobjects.c +++ b/lib/debugobjects.c @@ -398,9 +398,26 @@ static void fill_pool(void) atomic_inc(&cpus_allocating); while (pool_should_refill(&pool_global)) { + gfp_t gfp = __GFP_HIGH | __GFP_NOWARN; HLIST_HEAD(head); - if (!kmem_alloc_batch(&head, obj_cache, __GFP_HIGH | __GFP_NOWARN)) + /* + * Allow reclaim only in preemptible context and during + * early boot. If not preemptible, the caller might hold + * locks causing a deadlock in the allocator. + * + * If the reclaim flag is not set during early boot then + * allocations, which happen before deferred page + * initialization has completed, will fail. + * + * In preemptible context the flag is harmless and not a + * performance issue as that's usually invoked from slow + * path initialization context. + */ + if (preemptible() || system_state < SYSTEM_SCHEDULING) + gfp |= __GFP_KSWAPD_RECLAIM; + + if (!kmem_alloc_batch(&head, obj_cache, gfp)) break; guard(raw_spinlock_irqsave)(&pool_lock); diff --git a/lib/kstrtox.c b/lib/kstrtox.c index bdde40cd69d78..97be2a39f5371 100644 --- a/lib/kstrtox.c +++ b/lib/kstrtox.c @@ -340,8 +340,8 @@ EXPORT_SYMBOL(kstrtos8); * @s: input string * @res: result * - * This routine returns 0 iff the first character is one of 'YyTt1NnFf0', or - * [oO][NnFf] for "on" and "off". Otherwise it will return -EINVAL. Value + * This routine returns 0 iff the first character is one of 'EeYyTt1DdNnFf0', + * or [oO][NnFf] for "on" and "off". Otherwise it will return -EINVAL. Value * pointed to by res is updated upon finding a match. */ noinline diff --git a/lib/objpool.c b/lib/objpool.c index b998b720c7329..d98fadf1de169 100644 --- a/lib/objpool.c +++ b/lib/objpool.c @@ -142,7 +142,7 @@ int objpool_init(struct objpool_head *pool, int nr_objs, int object_size, pool->gfp = gfp & ~__GFP_ZERO; pool->context = context; pool->release = release; - slot_size = nr_cpu_ids * sizeof(struct objpool_slot); + slot_size = nr_cpu_ids * sizeof(struct objpool_slot *); pool->cpu_slots = kzalloc(slot_size, pool->gfp); if (!pool->cpu_slots) return -ENOMEM; diff --git a/mm/highmem.c b/mm/highmem.c index b5c8e4c2d5d49..a33e411839517 100644 --- a/mm/highmem.c +++ b/mm/highmem.c @@ -180,12 +180,13 @@ struct page *__kmap_to_page(void *vaddr) for (i = 0; i < kctrl->idx; i++) { unsigned long base_addr; int idx; + pte_t pteval = kctrl->pteval[i]; idx = arch_kmap_local_map_idx(i, pte_pfn(pteval)); base_addr = __fix_to_virt(FIX_KMAP_BEGIN + idx); if (base_addr == base) - return pte_page(kctrl->pteval[i]); + return pte_page(pteval); } } diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 8ad170b9855a5..35ec12c4d7766 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -94,6 +94,9 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma) inode = file_inode(vma->vm_file); + if (IS_ANON_FILE(inode)) + return false; + return !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode); } diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 6a60af4798bee..13293976e0568 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5797,7 +5797,7 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma, unsigned long last_addr_mask; pte_t *src_pte, *dst_pte; struct mmu_notifier_range range; - bool shared_pmd = false; + struct mmu_gather tlb; mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, old_addr, old_end); @@ -5807,6 +5807,7 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma, * range. */ flush_cache_range(vma, range.start, range.end); + tlb_gather_mmu_vma(&tlb, vma); mmu_notifier_invalidate_range_start(&range); last_addr_mask = hugetlb_mask_last_page(h); @@ -5823,8 +5824,7 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma, if (huge_pte_none(huge_ptep_get(mm, old_addr, src_pte))) continue; - if (huge_pmd_unshare(mm, vma, old_addr, src_pte)) { - shared_pmd = true; + if (huge_pmd_unshare(&tlb, vma, old_addr, src_pte)) { old_addr |= last_addr_mask; new_addr |= last_addr_mask; continue; @@ -5835,15 +5835,16 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma, break; move_huge_pte(vma, old_addr, new_addr, src_pte, dst_pte, sz); + tlb_remove_huge_tlb_entry(h, &tlb, src_pte, old_addr); } - if (shared_pmd) - flush_hugetlb_tlb_range(vma, range.start, range.end); - else - flush_hugetlb_tlb_range(vma, old_end - len, old_end); + tlb_flush_mmu_tlbonly(&tlb); + huge_pmd_unshare_flush(&tlb, vma); + mmu_notifier_invalidate_range_end(&range); i_mmap_unlock_write(mapping); hugetlb_vma_unlock_write(vma); + tlb_finish_mmu(&tlb); return len + old_addr - old_end; } @@ -5862,7 +5863,6 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, unsigned long sz = huge_page_size(h); bool adjust_reservation; unsigned long last_addr_mask; - bool force_flush = false; WARN_ON(!is_vm_hugetlb_page(vma)); BUG_ON(start & ~huge_page_mask(h)); @@ -5885,10 +5885,8 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, } ptl = huge_pte_lock(h, mm, ptep); - if (huge_pmd_unshare(mm, vma, address, ptep)) { + if (huge_pmd_unshare(tlb, vma, address, ptep)) { spin_unlock(ptl); - tlb_flush_pmd_range(tlb, address & PUD_MASK, PUD_SIZE); - force_flush = true; address |= last_addr_mask; continue; } @@ -6004,14 +6002,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, } tlb_end_vma(tlb, vma); - /* - * There is nothing protecting a previously-shared page table that we - * unshared through huge_pmd_unshare() from getting freed after we - * release i_mmap_rwsem, so flush the TLB now. If huge_pmd_unshare() - * succeeded, flush the range corresponding to the pud. - */ - if (force_flush) - tlb_flush_mmu_tlbonly(tlb); + huge_pmd_unshare_flush(tlb, vma); } void __hugetlb_zap_begin(struct vm_area_struct *vma, @@ -7104,11 +7095,11 @@ long hugetlb_change_protection(struct vm_area_struct *vma, pte_t pte; struct hstate *h = hstate_vma(vma); long pages = 0, psize = huge_page_size(h); - bool shared_pmd = false; struct mmu_notifier_range range; unsigned long last_addr_mask; bool uffd_wp = cp_flags & MM_CP_UFFD_WP; bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; + struct mmu_gather tlb; /* * In the case of shared PMDs, the area to flush could be beyond @@ -7121,6 +7112,7 @@ long hugetlb_change_protection(struct vm_area_struct *vma, BUG_ON(address >= end); flush_cache_range(vma, range.start, range.end); + tlb_gather_mmu_vma(&tlb, vma); mmu_notifier_invalidate_range_start(&range); hugetlb_vma_lock_write(vma); @@ -7145,7 +7137,7 @@ long hugetlb_change_protection(struct vm_area_struct *vma, } } ptl = huge_pte_lock(h, mm, ptep); - if (huge_pmd_unshare(mm, vma, address, ptep)) { + if (huge_pmd_unshare(&tlb, vma, address, ptep)) { /* * When uffd-wp is enabled on the vma, unshare * shouldn't happen at all. Warn about it if it @@ -7154,7 +7146,6 @@ long hugetlb_change_protection(struct vm_area_struct *vma, WARN_ON_ONCE(uffd_wp || uffd_wp_resolve); pages++; spin_unlock(ptl); - shared_pmd = true; address |= last_addr_mask; continue; } @@ -7206,6 +7197,7 @@ long hugetlb_change_protection(struct vm_area_struct *vma, pte = huge_pte_clear_uffd_wp(pte); huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte); pages++; + tlb_remove_huge_tlb_entry(h, &tlb, ptep, address); } else { /* None pte */ if (unlikely(uffd_wp)) @@ -7218,16 +7210,9 @@ long hugetlb_change_protection(struct vm_area_struct *vma, cond_resched(); } - /* - * There is nothing protecting a previously-shared page table that we - * unshared through huge_pmd_unshare() from getting freed after we - * release i_mmap_rwsem, so flush the TLB now. If huge_pmd_unshare() - * succeeded, flush the range corresponding to the pud. - */ - if (shared_pmd) - flush_hugetlb_tlb_range(vma, range.start, range.end); - else - flush_hugetlb_tlb_range(vma, start, end); + + tlb_flush_mmu_tlbonly(&tlb); + huge_pmd_unshare_flush(&tlb, vma); /* * No need to call mmu_notifier_arch_invalidate_secondary_tlbs() we are * downgrading page table protection not changing it to point to a new @@ -7238,6 +7223,7 @@ long hugetlb_change_protection(struct vm_area_struct *vma, i_mmap_unlock_write(vma->vm_file->f_mapping); hugetlb_vma_unlock_write(vma); mmu_notifier_invalidate_range_end(&range); + tlb_finish_mmu(&tlb); return pages > 0 ? (pages << h->order) : pages; } @@ -7400,6 +7386,15 @@ long hugetlb_reserve_pages(struct inode *inode, */ hugetlb_acct_memory(h, -gbl_resv); } + /* Restore used_hpages for pages that failed global reservation */ + if (gbl_reserve && spool) { + unsigned long flags; + + spin_lock_irqsave(&spool->lock, flags); + if (spool->max_hpages != -1) + spool->used_hpages -= gbl_reserve; + unlock_or_release_subpool(spool, flags); + } out_uncharge_cgroup: hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h), chg * pages_per_huge_page(h), h_cg); @@ -7590,18 +7585,27 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, return pte; } -/* - * unmap huge page backed by shared pte. +/** + * huge_pmd_unshare - Unmap a pmd table if it is shared by multiple users + * @tlb: the current mmu_gather. + * @vma: the vma covering the pmd table. + * @addr: the address we are trying to unshare. + * @ptep: pointer into the (pmd) page table. + * + * Called with the page table lock held, the i_mmap_rwsem held in write mode + * and the hugetlb vma lock held in write mode. * - * Called with page table lock held. + * Note: The caller must call huge_pmd_unshare_flush() before dropping the + * i_mmap_rwsem. * - * returns: 1 successfully unmapped a shared pte page - * 0 the underlying pte page is not shared, or it is the last user + * Returns: 1 if it was a shared PMD table and it got unmapped, or 0 if it + * was not a shared PMD table. */ -int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, - unsigned long addr, pte_t *ptep) +int huge_pmd_unshare(struct mmu_gather *tlb, struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) { unsigned long sz = huge_page_size(hstate_vma(vma)); + struct mm_struct *mm = vma->vm_mm; pgd_t *pgd = pgd_offset(mm, addr); p4d_t *p4d = p4d_offset(pgd, addr); pud_t *pud = pud_offset(p4d, addr); @@ -7613,18 +7617,36 @@ int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, i_mmap_assert_write_locked(vma->vm_file->f_mapping); hugetlb_vma_assert_locked(vma); pud_clear(pud); - /* - * Once our caller drops the rmap lock, some other process might be - * using this page table as a normal, non-hugetlb page table. - * Wait for pending gup_fast() in other threads to finish before letting - * that happen. - */ - tlb_remove_table_sync_one(); - ptdesc_pmd_pts_dec(virt_to_ptdesc(ptep)); + + tlb_unshare_pmd_ptdesc(tlb, virt_to_ptdesc(ptep), addr); + mm_dec_nr_pmds(mm); return 1; } +/* + * huge_pmd_unshare_flush - Complete a sequence of huge_pmd_unshare() calls + * @tlb: the current mmu_gather. + * @vma: the vma covering the pmd table. + * + * Perform necessary TLB flushes or IPI broadcasts to synchronize PMD table + * unsharing with concurrent page table walkers. + * + * This function must be called after a sequence of huge_pmd_unshare() + * calls while still holding the i_mmap_rwsem. + */ +void huge_pmd_unshare_flush(struct mmu_gather *tlb, struct vm_area_struct *vma) +{ + /* + * We must synchronize page table unsharing such that nobody will + * try reusing a previously-shared page table while it might still + * be in use by previous sharers (TLB, GUP_fast). + */ + i_mmap_assert_write_locked(vma->vm_file->f_mapping); + + tlb_flush_unshared_tables(tlb); +} + #else /* !CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING */ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, @@ -7633,12 +7655,16 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, return NULL; } -int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, - unsigned long addr, pte_t *ptep) +int huge_pmd_unshare(struct mmu_gather *tlb, struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) { return 0; } +void huge_pmd_unshare_flush(struct mmu_gather *tlb, struct vm_area_struct *vma) +{ +} + void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma, unsigned long *start, unsigned long *end) { @@ -7905,6 +7931,7 @@ static void hugetlb_unshare_pmds(struct vm_area_struct *vma, unsigned long sz = huge_page_size(h); struct mm_struct *mm = vma->vm_mm; struct mmu_notifier_range range; + struct mmu_gather tlb; unsigned long address; spinlock_t *ptl; pte_t *ptep; @@ -7916,6 +7943,8 @@ static void hugetlb_unshare_pmds(struct vm_area_struct *vma, return; flush_cache_range(vma, start, end); + tlb_gather_mmu_vma(&tlb, vma); + /* * No need to call adjust_range_if_pmd_sharing_possible(), because * we have already done the PUD_SIZE alignment. @@ -7934,10 +7963,10 @@ static void hugetlb_unshare_pmds(struct vm_area_struct *vma, if (!ptep) continue; ptl = huge_pte_lock(h, mm, ptep); - huge_pmd_unshare(mm, vma, address, ptep); + huge_pmd_unshare(&tlb, vma, address, ptep); spin_unlock(ptl); } - flush_hugetlb_tlb_range(vma, start, end); + huge_pmd_unshare_flush(&tlb, vma); if (take_locks) { i_mmap_unlock_write(vma->vm_file->f_mapping); hugetlb_vma_unlock_write(vma); @@ -7947,6 +7976,7 @@ static void hugetlb_unshare_pmds(struct vm_area_struct *vma, * Documentation/mm/mmu_notifier.rst. */ mmu_notifier_invalidate_range_end(&range); + tlb_finish_mmu(&tlb); } /* diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 4deda33625f41..ab25d540f0b8f 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5597,7 +5597,19 @@ subsys_initcall(mem_cgroup_swap_init); #endif /* CONFIG_SWAP */ -bool mem_cgroup_node_allowed(struct mem_cgroup *memcg, int nid) +void mem_cgroup_node_filter_allowed(struct mem_cgroup *memcg, nodemask_t *mask) { - return memcg ? cpuset_node_allowed(memcg->css.cgroup, nid) : true; + nodemask_t allowed; + + if (!memcg) + return; + + /* + * Since this interface is intended for use by migration paths, and + * reclaim and migration are subject to race conditions such as changes + * in effective_mems and hot-unpluging of nodes, inaccurate allowed + * mask is acceptable. + */ + cpuset_nodes_allowed(memcg->css.cgroup, &allowed); + nodes_and(*mask, *mask, allowed); } diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index 374aa6f021c6b..5619ee967f3c7 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -426,6 +427,7 @@ static void __tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, #endif tlb->vma_pfn = 0; + tlb->fully_unshared_tables = 0; __tlb_reset_range(tlb); inc_tlb_flush_pending(tlb->mm); } @@ -459,6 +461,31 @@ void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm) __tlb_gather_mmu(tlb, mm, true); } +/** + * tlb_gather_mmu_vma - initialize an mmu_gather structure for operating on a + * single VMA + * @tlb: the mmu_gather structure to initialize + * @vma: the vm_area_struct + * + * Called to initialize an (on-stack) mmu_gather structure for operating on + * a single VMA. In contrast to tlb_gather_mmu(), calling this function will + * not require another call to tlb_start_vma(). In contrast to tlb_start_vma(), + * this function will *not* call flush_cache_range(). + * + * For hugetlb VMAs, this function will also initialize the mmu_gather + * page_size accordingly, not requiring a separate call to + * tlb_change_page_size(). + * + */ +void tlb_gather_mmu_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) +{ + tlb_gather_mmu(tlb, vma->vm_mm); + tlb_update_vma_flags(tlb, vma); + if (is_vm_hugetlb_page(vma)) + /* All entries have the same size. */ + tlb_change_page_size(tlb, huge_page_size(hstate_vma(vma))); +} + /** * tlb_finish_mmu - finish an mmu_gather structure * @tlb: the mmu_gather structure to finish @@ -468,6 +495,12 @@ void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm) */ void tlb_finish_mmu(struct mmu_gather *tlb) { + /* + * We expect an earlier huge_pmd_unshare_flush() call to sort this out, + * due to complicated locking requirements with page table unsharing. + */ + VM_WARN_ON_ONCE(tlb->fully_unshared_tables); + /* * If there are parallel threads are doing PTE changes on same range * under non-exclusive lock (e.g., mmap_lock read-side) but defer TLB diff --git a/mm/numa_memblks.c b/mm/numa_memblks.c index 8f5735fda0a21..3f53464240e8d 100644 --- a/mm/numa_memblks.c +++ b/mm/numa_memblks.c @@ -570,15 +570,16 @@ static int meminfo_to_nid(struct numa_meminfo *mi, u64 start) int phys_to_target_node(u64 start) { int nid = meminfo_to_nid(&numa_meminfo, start); + int reserved_nid = meminfo_to_nid(&numa_reserved_meminfo, start); /* - * Prefer online nodes, but if reserved memory might be - * hot-added continue the search with reserved ranges. + * Prefer online nodes unless the address is also described + * by reserved ranges, in which case use the reserved nid. */ - if (nid != NUMA_NO_NODE) + if (nid != NUMA_NO_NODE && reserved_nid == NUMA_NO_NODE) return nid; - return meminfo_to_nid(&numa_reserved_meminfo, start); + return reserved_nid; } EXPORT_SYMBOL_GPL(phys_to_target_node); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 623f6e5b583ab..d76f0f60f080d 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1339,8 +1339,8 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) #endif /* CONFIG_MEM_ALLOC_PROFILING */ -__always_inline bool free_pages_prepare(struct page *page, - unsigned int order) +__always_inline bool __free_pages_prepare(struct page *page, + unsigned int order, fpi_t fpi_flags) { int bad = 0; bool skip_kasan_poison = should_skip_kasan_poison(page); @@ -1429,11 +1429,12 @@ __always_inline bool free_pages_prepare(struct page *page, page_cpupid_reset_last(page); page->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP; + page->private = 0; reset_page_owner(page, order); page_table_check_free(page, order); pgalloc_tag_sub(page, 1 << order); - if (!PageHighMem(page)) { + if (!PageHighMem(page) && !(fpi_flags & FPI_TRYLOCK)) { debug_check_no_locks_freed(page_address(page), PAGE_SIZE << order); debug_check_no_obj_freed(page_address(page), @@ -1472,6 +1473,11 @@ __always_inline bool free_pages_prepare(struct page *page, return true; } +bool free_pages_prepare(struct page *page, unsigned int order) +{ + return __free_pages_prepare(page, order, FPI_NONE); +} + /* * Frees a number of pages from the PCP lists * Assumes all pages on list are in same zone. @@ -1605,7 +1611,7 @@ static void __free_pages_ok(struct page *page, unsigned int order, unsigned long pfn = page_to_pfn(page); struct zone *zone = page_zone(page); - if (free_pages_prepare(page, order)) + if (__free_pages_prepare(page, order, fpi_flags)) free_one_page(zone, page, pfn, order, fpi_flags); } @@ -2931,7 +2937,7 @@ static void __free_frozen_pages(struct page *page, unsigned int order, return; } - if (!free_pages_prepare(page, order)) + if (!__free_pages_prepare(page, order, fpi_flags)) return; /* @@ -2988,7 +2994,7 @@ void free_unref_folios(struct folio_batch *folios) unsigned long pfn = folio_pfn(folio); unsigned int order = folio_order(folio); - if (!free_pages_prepare(&folio->page, order)) + if (!__free_pages_prepare(&folio->page, order, FPI_NONE)) continue; /* * Free orders not handled on the PCP directly to the @@ -4789,6 +4795,20 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, compact_result == COMPACT_DEFERRED) goto nopage; + /* + * THP page faults may attempt local node only first, + * but are then allowed to only compact, not reclaim, + * see alloc_pages_mpol(). + * + * Compaction can fail for other reasons than those + * checked above and we don't want such THP allocations + * to put reclaim pressure on a single node in a + * situation where other nodes might have plenty of + * available memory. + */ + if (gfp_mask & __GFP_THISNODE) + goto nopage; + /* * Looks like reclaim/compaction is worth trying, but * sync compaction could be very expensive, so keep diff --git a/mm/rmap.c b/mm/rmap.c index d52055a026a05..f1e6a97cf4604 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -76,7 +76,7 @@ #include #include -#include +#include #define CREATE_TRACE_POINTS #include @@ -2019,13 +2019,17 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * if unsuccessful. */ if (!anon) { + struct mmu_gather tlb; + VM_BUG_ON(!(flags & TTU_RMAP_LOCKED)); if (!hugetlb_vma_trylock_write(vma)) goto walk_abort; - if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) { + + tlb_gather_mmu_vma(&tlb, vma); + if (huge_pmd_unshare(&tlb, vma, address, pvmw.pte)) { hugetlb_vma_unlock_write(vma); - flush_tlb_range(vma, - range.start, range.end); + huge_pmd_unshare_flush(&tlb, vma); + tlb_finish_mmu(&tlb); /* * The PMD table was unmapped, * consequently unmapping the folio. @@ -2033,6 +2037,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, goto walk_done; } hugetlb_vma_unlock_write(vma); + tlb_finish_mmu(&tlb); } pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); if (pte_dirty(pteval)) @@ -2398,17 +2403,20 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, * fail if unsuccessful. */ if (!anon) { + struct mmu_gather tlb; + VM_BUG_ON(!(flags & TTU_RMAP_LOCKED)); if (!hugetlb_vma_trylock_write(vma)) { page_vma_mapped_walk_done(&pvmw); ret = false; break; } - if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) { - hugetlb_vma_unlock_write(vma); - flush_tlb_range(vma, - range.start, range.end); + tlb_gather_mmu_vma(&tlb, vma); + if (huge_pmd_unshare(&tlb, vma, address, pvmw.pte)) { + hugetlb_vma_unlock_write(vma); + huge_pmd_unshare_flush(&tlb, vma); + tlb_finish_mmu(&tlb); /* * The PMD table was unmapped, * consequently unmapping the folio. @@ -2417,6 +2425,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, break; } hugetlb_vma_unlock_write(vma); + tlb_finish_mmu(&tlb); } /* Nuke the hugetlb page table entry */ pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); diff --git a/mm/slab.h b/mm/slab.h index bf9d8940b8f21..36893299fa67c 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -236,10 +236,8 @@ struct kmem_cache_order_objects { * Slab cache management. */ struct kmem_cache { -#ifndef CONFIG_SLUB_TINY struct kmem_cache_cpu __percpu *cpu_slab; struct lock_class_key lock_key; -#endif struct slub_percpu_sheaves __percpu *cpu_sheaves; /* Used for retrieving partial slabs, etc. */ slab_flags_t flags; diff --git a/mm/slab_common.c b/mm/slab_common.c index 87bde1d8916be..0f58265ca200e 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -2134,8 +2134,11 @@ EXPORT_SYMBOL_GPL(kvfree_rcu_barrier); */ void kvfree_rcu_barrier_on_cache(struct kmem_cache *s) { - if (s->cpu_sheaves) + if (s->cpu_sheaves) { flush_rcu_sheaves_on_cache(s); + rcu_barrier(); + } + /* * TODO: Introduce a version of __kvfree_rcu_barrier() that works * on a specific slab cache. diff --git a/mm/slub.c b/mm/slub.c index e01641cea143b..870b8e00a938b 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -410,7 +411,6 @@ enum stat_item { NR_SLUB_STAT_ITEMS }; -#ifndef CONFIG_SLUB_TINY /* * When changing the layout, make sure freelist and tid are still compatible * with this_cpu_cmpxchg_double() alignment requirements. @@ -432,7 +432,6 @@ struct kmem_cache_cpu { unsigned int stat[NR_SLUB_STAT_ITEMS]; #endif }; -#endif /* CONFIG_SLUB_TINY */ static inline void stat(const struct kmem_cache *s, enum stat_item si) { @@ -594,12 +593,10 @@ static inline void *get_freepointer(struct kmem_cache *s, void *object) return freelist_ptr_decode(s, p, ptr_addr); } -#ifndef CONFIG_SLUB_TINY static void prefetch_freepointer(const struct kmem_cache *s, void *object) { prefetchw(object + s->offset); } -#endif /* * When running under KMSAN, get_freepointer_safe() may return an uninitialized @@ -711,10 +708,12 @@ static inline unsigned int slub_get_cpu_partial(struct kmem_cache *s) return s->cpu_partial_slabs; } #else +#ifdef SLAB_SUPPORTS_SYSFS static inline void slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects) { } +#endif static inline unsigned int slub_get_cpu_partial(struct kmem_cache *s) { @@ -863,7 +862,7 @@ static inline bool slab_update_freelist(struct kmem_cache *s, struct slab *slab, * request size in the meta data area, for better debug and sanity check. */ static inline void set_orig_size(struct kmem_cache *s, - void *object, unsigned int orig_size) + void *object, unsigned long orig_size) { void *p = kasan_reset_tag(object); @@ -873,10 +872,10 @@ static inline void set_orig_size(struct kmem_cache *s, p += get_info_end(s); p += sizeof(struct track) * 2; - *(unsigned int *)p = orig_size; + *(unsigned long *)p = orig_size; } -static inline unsigned int get_orig_size(struct kmem_cache *s, void *object) +static inline unsigned long get_orig_size(struct kmem_cache *s, void *object) { void *p = kasan_reset_tag(object); @@ -889,7 +888,7 @@ static inline unsigned int get_orig_size(struct kmem_cache *s, void *object) p += get_info_end(s); p += sizeof(struct track) * 2; - return *(unsigned int *)p; + return *(unsigned long *)p; } #ifdef CONFIG_SLUB_DEBUG @@ -1204,7 +1203,7 @@ static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p) off += 2 * sizeof(struct track); if (slub_debug_orig_size(s)) - off += sizeof(unsigned int); + off += sizeof(unsigned long); off += kasan_metadata_size(s, false); @@ -1400,7 +1399,7 @@ static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p) off += 2 * sizeof(struct track); if (s->flags & SLAB_KMALLOC) - off += sizeof(unsigned int); + off += sizeof(unsigned long); } off += kasan_metadata_size(s, false); @@ -2023,13 +2022,11 @@ static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects) {} static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects) {} -#ifndef CONFIG_SLUB_TINY static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab, void **freelist, void *nextfree) { return false; } -#endif #endif /* CONFIG_SLUB_DEBUG */ #ifdef CONFIG_SLAB_OBJ_EXT @@ -2099,6 +2096,49 @@ static inline void init_slab_obj_exts(struct slab *slab) slab->obj_exts = 0; } +/* + * Calculate the allocation size for slabobj_ext array. + * + * When memory allocation profiling is enabled, the obj_exts array + * could be allocated from the same slab cache it's being allocated for. + * This would prevent the slab from ever being freed because it would + * always contain at least one allocated object (its own obj_exts array). + * + * To avoid this, increase the allocation size when we detect the array + * may come from the same cache, forcing it to use a different cache. + */ +static inline size_t obj_exts_alloc_size(struct kmem_cache *s, + struct slab *slab, gfp_t gfp) +{ + size_t sz = sizeof(struct slabobj_ext) * slab->objects; + struct kmem_cache *obj_exts_cache; + + /* + * slabobj_ext array for KMALLOC_CGROUP allocations + * are served from KMALLOC_NORMAL caches. + */ + if (!mem_alloc_profiling_enabled()) + return sz; + + if (sz > KMALLOC_MAX_CACHE_SIZE) + return sz; + + if (!is_kmalloc_normal(s)) + return sz; + + obj_exts_cache = kmalloc_slab(sz, NULL, gfp, 0); + /* + * We can't simply compare s with obj_exts_cache, because random kmalloc + * caches have multiple caches per size, selected by caller address. + * Since caller address may differ between kmalloc_slab() and actual + * allocation, bump size when sizes are equal. + */ + if (s->object_size == obj_exts_cache->object_size) + return obj_exts_cache->object_size + 1; + + return sz; +} + int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, gfp_t gfp, bool new_slab) { @@ -2107,26 +2147,26 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, unsigned long new_exts; unsigned long old_exts; struct slabobj_ext *vec; + size_t sz; gfp &= ~OBJCGS_CLEAR_MASK; /* Prevent recursive extension vector allocation */ gfp |= __GFP_NO_OBJ_EXT; + sz = obj_exts_alloc_size(s, slab, gfp); + /* * Note that allow_spin may be false during early boot and its * restricted GFP_BOOT_MASK. Due to kmalloc_nolock() only supporting * architectures with cmpxchg16b, early obj_exts will be missing for * very early allocations on those. */ - if (unlikely(!allow_spin)) { - size_t sz = objects * sizeof(struct slabobj_ext); - + if (unlikely(!allow_spin)) vec = kmalloc_nolock(sz, __GFP_ZERO | __GFP_NO_OBJ_EXT, slab_nid(slab)); - } else { - vec = kcalloc_node(objects, sizeof(struct slabobj_ext), gfp, - slab_nid(slab)); - } + else + vec = kmalloc_node(sz, gfp | __GFP_ZERO, slab_nid(slab)); + if (!vec) { /* * Try to mark vectors which failed to allocate. @@ -2140,6 +2180,9 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, return -ENOMEM; } + VM_WARN_ON_ONCE(virt_to_slab(vec) != NULL && + virt_to_slab(vec)->slab_cache == s); + new_exts = (unsigned long)vec; if (unlikely(!allow_spin)) new_exts |= OBJEXTS_NOSPIN_ALLOC; @@ -3134,8 +3177,11 @@ static void *next_freelist_entry(struct kmem_cache *s, return (char *)start + idx; } +static DEFINE_PER_CPU(struct rnd_state, slab_rnd_state); + /* Shuffle the single linked freelist based on a random pre-computed sequence */ -static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab) +static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab, + bool allow_spin) { void *start; void *cur; @@ -3146,7 +3192,19 @@ static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab) return false; freelist_count = oo_objects(s->oo); - pos = get_random_u32_below(freelist_count); + if (allow_spin) { + pos = get_random_u32_below(freelist_count); + } else { + struct rnd_state *state; + + /* + * An interrupt or NMI handler might interrupt and change + * the state in the middle, but that's safe. + */ + state = &get_cpu_var(slab_rnd_state); + pos = prandom_u32_state(state) % freelist_count; + put_cpu_var(slab_rnd_state); + } page_limit = slab->objects * s->size; start = fixup_red_left(s, slab_address(slab)); @@ -3173,7 +3231,8 @@ static inline int init_cache_random_seq(struct kmem_cache *s) return 0; } static inline void init_freelist_randomization(void) { } -static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab) +static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab, + bool allow_spin) { return false; } @@ -3258,7 +3317,7 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) setup_slab_debug(s, slab, start); - shuffle = shuffle_freelist(s, slab); + shuffle = shuffle_freelist(s, slab, allow_spin); if (!shuffle) { start = fixup_red_left(s, start); @@ -3548,6 +3607,7 @@ static struct slab *get_any_partial(struct kmem_cache *s, enum zone_type highest_zoneidx = gfp_zone(pc->flags); struct slab *slab; unsigned int cpuset_mems_cookie; + bool allow_spin = gfpflags_allow_spinning(pc->flags); /* * The defrag ratio allows a configuration of the tradeoffs between @@ -3572,7 +3632,15 @@ static struct slab *get_any_partial(struct kmem_cache *s, return NULL; do { - cpuset_mems_cookie = read_mems_allowed_begin(); + /* + * read_mems_allowed_begin() accesses current->mems_allowed_seq, + * a seqcount_spinlock_t that is not NMI-safe. Do not access + * current->mems_allowed_seq and avoid retry when GFP flags + * indicate spinning is not allowed. + */ + if (allow_spin) + cpuset_mems_cookie = read_mems_allowed_begin(); + zonelist = node_zonelist(mempolicy_slab_node(), pc->flags); for_each_zone_zonelist(zone, z, zonelist, highest_zoneidx) { struct kmem_cache_node *n; @@ -3594,7 +3662,7 @@ static struct slab *get_any_partial(struct kmem_cache *s, } } } - } while (read_mems_allowed_retry(cpuset_mems_cookie)); + } while (allow_spin && read_mems_allowed_retry(cpuset_mems_cookie)); #endif /* CONFIG_NUMA */ return NULL; } @@ -3618,8 +3686,6 @@ static struct slab *get_partial(struct kmem_cache *s, int node, return get_any_partial(s, pc); } -#ifndef CONFIG_SLUB_TINY - #ifdef CONFIG_PREEMPTION /* * Calculate the next globally unique transaction for disambiguation @@ -4019,12 +4085,6 @@ static bool has_cpu_slab(int cpu, struct kmem_cache *s) return c->slab || slub_percpu_partial(c); } -#else /* CONFIG_SLUB_TINY */ -static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) { } -static inline bool has_cpu_slab(int cpu, struct kmem_cache *s) { return false; } -static inline void flush_this_cpu_slab(struct kmem_cache *s) { } -#endif /* CONFIG_SLUB_TINY */ - static bool has_pcs_used(int cpu, struct kmem_cache *s) { struct slub_percpu_sheaves *pcs; @@ -4370,7 +4430,6 @@ static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags) return true; } -#ifndef CONFIG_SLUB_TINY static inline bool __update_cpu_freelist_fast(struct kmem_cache *s, void *freelist_old, void *freelist_new, @@ -4634,7 +4693,7 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, pc.orig_size = orig_size; slab = get_partial(s, node, &pc); if (slab) { - if (kmem_cache_debug(s)) { + if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { freelist = pc.object; /* * For debug caches here we had to go through @@ -4672,7 +4731,7 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, stat(s, ALLOC_SLAB); - if (kmem_cache_debug(s)) { + if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { freelist = alloc_single_from_new_slab(s, slab, orig_size, gfpflags); if (unlikely(!freelist)) { @@ -4884,32 +4943,6 @@ static __always_inline void *__slab_alloc_node(struct kmem_cache *s, return object; } -#else /* CONFIG_SLUB_TINY */ -static void *__slab_alloc_node(struct kmem_cache *s, - gfp_t gfpflags, int node, unsigned long addr, size_t orig_size) -{ - struct partial_context pc; - struct slab *slab; - void *object; - - pc.flags = gfpflags; - pc.orig_size = orig_size; - slab = get_partial(s, node, &pc); - - if (slab) - return pc.object; - - slab = new_slab(s, gfpflags, node); - if (unlikely(!slab)) { - slab_out_of_memory(s, gfpflags, node); - return NULL; - } - - object = alloc_single_from_new_slab(s, slab, orig_size, gfpflags); - - return object; -} -#endif /* CONFIG_SLUB_TINY */ /* * If the object has been wiped upon free, make sure it's fully initialized by @@ -5732,9 +5765,7 @@ void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node) * it did local_lock_irqsave(&s->cpu_slab->lock, flags). * In this case fast path with __update_cpu_freelist_fast() is not safe. */ -#ifndef CONFIG_SLUB_TINY if (!in_nmi() || !local_lock_is_locked(&s->cpu_slab->lock)) -#endif ret = __slab_alloc_node(s, alloc_gfp, node, _RET_IP_, size); if (PTR_ERR(ret) == -EBUSY) { @@ -6232,11 +6263,29 @@ static void rcu_free_sheaf(struct rcu_head *head) free_empty_sheaf(s, sheaf); } +/* + * kvfree_call_rcu() can be called while holding a raw_spinlock_t. Since + * __kfree_rcu_sheaf() may acquire a spinlock_t (sleeping lock on PREEMPT_RT), + * this would violate lock nesting rules. Therefore, kvfree_call_rcu() avoids + * this problem by bypassing the sheaves layer entirely on PREEMPT_RT. + * + * However, lockdep still complains that it is invalid to acquire spinlock_t + * while holding raw_spinlock_t, even on !PREEMPT_RT where spinlock_t is a + * spinning lock. Tell lockdep that acquiring spinlock_t is valid here + * by temporarily raising the wait-type to LD_WAIT_CONFIG. + */ +static DEFINE_WAIT_OVERRIDE_MAP(kfree_rcu_sheaf_map, LD_WAIT_CONFIG); + bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) { struct slub_percpu_sheaves *pcs; struct slab_sheaf *rcu_sheaf; + if (WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT))) + return false; + + lock_map_acquire_try(&kfree_rcu_sheaf_map); + if (!local_trylock(&s->cpu_sheaves->lock)) goto fail; @@ -6313,10 +6362,12 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) local_unlock(&s->cpu_sheaves->lock); stat(s, FREE_RCU_SHEAF); + lock_map_release(&kfree_rcu_sheaf_map); return true; fail: stat(s, FREE_RCU_SHEAF_FAIL); + lock_map_release(&kfree_rcu_sheaf_map); return false; } @@ -6496,14 +6547,10 @@ static void free_deferred_objects(struct irq_work *work) llist_for_each_safe(pos, t, llnode) { struct slab *slab = container_of(pos, struct slab, llnode); -#ifdef CONFIG_SLUB_TINY - free_slab(slab->slab_cache, slab); -#else if (slab->frozen) deactivate_slab(slab->slab_cache, slab, slab->flush_freelist); else free_slab(slab->slab_cache, slab); -#endif } } @@ -6541,7 +6588,6 @@ void defer_free_barrier(void) irq_work_sync(&per_cpu_ptr(&defer_free_objects, cpu)->work); } -#ifndef CONFIG_SLUB_TINY /* * Fastpath with forced inlining to produce a kfree and kmem_cache_free that * can perform fastpath freeing without additional function calls. @@ -6634,14 +6680,6 @@ static __always_inline void do_slab_free(struct kmem_cache *s, } stat_add(s, FREE_FASTPATH, cnt); } -#else /* CONFIG_SLUB_TINY */ -static void do_slab_free(struct kmem_cache *s, - struct slab *slab, void *head, void *tail, - int cnt, unsigned long addr) -{ - __slab_free(s, slab, head, tail, cnt, addr); -} -#endif /* CONFIG_SLUB_TINY */ static __fastpath_inline void slab_free(struct kmem_cache *s, struct slab *slab, void *object, @@ -6922,11 +6960,7 @@ void kfree_nolock(const void *object) * since kasan quarantine takes locks and not supported from NMI. */ kasan_slab_free(s, x, false, false, /* skip quarantine */true); -#ifndef CONFIG_SLUB_TINY do_slab_free(s, slab, x, x, 0, _RET_IP_); -#else - defer_free(s, x); -#endif } EXPORT_SYMBOL_GPL(kfree_nolock); @@ -7376,7 +7410,6 @@ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) } EXPORT_SYMBOL(kmem_cache_free_bulk); -#ifndef CONFIG_SLUB_TINY static inline int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, void **p) @@ -7447,35 +7480,6 @@ int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, return 0; } -#else /* CONFIG_SLUB_TINY */ -static int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, - size_t size, void **p) -{ - int i; - - for (i = 0; i < size; i++) { - void *object = kfence_alloc(s, s->object_size, flags); - - if (unlikely(object)) { - p[i] = object; - continue; - } - - p[i] = __slab_alloc_node(s, flags, NUMA_NO_NODE, - _RET_IP_, s->object_size); - if (unlikely(!p[i])) - goto error; - - maybe_wipe_obj_freeptr(s, p[i]); - } - - return i; - -error: - __kmem_cache_free_bulk(s, i, p); - return 0; -} -#endif /* CONFIG_SLUB_TINY */ /* Note that interrupts must be enabled when calling this function. */ int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, @@ -7666,7 +7670,6 @@ init_kmem_cache_node(struct kmem_cache_node *n, struct node_barn *barn) barn_init(barn); } -#ifndef CONFIG_SLUB_TINY static inline int alloc_kmem_cache_cpus(struct kmem_cache *s) { BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE < @@ -7687,12 +7690,6 @@ static inline int alloc_kmem_cache_cpus(struct kmem_cache *s) return 1; } -#else -static inline int alloc_kmem_cache_cpus(struct kmem_cache *s) -{ - return 1; -} -#endif /* CONFIG_SLUB_TINY */ static int init_percpu_sheaves(struct kmem_cache *s) { @@ -7782,13 +7779,11 @@ void __kmem_cache_release(struct kmem_cache *s) cache_random_seq_destroy(s); if (s->cpu_sheaves) pcs_destroy(s); -#ifndef CONFIG_SLUB_TINY #ifdef CONFIG_PREEMPT_RT if (s->cpu_slab) lockdep_unregister_key(&s->lock_key); #endif free_percpu(s->cpu_slab); -#endif free_kmem_cache_nodes(s); } @@ -7947,7 +7942,7 @@ static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s) /* Save the original kmalloc request size */ if (flags & SLAB_KMALLOC) - size += sizeof(unsigned int); + size += sizeof(unsigned long); } #endif @@ -8531,9 +8526,10 @@ void __init kmem_cache_init(void) void __init kmem_cache_init_late(void) { -#ifndef CONFIG_SLUB_TINY flushwq = alloc_workqueue("slub_flushwq", WQ_MEM_RECLAIM, 0); WARN_ON(!flushwq); +#ifdef CONFIG_SLAB_FREELIST_RANDOM + prandom_init_once(&slab_rnd_state); #endif } diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 4fbd6e7dc479a..e2f526ad7abba 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -2241,11 +2241,14 @@ decay_va_pool_node(struct vmap_node *vn, bool full_decay) reclaim_list_global(&decay_list); } +#define KASAN_RELEASE_BATCH_SIZE 32 + static void kasan_release_vmalloc_node(struct vmap_node *vn) { struct vmap_area *va; unsigned long start, end; + unsigned int batch_count = 0; start = list_first_entry(&vn->purge_list, struct vmap_area, list)->va_start; end = list_last_entry(&vn->purge_list, struct vmap_area, list)->va_end; @@ -2255,6 +2258,11 @@ kasan_release_vmalloc_node(struct vmap_node *vn) kasan_release_vmalloc(va->va_start, va->va_end, va->va_start, va->va_end, KASAN_VMALLOC_PAGE_RANGE); + + if (need_resched() || (++batch_count >= KASAN_RELEASE_BATCH_SIZE)) { + cond_resched(); + batch_count = 0; + } } kasan_release_vmalloc(start, end, start, end, KASAN_VMALLOC_TLB_FLUSH); diff --git a/mm/vmscan.c b/mm/vmscan.c index 06071995dacc9..05d9354a59c65 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -344,19 +344,21 @@ static void flush_reclaim_state(struct scan_control *sc) static bool can_demote(int nid, struct scan_control *sc, struct mem_cgroup *memcg) { - int demotion_nid; + struct pglist_data *pgdat = NODE_DATA(nid); + nodemask_t allowed_mask; - if (!numa_demotion_enabled) + if (!pgdat || !numa_demotion_enabled) return false; if (sc && sc->no_demotion) return false; - demotion_nid = next_demotion_node(nid); - if (demotion_nid == NUMA_NO_NODE) + node_get_allowed_targets(pgdat, &allowed_mask); + if (nodes_empty(allowed_mask)) return false; - /* If demotion node isn't in the cgroup's mems_allowed, fall back */ - return mem_cgroup_node_allowed(memcg, demotion_nid); + /* Filter out nodes that are not in cgroup's mems_allowed. */ + mem_cgroup_node_filter_allowed(memcg, &allowed_mask); + return !nodes_empty(allowed_mask); } static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg, @@ -1042,9 +1044,10 @@ static struct folio *alloc_demote_folio(struct folio *src, * Folios which are not demoted are left on @demote_folios. */ static unsigned int demote_folio_list(struct list_head *demote_folios, - struct pglist_data *pgdat) + struct pglist_data *pgdat, + struct mem_cgroup *memcg) { - int target_nid = next_demotion_node(pgdat->node_id); + int target_nid; unsigned int nr_succeeded; nodemask_t allowed_mask; @@ -1056,7 +1059,6 @@ static unsigned int demote_folio_list(struct list_head *demote_folios, */ .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | __GFP_NOWARN | __GFP_NOMEMALLOC | GFP_NOWAIT, - .nid = target_nid, .nmask = &allowed_mask, .reason = MR_DEMOTION, }; @@ -1064,10 +1066,18 @@ static unsigned int demote_folio_list(struct list_head *demote_folios, if (list_empty(demote_folios)) return 0; - if (target_nid == NUMA_NO_NODE) + node_get_allowed_targets(pgdat, &allowed_mask); + mem_cgroup_node_filter_allowed(memcg, &allowed_mask); + if (nodes_empty(allowed_mask)) return 0; - node_get_allowed_targets(pgdat, &allowed_mask); + target_nid = next_demotion_node(pgdat->node_id); + if (target_nid == NUMA_NO_NODE) + /* No lower-tier nodes or nodes were hot-unplugged. */ + return 0; + if (!node_isset(target_nid, allowed_mask)) + target_nid = node_random(&allowed_mask); + mtc.nid = target_nid; /* Demotion ignores all cpuset and mempolicy settings */ migrate_pages(demote_folios, alloc_demote_folio, NULL, @@ -1601,7 +1611,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, /* 'folio_list' is always empty here */ /* Migrate folios selected for demotion */ - nr_demoted = demote_folio_list(&demote_folios, pgdat); + nr_demoted = demote_folio_list(&demote_folios, pgdat, memcg); nr_reclaimed += nr_demoted; stat->nr_demoted += nr_demoted; /* Folios that could not be demoted are still in @demote_folios */ diff --git a/net/9p/trans_xen.c b/net/9p/trans_xen.c index b9ff69c7522a1..068d57515dd58 100644 --- a/net/9p/trans_xen.c +++ b/net/9p/trans_xen.c @@ -274,45 +274,52 @@ static void xen_9pfs_front_free(struct xen_9pfs_front_priv *priv) { int i, j; - write_lock(&xen_9pfs_lock); - list_del(&priv->list); - write_unlock(&xen_9pfs_lock); - - for (i = 0; i < XEN_9PFS_NUM_RINGS; i++) { - struct xen_9pfs_dataring *ring = &priv->rings[i]; - - cancel_work_sync(&ring->work); - - if (!priv->rings[i].intf) - break; - if (priv->rings[i].irq > 0) - unbind_from_irqhandler(priv->rings[i].irq, ring); - if (priv->rings[i].data.in) { - for (j = 0; - j < (1 << priv->rings[i].intf->ring_order); - j++) { - grant_ref_t ref; - - ref = priv->rings[i].intf->ref[j]; - gnttab_end_foreign_access(ref, NULL); - } - free_pages_exact(priv->rings[i].data.in, + if (priv->rings) { + for (i = 0; i < XEN_9PFS_NUM_RINGS; i++) { + struct xen_9pfs_dataring *ring = &priv->rings[i]; + + cancel_work_sync(&ring->work); + + if (!priv->rings[i].intf) + break; + if (priv->rings[i].irq > 0) + unbind_from_irqhandler(priv->rings[i].irq, ring); + if (priv->rings[i].data.in) { + for (j = 0; + j < (1 << priv->rings[i].intf->ring_order); + j++) { + grant_ref_t ref; + + ref = priv->rings[i].intf->ref[j]; + gnttab_end_foreign_access(ref, NULL); + } + free_pages_exact(priv->rings[i].data.in, 1UL << (priv->rings[i].intf->ring_order + XEN_PAGE_SHIFT)); + } + gnttab_end_foreign_access(priv->rings[i].ref, NULL); + free_page((unsigned long)priv->rings[i].intf); } - gnttab_end_foreign_access(priv->rings[i].ref, NULL); - free_page((unsigned long)priv->rings[i].intf); + kfree(priv->rings); } - kfree(priv->rings); kfree(priv->tag); kfree(priv); } static void xen_9pfs_front_remove(struct xenbus_device *dev) { - struct xen_9pfs_front_priv *priv = dev_get_drvdata(&dev->dev); + struct xen_9pfs_front_priv *priv; + write_lock(&xen_9pfs_lock); + priv = dev_get_drvdata(&dev->dev); + if (priv == NULL) { + write_unlock(&xen_9pfs_lock); + return; + } dev_set_drvdata(&dev->dev, NULL); + list_del(&priv->list); + write_unlock(&xen_9pfs_lock); + xen_9pfs_front_free(priv); } @@ -379,7 +386,7 @@ static int xen_9pfs_front_init(struct xenbus_device *dev) { int ret, i; struct xenbus_transaction xbt; - struct xen_9pfs_front_priv *priv = dev_get_drvdata(&dev->dev); + struct xen_9pfs_front_priv *priv; char *versions, *v; unsigned int max_rings, max_ring_order, len = 0; @@ -407,6 +414,10 @@ static int xen_9pfs_front_init(struct xenbus_device *dev) if (p9_xen_trans.maxsize > XEN_FLEX_RING_SIZE(max_ring_order)) p9_xen_trans.maxsize = XEN_FLEX_RING_SIZE(max_ring_order) / 2; + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + priv->dev = dev; priv->rings = kcalloc(XEN_9PFS_NUM_RINGS, sizeof(*priv->rings), GFP_KERNEL); if (!priv->rings) { @@ -465,6 +476,11 @@ static int xen_9pfs_front_init(struct xenbus_device *dev) goto error; } + write_lock(&xen_9pfs_lock); + dev_set_drvdata(&dev->dev, priv); + list_add_tail(&priv->list, &xen_9pfs_devs); + write_unlock(&xen_9pfs_lock); + xenbus_switch_state(dev, XenbusStateInitialised); return 0; @@ -479,19 +495,6 @@ static int xen_9pfs_front_init(struct xenbus_device *dev) static int xen_9pfs_front_probe(struct xenbus_device *dev, const struct xenbus_device_id *id) { - struct xen_9pfs_front_priv *priv = NULL; - - priv = kzalloc(sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; - - priv->dev = dev; - dev_set_drvdata(&dev->dev, priv); - - write_lock(&xen_9pfs_lock); - list_add_tail(&priv->list, &xen_9pfs_devs); - write_unlock(&xen_9pfs_lock); - return 0; } diff --git a/net/atm/lec.c b/net/atm/lec.c index afb8d3eb21850..c39dc5d367979 100644 --- a/net/atm/lec.c +++ b/net/atm/lec.c @@ -1260,24 +1260,28 @@ static void lec_arp_clear_vccs(struct lec_arp_table *entry) struct lec_vcc_priv *vpriv = LEC_VCC_PRIV(vcc); struct net_device *dev = (struct net_device *)vcc->proto_data; - vcc->pop = vpriv->old_pop; - if (vpriv->xoff) - netif_wake_queue(dev); - kfree(vpriv); - vcc->user_back = NULL; - vcc->push = entry->old_push; - vcc_release_async(vcc, -EPIPE); + if (vpriv) { + vcc->pop = vpriv->old_pop; + if (vpriv->xoff) + netif_wake_queue(dev); + kfree(vpriv); + vcc->user_back = NULL; + vcc->push = entry->old_push; + vcc_release_async(vcc, -EPIPE); + } entry->vcc = NULL; } if (entry->recv_vcc) { struct atm_vcc *vcc = entry->recv_vcc; struct lec_vcc_priv *vpriv = LEC_VCC_PRIV(vcc); - kfree(vpriv); - vcc->user_back = NULL; + if (vpriv) { + kfree(vpriv); + vcc->user_back = NULL; - entry->recv_vcc->push = entry->old_recv_push; - vcc_release_async(entry->recv_vcc, -EPIPE); + entry->recv_vcc->push = entry->old_recv_push; + vcc_release_async(entry->recv_vcc, -EPIPE); + } entry->recv_vcc = NULL; } } diff --git a/net/atm/signaling.c b/net/atm/signaling.c index e70ae2c113f95..358fbe5e4d1d0 100644 --- a/net/atm/signaling.c +++ b/net/atm/signaling.c @@ -22,6 +22,36 @@ struct atm_vcc *sigd = NULL; +/* + * find_get_vcc - validate and get a reference to a vcc pointer + * @vcc: the vcc pointer to validate + * + * This function validates that @vcc points to a registered VCC in vcc_hash. + * If found, it increments the socket reference count and returns the vcc. + * The caller must call sock_put(sk_atm(vcc)) when done. + * + * Returns the vcc pointer if valid, NULL otherwise. + */ +static struct atm_vcc *find_get_vcc(struct atm_vcc *vcc) +{ + int i; + + read_lock(&vcc_sklist_lock); + for (i = 0; i < VCC_HTABLE_SIZE; i++) { + struct sock *s; + + sk_for_each(s, &vcc_hash[i]) { + if (atm_sk(s) == vcc) { + sock_hold(s); + read_unlock(&vcc_sklist_lock); + return vcc; + } + } + } + read_unlock(&vcc_sklist_lock); + return NULL; +} + static void sigd_put_skb(struct sk_buff *skb) { if (!sigd) { @@ -69,7 +99,14 @@ static int sigd_send(struct atm_vcc *vcc, struct sk_buff *skb) msg = (struct atmsvc_msg *) skb->data; WARN_ON(refcount_sub_and_test(skb->truesize, &sk_atm(vcc)->sk_wmem_alloc)); - vcc = *(struct atm_vcc **) &msg->vcc; + + vcc = find_get_vcc(*(struct atm_vcc **)&msg->vcc); + if (!vcc) { + pr_debug("invalid vcc pointer in msg\n"); + dev_kfree_skb(skb); + return -EINVAL; + } + pr_debug("%d (0x%lx)\n", (int)msg->type, (unsigned long)vcc); sk = sk_atm(vcc); @@ -100,7 +137,16 @@ static int sigd_send(struct atm_vcc *vcc, struct sk_buff *skb) clear_bit(ATM_VF_WAITING, &vcc->flags); break; case as_indicate: - vcc = *(struct atm_vcc **)&msg->listen_vcc; + /* Release the reference from msg->vcc, we'll use msg->listen_vcc instead */ + sock_put(sk); + + vcc = find_get_vcc(*(struct atm_vcc **)&msg->listen_vcc); + if (!vcc) { + pr_debug("invalid listen_vcc pointer in msg\n"); + dev_kfree_skb(skb); + return -EINVAL; + } + sk = sk_atm(vcc); pr_debug("as_indicate!!!\n"); lock_sock(sk); @@ -115,6 +161,8 @@ static int sigd_send(struct atm_vcc *vcc, struct sk_buff *skb) sk->sk_state_change(sk); as_indicate_complete: release_sock(sk); + /* Paired with find_get_vcc(msg->listen_vcc) above */ + sock_put(sk); return 0; case as_close: set_bit(ATM_VF_RELEASED, &vcc->flags); @@ -131,11 +179,15 @@ static int sigd_send(struct atm_vcc *vcc, struct sk_buff *skb) break; default: pr_alert("bad message type %d\n", (int)msg->type); + /* Paired with find_get_vcc(msg->vcc) above */ + sock_put(sk); return -EINVAL; } sk->sk_state_change(sk); out: dev_kfree_skb(skb); + /* Paired with find_get_vcc(msg->vcc) above */ + sock_put(sk); return 0; } diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c index 6fc0692abf057..6a27ac5a751ca 100644 --- a/net/bluetooth/hci_conn.c +++ b/net/bluetooth/hci_conn.c @@ -990,6 +990,7 @@ static struct hci_conn *__hci_conn_add(struct hci_dev *hdev, int type, bdaddr_t switch (type) { case ACL_LINK: conn->pkt_type = hdev->pkt_type & ACL_PTYPE_MASK; + conn->link_policy = hdev->link_policy; conn->mtu = hdev->acl_mtu; break; case LE_LINK: @@ -2591,8 +2592,8 @@ void hci_conn_enter_active_mode(struct hci_conn *conn, __u8 force_active) timer: if (hdev->idle_timeout > 0) - queue_delayed_work(hdev->workqueue, &conn->idle_work, - msecs_to_jiffies(hdev->idle_timeout)); + mod_delayed_work(hdev->workqueue, &conn->idle_work, + msecs_to_jiffies(hdev->idle_timeout)); } /* Drop all connection on the device */ diff --git a/net/bluetooth/hci_sock.c b/net/bluetooth/hci_sock.c index ad19022ae127a..5295ff001e38c 100644 --- a/net/bluetooth/hci_sock.c +++ b/net/bluetooth/hci_sock.c @@ -2166,6 +2166,7 @@ static void hci_sock_destruct(struct sock *sk) mgmt_cleanup(sk); skb_queue_purge(&sk->sk_receive_queue); skb_queue_purge(&sk->sk_write_queue); + skb_queue_purge(&sk->sk_error_queue); } static const struct proto_ops hci_sock_ops = { diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c index f5896c023a9fa..9f01837250a5e 100644 --- a/net/bluetooth/hci_sync.c +++ b/net/bluetooth/hci_sync.c @@ -4546,7 +4546,7 @@ static int hci_le_set_host_feature_sync(struct hci_dev *hdev) { struct hci_cp_le_set_host_feature cp; - if (!iso_capable(hdev)) + if (!cis_capable(hdev)) return 0; memset(&cp, 0, sizeof(cp)); @@ -6879,8 +6879,6 @@ static int hci_acl_create_conn_sync(struct hci_dev *hdev, void *data) conn->attempt++; - conn->link_policy = hdev->link_policy; - memset(&cp, 0, sizeof(cp)); bacpy(&cp.bdaddr, &conn->dst); cp.pscan_rep_mode = 0x02; diff --git a/net/bluetooth/iso.c b/net/bluetooth/iso.c index 616c2fef91d24..46ebd69026fe2 100644 --- a/net/bluetooth/iso.c +++ b/net/bluetooth/iso.c @@ -728,6 +728,7 @@ static void iso_sock_destruct(struct sock *sk) skb_queue_purge(&sk->sk_receive_queue); skb_queue_purge(&sk->sk_write_queue); + skb_queue_purge(&sk->sk_error_queue); } static void iso_sock_cleanup_listen(struct sock *parent) diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index 07b493331fd78..72a4bb1fee46a 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -4900,6 +4900,13 @@ static int l2cap_le_connect_req(struct l2cap_conn *conn, goto response_unlock; } + /* Check if Key Size is sufficient for the security level */ + if (!l2cap_check_enc_key_size(conn->hcon, pchan)) { + result = L2CAP_CR_LE_BAD_KEY_SIZE; + chan = NULL; + goto response_unlock; + } + /* Check for valid dynamic CID range */ if (scid < L2CAP_CID_DYN_START || scid > L2CAP_CID_LE_DYN_END) { result = L2CAP_CR_LE_INVALID_SCID; @@ -5035,13 +5042,15 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, struct l2cap_chan *chan, *pchan; u16 mtu, mps; __le16 psm; - u8 result, len = 0; + u8 result, rsp_len = 0; int i, num_scid; bool defer = false; if (!enable_ecred) return -EINVAL; + memset(pdu, 0, sizeof(*pdu)); + if (cmd_len < sizeof(*req) || (cmd_len - sizeof(*req)) % sizeof(u16)) { result = L2CAP_CR_LE_INVALID_PARAMS; goto response; @@ -5050,6 +5059,9 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, cmd_len -= sizeof(*req); num_scid = cmd_len / sizeof(u16); + /* Always respond with the same number of scids as in the request */ + rsp_len = cmd_len; + if (num_scid > L2CAP_ECRED_MAX_CID) { result = L2CAP_CR_LE_INVALID_PARAMS; goto response; @@ -5059,7 +5071,7 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, mps = __le16_to_cpu(req->mps); if (mtu < L2CAP_ECRED_MIN_MTU || mps < L2CAP_ECRED_MIN_MPS) { - result = L2CAP_CR_LE_UNACCEPT_PARAMS; + result = L2CAP_CR_LE_INVALID_PARAMS; goto response; } @@ -5079,8 +5091,6 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, BT_DBG("psm 0x%2.2x mtu %u mps %u", __le16_to_cpu(psm), mtu, mps); - memset(pdu, 0, sizeof(*pdu)); - /* Check if we have socket listening on psm */ pchan = l2cap_global_chan_by_psm(BT_LISTEN, psm, &conn->hcon->src, &conn->hcon->dst, LE_LINK); @@ -5093,7 +5103,16 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, if (!smp_sufficient_security(conn->hcon, pchan->sec_level, SMP_ALLOW_STK)) { - result = L2CAP_CR_LE_AUTHENTICATION; + result = pchan->sec_level == BT_SECURITY_MEDIUM ? + L2CAP_CR_LE_ENCRYPTION : L2CAP_CR_LE_AUTHENTICATION; + goto unlock; + } + + /* Check if the listening channel has set an output MTU then the + * requested MTU shall be less than or equal to that value. + */ + if (pchan->omtu && mtu < pchan->omtu) { + result = L2CAP_CR_LE_UNACCEPT_PARAMS; goto unlock; } @@ -5105,7 +5124,6 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, BT_DBG("scid[%d] 0x%4.4x", i, scid); pdu->dcid[i] = 0x0000; - len += sizeof(*pdu->dcid); /* Check for valid dynamic CID range */ if (scid < L2CAP_CID_DYN_START || scid > L2CAP_CID_LE_DYN_END) { @@ -5172,7 +5190,7 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, return 0; l2cap_send_cmd(conn, cmd->ident, L2CAP_ECRED_CONN_RSP, - sizeof(*pdu) + len, pdu); + sizeof(*pdu) + rsp_len, pdu); return 0; } @@ -5294,14 +5312,14 @@ static inline int l2cap_ecred_reconf_req(struct l2cap_conn *conn, struct l2cap_ecred_reconf_req *req = (void *) data; struct l2cap_ecred_reconf_rsp rsp; u16 mtu, mps, result; - struct l2cap_chan *chan; + struct l2cap_chan *chan[L2CAP_ECRED_MAX_CID] = {}; int i, num_scid; if (!enable_ecred) return -EINVAL; - if (cmd_len < sizeof(*req) || cmd_len - sizeof(*req) % sizeof(u16)) { - result = L2CAP_CR_LE_INVALID_PARAMS; + if (cmd_len < sizeof(*req) || (cmd_len - sizeof(*req)) % sizeof(u16)) { + result = L2CAP_RECONF_INVALID_CID; goto respond; } @@ -5311,42 +5329,69 @@ static inline int l2cap_ecred_reconf_req(struct l2cap_conn *conn, BT_DBG("mtu %u mps %u", mtu, mps); if (mtu < L2CAP_ECRED_MIN_MTU) { - result = L2CAP_RECONF_INVALID_MTU; + result = L2CAP_RECONF_INVALID_PARAMS; goto respond; } if (mps < L2CAP_ECRED_MIN_MPS) { - result = L2CAP_RECONF_INVALID_MPS; + result = L2CAP_RECONF_INVALID_PARAMS; goto respond; } cmd_len -= sizeof(*req); num_scid = cmd_len / sizeof(u16); + + if (num_scid > L2CAP_ECRED_MAX_CID) { + result = L2CAP_RECONF_INVALID_PARAMS; + goto respond; + } + result = L2CAP_RECONF_SUCCESS; + /* Check if each SCID, MTU and MPS are valid */ for (i = 0; i < num_scid; i++) { u16 scid; scid = __le16_to_cpu(req->scid[i]); - if (!scid) - return -EPROTO; + if (!scid) { + result = L2CAP_RECONF_INVALID_CID; + goto respond; + } - chan = __l2cap_get_chan_by_dcid(conn, scid); - if (!chan) - continue; + chan[i] = __l2cap_get_chan_by_dcid(conn, scid); + if (!chan[i]) { + result = L2CAP_RECONF_INVALID_CID; + goto respond; + } - /* If the MTU value is decreased for any of the included - * channels, then the receiver shall disconnect all - * included channels. + /* The MTU field shall be greater than or equal to the greatest + * current MTU size of these channels. */ - if (chan->omtu > mtu) { - BT_ERR("chan %p decreased MTU %u -> %u", chan, - chan->omtu, mtu); + if (chan[i]->omtu > mtu) { + BT_ERR("chan %p decreased MTU %u -> %u", chan[i], + chan[i]->omtu, mtu); result = L2CAP_RECONF_INVALID_MTU; + goto respond; } - chan->omtu = mtu; - chan->remote_mps = mps; + /* If more than one channel is being configured, the MPS field + * shall be greater than or equal to the current MPS size of + * each of these channels. If only one channel is being + * configured, the MPS field may be less than the current MPS + * of that channel. + */ + if (chan[i]->remote_mps >= mps && i) { + BT_ERR("chan %p decreased MPS %u -> %u", chan[i], + chan[i]->remote_mps, mps); + result = L2CAP_RECONF_INVALID_MPS; + goto respond; + } + } + + /* Commit the new MTU and MPS values after checking they are valid */ + for (i = 0; i < num_scid; i++) { + chan[i]->omtu = mtu; + chan[i]->remote_mps = mps; } respond: diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c index 814fb8610ac43..ab86aeef98d18 100644 --- a/net/bluetooth/l2cap_sock.c +++ b/net/bluetooth/l2cap_sock.c @@ -1029,10 +1029,17 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, break; } - /* Setting is not supported as it's the remote side that - * decides this. - */ - err = -EPERM; + /* Only allow setting output MTU when not connected */ + if (sk->sk_state == BT_CONNECTED) { + err = -EISCONN; + break; + } + + err = copy_safe_from_sockptr(&mtu, sizeof(mtu), optval, optlen); + if (err) + break; + + chan->omtu = mtu; break; case BT_RCVMTU: @@ -1799,6 +1806,7 @@ static void l2cap_sock_destruct(struct sock *sk) skb_queue_purge(&sk->sk_receive_queue); skb_queue_purge(&sk->sk_write_queue); + skb_queue_purge(&sk->sk_error_queue); } static void l2cap_skb_msg_name(struct sk_buff *skb, void *msg_name, diff --git a/net/bluetooth/sco.c b/net/bluetooth/sco.c index 298c2a9ab4df8..49a47eaa674da 100644 --- a/net/bluetooth/sco.c +++ b/net/bluetooth/sco.c @@ -470,6 +470,7 @@ static void sco_sock_destruct(struct sock *sk) skb_queue_purge(&sk->sk_receive_queue); skb_queue_purge(&sk->sk_write_queue); + skb_queue_purge(&sk->sk_error_queue); } static void sco_sock_cleanup_listen(struct sock *parent) diff --git a/net/bridge/br_device.c b/net/bridge/br_device.c index a818fdc22da9a..525d4eccd194a 100644 --- a/net/bridge/br_device.c +++ b/net/bridge/br_device.c @@ -74,7 +74,7 @@ netdev_tx_t br_dev_xmit(struct sk_buff *skb, struct net_device *dev) eth_hdr(skb)->h_proto == htons(ETH_P_RARP)) && br_opt_get(br, BROPT_NEIGH_SUPPRESS_ENABLED)) { br_do_proxy_suppress_arp(skb, br, vid, NULL); - } else if (IS_ENABLED(CONFIG_IPV6) && + } else if (ipv6_mod_enabled() && skb->protocol == htons(ETH_P_IPV6) && br_opt_get(br, BROPT_NEIGH_SUPPRESS_ENABLED) && pskb_may_pull(skb, sizeof(struct ipv6hdr) + diff --git a/net/bridge/br_input.c b/net/bridge/br_input.c index 1405f1061a549..2cbae0f9ae1f0 100644 --- a/net/bridge/br_input.c +++ b/net/bridge/br_input.c @@ -170,7 +170,7 @@ int br_handle_frame_finish(struct net *net, struct sock *sk, struct sk_buff *skb (skb->protocol == htons(ETH_P_ARP) || skb->protocol == htons(ETH_P_RARP))) { br_do_proxy_suppress_arp(skb, br, vid, p); - } else if (IS_ENABLED(CONFIG_IPV6) && + } else if (ipv6_mod_enabled() && skb->protocol == htons(ETH_P_IPV6) && br_opt_get(br, BROPT_NEIGH_SUPPRESS_ENABLED) && pskb_may_pull(skb, sizeof(struct ipv6hdr) + diff --git a/net/bridge/br_multicast.c b/net/bridge/br_multicast.c index 22d12e5459668..5855eb0502085 100644 --- a/net/bridge/br_multicast.c +++ b/net/bridge/br_multicast.c @@ -244,14 +244,11 @@ br_multicast_port_vid_to_port_ctx(struct net_bridge_port *port, u16 vid) lockdep_assert_held_once(&port->br->multicast_lock); - if (!br_opt_get(port->br, BROPT_MCAST_VLAN_SNOOPING_ENABLED)) - return NULL; - /* Take RCU to access the vlan. */ rcu_read_lock(); vlan = br_vlan_find(nbp_vlan_group_rcu(port), vid); - if (vlan && !br_multicast_port_ctx_vlan_disabled(&vlan->port_mcast_ctx)) + if (vlan) pmctx = &vlan->port_mcast_ctx; rcu_read_unlock(); @@ -701,7 +698,10 @@ br_multicast_port_ngroups_inc_one(struct net_bridge_mcast_port *pmctx, u32 max = READ_ONCE(pmctx->mdb_max_entries); u32 n = READ_ONCE(pmctx->mdb_n_entries); - if (max && n >= max) { + /* enforce the max limit when it's a port pmctx or a port-vlan pmctx + * with snooping enabled + */ + if (!br_multicast_port_ctx_vlan_disabled(pmctx) && max && n >= max) { NL_SET_ERR_MSG_FMT_MOD(extack, "%s is already in %u groups, and mcast_max_groups=%u", what, n, max); return -E2BIG; @@ -736,9 +736,7 @@ static int br_multicast_port_ngroups_inc(struct net_bridge_port *port, return err; } - /* Only count on the VLAN context if VID is given, and if snooping on - * that VLAN is enabled. - */ + /* Only count on the VLAN context if VID is given */ if (!group->vid) return 0; @@ -2011,6 +2009,18 @@ void br_multicast_port_ctx_init(struct net_bridge_port *port, timer_setup(&pmctx->ip6_own_query.timer, br_ip6_multicast_port_query_expired, 0); #endif + /* initialize mdb_n_entries if a new port vlan is being created */ + if (vlan) { + struct net_bridge_port_group *pg; + u32 n = 0; + + spin_lock_bh(&port->br->multicast_lock); + hlist_for_each_entry(pg, &port->mglist, mglist) + if (pg->key.addr.vid == vlan->vid) + n++; + WRITE_ONCE(pmctx->mdb_n_entries, n); + spin_unlock_bh(&port->br->multicast_lock); + } } void br_multicast_port_ctx_deinit(struct net_bridge_mcast_port *pmctx) @@ -2094,25 +2104,6 @@ static void __br_multicast_enable_port_ctx(struct net_bridge_mcast_port *pmctx) br_ip4_multicast_add_router(brmctx, pmctx); br_ip6_multicast_add_router(brmctx, pmctx); } - - if (br_multicast_port_ctx_is_vlan(pmctx)) { - struct net_bridge_port_group *pg; - u32 n = 0; - - /* The mcast_n_groups counter might be wrong. First, - * BR_VLFLAG_MCAST_ENABLED is toggled before temporary entries - * are flushed, thus mcast_n_groups after the toggle does not - * reflect the true values. And second, permanent entries added - * while BR_VLFLAG_MCAST_ENABLED was disabled, are not reflected - * either. Thus we have to refresh the counter. - */ - - hlist_for_each_entry(pg, &pmctx->port->mglist, mglist) { - if (pg->key.addr.vid == pmctx->vlan->vid) - n++; - } - WRITE_ONCE(pmctx->mdb_n_entries, n); - } } static void br_multicast_enable_port_ctx(struct net_bridge_mcast_port *pmctx) diff --git a/net/can/bcm.c b/net/can/bcm.c index 5e690a2377e48..756acfd20c6c2 100644 --- a/net/can/bcm.c +++ b/net/can/bcm.c @@ -1170,6 +1170,7 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, if (!op) return -ENOMEM; + spin_lock_init(&op->bcm_tx_lock); op->can_id = msg_head->can_id; op->nframes = msg_head->nframes; op->cfsiz = CFSIZ(msg_head->flags); diff --git a/net/ceph/crypto.c b/net/ceph/crypto.c index 01b2ce1e8fc06..5601732cf4faa 100644 --- a/net/ceph/crypto.c +++ b/net/ceph/crypto.c @@ -37,9 +37,6 @@ static int set_secret(struct ceph_crypto_key *key, void *buf) return -ENOTSUPP; } - if (!key->len) - return -EINVAL; - key->key = kmemdup(buf, key->len, GFP_NOIO); if (!key->key) { ret = -ENOMEM; @@ -83,6 +80,11 @@ int ceph_crypto_key_decode(struct ceph_crypto_key *key, void **p, void *end) ceph_decode_copy(p, &key->created, sizeof(key->created)); key->len = ceph_decode_16(p); ceph_decode_need(p, end, key->len, bad); + if (key->len > CEPH_MAX_KEY_LEN) { + pr_err("secret too big %d\n", key->len); + return -EINVAL; + } + ret = set_secret(key, *p); memzero_explicit(*p, key->len); *p += key->len; diff --git a/net/ceph/crypto.h b/net/ceph/crypto.h index 23de29fc613cf..a20bad6d1e964 100644 --- a/net/ceph/crypto.h +++ b/net/ceph/crypto.h @@ -5,7 +5,7 @@ #include #include -#define CEPH_KEY_LEN 16 +#define CEPH_MAX_KEY_LEN 16 #define CEPH_MAX_CON_SECRET_LEN 64 /* diff --git a/net/ceph/messenger_v2.c b/net/ceph/messenger_v2.c index 061eaa047f765..b67f2b582bc76 100644 --- a/net/ceph/messenger_v2.c +++ b/net/ceph/messenger_v2.c @@ -2361,7 +2361,7 @@ static int process_auth_reply_more(struct ceph_connection *con, */ static int process_auth_done(struct ceph_connection *con, void *p, void *end) { - u8 session_key_buf[CEPH_KEY_LEN + 16]; + u8 session_key_buf[CEPH_MAX_KEY_LEN + 16]; u8 con_secret_buf[CEPH_MAX_CON_SECRET_LEN + 16]; u8 *session_key = PTR_ALIGN(&session_key_buf[0], 16); u8 *con_secret = PTR_ALIGN(&con_secret_buf[0], 16); diff --git a/net/core/dev.c b/net/core/dev.c index 5b536860138d1..c8e49eef45198 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -738,7 +738,7 @@ static struct net_device_path *dev_fwd_path(struct net_device_path_stack *stack) { int k = stack->num_paths++; - if (WARN_ON_ONCE(k >= NET_DEVICE_PATH_STACK_MAX)) + if (k >= NET_DEVICE_PATH_STACK_MAX) return NULL; return &stack->path[k]; @@ -3975,7 +3975,7 @@ static struct sk_buff *validate_xmit_unreadable_skb(struct sk_buff *skb, if (shinfo->nr_frags > 0) { niov = netmem_to_net_iov(skb_frag_netmem(&shinfo->frags[0])); if (net_is_devmem_iov(niov) && - net_devmem_iov_binding(niov)->dev != dev) + READ_ONCE(net_devmem_iov_binding(niov)->dev) != dev) goto out_free; } @@ -4758,10 +4758,9 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) if (dev->flags & IFF_UP) { int cpu = smp_processor_id(); /* ok because BHs are off */ - /* Other cpus might concurrently change txq->xmit_lock_owner - * to -1 or to their cpu id, but not to our id. - */ - if (READ_ONCE(txq->xmit_lock_owner) != cpu) { + if (!netif_tx_owned(txq, cpu)) { + bool is_list = false; + if (dev_xmit_recursion()) goto recursion_alert; @@ -4772,17 +4771,28 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) HARD_TX_LOCK(dev, txq, cpu); if (!netif_xmit_stopped(txq)) { + is_list = !!skb->next; + dev_xmit_recursion_inc(); skb = dev_hard_start_xmit(skb, dev, txq, &rc); dev_xmit_recursion_dec(); - if (dev_xmit_complete(rc)) { - HARD_TX_UNLOCK(dev, txq); - goto out; - } + + /* GSO segments a single SKB into + * a list of frames. TCP expects error + * to mean none of the data was sent. + */ + if (is_list) + rc = NETDEV_TX_OK; } HARD_TX_UNLOCK(dev, txq); + if (!skb) /* xmit completed */ + goto out; + net_crit_ratelimited("Virtual device %s asks to queue packet!\n", dev->name); + /* NETDEV_TX_BUSY or queue was stopped */ + if (!is_list) + rc = -ENETDOWN; } else { /* Recursion is detected! It is possible, * unfortunately @@ -4790,10 +4800,10 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) recursion_alert: net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n", dev->name); + rc = -ENETDOWN; } } - rc = -ENETDOWN; rcu_read_unlock_bh(); dev_core_stats_tx_dropped_inc(dev); @@ -4932,8 +4942,7 @@ static bool rps_flow_is_active(struct rps_dev_flow *rflow, static struct rps_dev_flow * set_rps_cpu(struct net_device *dev, struct sk_buff *skb, - struct rps_dev_flow *rflow, u16 next_cpu, u32 hash, - u32 flow_id) + struct rps_dev_flow *rflow, u16 next_cpu, u32 hash) { if (next_cpu < nr_cpu_ids) { u32 head; @@ -4944,6 +4953,7 @@ set_rps_cpu(struct net_device *dev, struct sk_buff *skb, struct rps_dev_flow *tmp_rflow; unsigned int tmp_cpu; u16 rxq_index; + u32 flow_id; int rc; /* Should we steer this flow to a different hardware queue? */ @@ -4959,6 +4969,7 @@ set_rps_cpu(struct net_device *dev, struct sk_buff *skb, if (!flow_table) goto out; + flow_id = rfs_slot(hash, flow_table); tmp_rflow = &flow_table->flows[flow_id]; tmp_cpu = READ_ONCE(tmp_rflow->cpu); @@ -5006,7 +5017,6 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, struct rps_dev_flow_table *flow_table; struct rps_map *map; int cpu = -1; - u32 flow_id; u32 tcpu; u32 hash; @@ -5053,8 +5063,7 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, /* OK, now we know there is a match, * we can look at the local (per receive queue) flow table */ - flow_id = rfs_slot(hash, flow_table); - rflow = &flow_table->flows[flow_id]; + rflow = &flow_table->flows[rfs_slot(hash, flow_table)]; tcpu = rflow->cpu; /* @@ -5073,8 +5082,7 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, ((int)(READ_ONCE(per_cpu(softnet_data, tcpu).input_queue_head) - rflow->last_qtail)) >= 0)) { tcpu = next_cpu; - rflow = set_rps_cpu(dev, skb, rflow, next_cpu, hash, - flow_id); + rflow = set_rps_cpu(dev, skb, rflow, next_cpu, hash); } if (tcpu < nr_cpu_ids && cpu_online(tcpu)) { @@ -12659,6 +12667,94 @@ netdev_features_t netdev_increment_features(netdev_features_t all, } EXPORT_SYMBOL(netdev_increment_features); +/** + * netdev_compute_master_upper_features - compute feature from lowers + * @dev: the upper device + * @update_header: whether to update upper device's header_len/headroom/tailroom + * + * Recompute the upper device's feature based on all lower devices. + */ +void netdev_compute_master_upper_features(struct net_device *dev, bool update_header) +{ + unsigned int dst_release_flag = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; + netdev_features_t gso_partial_features = MASTER_UPPER_DEV_GSO_PARTIAL_FEATURES; + netdev_features_t xfrm_features = MASTER_UPPER_DEV_XFRM_FEATURES; + netdev_features_t mpls_features = MASTER_UPPER_DEV_MPLS_FEATURES; + netdev_features_t vlan_features = MASTER_UPPER_DEV_VLAN_FEATURES; + netdev_features_t enc_features = MASTER_UPPER_DEV_ENC_FEATURES; + unsigned short max_header_len = ETH_HLEN; + unsigned int tso_max_size = TSO_MAX_SIZE; + unsigned short max_headroom = 0; + unsigned short max_tailroom = 0; + u16 tso_max_segs = TSO_MAX_SEGS; + struct net_device *lower_dev; + struct list_head *iter; + + mpls_features = netdev_base_features(mpls_features); + vlan_features = netdev_base_features(vlan_features); + enc_features = netdev_base_features(enc_features); + + netdev_for_each_lower_dev(dev, lower_dev, iter) { + gso_partial_features = netdev_increment_features(gso_partial_features, + lower_dev->gso_partial_features, + MASTER_UPPER_DEV_GSO_PARTIAL_FEATURES); + + vlan_features = netdev_increment_features(vlan_features, + lower_dev->vlan_features, + MASTER_UPPER_DEV_VLAN_FEATURES); + + enc_features = netdev_increment_features(enc_features, + lower_dev->hw_enc_features, + MASTER_UPPER_DEV_ENC_FEATURES); + + if (IS_ENABLED(CONFIG_XFRM_OFFLOAD)) + xfrm_features = netdev_increment_features(xfrm_features, + lower_dev->hw_enc_features, + MASTER_UPPER_DEV_XFRM_FEATURES); + + mpls_features = netdev_increment_features(mpls_features, + lower_dev->mpls_features, + MASTER_UPPER_DEV_MPLS_FEATURES); + + dst_release_flag &= lower_dev->priv_flags; + + if (update_header) { + max_header_len = max(max_header_len, lower_dev->hard_header_len); + max_headroom = max(max_headroom, lower_dev->needed_headroom); + max_tailroom = max(max_tailroom, lower_dev->needed_tailroom); + } + + tso_max_size = min(tso_max_size, lower_dev->tso_max_size); + tso_max_segs = min(tso_max_segs, lower_dev->tso_max_segs); + } + + dev->gso_partial_features = gso_partial_features; + dev->vlan_features = vlan_features; + dev->hw_enc_features = enc_features | NETIF_F_GSO_ENCAP_ALL | + NETIF_F_HW_VLAN_CTAG_TX | + NETIF_F_HW_VLAN_STAG_TX; + if (IS_ENABLED(CONFIG_XFRM_OFFLOAD)) + dev->hw_enc_features |= xfrm_features; + dev->mpls_features = mpls_features; + + dev->priv_flags &= ~IFF_XMIT_DST_RELEASE; + if ((dev->priv_flags & IFF_XMIT_DST_RELEASE_PERM) && + dst_release_flag == (IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM)) + dev->priv_flags |= IFF_XMIT_DST_RELEASE; + + if (update_header) { + dev->hard_header_len = max_header_len; + dev->needed_headroom = max_headroom; + dev->needed_tailroom = max_tailroom; + } + + netif_set_tso_max_segs(dev, tso_max_segs); + netif_set_tso_max_size(dev, tso_max_size); + + netdev_change_features(dev); +} +EXPORT_SYMBOL(netdev_compute_master_upper_features); + static struct hlist_head * __net_init netdev_create_hash(void) { int i; diff --git a/net/core/devmem.c b/net/core/devmem.c index 1d04754bc756d..448f6582ac1ae 100644 --- a/net/core/devmem.c +++ b/net/core/devmem.c @@ -387,7 +387,8 @@ struct net_devmem_dmabuf_binding *net_devmem_get_binding(struct sock *sk, * net_device. */ dst_dev = dst_dev_rcu(dst); - if (unlikely(!dst_dev) || unlikely(dst_dev != binding->dev)) { + if (unlikely(!dst_dev) || + unlikely(dst_dev != READ_ONCE(binding->dev))) { err = -ENODEV; goto out_unlock; } @@ -504,7 +505,8 @@ static void mp_dmabuf_devmem_uninstall(void *mp_priv, xa_erase(&binding->bound_rxqs, xa_idx); if (xa_empty(&binding->bound_rxqs)) { mutex_lock(&binding->lock); - binding->dev = NULL; + ASSERT_EXCLUSIVE_WRITER(binding->dev); + WRITE_ONCE(binding->dev, NULL); mutex_unlock(&binding->lock); } break; diff --git a/net/core/filter.c b/net/core/filter.c index 88b265f6ccf89..3d4bf4d2a1a4b 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -4133,7 +4133,7 @@ static const struct bpf_func_proto bpf_xdp_store_bytes_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, - .arg3_type = ARG_PTR_TO_UNINIT_MEM, + .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg4_type = ARG_CONST_SIZE, }; @@ -4147,12 +4147,14 @@ static int bpf_xdp_frags_increase_tail(struct xdp_buff *xdp, int offset) struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); skb_frag_t *frag = &sinfo->frags[sinfo->nr_frags - 1]; struct xdp_rxq_info *rxq = xdp->rxq; - unsigned int tailroom; + int tailroom; if (!rxq->frag_size || rxq->frag_size > xdp->frame_sz) return -EOPNOTSUPP; - tailroom = rxq->frag_size - skb_frag_size(frag) - skb_frag_off(frag); + tailroom = rxq->frag_size - skb_frag_size(frag) - + skb_frag_off(frag) % rxq->frag_size; + WARN_ON_ONCE(tailroom < 0); if (unlikely(offset > tailroom)) return -EINVAL; @@ -6325,7 +6327,7 @@ static const struct bpf_func_proto bpf_xdp_fib_lookup_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, .arg3_type = ARG_CONST_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -6380,7 +6382,7 @@ static const struct bpf_func_proto bpf_skb_fib_lookup_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, .arg3_type = ARG_CONST_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -7934,9 +7936,9 @@ static const struct bpf_func_proto bpf_tcp_raw_gen_syncookie_ipv4_proto = { .gpl_only = true, /* __cookie_v4_init_sequence() is GPL */ .pkt_access = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg1_size = sizeof(struct iphdr), - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE_OR_ZERO, }; @@ -7966,9 +7968,9 @@ static const struct bpf_func_proto bpf_tcp_raw_gen_syncookie_ipv6_proto = { .gpl_only = true, /* __cookie_v6_init_sequence() is GPL */ .pkt_access = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg1_size = sizeof(struct ipv6hdr), - .arg2_type = ARG_PTR_TO_MEM, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_CONST_SIZE_OR_ZERO, }; @@ -7986,9 +7988,9 @@ static const struct bpf_func_proto bpf_tcp_raw_check_syncookie_ipv4_proto = { .gpl_only = true, /* __cookie_v4_check is GPL */ .pkt_access = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg1_size = sizeof(struct iphdr), - .arg2_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg2_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg2_size = sizeof(struct tcphdr), }; @@ -8010,9 +8012,9 @@ static const struct bpf_func_proto bpf_tcp_raw_check_syncookie_ipv6_proto = { .gpl_only = true, /* __cookie_v6_check is GPL */ .pkt_access = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg1_size = sizeof(struct ipv6hdr), - .arg2_type = ARG_PTR_TO_FIXED_SIZE_MEM, + .arg2_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg2_size = sizeof(struct tcphdr), }; #endif /* CONFIG_SYN_COOKIES */ diff --git a/net/core/gro.c b/net/core/gro.c index 482fa7d7f5981..ef61695fbdbb6 100644 --- a/net/core/gro.c +++ b/net/core/gro.c @@ -417,7 +417,7 @@ static void gro_pull_from_frag0(struct sk_buff *skb, int grow) { struct skb_shared_info *pinfo = skb_shinfo(skb); - BUG_ON(skb->end - skb->tail < grow); + DEBUG_NET_WARN_ON_ONCE(skb->end - skb->tail < grow); memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow); diff --git a/net/core/netpoll.c b/net/core/netpoll.c index 09f72f10813cc..5af14f14a3623 100644 --- a/net/core/netpoll.c +++ b/net/core/netpoll.c @@ -132,7 +132,7 @@ static int netif_local_xmit_active(struct net_device *dev) for (i = 0; i < dev->num_tx_queues; i++) { struct netdev_queue *txq = netdev_get_tx_queue(dev, i); - if (READ_ONCE(txq->xmit_lock_owner) == smp_processor_id()) + if (netif_tx_owned(txq, smp_processor_id())) return 1; } diff --git a/net/core/secure_seq.c b/net/core/secure_seq.c index 9a39656804513..6a6f2cda5aaef 100644 --- a/net/core/secure_seq.c +++ b/net/core/secure_seq.c @@ -20,7 +20,6 @@ #include static siphash_aligned_key_t net_secret; -static siphash_aligned_key_t ts_secret; #define EPHEMERAL_PORT_SHUFFLE_PERIOD (10 * HZ) @@ -28,11 +27,6 @@ static __always_inline void net_secret_init(void) { net_get_random_once(&net_secret, sizeof(net_secret)); } - -static __always_inline void ts_secret_init(void) -{ - net_get_random_once(&ts_secret, sizeof(ts_secret)); -} #endif #ifdef CONFIG_INET @@ -53,28 +47,9 @@ static u32 seq_scale(u32 seq) #endif #if IS_ENABLED(CONFIG_IPV6) -u32 secure_tcpv6_ts_off(const struct net *net, - const __be32 *saddr, const __be32 *daddr) -{ - const struct { - struct in6_addr saddr; - struct in6_addr daddr; - } __aligned(SIPHASH_ALIGNMENT) combined = { - .saddr = *(struct in6_addr *)saddr, - .daddr = *(struct in6_addr *)daddr, - }; - - if (READ_ONCE(net->ipv4.sysctl_tcp_timestamps) != 1) - return 0; - - ts_secret_init(); - return siphash(&combined, offsetofend(typeof(combined), daddr), - &ts_secret); -} -EXPORT_IPV6_MOD(secure_tcpv6_ts_off); - -u32 secure_tcpv6_seq(const __be32 *saddr, const __be32 *daddr, - __be16 sport, __be16 dport) +union tcp_seq_and_ts_off +secure_tcpv6_seq_and_ts_off(const struct net *net, const __be32 *saddr, + const __be32 *daddr, __be16 sport, __be16 dport) { const struct { struct in6_addr saddr; @@ -87,14 +62,20 @@ u32 secure_tcpv6_seq(const __be32 *saddr, const __be32 *daddr, .sport = sport, .dport = dport }; - u32 hash; + union tcp_seq_and_ts_off st; net_secret_init(); - hash = siphash(&combined, offsetofend(typeof(combined), dport), - &net_secret); - return seq_scale(hash); + + st.hash64 = siphash(&combined, offsetofend(typeof(combined), dport), + &net_secret); + + if (READ_ONCE(net->ipv4.sysctl_tcp_timestamps) != 1) + st.ts_off = 0; + + st.seq = seq_scale(st.seq); + return st; } -EXPORT_SYMBOL(secure_tcpv6_seq); +EXPORT_SYMBOL(secure_tcpv6_seq_and_ts_off); u64 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr, __be16 dport) @@ -118,33 +99,30 @@ EXPORT_SYMBOL(secure_ipv6_port_ephemeral); #endif #ifdef CONFIG_INET -u32 secure_tcp_ts_off(const struct net *net, __be32 saddr, __be32 daddr) -{ - if (READ_ONCE(net->ipv4.sysctl_tcp_timestamps) != 1) - return 0; - - ts_secret_init(); - return siphash_2u32((__force u32)saddr, (__force u32)daddr, - &ts_secret); -} - /* secure_tcp_seq_and_tsoff(a, b, 0, d) == secure_ipv4_port_ephemeral(a, b, d), * but fortunately, `sport' cannot be 0 in any circumstances. If this changes, * it would be easy enough to have the former function use siphash_4u32, passing * the arguments as separate u32. */ -u32 secure_tcp_seq(__be32 saddr, __be32 daddr, - __be16 sport, __be16 dport) +union tcp_seq_and_ts_off +secure_tcp_seq_and_ts_off(const struct net *net, __be32 saddr, __be32 daddr, + __be16 sport, __be16 dport) { - u32 hash; + u32 ports = (__force u32)sport << 16 | (__force u32)dport; + union tcp_seq_and_ts_off st; net_secret_init(); - hash = siphash_3u32((__force u32)saddr, (__force u32)daddr, - (__force u32)sport << 16 | (__force u32)dport, - &net_secret); - return seq_scale(hash); + + st.hash64 = siphash_3u32((__force u32)saddr, (__force u32)daddr, + ports, &net_secret); + + if (READ_ONCE(net->ipv4.sysctl_tcp_timestamps) != 1) + st.ts_off = 0; + + st.seq = seq_scale(st.seq); + return st; } -EXPORT_SYMBOL_GPL(secure_tcp_seq); +EXPORT_SYMBOL_GPL(secure_tcp_seq_and_ts_off); u64 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport) { diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 9a763d120925a..60d89899fdb9b 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -5513,15 +5513,28 @@ static void __skb_complete_tx_timestamp(struct sk_buff *skb, static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly) { - bool ret; + struct socket *sock; + struct file *file; + bool ret = false; if (likely(tsonly || READ_ONCE(sock_net(sk)->core.sysctl_tstamp_allow_data))) return true; - read_lock_bh(&sk->sk_callback_lock); - ret = sk->sk_socket && sk->sk_socket->file && - file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW); - read_unlock_bh(&sk->sk_callback_lock); + /* The sk pointer remains valid as long as the skb is. The sk_socket and + * file pointer may become NULL if the socket is closed. Both structures + * (including file->cred) are RCU freed which means they can be accessed + * within a RCU read section. + */ + rcu_read_lock(); + sock = READ_ONCE(sk->sk_socket); + if (!sock) + goto out; + file = READ_ONCE(sock->file); + if (!file) + goto out; + ret = file_ns_capable(file, &init_user_ns, CAP_NET_RAW); +out: + rcu_read_unlock(); return ret; } diff --git a/net/core/skmsg.c b/net/core/skmsg.c index 2ac7731e1e0a7..12fbb0545c712 100644 --- a/net/core/skmsg.c +++ b/net/core/skmsg.c @@ -409,22 +409,26 @@ int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from, } EXPORT_SYMBOL_GPL(sk_msg_memcopy_from_iter); -/* Receive sk_msg from psock->ingress_msg to @msg. */ -int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, - int len, int flags) +int __sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, + int len, int flags, int *copied_from_self) { struct iov_iter *iter = &msg->msg_iter; int peek = flags & MSG_PEEK; struct sk_msg *msg_rx; int i, copied = 0; + bool from_self; msg_rx = sk_psock_peek_msg(psock); + if (copied_from_self) + *copied_from_self = 0; + while (copied != len) { struct scatterlist *sge; if (unlikely(!msg_rx)) break; + from_self = msg_rx->sk == sk; i = msg_rx->sg.start; do { struct page *page; @@ -443,6 +447,9 @@ int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, } copied += copy; + if (from_self && copied_from_self) + *copied_from_self += copy; + if (likely(!peek)) { sge->offset += copy; sge->length -= copy; @@ -451,6 +458,7 @@ int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, atomic_sub(copy, &sk->sk_rmem_alloc); } msg_rx->sg.size -= copy; + sk_psock_msg_len_add(psock, -copy); if (!sge->length) { sk_msg_iter_var_next(i); @@ -487,6 +495,13 @@ int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, out: return copied; } + +/* Receive sk_msg from psock->ingress_msg to @msg. */ +int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, + int len, int flags) +{ + return __sk_msg_recvmsg(sk, psock, msg, len, flags, NULL); +} EXPORT_SYMBOL_GPL(sk_msg_recvmsg); bool sk_msg_is_readable(struct sock *sk) @@ -616,6 +631,12 @@ static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb if (unlikely(!msg)) return -EAGAIN; skb_set_owner_r(skb, sk); + + /* This is used in tcp_bpf_recvmsg_parser() to determine whether the + * data originates from the socket's own protocol stack. No need to + * refcount sk because msg's lifetime is bound to sk via the ingress_msg. + */ + msg->sk = sk; err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg, take_ref); if (err < 0) kfree(msg); @@ -801,9 +822,11 @@ static void __sk_psock_purge_ingress_msg(struct sk_psock *psock) list_del(&msg->list); if (!msg->skb) atomic_sub(msg->sg.size, &psock->sk->sk_rmem_alloc); + sk_psock_msg_len_add(psock, -msg->sg.size); sk_msg_free(psock->sk, msg); kfree(msg); } + WARN_ON_ONCE(psock->msg_tot_len); } static void __sk_psock_zap_ingress(struct sk_psock *psock) @@ -909,6 +932,7 @@ int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock, sk_msg_compute_data_pointers(msg); msg->sk = sk; ret = bpf_prog_run_pin_on_cpu(prog, msg); + msg->sk = NULL; ret = sk_psock_map_verd(ret, msg->sk_redir); psock->apply_bytes = msg->apply_bytes; if (ret == __SK_REDIRECT) { @@ -1181,8 +1205,8 @@ void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock) return; psock->saved_data_ready = sk->sk_data_ready; - sk->sk_data_ready = sk_psock_strp_data_ready; - sk->sk_write_space = sk_psock_write_space; + WRITE_ONCE(sk->sk_data_ready, sk_psock_strp_data_ready); + WRITE_ONCE(sk->sk_write_space, sk_psock_write_space); } void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock) @@ -1192,8 +1216,8 @@ void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock) if (!psock->saved_data_ready) return; - sk->sk_data_ready = psock->saved_data_ready; - psock->saved_data_ready = NULL; + WRITE_ONCE(sk->sk_data_ready, psock->saved_data_ready); + WRITE_ONCE(psock->saved_data_ready, NULL); strp_stop(&psock->strp); } @@ -1272,8 +1296,8 @@ void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock) return; psock->saved_data_ready = sk->sk_data_ready; - sk->sk_data_ready = sk_psock_verdict_data_ready; - sk->sk_write_space = sk_psock_write_space; + WRITE_ONCE(sk->sk_data_ready, sk_psock_verdict_data_ready); + WRITE_ONCE(sk->sk_write_space, sk_psock_write_space); } void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock) @@ -1284,6 +1308,6 @@ void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock) if (!psock->saved_data_ready) return; - sk->sk_data_ready = psock->saved_data_ready; + WRITE_ONCE(sk->sk_data_ready, psock->saved_data_ready); psock->saved_data_ready = NULL; } diff --git a/net/ipv4/fib_lookup.h b/net/ipv4/fib_lookup.h index f9b9e26c32c19..0b72796dd1ad3 100644 --- a/net/ipv4/fib_lookup.h +++ b/net/ipv4/fib_lookup.h @@ -28,8 +28,10 @@ struct fib_alias { /* Don't write on fa_state unless needed, to keep it shared on all cpus */ static inline void fib_alias_accessed(struct fib_alias *fa) { - if (!(fa->fa_state & FA_S_ACCESSED)) - fa->fa_state |= FA_S_ACCESSED; + u8 fa_state = READ_ONCE(fa->fa_state); + + if (!(fa_state & FA_S_ACCESSED)) + WRITE_ONCE(fa->fa_state, fa_state | FA_S_ACCESSED); } /* Exported by fib_semantics.c */ diff --git a/net/ipv4/fib_trie.c b/net/ipv4/fib_trie.c index 7e2c17fec3fc4..1308213791f19 100644 --- a/net/ipv4/fib_trie.c +++ b/net/ipv4/fib_trie.c @@ -1280,7 +1280,7 @@ int fib_table_insert(struct net *net, struct fib_table *tb, new_fa->fa_dscp = fa->fa_dscp; new_fa->fa_info = fi; new_fa->fa_type = cfg->fc_type; - state = fa->fa_state; + state = READ_ONCE(fa->fa_state); new_fa->fa_state = state & ~FA_S_ACCESSED; new_fa->fa_slen = fa->fa_slen; new_fa->tb_id = tb->tb_id; @@ -1745,7 +1745,7 @@ int fib_table_delete(struct net *net, struct fib_table *tb, fib_remove_alias(t, tp, l, fa_to_delete); - if (fa_to_delete->fa_state & FA_S_ACCESSED) + if (READ_ONCE(fa_to_delete->fa_state) & FA_S_ACCESSED) rt_cache_flush(cfg->fc_nlinfo.nl_net); fib_release_info(fa_to_delete->fa_info); diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c index 1b7fb5d935edf..3e19a5d465b83 100644 --- a/net/ipv4/icmp.c +++ b/net/ipv4/icmp.c @@ -248,7 +248,8 @@ bool icmp_global_allow(struct net *net) if (delta < HZ / 50) return false; - incr = READ_ONCE(net->ipv4.sysctl_icmp_msgs_per_sec) * delta / HZ; + incr = READ_ONCE(net->ipv4.sysctl_icmp_msgs_per_sec); + incr = div_u64((u64)incr * delta, HZ); if (!incr) return false; @@ -554,6 +555,21 @@ static struct rtable *icmp_route_lookup(struct net *net, struct flowi4 *fl4, /* steal dst entry from skb_in, don't drop refcnt */ skb_dstref_steal(skb_in); skb_dstref_restore(skb_in, orefdst); + + /* + * At this point, fl4_dec.daddr should NOT be local (we + * checked fl4_dec.saddr above). However, a race condition + * may occur if the address is added to the interface + * concurrently. In that case, ip_route_input() returns a + * LOCAL route with dst.output=ip_rt_bug, which must not + * be used for output. + */ + if (!err && rt2 && rt2->rt_type == RTN_LOCAL) { + net_warn_ratelimited("detected local route for %pI4 during ICMP sending, src %pI4\n", + &fl4_dec.daddr, &fl4_dec.saddr); + dst_release(&rt2->dst); + err = -EINVAL; + } } if (err) @@ -843,16 +859,22 @@ static void icmp_socket_deliver(struct sk_buff *skb, u32 info) /* Checkin full IP header plus 8 bytes of protocol to * avoid additional coding at protocol handlers. */ - if (!pskb_may_pull(skb, iph->ihl * 4 + 8)) { - __ICMP_INC_STATS(dev_net_rcu(skb->dev), ICMP_MIB_INERRORS); - return; - } + if (!pskb_may_pull(skb, iph->ihl * 4 + 8)) + goto out; + + /* IPPROTO_RAW sockets are not supposed to receive anything. */ + if (protocol == IPPROTO_RAW) + goto out; raw_icmp_error(skb, protocol, info); ipprot = rcu_dereference(inet_protos[protocol]); if (ipprot && ipprot->err_handler) ipprot->err_handler(skb, info); + return; + +out: + __ICMP_INC_STATS(dev_net_rcu(skb->dev), ICMP_MIB_INERRORS); } static bool icmp_tag_validation(int proto) diff --git a/net/ipv4/igmp.c b/net/ipv4/igmp.c index 7182f1419c2a4..0adc993c211d7 100644 --- a/net/ipv4/igmp.c +++ b/net/ipv4/igmp.c @@ -227,7 +227,7 @@ static void igmp_start_timer(struct ip_mc_list *im, int max_delay) static void igmp_gq_start_timer(struct in_device *in_dev) { - int tv = get_random_u32_below(in_dev->mr_maxdelay); + int tv = get_random_u32_below(READ_ONCE(in_dev->mr_maxdelay)); unsigned long exp = jiffies + tv + 2; if (in_dev->mr_gq_running && @@ -1009,7 +1009,7 @@ static bool igmp_heard_query(struct in_device *in_dev, struct sk_buff *skb, max_delay = IGMPV3_MRC(ih3->code)*(HZ/IGMP_TIMER_SCALE); if (!max_delay) max_delay = 1; /* can't mod w/ 0 */ - in_dev->mr_maxdelay = max_delay; + WRITE_ONCE(in_dev->mr_maxdelay, max_delay); /* RFC3376, 4.1.6. QRV and 4.1.7. QQIC, when the most recently * received value was zero, use the default or statically diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c index b7024e3d9ac3d..a57e33ff92d77 100644 --- a/net/ipv4/inet_hashtables.c +++ b/net/ipv4/inet_hashtables.c @@ -200,7 +200,7 @@ static bool inet_bind2_bucket_addr_match(const struct inet_bind2_bucket *tb2, void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb, struct inet_bind2_bucket *tb2, unsigned short port) { - inet_sk(sk)->inet_num = port; + WRITE_ONCE(inet_sk(sk)->inet_num, port); inet_csk(sk)->icsk_bind_hash = tb; inet_csk(sk)->icsk_bind2_hash = tb2; sk_add_bind_node(sk, &tb2->owners); @@ -224,7 +224,7 @@ static void __inet_put_port(struct sock *sk) spin_lock(&head->lock); tb = inet_csk(sk)->icsk_bind_hash; inet_csk(sk)->icsk_bind_hash = NULL; - inet_sk(sk)->inet_num = 0; + WRITE_ONCE(inet_sk(sk)->inet_num, 0); sk->sk_userlocks &= ~SOCK_CONNECT_BIND; spin_lock(&head2->lock); @@ -352,7 +352,7 @@ static inline int compute_score(struct sock *sk, const struct net *net, { int score = -1; - if (net_eq(sock_net(sk), net) && sk->sk_num == hnum && + if (net_eq(sock_net(sk), net) && READ_ONCE(sk->sk_num) == hnum && !ipv6_only_sock(sk)) { if (sk->sk_rcv_saddr != daddr) return -1; @@ -1202,7 +1202,7 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, sk->sk_hash = 0; inet_sk(sk)->inet_sport = 0; - inet_sk(sk)->inet_num = 0; + WRITE_ONCE(inet_sk(sk)->inet_num, 0); if (tw) inet_twsk_bind_unhash(tw, hinfo); diff --git a/net/ipv4/ping.c b/net/ipv4/ping.c index a5227d23bb0b5..690f486173e01 100644 --- a/net/ipv4/ping.c +++ b/net/ipv4/ping.c @@ -148,7 +148,7 @@ void ping_unhash(struct sock *sk) pr_debug("ping_unhash(isk=%p,isk->num=%u)\n", isk, isk->inet_num); spin_lock(&ping_table.lock); if (sk_del_node_init_rcu(sk)) { - isk->inet_num = 0; + WRITE_ONCE(isk->inet_num, 0); isk->inet_sport = 0; sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); } @@ -181,31 +181,35 @@ static struct sock *ping_lookup(struct net *net, struct sk_buff *skb, u16 ident) } sk_for_each_rcu(sk, hslot) { + int bound_dev_if; + if (!net_eq(sock_net(sk), net)) continue; isk = inet_sk(sk); pr_debug("iterate\n"); - if (isk->inet_num != ident) + if (READ_ONCE(isk->inet_num) != ident) continue; + bound_dev_if = READ_ONCE(sk->sk_bound_dev_if); if (skb->protocol == htons(ETH_P_IP) && sk->sk_family == AF_INET) { + __be32 rcv_saddr = READ_ONCE(isk->inet_rcv_saddr); + pr_debug("found: %p: num=%d, daddr=%pI4, dif=%d\n", sk, - (int) isk->inet_num, &isk->inet_rcv_saddr, - sk->sk_bound_dev_if); + ident, &rcv_saddr, + bound_dev_if); - if (isk->inet_rcv_saddr && - isk->inet_rcv_saddr != ip_hdr(skb)->daddr) + if (rcv_saddr && rcv_saddr != ip_hdr(skb)->daddr) continue; #if IS_ENABLED(CONFIG_IPV6) } else if (skb->protocol == htons(ETH_P_IPV6) && sk->sk_family == AF_INET6) { pr_debug("found: %p: num=%d, daddr=%pI6c, dif=%d\n", sk, - (int) isk->inet_num, + ident, &sk->sk_v6_rcv_saddr, - sk->sk_bound_dev_if); + bound_dev_if); if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr) && !ipv6_addr_equal(&sk->sk_v6_rcv_saddr, @@ -216,8 +220,8 @@ static struct sock *ping_lookup(struct net *net, struct sk_buff *skb, u16 ident) continue; } - if (sk->sk_bound_dev_if && sk->sk_bound_dev_if != dif && - sk->sk_bound_dev_if != sdif) + if (bound_dev_if && bound_dev_if != dif && + bound_dev_if != sdif) continue; goto exit; @@ -392,7 +396,9 @@ static void ping_set_saddr(struct sock *sk, struct sockaddr *saddr) if (saddr->sa_family == AF_INET) { struct inet_sock *isk = inet_sk(sk); struct sockaddr_in *addr = (struct sockaddr_in *) saddr; - isk->inet_rcv_saddr = isk->inet_saddr = addr->sin_addr.s_addr; + + isk->inet_saddr = addr->sin_addr.s_addr; + WRITE_ONCE(isk->inet_rcv_saddr, addr->sin_addr.s_addr); #if IS_ENABLED(CONFIG_IPV6) } else if (saddr->sa_family == AF_INET6) { struct sockaddr_in6 *addr = (struct sockaddr_in6 *) saddr; @@ -849,7 +855,8 @@ int ping_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags, struct sk_buff *skb; int copied, err; - pr_debug("ping_recvmsg(sk=%p,sk->num=%u)\n", isk, isk->inet_num); + pr_debug("ping_recvmsg(sk=%p,sk->num=%u)\n", isk, + READ_ONCE(isk->inet_num)); err = -EOPNOTSUPP; if (flags & MSG_OOB) diff --git a/net/ipv4/syncookies.c b/net/ipv4/syncookies.c index 569befcf021ba..fc3affd9c8014 100644 --- a/net/ipv4/syncookies.c +++ b/net/ipv4/syncookies.c @@ -203,7 +203,7 @@ struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb, bool own_req; child = icsk->icsk_af_ops->syn_recv_sock(sk, skb, req, dst, - NULL, &own_req); + NULL, &own_req, NULL); if (child) { refcount_set(&req->rsk_refcnt, 1); sock_rps_save_rxhash(child, skb); @@ -378,9 +378,14 @@ static struct request_sock *cookie_tcp_check(struct net *net, struct sock *sk, tcp_parse_options(net, skb, &tcp_opt, 0, NULL); if (tcp_opt.saw_tstamp && tcp_opt.rcv_tsecr) { - tsoff = secure_tcp_ts_off(net, - ip_hdr(skb)->daddr, - ip_hdr(skb)->saddr); + union tcp_seq_and_ts_off st; + + st = secure_tcp_seq_and_ts_off(net, + ip_hdr(skb)->daddr, + ip_hdr(skb)->saddr, + tcp_hdr(skb)->dest, + tcp_hdr(skb)->source); + tsoff = st.ts_off; tcp_opt.rcv_tsecr -= tsoff; } diff --git a/net/ipv4/sysctl_net_ipv4.c b/net/ipv4/sysctl_net_ipv4.c index 24dbc603cc44d..0f1dd75dbf37b 100644 --- a/net/ipv4/sysctl_net_ipv4.c +++ b/net/ipv4/sysctl_net_ipv4.c @@ -484,7 +484,8 @@ static void proc_fib_multipath_hash_set_seed(struct net *net, u32 user_seed) proc_fib_multipath_hash_rand_seed), }; - WRITE_ONCE(net->ipv4.sysctl_fib_multipath_hash_seed, new); + WRITE_ONCE(net->ipv4.sysctl_fib_multipath_hash_seed.user_seed, new.user_seed); + WRITE_ONCE(net->ipv4.sysctl_fib_multipath_hash_seed.mp_seed, new.mp_seed); } static int proc_fib_multipath_hash_seed(const struct ctl_table *table, int write, @@ -498,7 +499,7 @@ static int proc_fib_multipath_hash_seed(const struct ctl_table *table, int write int ret; mphs = &net->ipv4.sysctl_fib_multipath_hash_seed; - user_seed = mphs->user_seed; + user_seed = READ_ONCE(mphs->user_seed); tmp = *table; tmp.data = &user_seed; diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 74079eab89804..f665c87edc0f7 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -503,6 +503,9 @@ static void tcp_tx_timestamp(struct sock *sk, struct sockcm_cookie *sockc) struct sk_buff *skb = tcp_write_queue_tail(sk); u32 tsflags = sockc->tsflags; + if (unlikely(!skb)) + skb = skb_rb_last(&sk->tcp_rtx_queue); + if (tsflags && skb) { struct skb_shared_info *shinfo = skb_shinfo(skb); struct tcp_skb_cb *tcb = TCP_SKB_CB(skb); @@ -1395,7 +1398,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) err = sk_stream_error(sk, flags, err); /* make sure we wake any epoll edge trigger waiter */ if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) { - sk->sk_write_space(sk); + READ_ONCE(sk->sk_write_space)(sk); tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED); } if (binding) @@ -4108,7 +4111,7 @@ int do_tcp_setsockopt(struct sock *sk, int level, int optname, break; case TCP_NOTSENT_LOWAT: WRITE_ONCE(tp->notsent_lowat, val); - sk->sk_write_space(sk); + READ_ONCE(sk->sk_write_space)(sk); break; case TCP_INQ: if (val > 1 || val < 0) diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c index a268e1595b22a..d3d6a47af5270 100644 --- a/net/ipv4/tcp_bpf.c +++ b/net/ipv4/tcp_bpf.c @@ -10,6 +10,7 @@ #include #include +#include void tcp_eat_skb(struct sock *sk, struct sk_buff *skb) { @@ -226,6 +227,7 @@ static int tcp_bpf_recvmsg_parser(struct sock *sk, int peek = flags & MSG_PEEK; struct sk_psock *psock; struct tcp_sock *tcp; + int copied_from_self = 0; int copied = 0; u32 seq; @@ -262,7 +264,7 @@ static int tcp_bpf_recvmsg_parser(struct sock *sk, } msg_bytes_ready: - copied = sk_msg_recvmsg(sk, psock, msg, len, flags); + copied = __sk_msg_recvmsg(sk, psock, msg, len, flags, &copied_from_self); /* The typical case for EFAULT is the socket was gracefully * shutdown with a FIN pkt. So check here the other case is * some error on copy_page_to_iter which would be unexpected. @@ -277,7 +279,7 @@ static int tcp_bpf_recvmsg_parser(struct sock *sk, goto out; } } - seq += copied; + seq += copied_from_self; if (!copied) { long timeo; int data; @@ -331,6 +333,24 @@ static int tcp_bpf_recvmsg_parser(struct sock *sk, return copied; } +static int tcp_bpf_ioctl(struct sock *sk, int cmd, int *karg) +{ + bool slow; + + if (cmd != SIOCINQ) + return tcp_ioctl(sk, cmd, karg); + + /* works similar as tcp_ioctl */ + if (sk->sk_state == TCP_LISTEN) + return -EINVAL; + + slow = lock_sock_fast(sk); + *karg = sk_psock_msg_inq(sk); + unlock_sock_fast(sk, slow); + + return 0; +} + static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags, int *addr_len) { @@ -609,6 +629,7 @@ static void tcp_bpf_rebuild_protos(struct proto prot[TCP_BPF_NUM_CFGS], prot[TCP_BPF_BASE].close = sock_map_close; prot[TCP_BPF_BASE].recvmsg = tcp_bpf_recvmsg; prot[TCP_BPF_BASE].sock_is_readable = sk_msg_is_readable; + prot[TCP_BPF_BASE].ioctl = tcp_bpf_ioctl; prot[TCP_BPF_TX] = prot[TCP_BPF_BASE]; prot[TCP_BPF_TX].sendmsg = tcp_bpf_sendmsg; @@ -704,7 +725,7 @@ int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore) WRITE_ONCE(sk->sk_prot->unhash, psock->saved_unhash); tcp_update_ulp(sk, psock->sk_proto, psock->saved_write_space); } else { - sk->sk_write_space = psock->saved_write_space; + WRITE_ONCE(sk->sk_write_space, psock->saved_write_space); /* Pairs with lockless read in sk_clone_lock() */ sock_replace_proto(sk, psock->sk_proto); } diff --git a/net/ipv4/tcp_cong.c b/net/ipv4/tcp_cong.c index df758adbb445f..e9f6c77e06316 100644 --- a/net/ipv4/tcp_cong.c +++ b/net/ipv4/tcp_cong.c @@ -16,6 +16,7 @@ #include #include #include +#include #include static DEFINE_SPINLOCK(tcp_cong_list_lock); @@ -227,7 +228,7 @@ void tcp_assign_congestion_control(struct sock *sk) memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv)); if (ca->flags & TCP_CONG_NEEDS_ECN) - INET_ECN_xmit(sk); + INET_ECN_xmit_ect_1_negotiation(sk); else INET_ECN_dontxmit(sk); } @@ -257,7 +258,7 @@ static void tcp_reinit_congestion_control(struct sock *sk, memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv)); if (ca->flags & TCP_CONG_NEEDS_ECN) - INET_ECN_xmit(sk); + INET_ECN_xmit_ect_1_negotiation(sk); else INET_ECN_dontxmit(sk); diff --git a/net/ipv4/tcp_diag.c b/net/ipv4/tcp_diag.c index d83efd91f461c..7935702e394b2 100644 --- a/net/ipv4/tcp_diag.c +++ b/net/ipv4/tcp_diag.c @@ -509,7 +509,7 @@ static void tcp_diag_dump(struct sk_buff *skb, struct netlink_callback *cb, if (r->sdiag_family != AF_UNSPEC && sk->sk_family != r->sdiag_family) goto next_normal; - if (r->id.idiag_sport != htons(sk->sk_num) && + if (r->id.idiag_sport != htons(READ_ONCE(sk->sk_num)) && r->id.idiag_sport) goto next_normal; if (r->id.idiag_dport != sk->sk_dport && diff --git a/net/ipv4/tcp_fastopen.c b/net/ipv4/tcp_fastopen.c index 7d945a527daf0..444306af444ae 100644 --- a/net/ipv4/tcp_fastopen.c +++ b/net/ipv4/tcp_fastopen.c @@ -247,7 +247,7 @@ static struct sock *tcp_fastopen_create_child(struct sock *sk, bool own_req; child = inet_csk(sk)->icsk_af_ops->syn_recv_sock(sk, skb, req, NULL, - NULL, &own_req); + NULL, &own_req, NULL); if (!child) return NULL; diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index e4a979b75cc66..96486eea26724 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -4595,15 +4595,24 @@ static enum skb_drop_reason tcp_disordered_ack_check(const struct sock *sk, */ static enum skb_drop_reason tcp_sequence(const struct sock *sk, - u32 seq, u32 end_seq) + u32 seq, u32 end_seq, + const struct tcphdr *th) { const struct tcp_sock *tp = tcp_sk(sk); + u32 seq_limit; if (before(end_seq, tp->rcv_wup)) return SKB_DROP_REASON_TCP_OLD_SEQUENCE; - if (after(end_seq, tp->rcv_nxt + tcp_receive_window(tp))) { - if (after(seq, tp->rcv_nxt + tcp_receive_window(tp))) + seq_limit = tp->rcv_nxt + tcp_receive_window(tp); + if (unlikely(after(end_seq, seq_limit))) { + /* Some stacks are known to handle FIN incorrectly; allow the + * FIN to extend beyond the window and check it in detail later. + */ + if (!after(end_seq - th->fin, seq_limit)) + return SKB_NOT_DROPPED_YET; + + if (after(seq, seq_limit)) return SKB_DROP_REASON_TCP_INVALID_SEQUENCE; /* Only accept this packet if receive queue is empty. */ @@ -5091,25 +5100,11 @@ static void tcp_ofo_queue(struct sock *sk) static bool tcp_prune_ofo_queue(struct sock *sk, const struct sk_buff *in_skb); static int tcp_prune_queue(struct sock *sk, const struct sk_buff *in_skb); -/* Check if this incoming skb can be added to socket receive queues - * while satisfying sk->sk_rcvbuf limit. - * - * In theory we should use skb->truesize, but this can cause problems - * when applications use too small SO_RCVBUF values. - * When LRO / hw gro is used, the socket might have a high tp->scaling_ratio, - * allowing RWIN to be close to available space. - * Whenever the receive queue gets full, we can receive a small packet - * filling RWIN, but with a high skb->truesize, because most NIC use 4K page - * plus sk_buff metadata even when receiving less than 1500 bytes of payload. - * - * Note that we use skb->len to decide to accept or drop this packet, - * but sk->sk_rmem_alloc is the sum of all skb->truesize. - */ static bool tcp_can_ingest(const struct sock *sk, const struct sk_buff *skb) { unsigned int rmem = atomic_read(&sk->sk_rmem_alloc); - return rmem + skb->len <= sk->sk_rcvbuf; + return rmem <= sk->sk_rcvbuf; } static int tcp_try_rmem_schedule(struct sock *sk, const struct sk_buff *skb, @@ -5142,7 +5137,7 @@ static void tcp_data_queue_ofo(struct sock *sk, struct sk_buff *skb) if (unlikely(tcp_try_rmem_schedule(sk, skb, skb->truesize))) { NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPOFODROP); - sk->sk_data_ready(sk); + READ_ONCE(sk->sk_data_ready)(sk); tcp_drop_reason(sk, skb, SKB_DROP_REASON_PROTO_MEM); return; } @@ -5352,7 +5347,7 @@ int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size) void tcp_data_ready(struct sock *sk) { if (tcp_epollin_ready(sk, sk->sk_rcvlowat) || sock_flag(sk, SOCK_DONE)) - sk->sk_data_ready(sk); + READ_ONCE(sk->sk_data_ready)(sk); } static void tcp_data_queue(struct sock *sk, struct sk_buff *skb) @@ -5408,7 +5403,7 @@ static void tcp_data_queue(struct sock *sk, struct sk_buff *skb) inet_csk(sk)->icsk_ack.pending |= (ICSK_ACK_NOMEM | ICSK_ACK_NOW); inet_csk_schedule_ack(sk); - sk->sk_data_ready(sk); + READ_ONCE(sk->sk_data_ready)(sk); if (skb_queue_len(&sk->sk_receive_queue) && skb->len) { reason = SKB_DROP_REASON_PROTO_MEM; @@ -5850,7 +5845,9 @@ static void tcp_new_space(struct sock *sk) tp->snd_cwnd_stamp = tcp_jiffies32; } - INDIRECT_CALL_1(sk->sk_write_space, sk_stream_write_space, sk); + INDIRECT_CALL_1(READ_ONCE(sk->sk_write_space), + sk_stream_write_space, + sk); } /* Caller made space either from: @@ -6056,7 +6053,7 @@ static void tcp_urg(struct sock *sk, struct sk_buff *skb, const struct tcphdr *t BUG(); WRITE_ONCE(tp->urg_data, TCP_URG_VALID | tmp); if (!sock_flag(sk, SOCK_DEAD)) - sk->sk_data_ready(sk); + READ_ONCE(sk->sk_data_ready)(sk); } } } @@ -6119,7 +6116,8 @@ static bool tcp_validate_incoming(struct sock *sk, struct sk_buff *skb, step1: /* Step 1: check sequence number */ - reason = tcp_sequence(sk, TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq); + reason = tcp_sequence(sk, TCP_SKB_CB(skb)->seq, + TCP_SKB_CB(skb)->end_seq, th); if (reason) { /* RFC793, page 37: "In all states except SYN-SENT, all reset * (RST) segments are validated by checking their SEQ-fields." @@ -6817,7 +6815,7 @@ static int tcp_rcv_synsent_state_process(struct sock *sk, struct sk_buff *skb, tp->snd_wl1 = TCP_SKB_CB(skb)->seq; tp->max_window = tp->snd_wnd; - tcp_ecn_rcv_syn(tp, th, skb); + tcp_ecn_rcv_syn(sk, th, skb); tcp_mtup_init(sk); tcp_sync_mss(sk, icsk->icsk_pmtu_cookie); @@ -7222,7 +7220,8 @@ static void tcp_ecn_create_request(struct request_sock *req, u32 ecn_ok_dst; if (tcp_accecn_syn_requested(th) && - READ_ONCE(net->ipv4.sysctl_tcp_ecn) >= 3) { + (READ_ONCE(net->ipv4.sysctl_tcp_ecn) >= 3 || + tcp_ca_needs_accecn(listen_sk))) { inet_rsk(req)->ecn_ok = 1; tcp_rsk(req)->accecn_ok = 1; tcp_rsk(req)->syn_ect_rcv = TCP_SKB_CB(skb)->ip_dsfield & @@ -7386,6 +7385,7 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops, const struct tcp_sock *tp = tcp_sk(sk); struct net *net = sock_net(sk); struct sock *fastopen_sk = NULL; + union tcp_seq_and_ts_off st; struct request_sock *req; bool want_cookie = false; struct dst_entry *dst; @@ -7455,9 +7455,12 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops, if (!dst) goto drop_and_free; + if (tmp_opt.tstamp_ok || (!want_cookie && !isn)) + st = af_ops->init_seq_and_ts_off(net, skb); + if (tmp_opt.tstamp_ok) { tcp_rsk(req)->req_usec_ts = dst_tcp_usec_ts(dst); - tcp_rsk(req)->ts_off = af_ops->init_ts_off(net, skb); + tcp_rsk(req)->ts_off = st.ts_off; } if (!want_cookie && !isn) { int max_syn_backlog = READ_ONCE(net->ipv4.sysctl_max_syn_backlog); @@ -7479,7 +7482,7 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops, goto drop_and_release; } - isn = af_ops->init_seq(skb); + isn = st.seq; } tcp_ecn_create_request(req, skb, sk, dst); @@ -7520,7 +7523,7 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops, sock_put(fastopen_sk); goto drop_and_free; } - sk->sk_data_ready(sk); + READ_ONCE(sk->sk_data_ready)(sk); bh_unlock_sock(fastopen_sk); sock_put(fastopen_sk); } else { diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index b1fcf3e4e1ce0..75a11d7feb260 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -104,17 +104,14 @@ static DEFINE_PER_CPU(struct sock_bh_locked, ipv4_tcp_sk) = { static DEFINE_MUTEX(tcp_exit_batch_mutex); -static u32 tcp_v4_init_seq(const struct sk_buff *skb) +static union tcp_seq_and_ts_off +tcp_v4_init_seq_and_ts_off(const struct net *net, const struct sk_buff *skb) { - return secure_tcp_seq(ip_hdr(skb)->daddr, - ip_hdr(skb)->saddr, - tcp_hdr(skb)->dest, - tcp_hdr(skb)->source); -} - -static u32 tcp_v4_init_ts_off(const struct net *net, const struct sk_buff *skb) -{ - return secure_tcp_ts_off(net, ip_hdr(skb)->daddr, ip_hdr(skb)->saddr); + return secure_tcp_seq_and_ts_off(net, + ip_hdr(skb)->daddr, + ip_hdr(skb)->saddr, + tcp_hdr(skb)->dest, + tcp_hdr(skb)->source); } int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp) @@ -326,15 +323,16 @@ int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) rt = NULL; if (likely(!tp->repair)) { + union tcp_seq_and_ts_off st; + + st = secure_tcp_seq_and_ts_off(net, + inet->inet_saddr, + inet->inet_daddr, + inet->inet_sport, + usin->sin_port); if (!tp->write_seq) - WRITE_ONCE(tp->write_seq, - secure_tcp_seq(inet->inet_saddr, - inet->inet_daddr, - inet->inet_sport, - usin->sin_port)); - WRITE_ONCE(tp->tsoffset, - secure_tcp_ts_off(net, inet->inet_saddr, - inet->inet_daddr)); + WRITE_ONCE(tp->write_seq, st.seq); + WRITE_ONCE(tp->tsoffset, st.ts_off); } atomic_set(&inet->inet_id, get_random_u16()); @@ -1727,8 +1725,7 @@ const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops = { .cookie_init_seq = cookie_v4_init_sequence, #endif .route_req = tcp_v4_route_req, - .init_seq = tcp_v4_init_seq, - .init_ts_off = tcp_v4_init_ts_off, + .init_seq_and_ts_off = tcp_v4_init_seq_and_ts_off, .send_synack = tcp_v4_send_synack, }; @@ -1756,7 +1753,9 @@ struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, struct request_sock *req, struct dst_entry *dst, struct request_sock *req_unhash, - bool *own_req) + bool *own_req, + void (*opt_child_init)(struct sock *newsk, + const struct sock *sk)) { struct inet_request_sock *ireq; bool found_dup_sk = false; @@ -1808,6 +1807,10 @@ struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, } sk_setup_caps(newsk, dst); +#if IS_ENABLED(CONFIG_IPV6) + if (opt_child_init) + opt_child_init(newsk, sk); +#endif tcp_ca_openreq_child(newsk, dst); tcp_sync_mss(newsk, dst_mtu(dst)); diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c index 2ec8c6f1cdccc..c70c29a3a0905 100644 --- a/net/ipv4/tcp_minisocks.c +++ b/net/ipv4/tcp_minisocks.c @@ -488,9 +488,10 @@ static void tcp_ecn_openreq_child(struct sock *sk, tp->accecn_opt_demand = 1; tcp_ecn_received_counters_payload(sk, skb); } else { - tcp_ecn_mode_set(tp, inet_rsk(req)->ecn_ok ? - TCP_ECN_MODE_RFC3168 : - TCP_ECN_DISABLED); + if (inet_rsk(req)->ecn_ok && !tcp_ca_no_fallback_rfc3168(sk)) + tcp_ecn_mode_set(tp, TCP_ECN_MODE_RFC3168); + else + tcp_ecn_mode_set(tp, TCP_ECN_DISABLED); } } @@ -910,7 +911,7 @@ struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb, * socket is created, wait for troubles. */ child = inet_csk(sk)->icsk_af_ops->syn_recv_sock(sk, skb, req, NULL, - req, &own_req); + req, &own_req, NULL); if (!child) goto listen_overflow; @@ -989,7 +990,7 @@ enum skb_drop_reason tcp_child_process(struct sock *parent, struct sock *child, reason = tcp_rcv_state_process(child, skb); /* Wakeup parent, send SIGIO */ if (state == TCP_SYN_RECV && child->sk_state != state) - parent->sk_data_ready(parent); + READ_ONCE(parent->sk_data_ready)(parent); } else { /* Alas, it is possible again, because we do lookup * in main socket hash table and lock on listening diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 860bd61ff047f..024cb4f5978c1 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -1786,7 +1786,7 @@ int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb) * using prepare_to_wait_exclusive(). */ while (nb) { - INDIRECT_CALL_1(sk->sk_data_ready, + INDIRECT_CALL_1(READ_ONCE(sk->sk_data_ready), sock_def_readable, sk); nb--; } @@ -2266,7 +2266,6 @@ void udp_lib_rehash(struct sock *sk, u16 newhash, u16 newhash4) udp_sk(sk)->udp_port_hash); hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); nhslot2 = udp_hashslot2(udptable, newhash); - udp_sk(sk)->udp_portaddr_hash = newhash; if (hslot2 != nhslot2 || rcu_access_pointer(sk->sk_reuseport_cb)) { @@ -2300,19 +2299,25 @@ void udp_lib_rehash(struct sock *sk, u16 newhash, u16 newhash4) if (udp_hashed4(sk)) { spin_lock_bh(&hslot->lock); - udp_rehash4(udptable, sk, newhash4); - if (hslot2 != nhslot2) { - spin_lock(&hslot2->lock); - udp_hash4_dec(hslot2); - spin_unlock(&hslot2->lock); - - spin_lock(&nhslot2->lock); - udp_hash4_inc(nhslot2); - spin_unlock(&nhslot2->lock); + if (inet_rcv_saddr_any(sk)) { + udp_unhash4(udptable, sk); + } else { + udp_rehash4(udptable, sk, newhash4); + if (hslot2 != nhslot2) { + spin_lock(&hslot2->lock); + udp_hash4_dec(hslot2); + spin_unlock(&hslot2->lock); + + spin_lock(&nhslot2->lock); + udp_hash4_inc(nhslot2); + spin_unlock(&nhslot2->lock); + } } spin_unlock_bh(&hslot->lock); } + + udp_sk(sk)->udp_portaddr_hash = newhash; } } EXPORT_IPV6_MOD(udp_lib_rehash); diff --git a/net/ipv4/udp_bpf.c b/net/ipv4/udp_bpf.c index 0735d820e413f..779a3a03762f1 100644 --- a/net/ipv4/udp_bpf.c +++ b/net/ipv4/udp_bpf.c @@ -5,6 +5,7 @@ #include #include #include +#include #include "udp_impl.h" @@ -111,12 +112,26 @@ enum { static DEFINE_SPINLOCK(udpv6_prot_lock); static struct proto udp_bpf_prots[UDP_BPF_NUM_PROTS]; +static int udp_bpf_ioctl(struct sock *sk, int cmd, int *karg) +{ + if (cmd != SIOCINQ) + return udp_ioctl(sk, cmd, karg); + + /* Since we don't hold a lock, sk_receive_queue may contain data. + * BPF might only be processing this data at the moment. We only + * care about the data in the ingress_msg here. + */ + *karg = sk_msg_first_len(sk); + return 0; +} + static void udp_bpf_rebuild_protos(struct proto *prot, const struct proto *base) { - *prot = *base; - prot->close = sock_map_close; - prot->recvmsg = udp_bpf_recvmsg; - prot->sock_is_readable = sk_msg_is_readable; + *prot = *base; + prot->close = sock_map_close; + prot->recvmsg = udp_bpf_recvmsg; + prot->sock_is_readable = sk_msg_is_readable; + prot->ioctl = udp_bpf_ioctl; } static void udp_bpf_check_v6_needs_rebuild(struct proto *ops) @@ -143,7 +158,7 @@ int udp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore) int family = sk->sk_family == AF_INET ? UDP_BPF_IPV4 : UDP_BPF_IPV6; if (restore) { - sk->sk_write_space = psock->saved_write_space; + WRITE_ONCE(sk->sk_write_space, psock->saved_write_space); sock_replace_proto(sk, psock->sk_proto); return 0; } diff --git a/net/ipv4/udplite.c b/net/ipv4/udplite.c index d3e621a11a1aa..826e9e79eb19c 100644 --- a/net/ipv4/udplite.c +++ b/net/ipv4/udplite.c @@ -20,10 +20,9 @@ EXPORT_SYMBOL(udplite_table); /* Designate sk as UDP-Lite socket */ static int udplite_sk_init(struct sock *sk) { - udp_init_sock(sk); pr_warn_once("UDP-Lite is deprecated and scheduled to be removed in 2025, " "please contact the netdev mailing list\n"); - return 0; + return udp_init_sock(sk); } static int udplite_rcv(struct sk_buff *skb) diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c index 1b0314644e0cc..3709f213d33de 100644 --- a/net/ipv6/af_inet6.c +++ b/net/ipv6/af_inet6.c @@ -224,8 +224,8 @@ static int inet6_create(struct net *net, struct socket *sock, int protocol, inet6_set_bit(MC6_LOOP, sk); inet6_set_bit(MC6_ALL, sk); np->pmtudisc = IPV6_PMTUDISC_WANT; - inet6_assign_bit(REPFLOW, sk, net->ipv6.sysctl.flowlabel_reflect & - FLOWLABEL_REFLECT_ESTABLISHED); + inet6_assign_bit(REPFLOW, sk, READ_ONCE(net->ipv6.sysctl.flowlabel_reflect) & + FLOWLABEL_REFLECT_ESTABLISHED); sk->sk_ipv6only = net->ipv6.sysctl.bindv6only; sk->sk_txrehash = READ_ONCE(net->core.sysctl_txrehash); @@ -955,7 +955,7 @@ static int __net_init inet6_net_init(struct net *net) int err = 0; net->ipv6.sysctl.bindv6only = 0; - net->ipv6.sysctl.icmpv6_time = 1*HZ; + net->ipv6.sysctl.icmpv6_time = HZ / 10; net->ipv6.sysctl.icmpv6_echo_ignore_all = 0; net->ipv6.sysctl.icmpv6_echo_ignore_multicast = 0; net->ipv6.sysctl.icmpv6_echo_ignore_anycast = 0; diff --git a/net/ipv6/exthdrs.c b/net/ipv6/exthdrs.c index a23eb8734e151..310836a0cf17b 100644 --- a/net/ipv6/exthdrs.c +++ b/net/ipv6/exthdrs.c @@ -314,7 +314,7 @@ static int ipv6_destopt_rcv(struct sk_buff *skb) } extlen = (skb_transport_header(skb)[1] + 1) << 3; - if (extlen > net->ipv6.sysctl.max_dst_opts_len) + if (extlen > READ_ONCE(net->ipv6.sysctl.max_dst_opts_len)) goto fail_and_free; opt->lastopt = opt->dst1 = skb_network_header_len(skb); @@ -322,7 +322,8 @@ static int ipv6_destopt_rcv(struct sk_buff *skb) dstbuf = opt->dst1; #endif - if (ip6_parse_tlv(false, skb, net->ipv6.sysctl.max_dst_opts_cnt)) { + if (ip6_parse_tlv(false, skb, + READ_ONCE(net->ipv6.sysctl.max_dst_opts_cnt))) { skb->transport_header += extlen; opt = IP6CB(skb); #if IS_ENABLED(CONFIG_IPV6_MIP6) @@ -930,6 +931,11 @@ static bool ipv6_hop_ioam(struct sk_buff *skb, int optoff) if (hdr->opt_len < 2 + sizeof(*trace) + trace->remlen * 4) goto drop; + /* Inconsistent Pre-allocated Trace header */ + if (trace->nodelen != + ioam6_trace_compute_nodelen(be32_to_cpu(trace->type_be32))) + goto drop; + /* Ignore if the IOAM namespace is unknown */ ns = ioam6_namespace(dev_net(skb->dev), trace->namespace_id); if (!ns) @@ -1049,11 +1055,12 @@ int ipv6_parse_hopopts(struct sk_buff *skb) } extlen = (skb_transport_header(skb)[1] + 1) << 3; - if (extlen > net->ipv6.sysctl.max_hbh_opts_len) + if (extlen > READ_ONCE(net->ipv6.sysctl.max_hbh_opts_len)) goto fail_and_free; opt->flags |= IP6SKB_HOPBYHOP; - if (ip6_parse_tlv(true, skb, net->ipv6.sysctl.max_hbh_opts_cnt)) { + if (ip6_parse_tlv(true, skb, + READ_ONCE(net->ipv6.sysctl.max_hbh_opts_cnt))) { skb->transport_header += extlen; opt = IP6CB(skb); opt->nhoff = sizeof(struct ipv6hdr); diff --git a/net/ipv6/icmp.c b/net/ipv6/icmp.c index cf6455cbe2cc9..54ad4c7578679 100644 --- a/net/ipv6/icmp.c +++ b/net/ipv6/icmp.c @@ -217,14 +217,9 @@ static bool icmpv6_xrlim_allow(struct sock *sk, u8 type, } else if (dev && (dev->flags & IFF_LOOPBACK)) { res = true; } else { - struct rt6_info *rt = dst_rt6_info(dst); - int tmo = net->ipv6.sysctl.icmpv6_time; + int tmo = READ_ONCE(net->ipv6.sysctl.icmpv6_time); struct inet_peer *peer; - /* Give more bandwidth to wider prefixes. */ - if (rt->rt6i_dst.plen < 128) - tmo >>= ((128 - rt->rt6i_dst.plen)>>5); - peer = inet_getpeer_v6(net->ipv6.peers, &fl6->daddr); res = inet_peer_xrlim_allow(peer, tmo); } @@ -762,7 +757,8 @@ static enum skb_drop_reason icmpv6_echo_reply(struct sk_buff *skb) tmp_hdr.icmp6_type = type; memset(&fl6, 0, sizeof(fl6)); - if (net->ipv6.sysctl.flowlabel_reflect & FLOWLABEL_REFLECT_ICMPV6_ECHO_REPLIES) + if (READ_ONCE(net->ipv6.sysctl.flowlabel_reflect) & + FLOWLABEL_REFLECT_ICMPV6_ECHO_REPLIES) fl6.flowlabel = ip6_flowlabel(ipv6_hdr(skb)); fl6.flowi6_proto = IPPROTO_ICMPV6; @@ -870,6 +866,12 @@ enum skb_drop_reason icmpv6_notify(struct sk_buff *skb, u8 type, if (reason != SKB_NOT_DROPPED_YET) goto out; + if (nexthdr == IPPROTO_RAW) { + /* Add a more specific reason later ? */ + reason = SKB_DROP_REASON_NOT_SPECIFIED; + goto out; + } + /* BUGGG_FUTURE: we should try to parse exthdrs in this packet. Without this we will not able f.e. to make source routed pmtu discovery. diff --git a/net/ipv6/inet6_hashtables.c b/net/ipv6/inet6_hashtables.c index 5e1da088d8e11..182d38e6d6d8d 100644 --- a/net/ipv6/inet6_hashtables.c +++ b/net/ipv6/inet6_hashtables.c @@ -95,7 +95,8 @@ static inline int compute_score(struct sock *sk, const struct net *net, { int score = -1; - if (net_eq(sock_net(sk), net) && inet_sk(sk)->inet_num == hnum && + if (net_eq(sock_net(sk), net) && + READ_ONCE(inet_sk(sk)->inet_num) == hnum && sk->sk_family == PF_INET6) { if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr)) return -1; diff --git a/net/ipv6/ioam6.c b/net/ipv6/ioam6.c index 9553a32000813..08b7ac8c99b7e 100644 --- a/net/ipv6/ioam6.c +++ b/net/ipv6/ioam6.c @@ -690,6 +690,20 @@ struct ioam6_namespace *ioam6_namespace(struct net *net, __be16 id) return rhashtable_lookup_fast(&nsdata->namespaces, &id, rht_ns_params); } +#define IOAM6_MASK_SHORT_FIELDS 0xff1ffc00 +#define IOAM6_MASK_WIDE_FIELDS 0x00e00000 + +u8 ioam6_trace_compute_nodelen(u32 trace_type) +{ + u8 nodelen = hweight32(trace_type & IOAM6_MASK_SHORT_FIELDS) + * (sizeof(__be32) / 4); + + nodelen += hweight32(trace_type & IOAM6_MASK_WIDE_FIELDS) + * (sizeof(__be64) / 4); + + return nodelen; +} + static void __ioam6_fill_trace_data(struct sk_buff *skb, struct ioam6_namespace *ns, struct ioam6_trace_hdr *trace, diff --git a/net/ipv6/ioam6_iptunnel.c b/net/ipv6/ioam6_iptunnel.c index 1fe7894f14dd9..b9f6d892a566c 100644 --- a/net/ipv6/ioam6_iptunnel.c +++ b/net/ipv6/ioam6_iptunnel.c @@ -22,9 +22,6 @@ #include #include -#define IOAM6_MASK_SHORT_FIELDS 0xff100000 -#define IOAM6_MASK_WIDE_FIELDS 0xe00000 - struct ioam6_lwt_encap { struct ipv6_hopopt_hdr eh; u8 pad[2]; /* 2-octet padding for 4n-alignment */ @@ -93,13 +90,8 @@ static bool ioam6_validate_trace_hdr(struct ioam6_trace_hdr *trace) trace->type.bit21 | trace->type.bit23) return false; - trace->nodelen = 0; fields = be32_to_cpu(trace->type_be32); - - trace->nodelen += hweight32(fields & IOAM6_MASK_SHORT_FIELDS) - * (sizeof(__be32) / 4); - trace->nodelen += hweight32(fields & IOAM6_MASK_WIDE_FIELDS) - * (sizeof(__be64) / 4); + trace->nodelen = ioam6_trace_compute_nodelen(fields); return true; } diff --git a/net/ipv6/ip6_fib.c b/net/ipv6/ip6_fib.c index c6439e30e892a..cc149227b49f4 100644 --- a/net/ipv6/ip6_fib.c +++ b/net/ipv6/ip6_fib.c @@ -1139,7 +1139,7 @@ static int fib6_add_rt2node(struct fib6_node *fn, struct fib6_info *rt, fib6_add_gc_list(iter); } if (!(rt->fib6_flags & (RTF_ADDRCONF | RTF_PREFIX_RT)) && - !iter->fib6_nh->fib_nh_gw_family) { + (iter->nh || !iter->fib6_nh->fib_nh_gw_family)) { iter->fib6_flags &= ~RTF_ADDRCONF; iter->fib6_flags &= ~RTF_PREFIX_RT; } diff --git a/net/ipv6/ip6_flowlabel.c b/net/ipv6/ip6_flowlabel.c index a3ff575798dda..60d0be47a9f31 100644 --- a/net/ipv6/ip6_flowlabel.c +++ b/net/ipv6/ip6_flowlabel.c @@ -66,8 +66,8 @@ EXPORT_SYMBOL(ipv6_flowlabel_exclusive); fl != NULL; \ fl = rcu_dereference(fl->next)) -#define for_each_sk_fl_rcu(np, sfl) \ - for (sfl = rcu_dereference(np->ipv6_fl_list); \ +#define for_each_sk_fl_rcu(sk, sfl) \ + for (sfl = rcu_dereference(inet_sk(sk)->ipv6_fl_list); \ sfl != NULL; \ sfl = rcu_dereference(sfl->next)) @@ -262,12 +262,11 @@ static struct ip6_flowlabel *fl_intern(struct net *net, struct ip6_flowlabel *__fl6_sock_lookup(struct sock *sk, __be32 label) { struct ipv6_fl_socklist *sfl; - struct ipv6_pinfo *np = inet6_sk(sk); label &= IPV6_FLOWLABEL_MASK; rcu_read_lock(); - for_each_sk_fl_rcu(np, sfl) { + for_each_sk_fl_rcu(sk, sfl) { struct ip6_flowlabel *fl = sfl->fl; if (fl->label == label && atomic_inc_not_zero(&fl->users)) { @@ -283,16 +282,16 @@ EXPORT_SYMBOL_GPL(__fl6_sock_lookup); void fl6_free_socklist(struct sock *sk) { - struct ipv6_pinfo *np = inet6_sk(sk); + struct inet_sock *inet = inet_sk(sk); struct ipv6_fl_socklist *sfl; - if (!rcu_access_pointer(np->ipv6_fl_list)) + if (!rcu_access_pointer(inet->ipv6_fl_list)) return; spin_lock_bh(&ip6_sk_fl_lock); - while ((sfl = rcu_dereference_protected(np->ipv6_fl_list, + while ((sfl = rcu_dereference_protected(inet->ipv6_fl_list, lockdep_is_held(&ip6_sk_fl_lock))) != NULL) { - np->ipv6_fl_list = sfl->next; + inet->ipv6_fl_list = sfl->next; spin_unlock_bh(&ip6_sk_fl_lock); fl_release(sfl->fl); @@ -470,16 +469,15 @@ fl_create(struct net *net, struct sock *sk, struct in6_flowlabel_req *freq, static int mem_check(struct sock *sk) { - struct ipv6_pinfo *np = inet6_sk(sk); - struct ipv6_fl_socklist *sfl; int room = FL_MAX_SIZE - atomic_read(&fl_size); + struct ipv6_fl_socklist *sfl; int count = 0; if (room > FL_MAX_SIZE - FL_MAX_PER_SOCK) return 0; rcu_read_lock(); - for_each_sk_fl_rcu(np, sfl) + for_each_sk_fl_rcu(sk, sfl) count++; rcu_read_unlock(); @@ -492,13 +490,15 @@ static int mem_check(struct sock *sk) return 0; } -static inline void fl_link(struct ipv6_pinfo *np, struct ipv6_fl_socklist *sfl, - struct ip6_flowlabel *fl) +static inline void fl_link(struct sock *sk, struct ipv6_fl_socklist *sfl, + struct ip6_flowlabel *fl) { + struct inet_sock *inet = inet_sk(sk); + spin_lock_bh(&ip6_sk_fl_lock); sfl->fl = fl; - sfl->next = np->ipv6_fl_list; - rcu_assign_pointer(np->ipv6_fl_list, sfl); + sfl->next = inet->ipv6_fl_list; + rcu_assign_pointer(inet->ipv6_fl_list, sfl); spin_unlock_bh(&ip6_sk_fl_lock); } @@ -520,7 +520,7 @@ int ipv6_flowlabel_opt_get(struct sock *sk, struct in6_flowlabel_req *freq, rcu_read_lock(); - for_each_sk_fl_rcu(np, sfl) { + for_each_sk_fl_rcu(sk, sfl) { if (sfl->fl->label == (np->flow_label & IPV6_FLOWLABEL_MASK)) { spin_lock_bh(&ip6_fl_lock); freq->flr_label = sfl->fl->label; @@ -559,7 +559,7 @@ static int ipv6_flowlabel_put(struct sock *sk, struct in6_flowlabel_req *freq) } spin_lock_bh(&ip6_sk_fl_lock); - for (sflp = &np->ipv6_fl_list; + for (sflp = &inet_sk(sk)->ipv6_fl_list; (sfl = socklist_dereference(*sflp)) != NULL; sflp = &sfl->next) { if (sfl->fl->label == freq->flr_label) @@ -579,13 +579,12 @@ static int ipv6_flowlabel_put(struct sock *sk, struct in6_flowlabel_req *freq) static int ipv6_flowlabel_renew(struct sock *sk, struct in6_flowlabel_req *freq) { - struct ipv6_pinfo *np = inet6_sk(sk); struct net *net = sock_net(sk); struct ipv6_fl_socklist *sfl; int err; rcu_read_lock(); - for_each_sk_fl_rcu(np, sfl) { + for_each_sk_fl_rcu(sk, sfl) { if (sfl->fl->label == freq->flr_label) { err = fl6_renew(sfl->fl, freq->flr_linger, freq->flr_expires); @@ -614,7 +613,6 @@ static int ipv6_flowlabel_get(struct sock *sk, struct in6_flowlabel_req *freq, { struct ipv6_fl_socklist *sfl, *sfl1 = NULL; struct ip6_flowlabel *fl, *fl1 = NULL; - struct ipv6_pinfo *np = inet6_sk(sk); struct net *net = sock_net(sk); int err; @@ -645,7 +643,7 @@ static int ipv6_flowlabel_get(struct sock *sk, struct in6_flowlabel_req *freq, if (freq->flr_label) { err = -EEXIST; rcu_read_lock(); - for_each_sk_fl_rcu(np, sfl) { + for_each_sk_fl_rcu(sk, sfl) { if (sfl->fl->label == freq->flr_label) { if (freq->flr_flags & IPV6_FL_F_EXCL) { rcu_read_unlock(); @@ -682,7 +680,7 @@ static int ipv6_flowlabel_get(struct sock *sk, struct in6_flowlabel_req *freq, fl1->linger = fl->linger; if ((long)(fl->expires - fl1->expires) > 0) fl1->expires = fl->expires; - fl_link(np, sfl1, fl1); + fl_link(sk, sfl1, fl1); fl_free(fl); return 0; @@ -716,7 +714,7 @@ static int ipv6_flowlabel_get(struct sock *sk, struct in6_flowlabel_req *freq, } } - fl_link(np, sfl1, fl); + fl_link(sk, sfl1, fl); return 0; done: fl_free(fl); diff --git a/net/ipv6/route.c b/net/ipv6/route.c index e3a260a5564ba..e01331d965313 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -1063,7 +1063,8 @@ static struct net_device *ip6_rt_get_dev_rcu(const struct fib6_result *res) */ if (netif_is_l3_slave(dev) && !rt6_need_strict(&res->f6i->fib6_dst.addr)) - dev = l3mdev_master_dev_rcu(dev); + dev = l3mdev_master_dev_rcu(dev) ? : + dev_net(dev)->loopback_dev; else if (!netif_is_l3_master(dev)) dev = dev_net(dev)->loopback_dev; /* last case is netif_is_l3_master(dev) is true in which @@ -2895,7 +2896,7 @@ static void rt6_do_update_pmtu(struct rt6_info *rt, u32 mtu) dst_metric_set(&rt->dst, RTAX_MTU, mtu); rt->rt6i_flags |= RTF_MODIFIED; - rt6_update_expires(rt, net->ipv6.sysctl.ip6_rt_mtu_expires); + rt6_update_expires(rt, READ_ONCE(net->ipv6.sysctl.ip6_rt_mtu_expires)); } static bool rt6_cache_allowed_for_pmtu(const struct rt6_info *rt) @@ -3256,8 +3257,8 @@ static unsigned int ip6_default_advmss(const struct dst_entry *dst) rcu_read_lock(); net = dst_dev_net_rcu(dst); - if (mtu < net->ipv6.sysctl.ip6_rt_min_advmss) - mtu = net->ipv6.sysctl.ip6_rt_min_advmss; + mtu = max_t(unsigned int, mtu, + READ_ONCE(net->ipv6.sysctl.ip6_rt_min_advmss)); rcu_read_unlock(); @@ -3359,10 +3360,10 @@ struct dst_entry *icmp6_dst_alloc(struct net_device *dev, static void ip6_dst_gc(struct dst_ops *ops) { struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops); - int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval; - int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity; - int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout; - unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc; + int rt_min_interval = READ_ONCE(net->ipv6.sysctl.ip6_rt_gc_min_interval); + int rt_elasticity = READ_ONCE(net->ipv6.sysctl.ip6_rt_gc_elasticity); + int rt_gc_timeout = READ_ONCE(net->ipv6.sysctl.ip6_rt_gc_timeout); + unsigned long rt_last_gc = READ_ONCE(net->ipv6.ip6_rt_last_gc); unsigned int val; int entries; @@ -3583,7 +3584,6 @@ int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh, netdevice_tracker *dev_tracker = &fib6_nh->fib_nh_dev_tracker; struct net_device *dev = NULL; struct inet6_dev *idev = NULL; - int addr_type; int err; fib6_nh->fib_nh_family = AF_INET6; @@ -3625,11 +3625,10 @@ int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh, fib6_nh->fib_nh_weight = 1; - /* We cannot add true routes via loopback here, - * they would result in kernel looping; promote them to reject routes + /* Reset the nexthop device to the loopback device in case of reject + * routes. */ - addr_type = ipv6_addr_type(&cfg->fc_dst); - if (fib6_is_reject(cfg->fc_flags, dev, addr_type)) { + if (cfg->fc_flags & RTF_REJECT) { /* hold loopback dev/idev if we haven't done so. */ if (dev != net->loopback_dev) { if (dev) { @@ -5008,7 +5007,7 @@ void rt6_sync_down_dev(struct net_device *dev, unsigned long event) }; struct net *net = dev_net(dev); - if (net->ipv6.sysctl.skip_notify_on_dev_down) + if (READ_ONCE(net->ipv6.sysctl.skip_notify_on_dev_down)) fib6_clean_all_skip_notify(net, fib6_ifdown, &arg); else fib6_clean_all(net, fib6_ifdown, &arg); @@ -6408,6 +6407,7 @@ void fib6_rt_update(struct net *net, struct fib6_info *rt, void fib6_info_hw_flags_set(struct net *net, struct fib6_info *f6i, bool offload, bool trap, bool offload_failed) { + u8 fib_notify_on_flag_change; struct sk_buff *skb; int err; @@ -6419,8 +6419,9 @@ void fib6_info_hw_flags_set(struct net *net, struct fib6_info *f6i, WRITE_ONCE(f6i->offload, offload); WRITE_ONCE(f6i->trap, trap); + fib_notify_on_flag_change = READ_ONCE(net->ipv6.sysctl.fib_notify_on_flag_change); /* 2 means send notifications only if offload_failed was changed. */ - if (net->ipv6.sysctl.fib_notify_on_flag_change == 2 && + if (fib_notify_on_flag_change == 2 && READ_ONCE(f6i->offload_failed) == offload_failed) return; @@ -6432,7 +6433,7 @@ void fib6_info_hw_flags_set(struct net *net, struct fib6_info *f6i, */ return; - if (!net->ipv6.sysctl.fib_notify_on_flag_change) + if (!fib_notify_on_flag_change) return; skb = nlmsg_new(rt6_nlmsg_size(f6i), GFP_KERNEL); @@ -6529,7 +6530,7 @@ static int ipv6_sysctl_rtcache_flush(const struct ctl_table *ctl, int write, return ret; net = (struct net *)ctl->extra1; - delay = net->ipv6.sysctl.flush_delay; + delay = READ_ONCE(net->ipv6.sysctl.flush_delay); fib6_run_gc(delay <= 0 ? 0 : (unsigned long)delay, net, delay > 0); return 0; } diff --git a/net/ipv6/syncookies.c b/net/ipv6/syncookies.c index 7e007f013ec82..4f6f0d751d6c5 100644 --- a/net/ipv6/syncookies.c +++ b/net/ipv6/syncookies.c @@ -151,9 +151,14 @@ static struct request_sock *cookie_tcp_check(struct net *net, struct sock *sk, tcp_parse_options(net, skb, &tcp_opt, 0, NULL); if (tcp_opt.saw_tstamp && tcp_opt.rcv_tsecr) { - tsoff = secure_tcpv6_ts_off(net, - ipv6_hdr(skb)->daddr.s6_addr32, - ipv6_hdr(skb)->saddr.s6_addr32); + union tcp_seq_and_ts_off st; + + st = secure_tcpv6_seq_and_ts_off(net, + ipv6_hdr(skb)->daddr.s6_addr32, + ipv6_hdr(skb)->saddr.s6_addr32, + tcp_hdr(skb)->dest, + tcp_hdr(skb)->source); + tsoff = st.ts_off; tcp_opt.rcv_tsecr -= tsoff; } diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index 59c4977a811a0..90afe81bc8e5d 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -105,18 +105,14 @@ static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb) } } -static u32 tcp_v6_init_seq(const struct sk_buff *skb) +static union tcp_seq_and_ts_off +tcp_v6_init_seq_and_ts_off(const struct net *net, const struct sk_buff *skb) { - return secure_tcpv6_seq(ipv6_hdr(skb)->daddr.s6_addr32, - ipv6_hdr(skb)->saddr.s6_addr32, - tcp_hdr(skb)->dest, - tcp_hdr(skb)->source); -} - -static u32 tcp_v6_init_ts_off(const struct net *net, const struct sk_buff *skb) -{ - return secure_tcpv6_ts_off(net, ipv6_hdr(skb)->daddr.s6_addr32, - ipv6_hdr(skb)->saddr.s6_addr32); + return secure_tcpv6_seq_and_ts_off(net, + ipv6_hdr(skb)->daddr.s6_addr32, + ipv6_hdr(skb)->saddr.s6_addr32, + tcp_hdr(skb)->dest, + tcp_hdr(skb)->source); } static int tcp_v6_pre_connect(struct sock *sk, struct sockaddr *uaddr, @@ -319,14 +315,16 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr, sk_set_txhash(sk); if (likely(!tp->repair)) { + union tcp_seq_and_ts_off st; + + st = secure_tcpv6_seq_and_ts_off(net, + np->saddr.s6_addr32, + sk->sk_v6_daddr.s6_addr32, + inet->inet_sport, + inet->inet_dport); if (!tp->write_seq) - WRITE_ONCE(tp->write_seq, - secure_tcpv6_seq(np->saddr.s6_addr32, - sk->sk_v6_daddr.s6_addr32, - inet->inet_sport, - inet->inet_dport)); - tp->tsoffset = secure_tcpv6_ts_off(net, np->saddr.s6_addr32, - sk->sk_v6_daddr.s6_addr32); + WRITE_ONCE(tp->write_seq, st.seq); + tp->tsoffset = st.ts_off; } if (tcp_fastopen_defer_connect(sk, &err)) @@ -859,8 +857,7 @@ const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops = { .cookie_init_seq = cookie_v6_init_sequence, #endif .route_req = tcp_v6_route_req, - .init_seq = tcp_v6_init_seq, - .init_ts_off = tcp_v6_init_ts_off, + .init_seq_and_ts_off = tcp_v6_init_seq_and_ts_off, .send_synack = tcp_v6_send_synack, }; @@ -1131,7 +1128,8 @@ static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb, txhash = inet_twsk(sk)->tw_txhash; } } else { - if (net->ipv6.sysctl.flowlabel_reflect & FLOWLABEL_REFLECT_TCP_RESET) + if (READ_ONCE(net->ipv6.sysctl.flowlabel_reflect) & + FLOWLABEL_REFLECT_TCP_RESET) label = ip6_flowlabel(ipv6h); } @@ -1355,11 +1353,48 @@ static void tcp_v6_restore_cb(struct sk_buff *skb) sizeof(struct inet6_skb_parm)); } +/* Called from tcp_v4_syn_recv_sock() for v6_mapped children. */ +static void tcp_v6_mapped_child_init(struct sock *newsk, const struct sock *sk) +{ + struct inet_sock *newinet = inet_sk(newsk); + struct ipv6_pinfo *newnp; + + newinet->pinet6 = newnp = tcp_inet6_sk(newsk); + newinet->ipv6_fl_list = NULL; + + memcpy(newnp, tcp_inet6_sk(sk), sizeof(struct ipv6_pinfo)); + + newnp->saddr = newsk->sk_v6_rcv_saddr; + + inet_csk(newsk)->icsk_af_ops = &ipv6_mapped; + if (sk_is_mptcp(newsk)) + mptcpv6_handle_mapped(newsk, true); + newsk->sk_backlog_rcv = tcp_v4_do_rcv; +#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) + tcp_sk(newsk)->af_specific = &tcp_sock_ipv6_mapped_specific; +#endif + + newnp->ipv6_mc_list = NULL; + newnp->ipv6_ac_list = NULL; + newnp->pktoptions = NULL; + newnp->opt = NULL; + + /* tcp_v4_syn_recv_sock() has initialized newinet->mc_{index,ttl} */ + newnp->mcast_oif = newinet->mc_index; + newnp->mcast_hops = newinet->mc_ttl; + + newnp->rcv_flowinfo = 0; + if (inet6_test_bit(REPFLOW, sk)) + newnp->flow_label = 0; +} + static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, struct request_sock *req, struct dst_entry *dst, struct request_sock *req_unhash, - bool *own_req) + bool *own_req, + void (*opt_child_init)(struct sock *newsk, + const struct sock *sk)) { struct inet_request_sock *ireq; struct ipv6_pinfo *newnp; @@ -1375,60 +1410,10 @@ static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff * #endif struct flowi6 fl6; - if (skb->protocol == htons(ETH_P_IP)) { - /* - * v6 mapped - */ - - newsk = tcp_v4_syn_recv_sock(sk, skb, req, dst, - req_unhash, own_req); - - if (!newsk) - return NULL; - - inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk); - - newnp = tcp_inet6_sk(newsk); - newtp = tcp_sk(newsk); - - memcpy(newnp, np, sizeof(struct ipv6_pinfo)); - - newnp->saddr = newsk->sk_v6_rcv_saddr; - - inet_csk(newsk)->icsk_af_ops = &ipv6_mapped; - if (sk_is_mptcp(newsk)) - mptcpv6_handle_mapped(newsk, true); - newsk->sk_backlog_rcv = tcp_v4_do_rcv; -#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) - newtp->af_specific = &tcp_sock_ipv6_mapped_specific; -#endif - - newnp->ipv6_mc_list = NULL; - newnp->ipv6_ac_list = NULL; - newnp->ipv6_fl_list = NULL; - newnp->pktoptions = NULL; - newnp->opt = NULL; - newnp->mcast_oif = inet_iif(skb); - newnp->mcast_hops = ip_hdr(skb)->ttl; - newnp->rcv_flowinfo = 0; - if (inet6_test_bit(REPFLOW, sk)) - newnp->flow_label = 0; - - /* - * No need to charge this sock to the relevant IPv6 refcnt debug socks count - * here, tcp_create_openreq_child now does this for us, see the comment in - * that function for the gory details. -acme - */ - - /* It is tricky place. Until this moment IPv4 tcp - worked with IPv6 icsk.icsk_af_ops. - Sync it now. - */ - tcp_sync_mss(newsk, inet_csk(newsk)->icsk_pmtu_cookie); - - return newsk; - } - + if (skb->protocol == htons(ETH_P_IP)) + return tcp_v4_syn_recv_sock(sk, skb, req, dst, + req_unhash, own_req, + tcp_v6_mapped_child_init); ireq = inet_rsk(req); if (sk_acceptq_is_full(sk)) @@ -1453,10 +1438,12 @@ static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff * newsk->sk_gso_type = SKB_GSO_TCPV6; inet6_sk_rx_dst_set(newsk, skb); - inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk); + newinet = inet_sk(newsk); + newinet->pinet6 = tcp_inet6_sk(newsk); + newinet->ipv6_fl_list = NULL; + newinet->inet_opt = NULL; newtp = tcp_sk(newsk); - newinet = inet_sk(newsk); newnp = tcp_inet6_sk(newsk); memcpy(newnp, np, sizeof(struct ipv6_pinfo)); @@ -1469,10 +1456,8 @@ static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff * First: no IPv4 options. */ - newinet->inet_opt = NULL; newnp->ipv6_mc_list = NULL; newnp->ipv6_ac_list = NULL; - newnp->ipv6_fl_list = NULL; /* Clone RX bits */ newnp->rxopt.all = np->rxopt.all; diff --git a/net/ipv6/udplite.c b/net/ipv6/udplite.c index 2cec542437f74..e867721cda4d3 100644 --- a/net/ipv6/udplite.c +++ b/net/ipv6/udplite.c @@ -16,10 +16,9 @@ static int udplitev6_sk_init(struct sock *sk) { - udpv6_init_sock(sk); pr_warn_once("UDP-Lite is deprecated and scheduled to be removed in 2025, " "please contact the netdev mailing list\n"); - return 0; + return udpv6_init_sock(sk); } static int udplitev6_rcv(struct sk_buff *skb) diff --git a/net/ipv6/xfrm6_policy.c b/net/ipv6/xfrm6_policy.c index 1f19b6f14484c..125ea9a5b8a08 100644 --- a/net/ipv6/xfrm6_policy.c +++ b/net/ipv6/xfrm6_policy.c @@ -57,6 +57,7 @@ static int xfrm6_get_saddr(xfrm_address_t *saddr, struct dst_entry *dst; struct net_device *dev; struct inet6_dev *idev; + int err; dst = xfrm6_dst_lookup(params); if (IS_ERR(dst)) @@ -68,9 +69,11 @@ static int xfrm6_get_saddr(xfrm_address_t *saddr, return -EHOSTUNREACH; } dev = idev->dev; - ipv6_dev_get_saddr(dev_net(dev), dev, ¶ms->daddr->in6, 0, - &saddr->in6); + err = ipv6_dev_get_saddr(dev_net(dev), dev, ¶ms->daddr->in6, 0, + &saddr->in6); dst_release(dst); + if (err) + return -EHOSTUNREACH; return 0; } diff --git a/net/kcm/kcmsock.c b/net/kcm/kcmsock.c index b4f01cb075615..f6d44481954cd 100644 --- a/net/kcm/kcmsock.c +++ b/net/kcm/kcmsock.c @@ -628,7 +628,7 @@ static int kcm_write_msgs(struct kcm_sock *kcm) skb = txm->frag_skb; } - if (WARN_ON(!skb_shinfo(skb)->nr_frags) || + if (WARN_ON_ONCE(!skb_shinfo(skb)->nr_frags) || WARN_ON_ONCE(!skb_frag_page(&skb_shinfo(skb)->frags[0]))) { ret = -EINVAL; goto out; @@ -749,7 +749,7 @@ static int kcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) { struct sock *sk = sock->sk; struct kcm_sock *kcm = kcm_sk(sk); - struct sk_buff *skb = NULL, *head = NULL; + struct sk_buff *skb = NULL, *head = NULL, *frag_prev = NULL; size_t copy, copied = 0; long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); int eor = (sock->type == SOCK_DGRAM) ? @@ -824,6 +824,7 @@ static int kcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) else skb->next = tskb; + frag_prev = skb; skb = tskb; skb->ip_summed = CHECKSUM_UNNECESSARY; continue; @@ -933,6 +934,22 @@ static int kcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) out_error: kcm_push(kcm); + /* When MAX_SKB_FRAGS was reached, a new skb was allocated and + * linked into the frag_list before data copy. If the copy + * subsequently failed, this skb has zero frags. Remove it from + * the frag_list to prevent kcm_write_msgs from later hitting + * WARN_ON(!skb_shinfo(skb)->nr_frags). + */ + if (frag_prev && !skb_shinfo(skb)->nr_frags) { + if (head == frag_prev) + skb_shinfo(head)->frag_list = NULL; + else + frag_prev->next = NULL; + kfree_skb(skb); + /* Update skb as it may be saved in partial_message via goto */ + skb = frag_prev; + } + if (sock->type == SOCK_SEQPACKET) { /* Wrote some bytes before encountering an * error, return partial success. diff --git a/net/mac80211/mesh.c b/net/mac80211/mesh.c index f37068a533f4e..e235ab7a5651c 100644 --- a/net/mac80211/mesh.c +++ b/net/mac80211/mesh.c @@ -1629,6 +1629,9 @@ static void mesh_rx_csa_frame(struct ieee80211_sub_if_data *sdata, if (!mesh_matches_local(sdata, elems)) goto free; + if (!elems->mesh_chansw_params_ie) + goto free; + ifmsh->chsw_ttl = elems->mesh_chansw_params_ie->mesh_ttl; if (!--ifmsh->chsw_ttl) fwd_csa = false; diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index 8ba199cd38c0f..f119149bcc1c1 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -6977,6 +6977,9 @@ static void ieee80211_ml_reconfiguration(struct ieee80211_sub_if_data *sdata, control = le16_to_cpu(prof->control); link_id = control & IEEE80211_MLE_STA_RECONF_CONTROL_LINK_ID; + if (link_id >= IEEE80211_MLD_MAX_NUM_LINKS) + continue; + removed_links |= BIT(link_id); /* the MAC address should not be included, but handle it */ diff --git a/net/mptcp/pm.c b/net/mptcp/pm.c index 9604b91902b8b..e22d53af0ffa0 100644 --- a/net/mptcp/pm.c +++ b/net/mptcp/pm.c @@ -212,9 +212,24 @@ void mptcp_pm_send_ack(struct mptcp_sock *msk, spin_lock_bh(&msk->pm.lock); } -void mptcp_pm_addr_send_ack(struct mptcp_sock *msk) +static bool subflow_in_rm_list(const struct mptcp_subflow_context *subflow, + const struct mptcp_rm_list *rm_list) +{ + u8 i, id = subflow_get_local_id(subflow); + + for (i = 0; i < rm_list->nr; i++) { + if (rm_list->ids[i] == id) + return true; + } + + return false; +} + +static void +mptcp_pm_addr_send_ack_avoid_list(struct mptcp_sock *msk, + const struct mptcp_rm_list *rm_list) { - struct mptcp_subflow_context *subflow, *alt = NULL; + struct mptcp_subflow_context *subflow, *stale = NULL, *same_id = NULL; msk_owned_by_me(msk); lockdep_assert_held(&msk->pm.lock); @@ -224,19 +239,35 @@ void mptcp_pm_addr_send_ack(struct mptcp_sock *msk) return; mptcp_for_each_subflow(msk, subflow) { - if (__mptcp_subflow_active(subflow)) { - if (!subflow->stale) { - mptcp_pm_send_ack(msk, subflow, false, false); - return; - } + if (!__mptcp_subflow_active(subflow)) + continue; - if (!alt) - alt = subflow; + if (unlikely(subflow->stale)) { + if (!stale) + stale = subflow; + } else if (unlikely(rm_list && + subflow_in_rm_list(subflow, rm_list))) { + if (!same_id) + same_id = subflow; + } else { + goto send_ack; } } - if (alt) - mptcp_pm_send_ack(msk, alt, false, false); + if (same_id) + subflow = same_id; + else if (stale) + subflow = stale; + else + return; + +send_ack: + mptcp_pm_send_ack(msk, subflow, false, false); +} + +void mptcp_pm_addr_send_ack(struct mptcp_sock *msk) +{ + mptcp_pm_addr_send_ack_avoid_list(msk, NULL); } int mptcp_pm_mp_prio_send_ack(struct mptcp_sock *msk, @@ -470,7 +501,7 @@ int mptcp_pm_remove_addr(struct mptcp_sock *msk, const struct mptcp_rm_list *rm_ msk->pm.rm_list_tx = *rm_list; rm_addr |= BIT(MPTCP_RM_ADDR_SIGNAL); WRITE_ONCE(msk->pm.addr_signal, rm_addr); - mptcp_pm_addr_send_ack(msk); + mptcp_pm_addr_send_ack_avoid_list(msk, rm_list); return 0; } diff --git a/net/mptcp/pm_kernel.c b/net/mptcp/pm_kernel.c index 1b517a81e0299..6fd393f451bf4 100644 --- a/net/mptcp/pm_kernel.c +++ b/net/mptcp/pm_kernel.c @@ -407,6 +407,15 @@ static void mptcp_pm_create_subflow_or_signal_addr(struct mptcp_sock *msk) } exit: + /* If an endpoint has both the signal and subflow flags, but it is not + * possible to create subflows -- the 'while' loop body above never + * executed -- then still mark the endp as used, which is somehow the + * case. This avoids issues later when removing the endpoint and calling + * __mark_subflow_endp_available(), which expects the increment here. + */ + if (signal_and_subflow && local.addr.id != msk->mpc_endpoint_id) + msk->pm.local_addr_used++; + mptcp_pm_nl_check_work_pending(msk); } @@ -1042,10 +1051,8 @@ static bool mptcp_pm_remove_anno_addr(struct mptcp_sock *msk, ret = mptcp_remove_anno_list_by_saddr(msk, addr); if (ret || force) { spin_lock_bh(&msk->pm.lock); - if (ret) { - __set_bit(addr->id, msk->pm.id_avail_bitmap); + if (ret) msk->pm.add_addr_signaled--; - } mptcp_pm_remove_addr(msk, &list); spin_unlock_bh(&msk->pm.lock); } @@ -1083,17 +1090,15 @@ static int mptcp_nl_remove_subflow_and_signal_addr(struct net *net, !(entry->flags & MPTCP_PM_ADDR_FLAG_IMPLICIT)); list.ids[0] = mptcp_endp_get_local_id(msk, addr); - if (remove_subflow) { - spin_lock_bh(&msk->pm.lock); - mptcp_pm_rm_subflow(msk, &list); - spin_unlock_bh(&msk->pm.lock); - } - if (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW) { - spin_lock_bh(&msk->pm.lock); + spin_lock_bh(&msk->pm.lock); + if (remove_subflow) + mptcp_pm_rm_subflow(msk, &list); + if (entry->flags & MPTCP_PM_ADDR_FLAG_SUBFLOW) __mark_subflow_endp_available(msk, list.ids[0]); - spin_unlock_bh(&msk->pm.lock); - } + else /* mark endp ID as available, e.g. Signal or MPC endp */ + __set_bit(addr->id, msk->pm.id_avail_bitmap); + spin_unlock_bh(&msk->pm.lock); if (msk->mpc_endpoint_id == entry->addr.id) msk->mpc_endpoint_id = 0; diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index e4bb7e2d7b19a..8f18509204b67 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -224,9 +224,6 @@ static bool mptcp_rcvbuf_grow(struct sock *sk, u32 newval) do_div(grow, oldval); rcvwin += grow << 1; - if (!RB_EMPTY_ROOT(&msk->out_of_order_queue)) - rcvwin += MPTCP_SKB_CB(msk->ooo_last_skb)->end_seq - msk->ack_seq; - cap = READ_ONCE(net->ipv4.sysctl_tcp_rmem[2]); rcvbuf = min_t(u32, mptcp_space_from_win(sk, rcvwin), cap); @@ -350,9 +347,6 @@ static void mptcp_data_queue_ofo(struct mptcp_sock *msk, struct sk_buff *skb) end: skb_condense(skb); skb_set_owner_r(skb, sk); - /* do not grow rcvbuf for not-yet-accepted or orphaned sockets. */ - if (sk->sk_socket) - mptcp_rcvbuf_grow(sk, msk->rcvq_space.space); } static void mptcp_init_skb(struct sock *ssk, struct sk_buff *skb, int offset, @@ -2062,8 +2056,8 @@ static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) msk->rcvq_space.copied += copied; - mstamp = div_u64(tcp_clock_ns(), NSEC_PER_USEC); - time = tcp_stamp_us_delta(mstamp, msk->rcvq_space.time); + mstamp = mptcp_stamp(); + time = tcp_stamp_us_delta(mstamp, READ_ONCE(msk->rcvq_space.time)); rtt_us = msk->rcvq_space.rtt_us; if (rtt_us && time < (rtt_us >> 3)) @@ -3427,6 +3421,7 @@ struct sock *mptcp_sk_clone_init(const struct sock *sk, __mptcp_propagate_sndbuf(nsk, ssk); mptcp_rcv_space_init(msk, ssk); + msk->rcvq_space.time = mptcp_stamp(); if (mp_opt->suboptions & OPTION_MPTCP_MPC_ACK) __mptcp_subflow_fully_established(msk, subflow, mp_opt); @@ -3444,8 +3439,6 @@ void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk) msk->rcvq_space.copied = 0; msk->rcvq_space.rtt_us = 0; - msk->rcvq_space.time = tp->tcp_mstamp; - /* initial rcv_space offering made to peer */ msk->rcvq_space.space = min_t(u32, tp->rcv_wnd, TCP_INIT_CWND * tp->advmss); @@ -3658,6 +3651,7 @@ void mptcp_finish_connect(struct sock *ssk) * accessing the field below */ WRITE_ONCE(msk->local_key, subflow->local_key); + WRITE_ONCE(msk->rcvq_space.time, mptcp_stamp()); mptcp_pm_new_connection(msk, ssk, 0); } diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h index 30d5e57197936..27b1698c5aa2d 100644 --- a/net/mptcp/protocol.h +++ b/net/mptcp/protocol.h @@ -870,6 +870,11 @@ static inline bool mptcp_is_fully_established(struct sock *sk) READ_ONCE(mptcp_sk(sk)->fully_established); } +static inline u64 mptcp_stamp(void) +{ + return div_u64(tcp_clock_ns(), NSEC_PER_USEC); +} + void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk); void mptcp_data_ready(struct sock *sk, struct sock *ssk); bool mptcp_finish_join(struct sock *sk); diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c index a8c281bc53778..838203e2740e5 100644 --- a/net/mptcp/subflow.c +++ b/net/mptcp/subflow.c @@ -807,7 +807,9 @@ static struct sock *subflow_syn_recv_sock(const struct sock *sk, struct request_sock *req, struct dst_entry *dst, struct request_sock *req_unhash, - bool *own_req) + bool *own_req, + void (*opt_child_init)(struct sock *newsk, + const struct sock *sk)) { struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk); struct mptcp_subflow_request_sock *subflow_req; @@ -854,7 +856,7 @@ static struct sock *subflow_syn_recv_sock(const struct sock *sk, create_child: child = listener->icsk_af_ops->syn_recv_sock(sk, skb, req, dst, - req_unhash, own_req); + req_unhash, own_req, opt_child_init); if (child && *own_req) { struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(child); diff --git a/net/netfilter/ipvs/ip_vs_xmit.c b/net/netfilter/ipvs/ip_vs_xmit.c index 618fbe1240b54..ecbcdc43263d6 100644 --- a/net/netfilter/ipvs/ip_vs_xmit.c +++ b/net/netfilter/ipvs/ip_vs_xmit.c @@ -295,6 +295,12 @@ static inline bool decrement_ttl(struct netns_ipvs *ipvs, return true; } +/* rt has device that is down */ +static bool rt_dev_is_down(const struct net_device *dev) +{ + return dev && !netif_running(dev); +} + /* Get route to destination or remote server */ static int __ip_vs_get_out_rt(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, @@ -310,9 +316,11 @@ __ip_vs_get_out_rt(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, if (dest) { dest_dst = __ip_vs_dst_check(dest); - if (likely(dest_dst)) + if (likely(dest_dst)) { rt = dst_rtable(dest_dst->dst_cache); - else { + if (ret_saddr) + *ret_saddr = dest_dst->dst_saddr.ip; + } else { dest_dst = ip_vs_dest_dst_alloc(); spin_lock_bh(&dest->dst_lock); if (!dest_dst) { @@ -328,14 +336,22 @@ __ip_vs_get_out_rt(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, ip_vs_dest_dst_free(dest_dst); goto err_unreach; } - __ip_vs_dst_set(dest, dest_dst, &rt->dst, 0); + /* It is forbidden to attach dest->dest_dst if + * device is going down. + */ + if (!rt_dev_is_down(dst_dev_rcu(&rt->dst))) + __ip_vs_dst_set(dest, dest_dst, &rt->dst, 0); + else + noref = 0; spin_unlock_bh(&dest->dst_lock); IP_VS_DBG(10, "new dst %pI4, src %pI4, refcnt=%d\n", &dest->addr.ip, &dest_dst->dst_saddr.ip, rcuref_read(&rt->dst.__rcuref)); + if (ret_saddr) + *ret_saddr = dest_dst->dst_saddr.ip; + if (!noref) + ip_vs_dest_dst_free(dest_dst); } - if (ret_saddr) - *ret_saddr = dest_dst->dst_saddr.ip; } else { noref = 0; @@ -472,9 +488,11 @@ __ip_vs_get_out_rt_v6(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, if (dest) { dest_dst = __ip_vs_dst_check(dest); - if (likely(dest_dst)) + if (likely(dest_dst)) { rt = dst_rt6_info(dest_dst->dst_cache); - else { + if (ret_saddr) + *ret_saddr = dest_dst->dst_saddr.in6; + } else { u32 cookie; dest_dst = ip_vs_dest_dst_alloc(); @@ -495,14 +513,22 @@ __ip_vs_get_out_rt_v6(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, } rt = dst_rt6_info(dst); cookie = rt6_get_cookie(rt); - __ip_vs_dst_set(dest, dest_dst, &rt->dst, cookie); + /* It is forbidden to attach dest->dest_dst if + * device is going down. + */ + if (!rt_dev_is_down(dst_dev_rcu(&rt->dst))) + __ip_vs_dst_set(dest, dest_dst, &rt->dst, cookie); + else + noref = 0; spin_unlock_bh(&dest->dst_lock); IP_VS_DBG(10, "new dst %pI6, src %pI6, refcnt=%d\n", &dest->addr.in6, &dest_dst->dst_saddr.in6, rcuref_read(&rt->dst.__rcuref)); + if (ret_saddr) + *ret_saddr = dest_dst->dst_saddr.in6; + if (!noref) + ip_vs_dest_dst_free(dest_dst); } - if (ret_saddr) - *ret_saddr = dest_dst->dst_saddr.in6; } else { noref = 0; dst = __ip_vs_route_output_v6(net, daddr, ret_saddr, do_xfrm, diff --git a/net/netfilter/nf_conncount.c b/net/netfilter/nf_conncount.c index 828d5c64c68a3..14e62b3263cd9 100644 --- a/net/netfilter/nf_conncount.c +++ b/net/netfilter/nf_conncount.c @@ -34,8 +34,9 @@ #define CONNCOUNT_SLOTS 256U -#define CONNCOUNT_GC_MAX_NODES 8 -#define MAX_KEYLEN 5 +#define CONNCOUNT_GC_MAX_NODES 8 +#define CONNCOUNT_GC_MAX_COLLECT 64 +#define MAX_KEYLEN 5 /* we will save the tuples of all connections we care about */ struct nf_conncount_tuple { @@ -178,16 +179,28 @@ static int __nf_conncount_add(struct net *net, return -ENOENT; if (ct && nf_ct_is_confirmed(ct)) { - err = -EEXIST; - goto out_put; + /* local connections are confirmed in postrouting so confirmation + * might have happened before hitting connlimit + */ + if (skb->skb_iif != LOOPBACK_IFINDEX) { + err = -EEXIST; + goto out_put; + } + + /* this is likely a local connection, skip optimization to avoid + * adding duplicates from a 'packet train' + */ + goto check_connections; } - if ((u32)jiffies == list->last_gc) + if ((u32)jiffies == list->last_gc && + (list->count - list->last_gc_count) < CONNCOUNT_GC_MAX_COLLECT) goto add_new_node; +check_connections: /* check the saved connections */ list_for_each_entry_safe(conn, conn_n, &list->head, node) { - if (collect > CONNCOUNT_GC_MAX_NODES) + if (collect > CONNCOUNT_GC_MAX_COLLECT) break; found = find_or_evict(net, list, conn); @@ -230,6 +243,7 @@ static int __nf_conncount_add(struct net *net, nf_ct_put(found_ct); } list->last_gc = (u32)jiffies; + list->last_gc_count = list->count; add_new_node: if (WARN_ON_ONCE(list->count > INT_MAX)) { @@ -277,13 +291,14 @@ void nf_conncount_list_init(struct nf_conncount_list *list) spin_lock_init(&list->list_lock); INIT_LIST_HEAD(&list->head); list->count = 0; + list->last_gc_count = 0; list->last_gc = (u32)jiffies; } EXPORT_SYMBOL_GPL(nf_conncount_list_init); /* Return true if the list is empty. Must be called with BH disabled. */ -bool nf_conncount_gc_list(struct net *net, - struct nf_conncount_list *list) +static bool __nf_conncount_gc_list(struct net *net, + struct nf_conncount_list *list) { const struct nf_conntrack_tuple_hash *found; struct nf_conncount_tuple *conn, *conn_n; @@ -295,10 +310,6 @@ bool nf_conncount_gc_list(struct net *net, if ((u32)jiffies == READ_ONCE(list->last_gc)) return false; - /* don't bother if other cpu is already doing GC */ - if (!spin_trylock(&list->list_lock)) - return false; - list_for_each_entry_safe(conn, conn_n, &list->head, node) { found = find_or_evict(net, list, conn); if (IS_ERR(found)) { @@ -320,14 +331,29 @@ bool nf_conncount_gc_list(struct net *net, } nf_ct_put(found_ct); - if (collected > CONNCOUNT_GC_MAX_NODES) + if (collected > CONNCOUNT_GC_MAX_COLLECT) break; } if (!list->count) ret = true; list->last_gc = (u32)jiffies; - spin_unlock(&list->list_lock); + list->last_gc_count = list->count; + + return ret; +} + +bool nf_conncount_gc_list(struct net *net, + struct nf_conncount_list *list) +{ + bool ret; + + /* don't bother if other cpu is already doing GC */ + if (!spin_trylock_bh(&list->list_lock)) + return false; + + ret = __nf_conncount_gc_list(net, list); + spin_unlock_bh(&list->list_lock); return ret; } diff --git a/net/netfilter/nf_conntrack_h323_asn1.c b/net/netfilter/nf_conntrack_h323_asn1.c index 540d97715bd23..62aa22a078769 100644 --- a/net/netfilter/nf_conntrack_h323_asn1.c +++ b/net/netfilter/nf_conntrack_h323_asn1.c @@ -796,7 +796,7 @@ static int decode_choice(struct bitstr *bs, const struct field_t *f, if (ext || (son->attr & OPEN)) { BYTE_ALIGN(bs); - if (nf_h323_error_boundary(bs, len, 0)) + if (nf_h323_error_boundary(bs, 2, 0)) return H323_ERROR_BOUND; len = get_len(bs); if (nf_h323_error_boundary(bs, len, 0)) diff --git a/net/netfilter/nf_conntrack_h323_main.c b/net/netfilter/nf_conntrack_h323_main.c index 14f73872f6477..e35814d68ce30 100644 --- a/net/netfilter/nf_conntrack_h323_main.c +++ b/net/netfilter/nf_conntrack_h323_main.c @@ -1186,13 +1186,13 @@ static struct nf_conntrack_expect *find_expect(struct nf_conn *ct, { struct net *net = nf_ct_net(ct); struct nf_conntrack_expect *exp; - struct nf_conntrack_tuple tuple; + struct nf_conntrack_tuple tuple = { + .src.l3num = nf_ct_l3num(ct), + .dst.protonum = IPPROTO_TCP, + .dst.u.tcp.port = port, + }; - memset(&tuple.src.u3, 0, sizeof(tuple.src.u3)); - tuple.src.u.tcp.port = 0; memcpy(&tuple.dst.u3, addr, sizeof(tuple.dst.u3)); - tuple.dst.u.tcp.port = port; - tuple.dst.protonum = IPPROTO_TCP; exp = __nf_ct_expect_find(net, nf_ct_zone(ct), &tuple); if (exp && exp->master == ct) diff --git a/net/netfilter/nf_conntrack_proto_generic.c b/net/netfilter/nf_conntrack_proto_generic.c index e831637bc8ca8..cb260eb3d012c 100644 --- a/net/netfilter/nf_conntrack_proto_generic.c +++ b/net/netfilter/nf_conntrack_proto_generic.c @@ -67,6 +67,7 @@ void nf_conntrack_generic_init_net(struct net *net) const struct nf_conntrack_l4proto nf_conntrack_l4proto_generic = { .l4proto = 255, + .allow_clash = true, #ifdef CONFIG_NF_CONNTRACK_TIMEOUT .ctnl_timeout = { .nlattr_to_obj = generic_timeout_nlattr_to_obj, diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index 6059a299004d4..598a9fe03fb0b 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -832,6 +832,11 @@ static void nft_map_catchall_deactivate(const struct nft_ctx *ctx, } } +/* Use NFT_ITER_UPDATE iterator even if this may be called from the preparation + * phase, the set clone might already exist from a previous command, or it might + * be a set that is going away and does not require a clone. The netns and + * netlink release paths also need to work on the live set. + */ static void nft_map_deactivate(const struct nft_ctx *ctx, struct nft_set *set) { struct nft_set_iter iter = { @@ -2822,6 +2827,7 @@ static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 policy, err_register_hook: nft_chain_del(chain); + synchronize_rcu(); err_chain_add: nft_trans_destroy(trans); err_trans: @@ -7287,6 +7293,7 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, struct nft_data_desc desc; enum nft_registers dreg; struct nft_trans *trans; + bool set_full = false; u64 expiration; u64 timeout; int err, i; @@ -7573,10 +7580,18 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, if (err < 0) goto err_elem_free; + if (!(flags & NFT_SET_ELEM_CATCHALL)) { + unsigned int max = nft_set_maxsize(set), nelems; + + nelems = atomic_inc_return(&set->nelems); + if (nelems > max) + set_full = true; + } + trans = nft_trans_elem_alloc(ctx, NFT_MSG_NEWSETELEM, set); if (trans == NULL) { err = -ENOMEM; - goto err_elem_free; + goto err_set_size; } ext->genmask = nft_genmask_cur(ctx->net); @@ -7628,7 +7643,7 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, ue->priv = elem_priv; nft_trans_commit_list_add_elem(ctx->net, trans); - goto err_elem_free; + goto err_set_size; } } } @@ -7637,27 +7652,25 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, * and an existing one. */ err = -EEXIST; + } else if (err == -ECANCELED) { + /* ECANCELED reports an existing nul-element in + * interval sets. + */ + err = 0; } goto err_element_clash; } - if (!(flags & NFT_SET_ELEM_CATCHALL)) { - unsigned int max = nft_set_maxsize(set); - - if (!atomic_add_unless(&set->nelems, 1, max)) { - err = -ENFILE; - goto err_set_full; - } - } - nft_trans_container_elem(trans)->elems[0].priv = elem.priv; nft_trans_commit_list_add_elem(ctx->net, trans); - return 0; -err_set_full: - nft_setelem_remove(ctx->net, set, elem.priv); + return set_full ? -ENFILE : 0; + err_element_clash: kfree(trans); +err_set_size: + if (!(flags & NFT_SET_ELEM_CATCHALL)) + atomic_dec(&set->nelems); err_elem_free: nf_tables_set_elem_destroy(ctx, set, elem.priv); err_parse_data: @@ -7808,7 +7821,8 @@ static bool nft_trans_elems_new_abort(const struct nft_ctx *ctx, continue; } - if (!te->set->ops->abort || nft_setelem_is_catchall(te->set, te->elems[i].priv)) + if (!te->set->ops->abort_skip_removal || + nft_setelem_is_catchall(te->set, te->elems[i].priv)) nft_setelem_remove(ctx->net, te->set, te->elems[i].priv); if (!nft_setelem_is_catchall(te->set, te->elems[i].priv)) @@ -8001,9 +8015,12 @@ static int nft_set_catchall_flush(const struct nft_ctx *ctx, static int nft_set_flush(struct nft_ctx *ctx, struct nft_set *set, u8 genmask) { + /* The set backend might need to clone the set, do it now from the + * preparation phase, use NFT_ITER_UPDATE_CLONE iterator type. + */ struct nft_set_iter iter = { .genmask = genmask, - .type = NFT_ITER_UPDATE, + .type = NFT_ITER_UPDATE_CLONE, .fn = nft_setelem_flush, }; @@ -10629,11 +10646,6 @@ static void nft_trans_gc_queue_work(struct nft_trans_gc *trans) schedule_work(&trans_gc_work); } -static int nft_trans_gc_space(struct nft_trans_gc *trans) -{ - return NFT_TRANS_GC_BATCHCOUNT - trans->count; -} - struct nft_trans_gc *nft_trans_gc_queue_async(struct nft_trans_gc *gc, unsigned int gc_seq, gfp_t gfp) { @@ -11538,6 +11550,13 @@ static int nf_tables_abort(struct net *net, struct sk_buff *skb, ret = __nf_tables_abort(net, action); nft_gc_seq_end(nft_net, gc_seq); + if (action == NFNL_ABORT_NONE) { + struct nft_table *table; + + list_for_each_entry(table, &nft_net->tables, list) + table->validate_state = NFT_VALIDATE_SKIP; + } + WARN_ON_ONCE(!list_empty(&nft_net->commit_list)); /* module autoload needs to happen after GC sequence update because it diff --git a/net/netfilter/nfnetlink_queue.c b/net/netfilter/nfnetlink_queue.c index 8b7b39d8a1091..34548213f2f14 100644 --- a/net/netfilter/nfnetlink_queue.c +++ b/net/netfilter/nfnetlink_queue.c @@ -30,6 +30,8 @@ #include #include #include +#include +#include #include #include #include @@ -47,6 +49,8 @@ #endif #define NFQNL_QMAX_DEFAULT 1024 +#define NFQNL_HASH_MIN 1024 +#define NFQNL_HASH_MAX 1048576 /* We're using struct nlattr which has 16bit nla_len. Note that nla_len * includes the header length. Thus, the maximum packet length that we @@ -56,6 +60,26 @@ */ #define NFQNL_MAX_COPY_RANGE (0xffff - NLA_HDRLEN) +/* Composite key for packet lookup: (net, queue_num, packet_id) */ +struct nfqnl_packet_key { + possible_net_t net; + u32 packet_id; + u16 queue_num; +} __aligned(sizeof(u32)); /* jhash2 requires 32-bit alignment */ + +/* Global rhashtable - one for entire system, all netns */ +static struct rhashtable nfqnl_packet_map __read_mostly; + +/* Helper to initialize composite key */ +static inline void nfqnl_init_key(struct nfqnl_packet_key *key, + struct net *net, u32 packet_id, u16 queue_num) +{ + memset(key, 0, sizeof(*key)); + write_pnet(&key->net, net); + key->packet_id = packet_id; + key->queue_num = queue_num; +} + struct nfqnl_instance { struct hlist_node hlist; /* global list of queues */ struct rcu_head rcu; @@ -100,6 +124,39 @@ static inline u_int8_t instance_hashfn(u_int16_t queue_num) return ((queue_num >> 8) ^ queue_num) % INSTANCE_BUCKETS; } +/* Extract composite key from nf_queue_entry for hashing */ +static u32 nfqnl_packet_obj_hashfn(const void *data, u32 len, u32 seed) +{ + const struct nf_queue_entry *entry = data; + struct nfqnl_packet_key key; + + nfqnl_init_key(&key, entry->state.net, entry->id, entry->queue_num); + + return jhash2((u32 *)&key, sizeof(key) / sizeof(u32), seed); +} + +/* Compare stack-allocated key against entry */ +static int nfqnl_packet_obj_cmpfn(struct rhashtable_compare_arg *arg, + const void *obj) +{ + const struct nfqnl_packet_key *key = arg->key; + const struct nf_queue_entry *entry = obj; + + return !net_eq(entry->state.net, read_pnet(&key->net)) || + entry->queue_num != key->queue_num || + entry->id != key->packet_id; +} + +static const struct rhashtable_params nfqnl_rhashtable_params = { + .head_offset = offsetof(struct nf_queue_entry, hash_node), + .key_len = sizeof(struct nfqnl_packet_key), + .obj_hashfn = nfqnl_packet_obj_hashfn, + .obj_cmpfn = nfqnl_packet_obj_cmpfn, + .automatic_shrinking = true, + .min_size = NFQNL_HASH_MIN, + .max_size = NFQNL_HASH_MAX, +}; + static struct nfqnl_instance * instance_lookup(struct nfnl_queue_net *q, u_int16_t queue_num) { @@ -191,33 +248,45 @@ instance_destroy(struct nfnl_queue_net *q, struct nfqnl_instance *inst) spin_unlock(&q->instances_lock); } -static inline void +static int __enqueue_entry(struct nfqnl_instance *queue, struct nf_queue_entry *entry) { - list_add_tail(&entry->list, &queue->queue_list); - queue->queue_total++; + int err; + + entry->queue_num = queue->queue_num; + + err = rhashtable_insert_fast(&nfqnl_packet_map, &entry->hash_node, + nfqnl_rhashtable_params); + if (unlikely(err)) + return err; + + list_add_tail(&entry->list, &queue->queue_list); + queue->queue_total++; + + return 0; } static void __dequeue_entry(struct nfqnl_instance *queue, struct nf_queue_entry *entry) { + rhashtable_remove_fast(&nfqnl_packet_map, &entry->hash_node, + nfqnl_rhashtable_params); list_del(&entry->list); queue->queue_total--; } static struct nf_queue_entry * -find_dequeue_entry(struct nfqnl_instance *queue, unsigned int id) +find_dequeue_entry(struct nfqnl_instance *queue, unsigned int id, + struct net *net) { - struct nf_queue_entry *entry = NULL, *i; + struct nfqnl_packet_key key; + struct nf_queue_entry *entry; - spin_lock_bh(&queue->lock); + nfqnl_init_key(&key, net, id, queue->queue_num); - list_for_each_entry(i, &queue->queue_list, list) { - if (i->id == id) { - entry = i; - break; - } - } + spin_lock_bh(&queue->lock); + entry = rhashtable_lookup_fast(&nfqnl_packet_map, &key, + nfqnl_rhashtable_params); if (entry) __dequeue_entry(queue, entry); @@ -369,6 +438,34 @@ static void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict) nf_queue_entry_free(entry); } +/* return true if the entry has an unconfirmed conntrack attached that isn't owned by us + * exclusively. + */ +static bool nf_ct_drop_unconfirmed(const struct nf_queue_entry *entry, bool *is_unconfirmed) +{ +#if IS_ENABLED(CONFIG_NF_CONNTRACK) + struct nf_conn *ct = (void *)skb_nfct(entry->skb); + + if (!ct || nf_ct_is_confirmed(ct)) + return false; + + if (is_unconfirmed) + *is_unconfirmed = true; + + /* in some cases skb_clone() can occur after initial conntrack + * pickup, but conntrack assumes exclusive skb->_nfct ownership for + * unconfirmed entries. + * + * This happens for br_netfilter and with ip multicast routing. + * This can't be solved with serialization here because one clone + * could have been queued for local delivery or could be transmitted + * in parallel on another CPU. + */ + return refcount_read(&ct->ct_general.use) > 1; +#endif + return false; +} + static void nfqnl_reinject(struct nf_queue_entry *entry, unsigned int verdict) { const struct nf_ct_hook *ct_hook; @@ -396,6 +493,24 @@ static void nfqnl_reinject(struct nf_queue_entry *entry, unsigned int verdict) break; } } + + if (verdict != NF_DROP && entry->nf_ct_is_unconfirmed) { + /* If first queued segment was already reinjected then + * there is a good chance the ct entry is now confirmed. + * + * Handle the rare cases: + * - out-of-order verdict + * - threaded userspace reinjecting in parallel + * - first segment was dropped + * + * In all of those cases we can't handle this packet + * because we can't be sure that another CPU won't modify + * nf_conn->ext in parallel which isn't allowed. + */ + if (nf_ct_drop_unconfirmed(entry, NULL)) + verdict = NF_DROP; + } + nf_reinject(entry, verdict); } @@ -407,8 +522,7 @@ nfqnl_flush(struct nfqnl_instance *queue, nfqnl_cmpfn cmpfn, unsigned long data) spin_lock_bh(&queue->lock); list_for_each_entry_safe(entry, next, &queue->queue_list, list) { if (!cmpfn || cmpfn(entry, data)) { - list_del(&entry->list); - queue->queue_total--; + __dequeue_entry(queue, entry); nfqnl_reinject(entry, NF_DROP); } } @@ -826,49 +940,6 @@ nfqnl_build_packet_message(struct net *net, struct nfqnl_instance *queue, return NULL; } -static bool nf_ct_drop_unconfirmed(const struct nf_queue_entry *entry) -{ -#if IS_ENABLED(CONFIG_NF_CONNTRACK) - static const unsigned long flags = IPS_CONFIRMED | IPS_DYING; - struct nf_conn *ct = (void *)skb_nfct(entry->skb); - unsigned long status; - unsigned int use; - - if (!ct) - return false; - - status = READ_ONCE(ct->status); - if ((status & flags) == IPS_DYING) - return true; - - if (status & IPS_CONFIRMED) - return false; - - /* in some cases skb_clone() can occur after initial conntrack - * pickup, but conntrack assumes exclusive skb->_nfct ownership for - * unconfirmed entries. - * - * This happens for br_netfilter and with ip multicast routing. - * We can't be solved with serialization here because one clone could - * have been queued for local delivery. - */ - use = refcount_read(&ct->ct_general.use); - if (likely(use == 1)) - return false; - - /* Can't decrement further? Exclusive ownership. */ - if (!refcount_dec_not_one(&ct->ct_general.use)) - return false; - - skb_set_nfct(entry->skb, 0); - /* No nf_ct_put(): we already decremented .use and it cannot - * drop down to 0. - */ - return true; -#endif - return false; -} - static int __nfqnl_enqueue_packet(struct net *net, struct nfqnl_instance *queue, struct nf_queue_entry *entry) @@ -885,26 +956,23 @@ __nfqnl_enqueue_packet(struct net *net, struct nfqnl_instance *queue, } spin_lock_bh(&queue->lock); - if (nf_ct_drop_unconfirmed(entry)) - goto err_out_free_nskb; + if (queue->queue_total >= queue->queue_maxlen) + goto err_out_queue_drop; - if (queue->queue_total >= queue->queue_maxlen) { - if (queue->flags & NFQA_CFG_F_FAIL_OPEN) { - failopen = 1; - err = 0; - } else { - queue->queue_dropped++; - net_warn_ratelimited("nf_queue: full at %d entries, dropping packets(s)\n", - queue->queue_total); - } - goto err_out_free_nskb; - } entry->id = ++queue->id_sequence; *packet_id_ptr = htonl(entry->id); + /* Insert into hash BEFORE unicast. If failure don't send to userspace. */ + err = __enqueue_entry(queue, entry); + if (unlikely(err)) + goto err_out_queue_drop; + /* nfnetlink_unicast will either free the nskb or add it to a socket */ err = nfnetlink_unicast(nskb, net, queue->peer_portid); if (err < 0) { + /* Unicast failed - remove entry we just inserted */ + __dequeue_entry(queue, entry); + if (queue->flags & NFQA_CFG_F_FAIL_OPEN) { failopen = 1; err = 0; @@ -914,12 +982,22 @@ __nfqnl_enqueue_packet(struct net *net, struct nfqnl_instance *queue, goto err_out_unlock; } - __enqueue_entry(queue, entry); - spin_unlock_bh(&queue->lock); return 0; -err_out_free_nskb: +err_out_queue_drop: + if (queue->flags & NFQA_CFG_F_FAIL_OPEN) { + failopen = 1; + err = 0; + } else { + queue->queue_dropped++; + + if (queue->queue_total >= queue->queue_maxlen) + net_warn_ratelimited("nf_queue: full at %d entries, dropping packets(s)\n", + queue->queue_total); + else + net_warn_ratelimited("nf_queue: hash insert failed: %d\n", err); + } kfree_skb(nskb); err_out_unlock: spin_unlock_bh(&queue->lock); @@ -998,9 +1076,10 @@ __nfqnl_enqueue_packet_gso(struct net *net, struct nfqnl_instance *queue, static int nfqnl_enqueue_packet(struct nf_queue_entry *entry, unsigned int queuenum) { - unsigned int queued; - struct nfqnl_instance *queue; struct sk_buff *skb, *segs, *nskb; + bool ct_is_unconfirmed = false; + struct nfqnl_instance *queue; + unsigned int queued; int err = -ENOBUFS; struct net *net = entry->state.net; struct nfnl_queue_net *q = nfnl_queue_pernet(net); @@ -1024,6 +1103,15 @@ nfqnl_enqueue_packet(struct nf_queue_entry *entry, unsigned int queuenum) break; } + /* Check if someone already holds another reference to + * unconfirmed ct. If so, we cannot queue the skb: + * concurrent modifications of nf_conn->ext are not + * allowed and we can't know if another CPU isn't + * processing the same nf_conn entry in parallel. + */ + if (nf_ct_drop_unconfirmed(entry, &ct_is_unconfirmed)) + return -EINVAL; + if (!skb_is_gso(skb) || ((queue->flags & NFQA_CFG_F_GSO) && !skb_is_gso_sctp(skb))) return __nfqnl_enqueue_packet(net, queue, entry); @@ -1037,7 +1125,23 @@ nfqnl_enqueue_packet(struct nf_queue_entry *entry, unsigned int queuenum) goto out_err; queued = 0; err = 0; + skb_list_walk_safe(segs, segs, nskb) { + if (ct_is_unconfirmed && queued > 0) { + /* skb_gso_segment() increments the ct refcount. + * This is a problem for unconfirmed (not in hash) + * entries, those can race when reinjections happen + * in parallel. + * + * Annotate this for all queued entries except the + * first one. + * + * As long as the first one is reinjected first it + * will do the confirmation for us. + */ + entry->nf_ct_is_unconfirmed = ct_is_unconfirmed; + } + if (err == 0) err = __nfqnl_enqueue_packet_gso(net, queue, segs, entry); @@ -1430,7 +1534,7 @@ static int nfqnl_recv_verdict(struct sk_buff *skb, const struct nfnl_info *info, verdict = ntohl(vhdr->verdict); - entry = find_dequeue_entry(queue, ntohl(vhdr->id)); + entry = find_dequeue_entry(queue, ntohl(vhdr->id), info->net); if (entry == NULL) return -ENOENT; @@ -1781,10 +1885,14 @@ static int __init nfnetlink_queue_init(void) { int status; + status = rhashtable_init(&nfqnl_packet_map, &nfqnl_rhashtable_params); + if (status < 0) + return status; + status = register_pernet_subsys(&nfnl_queue_net_ops); if (status < 0) { pr_err("failed to register pernet ops\n"); - goto out; + goto cleanup_rhashtable; } netlink_register_notifier(&nfqnl_rtnl_notifier); @@ -1809,7 +1917,8 @@ static int __init nfnetlink_queue_init(void) cleanup_netlink_notifier: netlink_unregister_notifier(&nfqnl_rtnl_notifier); unregister_pernet_subsys(&nfnl_queue_net_ops); -out: +cleanup_rhashtable: + rhashtable_destroy(&nfqnl_packet_map); return status; } @@ -1821,6 +1930,8 @@ static void __exit nfnetlink_queue_fini(void) netlink_unregister_notifier(&nfqnl_rtnl_notifier); unregister_pernet_subsys(&nfnl_queue_net_ops); + rhashtable_destroy(&nfqnl_packet_map); + rcu_barrier(); /* Wait for completion of call_rcu()'s */ } diff --git a/net/netfilter/nft_compat.c b/net/netfilter/nft_compat.c index 72711d62fddfa..08f620311b03f 100644 --- a/net/netfilter/nft_compat.c +++ b/net/netfilter/nft_compat.c @@ -134,7 +134,8 @@ static void nft_target_eval_bridge(const struct nft_expr *expr, } static const struct nla_policy nft_target_policy[NFTA_TARGET_MAX + 1] = { - [NFTA_TARGET_NAME] = { .type = NLA_NUL_STRING }, + [NFTA_TARGET_NAME] = { .type = NLA_NUL_STRING, + .len = XT_EXTENSION_MAXNAMELEN, }, [NFTA_TARGET_REV] = NLA_POLICY_MAX(NLA_BE32, 255), [NFTA_TARGET_INFO] = { .type = NLA_BINARY }, }; @@ -434,7 +435,8 @@ static void nft_match_eval(const struct nft_expr *expr, } static const struct nla_policy nft_match_policy[NFTA_MATCH_MAX + 1] = { - [NFTA_MATCH_NAME] = { .type = NLA_NUL_STRING }, + [NFTA_MATCH_NAME] = { .type = NLA_NUL_STRING, + .len = XT_EXTENSION_MAXNAMELEN }, [NFTA_MATCH_REV] = NLA_POLICY_MAX(NLA_BE32, 255), [NFTA_MATCH_INFO] = { .type = NLA_BINARY }, }; @@ -693,7 +695,12 @@ static int nfnl_compat_get_rcu(struct sk_buff *skb, name = nla_data(tb[NFTA_COMPAT_NAME]); rev = ntohl(nla_get_be32(tb[NFTA_COMPAT_REV])); - target = ntohl(nla_get_be32(tb[NFTA_COMPAT_TYPE])); + /* x_tables api checks for 'target == 1' to mean target, + * everything else means 'match'. + * In x_tables world, the number is set by kernel, not + * userspace. + */ + target = nla_get_be32(tb[NFTA_COMPAT_TYPE]) == htonl(1); switch(family) { case AF_INET: diff --git a/net/netfilter/nft_connlimit.c b/net/netfilter/nft_connlimit.c index d4964087bbc5e..714a594859357 100644 --- a/net/netfilter/nft_connlimit.c +++ b/net/netfilter/nft_connlimit.c @@ -232,13 +232,8 @@ static void nft_connlimit_destroy_clone(const struct nft_ctx *ctx, static bool nft_connlimit_gc(struct net *net, const struct nft_expr *expr) { struct nft_connlimit *priv = nft_expr_priv(expr); - bool ret; - local_bh_disable(); - ret = nf_conncount_gc_list(net, priv->list); - local_bh_enable(); - - return ret; + return nf_conncount_gc_list(net, priv->list); } static struct nft_expr_type nft_connlimit_type; diff --git a/net/netfilter/nft_counter.c b/net/netfilter/nft_counter.c index cc73253294963..0d70325280cc5 100644 --- a/net/netfilter/nft_counter.c +++ b/net/netfilter/nft_counter.c @@ -117,8 +117,8 @@ static void nft_counter_reset(struct nft_counter_percpu_priv *priv, nft_sync = this_cpu_ptr(&nft_counter_sync); u64_stats_update_begin(nft_sync); - u64_stats_add(&this_cpu->packets, -total->packets); - u64_stats_add(&this_cpu->bytes, -total->bytes); + u64_stats_sub(&this_cpu->packets, total->packets); + u64_stats_sub(&this_cpu->bytes, total->bytes); u64_stats_update_end(nft_sync); local_bh_enable(); diff --git a/net/netfilter/nft_set_hash.c b/net/netfilter/nft_set_hash.c index ba01ce75d6dea..b0e571c8e3f38 100644 --- a/net/netfilter/nft_set_hash.c +++ b/net/netfilter/nft_set_hash.c @@ -374,6 +374,7 @@ static void nft_rhash_walk(const struct nft_ctx *ctx, struct nft_set *set, { switch (iter->type) { case NFT_ITER_UPDATE: + case NFT_ITER_UPDATE_CLONE: /* only relevant for netlink dumps which use READ type */ WARN_ON_ONCE(iter->skip != 0); @@ -619,15 +620,20 @@ static struct nft_elem_priv * nft_hash_get(const struct net *net, const struct nft_set *set, const struct nft_set_elem *elem, unsigned int flags) { + const u32 *key = (const u32 *)&elem->key.val; struct nft_hash *priv = nft_set_priv(set); u8 genmask = nft_genmask_cur(net); struct nft_hash_elem *he; u32 hash; - hash = jhash(elem->key.val.data, set->klen, priv->seed); + if (set->klen == 4) + hash = jhash_1word(*key, priv->seed); + else + hash = jhash(key, set->klen, priv->seed); + hash = reciprocal_scale(hash, priv->buckets); hlist_for_each_entry_rcu(he, &priv->table[hash], node) { - if (!memcmp(nft_set_ext_key(&he->ext), elem->key.val.data, set->klen) && + if (!memcmp(nft_set_ext_key(&he->ext), key, set->klen) && nft_set_elem_active(&he->ext, genmask)) return &he->priv; } diff --git a/net/netfilter/nft_set_pipapo.c b/net/netfilter/nft_set_pipapo.c index 6d77a5f0088ad..d9b74d588c768 100644 --- a/net/netfilter/nft_set_pipapo.c +++ b/net/netfilter/nft_set_pipapo.c @@ -1681,11 +1681,11 @@ static void nft_pipapo_gc_deactivate(struct net *net, struct nft_set *set, } /** - * pipapo_gc() - Drop expired entries from set, destroy start and end elements + * pipapo_gc_scan() - Drop expired entries from set and link them to gc list * @set: nftables API set representation * @m: Matching data */ -static void pipapo_gc(struct nft_set *set, struct nft_pipapo_match *m) +static void pipapo_gc_scan(struct nft_set *set, struct nft_pipapo_match *m) { struct nft_pipapo *priv = nft_set_priv(set); struct net *net = read_pnet(&set->net); @@ -1698,6 +1698,8 @@ static void pipapo_gc(struct nft_set *set, struct nft_pipapo_match *m) if (!gc) return; + list_add(&gc->list, &priv->gc_head); + while ((rules_f0 = pipapo_rules_same_key(m->f, first_rule))) { union nft_pipapo_map_bucket rulemap[NFT_PIPAPO_MAX_FIELDS]; const struct nft_pipapo_field *f; @@ -1725,9 +1727,13 @@ static void pipapo_gc(struct nft_set *set, struct nft_pipapo_match *m) * NFT_SET_ELEM_DEAD_BIT. */ if (__nft_set_elem_expired(&e->ext, tstamp)) { - gc = nft_trans_gc_queue_sync(gc, GFP_KERNEL); - if (!gc) - return; + if (!nft_trans_gc_space(gc)) { + gc = nft_trans_gc_alloc(set, 0, GFP_KERNEL); + if (!gc) + return; + + list_add(&gc->list, &priv->gc_head); + } nft_pipapo_gc_deactivate(net, set, e); pipapo_drop(m, rulemap); @@ -1741,10 +1747,30 @@ static void pipapo_gc(struct nft_set *set, struct nft_pipapo_match *m) } } - gc = nft_trans_gc_catchall_sync(gc); + priv->last_gc = jiffies; +} + +/** + * pipapo_gc_queue() - Free expired elements + * @set: nftables API set representation + */ +static void pipapo_gc_queue(struct nft_set *set) +{ + struct nft_pipapo *priv = nft_set_priv(set); + struct nft_trans_gc *gc, *next; + + /* always do a catchall cycle: */ + gc = nft_trans_gc_alloc(set, 0, GFP_KERNEL); if (gc) { + gc = nft_trans_gc_catchall_sync(gc); + if (gc) + nft_trans_gc_queue_sync_done(gc); + } + + /* always purge queued gc elements. */ + list_for_each_entry_safe(gc, next, &priv->gc_head, list) { + list_del(&gc->list); nft_trans_gc_queue_sync_done(gc); - priv->last_gc = jiffies; } } @@ -1798,6 +1824,10 @@ static void pipapo_reclaim_match(struct rcu_head *rcu) * * We also need to create a new working copy for subsequent insertions and * deletions. + * + * After the live copy has been replaced by the clone, we can safely queue + * expired elements that have been collected by pipapo_gc_scan() for + * memory reclaim. */ static void nft_pipapo_commit(struct nft_set *set) { @@ -1808,7 +1838,7 @@ static void nft_pipapo_commit(struct nft_set *set) return; if (time_after_eq(jiffies, priv->last_gc + nft_set_gc_interval(set))) - pipapo_gc(set, priv->clone); + pipapo_gc_scan(set, priv->clone); old = rcu_replace_pointer(priv->match, priv->clone, nft_pipapo_transaction_mutex_held(set)); @@ -1816,6 +1846,8 @@ static void nft_pipapo_commit(struct nft_set *set) if (old) call_rcu(&old->rcu, pipapo_reclaim_match); + + pipapo_gc_queue(set); } static void nft_pipapo_abort(const struct nft_set *set) @@ -2145,13 +2177,20 @@ static void nft_pipapo_walk(const struct nft_ctx *ctx, struct nft_set *set, const struct nft_pipapo_match *m; switch (iter->type) { - case NFT_ITER_UPDATE: + case NFT_ITER_UPDATE_CLONE: m = pipapo_maybe_clone(set); if (!m) { iter->err = -ENOMEM; return; } - + nft_pipapo_do_walk(ctx, set, m, iter); + break; + case NFT_ITER_UPDATE: + if (priv->clone) + m = priv->clone; + else + m = rcu_dereference_protected(priv->match, + nft_pipapo_transaction_mutex_held(set)); nft_pipapo_do_walk(ctx, set, m, iter); break; case NFT_ITER_READ: @@ -2273,6 +2312,7 @@ static int nft_pipapo_init(const struct nft_set *set, f->mt = NULL; } + INIT_LIST_HEAD(&priv->gc_head); rcu_assign_pointer(priv->match, m); return 0; @@ -2322,6 +2362,8 @@ static void nft_pipapo_destroy(const struct nft_ctx *ctx, struct nft_pipapo *priv = nft_set_priv(set); struct nft_pipapo_match *m; + WARN_ON_ONCE(!list_empty(&priv->gc_head)); + m = rcu_dereference_protected(priv->match, true); if (priv->clone) { @@ -2370,6 +2412,7 @@ const struct nft_set_type nft_set_pipapo_type = { .gc_init = nft_pipapo_gc_init, .commit = nft_pipapo_commit, .abort = nft_pipapo_abort, + .abort_skip_removal = true, .elemsize = offsetof(struct nft_pipapo_elem, ext), }, }; @@ -2394,6 +2437,7 @@ const struct nft_set_type nft_set_pipapo_avx2_type = { .gc_init = nft_pipapo_gc_init, .commit = nft_pipapo_commit, .abort = nft_pipapo_abort, + .abort_skip_removal = true, .elemsize = offsetof(struct nft_pipapo_elem, ext), }, }; diff --git a/net/netfilter/nft_set_pipapo.h b/net/netfilter/nft_set_pipapo.h index eaab422aa56ab..9aee9a9eaeb75 100644 --- a/net/netfilter/nft_set_pipapo.h +++ b/net/netfilter/nft_set_pipapo.h @@ -156,12 +156,14 @@ struct nft_pipapo_match { * @clone: Copy where pending insertions and deletions are kept * @width: Total bytes to be matched for one packet, including padding * @last_gc: Timestamp of last garbage collection run, jiffies + * @gc_head: list of nft_trans_gc to queue up for mem reclaim */ struct nft_pipapo { struct nft_pipapo_match __rcu *match; struct nft_pipapo_match *clone; int width; unsigned long last_gc; + struct list_head gc_head; }; struct nft_pipapo_elem; diff --git a/net/netfilter/nft_set_rbtree.c b/net/netfilter/nft_set_rbtree.c index ca594161b8402..5d91b7d08d33a 100644 --- a/net/netfilter/nft_set_rbtree.c +++ b/net/netfilter/nft_set_rbtree.c @@ -10,21 +10,41 @@ #include #include #include +#include #include #include #include #include +struct nft_array_interval { + struct nft_set_ext *from; + struct nft_set_ext *to; +}; + +struct nft_array { + u32 max_intervals; + u32 num_intervals; + struct nft_array_interval *intervals; + struct rcu_head rcu_head; +}; + struct nft_rbtree { struct rb_root root; rwlock_t lock; - seqcount_rwlock_t count; + struct nft_array __rcu *array; + struct nft_array *array_next; + unsigned long start_rbe_cookie; unsigned long last_gc; + struct list_head expired; + u64 last_tstamp; }; struct nft_rbtree_elem { struct nft_elem_priv priv; - struct rb_node node; + union { + struct rb_node node; + struct list_head list; + }; struct nft_set_ext ext; }; @@ -39,6 +59,13 @@ static bool nft_rbtree_interval_start(const struct nft_rbtree_elem *rbe) return !nft_rbtree_interval_end(rbe); } +static bool nft_rbtree_interval_null(const struct nft_set *set, + const struct nft_rbtree_elem *rbe) +{ + return (!memchr_inv(nft_set_ext_key(&rbe->ext), 0, set->klen) && + nft_rbtree_interval_end(rbe)); +} + static int nft_rbtree_cmp(const struct nft_set *set, const struct nft_rbtree_elem *e1, const struct nft_rbtree_elem *e2) @@ -47,67 +74,33 @@ static int nft_rbtree_cmp(const struct nft_set *set, set->klen); } -static bool nft_rbtree_elem_expired(const struct nft_rbtree_elem *rbe) -{ - return nft_set_elem_expired(&rbe->ext); -} +struct nft_array_lookup_ctx { + const u32 *key; + u32 klen; +}; -static const struct nft_set_ext * -__nft_rbtree_lookup(const struct net *net, const struct nft_set *set, - const u32 *key, unsigned int seq) +static int nft_array_lookup_cmp(const void *pkey, const void *entry) { - struct nft_rbtree *priv = nft_set_priv(set); - const struct nft_rbtree_elem *rbe, *interval = NULL; - u8 genmask = nft_genmask_cur(net); - const struct rb_node *parent; - int d; + const struct nft_array_interval *interval = entry; + const struct nft_array_lookup_ctx *ctx = pkey; + int a, b; - parent = rcu_dereference_raw(priv->root.rb_node); - while (parent != NULL) { - if (read_seqcount_retry(&priv->count, seq)) - return NULL; + if (!interval->from) + return 1; - rbe = rb_entry(parent, struct nft_rbtree_elem, node); - - d = memcmp(nft_set_ext_key(&rbe->ext), key, set->klen); - if (d < 0) { - parent = rcu_dereference_raw(parent->rb_left); - if (interval && - !nft_rbtree_cmp(set, rbe, interval) && - nft_rbtree_interval_end(rbe) && - nft_rbtree_interval_start(interval)) - continue; - if (nft_set_elem_active(&rbe->ext, genmask) && - !nft_rbtree_elem_expired(rbe)) - interval = rbe; - } else if (d > 0) - parent = rcu_dereference_raw(parent->rb_right); - else { - if (!nft_set_elem_active(&rbe->ext, genmask)) { - parent = rcu_dereference_raw(parent->rb_left); - continue; - } - - if (nft_rbtree_elem_expired(rbe)) - return NULL; - - if (nft_rbtree_interval_end(rbe)) { - if (nft_set_is_anonymous(set)) - return NULL; - parent = rcu_dereference_raw(parent->rb_left); - interval = NULL; - continue; - } + a = memcmp(ctx->key, nft_set_ext_key(interval->from), ctx->klen); + if (!interval->to) + b = -1; + else + b = memcmp(ctx->key, nft_set_ext_key(interval->to), ctx->klen); - return &rbe->ext; - } - } + if (a >= 0 && b < 0) + return 0; - if (set->flags & NFT_SET_INTERVAL && interval != NULL && - nft_rbtree_interval_start(interval)) - return &interval->ext; + if (a < 0) + return -1; - return NULL; + return 1; } INDIRECT_CALLABLE_SCOPE @@ -116,83 +109,57 @@ nft_rbtree_lookup(const struct net *net, const struct nft_set *set, const u32 *key) { struct nft_rbtree *priv = nft_set_priv(set); - unsigned int seq = read_seqcount_begin(&priv->count); - const struct nft_set_ext *ext; - - ext = __nft_rbtree_lookup(net, set, key, seq); - if (ext || !read_seqcount_retry(&priv->count, seq)) - return ext; - - read_lock_bh(&priv->lock); - seq = read_seqcount_begin(&priv->count); - ext = __nft_rbtree_lookup(net, set, key, seq); - read_unlock_bh(&priv->lock); - - return ext; + struct nft_array *array = rcu_dereference(priv->array); + const struct nft_array_interval *interval; + struct nft_array_lookup_ctx ctx = { + .key = key, + .klen = set->klen, + }; + + if (!array) + return NULL; + + interval = bsearch(&ctx, array->intervals, array->num_intervals, + sizeof(struct nft_array_interval), + nft_array_lookup_cmp); + if (!interval || nft_set_elem_expired(interval->from)) + return NULL; + + return interval->from; } -static bool __nft_rbtree_get(const struct net *net, const struct nft_set *set, - const u32 *key, struct nft_rbtree_elem **elem, - unsigned int seq, unsigned int flags, u8 genmask) -{ - struct nft_rbtree_elem *rbe, *interval = NULL; - struct nft_rbtree *priv = nft_set_priv(set); - const struct rb_node *parent; - const void *this; - int d; - - parent = rcu_dereference_raw(priv->root.rb_node); - while (parent != NULL) { - if (read_seqcount_retry(&priv->count, seq)) - return false; - - rbe = rb_entry(parent, struct nft_rbtree_elem, node); - - this = nft_set_ext_key(&rbe->ext); - d = memcmp(this, key, set->klen); - if (d < 0) { - parent = rcu_dereference_raw(parent->rb_left); - if (!(flags & NFT_SET_ELEM_INTERVAL_END)) - interval = rbe; - } else if (d > 0) { - parent = rcu_dereference_raw(parent->rb_right); - if (flags & NFT_SET_ELEM_INTERVAL_END) - interval = rbe; - } else { - if (!nft_set_elem_active(&rbe->ext, genmask)) { - parent = rcu_dereference_raw(parent->rb_left); - continue; - } +struct nft_array_get_ctx { + const u32 *key; + unsigned int flags; + u32 klen; +}; - if (nft_set_elem_expired(&rbe->ext)) - return false; +static int nft_array_get_cmp(const void *pkey, const void *entry) +{ + const struct nft_array_interval *interval = entry; + const struct nft_array_get_ctx *ctx = pkey; + int a, b; - if (!nft_set_ext_exists(&rbe->ext, NFT_SET_EXT_FLAGS) || - (*nft_set_ext_flags(&rbe->ext) & NFT_SET_ELEM_INTERVAL_END) == - (flags & NFT_SET_ELEM_INTERVAL_END)) { - *elem = rbe; - return true; - } + if (!interval->from) + return 1; - if (nft_rbtree_interval_end(rbe)) - interval = NULL; + a = memcmp(ctx->key, nft_set_ext_key(interval->from), ctx->klen); + if (!interval->to) + b = -1; + else + b = memcmp(ctx->key, nft_set_ext_key(interval->to), ctx->klen); - parent = rcu_dereference_raw(parent->rb_left); - } + if (a >= 0) { + if (ctx->flags & NFT_SET_ELEM_INTERVAL_END && b <= 0) + return 0; + else if (b < 0) + return 0; } - if (set->flags & NFT_SET_INTERVAL && interval != NULL && - nft_set_elem_active(&interval->ext, genmask) && - !nft_set_elem_expired(&interval->ext) && - ((!nft_rbtree_interval_end(interval) && - !(flags & NFT_SET_ELEM_INTERVAL_END)) || - (nft_rbtree_interval_end(interval) && - (flags & NFT_SET_ELEM_INTERVAL_END)))) { - *elem = interval; - return true; - } + if (a < 0) + return -1; - return false; + return 1; } static struct nft_elem_priv * @@ -200,34 +167,41 @@ nft_rbtree_get(const struct net *net, const struct nft_set *set, const struct nft_set_elem *elem, unsigned int flags) { struct nft_rbtree *priv = nft_set_priv(set); - unsigned int seq = read_seqcount_begin(&priv->count); - struct nft_rbtree_elem *rbe = ERR_PTR(-ENOENT); - const u32 *key = (const u32 *)&elem->key.val; - u8 genmask = nft_genmask_cur(net); - bool ret; - - ret = __nft_rbtree_get(net, set, key, &rbe, seq, flags, genmask); - if (ret || !read_seqcount_retry(&priv->count, seq)) - return &rbe->priv; - - read_lock_bh(&priv->lock); - seq = read_seqcount_begin(&priv->count); - ret = __nft_rbtree_get(net, set, key, &rbe, seq, flags, genmask); - read_unlock_bh(&priv->lock); - - if (!ret) + struct nft_array *array = rcu_dereference(priv->array); + const struct nft_array_interval *interval; + struct nft_array_get_ctx ctx = { + .key = (const u32 *)&elem->key.val, + .flags = flags, + .klen = set->klen, + }; + struct nft_rbtree_elem *rbe; + + if (!array) return ERR_PTR(-ENOENT); + interval = bsearch(&ctx, array->intervals, array->num_intervals, + sizeof(struct nft_array_interval), nft_array_get_cmp); + if (!interval || nft_set_elem_expired(interval->from)) + return ERR_PTR(-ENOENT); + + if (flags & NFT_SET_ELEM_INTERVAL_END) + rbe = container_of(interval->to, struct nft_rbtree_elem, ext); + else + rbe = container_of(interval->from, struct nft_rbtree_elem, ext); + return &rbe->priv; } -static void nft_rbtree_gc_elem_remove(struct net *net, struct nft_set *set, - struct nft_rbtree *priv, - struct nft_rbtree_elem *rbe) +static void nft_rbtree_gc_elem_move(struct net *net, struct nft_set *set, + struct nft_rbtree *priv, + struct nft_rbtree_elem *rbe) { lockdep_assert_held_write(&priv->lock); nft_setelem_data_deactivate(net, set, &rbe->priv); rb_erase(&rbe->node, &priv->root); + + /* collected later on in commit callback */ + list_add(&rbe->list, &priv->expired); } static const struct nft_rbtree_elem * @@ -238,11 +212,6 @@ nft_rbtree_gc_elem(const struct nft_set *__set, struct nft_rbtree *priv, struct rb_node *prev = rb_prev(&rbe->node); struct net *net = read_pnet(&set->net); struct nft_rbtree_elem *rbe_prev; - struct nft_trans_gc *gc; - - gc = nft_trans_gc_alloc(set, 0, GFP_ATOMIC); - if (!gc) - return ERR_PTR(-ENOMEM); /* search for end interval coming before this element. * end intervals don't carry a timeout extension, they @@ -260,28 +229,10 @@ nft_rbtree_gc_elem(const struct nft_set *__set, struct nft_rbtree *priv, rbe_prev = NULL; if (prev) { rbe_prev = rb_entry(prev, struct nft_rbtree_elem, node); - nft_rbtree_gc_elem_remove(net, set, priv, rbe_prev); - - /* There is always room in this trans gc for this element, - * memory allocation never actually happens, hence, the warning - * splat in such case. No need to set NFT_SET_ELEM_DEAD_BIT, - * this is synchronous gc which never fails. - */ - gc = nft_trans_gc_queue_sync(gc, GFP_ATOMIC); - if (WARN_ON_ONCE(!gc)) - return ERR_PTR(-ENOMEM); - - nft_trans_gc_elem_add(gc, rbe_prev); + nft_rbtree_gc_elem_move(net, set, priv, rbe_prev); } - nft_rbtree_gc_elem_remove(net, set, priv, rbe); - gc = nft_trans_gc_queue_sync(gc, GFP_ATOMIC); - if (WARN_ON_ONCE(!gc)) - return ERR_PTR(-ENOMEM); - - nft_trans_gc_elem_add(gc, rbe); - - nft_trans_gc_queue_sync_done(gc); + nft_rbtree_gc_elem_move(net, set, priv, rbe); return rbe_prev; } @@ -302,16 +253,97 @@ static bool nft_rbtree_update_first(const struct nft_set *set, return false; } +/* Only for anonymous sets which do not allow updates, all element are active. */ +static struct nft_rbtree_elem *nft_rbtree_prev_active(struct nft_rbtree_elem *rbe) +{ + struct rb_node *node; + + node = rb_prev(&rbe->node); + if (!node) + return NULL; + + return rb_entry(node, struct nft_rbtree_elem, node); +} + +static struct nft_rbtree_elem * +__nft_rbtree_next_active(struct rb_node *node, u8 genmask) +{ + struct nft_rbtree_elem *next_rbe; + + while (node) { + next_rbe = rb_entry(node, struct nft_rbtree_elem, node); + if (!nft_set_elem_active(&next_rbe->ext, genmask)) { + node = rb_next(node); + continue; + } + + return next_rbe; + } + + return NULL; +} + +static struct nft_rbtree_elem * +nft_rbtree_next_active(struct nft_rbtree_elem *rbe, u8 genmask) +{ + return __nft_rbtree_next_active(rb_next(&rbe->node), genmask); +} + +static void nft_rbtree_maybe_reset_start_cookie(struct nft_rbtree *priv, + u64 tstamp) +{ + if (priv->last_tstamp != tstamp) { + priv->start_rbe_cookie = 0; + priv->last_tstamp = tstamp; + } +} + +static void nft_rbtree_set_start_cookie(struct nft_rbtree *priv, + const struct nft_rbtree_elem *rbe) +{ + priv->start_rbe_cookie = (unsigned long)rbe; +} + +static bool nft_rbtree_cmp_start_cookie(struct nft_rbtree *priv, + const struct nft_rbtree_elem *rbe) +{ + return priv->start_rbe_cookie == (unsigned long)rbe; +} + +static bool nft_rbtree_insert_same_interval(const struct net *net, + struct nft_rbtree *priv, + struct nft_rbtree_elem *rbe) +{ + u8 genmask = nft_genmask_next(net); + struct nft_rbtree_elem *next_rbe; + + if (!priv->start_rbe_cookie) + return true; + + next_rbe = nft_rbtree_next_active(rbe, genmask); + if (next_rbe) { + /* Closest start element differs from last element added. */ + if (nft_rbtree_interval_start(next_rbe) && + nft_rbtree_cmp_start_cookie(priv, next_rbe)) { + priv->start_rbe_cookie = 0; + return true; + } + } + + priv->start_rbe_cookie = 0; + + return false; +} + static int __nft_rbtree_insert(const struct net *net, const struct nft_set *set, struct nft_rbtree_elem *new, - struct nft_elem_priv **elem_priv) + struct nft_elem_priv **elem_priv, u64 tstamp) { - struct nft_rbtree_elem *rbe, *rbe_le = NULL, *rbe_ge = NULL; + struct nft_rbtree_elem *rbe, *rbe_le = NULL, *rbe_ge = NULL, *rbe_prev; struct rb_node *node, *next, *parent, **p, *first = NULL; struct nft_rbtree *priv = nft_set_priv(set); u8 cur_genmask = nft_genmask_cur(net); u8 genmask = nft_genmask_next(net); - u64 tstamp = nft_net_tstamp(net); int d; /* Descend the tree to search for an existing element greater than the @@ -417,12 +449,18 @@ static int __nft_rbtree_insert(const struct net *net, const struct nft_set *set, } } + if (nft_rbtree_interval_null(set, new)) + priv->start_rbe_cookie = 0; + else if (nft_rbtree_interval_start(new) && priv->start_rbe_cookie) + priv->start_rbe_cookie = 0; + /* - new start element matching existing start element: full overlap * reported as -EEXIST, cleared by caller if NLM_F_EXCL is not given. */ if (rbe_ge && !nft_rbtree_cmp(set, new, rbe_ge) && nft_rbtree_interval_start(rbe_ge) == nft_rbtree_interval_start(new)) { *elem_priv = &rbe_ge->priv; + nft_rbtree_set_start_cookie(priv, rbe_ge); return -EEXIST; } @@ -431,18 +469,37 @@ static int __nft_rbtree_insert(const struct net *net, const struct nft_set *set, */ if (rbe_le && !nft_rbtree_cmp(set, new, rbe_le) && nft_rbtree_interval_end(rbe_le) == nft_rbtree_interval_end(new)) { + /* - ignore null interval, otherwise NLM_F_CREATE bogusly + * reports EEXIST. + */ + if (nft_rbtree_interval_null(set, new)) + return -ECANCELED; + *elem_priv = &rbe_le->priv; + + /* - start and end element belong to the same interval. */ + if (!nft_rbtree_insert_same_interval(net, priv, rbe_le)) + return -ENOTEMPTY; + return -EEXIST; } /* - new start element with existing closest, less or equal key value * being a start element: partial overlap, reported as -ENOTEMPTY. * Anonymous sets allow for two consecutive start element since they - * are constant, skip them to avoid bogus overlap reports. + * are constant, but validate that this new start element does not + * sit in between an existing start and end elements: partial overlap, + * reported as -ENOTEMPTY. */ - if (!nft_set_is_anonymous(set) && rbe_le && - nft_rbtree_interval_start(rbe_le) && nft_rbtree_interval_start(new)) - return -ENOTEMPTY; + if (rbe_le && + nft_rbtree_interval_start(rbe_le) && nft_rbtree_interval_start(new)) { + if (!nft_set_is_anonymous(set)) + return -ENOTEMPTY; + + rbe_prev = nft_rbtree_prev_active(rbe_le); + if (rbe_prev && nft_rbtree_interval_end(rbe_prev)) + return -ENOTEMPTY; + } /* - new end element with existing closest, less or equal key value * being a end element: partial overlap, reported as -ENOTEMPTY. @@ -481,14 +538,101 @@ static int __nft_rbtree_insert(const struct net *net, const struct nft_set *set, return 0; } +static int nft_array_intervals_alloc(struct nft_array *array, u32 max_intervals) +{ + struct nft_array_interval *intervals; + + intervals = kvcalloc(max_intervals, sizeof(struct nft_array_interval), + GFP_KERNEL_ACCOUNT); + if (!intervals) + return -ENOMEM; + + if (array->intervals) + kvfree(array->intervals); + + array->intervals = intervals; + array->max_intervals = max_intervals; + + return 0; +} + +static struct nft_array *nft_array_alloc(u32 max_intervals) +{ + struct nft_array *array; + + array = kzalloc(sizeof(*array), GFP_KERNEL_ACCOUNT); + if (!array) + return NULL; + + if (nft_array_intervals_alloc(array, max_intervals) < 0) { + kfree(array); + return NULL; + } + + return array; +} + +#define NFT_ARRAY_EXTRA_SIZE 10240 + +/* Similar to nft_rbtree_{u,k}size to hide details to userspace, but consider + * packed representation coming from userspace for anonymous sets too. + */ +static u32 nft_array_elems(const struct nft_set *set) +{ + u32 nelems = atomic_read(&set->nelems); + + /* Adjacent intervals are represented with a single start element in + * anonymous sets, use the current element counter as is. + */ + if (nft_set_is_anonymous(set)) + return nelems; + + /* Add extra room for never matching interval at the beginning and open + * interval at the end which only use a single element to represent it. + * The conversion to array will compact intervals, this allows reduce + * memory consumption. + */ + return (nelems / 2) + 2; +} + +static int nft_array_may_resize(const struct nft_set *set) +{ + u32 nelems = nft_array_elems(set), new_max_intervals; + struct nft_rbtree *priv = nft_set_priv(set); + struct nft_array *array; + + if (!priv->array_next) { + array = nft_array_alloc(nelems + NFT_ARRAY_EXTRA_SIZE); + if (!array) + return -ENOMEM; + + priv->array_next = array; + } + + if (nelems < priv->array_next->max_intervals) + return 0; + + new_max_intervals = priv->array_next->max_intervals + NFT_ARRAY_EXTRA_SIZE; + if (nft_array_intervals_alloc(priv->array_next, new_max_intervals) < 0) + return -ENOMEM; + + return 0; +} + static int nft_rbtree_insert(const struct net *net, const struct nft_set *set, const struct nft_set_elem *elem, struct nft_elem_priv **elem_priv) { struct nft_rbtree_elem *rbe = nft_elem_priv_cast(elem->priv); struct nft_rbtree *priv = nft_set_priv(set); + u64 tstamp = nft_net_tstamp(net); int err; + nft_rbtree_maybe_reset_start_cookie(priv, tstamp); + + if (nft_array_may_resize(set) < 0) + return -ENOMEM; + do { if (fatal_signal_pending(current)) return -EINTR; @@ -496,9 +640,7 @@ static int nft_rbtree_insert(const struct net *net, const struct nft_set *set, cond_resched(); write_lock_bh(&priv->lock); - write_seqcount_begin(&priv->count); - err = __nft_rbtree_insert(net, set, rbe, elem_priv); - write_seqcount_end(&priv->count); + err = __nft_rbtree_insert(net, set, rbe, elem_priv, tstamp); write_unlock_bh(&priv->lock); } while (err == -EAGAIN); @@ -508,9 +650,7 @@ static int nft_rbtree_insert(const struct net *net, const struct nft_set *set, static void nft_rbtree_erase(struct nft_rbtree *priv, struct nft_rbtree_elem *rbe) { write_lock_bh(&priv->lock); - write_seqcount_begin(&priv->count); rb_erase(&rbe->node, &priv->root); - write_seqcount_end(&priv->count); write_unlock_bh(&priv->lock); } @@ -533,6 +673,48 @@ static void nft_rbtree_activate(const struct net *net, nft_clear(net, &rbe->ext); } +static struct nft_rbtree_elem * +nft_rbtree_next_inactive(struct nft_rbtree_elem *rbe, u8 genmask) +{ + struct nft_rbtree_elem *next_rbe; + struct rb_node *node; + + node = rb_next(&rbe->node); + if (node) { + next_rbe = rb_entry(node, struct nft_rbtree_elem, node); + if (nft_rbtree_interval_start(next_rbe) && + !nft_set_elem_active(&next_rbe->ext, genmask)) + return next_rbe; + } + + return NULL; +} + +static bool nft_rbtree_deactivate_same_interval(const struct net *net, + struct nft_rbtree *priv, + struct nft_rbtree_elem *rbe) +{ + u8 genmask = nft_genmask_next(net); + struct nft_rbtree_elem *next_rbe; + + if (!priv->start_rbe_cookie) + return true; + + next_rbe = nft_rbtree_next_inactive(rbe, genmask); + if (next_rbe) { + /* Closest start element differs from last element added. */ + if (nft_rbtree_interval_start(next_rbe) && + nft_rbtree_cmp_start_cookie(priv, next_rbe)) { + priv->start_rbe_cookie = 0; + return true; + } + } + + priv->start_rbe_cookie = 0; + + return false; +} + static void nft_rbtree_flush(const struct net *net, const struct nft_set *set, struct nft_elem_priv *elem_priv) @@ -547,12 +729,21 @@ nft_rbtree_deactivate(const struct net *net, const struct nft_set *set, const struct nft_set_elem *elem) { struct nft_rbtree_elem *rbe, *this = nft_elem_priv_cast(elem->priv); - const struct nft_rbtree *priv = nft_set_priv(set); + struct nft_rbtree *priv = nft_set_priv(set); const struct rb_node *parent = priv->root.rb_node; u8 genmask = nft_genmask_next(net); u64 tstamp = nft_net_tstamp(net); int d; + nft_rbtree_maybe_reset_start_cookie(priv, tstamp); + + if (nft_rbtree_interval_start(this) || + nft_rbtree_interval_null(set, this)) + priv->start_rbe_cookie = 0; + + if (nft_array_may_resize(set) < 0) + return NULL; + while (parent != NULL) { rbe = rb_entry(parent, struct nft_rbtree_elem, node); @@ -577,6 +768,12 @@ nft_rbtree_deactivate(const struct net *net, const struct nft_set *set, parent = parent->rb_left; continue; } + + if (nft_rbtree_interval_start(rbe)) + nft_rbtree_set_start_cookie(priv, rbe); + else if (!nft_rbtree_deactivate_same_interval(net, priv, rbe)) + return NULL; + nft_rbtree_flush(net, set, &rbe->priv); return &rbe->priv; } @@ -613,8 +810,15 @@ static void nft_rbtree_walk(const struct nft_ctx *ctx, struct nft_rbtree *priv = nft_set_priv(set); switch (iter->type) { + case NFT_ITER_UPDATE_CLONE: + if (nft_array_may_resize(set) < 0) { + iter->err = -ENOMEM; + break; + } + fallthrough; case NFT_ITER_UPDATE: lockdep_assert_held(&nft_pernet(ctx->net)->commit_mutex); + nft_rbtree_do_walk(ctx, set, iter); break; case NFT_ITER_READ: @@ -629,29 +833,13 @@ static void nft_rbtree_walk(const struct nft_ctx *ctx, } } -static void nft_rbtree_gc_remove(struct net *net, struct nft_set *set, - struct nft_rbtree *priv, - struct nft_rbtree_elem *rbe) -{ - nft_setelem_data_deactivate(net, set, &rbe->priv); - nft_rbtree_erase(priv, rbe); -} - -static void nft_rbtree_gc(struct nft_set *set) +static void nft_rbtree_gc_scan(struct nft_set *set) { struct nft_rbtree *priv = nft_set_priv(set); struct nft_rbtree_elem *rbe, *rbe_end = NULL; struct net *net = read_pnet(&set->net); u64 tstamp = nft_net_tstamp(net); struct rb_node *node, *next; - struct nft_trans_gc *gc; - - set = nft_set_container_of(priv); - net = read_pnet(&set->net); - - gc = nft_trans_gc_alloc(set, 0, GFP_KERNEL); - if (!gc) - return; for (node = rb_first(&priv->root); node ; node = next) { next = rb_next(node); @@ -669,34 +857,46 @@ static void nft_rbtree_gc(struct nft_set *set) if (!__nft_set_elem_expired(&rbe->ext, tstamp)) continue; - gc = nft_trans_gc_queue_sync(gc, GFP_KERNEL); - if (!gc) - goto try_later; - /* end element needs to be removed first, it has * no timeout extension. */ + write_lock_bh(&priv->lock); if (rbe_end) { - nft_rbtree_gc_remove(net, set, priv, rbe_end); - nft_trans_gc_elem_add(gc, rbe_end); + nft_rbtree_gc_elem_move(net, set, priv, rbe_end); rbe_end = NULL; } - gc = nft_trans_gc_queue_sync(gc, GFP_KERNEL); - if (!gc) - goto try_later; - - nft_rbtree_gc_remove(net, set, priv, rbe); - nft_trans_gc_elem_add(gc, rbe); + nft_rbtree_gc_elem_move(net, set, priv, rbe); + write_unlock_bh(&priv->lock); } -try_later: + priv->last_gc = jiffies; +} + +static void nft_rbtree_gc_queue(struct nft_set *set) +{ + struct nft_rbtree *priv = nft_set_priv(set); + struct nft_rbtree_elem *rbe, *rbe_end; + struct nft_trans_gc *gc; + + if (list_empty(&priv->expired)) + return; + + gc = nft_trans_gc_alloc(set, 0, GFP_KERNEL); + if (!gc) + return; + + list_for_each_entry_safe(rbe, rbe_end, &priv->expired, list) { + list_del(&rbe->list); + nft_trans_gc_elem_add(gc, rbe); - if (gc) { - gc = nft_trans_gc_catchall_sync(gc); - nft_trans_gc_queue_sync_done(gc); - priv->last_gc = jiffies; + gc = nft_trans_gc_queue_sync(gc, GFP_KERNEL); + if (!gc) + return; } + + gc = nft_trans_gc_catchall_sync(gc); + nft_trans_gc_queue_sync_done(gc); } static u64 nft_rbtree_privsize(const struct nlattr * const nla[], @@ -714,24 +914,45 @@ static int nft_rbtree_init(const struct nft_set *set, BUILD_BUG_ON(offsetof(struct nft_rbtree_elem, priv) != 0); rwlock_init(&priv->lock); - seqcount_rwlock_init(&priv->count, &priv->lock); priv->root = RB_ROOT; + INIT_LIST_HEAD(&priv->expired); + + priv->array = NULL; + priv->array_next = NULL; return 0; } +static void __nft_array_free(struct nft_array *array) +{ + kvfree(array->intervals); + kfree(array); +} + static void nft_rbtree_destroy(const struct nft_ctx *ctx, const struct nft_set *set) { struct nft_rbtree *priv = nft_set_priv(set); - struct nft_rbtree_elem *rbe; + struct nft_rbtree_elem *rbe, *next; + struct nft_array *array; struct rb_node *node; + list_for_each_entry_safe(rbe, next, &priv->expired, list) { + list_del(&rbe->list); + nf_tables_set_elem_destroy(ctx, set, &rbe->priv); + } + while ((node = priv->root.rb_node) != NULL) { rb_erase(node, &priv->root); rbe = rb_entry(node, struct nft_rbtree_elem, node); nf_tables_set_elem_destroy(ctx, set, &rbe->priv); } + + array = rcu_dereference_protected(priv->array, true); + if (array) + __nft_array_free(array); + if (priv->array_next) + __nft_array_free(priv->array_next); } static bool nft_rbtree_estimate(const struct nft_set_desc *desc, u32 features, @@ -752,12 +973,105 @@ static bool nft_rbtree_estimate(const struct nft_set_desc *desc, u32 features, return true; } +static void nft_array_free_rcu(struct rcu_head *rcu_head) +{ + struct nft_array *array = container_of(rcu_head, struct nft_array, rcu_head); + + __nft_array_free(array); +} + static void nft_rbtree_commit(struct nft_set *set) { struct nft_rbtree *priv = nft_set_priv(set); + struct nft_rbtree_elem *rbe, *prev_rbe; + struct nft_array *old; + u32 num_intervals = 0; + struct rb_node *node; + + /* No changes, skip, eg. elements updates only. */ + if (!priv->array_next) + return; + /* GC can be performed if the binary search blob is going + * to be rebuilt. It has to be done in two phases: first + * scan tree and move all expired elements to the expired + * list. + * + * Then, after blob has been re-built and published to other + * CPUs, queue collected entries for freeing. + */ if (time_after_eq(jiffies, priv->last_gc + nft_set_gc_interval(set))) - nft_rbtree_gc(set); + nft_rbtree_gc_scan(set); + + /* Reverse walk to create an array from smaller to largest interval. */ + node = rb_last(&priv->root); + if (node) + prev_rbe = rb_entry(node, struct nft_rbtree_elem, node); + else + prev_rbe = NULL; + + while (prev_rbe) { + rbe = prev_rbe; + + if (nft_rbtree_interval_start(rbe)) + priv->array_next->intervals[num_intervals].from = &rbe->ext; + else if (nft_rbtree_interval_end(rbe)) + priv->array_next->intervals[num_intervals++].to = &rbe->ext; + + if (num_intervals >= priv->array_next->max_intervals) { + pr_warn_once("malformed interval set from userspace?"); + goto err_out; + } + + node = rb_prev(node); + if (!node) + break; + + prev_rbe = rb_entry(node, struct nft_rbtree_elem, node); + + /* For anonymous sets, when adjacent ranges are found, + * the end element is not added to the set to pack the set + * representation. Use next start element to complete this + * interval. + */ + if (nft_rbtree_interval_start(rbe) && + nft_rbtree_interval_start(prev_rbe) && + priv->array_next->intervals[num_intervals].from) + priv->array_next->intervals[num_intervals++].to = &prev_rbe->ext; + + if (num_intervals >= priv->array_next->max_intervals) { + pr_warn_once("malformed interval set from userspace?"); + goto err_out; + } + } + + if (priv->array_next->intervals[num_intervals].from) + num_intervals++; +err_out: + priv->array_next->num_intervals = num_intervals; + old = rcu_replace_pointer(priv->array, priv->array_next, + lockdep_is_held(&nft_pernet(read_pnet(&set->net))->commit_mutex)); + priv->array_next = NULL; + if (old) + call_rcu(&old->rcu_head, nft_array_free_rcu); + + /* New blob is public, queue collected entries for freeing. + * call_rcu ensures elements stay around until readers are done. + */ + nft_rbtree_gc_queue(set); +} + +static void nft_rbtree_abort(const struct nft_set *set) +{ + struct nft_rbtree *priv = nft_set_priv(set); + struct nft_array *array_next; + + if (!priv->array_next) + return; + + array_next = priv->array_next; + priv->array_next = NULL; + __nft_array_free(array_next); } static void nft_rbtree_gc_init(const struct nft_set *set) @@ -821,6 +1135,7 @@ const struct nft_set_type nft_set_rbtree_type = { .flush = nft_rbtree_flush, .activate = nft_rbtree_activate, .commit = nft_rbtree_commit, + .abort = nft_rbtree_abort, .gc_init = nft_rbtree_gc_init, .lookup = nft_rbtree_lookup, .walk = nft_rbtree_walk, diff --git a/net/netfilter/xt_tcpmss.c b/net/netfilter/xt_tcpmss.c index 37704ab017992..0d32d4841cb32 100644 --- a/net/netfilter/xt_tcpmss.c +++ b/net/netfilter/xt_tcpmss.c @@ -61,7 +61,7 @@ tcpmss_mt(const struct sk_buff *skb, struct xt_action_param *par) return (mssval >= info->mss_min && mssval <= info->mss_max) ^ info->invert; } - if (op[i] < 2) + if (op[i] < 2 || i == optlen - 1) i++; else i += op[i+1] ? : 1; diff --git a/net/nfc/hci/llc_shdlc.c b/net/nfc/hci/llc_shdlc.c index 4fc37894860c9..08c8aa1530d8a 100644 --- a/net/nfc/hci/llc_shdlc.c +++ b/net/nfc/hci/llc_shdlc.c @@ -762,6 +762,14 @@ static void llc_shdlc_deinit(struct nfc_llc *llc) { struct llc_shdlc *shdlc = nfc_llc_get_data(llc); + timer_shutdown_sync(&shdlc->connect_timer); + timer_shutdown_sync(&shdlc->t1_timer); + timer_shutdown_sync(&shdlc->t2_timer); + shdlc->t1_active = false; + shdlc->t2_active = false; + + cancel_work_sync(&shdlc->sm_work); + skb_queue_purge(&shdlc->rcv_q); skb_queue_purge(&shdlc->send_q); skb_queue_purge(&shdlc->ack_pending_q); diff --git a/net/nfc/nci/core.c b/net/nfc/nci/core.c index e419e020a70a3..d334b7aa8c172 100644 --- a/net/nfc/nci/core.c +++ b/net/nfc/nci/core.c @@ -567,6 +567,10 @@ static int nci_close_device(struct nci_dev *ndev) flush_workqueue(ndev->cmd_wq); timer_delete_sync(&ndev->cmd_timer); timer_delete_sync(&ndev->data_timer); + if (test_bit(NCI_DATA_EXCHANGE, &ndev->flags)) + nci_data_exchange_complete(ndev, NULL, + ndev->cur_conn_id, + -ENODEV); mutex_unlock(&ndev->req_lock); return 0; } @@ -598,6 +602,11 @@ static int nci_close_device(struct nci_dev *ndev) flush_workqueue(ndev->cmd_wq); timer_delete_sync(&ndev->cmd_timer); + timer_delete_sync(&ndev->data_timer); + + if (test_bit(NCI_DATA_EXCHANGE, &ndev->flags)) + nci_data_exchange_complete(ndev, NULL, ndev->cur_conn_id, + -ENODEV); /* Clear flags except NCI_UNREG */ ndev->flags &= BIT(NCI_UNREG); @@ -1035,18 +1044,23 @@ static int nci_transceive(struct nfc_dev *nfc_dev, struct nfc_target *target, struct nci_conn_info *conn_info; conn_info = ndev->rf_conn_info; - if (!conn_info) + if (!conn_info) { + kfree_skb(skb); return -EPROTO; + } pr_debug("target_idx %d, len %d\n", target->idx, skb->len); if (!ndev->target_active_prot) { pr_err("unable to exchange data, no active target\n"); + kfree_skb(skb); return -EINVAL; } - if (test_and_set_bit(NCI_DATA_EXCHANGE, &ndev->flags)) + if (test_and_set_bit(NCI_DATA_EXCHANGE, &ndev->flags)) { + kfree_skb(skb); return -EBUSY; + } /* store cb and context to be used on receiving data */ conn_info->data_exchange_cb = cb; @@ -1482,10 +1496,20 @@ static bool nci_valid_size(struct sk_buff *skb) unsigned int hdr_size = NCI_CTRL_HDR_SIZE; if (skb->len < hdr_size || - !nci_plen(skb->data) || skb->len < hdr_size + nci_plen(skb->data)) { return false; } + + if (!nci_plen(skb->data)) { + /* Allow zero length in proprietary notifications (0x20 - 0x3F). */ + if (nci_opcode_oid(nci_opcode(skb->data)) >= 0x20 && + nci_mt(skb->data) == NCI_MT_NTF_PKT) + return true; + + /* Disallow zero length otherwise. */ + return false; + } + return true; } diff --git a/net/nfc/nci/data.c b/net/nfc/nci/data.c index 78f4131af3cf3..5f98c73db5afd 100644 --- a/net/nfc/nci/data.c +++ b/net/nfc/nci/data.c @@ -33,7 +33,8 @@ void nci_data_exchange_complete(struct nci_dev *ndev, struct sk_buff *skb, conn_info = nci_get_conn_info_by_conn_id(ndev, conn_id); if (!conn_info) { kfree_skb(skb); - goto exit; + clear_bit(NCI_DATA_EXCHANGE, &ndev->flags); + return; } cb = conn_info->data_exchange_cb; @@ -45,6 +46,12 @@ void nci_data_exchange_complete(struct nci_dev *ndev, struct sk_buff *skb, timer_delete_sync(&ndev->data_timer); clear_bit(NCI_DATA_EXCHANGE_TO, &ndev->flags); + /* Mark the exchange as done before calling the callback. + * The callback (e.g. rawsock_data_exchange_complete) may + * want to immediately queue another data exchange. + */ + clear_bit(NCI_DATA_EXCHANGE, &ndev->flags); + if (cb) { /* forward skb to nfc core */ cb(cb_context, skb, err); @@ -54,9 +61,6 @@ void nci_data_exchange_complete(struct nci_dev *ndev, struct sk_buff *skb, /* no waiting callback, free skb */ kfree_skb(skb); } - -exit: - clear_bit(NCI_DATA_EXCHANGE, &ndev->flags); } /* ----------------- NCI TX Data ----------------- */ diff --git a/net/nfc/nci/ntf.c b/net/nfc/nci/ntf.c index 418b84e2b2605..c96512bb86531 100644 --- a/net/nfc/nci/ntf.c +++ b/net/nfc/nci/ntf.c @@ -58,7 +58,7 @@ static int nci_core_conn_credits_ntf_packet(struct nci_dev *ndev, struct nci_conn_info *conn_info; int i; - if (skb->len < sizeof(struct nci_core_conn_credit_ntf)) + if (skb->len < offsetofend(struct nci_core_conn_credit_ntf, num_entries)) return -EINVAL; ntf = (struct nci_core_conn_credit_ntf *)skb->data; @@ -68,6 +68,10 @@ static int nci_core_conn_credits_ntf_packet(struct nci_dev *ndev, if (ntf->num_entries > NCI_MAX_NUM_CONN) ntf->num_entries = NCI_MAX_NUM_CONN; + if (skb->len < offsetofend(struct nci_core_conn_credit_ntf, num_entries) + + ntf->num_entries * sizeof(struct conn_credit_entry)) + return -EINVAL; + /* update the credits */ for (i = 0; i < ntf->num_entries; i++) { ntf->conn_entries[i].conn_id = @@ -138,23 +142,48 @@ static int nci_core_conn_intf_error_ntf_packet(struct nci_dev *ndev, static const __u8 * nci_extract_rf_params_nfca_passive_poll(struct nci_dev *ndev, struct rf_tech_specific_params_nfca_poll *nfca_poll, - const __u8 *data) + const __u8 *data, ssize_t data_len) { + /* Check if we have enough data for sens_res (2 bytes) */ + if (data_len < 2) + return ERR_PTR(-EINVAL); + nfca_poll->sens_res = __le16_to_cpu(*((__le16 *)data)); data += 2; + data_len -= 2; + + /* Check if we have enough data for nfcid1_len (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); nfca_poll->nfcid1_len = min_t(__u8, *data++, NFC_NFCID1_MAXSIZE); + data_len--; pr_debug("sens_res 0x%x, nfcid1_len %d\n", nfca_poll->sens_res, nfca_poll->nfcid1_len); + /* Check if we have enough data for nfcid1 */ + if (data_len < nfca_poll->nfcid1_len) + return ERR_PTR(-EINVAL); + memcpy(nfca_poll->nfcid1, data, nfca_poll->nfcid1_len); data += nfca_poll->nfcid1_len; + data_len -= nfca_poll->nfcid1_len; + + /* Check if we have enough data for sel_res_len (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); nfca_poll->sel_res_len = *data++; + data_len--; + + if (nfca_poll->sel_res_len != 0) { + /* Check if we have enough data for sel_res (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); - if (nfca_poll->sel_res_len != 0) nfca_poll->sel_res = *data++; + } pr_debug("sel_res_len %d, sel_res 0x%x\n", nfca_poll->sel_res_len, @@ -166,12 +195,21 @@ nci_extract_rf_params_nfca_passive_poll(struct nci_dev *ndev, static const __u8 * nci_extract_rf_params_nfcb_passive_poll(struct nci_dev *ndev, struct rf_tech_specific_params_nfcb_poll *nfcb_poll, - const __u8 *data) + const __u8 *data, ssize_t data_len) { + /* Check if we have enough data for sensb_res_len (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); + nfcb_poll->sensb_res_len = min_t(__u8, *data++, NFC_SENSB_RES_MAXSIZE); + data_len--; pr_debug("sensb_res_len %d\n", nfcb_poll->sensb_res_len); + /* Check if we have enough data for sensb_res */ + if (data_len < nfcb_poll->sensb_res_len) + return ERR_PTR(-EINVAL); + memcpy(nfcb_poll->sensb_res, data, nfcb_poll->sensb_res_len); data += nfcb_poll->sensb_res_len; @@ -181,14 +219,29 @@ nci_extract_rf_params_nfcb_passive_poll(struct nci_dev *ndev, static const __u8 * nci_extract_rf_params_nfcf_passive_poll(struct nci_dev *ndev, struct rf_tech_specific_params_nfcf_poll *nfcf_poll, - const __u8 *data) + const __u8 *data, ssize_t data_len) { + /* Check if we have enough data for bit_rate (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); + nfcf_poll->bit_rate = *data++; + data_len--; + + /* Check if we have enough data for sensf_res_len (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); + nfcf_poll->sensf_res_len = min_t(__u8, *data++, NFC_SENSF_RES_MAXSIZE); + data_len--; pr_debug("bit_rate %d, sensf_res_len %d\n", nfcf_poll->bit_rate, nfcf_poll->sensf_res_len); + /* Check if we have enough data for sensf_res */ + if (data_len < nfcf_poll->sensf_res_len) + return ERR_PTR(-EINVAL); + memcpy(nfcf_poll->sensf_res, data, nfcf_poll->sensf_res_len); data += nfcf_poll->sensf_res_len; @@ -198,22 +251,49 @@ nci_extract_rf_params_nfcf_passive_poll(struct nci_dev *ndev, static const __u8 * nci_extract_rf_params_nfcv_passive_poll(struct nci_dev *ndev, struct rf_tech_specific_params_nfcv_poll *nfcv_poll, - const __u8 *data) + const __u8 *data, ssize_t data_len) { + /* Skip 1 byte (reserved) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); + ++data; + data_len--; + + /* Check if we have enough data for dsfid (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); + nfcv_poll->dsfid = *data++; + data_len--; + + /* Check if we have enough data for uid (8 bytes) */ + if (data_len < NFC_ISO15693_UID_MAXSIZE) + return ERR_PTR(-EINVAL); + memcpy(nfcv_poll->uid, data, NFC_ISO15693_UID_MAXSIZE); data += NFC_ISO15693_UID_MAXSIZE; + return data; } static const __u8 * nci_extract_rf_params_nfcf_passive_listen(struct nci_dev *ndev, struct rf_tech_specific_params_nfcf_listen *nfcf_listen, - const __u8 *data) + const __u8 *data, ssize_t data_len) { + /* Check if we have enough data for local_nfcid2_len (1 byte) */ + if (data_len < 1) + return ERR_PTR(-EINVAL); + nfcf_listen->local_nfcid2_len = min_t(__u8, *data++, NFC_NFCID2_MAXSIZE); + data_len--; + + /* Check if we have enough data for local_nfcid2 */ + if (data_len < nfcf_listen->local_nfcid2_len) + return ERR_PTR(-EINVAL); + memcpy(nfcf_listen->local_nfcid2, data, nfcf_listen->local_nfcid2_len); data += nfcf_listen->local_nfcid2_len; @@ -364,7 +444,7 @@ static int nci_rf_discover_ntf_packet(struct nci_dev *ndev, const __u8 *data; bool add_target = true; - if (skb->len < sizeof(struct nci_rf_discover_ntf)) + if (skb->len < offsetofend(struct nci_rf_discover_ntf, rf_tech_specific_params_len)) return -EINVAL; data = skb->data; @@ -380,26 +460,42 @@ static int nci_rf_discover_ntf_packet(struct nci_dev *ndev, pr_debug("rf_tech_specific_params_len %d\n", ntf.rf_tech_specific_params_len); + if (skb->len < (data - skb->data) + + ntf.rf_tech_specific_params_len + sizeof(ntf.ntf_type)) + return -EINVAL; + if (ntf.rf_tech_specific_params_len > 0) { switch (ntf.rf_tech_and_mode) { case NCI_NFC_A_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfca_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfca_poll), data); + &(ntf.rf_tech_specific_params.nfca_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return PTR_ERR(data); break; case NCI_NFC_B_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfcb_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfcb_poll), data); + &(ntf.rf_tech_specific_params.nfcb_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return PTR_ERR(data); break; case NCI_NFC_F_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfcf_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfcf_poll), data); + &(ntf.rf_tech_specific_params.nfcf_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return PTR_ERR(data); break; case NCI_NFC_V_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfcv_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfcv_poll), data); + &(ntf.rf_tech_specific_params.nfcv_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return PTR_ERR(data); break; default: @@ -596,7 +692,7 @@ static int nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, const __u8 *data; int err = NCI_STATUS_OK; - if (skb->len < sizeof(struct nci_rf_intf_activated_ntf)) + if (skb->len < offsetofend(struct nci_rf_intf_activated_ntf, rf_tech_specific_params_len)) return -EINVAL; data = skb->data; @@ -628,26 +724,41 @@ static int nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, if (ntf.rf_interface == NCI_RF_INTERFACE_NFCEE_DIRECT) goto listen; + if (skb->len < (data - skb->data) + ntf.rf_tech_specific_params_len) + return -EINVAL; + if (ntf.rf_tech_specific_params_len > 0) { switch (ntf.activation_rf_tech_and_mode) { case NCI_NFC_A_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfca_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfca_poll), data); + &(ntf.rf_tech_specific_params.nfca_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return -EINVAL; break; case NCI_NFC_B_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfcb_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfcb_poll), data); + &(ntf.rf_tech_specific_params.nfcb_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return -EINVAL; break; case NCI_NFC_F_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfcf_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfcf_poll), data); + &(ntf.rf_tech_specific_params.nfcf_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return -EINVAL; break; case NCI_NFC_V_PASSIVE_POLL_MODE: data = nci_extract_rf_params_nfcv_passive_poll(ndev, - &(ntf.rf_tech_specific_params.nfcv_poll), data); + &(ntf.rf_tech_specific_params.nfcv_poll), data, + ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return -EINVAL; break; case NCI_NFC_A_PASSIVE_LISTEN_MODE: @@ -657,7 +768,9 @@ static int nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, case NCI_NFC_F_PASSIVE_LISTEN_MODE: data = nci_extract_rf_params_nfcf_passive_listen(ndev, &(ntf.rf_tech_specific_params.nfcf_listen), - data); + data, ntf.rf_tech_specific_params_len); + if (IS_ERR(data)) + return -EINVAL; break; default: @@ -668,6 +781,13 @@ static int nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, } } + if (skb->len < (data - skb->data) + + sizeof(ntf.data_exch_rf_tech_and_mode) + + sizeof(ntf.data_exch_tx_bit_rate) + + sizeof(ntf.data_exch_rx_bit_rate) + + sizeof(ntf.activation_params_len)) + return -EINVAL; + ntf.data_exch_rf_tech_and_mode = *data++; ntf.data_exch_tx_bit_rate = *data++; ntf.data_exch_rx_bit_rate = *data++; @@ -679,6 +799,9 @@ static int nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, pr_debug("data_exch_rx_bit_rate 0x%x\n", ntf.data_exch_rx_bit_rate); pr_debug("activation_params_len %d\n", ntf.activation_params_len); + if (skb->len < (data - skb->data) + ntf.activation_params_len) + return -EINVAL; + if (ntf.activation_params_len > 0) { switch (ntf.rf_interface) { case NCI_RF_INTERFACE_ISO_DEP: diff --git a/net/nfc/rawsock.c b/net/nfc/rawsock.c index 5125392bb68eb..028b4daafaf83 100644 --- a/net/nfc/rawsock.c +++ b/net/nfc/rawsock.c @@ -67,6 +67,17 @@ static int rawsock_release(struct socket *sock) if (sock->type == SOCK_RAW) nfc_sock_unlink(&raw_sk_list, sk); + if (sk->sk_state == TCP_ESTABLISHED) { + /* Prevent rawsock_tx_work from starting new transmits and + * wait for any in-progress work to finish. This must happen + * before the socket is orphaned to avoid a race where + * rawsock_tx_work runs after the NCI device has been freed. + */ + sk->sk_shutdown |= SEND_SHUTDOWN; + cancel_work_sync(&nfc_rawsock(sk)->tx_work); + rawsock_write_queue_purge(sk); + } + sock_orphan(sk); sock_put(sk); diff --git a/net/psp/Kconfig b/net/psp/Kconfig index 371e8771f3bd3..84d6b0f254606 100644 --- a/net/psp/Kconfig +++ b/net/psp/Kconfig @@ -6,6 +6,7 @@ config INET_PSP bool "PSP Security Protocol support" depends on INET select SKB_DECRYPTED + select SKB_EXTENSIONS select SOCK_VALIDATE_XMIT help Enable kernel support for the PSP Security Protocol (PSP). diff --git a/net/psp/psp_main.c b/net/psp/psp_main.c index 481aaf0fc9fcf..53c0313633689 100644 --- a/net/psp/psp_main.c +++ b/net/psp/psp_main.c @@ -165,9 +165,46 @@ static void psp_write_headers(struct net *net, struct sk_buff *skb, __be32 spi, { struct udphdr *uh = udp_hdr(skb); struct psphdr *psph = (struct psphdr *)(uh + 1); + const struct sock *sk = skb->sk; uh->dest = htons(PSP_DEFAULT_UDP_PORT); - uh->source = udp_flow_src_port(net, skb, 0, 0, false); + + /* A bit of theory: Selection of the source port. + * + * We need some entropy, so that multiple flows use different + * source ports for better RSS spreading at the receiver. + * + * We also need that all packets belonging to one TCP flow + * use the same source port through their duration, + * so that all these packets land in the same receive queue. + * + * udp_flow_src_port() is using sk_txhash, inherited from + * skb_set_hash_from_sk() call in __tcp_transmit_skb(). + * This field is subject to reshuffling, thanks to + * sk_rethink_txhash() calls in various TCP functions. + * + * Instead, use sk->sk_hash which is constant through + * the whole flow duration. + */ + if (likely(sk)) { + u32 hash = sk->sk_hash; + int min, max; + + /* These operations are cheap, no need to cache the result + * in another socket field. + */ + inet_get_local_port_range(net, &min, &max); + /* Since this is being sent on the wire obfuscate hash a bit + * to minimize possibility that any useful information to an + * attacker is leaked. Only upper 16 bits are relevant in the + * computation for 16 bit port value because we use a + * reciprocal divide. + */ + hash ^= hash << 16; + uh->source = htons((((u64)hash * (max - min)) >> 32) + min); + } else { + uh->source = udp_flow_src_port(net, skb, 0, 0, false); + } uh->check = 0; uh->len = htons(udp_len); diff --git a/net/qrtr/mhi.c b/net/qrtr/mhi.c index 69f53625a049d..80e341d2f8a45 100644 --- a/net/qrtr/mhi.c +++ b/net/qrtr/mhi.c @@ -24,13 +24,25 @@ static void qcom_mhi_qrtr_dl_callback(struct mhi_device *mhi_dev, struct qrtr_mhi_dev *qdev = dev_get_drvdata(&mhi_dev->dev); int rc; - if (!qdev || mhi_res->transaction_status) + if (!qdev || (mhi_res->transaction_status && mhi_res->transaction_status != -ENOTCONN)) return; + /* Channel got reset. So just free the buffer */ + if (mhi_res->transaction_status == -ENOTCONN) { + devm_kfree(&mhi_dev->dev, mhi_res->buf_addr); + return; + } + rc = qrtr_endpoint_post(&qdev->ep, mhi_res->buf_addr, mhi_res->bytes_xferd); if (rc == -EINVAL) dev_err(qdev->dev, "invalid ipcrouter packet\n"); + + /* Done with the buffer, now recycle it for future use */ + rc = mhi_queue_buf(mhi_dev, DMA_FROM_DEVICE, mhi_res->buf_addr, + mhi_dev->mhi_cntrl->buffer_len, MHI_EOT); + if (rc) + dev_err(&mhi_dev->dev, "Failed to recycle the buffer: %d\n", rc); } /* From QRTR to MHI */ @@ -72,6 +84,29 @@ static int qcom_mhi_qrtr_send(struct qrtr_endpoint *ep, struct sk_buff *skb) return rc; } +static int qcom_mhi_qrtr_queue_dl_buffers(struct mhi_device *mhi_dev) +{ + u32 free_desc; + void *buf; + int ret; + + free_desc = mhi_get_free_desc_count(mhi_dev, DMA_FROM_DEVICE); + while (free_desc--) { + buf = devm_kmalloc(&mhi_dev->dev, mhi_dev->mhi_cntrl->buffer_len, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + ret = mhi_queue_buf(mhi_dev, DMA_FROM_DEVICE, buf, mhi_dev->mhi_cntrl->buffer_len, + MHI_EOT); + if (ret) { + dev_err(&mhi_dev->dev, "Failed to queue buffer: %d\n", ret); + return ret; + } + } + + return 0; +} + static int qcom_mhi_qrtr_probe(struct mhi_device *mhi_dev, const struct mhi_device_id *id) { @@ -87,20 +122,30 @@ static int qcom_mhi_qrtr_probe(struct mhi_device *mhi_dev, qdev->ep.xmit = qcom_mhi_qrtr_send; dev_set_drvdata(&mhi_dev->dev, qdev); - rc = qrtr_endpoint_register(&qdev->ep, QRTR_EP_NID_AUTO); - if (rc) - return rc; /* start channels */ - rc = mhi_prepare_for_transfer_autoqueue(mhi_dev); - if (rc) { - qrtr_endpoint_unregister(&qdev->ep); + rc = mhi_prepare_for_transfer(mhi_dev); + if (rc) return rc; - } + + rc = qrtr_endpoint_register(&qdev->ep, QRTR_EP_NID_AUTO); + if (rc) + goto err_unprepare; + + rc = qcom_mhi_qrtr_queue_dl_buffers(mhi_dev); + if (rc) + goto err_unregister; dev_dbg(qdev->dev, "Qualcomm MHI QRTR driver probed\n"); return 0; + +err_unregister: + qrtr_endpoint_unregister(&qdev->ep); +err_unprepare: + mhi_unprepare_from_transfer(mhi_dev); + + return rc; } static void qcom_mhi_qrtr_remove(struct mhi_device *mhi_dev) @@ -151,11 +196,13 @@ static int __maybe_unused qcom_mhi_qrtr_pm_resume_early(struct device *dev) if (state == MHI_STATE_M3) return 0; - rc = mhi_prepare_for_transfer_autoqueue(mhi_dev); - if (rc) + rc = mhi_prepare_for_transfer(mhi_dev); + if (rc) { dev_err(dev, "failed to prepare for autoqueue transfer %d\n", rc); + return rc; + } - return rc; + return qcom_mhi_qrtr_queue_dl_buffers(mhi_dev); } static const struct dev_pm_ops qcom_mhi_qrtr_pm_ops = { diff --git a/net/rds/connection.c b/net/rds/connection.c index 68bc88cce84ec..dbfea6fa11260 100644 --- a/net/rds/connection.c +++ b/net/rds/connection.c @@ -382,6 +382,8 @@ void rds_conn_shutdown(struct rds_conn_path *cp) if (!rds_conn_path_transition(cp, RDS_CONN_UP, RDS_CONN_DISCONNECTING) && !rds_conn_path_transition(cp, RDS_CONN_ERROR, + RDS_CONN_DISCONNECTING) && + !rds_conn_path_transition(cp, RDS_CONN_RESETTING, RDS_CONN_DISCONNECTING)) { rds_conn_path_error(cp, "shutdown called in state %d\n", @@ -427,6 +429,8 @@ void rds_conn_shutdown(struct rds_conn_path *cp) * to the conn hash, so we never trigger a reconnect on this * conn - the reconnect is always triggered by the active peer. */ cancel_delayed_work_sync(&cp->cp_conn_w); + + clear_bit(RDS_RECONNECT_PENDING, &cp->cp_flags); rcu_read_lock(); if (!hlist_unhashed(&conn->c_hash_node)) { rcu_read_unlock(); diff --git a/net/rds/send.c b/net/rds/send.c index 0b3d0ef2f008b..071c5dca969a2 100644 --- a/net/rds/send.c +++ b/net/rds/send.c @@ -1382,9 +1382,11 @@ int rds_sendmsg(struct socket *sock, struct msghdr *msg, size_t payload_len) else queue_delayed_work(rds_wq, &cpath->cp_send_w, 1); rcu_read_unlock(); + + if (ret) + goto out; } - if (ret) - goto out; + rds_message_put(rm); for (ind = 0; ind < vct.indx; ind++) diff --git a/net/rds/tcp.c b/net/rds/tcp.c index 3cc2f303bf786..b66dfcc3efaa0 100644 --- a/net/rds/tcp.c +++ b/net/rds/tcp.c @@ -495,18 +495,24 @@ bool rds_tcp_tune(struct socket *sock) struct rds_tcp_net *rtn; tcp_sock_set_nodelay(sock->sk); - lock_sock(sk); /* TCP timer functions might access net namespace even after * a process which created this net namespace terminated. */ if (!sk->sk_net_refcnt) { - if (!maybe_get_net(net)) { - release_sock(sk); + if (!maybe_get_net(net)) return false; - } + /* + * sk_net_refcnt_upgrade() must be called before lock_sock() + * because it does a GFP_KERNEL allocation, which can trigger + * fs_reclaim and create a circular lock dependency with the + * socket lock. The fields it modifies (sk_net_refcnt, + * ns_tracker) are not accessed by any concurrent code path + * at this point. + */ sk_net_refcnt_upgrade(sk); put_net(net); } + lock_sock(sk); rtn = net_generic(net, rds_tcp_netid); if (rtn->sndbuf_size > 0) { sk->sk_sndbuf = rtn->sndbuf_size; diff --git a/net/rds/tcp_listen.c b/net/rds/tcp_listen.c index 91e34af3fe5d5..65c5425a02de4 100644 --- a/net/rds/tcp_listen.c +++ b/net/rds/tcp_listen.c @@ -59,9 +59,6 @@ void rds_tcp_keepalive(struct socket *sock) * socket and force a reconneect from smaller -> larger ip addr. The reason * we special case cp_index 0 is to allow the rds probe ping itself to itself * get through efficiently. - * Since reconnects are only initiated from the node with the numerically - * smaller ip address, we recycle conns in RDS_CONN_ERROR on the passive side - * by moving them to CONNECTING in this function. */ static struct rds_tcp_connection *rds_tcp_accept_one_path(struct rds_connection *conn) @@ -86,8 +83,6 @@ struct rds_tcp_connection *rds_tcp_accept_one_path(struct rds_connection *conn) struct rds_conn_path *cp = &conn->c_path[i]; if (rds_conn_path_transition(cp, RDS_CONN_DOWN, - RDS_CONN_CONNECTING) || - rds_conn_path_transition(cp, RDS_CONN_ERROR, RDS_CONN_CONNECTING)) { return cp->cp_transport_data; } diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index 6749a4a9a9cd0..b3c160ad590d2 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -1358,6 +1358,12 @@ static int tcf_ct_init(struct net *net, struct nlattr *nla, return -EINVAL; } + if (bind && !(flags & TCA_ACT_FLAGS_AT_INGRESS_OR_CLSACT)) { + NL_SET_ERR_MSG_MOD(extack, + "Attaching ct to a non ingress/clsact qdisc is unsupported"); + return -EOPNOTSUPP; + } + err = nla_parse_nested(tb, TCA_CT_MAX, nla, ct_policy, extack); if (err < 0) return err; diff --git a/net/sched/act_gate.c b/net/sched/act_gate.c index c1f75f2727576..d09013ae1892a 100644 --- a/net/sched/act_gate.c +++ b/net/sched/act_gate.c @@ -32,9 +32,12 @@ static ktime_t gate_get_time(struct tcf_gate *gact) return KTIME_MAX; } -static void gate_get_start_time(struct tcf_gate *gact, ktime_t *start) +static void tcf_gate_params_free_rcu(struct rcu_head *head); + +static void gate_get_start_time(struct tcf_gate *gact, + const struct tcf_gate_params *param, + ktime_t *start) { - struct tcf_gate_params *param = &gact->param; ktime_t now, base, cycle; u64 n; @@ -69,12 +72,14 @@ static enum hrtimer_restart gate_timer_func(struct hrtimer *timer) { struct tcf_gate *gact = container_of(timer, struct tcf_gate, hitimer); - struct tcf_gate_params *p = &gact->param; struct tcfg_gate_entry *next; + struct tcf_gate_params *p; ktime_t close_time, now; spin_lock(&gact->tcf_lock); + p = rcu_dereference_protected(gact->param, + lockdep_is_held(&gact->tcf_lock)); next = gact->next_entry; /* cycle start, clear pending bit, clear total octets */ @@ -225,6 +230,35 @@ static void release_entry_list(struct list_head *entries) } } +static int tcf_gate_copy_entries(struct tcf_gate_params *dst, + const struct tcf_gate_params *src, + struct netlink_ext_ack *extack) +{ + struct tcfg_gate_entry *entry; + int i = 0; + + list_for_each_entry(entry, &src->entries, list) { + struct tcfg_gate_entry *new; + + new = kzalloc(sizeof(*new), GFP_ATOMIC); + if (!new) { + NL_SET_ERR_MSG(extack, "Not enough memory for entry"); + return -ENOMEM; + } + + new->index = entry->index; + new->gate_state = entry->gate_state; + new->interval = entry->interval; + new->ipv = entry->ipv; + new->maxoctets = entry->maxoctets; + list_add_tail(&new->list, &dst->entries); + i++; + } + + dst->num_entries = i; + return 0; +} + static int parse_gate_list(struct nlattr *list_attr, struct tcf_gate_params *sched, struct netlink_ext_ack *extack) @@ -270,24 +304,44 @@ static int parse_gate_list(struct nlattr *list_attr, return err; } -static void gate_setup_timer(struct tcf_gate *gact, u64 basetime, - enum tk_offsets tko, s32 clockid, - bool do_init) +static bool gate_timer_needs_cancel(u64 basetime, u64 old_basetime, + enum tk_offsets tko, + enum tk_offsets old_tko, + s32 clockid, s32 old_clockid) { - if (!do_init) { - if (basetime == gact->param.tcfg_basetime && - tko == gact->tk_offset && - clockid == gact->param.tcfg_clockid) - return; + return basetime != old_basetime || + clockid != old_clockid || + tko != old_tko; +} - spin_unlock_bh(&gact->tcf_lock); - hrtimer_cancel(&gact->hitimer); - spin_lock_bh(&gact->tcf_lock); +static int gate_clock_resolve(s32 clockid, enum tk_offsets *tko, + struct netlink_ext_ack *extack) +{ + switch (clockid) { + case CLOCK_REALTIME: + *tko = TK_OFFS_REAL; + return 0; + case CLOCK_MONOTONIC: + *tko = TK_OFFS_MAX; + return 0; + case CLOCK_BOOTTIME: + *tko = TK_OFFS_BOOT; + return 0; + case CLOCK_TAI: + *tko = TK_OFFS_TAI; + return 0; + default: + NL_SET_ERR_MSG(extack, "Invalid 'clockid'"); + return -EINVAL; } - gact->param.tcfg_basetime = basetime; - gact->param.tcfg_clockid = clockid; - gact->tk_offset = tko; - hrtimer_setup(&gact->hitimer, gate_timer_func, clockid, HRTIMER_MODE_ABS_SOFT); +} + +static void gate_setup_timer(struct tcf_gate *gact, s32 clockid, + enum tk_offsets tko) +{ + WRITE_ONCE(gact->tk_offset, tko); + hrtimer_setup(&gact->hitimer, gate_timer_func, clockid, + HRTIMER_MODE_ABS_SOFT); } static int tcf_gate_init(struct net *net, struct nlattr *nla, @@ -296,15 +350,22 @@ static int tcf_gate_init(struct net *net, struct nlattr *nla, struct netlink_ext_ack *extack) { struct tc_action_net *tn = net_generic(net, act_gate_ops.net_id); - enum tk_offsets tk_offset = TK_OFFS_TAI; + u64 cycletime = 0, basetime = 0, cycletime_ext = 0; + struct tcf_gate_params *p = NULL, *old_p = NULL; + enum tk_offsets old_tk_offset = TK_OFFS_TAI; + const struct tcf_gate_params *cur_p = NULL; bool bind = flags & TCA_ACT_FLAGS_BIND; struct nlattr *tb[TCA_GATE_MAX + 1]; + enum tk_offsets tko = TK_OFFS_TAI; struct tcf_chain *goto_ch = NULL; - u64 cycletime = 0, basetime = 0; - struct tcf_gate_params *p; + s32 timer_clockid = CLOCK_TAI; + bool use_old_entries = false; + s32 old_clockid = CLOCK_TAI; + bool need_cancel = false; s32 clockid = CLOCK_TAI; struct tcf_gate *gact; struct tc_gate *parm; + u64 old_basetime = 0; int ret = 0, err; u32 gflags = 0; s32 prio = -1; @@ -321,26 +382,8 @@ static int tcf_gate_init(struct net *net, struct nlattr *nla, if (!tb[TCA_GATE_PARMS]) return -EINVAL; - if (tb[TCA_GATE_CLOCKID]) { + if (tb[TCA_GATE_CLOCKID]) clockid = nla_get_s32(tb[TCA_GATE_CLOCKID]); - switch (clockid) { - case CLOCK_REALTIME: - tk_offset = TK_OFFS_REAL; - break; - case CLOCK_MONOTONIC: - tk_offset = TK_OFFS_MAX; - break; - case CLOCK_BOOTTIME: - tk_offset = TK_OFFS_BOOT; - break; - case CLOCK_TAI: - tk_offset = TK_OFFS_TAI; - break; - default: - NL_SET_ERR_MSG(extack, "Invalid 'clockid'"); - return -EINVAL; - } - } parm = nla_data(tb[TCA_GATE_PARMS]); index = parm->index; @@ -366,6 +409,60 @@ static int tcf_gate_init(struct net *net, struct nlattr *nla, return -EEXIST; } + gact = to_gate(*a); + + err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack); + if (err < 0) + goto release_idr; + + p = kzalloc(sizeof(*p), GFP_KERNEL); + if (!p) { + err = -ENOMEM; + goto chain_put; + } + INIT_LIST_HEAD(&p->entries); + + use_old_entries = !tb[TCA_GATE_ENTRY_LIST]; + if (!use_old_entries) { + err = parse_gate_list(tb[TCA_GATE_ENTRY_LIST], p, extack); + if (err < 0) + goto err_free; + use_old_entries = !err; + } + + if (ret == ACT_P_CREATED && use_old_entries) { + NL_SET_ERR_MSG(extack, "The entry list is empty"); + err = -EINVAL; + goto err_free; + } + + if (ret != ACT_P_CREATED) { + rcu_read_lock(); + cur_p = rcu_dereference(gact->param); + + old_basetime = cur_p->tcfg_basetime; + old_clockid = cur_p->tcfg_clockid; + old_tk_offset = READ_ONCE(gact->tk_offset); + + basetime = old_basetime; + cycletime_ext = cur_p->tcfg_cycletime_ext; + prio = cur_p->tcfg_priority; + gflags = cur_p->tcfg_flags; + + if (!tb[TCA_GATE_CLOCKID]) + clockid = old_clockid; + + err = 0; + if (use_old_entries) { + err = tcf_gate_copy_entries(p, cur_p, extack); + if (!err && !tb[TCA_GATE_CYCLE_TIME]) + cycletime = cur_p->tcfg_cycletime; + } + rcu_read_unlock(); + if (err) + goto err_free; + } + if (tb[TCA_GATE_PRIORITY]) prio = nla_get_s32(tb[TCA_GATE_PRIORITY]); @@ -375,25 +472,26 @@ static int tcf_gate_init(struct net *net, struct nlattr *nla, if (tb[TCA_GATE_FLAGS]) gflags = nla_get_u32(tb[TCA_GATE_FLAGS]); - gact = to_gate(*a); - if (ret == ACT_P_CREATED) - INIT_LIST_HEAD(&gact->param.entries); + if (tb[TCA_GATE_CYCLE_TIME]) + cycletime = nla_get_u64(tb[TCA_GATE_CYCLE_TIME]); - err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack); - if (err < 0) - goto release_idr; + if (tb[TCA_GATE_CYCLE_TIME_EXT]) + cycletime_ext = nla_get_u64(tb[TCA_GATE_CYCLE_TIME_EXT]); - spin_lock_bh(&gact->tcf_lock); - p = &gact->param; + err = gate_clock_resolve(clockid, &tko, extack); + if (err) + goto err_free; + timer_clockid = clockid; - if (tb[TCA_GATE_CYCLE_TIME]) - cycletime = nla_get_u64(tb[TCA_GATE_CYCLE_TIME]); + need_cancel = ret != ACT_P_CREATED && + gate_timer_needs_cancel(basetime, old_basetime, + tko, old_tk_offset, + timer_clockid, old_clockid); - if (tb[TCA_GATE_ENTRY_LIST]) { - err = parse_gate_list(tb[TCA_GATE_ENTRY_LIST], p, extack); - if (err < 0) - goto chain_put; - } + if (need_cancel) + hrtimer_cancel(&gact->hitimer); + + spin_lock_bh(&gact->tcf_lock); if (!cycletime) { struct tcfg_gate_entry *entry; @@ -402,22 +500,20 @@ static int tcf_gate_init(struct net *net, struct nlattr *nla, list_for_each_entry(entry, &p->entries, list) cycle = ktime_add_ns(cycle, entry->interval); cycletime = cycle; - if (!cycletime) { - err = -EINVAL; - goto chain_put; - } } p->tcfg_cycletime = cycletime; + p->tcfg_cycletime_ext = cycletime_ext; - if (tb[TCA_GATE_CYCLE_TIME_EXT]) - p->tcfg_cycletime_ext = - nla_get_u64(tb[TCA_GATE_CYCLE_TIME_EXT]); - - gate_setup_timer(gact, basetime, tk_offset, clockid, - ret == ACT_P_CREATED); + if (need_cancel || ret == ACT_P_CREATED) + gate_setup_timer(gact, timer_clockid, tko); p->tcfg_priority = prio; p->tcfg_flags = gflags; - gate_get_start_time(gact, &start); + p->tcfg_basetime = basetime; + p->tcfg_clockid = timer_clockid; + gate_get_start_time(gact, p, &start); + + old_p = rcu_replace_pointer(gact->param, p, + lockdep_is_held(&gact->tcf_lock)); gact->current_close_time = start; gact->current_gate_status = GATE_ACT_GATE_OPEN | GATE_ACT_PENDING; @@ -434,11 +530,15 @@ static int tcf_gate_init(struct net *net, struct nlattr *nla, if (goto_ch) tcf_chain_put_by_act(goto_ch); + if (old_p) + call_rcu(&old_p->rcu, tcf_gate_params_free_rcu); + return ret; +err_free: + release_entry_list(&p->entries); + kfree(p); chain_put: - spin_unlock_bh(&gact->tcf_lock); - if (goto_ch) tcf_chain_put_by_act(goto_ch); release_idr: @@ -446,21 +546,29 @@ static int tcf_gate_init(struct net *net, struct nlattr *nla, * without taking tcf_lock. */ if (ret == ACT_P_CREATED) - gate_setup_timer(gact, gact->param.tcfg_basetime, - gact->tk_offset, gact->param.tcfg_clockid, - true); + gate_setup_timer(gact, timer_clockid, tko); + tcf_idr_release(*a, bind); return err; } +static void tcf_gate_params_free_rcu(struct rcu_head *head) +{ + struct tcf_gate_params *p = container_of(head, struct tcf_gate_params, rcu); + + release_entry_list(&p->entries); + kfree(p); +} + static void tcf_gate_cleanup(struct tc_action *a) { struct tcf_gate *gact = to_gate(a); struct tcf_gate_params *p; - p = &gact->param; hrtimer_cancel(&gact->hitimer); - release_entry_list(&p->entries); + p = rcu_dereference_protected(gact->param, 1); + if (p) + call_rcu(&p->rcu, tcf_gate_params_free_rcu); } static int dumping_entry(struct sk_buff *skb, @@ -509,10 +617,9 @@ static int tcf_gate_dump(struct sk_buff *skb, struct tc_action *a, struct nlattr *entry_list; struct tcf_t t; - spin_lock_bh(&gact->tcf_lock); - opt.action = gact->tcf_action; - - p = &gact->param; + rcu_read_lock(); + opt.action = READ_ONCE(gact->tcf_action); + p = rcu_dereference(gact->param); if (nla_put(skb, TCA_GATE_PARMS, sizeof(opt), &opt)) goto nla_put_failure; @@ -552,12 +659,12 @@ static int tcf_gate_dump(struct sk_buff *skb, struct tc_action *a, tcf_tm_dump(&t, &gact->tcf_tm); if (nla_put_64bit(skb, TCA_GATE_TM, sizeof(t), &t, TCA_GATE_PAD)) goto nla_put_failure; - spin_unlock_bh(&gact->tcf_lock); + rcu_read_unlock(); return skb->len; nla_put_failure: - spin_unlock_bh(&gact->tcf_lock); + rcu_read_unlock(); nlmsg_trim(skb, b); return -1; } diff --git a/net/sched/act_ife.c b/net/sched/act_ife.c index 8e8f6af731d51..4ad01d4e820db 100644 --- a/net/sched/act_ife.c +++ b/net/sched/act_ife.c @@ -293,8 +293,8 @@ static int load_metaops_and_vet(u32 metaid, void *val, int len, bool rtnl_held) /* called when adding new meta information */ static int __add_metainfo(const struct tcf_meta_ops *ops, - struct tcf_ife_info *ife, u32 metaid, void *metaval, - int len, bool atomic, bool exists) + struct tcf_ife_params *p, u32 metaid, void *metaval, + int len, bool atomic) { struct tcf_meta_info *mi = NULL; int ret = 0; @@ -313,45 +313,40 @@ static int __add_metainfo(const struct tcf_meta_ops *ops, } } - if (exists) - spin_lock_bh(&ife->tcf_lock); - list_add_tail(&mi->metalist, &ife->metalist); - if (exists) - spin_unlock_bh(&ife->tcf_lock); + list_add_tail(&mi->metalist, &p->metalist); return ret; } static int add_metainfo_and_get_ops(const struct tcf_meta_ops *ops, - struct tcf_ife_info *ife, u32 metaid, - bool exists) + struct tcf_ife_params *p, u32 metaid) { int ret; if (!try_module_get(ops->owner)) return -ENOENT; - ret = __add_metainfo(ops, ife, metaid, NULL, 0, true, exists); + ret = __add_metainfo(ops, p, metaid, NULL, 0, true); if (ret) module_put(ops->owner); return ret; } -static int add_metainfo(struct tcf_ife_info *ife, u32 metaid, void *metaval, - int len, bool exists) +static int add_metainfo(struct tcf_ife_params *p, u32 metaid, void *metaval, + int len) { const struct tcf_meta_ops *ops = find_ife_oplist(metaid); int ret; if (!ops) return -ENOENT; - ret = __add_metainfo(ops, ife, metaid, metaval, len, false, exists); + ret = __add_metainfo(ops, p, metaid, metaval, len, false); if (ret) /*put back what find_ife_oplist took */ module_put(ops->owner); return ret; } -static int use_all_metadata(struct tcf_ife_info *ife, bool exists) +static int use_all_metadata(struct tcf_ife_params *p) { struct tcf_meta_ops *o; int rc = 0; @@ -359,7 +354,7 @@ static int use_all_metadata(struct tcf_ife_info *ife, bool exists) read_lock(&ife_mod_lock); list_for_each_entry(o, &ifeoplist, list) { - rc = add_metainfo_and_get_ops(o, ife, o->metaid, exists); + rc = add_metainfo_and_get_ops(o, p, o->metaid); if (rc == 0) installed += 1; } @@ -371,7 +366,7 @@ static int use_all_metadata(struct tcf_ife_info *ife, bool exists) return -EINVAL; } -static int dump_metalist(struct sk_buff *skb, struct tcf_ife_info *ife) +static int dump_metalist(struct sk_buff *skb, struct tcf_ife_params *p) { struct tcf_meta_info *e; struct nlattr *nest; @@ -379,14 +374,14 @@ static int dump_metalist(struct sk_buff *skb, struct tcf_ife_info *ife) int total_encoded = 0; /*can only happen on decode */ - if (list_empty(&ife->metalist)) + if (list_empty(&p->metalist)) return 0; nest = nla_nest_start_noflag(skb, TCA_IFE_METALST); if (!nest) goto out_nlmsg_trim; - list_for_each_entry(e, &ife->metalist, metalist) { + list_for_each_entry(e, &p->metalist, metalist) { if (!e->ops->get(skb, e)) total_encoded += 1; } @@ -403,13 +398,11 @@ static int dump_metalist(struct sk_buff *skb, struct tcf_ife_info *ife) return -1; } -/* under ife->tcf_lock */ -static void _tcf_ife_cleanup(struct tc_action *a) +static void __tcf_ife_cleanup(struct tcf_ife_params *p) { - struct tcf_ife_info *ife = to_ife(a); struct tcf_meta_info *e, *n; - list_for_each_entry_safe(e, n, &ife->metalist, metalist) { + list_for_each_entry_safe(e, n, &p->metalist, metalist) { list_del(&e->metalist); if (e->metaval) { if (e->ops->release) @@ -422,18 +415,23 @@ static void _tcf_ife_cleanup(struct tc_action *a) } } +static void tcf_ife_cleanup_params(struct rcu_head *head) +{ + struct tcf_ife_params *p = container_of(head, struct tcf_ife_params, + rcu); + + __tcf_ife_cleanup(p); + kfree(p); +} + static void tcf_ife_cleanup(struct tc_action *a) { struct tcf_ife_info *ife = to_ife(a); struct tcf_ife_params *p; - spin_lock_bh(&ife->tcf_lock); - _tcf_ife_cleanup(a); - spin_unlock_bh(&ife->tcf_lock); - p = rcu_dereference_protected(ife->params, 1); if (p) - kfree_rcu(p, rcu); + call_rcu(&p->rcu, tcf_ife_cleanup_params); } static int load_metalist(struct nlattr **tb, bool rtnl_held) @@ -455,8 +453,7 @@ static int load_metalist(struct nlattr **tb, bool rtnl_held) return 0; } -static int populate_metalist(struct tcf_ife_info *ife, struct nlattr **tb, - bool exists, bool rtnl_held) +static int populate_metalist(struct tcf_ife_params *p, struct nlattr **tb) { int len = 0; int rc = 0; @@ -468,7 +465,7 @@ static int populate_metalist(struct tcf_ife_info *ife, struct nlattr **tb, val = nla_data(tb[i]); len = nla_len(tb[i]); - rc = add_metainfo(ife, i, val, len, exists); + rc = add_metainfo(p, i, val, len); if (rc) return rc; } @@ -523,6 +520,7 @@ static int tcf_ife_init(struct net *net, struct nlattr *nla, p = kzalloc(sizeof(*p), GFP_KERNEL); if (!p) return -ENOMEM; + INIT_LIST_HEAD(&p->metalist); if (tb[TCA_IFE_METALST]) { err = nla_parse_nested_deprecated(tb2, IFE_META_MAX, @@ -567,8 +565,6 @@ static int tcf_ife_init(struct net *net, struct nlattr *nla, } ife = to_ife(*a); - if (ret == ACT_P_CREATED) - INIT_LIST_HEAD(&ife->metalist); err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack); if (err < 0) @@ -600,8 +596,7 @@ static int tcf_ife_init(struct net *net, struct nlattr *nla, } if (tb[TCA_IFE_METALST]) { - err = populate_metalist(ife, tb2, exists, - !(flags & TCA_ACT_FLAGS_NO_RTNL)); + err = populate_metalist(p, tb2); if (err) goto metadata_parse_err; } else { @@ -610,7 +605,7 @@ static int tcf_ife_init(struct net *net, struct nlattr *nla, * as we can. You better have at least one else we are * going to bail out */ - err = use_all_metadata(ife, exists); + err = use_all_metadata(p); if (err) goto metadata_parse_err; } @@ -626,13 +621,14 @@ static int tcf_ife_init(struct net *net, struct nlattr *nla, if (goto_ch) tcf_chain_put_by_act(goto_ch); if (p) - kfree_rcu(p, rcu); + call_rcu(&p->rcu, tcf_ife_cleanup_params); return ret; metadata_parse_err: if (goto_ch) tcf_chain_put_by_act(goto_ch); release_idr: + __tcf_ife_cleanup(p); kfree(p); tcf_idr_release(*a, bind); return err; @@ -679,7 +675,7 @@ static int tcf_ife_dump(struct sk_buff *skb, struct tc_action *a, int bind, if (nla_put(skb, TCA_IFE_TYPE, 2, &p->eth_type)) goto nla_put_failure; - if (dump_metalist(skb, ife)) { + if (dump_metalist(skb, p)) { /*ignore failure to dump metalist */ pr_info("Failed to dump metalist\n"); } @@ -693,13 +689,13 @@ static int tcf_ife_dump(struct sk_buff *skb, struct tc_action *a, int bind, return -1; } -static int find_decode_metaid(struct sk_buff *skb, struct tcf_ife_info *ife, +static int find_decode_metaid(struct sk_buff *skb, struct tcf_ife_params *p, u16 metaid, u16 mlen, void *mdata) { struct tcf_meta_info *e; /* XXX: use hash to speed up */ - list_for_each_entry(e, &ife->metalist, metalist) { + list_for_each_entry_rcu(e, &p->metalist, metalist) { if (metaid == e->metaid) { if (e->ops) { /* We check for decode presence already */ @@ -716,10 +712,13 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a, { struct tcf_ife_info *ife = to_ife(a); int action = ife->tcf_action; + struct tcf_ife_params *p; u8 *ifehdr_end; u8 *tlv_data; u16 metalen; + p = rcu_dereference_bh(ife->params); + bstats_update(this_cpu_ptr(ife->common.cpu_bstats), skb); tcf_lastuse_update(&ife->tcf_tm); @@ -745,7 +744,7 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a, return TC_ACT_SHOT; } - if (find_decode_metaid(skb, ife, mtype, dlen, curr_data)) { + if (find_decode_metaid(skb, p, mtype, dlen, curr_data)) { /* abuse overlimits to count when we receive metadata * but dont have an ops for it */ @@ -769,12 +768,12 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a, /*XXX: check if we can do this at install time instead of current * send data path **/ -static int ife_get_sz(struct sk_buff *skb, struct tcf_ife_info *ife) +static int ife_get_sz(struct sk_buff *skb, struct tcf_ife_params *p) { - struct tcf_meta_info *e, *n; + struct tcf_meta_info *e; int tot_run_sz = 0, run_sz = 0; - list_for_each_entry_safe(e, n, &ife->metalist, metalist) { + list_for_each_entry_rcu(e, &p->metalist, metalist) { if (e->ops->check_presence) { run_sz = e->ops->check_presence(skb, e); tot_run_sz += run_sz; @@ -795,7 +794,7 @@ static int tcf_ife_encode(struct sk_buff *skb, const struct tc_action *a, OUTERHDR:TOTMETALEN:{TLVHDR:Metadatum:TLVHDR..}:ORIGDATA where ORIGDATA = original ethernet header ... */ - u16 metalen = ife_get_sz(skb, ife); + u16 metalen = ife_get_sz(skb, p); int hdrm = metalen + skb->dev->hard_header_len + IFE_METAHDRLEN; unsigned int skboff = 0; int new_len = skb->len + hdrm; @@ -833,25 +832,21 @@ static int tcf_ife_encode(struct sk_buff *skb, const struct tc_action *a, if (!ife_meta) goto drop; - spin_lock(&ife->tcf_lock); - /* XXX: we dont have a clever way of telling encode to * not repeat some of the computations that are done by * ops->presence_check... */ - list_for_each_entry(e, &ife->metalist, metalist) { + list_for_each_entry_rcu(e, &p->metalist, metalist) { if (e->ops->encode) { err = e->ops->encode(skb, (void *)(ife_meta + skboff), e); } if (err < 0) { /* too corrupt to keep around if overwritten */ - spin_unlock(&ife->tcf_lock); goto drop; } skboff += err; } - spin_unlock(&ife->tcf_lock); oethh = (struct ethhdr *)skb->data; if (!is_zero_ether_addr(p->eth_src)) diff --git a/net/sched/act_skbedit.c b/net/sched/act_skbedit.c index 8c1d1554f6575..5450c1293eb50 100644 --- a/net/sched/act_skbedit.c +++ b/net/sched/act_skbedit.c @@ -126,7 +126,7 @@ static int tcf_skbedit_init(struct net *net, struct nlattr *nla, struct tcf_skbedit *d; u32 flags = 0, *priority = NULL, *mark = NULL, *mask = NULL; u16 *queue_mapping = NULL, *ptype = NULL; - u16 mapping_mod = 1; + u32 mapping_mod = 1; bool exists = false; int ret = 0, err; u32 index; @@ -194,6 +194,10 @@ static int tcf_skbedit_init(struct net *net, struct nlattr *nla, } mapping_mod = *queue_mapping_max - *queue_mapping + 1; + if (mapping_mod > U16_MAX) { + NL_SET_ERR_MSG_MOD(extack, "The range of queue_mapping is invalid."); + return -EINVAL; + } flags |= SKBEDIT_F_TXQ_SKBHASH; } if (*pure_flags & SKBEDIT_F_INHERITDSFIELD) diff --git a/net/sched/bpf_qdisc.c b/net/sched/bpf_qdisc.c index adcb618a2bfca..e9bea9890777d 100644 --- a/net/sched/bpf_qdisc.c +++ b/net/sched/bpf_qdisc.c @@ -202,6 +202,12 @@ __bpf_kfunc void bpf_kfree_skb(struct sk_buff *skb) kfree_skb(skb); } +__bpf_kfunc void bpf_kfree_skb_dtor(void *skb) +{ + bpf_kfree_skb(skb); +} +CFI_NOSEAL(bpf_kfree_skb_dtor); + /* bpf_qdisc_skb_drop - Drop an skb by adding it to a deferred free list. * @skb: The skb whose reference to be released and dropped. * @to_free_list: The list of skbs to be dropped. @@ -449,7 +455,7 @@ static struct bpf_struct_ops bpf_Qdisc_ops = { .owner = THIS_MODULE, }; -BTF_ID_LIST_SINGLE(bpf_sk_buff_dtor_ids, func, bpf_kfree_skb) +BTF_ID_LIST_SINGLE(bpf_sk_buff_dtor_ids, func, bpf_kfree_skb_dtor) static int __init bpf_qdisc_kfunc_init(void) { diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index ecec0a1e1c1a0..bac9cd71ff8e9 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -2228,6 +2228,11 @@ static bool is_qdisc_ingress(__u32 classid) return (TC_H_MIN(classid) == TC_H_MIN(TC_H_MIN_INGRESS)); } +static bool is_ingress_or_clsact(struct tcf_block *block, struct Qdisc *q) +{ + return tcf_block_shared(block) || (q && !!(q->flags & TCQ_F_INGRESS)); +} + static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, struct netlink_ext_ack *extack) { @@ -2420,6 +2425,8 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, flags |= TCA_ACT_FLAGS_NO_RTNL; if (is_qdisc_ingress(parent)) flags |= TCA_ACT_FLAGS_AT_INGRESS; + if (is_ingress_or_clsact(block, q)) + flags |= TCA_ACT_FLAGS_AT_INGRESS_OR_CLSACT; err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh, flags, extack); if (err == 0) { diff --git a/net/sched/sch_ets.c b/net/sched/sch_ets.c index 306e046276d46..a4b07b661b775 100644 --- a/net/sched/sch_ets.c +++ b/net/sched/sch_ets.c @@ -115,12 +115,12 @@ static void ets_offload_change(struct Qdisc *sch) struct ets_sched *q = qdisc_priv(sch); struct tc_ets_qopt_offload qopt; unsigned int w_psum_prev = 0; - unsigned int q_psum = 0; - unsigned int q_sum = 0; unsigned int quantum; unsigned int w_psum; unsigned int weight; unsigned int i; + u64 q_psum = 0; + u64 q_sum = 0; if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc) return; @@ -138,8 +138,12 @@ static void ets_offload_change(struct Qdisc *sch) for (i = 0; i < q->nbands; i++) { quantum = q->classes[i].quantum; - q_psum += quantum; - w_psum = quantum ? q_psum * 100 / q_sum : 0; + if (quantum) { + q_psum += quantum; + w_psum = div64_u64(q_psum * 100, q_sum); + } else { + w_psum = 0; + } weight = w_psum - w_psum_prev; w_psum_prev = w_psum; diff --git a/net/sched/sch_fq.c b/net/sched/sch_fq.c index fee922da2f99c..5e41930079948 100644 --- a/net/sched/sch_fq.c +++ b/net/sched/sch_fq.c @@ -826,6 +826,7 @@ static void fq_reset(struct Qdisc *sch) for (idx = 0; idx < FQ_BANDS; idx++) { q->band_flows[idx].new_flows.first = NULL; q->band_flows[idx].old_flows.first = NULL; + q->band_pkt_count[idx] = 0; } q->delayed = RB_ROOT; q->flows = 0; diff --git a/net/sctp/ipv6.c b/net/sctp/ipv6.c index 568ff8797c393..d725b21587588 100644 --- a/net/sctp/ipv6.c +++ b/net/sctp/ipv6.c @@ -782,9 +782,10 @@ static struct sock *sctp_v6_create_accept_sk(struct sock *sk, struct sctp_association *asoc, bool kern) { - struct sock *newsk; struct ipv6_pinfo *newnp, *np = inet6_sk(sk); struct sctp6_sock *newsctp6sk; + struct inet_sock *newinet; + struct sock *newsk; newsk = sk_alloc(sock_net(sk), PF_INET6, GFP_KERNEL, sk->sk_prot, kern); if (!newsk) @@ -796,7 +797,9 @@ static struct sock *sctp_v6_create_accept_sk(struct sock *sk, sock_reset_flag(sk, SOCK_ZAPPED); newsctp6sk = (struct sctp6_sock *)newsk; - inet_sk(newsk)->pinet6 = &newsctp6sk->inet6; + newinet = inet_sk(newsk); + newinet->pinet6 = &newsctp6sk->inet6; + newinet->ipv6_fl_list = NULL; sctp_sk(newsk)->v4mapped = sctp_sk(sk)->v4mapped; @@ -805,7 +808,6 @@ static struct sock *sctp_v6_create_accept_sk(struct sock *sk, memcpy(newnp, np, sizeof(struct ipv6_pinfo)); newnp->ipv6_mc_list = NULL; newnp->ipv6_ac_list = NULL; - newnp->ipv6_fl_list = NULL; sctp_v6_copy_ip_options(sk, newsk); diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c index 77b99e8ef35a4..efdadb2d8d390 100644 --- a/net/smc/af_smc.c +++ b/net/smc/af_smc.c @@ -124,7 +124,9 @@ static struct sock *smc_tcp_syn_recv_sock(const struct sock *sk, struct request_sock *req, struct dst_entry *dst, struct request_sock *req_unhash, - bool *own_req) + bool *own_req, + void (*opt_child_init)(struct sock *newsk, + const struct sock *sk)) { struct smc_sock *smc; struct sock *child; @@ -142,7 +144,7 @@ static struct sock *smc_tcp_syn_recv_sock(const struct sock *sk, /* passthrough to original syn recv sock fct */ child = smc->ori_af_ops->syn_recv_sock(sk, skb, req, dst, req_unhash, - own_req); + own_req, opt_child_init); /* child must not inherit smc or its ops */ if (child) { rcu_assign_sk_user_data(child, NULL); diff --git a/net/socket.c b/net/socket.c index e8892b2187087..2b6e11b085ebd 100644 --- a/net/socket.c +++ b/net/socket.c @@ -674,7 +674,7 @@ static void __sock_release(struct socket *sock, struct inode *inode) iput(SOCK_INODE(sock)); return; } - sock->file = NULL; + WRITE_ONCE(sock->file, NULL); } /** diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c index 5c095cb8cb201..bb3c3db2713b1 100644 --- a/net/sunrpc/auth_gss/auth_gss.c +++ b/net/sunrpc/auth_gss/auth_gss.c @@ -39,6 +39,8 @@ static const struct rpc_authops authgss_ops; static const struct rpc_credops gss_credops; static const struct rpc_credops gss_nullops; +static void gss_free_callback(struct kref *kref); + #define GSS_RETRY_EXPIRED 5 static unsigned int gss_expired_cred_retry_delay = GSS_RETRY_EXPIRED; @@ -551,6 +553,7 @@ gss_alloc_msg(struct gss_auth *gss_auth, } return gss_msg; err_put_pipe_version: + kref_put(&gss_auth->kref, gss_free_callback); put_pipe_version(gss_auth->net); err_free_msg: kfree(gss_msg); diff --git a/net/sunrpc/auth_gss/gss_rpc_xdr.c b/net/sunrpc/auth_gss/gss_rpc_xdr.c index 7d2cdc2bd374e..f320c0a8e6049 100644 --- a/net/sunrpc/auth_gss/gss_rpc_xdr.c +++ b/net/sunrpc/auth_gss/gss_rpc_xdr.c @@ -320,29 +320,47 @@ static int gssx_dec_status(struct xdr_stream *xdr, /* status->minor_status */ p = xdr_inline_decode(xdr, 8); - if (unlikely(p == NULL)) - return -ENOSPC; + if (unlikely(p == NULL)) { + err = -ENOSPC; + goto out_free_mech; + } p = xdr_decode_hyper(p, &status->minor_status); /* status->major_status_string */ err = gssx_dec_buffer(xdr, &status->major_status_string); if (err) - return err; + goto out_free_mech; /* status->minor_status_string */ err = gssx_dec_buffer(xdr, &status->minor_status_string); if (err) - return err; + goto out_free_major_status_string; /* status->server_ctx */ err = gssx_dec_buffer(xdr, &status->server_ctx); if (err) - return err; + goto out_free_minor_status_string; /* we assume we have no options for now, so simply consume them */ /* status->options */ err = dummy_dec_opt_array(xdr, &status->options); + if (err) + goto out_free_server_ctx; + return 0; + +out_free_server_ctx: + kfree(status->server_ctx.data); + status->server_ctx.data = NULL; +out_free_minor_status_string: + kfree(status->minor_status_string.data); + status->minor_status_string.data = NULL; +out_free_major_status_string: + kfree(status->major_status_string.data); + status->major_status_string.data = NULL; +out_free_mech: + kfree(status->mech.data); + status->mech.data = NULL; return err; } @@ -505,28 +523,35 @@ static int gssx_dec_name(struct xdr_stream *xdr, /* name->name_type */ err = gssx_dec_buffer(xdr, &dummy_netobj); if (err) - return err; + goto out_free_display_name; /* name->exported_name */ err = gssx_dec_buffer(xdr, &dummy_netobj); if (err) - return err; + goto out_free_display_name; /* name->exported_composite_name */ err = gssx_dec_buffer(xdr, &dummy_netobj); if (err) - return err; + goto out_free_display_name; /* we assume we have no attributes for now, so simply consume them */ /* name->name_attributes */ err = dummy_dec_nameattr_array(xdr, &dummy_name_attr_array); if (err) - return err; + goto out_free_display_name; /* we assume we have no options for now, so simply consume them */ /* name->extensions */ err = dummy_dec_opt_array(xdr, &dummy_option_array); + if (err) + goto out_free_display_name; + return 0; + +out_free_display_name: + kfree(name->display_name.data); + name->display_name.data = NULL; return err; } @@ -649,32 +674,34 @@ static int gssx_dec_ctx(struct xdr_stream *xdr, /* ctx->state */ err = gssx_dec_buffer(xdr, &ctx->state); if (err) - return err; + goto out_free_exported_context_token; /* ctx->need_release */ err = gssx_dec_bool(xdr, &ctx->need_release); if (err) - return err; + goto out_free_state; /* ctx->mech */ err = gssx_dec_buffer(xdr, &ctx->mech); if (err) - return err; + goto out_free_state; /* ctx->src_name */ err = gssx_dec_name(xdr, &ctx->src_name); if (err) - return err; + goto out_free_mech; /* ctx->targ_name */ err = gssx_dec_name(xdr, &ctx->targ_name); if (err) - return err; + goto out_free_src_name; /* ctx->lifetime */ p = xdr_inline_decode(xdr, 8+8); - if (unlikely(p == NULL)) - return -ENOSPC; + if (unlikely(p == NULL)) { + err = -ENOSPC; + goto out_free_targ_name; + } p = xdr_decode_hyper(p, &ctx->lifetime); /* ctx->ctx_flags */ @@ -683,17 +710,36 @@ static int gssx_dec_ctx(struct xdr_stream *xdr, /* ctx->locally_initiated */ err = gssx_dec_bool(xdr, &ctx->locally_initiated); if (err) - return err; + goto out_free_targ_name; /* ctx->open */ err = gssx_dec_bool(xdr, &ctx->open); if (err) - return err; + goto out_free_targ_name; /* we assume we have no options for now, so simply consume them */ /* ctx->options */ err = dummy_dec_opt_array(xdr, &ctx->options); + if (err) + goto out_free_targ_name; + + return 0; +out_free_targ_name: + kfree(ctx->targ_name.display_name.data); + ctx->targ_name.display_name.data = NULL; +out_free_src_name: + kfree(ctx->src_name.display_name.data); + ctx->src_name.display_name.data = NULL; +out_free_mech: + kfree(ctx->mech.data); + ctx->mech.data = NULL; +out_free_state: + kfree(ctx->state.data); + ctx->state.data = NULL; +out_free_exported_context_token: + kfree(ctx->exported_context_token.data); + ctx->exported_context_token.data = NULL; return err; } diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c index 3d7f1413df023..12857381e8610 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_transport.c +++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c @@ -462,7 +462,10 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) newxprt->sc_max_bc_requests = 2; } - /* Arbitrary estimate of the needed number of rdma_rw contexts. + /* Estimate the needed number of rdma_rw contexts. The maximum + * Read and Write chunks have one segment each. Each request + * can involve one Read chunk and either a Write chunk or Reply + * chunk; thus a factor of three. */ maxpayload = min(xprt->xpt_server->sv_max_payload, RPCSVC_MAXPAYLOAD_RDMA); @@ -470,7 +473,8 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) rdma_rw_mr_factor(dev, newxprt->sc_port_num, maxpayload >> PAGE_SHIFT); - newxprt->sc_sq_depth = rq_depth + ctxts; + newxprt->sc_sq_depth = rq_depth + + rdma_rw_max_send_wr(dev, newxprt->sc_port_num, ctxts, 0); if (newxprt->sc_sq_depth > dev->attrs.max_qp_wr) newxprt->sc_sq_depth = dev->attrs.max_qp_wr; atomic_set(&newxprt->sc_sq_avail, newxprt->sc_sq_depth); diff --git a/net/tipc/crypto.c b/net/tipc/crypto.c index 970db62bd029b..a3f9ca28c3d53 100644 --- a/net/tipc/crypto.c +++ b/net/tipc/crypto.c @@ -460,7 +460,7 @@ static void tipc_aead_users_dec(struct tipc_aead __rcu *aead, int lim) rcu_read_lock(); tmp = rcu_dereference(aead); if (tmp) - atomic_add_unless(&rcu_dereference(aead)->users, -1, lim); + atomic_add_unless(&tmp->users, -1, lim); rcu_read_unlock(); } diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c index e74940eab3a47..7f42fb6a8481f 100644 --- a/net/tipc/name_table.c +++ b/net/tipc/name_table.c @@ -348,7 +348,8 @@ static bool tipc_service_insert_publ(struct net *net, /* Return if the publication already exists */ list_for_each_entry(_p, &sr->all_publ, all_publ) { - if (_p->key == key && (!_p->sk.node || _p->sk.node == node)) { + if (_p->key == key && _p->sk.ref == p->sk.ref && + (!_p->sk.node || _p->sk.node == node)) { pr_debug("Failed to bind duplicate %u,%u,%u/%u:%u/%u\n", p->sr.type, p->sr.lower, p->sr.upper, node, p->sk.ref, key); @@ -388,7 +389,8 @@ static struct publication *tipc_service_remove_publ(struct service_range *r, u32 node = sk->node; list_for_each_entry(p, &r->all_publ, all_publ) { - if (p->key != key || (node && node != p->sk.node)) + if (p->key != key || p->sk.ref != sk->ref || + (node && node != p->sk.node)) continue; list_del(&p->all_publ); list_del(&p->local_publ); diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index d171353699800..c6a708ee21dc8 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -2533,7 +2533,7 @@ void tls_sw_cancel_work_tx(struct tls_context *tls_ctx) set_bit(BIT_TX_CLOSING, &ctx->tx_bitmask); set_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask); - cancel_delayed_work_sync(&ctx->tx_work.work); + disable_delayed_work_sync(&ctx->tx_work.work); } void tls_sw_release_resources_tx(struct sock *sk) diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index c634a7fc86090..79943fb348064 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -1671,10 +1671,9 @@ static int unix_stream_connect(struct socket *sock, struct sockaddr *uaddr, timeo = sock_sndtimeo(sk, flags & O_NONBLOCK); - /* First of all allocate resources. - * If we will make it after state is locked, - * we will have to recheck all again in any case. - */ + err = prepare_peercred(&peercred); + if (err) + goto out; /* create new sock for complete connection */ newsk = unix_create1(net, NULL, 0, sock->type); @@ -1683,10 +1682,6 @@ static int unix_stream_connect(struct socket *sock, struct sockaddr *uaddr, goto out; } - err = prepare_peercred(&peercred); - if (err) - goto out; - /* Allocate skb for sending to listening sock */ skb = sock_wmalloc(newsk, 1, 0, GFP_KERNEL); if (!skb) { @@ -1811,7 +1806,7 @@ static int unix_stream_connect(struct socket *sock, struct sockaddr *uaddr, __skb_queue_tail(&other->sk_receive_queue, skb); spin_unlock(&other->sk_receive_queue.lock); unix_state_unlock(other); - other->sk_data_ready(other); + READ_ONCE(other->sk_data_ready)(other); sock_put(other); return 0; @@ -2306,7 +2301,7 @@ static int unix_dgram_sendmsg(struct socket *sock, struct msghdr *msg, scm_stat_add(other, skb); skb_queue_tail(&other->sk_receive_queue, skb); unix_state_unlock(other); - other->sk_data_ready(other); + READ_ONCE(other->sk_data_ready)(other); sock_put(other); scm_destroy(&scm); return len; @@ -2379,7 +2374,7 @@ static int queue_oob(struct sock *sk, struct msghdr *msg, struct sock *other, sk_send_sigurg(other); unix_state_unlock(other); - other->sk_data_ready(other); + READ_ONCE(other->sk_data_ready)(other); return 0; out_unlock: @@ -2507,7 +2502,7 @@ static int unix_stream_sendmsg(struct socket *sock, struct msghdr *msg, spin_unlock(&other->sk_receive_queue.lock); unix_state_unlock(other); - other->sk_data_ready(other); + READ_ONCE(other->sk_data_ready)(other); sent += size; } diff --git a/net/vmw_vsock/vmci_transport.c b/net/vmw_vsock/vmci_transport.c index 7eccd6708d664..aca3132689cf1 100644 --- a/net/vmw_vsock/vmci_transport.c +++ b/net/vmw_vsock/vmci_transport.c @@ -161,7 +161,7 @@ vmci_transport_packet_init(struct vmci_transport_packet *pkt, case VMCI_TRANSPORT_PACKET_TYPE_WAITING_READ: case VMCI_TRANSPORT_PACKET_TYPE_WAITING_WRITE: - memcpy(&pkt->u.wait, wait, sizeof(pkt->u.wait)); + pkt->u.wait = *wait; break; case VMCI_TRANSPORT_PACKET_TYPE_REQUEST2: diff --git a/net/wireless/core.c b/net/wireless/core.c index 5e5c1bc380a89..6b204ca91a140 100644 --- a/net/wireless/core.c +++ b/net/wireless/core.c @@ -658,12 +658,8 @@ int wiphy_verify_iface_combinations(struct wiphy *wiphy, c->limits[j].max > 1)) return -EINVAL; - /* Only a single NAN can be allowed, avoid this - * check for multi-radio global combination, since it - * hold the capabilities of all radio combinations. - */ - if (!combined_radio && - WARN_ON(types & BIT(NL80211_IFTYPE_NAN) && + /* Only a single NAN can be allowed */ + if (WARN_ON(types & BIT(NL80211_IFTYPE_NAN) && c->limits[j].max > 1)) return -EINVAL; @@ -1199,6 +1195,7 @@ void wiphy_unregister(struct wiphy *wiphy) /* this has nothing to do now but make sure it's gone */ cancel_work_sync(&rdev->wiphy_work); + cancel_work_sync(&rdev->rfkill_block); cancel_work_sync(&rdev->conn_work); flush_work(&rdev->event_work); cancel_delayed_work_sync(&rdev->dfs_update_channels_wk); @@ -1400,8 +1397,10 @@ void cfg80211_leave(struct cfg80211_registered_device *rdev, cfg80211_leave_ocb(rdev, dev); break; case NL80211_IFTYPE_P2P_DEVICE: + cfg80211_stop_p2p_device(rdev, wdev); + break; case NL80211_IFTYPE_NAN: - /* cannot happen, has no netdev */ + cfg80211_stop_nan(rdev, wdev); break; case NL80211_IFTYPE_AP_VLAN: case NL80211_IFTYPE_MONITOR: diff --git a/net/wireless/radiotap.c b/net/wireless/radiotap.c index 326faea38ca38..c85eaa583a466 100644 --- a/net/wireless/radiotap.c +++ b/net/wireless/radiotap.c @@ -239,14 +239,14 @@ int ieee80211_radiotap_iterator_next( default: if (!iterator->current_namespace || iterator->_arg_index >= iterator->current_namespace->n_bits) { - if (iterator->current_namespace == &radiotap_ns) - return -ENOENT; align = 0; } else { align = iterator->current_namespace->align_size[iterator->_arg_index].align; size = iterator->current_namespace->align_size[iterator->_arg_index].size; } if (!align) { + if (iterator->current_namespace == &radiotap_ns) + return -ENOENT; /* skip all subsequent data */ iterator->_arg = iterator->_next_ns_data; /* give up on this namespace */ diff --git a/net/wireless/scan.c b/net/wireless/scan.c index 90a9187a6b135..9a0c02c23dc56 100644 --- a/net/wireless/scan.c +++ b/net/wireless/scan.c @@ -1959,7 +1959,7 @@ cfg80211_update_known_bss(struct cfg80211_registered_device *rdev, ether_addr_copy(known->parent_bssid, new->parent_bssid); known->pub.max_bssid_indicator = new->pub.max_bssid_indicator; known->pub.bssid_index = new->pub.bssid_index; - known->pub.use_for &= new->pub.use_for; + known->pub.use_for = new->pub.use_for; known->pub.cannot_use_reasons = new->pub.cannot_use_reasons; known->bss_source = new->bss_source; diff --git a/net/wireless/wext-compat.c b/net/wireless/wext-compat.c index 1241fda78a68c..680500fa57cfd 100644 --- a/net/wireless/wext-compat.c +++ b/net/wireless/wext-compat.c @@ -684,7 +684,7 @@ static int cfg80211_wext_siwencodeext(struct net_device *dev, idx = erq->flags & IW_ENCODE_INDEX; if (cipher == WLAN_CIPHER_SUITE_AES_CMAC) { - if (idx < 4 || idx > 5) { + if (idx < 5 || idx > 6) { idx = wdev->wext.default_mgmt_key; if (idx < 0) return -EINVAL; diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index 69bbcca8ac753..a78cdc3356937 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -167,26 +167,32 @@ static int xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) struct xdp_buff_xsk *pos, *tmp; struct list_head *xskb_list; u32 contd = 0; + u32 num_desc; int err; - if (frags) - contd = XDP_PKT_CONTD; + if (likely(!frags)) { + err = __xsk_rcv_zc(xs, xskb, len, contd); + if (err) + goto err; + return 0; + } - err = __xsk_rcv_zc(xs, xskb, len, contd); - if (err) + contd = XDP_PKT_CONTD; + num_desc = xdp_get_shared_info_from_buff(xdp)->nr_frags + 1; + if (xskq_prod_nb_free(xs->rx, num_desc) < num_desc) { + xs->rx_queue_full++; + err = -ENOBUFS; goto err; - if (likely(!frags)) - return 0; + } + __xsk_rcv_zc(xs, xskb, len, contd); xskb_list = &xskb->pool->xskb_list; list_for_each_entry_safe(pos, tmp, xskb_list, list_node) { if (list_is_singular(xskb_list)) contd = 0; len = pos->xdp.data_end - pos->xdp.data; - err = __xsk_rcv_zc(xs, pos, len, contd); - if (err) - goto err; - list_del(&pos->list_node); + __xsk_rcv_zc(xs, pos, len, contd); + list_del_init(&pos->list_node); } return 0; diff --git a/net/xfrm/espintcp.c b/net/xfrm/espintcp.c index bf744ac9d5a73..8709df716e98e 100644 --- a/net/xfrm/espintcp.c +++ b/net/xfrm/espintcp.c @@ -536,7 +536,7 @@ static void espintcp_close(struct sock *sk, long timeout) sk->sk_prot = &tcp_prot; barrier(); - cancel_work_sync(&ctx->work); + disable_work_sync(&ctx->work); strp_done(&ctx->strp); skb_queue_purge(&ctx->out_queue); diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c index 52ae0e034d29e..550457e4c4f01 100644 --- a/net/xfrm/xfrm_device.c +++ b/net/xfrm/xfrm_device.c @@ -544,6 +544,14 @@ static int xfrm_dev_down(struct net_device *dev) return NOTIFY_DONE; } +static int xfrm_dev_unregister(struct net_device *dev) +{ + xfrm_dev_state_flush(dev_net(dev), dev, true); + xfrm_dev_policy_flush(dev_net(dev), dev, true); + + return NOTIFY_DONE; +} + static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr) { struct net_device *dev = netdev_notifier_info_to_dev(ptr); @@ -556,8 +564,10 @@ static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void return xfrm_api_check(dev); case NETDEV_DOWN: - case NETDEV_UNREGISTER: return xfrm_dev_down(dev); + + case NETDEV_UNREGISTER: + return xfrm_dev_unregister(dev); } return NOTIFY_DONE; } diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c index 62486f8669752..5428185196a1f 100644 --- a/net/xfrm/xfrm_policy.c +++ b/net/xfrm/xfrm_policy.c @@ -3801,8 +3801,8 @@ int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb, struct xfrm_tmpl *tp[XFRM_MAX_DEPTH]; struct xfrm_tmpl *stp[XFRM_MAX_DEPTH]; struct xfrm_tmpl **tpp = tp; + int i, k = 0; int ti = 0; - int i, k; sp = skb_sec_path(skb); if (!sp) @@ -3828,6 +3828,12 @@ int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb, tpp = stp; } + if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET && sp == &dummy) + /* This policy template was already checked by HW + * and secpath was removed in __xfrm_policy_check2. + */ + goto out; + /* For each tunnel xfrm, find the first matching tmpl. * For each tmpl before that, find corresponding xfrm. * Order is _important_. Later we will implement @@ -3837,7 +3843,7 @@ int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb, * verified to allow them to be skipped in future policy * checks (e.g. nested tunnels). */ - for (i = xfrm_nr-1, k = 0; i >= 0; i--) { + for (i = xfrm_nr - 1; i >= 0; i--) { k = xfrm_policy_ok(tpp[i], sp, k, family, if_id); if (k < 0) { if (k < -1) @@ -3853,6 +3859,7 @@ int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb, goto reject; } +out: xfrm_pols_put(pols, npols); sp->verified_cnt = k; diff --git a/rust/Makefile b/rust/Makefile index 7842ad0a4ea72..d4618f646b05b 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -437,6 +437,8 @@ $(obj)/$(libpin_init_internal_name): private rustc_target_flags = --cfg kernel $(obj)/$(libpin_init_internal_name): $(src)/pin-init/internal/src/lib.rs FORCE +$(call if_changed_dep,rustc_procmacro) +# `rustc` requires `-Zunstable-options` to use custom target specifications +# since Rust 1.95.0 (https://github.com/rust-lang/rust/pull/151534). quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L $@ cmd_rustc_library = \ OBJTREE=$(abspath $(objtree)) \ @@ -447,6 +449,7 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L --crate-type rlib -L$(objtree)/$(obj) \ --crate-name $(patsubst %.o,%,$(notdir $@)) $< \ --sysroot=/dev/null \ + -Zunstable-options \ $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@) \ $(cmd_objtool) diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs index 1a555fcb120a9..df5d9f6f43f3b 100644 --- a/rust/kernel/cpufreq.rs +++ b/rust/kernel/cpufreq.rs @@ -1015,6 +1015,8 @@ impl Registration { ..pin_init::zeroed() }; + // Always inline to optimize out error path of `build_assert`. + #[inline(always)] const fn copy_name(name: &'static CStr) -> [c_char; CPUFREQ_NAME_LEN] { let src = name.to_bytes_with_nul(); let mut dst = [0; CPUFREQ_NAME_LEN]; diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index a849b7dde2fd9..176531f54ed30 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -62,8 +62,9 @@ pub mod property; /// /// # Implementing Bus Devices /// -/// This section provides a guideline to implement bus specific devices, such as [`pci::Device`] or -/// [`platform::Device`]. +/// This section provides a guideline to implement bus specific devices, such as: +#[cfg_attr(CONFIG_PCI, doc = "* [`pci::Device`](kernel::pci::Device)")] +/// * [`platform::Device`] /// /// A bus specific device should be defined as follows. /// @@ -155,7 +156,6 @@ pub mod property; /// /// [`AlwaysRefCounted`]: kernel::types::AlwaysRefCounted /// [`impl_device_context_deref`]: kernel::impl_device_context_deref -/// [`pci::Device`]: kernel::pci::Device /// [`platform::Device`]: kernel::platform::Device #[repr(transparent)] pub struct Device(Opaque, PhantomData); diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 4e0af3e1a3b9a..777ea5c2431c7 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -26,8 +26,9 @@ pub type DmaAddress = bindings::dma_addr_t; /// Trait to be implemented by DMA capable bus devices. /// /// The [`dma::Device`](Device) trait should be implemented by bus specific device representations, -/// where the underlying bus is DMA capable, such as [`pci::Device`](::kernel::pci::Device) or -/// [`platform::Device`](::kernel::platform::Device). +/// where the underlying bus is DMA capable, such as: +#[cfg_attr(CONFIG_PCI, doc = "* [`pci::Device`](kernel::pci::Device)")] +/// * [`platform::Device`](::kernel::platform::Device) pub trait Device: AsRef> { /// Set up the device's DMA streaming addressing capabilities. /// diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs index 279e3af206822..16931b94d0d48 100644 --- a/rust/kernel/driver.rs +++ b/rust/kernel/driver.rs @@ -33,7 +33,14 @@ //! } //! ``` //! -//! For specific examples see [`auxiliary::Driver`], [`pci::Driver`] and [`platform::Driver`]. +//! For specific examples see: +//! +//! * [`platform::Driver`](kernel::platform::Driver) +#![cfg_attr( + CONFIG_AUXILIARY_BUS, + doc = "* [`auxiliary::Driver`](kernel::auxiliary::Driver)" +)] +#![cfg_attr(CONFIG_PCI, doc = "* [`pci::Driver`](kernel::pci::Driver)")] //! //! The `probe()` callback should return a `Result>>`, i.e. the driver's private //! data. The bus abstraction should store the pointer in the corresponding bus device. The generic @@ -79,7 +86,6 @@ //! //! For this purpose the generic infrastructure in [`device_id`] should be used. //! -//! [`auxiliary::Driver`]: kernel::auxiliary::Driver //! [`Core`]: device::Core //! [`Device`]: device::Device //! [`Device`]: device::Device @@ -87,8 +93,6 @@ //! [`DeviceContext`]: device::DeviceContext //! [`device_id`]: kernel::device_id //! [`module_driver`]: kernel::module_driver -//! [`pci::Driver`]: kernel::pci::Driver -//! [`platform::Driver`]: kernel::platform::Driver use crate::error::{Error, Result}; use crate::{acpi, device, of, str::CStr, try_pin_init, types::Opaque, ThisModule}; diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index f30ee4c6245cd..e09f977b5b519 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -121,7 +121,6 @@ pub trait Driver { pub struct Registration(ARef>); impl Registration { - /// Creates a new [`Registration`] and registers it. fn new(drm: &drm::Device, flags: usize) -> Result { // SAFETY: `drm.as_raw()` is valid by the invariants of `drm::Device`. to_result(unsafe { bindings::drm_dev_register(drm.as_raw(), flags) })?; @@ -129,8 +128,9 @@ impl Registration { Ok(Self(drm.into())) } - /// Same as [`Registration::new`}, but transfers ownership of the [`Registration`] to - /// [`devres::register`]. + /// Registers a new [`Device`](drm::Device) with userspace. + /// + /// Ownership of the [`Registration`] object is passed to [`devres::register`]. pub fn new_foreign_owned( drm: &drm::Device, dev: &device::Device, diff --git a/rust/kernel/irq/request.rs b/rust/kernel/irq/request.rs index b150563fdef80..2ceeaeb0543a4 100644 --- a/rust/kernel/irq/request.rs +++ b/rust/kernel/irq/request.rs @@ -261,7 +261,10 @@ impl Registration { /// # Safety /// /// This function should be only used as the callback in `request_irq`. -unsafe extern "C" fn handle_irq_callback(_irq: i32, ptr: *mut c_void) -> c_uint { +unsafe extern "C" fn handle_irq_callback( + _irq: i32, + ptr: *mut c_void, +) -> c_uint { // SAFETY: `ptr` is a pointer to `Registration` set in `Registration::new` let registration = unsafe { &*(ptr as *const Registration) }; // SAFETY: The irq callback is removed before the device is unbound, so the fact that the irq @@ -480,7 +483,7 @@ impl ThreadedRegistration { /// # Safety /// /// This function should be only used as the callback in `request_threaded_irq`. -unsafe extern "C" fn handle_threaded_irq_callback( +unsafe extern "C" fn handle_threaded_irq_callback( _irq: i32, ptr: *mut c_void, ) -> c_uint { @@ -496,7 +499,10 @@ unsafe extern "C" fn handle_threaded_irq_callback( /// # Safety /// /// This function should be only used as the callback in `request_threaded_irq`. -unsafe extern "C" fn thread_fn_callback(_irq: i32, ptr: *mut c_void) -> c_uint { +unsafe extern "C" fn thread_fn_callback( + _irq: i32, + ptr: *mut c_void, +) -> c_uint { // SAFETY: `ptr` is a pointer to `ThreadedRegistration` set in `ThreadedRegistration::new` let registration = unsafe { &*(ptr as *const ThreadedRegistration) }; // SAFETY: The irq callback is removed before the device is unbound, so the fact that the irq diff --git a/rust/kernel/kunit.rs b/rust/kernel/kunit.rs index 79436509dd73d..8907b6f89ece5 100644 --- a/rust/kernel/kunit.rs +++ b/rust/kernel/kunit.rs @@ -17,6 +17,10 @@ use crate::c_str; /// Public but hidden since it should only be used from KUnit generated code. #[doc(hidden)] pub fn err(args: fmt::Arguments<'_>) { + // `args` is unused if `CONFIG_PRINTK` is not set - this avoids a build-time warning. + #[cfg(not(CONFIG_PRINTK))] + let _ = args; + // SAFETY: The format string is null-terminated and the `%pA` specifier matches the argument we // are passing. #[cfg(CONFIG_PRINTK)] @@ -33,6 +37,10 @@ pub fn err(args: fmt::Arguments<'_>) { /// Public but hidden since it should only be used from KUnit generated code. #[doc(hidden)] pub fn info(args: fmt::Arguments<'_>) { + // `args` is unused if `CONFIG_PRINTK` is not set - this avoids a build-time warning. + #[cfg(not(CONFIG_PRINTK))] + let _ = args; + // SAFETY: The format string is null-terminated and the `%pA` specifier matches the argument we // are passing. #[cfg(CONFIG_PRINTK)] diff --git a/rust/kernel/list/impl_list_item_mod.rs b/rust/kernel/list/impl_list_item_mod.rs index 202bc6f97c132..ee53d0387e63d 100644 --- a/rust/kernel/list/impl_list_item_mod.rs +++ b/rust/kernel/list/impl_list_item_mod.rs @@ -84,11 +84,12 @@ macro_rules! impl_has_list_links_self_ptr { // right type. unsafe impl$(<$($generics)*>)? $crate::list::HasSelfPtr<$item_type $(, $id)?> for $self {} + // SAFETY: TODO. unsafe impl$(<$($generics)*>)? $crate::list::HasListLinks$(<$id>)? for $self { #[inline] unsafe fn raw_get_list_links(ptr: *mut Self) -> *mut $crate::list::ListLinks$(<$id>)? { - // SAFETY: The caller promises that the pointer is not dangling. let ptr: *mut $crate::list::ListLinksSelfPtr<$item_type $(, $id)?> = + // SAFETY: The caller promises that the pointer is not dangling. unsafe { ::core::ptr::addr_of_mut!((*ptr)$(.$field)*) }; ptr.cast() } @@ -217,7 +218,7 @@ macro_rules! impl_list_item { // SAFETY: `me` originates from the most recent call to `prepare_to_insert`, so it // points at the field `$field` in a value of type `Self`. Thus, reversing that // operation is still in-bounds of the allocation. - $crate::container_of!(me, Self, $($field).*) + unsafe { $crate::container_of!(me, Self, $($field).*) } } // GUARANTEES: @@ -242,7 +243,7 @@ macro_rules! impl_list_item { // SAFETY: `me` originates from the most recent call to `prepare_to_insert`, so it // points at the field `$field` in a value of type `Self`. Thus, reversing that // operation is still in-bounds of the allocation. - $crate::container_of!(me, Self, $($field).*) + unsafe { $crate::container_of!(me, Self, $($field).*) } } } )*}; @@ -270,9 +271,12 @@ macro_rules! impl_list_item { // SAFETY: The caller promises that `me` points at a valid value of type `Self`. let links_field = unsafe { >::view_links(me) }; - let container = $crate::container_of!( - links_field, $crate::list::ListLinksSelfPtr, inner - ); + // SAFETY: TODO. + let container = unsafe { + $crate::container_of!( + links_field, $crate::list::ListLinksSelfPtr, inner + ) + }; // SAFETY: By the same reasoning above, `links_field` is a valid pointer. let self_ptr = unsafe { @@ -319,9 +323,12 @@ macro_rules! impl_list_item { // `ListArc` containing `Self` until the next call to `post_remove`. The value cannot // be destroyed while a `ListArc` reference exists. unsafe fn view_value(links_field: *mut $crate::list::ListLinks<$num>) -> *const Self { - let container = $crate::container_of!( - links_field, $crate::list::ListLinksSelfPtr, inner - ); + // SAFETY: TODO. + let container = unsafe { + $crate::container_of!( + links_field, $crate::list::ListLinksSelfPtr, inner + ) + }; // SAFETY: By the same reasoning above, `links_field` is a valid pointer. let self_ptr = unsafe { diff --git a/rust/kernel/task.rs b/rust/kernel/task.rs index 49fad6de06740..cc907fb531bce 100644 --- a/rust/kernel/task.rs +++ b/rust/kernel/task.rs @@ -204,18 +204,6 @@ impl Task { self.0.get() } - /// Returns the group leader of the given task. - pub fn group_leader(&self) -> &Task { - // SAFETY: The group leader of a task never changes after initialization, so reading this - // field is not a data race. - let ptr = unsafe { *ptr::addr_of!((*self.as_ptr()).group_leader) }; - - // SAFETY: The lifetime of the returned task reference is tied to the lifetime of `self`, - // and given that a task has a reference to its group leader, we know it must be valid for - // the lifetime of the returned task reference. - unsafe { &*ptr.cast() } - } - /// Returns the PID of the given task. pub fn pid(&self) -> Pid { // SAFETY: The pid of a task never changes after initialization, so reading this field is @@ -345,6 +333,18 @@ impl CurrentTask { // `release_task()` call. Some(unsafe { PidNamespace::from_ptr(active_ns) }) } + + /// Returns the group leader of the current task. + pub fn group_leader(&self) -> &Task { + // SAFETY: The group leader of a task never changes while the task is running, and `self` + // is the current task, which is guaranteed running. + let ptr = unsafe { (*self.as_ptr()).group_leader }; + + // SAFETY: `current->group_leader` stays valid for at least the duration in which `current` + // is running, and the signature of this function ensures that the returned `&Task` can + // only be used while `current` is still valid, thus still running. + unsafe { &*ptr.cast() } + } } // SAFETY: The type invariants guarantee that `Task` is always refcounted. diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs index dd553212836e0..0128808579973 100644 --- a/rust/pin-init/src/lib.rs +++ b/rust/pin-init/src/lib.rs @@ -1276,13 +1276,13 @@ pub const unsafe fn init_from_closure( /// /// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a /// pointer must result in a valid `U`. -#[expect(clippy::let_and_return)] pub const unsafe fn cast_pin_init(init: impl PinInit) -> impl PinInit { // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety // requirements. let res = unsafe { pin_init_from_closure(|ptr: *mut U| init.__pinned_init(ptr.cast::())) }; // FIXME: remove the let statement once the nightly-MSRV allows it (1.78 otherwise encounters a // cycle when computing the type returned by this function) + #[allow(clippy::let_and_return)] res } @@ -1292,13 +1292,13 @@ pub const unsafe fn cast_pin_init(init: impl PinInit) -> impl Pin /// /// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a /// pointer must result in a valid `U`. -#[expect(clippy::let_and_return)] pub const unsafe fn cast_init(init: impl Init) -> impl Init { // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety // requirements. let res = unsafe { init_from_closure(|ptr: *mut U| init.__init(ptr.cast::())) }; // FIXME: remove the let statement once the nightly-MSRV allows it (1.78 otherwise encounters a // cycle when computing the type returned by this function) + #[allow(clippy::let_and_return)] res } diff --git a/scripts/cc-can-link.sh b/scripts/cc-can-link.sh index e67fd8d7b6841..58dc7dd6d5568 100755 --- a/scripts/cc-can-link.sh +++ b/scripts/cc-can-link.sh @@ -5,7 +5,7 @@ cat << "END" | $@ -Werror -Wl,--fatal-warnings -x c - -o /dev/null >/dev/null 2> #include int main(void) { - printf(""); + printf("\n"); return 0; } END diff --git a/scripts/gendwarfksyms/dwarf.c b/scripts/gendwarfksyms/dwarf.c index 3538a7d9cb070..e76d732f5f602 100644 --- a/scripts/gendwarfksyms/dwarf.c +++ b/scripts/gendwarfksyms/dwarf.c @@ -750,6 +750,7 @@ static void process_enumerator_type(struct state *state, struct die *cache, Dwarf_Die *die) { bool overridden = false; + unsigned long override; Dwarf_Word value; if (stable) { @@ -761,7 +762,8 @@ static void process_enumerator_type(struct state *state, struct die *cache, return; overridden = kabi_get_enumerator_value( - state->expand.current_fqn, cache->fqn, &value); + state->expand.current_fqn, cache->fqn, &override); + value = override; } process_list_comma(state, cache); diff --git a/scripts/gendwarfksyms/symbols.c b/scripts/gendwarfksyms/symbols.c index ecddcb5ffcdfb..42cd27c9cec4f 100644 --- a/scripts/gendwarfksyms/symbols.c +++ b/scripts/gendwarfksyms/symbols.c @@ -3,6 +3,7 @@ * Copyright (C) 2024 Google LLC */ +#include #include "gendwarfksyms.h" #define SYMBOL_HASH_BITS 12 @@ -242,7 +243,7 @@ static void elf_for_each_global(int fd, elf_symbol_callback_t func, void *arg) error("elf_getdata failed: %s", elf_errmsg(-1)); if (shdr->sh_entsize != sym_size) - error("expected sh_entsize (%lu) to be %zu", + error("expected sh_entsize (%" PRIu64 ") to be %zu", shdr->sh_entsize, sym_size); nsyms = shdr->sh_size / shdr->sh_entsize; @@ -292,7 +293,7 @@ static void set_symbol_addr(struct symbol *sym, void *arg) hash_add(symbol_addrs, &sym->addr_hash, symbol_addr_hash(&sym->addr)); - debug("%s -> { %u, %lx }", sym->name, sym->addr.section, + debug("%s -> { %u, %" PRIx64 " }", sym->name, sym->addr.section, sym->addr.address); } else if (sym->addr.section != addr->section || sym->addr.address != addr->address) { diff --git a/scripts/kernel-doc.py b/scripts/kernel-doc.py index d9fe2bcbd39cc..a6bb180b13d65 100755 --- a/scripts/kernel-doc.py +++ b/scripts/kernel-doc.py @@ -116,6 +116,8 @@ sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) +WERROR_RETURN_CODE = 3 + DESC = """ Read C language source or header FILEs, extract embedded documentation comments, and print formatted documentation to standard output. @@ -176,7 +178,21 @@ def format(self, record): return logging.Formatter.format(self, record) def main(): - """Main program""" + """ + Main program. + + By default, the return value is: + + - 0: success or Python version is not compatible with + kernel-doc. If -Werror is not used, it will also + return 0 if there are issues at kernel-doc markups; + + - 1: an abnormal condition happened; + + - 2: argparse issued an error; + + - 3: -Werror is used, and one or more unfiltered parse warnings happened. + """ parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description=DESC) @@ -323,16 +339,12 @@ def main(): if args.werror: print("%s warnings as errors" % error_count) # pylint: disable=C0209 - sys.exit(error_count) + sys.exit(WERROR_RETURN_CODE) if args.verbose: print("%s errors" % error_count) # pylint: disable=C0209 - if args.none: - sys.exit(0) - - sys.exit(error_count) - + sys.exit(0) # Call main method if __name__ == "__main__": diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index 47c8aa2a69392..133dfa16308a3 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -602,6 +602,10 @@ static int ignore_undef_symbol(struct elf_info *info, const char *symname) /* Special register function linked on all modules during final link of .ko */ if (strstarts(symname, "_restgpr0_") || strstarts(symname, "_savegpr0_") || + strstarts(symname, "_restgpr1_") || + strstarts(symname, "_savegpr1_") || + strstarts(symname, "_restfpr_") || + strstarts(symname, "_savefpr_") || strstarts(symname, "_restvr_") || strstarts(symname, "_savevr_") || strcmp(symname, ".TOC.") == 0) diff --git a/scripts/package/kernel.spec b/scripts/package/kernel.spec index 0f1c8de1bd95f..b3c956205af00 100644 --- a/scripts/package/kernel.spec +++ b/scripts/package/kernel.spec @@ -2,8 +2,6 @@ %{!?_arch: %define _arch dummy} %{!?make: %define make make} %define makeflags %{?_smp_mflags} ARCH=%{ARCH} -%define __spec_install_post /usr/lib/rpm/brp-compress || : -%define debug_package %{nil} Name: kernel Summary: The Linux Kernel @@ -47,13 +45,49 @@ This package provides kernel headers and makefiles sufficient to build modules against the %{version} kernel package. %endif -%if %{with_debuginfo} +%if %{with_debuginfo_manual} %package debuginfo Summary: Debug information package for the Linux kernel +Group: Development/Debug +AutoReq: 0 +AutoProv: 1 %description debuginfo This package provides debug information for the kernel image and modules from the %{version} package. +%define install_mod_strip 1 +%endif + +%if %{with_debuginfo_rpm} +# list of debuginfo-related options taken from distribution kernel.spec +# files +%undefine _include_minidebuginfo +%undefine _find_debuginfo_dwz_opts +%undefine _unique_build_ids +%undefine _unique_debug_names +%undefine _unique_debug_srcs +%undefine _debugsource_packages +%undefine _debuginfo_subpackages +%global _find_debuginfo_opts -r +%global _missing_build_ids_terminate_build 1 +%global _no_recompute_build_ids 1 +%{debug_package} + +# later, we make all modules executable so that find-debuginfo.sh strips +# them up. but they don't actually need to be executable, so remove the +# executable bit, taking care to do it _after_ find-debuginfo.sh has run +%define __spec_install_post \ + %{?__debug_package:%{__debug_install_post}} \ + %{__arch_install_post} \ + %{__os_install_post} \ + find %{buildroot}/lib/modules/%{KERNELRELEASE} -name "*.ko" -type f \\\ + | xargs --no-run-if-empty chmod u-x +%else +%define __spec_install_post /usr/lib/rpm/brp-compress || : %endif +# some (but not all) versions of rpmbuild emit %%debug_package with +# %%install. since we've already emitted it manually, that would cause +# a package redefinition error. ensure that doesn't happen +%define debug_package %{nil} %prep %setup -q -n linux @@ -67,7 +101,7 @@ patch -p1 < %{SOURCE2} mkdir -p %{buildroot}/lib/modules/%{KERNELRELEASE} cp $(%{make} %{makeflags} -s image_name) %{buildroot}/lib/modules/%{KERNELRELEASE}/vmlinuz # DEPMOD=true makes depmod no-op. We do not package depmod-generated files. -%{make} %{makeflags} INSTALL_MOD_PATH=%{buildroot} INSTALL_MOD_STRIP=1 DEPMOD=true modules_install +%{make} %{makeflags} INSTALL_MOD_PATH=%{buildroot} %{?install_mod_strip:INSTALL_MOD_STRIP=1} DEPMOD=true modules_install %{make} %{makeflags} INSTALL_HDR_PATH=%{buildroot}/usr headers_install cp System.map %{buildroot}/lib/modules/%{KERNELRELEASE} cp .config %{buildroot}/lib/modules/%{KERNELRELEASE}/config @@ -98,22 +132,30 @@ ln -fns /usr/src/kernels/%{KERNELRELEASE} %{buildroot}/lib/modules/%{KERNELRELEA echo "%exclude /lib/modules/%{KERNELRELEASE}/build" } > %{buildroot}/kernel.list -%if %{with_debuginfo} +%if 0%{with_debuginfo_manual}%{with_debuginfo_rpm} > 0 # copying vmlinux directly to the debug directory means it will not get # stripped (but its source paths will still be collected + fixed up) mkdir -p %{buildroot}/usr/lib/debug/lib/modules/%{KERNELRELEASE} cp vmlinux %{buildroot}/usr/lib/debug/lib/modules/%{KERNELRELEASE} +%endif -echo /usr/lib/debug/lib/modules/%{KERNELRELEASE}/vmlinux > %{buildroot}/debuginfo.list +%if %{with_debuginfo_rpm} +# make modules executable so that find-debuginfo.sh strips them. this +# will be undone later in %%__spec_install_post +find %{buildroot}/lib/modules/%{KERNELRELEASE} -name "*.ko" -type f \ + | xargs --no-run-if-empty chmod u+x +%endif +%if %{with_debuginfo_manual} +echo /usr/lib/debug/lib/modules/%{KERNELRELEASE}/vmlinux > %{buildroot}/debuginfo.list while read -r mod; do mod="${mod%.o}.ko" dbg="%{buildroot}/usr/lib/debug/lib/modules/%{KERNELRELEASE}/kernel/${mod}" - buildid=$("${READELF}" -n "${mod}" | sed -n 's@^.*Build ID: \(..\)\(.*\)@\1/\2@p') + buildid=$("${READELF:-readelf}" -n "${mod}" | sed -n 's@^.*Build ID: \(..\)\(.*\)@\1/\2@p') link="%{buildroot}/usr/lib/debug/.build-id/${buildid}.debug" mkdir -p "${dbg%/*}" "${link%/*}" - "${OBJCOPY}" --only-keep-debug "${mod}" "${dbg}" + "${OBJCOPY:-objcopy}" --only-keep-debug "${mod}" "${dbg}" ln -sf --relative "${dbg}" "${link}" echo "${dbg#%{buildroot}}" >> %{buildroot}/debuginfo.list @@ -123,6 +165,10 @@ done < modules.order %clean rm -rf %{buildroot} +%if %{with_debuginfo_rpm} +rm -f debugfiles.list debuglinks.list debugsourcefiles.list debugsources.list \ + elfbins.list +%endif %post if [ -x /usr/bin/kernel-install ]; then @@ -162,7 +208,7 @@ fi /lib/modules/%{KERNELRELEASE}/build %endif -%if %{with_debuginfo} +%if %{with_debuginfo_manual} %files -f %{buildroot}/debuginfo.list debuginfo %defattr (-, root, root) %exclude /debuginfo.list diff --git a/scripts/package/mkspec b/scripts/package/mkspec index c7375bfc25a9a..c604f8c174e2c 100755 --- a/scripts/package/mkspec +++ b/scripts/package/mkspec @@ -23,15 +23,47 @@ else echo '%define with_devel 0' fi +# use %{debug_package} machinery to generate -debuginfo +with_debuginfo_rpm=0 +# manually generate -debuginfo package +with_debuginfo_manual=0 # debuginfo package generation uses find-debuginfo.sh under the hood, # which only works on uncompressed modules that contain debuginfo if grep -q CONFIG_DEBUG_INFO=y include/config/auto.conf && (! grep -q CONFIG_MODULE_COMPRESS=y include/config/auto.conf) && (! grep -q CONFIG_DEBUG_INFO_SPLIT=y include/config/auto.conf); then -echo '%define with_debuginfo %{?_without_debuginfo: 0} %{?!_without_debuginfo: 1}' -else -echo '%define with_debuginfo 0' + # If module signing is enabled (which may be required to boot with + # lockdown enabled), the find-debuginfo.sh machinery cannot be used + # because the signatures will be stripped off the modules. However, due + # to an rpm bug in versions prior to 4.20.0 + # + # https://github.com/rpm-software-management/rpm/issues/3057 + # https://github.com/rpm-software-management/rpm/commit/49f906998f3cf1f4152162ca61ac0869251c380f + # + # We cannot provide our own debuginfo package because it does not listen + # to our custom files list, failing the build due to unpackaged files. + # Manually generate the debug info package if using rpm 4.20.0. If not + # using rpm 4.20.0, avoid generating a -debuginfo package altogether, + # as it is not safe. + if grep -q CONFIG_MODULE_SIG=y include/config/auto.conf; then + rpm_ver_str=$(rpm --version 2>/dev/null) + # Split the version on spaces + IFS=' ' + set -- $rpm_ver_str + if [ "${1:-}" = RPM -a "${2:-}" = version ]; then + IFS=. + set -- $3 + rpm_ver=$(( 1000000 * $1 + 10000 * $2 + 100 * $3 + ${4:-0} )) + if [ "$rpm_ver" -ge 4200000 ]; then + with_debuginfo_manual='%{?_without_debuginfo:0}%{?!_without_debuginfo:1}' + fi + fi + else + with_debuginfo_rpm='%{?_without_debuginfo:0}%{?!_without_debuginfo:1}' + fi fi +echo "%define with_debuginfo_manual $with_debuginfo_manual" +echo "%define with_debuginfo_rpm $with_debuginfo_rpm" cat<loaddata); + aa_put_i_loaddata(private->loaddata); kvfree(private); } @@ -153,6 +155,71 @@ static int aafs_show_path(struct seq_file *seq, struct dentry *dentry) return 0; } +static struct aa_ns *get_ns_common_ref(struct aa_common_ref *ref) +{ + if (ref) { + struct aa_label *reflabel = container_of(ref, struct aa_label, + count); + return aa_get_ns(labels_ns(reflabel)); + } + + return NULL; +} + +static struct aa_proxy *get_proxy_common_ref(struct aa_common_ref *ref) +{ + if (ref) + return aa_get_proxy(container_of(ref, struct aa_proxy, count)); + + return NULL; +} + +static struct aa_loaddata *get_loaddata_common_ref(struct aa_common_ref *ref) +{ + if (ref) + return aa_get_i_loaddata(container_of(ref, struct aa_loaddata, + count)); + return NULL; +} + +static void aa_put_common_ref(struct aa_common_ref *ref) +{ + if (!ref) + return; + + switch (ref->reftype) { + case REF_RAWDATA: + aa_put_i_loaddata(container_of(ref, struct aa_loaddata, + count)); + break; + case REF_PROXY: + aa_put_proxy(container_of(ref, struct aa_proxy, + count)); + break; + case REF_NS: + /* ns count is held on its unconfined label */ + aa_put_ns(labels_ns(container_of(ref, struct aa_label, count))); + break; + default: + AA_BUG(true, "unknown refcount type"); + break; + } +} + +static void aa_get_common_ref(struct aa_common_ref *ref) +{ + kref_get(&ref->count); +} + +static void aafs_evict(struct inode *inode) +{ + struct aa_common_ref *ref = inode->i_private; + + clear_inode(inode); + aa_put_common_ref(ref); + inode->i_private = (void *) IREF_POISON; +} + static void aafs_free_inode(struct inode *inode) { if (S_ISLNK(inode->i_mode)) @@ -162,6 +229,7 @@ static void aafs_free_inode(struct inode *inode) static const struct super_operations aafs_super_ops = { .statfs = simple_statfs, + .evict_inode = aafs_evict, .free_inode = aafs_free_inode, .show_path = aafs_show_path, }; @@ -262,7 +330,8 @@ static int __aafs_setup_d_inode(struct inode *dir, struct dentry *dentry, * aafs_remove(). Will return ERR_PTR on failure. */ static struct dentry *aafs_create(const char *name, umode_t mode, - struct dentry *parent, void *data, void *link, + struct dentry *parent, + struct aa_common_ref *data, void *link, const struct file_operations *fops, const struct inode_operations *iops) { @@ -299,6 +368,9 @@ static struct dentry *aafs_create(const char *name, umode_t mode, goto fail_dentry; inode_unlock(dir); + if (data) + aa_get_common_ref(data); + return dentry; fail_dentry: @@ -323,7 +395,8 @@ static struct dentry *aafs_create(const char *name, umode_t mode, * see aafs_create */ static struct dentry *aafs_create_file(const char *name, umode_t mode, - struct dentry *parent, void *data, + struct dentry *parent, + struct aa_common_ref *data, const struct file_operations *fops) { return aafs_create(name, mode, parent, data, NULL, fops, NULL); @@ -404,7 +477,8 @@ static struct aa_loaddata *aa_simple_write_to_buffer(const char __user *userbuf, data->size = copy_size; if (copy_from_user(data->data, userbuf, copy_size)) { - aa_put_loaddata(data); + /* trigger free - don't need to put pcount */ + aa_put_i_loaddata(data); return ERR_PTR(-EFAULT); } @@ -412,7 +486,8 @@ static struct aa_loaddata *aa_simple_write_to_buffer(const char __user *userbuf, } static ssize_t policy_update(u32 mask, const char __user *buf, size_t size, - loff_t *pos, struct aa_ns *ns) + loff_t *pos, struct aa_ns *ns, + const struct cred *ocred) { struct aa_loaddata *data; struct aa_label *label; @@ -423,7 +498,7 @@ static ssize_t policy_update(u32 mask, const char __user *buf, size_t size, /* high level check about policy management - fine grained in * below after unpack */ - error = aa_may_manage_policy(current_cred(), label, ns, mask); + error = aa_may_manage_policy(current_cred(), label, ns, ocred, mask); if (error) goto end_section; @@ -431,7 +506,10 @@ static ssize_t policy_update(u32 mask, const char __user *buf, size_t size, error = PTR_ERR(data); if (!IS_ERR(data)) { error = aa_replace_profiles(ns, label, mask, data); - aa_put_loaddata(data); + /* put pcount, which will put count and free if no + * profiles referencing it. + */ + aa_put_profile_loaddata(data); } end_section: end_current_label_crit_section(label); @@ -443,8 +521,9 @@ static ssize_t policy_update(u32 mask, const char __user *buf, size_t size, static ssize_t profile_load(struct file *f, const char __user *buf, size_t size, loff_t *pos) { - struct aa_ns *ns = aa_get_ns(f->f_inode->i_private); - int error = policy_update(AA_MAY_LOAD_POLICY, buf, size, pos, ns); + struct aa_ns *ns = get_ns_common_ref(f->f_inode->i_private); + int error = policy_update(AA_MAY_LOAD_POLICY, buf, size, pos, ns, + f->f_cred); aa_put_ns(ns); @@ -460,9 +539,9 @@ static const struct file_operations aa_fs_profile_load = { static ssize_t profile_replace(struct file *f, const char __user *buf, size_t size, loff_t *pos) { - struct aa_ns *ns = aa_get_ns(f->f_inode->i_private); + struct aa_ns *ns = get_ns_common_ref(f->f_inode->i_private); int error = policy_update(AA_MAY_LOAD_POLICY | AA_MAY_REPLACE_POLICY, - buf, size, pos, ns); + buf, size, pos, ns, f->f_cred); aa_put_ns(ns); return error; @@ -480,14 +559,14 @@ static ssize_t profile_remove(struct file *f, const char __user *buf, struct aa_loaddata *data; struct aa_label *label; ssize_t error; - struct aa_ns *ns = aa_get_ns(f->f_inode->i_private); + struct aa_ns *ns = get_ns_common_ref(f->f_inode->i_private); label = begin_current_label_crit_section(); /* high level check about policy management - fine grained in * below after unpack */ error = aa_may_manage_policy(current_cred(), label, ns, - AA_MAY_REMOVE_POLICY); + f->f_cred, AA_MAY_REMOVE_POLICY); if (error) goto out; @@ -501,7 +580,7 @@ static ssize_t profile_remove(struct file *f, const char __user *buf, if (!IS_ERR(data)) { data->data[size] = 0; error = aa_remove_profiles(ns, label, data->data, size); - aa_put_loaddata(data); + aa_put_profile_loaddata(data); } out: end_current_label_crit_section(label); @@ -570,7 +649,7 @@ static int ns_revision_open(struct inode *inode, struct file *file) if (!rev) return -ENOMEM; - rev->ns = aa_get_ns(inode->i_private); + rev->ns = get_ns_common_ref(inode->i_private); if (!rev->ns) rev->ns = aa_get_current_ns(); file->private_data = rev; @@ -1056,7 +1135,7 @@ static const struct file_operations seq_profile_ ##NAME ##_fops = { \ static int seq_profile_open(struct inode *inode, struct file *file, int (*show)(struct seq_file *, void *)) { - struct aa_proxy *proxy = aa_get_proxy(inode->i_private); + struct aa_proxy *proxy = get_proxy_common_ref(inode->i_private); int error = single_open(file, show, proxy); if (error) { @@ -1248,18 +1327,17 @@ static const struct file_operations seq_rawdata_ ##NAME ##_fops = { \ static int seq_rawdata_open(struct inode *inode, struct file *file, int (*show)(struct seq_file *, void *)) { - struct aa_loaddata *data = __aa_get_loaddata(inode->i_private); + struct aa_loaddata *data = get_loaddata_common_ref(inode->i_private); int error; if (!data) - /* lost race this ent is being reaped */ return -ENOENT; error = single_open(file, show, data); if (error) { AA_BUG(file->private_data && ((struct seq_file *)file->private_data)->private); - aa_put_loaddata(data); + aa_put_i_loaddata(data); } return error; @@ -1270,7 +1348,7 @@ static int seq_rawdata_release(struct inode *inode, struct file *file) struct seq_file *seq = (struct seq_file *) file->private_data; if (seq) - aa_put_loaddata(seq->private); + aa_put_i_loaddata(seq->private); return single_release(inode, file); } @@ -1382,9 +1460,8 @@ static int rawdata_open(struct inode *inode, struct file *file) if (!aa_current_policy_view_capable(NULL)) return -EACCES; - loaddata = __aa_get_loaddata(inode->i_private); + loaddata = get_loaddata_common_ref(inode->i_private); if (!loaddata) - /* lost race: this entry is being reaped */ return -ENOENT; private = rawdata_f_data_alloc(loaddata->size); @@ -1409,7 +1486,7 @@ static int rawdata_open(struct inode *inode, struct file *file) return error; fail_private_alloc: - aa_put_loaddata(loaddata); + aa_put_i_loaddata(loaddata); return error; } @@ -1426,7 +1503,6 @@ static void remove_rawdata_dents(struct aa_loaddata *rawdata) for (i = 0; i < AAFS_LOADDATA_NDENTS; i++) { if (!IS_ERR_OR_NULL(rawdata->dents[i])) { - /* no refcounts on i_private */ aafs_remove(rawdata->dents[i]); rawdata->dents[i] = NULL; } @@ -1469,35 +1545,37 @@ int __aa_fs_create_rawdata(struct aa_ns *ns, struct aa_loaddata *rawdata) return PTR_ERR(dir); rawdata->dents[AAFS_LOADDATA_DIR] = dir; - dent = aafs_create_file("abi", S_IFREG | 0444, dir, rawdata, + dent = aafs_create_file("abi", S_IFREG | 0444, dir, &rawdata->count, &seq_rawdata_abi_fops); if (IS_ERR(dent)) goto fail; rawdata->dents[AAFS_LOADDATA_ABI] = dent; - dent = aafs_create_file("revision", S_IFREG | 0444, dir, rawdata, - &seq_rawdata_revision_fops); + dent = aafs_create_file("revision", S_IFREG | 0444, dir, + &rawdata->count, + &seq_rawdata_revision_fops); if (IS_ERR(dent)) goto fail; rawdata->dents[AAFS_LOADDATA_REVISION] = dent; if (aa_g_hash_policy) { dent = aafs_create_file("sha256", S_IFREG | 0444, dir, - rawdata, &seq_rawdata_hash_fops); + &rawdata->count, + &seq_rawdata_hash_fops); if (IS_ERR(dent)) goto fail; rawdata->dents[AAFS_LOADDATA_HASH] = dent; } dent = aafs_create_file("compressed_size", S_IFREG | 0444, dir, - rawdata, + &rawdata->count, &seq_rawdata_compressed_size_fops); if (IS_ERR(dent)) goto fail; rawdata->dents[AAFS_LOADDATA_COMPRESSED_SIZE] = dent; - dent = aafs_create_file("raw_data", S_IFREG | 0444, - dir, rawdata, &rawdata_fops); + dent = aafs_create_file("raw_data", S_IFREG | 0444, dir, + &rawdata->count, &rawdata_fops); if (IS_ERR(dent)) goto fail; rawdata->dents[AAFS_LOADDATA_DATA] = dent; @@ -1505,13 +1583,11 @@ int __aa_fs_create_rawdata(struct aa_ns *ns, struct aa_loaddata *rawdata) rawdata->ns = aa_get_ns(ns); list_add(&rawdata->list, &ns->rawdata_list); - /* no refcount on inode rawdata */ return 0; fail: remove_rawdata_dents(rawdata); - return PTR_ERR(dent); } #endif /* CONFIG_SECURITY_APPARMOR_EXPORT_BINARY */ @@ -1535,13 +1611,10 @@ void __aafs_profile_rmdir(struct aa_profile *profile) __aafs_profile_rmdir(child); for (i = AAFS_PROF_SIZEOF - 1; i >= 0; --i) { - struct aa_proxy *proxy; if (!profile->dents[i]) continue; - proxy = d_inode(profile->dents[i])->i_private; aafs_remove(profile->dents[i]); - aa_put_proxy(proxy); profile->dents[i] = NULL; } } @@ -1575,14 +1648,7 @@ static struct dentry *create_profile_file(struct dentry *dir, const char *name, struct aa_profile *profile, const struct file_operations *fops) { - struct aa_proxy *proxy = aa_get_proxy(profile->label.proxy); - struct dentry *dent; - - dent = aafs_create_file(name, S_IFREG | 0444, dir, proxy, fops); - if (IS_ERR(dent)) - aa_put_proxy(proxy); - - return dent; + return aafs_create_file(name, S_IFREG | 0444, dir, &profile->label.proxy->count, fops); } #ifdef CONFIG_SECURITY_APPARMOR_EXPORT_BINARY @@ -1628,7 +1694,8 @@ static const char *rawdata_get_link_base(struct dentry *dentry, struct delayed_call *done, const char *name) { - struct aa_proxy *proxy = inode->i_private; + struct aa_common_ref *ref = inode->i_private; + struct aa_proxy *proxy = container_of(ref, struct aa_proxy, count); struct aa_label *label; struct aa_profile *profile; char *target; @@ -1639,6 +1706,15 @@ static const char *rawdata_get_link_base(struct dentry *dentry, label = aa_get_label_rcu(&proxy->label); profile = labels_profile(label); + + /* rawdata can be null when aa_g_export_binary is unset during + * runtime and a profile is replaced + */ + if (!profile->rawdata) { + aa_put_label(label); + return ERR_PTR(-ENOENT); + } + depth = profile_depth(profile); target = gen_symlink_name(depth, profile->rawdata->name, name); aa_put_label(label); @@ -1761,27 +1837,24 @@ int __aafs_profile_mkdir(struct aa_profile *profile, struct dentry *parent) if (profile->rawdata) { if (aa_g_hash_policy) { dent = aafs_create("raw_sha256", S_IFLNK | 0444, dir, - profile->label.proxy, NULL, NULL, - &rawdata_link_sha256_iops); + &profile->label.proxy->count, NULL, + NULL, &rawdata_link_sha256_iops); if (IS_ERR(dent)) goto fail; - aa_get_proxy(profile->label.proxy); profile->dents[AAFS_PROF_RAW_HASH] = dent; } dent = aafs_create("raw_abi", S_IFLNK | 0444, dir, - profile->label.proxy, NULL, NULL, + &profile->label.proxy->count, NULL, NULL, &rawdata_link_abi_iops); if (IS_ERR(dent)) goto fail; - aa_get_proxy(profile->label.proxy); profile->dents[AAFS_PROF_RAW_ABI] = dent; dent = aafs_create("raw_data", S_IFLNK | 0444, dir, - profile->label.proxy, NULL, NULL, + &profile->label.proxy->count, NULL, NULL, &rawdata_link_data_iops); if (IS_ERR(dent)) goto fail; - aa_get_proxy(profile->label.proxy); profile->dents[AAFS_PROF_RAW_DATA] = dent; } #endif /*CONFIG_SECURITY_APPARMOR_EXPORT_BINARY */ @@ -1812,13 +1885,13 @@ static struct dentry *ns_mkdir_op(struct mnt_idmap *idmap, struct inode *dir, int error; label = begin_current_label_crit_section(); - error = aa_may_manage_policy(current_cred(), label, NULL, + error = aa_may_manage_policy(current_cred(), label, NULL, NULL, AA_MAY_LOAD_POLICY); end_current_label_crit_section(label); if (error) return ERR_PTR(error); - parent = aa_get_ns(dir->i_private); + parent = get_ns_common_ref(dir->i_private); AA_BUG(d_inode(ns_subns_dir(parent)) != dir); /* we have to unlock and then relock to get locking order right @@ -1862,13 +1935,13 @@ static int ns_rmdir_op(struct inode *dir, struct dentry *dentry) int error; label = begin_current_label_crit_section(); - error = aa_may_manage_policy(current_cred(), label, NULL, + error = aa_may_manage_policy(current_cred(), label, NULL, NULL, AA_MAY_LOAD_POLICY); end_current_label_crit_section(label); if (error) return error; - parent = aa_get_ns(dir->i_private); + parent = get_ns_common_ref(dir->i_private); /* rmdir calls the generic securityfs functions to remove files * from the apparmor dir. It is up to the apparmor ns locking * to avoid races. @@ -1938,27 +2011,6 @@ void __aafs_ns_rmdir(struct aa_ns *ns) __aa_fs_list_remove_rawdata(ns); - if (ns_subns_dir(ns)) { - sub = d_inode(ns_subns_dir(ns))->i_private; - aa_put_ns(sub); - } - if (ns_subload(ns)) { - sub = d_inode(ns_subload(ns))->i_private; - aa_put_ns(sub); - } - if (ns_subreplace(ns)) { - sub = d_inode(ns_subreplace(ns))->i_private; - aa_put_ns(sub); - } - if (ns_subremove(ns)) { - sub = d_inode(ns_subremove(ns))->i_private; - aa_put_ns(sub); - } - if (ns_subrevision(ns)) { - sub = d_inode(ns_subrevision(ns))->i_private; - aa_put_ns(sub); - } - for (i = AAFS_NS_SIZEOF - 1; i >= 0; --i) { aafs_remove(ns->dents[i]); ns->dents[i] = NULL; @@ -1983,40 +2035,40 @@ static int __aafs_ns_mkdir_entries(struct aa_ns *ns, struct dentry *dir) return PTR_ERR(dent); ns_subdata_dir(ns) = dent; - dent = aafs_create_file("revision", 0444, dir, ns, + dent = aafs_create_file("revision", 0444, dir, + &ns->unconfined->label.count, &aa_fs_ns_revision_fops); if (IS_ERR(dent)) return PTR_ERR(dent); - aa_get_ns(ns); ns_subrevision(ns) = dent; - dent = aafs_create_file(".load", 0640, dir, ns, - &aa_fs_profile_load); + dent = aafs_create_file(".load", 0640, dir, + &ns->unconfined->label.count, + &aa_fs_profile_load); if (IS_ERR(dent)) return PTR_ERR(dent); - aa_get_ns(ns); ns_subload(ns) = dent; - dent = aafs_create_file(".replace", 0640, dir, ns, - &aa_fs_profile_replace); + dent = aafs_create_file(".replace", 0640, dir, + &ns->unconfined->label.count, + &aa_fs_profile_replace); if (IS_ERR(dent)) return PTR_ERR(dent); - aa_get_ns(ns); ns_subreplace(ns) = dent; - dent = aafs_create_file(".remove", 0640, dir, ns, - &aa_fs_profile_remove); + dent = aafs_create_file(".remove", 0640, dir, + &ns->unconfined->label.count, + &aa_fs_profile_remove); if (IS_ERR(dent)) return PTR_ERR(dent); - aa_get_ns(ns); ns_subremove(ns) = dent; /* use create_dentry so we can supply private data */ - dent = aafs_create("namespaces", S_IFDIR | 0755, dir, ns, NULL, NULL, - &ns_dir_inode_operations); + dent = aafs_create("namespaces", S_IFDIR | 0755, dir, + &ns->unconfined->label.count, + NULL, NULL, &ns_dir_inode_operations); if (IS_ERR(dent)) return PTR_ERR(dent); - aa_get_ns(ns); ns_subns_dir(ns) = dent; return 0; diff --git a/security/apparmor/file.c b/security/apparmor/file.c index c758204028780..7de23e85cd5d0 100644 --- a/security/apparmor/file.c +++ b/security/apparmor/file.c @@ -154,8 +154,12 @@ static int path_name(const char *op, const struct cred *subj_cred, const char *info = NULL; int error; - error = aa_path_name(path, flags, buffer, name, &info, - labels_profile(label)->disconnected); + /* don't reaudit files closed during inheritance */ + if (unlikely(path->dentry == aa_null.dentry)) + error = -EACCES; + else + error = aa_path_name(path, flags, buffer, name, &info, + labels_profile(label)->disconnected); if (error) { fn_for_each_confined(label, profile, aa_audit_file(subj_cred, @@ -578,6 +582,9 @@ static bool __unix_needs_revalidation(struct file *file, struct aa_label *label, return false; if (request & NET_PEER_MASK) return false; + /* sock and sock->sk can be NULL for sockets being set up or torn down */ + if (!sock || !sock->sk) + return false; if (sock->sk->sk_family == PF_UNIX) { struct aa_sk_ctx *ctx = aa_sock(sock->sk); @@ -613,6 +620,10 @@ int aa_file_perm(const char *op, const struct cred *subj_cred, AA_BUG(!label); AA_BUG(!file); + /* don't reaudit files closed during inheritance */ + if (unlikely(file->f_path.dentry == aa_null.dentry)) + return -EACCES; + fctx = file_ctx(file); rcu_read_lock(); diff --git a/security/apparmor/include/label.h b/security/apparmor/include/label.h index c0812dbc1b5b0..335f21930702a 100644 --- a/security/apparmor/include/label.h +++ b/security/apparmor/include/label.h @@ -102,7 +102,7 @@ enum label_flags { struct aa_label; struct aa_proxy { - struct kref count; + struct aa_common_ref count; struct aa_label __rcu *label; }; @@ -125,7 +125,7 @@ struct label_it { * vec: vector of profiles comprising the compound label */ struct aa_label { - struct kref count; + struct aa_common_ref count; struct rb_node node; struct rcu_head rcu; struct aa_proxy *proxy; @@ -357,7 +357,7 @@ int aa_label_match(struct aa_profile *profile, struct aa_ruleset *rules, */ static inline struct aa_label *__aa_get_label(struct aa_label *l) { - if (l && kref_get_unless_zero(&l->count)) + if (l && kref_get_unless_zero(&l->count.count)) return l; return NULL; @@ -366,7 +366,7 @@ static inline struct aa_label *__aa_get_label(struct aa_label *l) static inline struct aa_label *aa_get_label(struct aa_label *l) { if (l) - kref_get(&(l->count)); + kref_get(&(l->count.count)); return l; } @@ -386,7 +386,7 @@ static inline struct aa_label *aa_get_label_rcu(struct aa_label __rcu **l) rcu_read_lock(); do { c = rcu_dereference(*l); - } while (c && !kref_get_unless_zero(&c->count)); + } while (c && !kref_get_unless_zero(&c->count.count)); rcu_read_unlock(); return c; @@ -426,7 +426,7 @@ static inline struct aa_label *aa_get_newest_label(struct aa_label *l) static inline void aa_put_label(struct aa_label *l) { if (l) - kref_put(&l->count, aa_label_kref); + kref_put(&l->count.count, aa_label_kref); } /* wrapper fn to indicate semantics of the check */ @@ -443,7 +443,7 @@ void aa_proxy_kref(struct kref *kref); static inline struct aa_proxy *aa_get_proxy(struct aa_proxy *proxy) { if (proxy) - kref_get(&(proxy->count)); + kref_get(&(proxy->count.count)); return proxy; } @@ -451,7 +451,7 @@ static inline struct aa_proxy *aa_get_proxy(struct aa_proxy *proxy) static inline void aa_put_proxy(struct aa_proxy *proxy) { if (proxy) - kref_put(&proxy->count, aa_proxy_kref); + kref_put(&proxy->count.count, aa_proxy_kref); } void __aa_proxy_redirect(struct aa_label *orig, struct aa_label *new); diff --git a/security/apparmor/include/lib.h b/security/apparmor/include/lib.h index 444197075fd6b..26df19c1df4f8 100644 --- a/security/apparmor/include/lib.h +++ b/security/apparmor/include/lib.h @@ -85,6 +85,18 @@ void aa_info_message(const char *str); /* Security blob offsets */ extern struct lsm_blob_sizes apparmor_blob_sizes; +enum reftype { + REF_NS, + REF_PROXY, + REF_RAWDATA, +}; + +/* common reference count used by data the shows up in aafs */ +struct aa_common_ref { + struct kref count; + enum reftype reftype; +}; + /** * aa_strneq - compare null terminated @str to a non null terminated substring * @str: a null terminated string diff --git a/security/apparmor/include/match.h b/security/apparmor/include/match.h index 1fbe82f5021b1..7accb1c39849a 100644 --- a/security/apparmor/include/match.h +++ b/security/apparmor/include/match.h @@ -104,16 +104,18 @@ struct aa_dfa { struct table_header *tables[YYTD_ID_TSIZE]; }; -#define byte_to_byte(X) (X) - #define UNPACK_ARRAY(TABLE, BLOB, LEN, TTYPE, BTYPE, NTOHX) \ do { \ typeof(LEN) __i; \ TTYPE *__t = (TTYPE *) TABLE; \ BTYPE *__b = (BTYPE *) BLOB; \ - for (__i = 0; __i < LEN; __i++) { \ - __t[__i] = NTOHX(__b[__i]); \ - } \ + BUILD_BUG_ON(sizeof(TTYPE) != sizeof(BTYPE)); \ + if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) \ + memcpy(__t, __b, (LEN) * sizeof(BTYPE)); \ + else /* copy & convert from big-endian */ \ + for (__i = 0; __i < LEN; __i++) { \ + __t[__i] = NTOHX(&__b[__i]); \ + } \ } while (0) static inline size_t table_size(size_t len, size_t el_size) @@ -183,6 +185,7 @@ static inline void aa_put_dfa(struct aa_dfa *dfa) #define MATCH_FLAG_DIFF_ENCODE 0x80000000 #define MARK_DIFF_ENCODE 0x40000000 #define MATCH_FLAG_OOB_TRANSITION 0x20000000 +#define MARK_DIFF_ENCODE_VERIFIED 0x10000000 #define MATCH_FLAGS_MASK 0xff000000 #define MATCH_FLAGS_VALID (MATCH_FLAG_DIFF_ENCODE | MATCH_FLAG_OOB_TRANSITION) #define MATCH_FLAGS_INVALID (MATCH_FLAGS_MASK & ~MATCH_FLAGS_VALID) diff --git a/security/apparmor/include/policy.h b/security/apparmor/include/policy.h index 4c50875c9d13e..bf105ae9019d3 100644 --- a/security/apparmor/include/policy.h +++ b/security/apparmor/include/policy.h @@ -355,7 +355,7 @@ static inline bool profile_mediates_safe(struct aa_profile *profile, static inline struct aa_profile *aa_get_profile(struct aa_profile *p) { if (p) - kref_get(&(p->label.count)); + kref_get(&(p->label.count.count)); return p; } @@ -369,7 +369,7 @@ static inline struct aa_profile *aa_get_profile(struct aa_profile *p) */ static inline struct aa_profile *aa_get_profile_not0(struct aa_profile *p) { - if (p && kref_get_unless_zero(&p->label.count)) + if (p && kref_get_unless_zero(&p->label.count.count)) return p; return NULL; @@ -389,7 +389,7 @@ static inline struct aa_profile *aa_get_profile_rcu(struct aa_profile __rcu **p) rcu_read_lock(); do { c = rcu_dereference(*p); - } while (c && !kref_get_unless_zero(&c->label.count)); + } while (c && !kref_get_unless_zero(&c->label.count.count)); rcu_read_unlock(); return c; @@ -402,7 +402,7 @@ static inline struct aa_profile *aa_get_profile_rcu(struct aa_profile __rcu **p) static inline void aa_put_profile(struct aa_profile *p) { if (p) - kref_put(&p->label.count, aa_label_kref); + kref_put(&p->label.count.count, aa_label_kref); } static inline int AUDIT_MODE(struct aa_profile *profile) @@ -419,7 +419,7 @@ bool aa_policy_admin_capable(const struct cred *subj_cred, struct aa_label *label, struct aa_ns *ns); int aa_may_manage_policy(const struct cred *subj_cred, struct aa_label *label, struct aa_ns *ns, - u32 mask); + const struct cred *ocred, u32 mask); bool aa_current_policy_view_capable(struct aa_ns *ns); bool aa_current_policy_admin_capable(struct aa_ns *ns); diff --git a/security/apparmor/include/policy_ns.h b/security/apparmor/include/policy_ns.h index d646070fd966b..cc6e841518120 100644 --- a/security/apparmor/include/policy_ns.h +++ b/security/apparmor/include/policy_ns.h @@ -18,6 +18,8 @@ #include "label.h" #include "policy.h" +/* Match max depth of user namespaces */ +#define MAX_NS_DEPTH 32 /* struct aa_ns_acct - accounting of profiles in namespace * @max_size: maximum space allowed for all profiles in namespace diff --git a/security/apparmor/include/policy_unpack.h b/security/apparmor/include/policy_unpack.h index a6f4611ee50cf..e5a95dc4da1f7 100644 --- a/security/apparmor/include/policy_unpack.h +++ b/security/apparmor/include/policy_unpack.h @@ -87,17 +87,29 @@ struct aa_ext { u32 version; }; -/* - * struct aa_loaddata - buffer of policy raw_data set +/* struct aa_loaddata - buffer of policy raw_data set + * @count: inode/filesystem refcount - use aa_get_i_loaddata() + * @pcount: profile refcount - use aa_get_profile_loaddata() + * @list: list the loaddata is on + * @work: used to do a delayed cleanup + * @dents: refs to dents created in aafs + * @ns: the namespace this loaddata was loaded into + * @name: + * @size: the size of the data that was loaded + * @compressed_size: the size of the data when it is compressed + * @revision: unique revision count that this data was loaded as + * @abi: the abi number the loaddata uses + * @hash: a hash of the loaddata, used to help dedup data * - * there is no loaddata ref for being on ns list, nor a ref from - * d_inode(@dentry) when grab a ref from these, @ns->lock must be held - * && __aa_get_loaddata() needs to be used, and the return value - * checked, if NULL the loaddata is already being reaped and should be - * considered dead. + * There is no loaddata ref for being on ns->rawdata_list, so + * @ns->lock must be held when walking the list. Dentries and + * inode opens hold refs on @count; profiles hold refs on @pcount. + * When the last @pcount drops, do_ploaddata_rmfs() removes the + * fs entries and drops the associated @count ref. */ struct aa_loaddata { - struct kref count; + struct aa_common_ref count; + struct kref pcount; struct list_head list; struct work_struct work; struct dentry *dents[AAFS_LOADDATA_NDENTS]; @@ -119,50 +131,53 @@ struct aa_loaddata { int aa_unpack(struct aa_loaddata *udata, struct list_head *lh, const char **ns); /** - * __aa_get_loaddata - get a reference count to uncounted data reference + * aa_get_loaddata - get a reference count from a counted data reference * @data: reference to get a count on * - * Returns: pointer to reference OR NULL if race is lost and reference is - * being repeated. - * Requires: @data->ns->lock held, and the return code MUST be checked - * - * Use only from inode->i_private and @data->list found references + * Returns: pointer to reference + * Requires: @data to have a valid reference count on it. It is a bug + * if the race to reap can be encountered when it is used. */ static inline struct aa_loaddata * -__aa_get_loaddata(struct aa_loaddata *data) +aa_get_i_loaddata(struct aa_loaddata *data) { - if (data && kref_get_unless_zero(&(data->count))) - return data; - return NULL; + if (data) + kref_get(&(data->count.count)); + return data; } + /** - * aa_get_loaddata - get a reference count from a counted data reference + * aa_get_profile_loaddata - get a profile reference count on loaddata * @data: reference to get a count on * - * Returns: point to reference - * Requires: @data to have a valid reference count on it. It is a bug - * if the race to reap can be encountered when it is used. + * Returns: pointer to reference + * Requires: @data to have a valid reference count on it. */ static inline struct aa_loaddata * -aa_get_loaddata(struct aa_loaddata *data) +aa_get_profile_loaddata(struct aa_loaddata *data) { - struct aa_loaddata *tmp = __aa_get_loaddata(data); - - AA_BUG(data && !tmp); - - return tmp; + if (data) + kref_get(&(data->pcount)); + return data; } void __aa_loaddata_update(struct aa_loaddata *data, long revision); bool aa_rawdata_eq(struct aa_loaddata *l, struct aa_loaddata *r); void aa_loaddata_kref(struct kref *kref); +void aa_ploaddata_kref(struct kref *kref); struct aa_loaddata *aa_loaddata_alloc(size_t size); -static inline void aa_put_loaddata(struct aa_loaddata *data) +static inline void aa_put_i_loaddata(struct aa_loaddata *data) +{ + if (data) + kref_put(&data->count.count, aa_loaddata_kref); +} + +static inline void aa_put_profile_loaddata(struct aa_loaddata *data) { if (data) - kref_put(&data->count, aa_loaddata_kref); + kref_put(&data->pcount, aa_ploaddata_kref); } #if IS_ENABLED(CONFIG_KUNIT) diff --git a/security/apparmor/label.c b/security/apparmor/label.c index 913678f199c35..3bec1e33815e2 100644 --- a/security/apparmor/label.c +++ b/security/apparmor/label.c @@ -52,7 +52,8 @@ static void free_proxy(struct aa_proxy *proxy) void aa_proxy_kref(struct kref *kref) { - struct aa_proxy *proxy = container_of(kref, struct aa_proxy, count); + struct aa_proxy *proxy = container_of(kref, struct aa_proxy, + count.count); free_proxy(proxy); } @@ -63,7 +64,8 @@ struct aa_proxy *aa_alloc_proxy(struct aa_label *label, gfp_t gfp) new = kzalloc(sizeof(struct aa_proxy), gfp); if (new) { - kref_init(&new->count); + kref_init(&new->count.count); + new->count.reftype = REF_PROXY; rcu_assign_pointer(new->label, aa_get_label(label)); } return new; @@ -375,7 +377,8 @@ static void label_free_rcu(struct rcu_head *head) void aa_label_kref(struct kref *kref) { - struct aa_label *label = container_of(kref, struct aa_label, count); + struct aa_label *label = container_of(kref, struct aa_label, + count.count); struct aa_ns *ns = labels_ns(label); if (!ns) { @@ -412,7 +415,8 @@ bool aa_label_init(struct aa_label *label, int size, gfp_t gfp) label->size = size; /* doesn't include null */ label->vec[size] = NULL; /* null terminate */ - kref_init(&label->count); + kref_init(&label->count.count); + label->count.reftype = REF_NS; /* for aafs purposes */ RB_CLEAR_NODE(&label->node); return true; @@ -1278,7 +1282,7 @@ static inline aa_state_t match_component(struct aa_profile *profile, * @request: permissions to request * @perms: perms struct to set * - * Returns: 0 on success else ERROR + * Returns: state match stopped at or DFA_NOMATCH if aborted early * * For the label A//&B//&C this does the perm match for A//&B//&C * @perms should be preinitialized with allperms OR a previous permission @@ -1305,7 +1309,7 @@ static int label_compound_match(struct aa_profile *profile, /* no component visible */ *perms = allperms; - return 0; + return state; next: label_for_each_cont(i, label, tp) { @@ -1317,15 +1321,11 @@ static int label_compound_match(struct aa_profile *profile, goto fail; } *perms = *aa_lookup_perms(rules->policy, state); - aa_apply_modes_to_perms(profile, perms); - if ((perms->allow & request) != request) - return -EACCES; - - return 0; + return state; fail: *perms = nullperms; - return state; + return DFA_NOMATCH; } /** @@ -1338,7 +1338,7 @@ static int label_compound_match(struct aa_profile *profile, * @request: permissions to request * @perms: an initialized perms struct to add accumulation to * - * Returns: 0 on success else ERROR + * Returns: the state the match finished in, may be the none matching state * * For the label A//&B//&C this does the perm match for each of A and B and C * @perms should be preinitialized with allperms OR a previous permission @@ -1366,11 +1366,10 @@ static int label_components_match(struct aa_profile *profile, } /* no subcomponents visible - no change in perms */ - return 0; + return state; next: tmp = *aa_lookup_perms(rules->policy, state); - aa_apply_modes_to_perms(profile, &tmp); aa_perms_accum(perms, &tmp); label_for_each_cont(i, label, tp) { if (!aa_ns_visible(profile->ns, tp->ns, subns)) @@ -1379,18 +1378,17 @@ static int label_components_match(struct aa_profile *profile, if (!state) goto fail; tmp = *aa_lookup_perms(rules->policy, state); - aa_apply_modes_to_perms(profile, &tmp); aa_perms_accum(perms, &tmp); } if ((perms->allow & request) != request) - return -EACCES; + return DFA_NOMATCH; - return 0; + return state; fail: *perms = nullperms; - return -EACCES; + return DFA_NOMATCH; } /** @@ -1409,11 +1407,12 @@ int aa_label_match(struct aa_profile *profile, struct aa_ruleset *rules, struct aa_label *label, aa_state_t state, bool subns, u32 request, struct aa_perms *perms) { - int error = label_compound_match(profile, rules, label, state, subns, - request, perms); - if (!error) - return error; + aa_state_t tmp = label_compound_match(profile, rules, label, state, subns, + request, perms); + if ((perms->allow & request) == request) + return tmp; + /* failed compound_match try component matches */ *perms = allperms; return label_components_match(profile, rules, label, state, subns, request, perms); diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c index b3f7a3258a2cf..13c9bfdf65ff5 100644 --- a/security/apparmor/lsm.c +++ b/security/apparmor/lsm.c @@ -519,33 +519,26 @@ static void apparmor_file_free_security(struct file *file) aa_put_label(rcu_access_pointer(ctx->label)); } -static int common_file_perm(const char *op, struct file *file, u32 mask, - bool in_atomic) +static int common_file_perm(const char *op, struct file *file, u32 mask) { struct aa_label *label; int error = 0; - bool needput; - - /* don't reaudit files closed during inheritance */ - if (unlikely(file->f_path.dentry == aa_null.dentry)) - return -EACCES; - label = __begin_current_label_crit_section(&needput); - error = aa_file_perm(op, current_cred(), label, file, mask, in_atomic); - __end_current_label_crit_section(label, needput); + label = begin_current_label_crit_section(); + error = aa_file_perm(op, current_cred(), label, file, mask, false); + end_current_label_crit_section(label); return error; } static int apparmor_file_receive(struct file *file) { - return common_file_perm(OP_FRECEIVE, file, aa_map_file_to_perms(file), - false); + return common_file_perm(OP_FRECEIVE, file, aa_map_file_to_perms(file)); } static int apparmor_file_permission(struct file *file, int mask) { - return common_file_perm(OP_FPERM, file, mask, false); + return common_file_perm(OP_FPERM, file, mask); } static int apparmor_file_lock(struct file *file, unsigned int cmd) @@ -555,11 +548,11 @@ static int apparmor_file_lock(struct file *file, unsigned int cmd) if (cmd == F_WRLCK) mask |= MAY_WRITE; - return common_file_perm(OP_FLOCK, file, mask, false); + return common_file_perm(OP_FLOCK, file, mask); } static int common_mmap(const char *op, struct file *file, unsigned long prot, - unsigned long flags, bool in_atomic) + unsigned long flags) { int mask = 0; @@ -577,21 +570,20 @@ static int common_mmap(const char *op, struct file *file, unsigned long prot, if (prot & PROT_EXEC) mask |= AA_EXEC_MMAP; - return common_file_perm(op, file, mask, in_atomic); + return common_file_perm(op, file, mask); } static int apparmor_mmap_file(struct file *file, unsigned long reqprot, unsigned long prot, unsigned long flags) { - return common_mmap(OP_FMMAP, file, prot, flags, GFP_ATOMIC); + return common_mmap(OP_FMMAP, file, prot, flags); } static int apparmor_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, unsigned long prot) { return common_mmap(OP_FMPROT, vma->vm_file, prot, - !(vma->vm_flags & VM_SHARED) ? MAP_PRIVATE : 0, - false); + !(vma->vm_flags & VM_SHARED) ? MAP_PRIVATE : 0); } #ifdef CONFIG_IO_URING @@ -2144,7 +2136,8 @@ char *aa_get_buffer(bool in_atomic) if (!list_empty(&cache->head)) { aa_buf = list_first_entry(&cache->head, union aa_buffer, list); list_del(&aa_buf->list); - cache->hold--; + if (cache->hold) + cache->hold--; cache->count--; put_cpu_ptr(&aa_local_buffers); return &aa_buf->buffer[0]; diff --git a/security/apparmor/match.c b/security/apparmor/match.c index c5a91600842a1..0de249725efbf 100644 --- a/security/apparmor/match.c +++ b/security/apparmor/match.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "include/lib.h" #include "include/match.h" @@ -42,11 +43,11 @@ static struct table_header *unpack_table(char *blob, size_t bsize) /* loaded td_id's start at 1, subtract 1 now to avoid doing * it every time we use td_id as an index */ - th.td_id = be16_to_cpu(*(__be16 *) (blob)) - 1; + th.td_id = get_unaligned_be16(blob) - 1; if (th.td_id > YYTD_ID_MAX) goto out; - th.td_flags = be16_to_cpu(*(__be16 *) (blob + 2)); - th.td_lolen = be32_to_cpu(*(__be32 *) (blob + 8)); + th.td_flags = get_unaligned_be16(blob + 2); + th.td_lolen = get_unaligned_be32(blob + 8); blob += sizeof(struct table_header); if (!(th.td_flags == YYTD_DATA16 || th.td_flags == YYTD_DATA32 || @@ -66,14 +67,13 @@ static struct table_header *unpack_table(char *blob, size_t bsize) table->td_flags = th.td_flags; table->td_lolen = th.td_lolen; if (th.td_flags == YYTD_DATA8) - UNPACK_ARRAY(table->td_data, blob, th.td_lolen, - u8, u8, byte_to_byte); + memcpy(table->td_data, blob, th.td_lolen); else if (th.td_flags == YYTD_DATA16) UNPACK_ARRAY(table->td_data, blob, th.td_lolen, - u16, __be16, be16_to_cpu); + u16, __be16, get_unaligned_be16); else if (th.td_flags == YYTD_DATA32) UNPACK_ARRAY(table->td_data, blob, th.td_lolen, - u32, __be32, be32_to_cpu); + u32, __be32, get_unaligned_be32); else goto fail; /* if table was vmalloced make sure the page tables are synced @@ -160,9 +160,10 @@ static int verify_dfa(struct aa_dfa *dfa) if (state_count == 0) goto out; for (i = 0; i < state_count; i++) { - if (!(BASE_TABLE(dfa)[i] & MATCH_FLAG_DIFF_ENCODE) && - (DEFAULT_TABLE(dfa)[i] >= state_count)) + if (DEFAULT_TABLE(dfa)[i] >= state_count) { + pr_err("AppArmor DFA default state out of bounds"); goto out; + } if (BASE_TABLE(dfa)[i] & MATCH_FLAGS_INVALID) { pr_err("AppArmor DFA state with invalid match flags"); goto out; @@ -201,16 +202,31 @@ static int verify_dfa(struct aa_dfa *dfa) size_t j, k; for (j = i; - (BASE_TABLE(dfa)[j] & MATCH_FLAG_DIFF_ENCODE) && - !(BASE_TABLE(dfa)[j] & MARK_DIFF_ENCODE); + ((BASE_TABLE(dfa)[j] & MATCH_FLAG_DIFF_ENCODE) && + !(BASE_TABLE(dfa)[j] & MARK_DIFF_ENCODE_VERIFIED)); j = k) { + if (BASE_TABLE(dfa)[j] & MARK_DIFF_ENCODE) + /* loop in current chain */ + goto out; k = DEFAULT_TABLE(dfa)[j]; if (j == k) + /* self loop */ goto out; - if (k < j) - break; /* already verified */ BASE_TABLE(dfa)[j] |= MARK_DIFF_ENCODE; } + /* move mark to verified */ + for (j = i; + (BASE_TABLE(dfa)[j] & MATCH_FLAG_DIFF_ENCODE); + j = k) { + k = DEFAULT_TABLE(dfa)[j]; + if (j < i) + /* jumps to state/chain that has been + * verified + */ + break; + BASE_TABLE(dfa)[j] &= ~MARK_DIFF_ENCODE; + BASE_TABLE(dfa)[j] |= MARK_DIFF_ENCODE_VERIFIED; + } } error = 0; @@ -313,14 +329,14 @@ struct aa_dfa *aa_dfa_unpack(void *blob, size_t size, int flags) if (size < sizeof(struct table_set_header)) goto fail; - if (ntohl(*(__be32 *) data) != YYTH_MAGIC) + if (get_unaligned_be32(data) != YYTH_MAGIC) goto fail; - hsize = ntohl(*(__be32 *) (data + 4)); + hsize = get_unaligned_be32(data + 4); if (size < hsize) goto fail; - dfa->flags = ntohs(*(__be16 *) (data + 12)); + dfa->flags = get_unaligned_be16(data + 12); if (dfa->flags & ~(YYTH_FLAGS)) goto fail; @@ -329,7 +345,7 @@ struct aa_dfa *aa_dfa_unpack(void *blob, size_t size, int flags) * if (dfa->flags & YYTH_FLAGS_OOB_TRANS) { * if (hsize < 16 + 4) * goto fail; - * dfa->max_oob = ntol(*(__be32 *) (data + 16)); + * dfa->max_oob = get_unaligned_be32(data + 16); * if (dfa->max <= MAX_OOB_SUPPORTED) { * pr_err("AppArmor DFA OOB greater than supported\n"); * goto fail; @@ -463,13 +479,18 @@ aa_state_t aa_dfa_match_len(struct aa_dfa *dfa, aa_state_t start, if (dfa->tables[YYTD_ID_EC]) { /* Equivalence class table defined */ u8 *equiv = EQUIV_TABLE(dfa); - for (; len; len--) - match_char(state, def, base, next, check, - equiv[(u8) *str++]); + for (; len; len--) { + u8 c = equiv[(u8) *str]; + + match_char(state, def, base, next, check, c); + str++; + } } else { /* default is direct to next state */ - for (; len; len--) - match_char(state, def, base, next, check, (u8) *str++); + for (; len; len--) { + match_char(state, def, base, next, check, (u8) *str); + str++; + } } return state; @@ -503,13 +524,18 @@ aa_state_t aa_dfa_match(struct aa_dfa *dfa, aa_state_t start, const char *str) /* Equivalence class table defined */ u8 *equiv = EQUIV_TABLE(dfa); /* default is direct to next state */ - while (*str) - match_char(state, def, base, next, check, - equiv[(u8) *str++]); + while (*str) { + u8 c = equiv[(u8) *str]; + + match_char(state, def, base, next, check, c); + str++; + } } else { /* default is direct to next state */ - while (*str) - match_char(state, def, base, next, check, (u8) *str++); + while (*str) { + match_char(state, def, base, next, check, (u8) *str); + str++; + } } return state; diff --git a/security/apparmor/net.c b/security/apparmor/net.c index 45cf25605c345..44c04102062f3 100644 --- a/security/apparmor/net.c +++ b/security/apparmor/net.c @@ -326,8 +326,10 @@ int aa_sock_file_perm(const struct cred *subj_cred, struct aa_label *label, struct socket *sock = (struct socket *) file->private_data; AA_BUG(!label); - AA_BUG(!sock); - AA_BUG(!sock->sk); + + /* sock && sock->sk can be NULL for sockets being set up or torn down */ + if (!sock || !sock->sk) + return 0; if (sock->sk->sk_family == PF_UNIX) return aa_unix_file_perm(subj_cred, label, op, request, file); diff --git a/security/apparmor/policy.c b/security/apparmor/policy.c index 50d5345ff5cba..b92db1b2f26e2 100644 --- a/security/apparmor/policy.c +++ b/security/apparmor/policy.c @@ -183,19 +183,43 @@ static void __list_remove_profile(struct aa_profile *profile) } /** - * __remove_profile - remove old profile, and children - * @profile: profile to be replaced (NOT NULL) + * __remove_profile - remove profile, and children + * @profile: profile to be removed (NOT NULL) * * Requires: namespace list lock be held, or list not be shared */ static void __remove_profile(struct aa_profile *profile) { + struct aa_profile *curr, *to_remove; + AA_BUG(!profile); AA_BUG(!profile->ns); AA_BUG(!mutex_is_locked(&profile->ns->lock)); /* release any children lists first */ - __aa_profile_list_release(&profile->base.profiles); + if (!list_empty(&profile->base.profiles)) { + curr = list_first_entry(&profile->base.profiles, struct aa_profile, base.list); + + while (curr != profile) { + + while (!list_empty(&curr->base.profiles)) + curr = list_first_entry(&curr->base.profiles, + struct aa_profile, base.list); + + to_remove = curr; + if (!list_is_last(&to_remove->base.list, + &aa_deref_parent(curr)->base.profiles)) + curr = list_next_entry(to_remove, base.list); + else + curr = aa_deref_parent(curr); + + /* released by free_profile */ + aa_label_remove(&to_remove->label); + __aafs_profile_rmdir(to_remove); + __list_remove_profile(to_remove); + } + } + /* released by free_profile */ aa_label_remove(&profile->label); __aafs_profile_rmdir(profile); @@ -312,7 +336,7 @@ void aa_free_profile(struct aa_profile *profile) } kfree_sensitive(profile->hash); - aa_put_loaddata(profile->rawdata); + aa_put_profile_loaddata(profile->rawdata); aa_label_destroy(&profile->label); kfree_sensitive(profile); @@ -901,17 +925,44 @@ bool aa_current_policy_admin_capable(struct aa_ns *ns) return res; } +static bool is_subset_of_obj_privilege(const struct cred *cred, + struct aa_label *label, + const struct cred *ocred) +{ + if (cred == ocred) + return true; + + if (!aa_label_is_subset(label, cred_label(ocred))) + return false; + /* don't allow crossing userns for now */ + if (cred->user_ns != ocred->user_ns) + return false; + if (!cap_issubset(cred->cap_inheritable, ocred->cap_inheritable)) + return false; + if (!cap_issubset(cred->cap_permitted, ocred->cap_permitted)) + return false; + if (!cap_issubset(cred->cap_effective, ocred->cap_effective)) + return false; + if (!cap_issubset(cred->cap_bset, ocred->cap_bset)) + return false; + if (!cap_issubset(cred->cap_ambient, ocred->cap_ambient)) + return false; + return true; +} + + /** * aa_may_manage_policy - can the current task manage policy * @subj_cred: subjects cred * @label: label to check if it can manage policy * @ns: namespace being managed by @label (may be NULL if @label's ns) + * @ocred: object cred if request is coming from an open object * @mask: contains the policy manipulation operation being done * * Returns: 0 if the task is allowed to manipulate policy else error */ int aa_may_manage_policy(const struct cred *subj_cred, struct aa_label *label, - struct aa_ns *ns, u32 mask) + struct aa_ns *ns, const struct cred *ocred, u32 mask) { const char *op; @@ -927,6 +978,11 @@ int aa_may_manage_policy(const struct cred *subj_cred, struct aa_label *label, return audit_policy(label, op, NULL, NULL, "policy_locked", -EACCES); + if (ocred && !is_subset_of_obj_privilege(subj_cred, label, ocred)) + return audit_policy(label, op, NULL, NULL, + "not privileged for target profile", + -EACCES); + if (!aa_policy_admin_capable(subj_cred, label, ns)) return audit_policy(label, op, NULL, NULL, "not policy admin", -EACCES); @@ -1098,7 +1154,7 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, LIST_HEAD(lh); op = mask & AA_MAY_REPLACE_POLICY ? OP_PROF_REPL : OP_PROF_LOAD; - aa_get_loaddata(udata); + aa_get_profile_loaddata(udata); /* released below */ error = aa_unpack(udata, &lh, &ns_name); if (error) @@ -1125,6 +1181,7 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, goto fail; } ns_name = ent->ns_name; + ent->ns_name = NULL; } else count++; } @@ -1149,10 +1206,10 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, if (aa_rawdata_eq(rawdata_ent, udata)) { struct aa_loaddata *tmp; - tmp = __aa_get_loaddata(rawdata_ent); + tmp = aa_get_profile_loaddata(rawdata_ent); /* check we didn't fail the race */ if (tmp) { - aa_put_loaddata(udata); + aa_put_profile_loaddata(udata); udata = tmp; break; } @@ -1165,7 +1222,7 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, struct aa_profile *p; if (aa_g_export_binary) - ent->new->rawdata = aa_get_loaddata(udata); + ent->new->rawdata = aa_get_profile_loaddata(udata); error = __lookup_replace(ns, ent->new->base.hname, !(mask & AA_MAY_REPLACE_POLICY), &ent->old, &info); @@ -1298,7 +1355,7 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label, out: aa_put_ns(ns); - aa_put_loaddata(udata); + aa_put_profile_loaddata(udata); kfree(ns_name); if (error) diff --git a/security/apparmor/policy_ns.c b/security/apparmor/policy_ns.c index 64783ca3b0f2a..ff49a31ac2744 100644 --- a/security/apparmor/policy_ns.c +++ b/security/apparmor/policy_ns.c @@ -223,6 +223,8 @@ static struct aa_ns *__aa_create_ns(struct aa_ns *parent, const char *name, AA_BUG(!name); AA_BUG(!mutex_is_locked(&parent->lock)); + if (parent->level > MAX_NS_DEPTH) + return ERR_PTR(-ENOSPC); ns = alloc_ns(parent->base.hname, name); if (!ns) return ERR_PTR(-ENOMEM); diff --git a/security/apparmor/policy_unpack.c b/security/apparmor/policy_unpack.c index 7523971e37d9c..b0e18dd8d512b 100644 --- a/security/apparmor/policy_unpack.c +++ b/security/apparmor/policy_unpack.c @@ -109,34 +109,48 @@ bool aa_rawdata_eq(struct aa_loaddata *l, struct aa_loaddata *r) return memcmp(l->data, r->data, r->compressed_size ?: r->size) == 0; } +static void do_loaddata_free(struct aa_loaddata *d) +{ + kfree_sensitive(d->hash); + kfree_sensitive(d->name); + kvfree(d->data); + kfree_sensitive(d); +} + +void aa_loaddata_kref(struct kref *kref) +{ + struct aa_loaddata *d = container_of(kref, struct aa_loaddata, + count.count); + + do_loaddata_free(d); +} + /* * need to take the ns mutex lock which is NOT safe most places that * put_loaddata is called, so we have to delay freeing it */ -static void do_loaddata_free(struct work_struct *work) +static void do_ploaddata_rmfs(struct work_struct *work) { struct aa_loaddata *d = container_of(work, struct aa_loaddata, work); struct aa_ns *ns = aa_get_ns(d->ns); if (ns) { mutex_lock_nested(&ns->lock, ns->level); + /* remove fs ref to loaddata */ __aa_fs_remove_rawdata(d); mutex_unlock(&ns->lock); aa_put_ns(ns); } - - kfree_sensitive(d->hash); - kfree_sensitive(d->name); - kvfree(d->data); - kfree_sensitive(d); + /* called by dropping last pcount, so drop its associated icount */ + aa_put_i_loaddata(d); } -void aa_loaddata_kref(struct kref *kref) +void aa_ploaddata_kref(struct kref *kref) { - struct aa_loaddata *d = container_of(kref, struct aa_loaddata, count); + struct aa_loaddata *d = container_of(kref, struct aa_loaddata, pcount); if (d) { - INIT_WORK(&d->work, do_loaddata_free); + INIT_WORK(&d->work, do_ploaddata_rmfs); schedule_work(&d->work); } } @@ -153,7 +167,9 @@ struct aa_loaddata *aa_loaddata_alloc(size_t size) kfree(d); return ERR_PTR(-ENOMEM); } - kref_init(&d->count); + kref_init(&d->count.count); + d->count.reftype = REF_RAWDATA; + kref_init(&d->pcount); INIT_LIST_HEAD(&d->list); return d; @@ -687,8 +703,10 @@ static ssize_t unpack_perms_table(struct aa_ext *e, struct aa_perms **perms) if (!aa_unpack_array(e, NULL, &size)) goto fail_reset; *perms = kcalloc(size, sizeof(struct aa_perms), GFP_KERNEL); - if (!*perms) - goto fail_reset; + if (!*perms) { + e->pos = pos; + return -ENOMEM; + } for (i = 0; i < size; i++) { if (!unpack_perm(e, version, &(*perms)[i])) goto fail; @@ -768,7 +786,17 @@ static int unpack_pdb(struct aa_ext *e, struct aa_policydb **policy, if (!aa_unpack_u32(e, &pdb->start[AA_CLASS_FILE], "dfa_start")) { /* default start state for xmatch and file dfa */ pdb->start[AA_CLASS_FILE] = DFA_START; - } /* setup class index */ + } + + size_t state_count = pdb->dfa->tables[YYTD_ID_BASE]->td_lolen; + + if (pdb->start[0] >= state_count || + pdb->start[AA_CLASS_FILE] >= state_count) { + *info = "invalid dfa start state"; + goto fail; + } + + /* setup class index */ for (i = AA_CLASS_FILE + 1; i <= AA_CLASS_LAST; i++) { pdb->start[i] = aa_dfa_next(pdb->dfa, pdb->start[0], i); @@ -1165,7 +1193,6 @@ static int verify_header(struct aa_ext *e, int required, const char **ns) { int error = -EPROTONOSUPPORT; const char *name = NULL; - *ns = NULL; /* get the interface version */ if (!aa_unpack_u32(e, &e->version, "version")) { diff --git a/security/apparmor/resource.c b/security/apparmor/resource.c index 8e80db3ae21c0..64212b39ba4bb 100644 --- a/security/apparmor/resource.c +++ b/security/apparmor/resource.c @@ -196,6 +196,11 @@ void __aa_transition_rlimits(struct aa_label *old_l, struct aa_label *new_l) rules->rlimits.limits[j].rlim_max); /* soft limit should not exceed hard limit */ rlim->rlim_cur = min(rlim->rlim_cur, rlim->rlim_max); + if (j == RLIMIT_CPU && + rlim->rlim_cur != RLIM_INFINITY && + IS_ENABLED(CONFIG_POSIX_TIMERS)) + (void) update_rlimit_cpu(current->group_leader, + rlim->rlim_cur); } } } diff --git a/security/integrity/evm/evm_crypto.c b/security/integrity/evm/evm_crypto.c index a5e730ffda57f..5a8cef45bacf0 100644 --- a/security/integrity/evm/evm_crypto.c +++ b/security/integrity/evm/evm_crypto.c @@ -401,6 +401,7 @@ int evm_init_hmac(struct inode *inode, const struct xattr *xattrs, { struct shash_desc *desc; const struct xattr *xattr; + struct xattr_list *xattr_entry; desc = init_desc(EVM_XATTR_HMAC, HASH_ALGO_SHA1); if (IS_ERR(desc)) { @@ -408,11 +409,16 @@ int evm_init_hmac(struct inode *inode, const struct xattr *xattrs, return PTR_ERR(desc); } - for (xattr = xattrs; xattr->name; xattr++) { - if (!evm_protected_xattr(xattr->name)) - continue; + list_for_each_entry_lockless(xattr_entry, &evm_config_xattrnames, + list) { + for (xattr = xattrs; xattr->name; xattr++) { + if (strcmp(xattr_entry->name + + XATTR_SECURITY_PREFIX_LEN, xattr->name) != 0) + continue; - crypto_shash_update(desc, xattr->value, xattr->value_len); + crypto_shash_update(desc, xattr->value, + xattr->value_len); + } } hmac_add_misc(desc, inode, EVM_XATTR_HMAC, hmac_val); diff --git a/security/integrity/ima/ima_kexec.c b/security/integrity/ima/ima_kexec.c index 5beb69edd12fd..36a34c54de58b 100644 --- a/security/integrity/ima/ima_kexec.c +++ b/security/integrity/ima/ima_kexec.c @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include #include "ima.h" @@ -294,3 +296,36 @@ void __init ima_load_kexec_buffer(void) pr_debug("Error restoring the measurement list: %d\n", rc); } } + +/* + * ima_validate_range - verify a physical buffer lies in addressable RAM + * @phys: physical start address of the buffer from previous kernel + * @size: size of the buffer + * + * On success return 0. On failure returns -EINVAL so callers can skip + * restoring. + */ +int ima_validate_range(phys_addr_t phys, size_t size) +{ + unsigned long start_pfn, end_pfn; + phys_addr_t end_phys; + + if (check_add_overflow(phys, (phys_addr_t)size - 1, &end_phys)) + return -EINVAL; + + start_pfn = PHYS_PFN(phys); + end_pfn = PHYS_PFN(end_phys); + +#ifdef CONFIG_X86 + if (!pfn_range_is_mapped(start_pfn, end_pfn)) +#else + if (!page_is_ram(start_pfn) || !page_is_ram(end_pfn)) +#endif + { + pr_warn("IMA: previous kernel measurement buffer %pa (size 0x%zx) lies outside available memory\n", + &phys, size); + return -EINVAL; + } + + return 0; +} diff --git a/security/smack/smackfs.c b/security/smack/smackfs.c index b1e5e62f5cbd1..d27d9140dda2f 100644 --- a/security/smack/smackfs.c +++ b/security/smack/smackfs.c @@ -70,6 +70,7 @@ enum smk_inos { static DEFINE_MUTEX(smack_cipso_lock); static DEFINE_MUTEX(smack_ambient_lock); static DEFINE_MUTEX(smk_net4addr_lock); +static DEFINE_MUTEX(smk_cipso_doi_lock); #if IS_ENABLED(CONFIG_IPV6) static DEFINE_MUTEX(smk_net6addr_lock); #endif /* CONFIG_IPV6 */ @@ -141,7 +142,7 @@ struct smack_parsed_rule { int smk_access2; }; -static int smk_cipso_doi_value = SMACK_CIPSO_DOI_DEFAULT; +static u32 smk_cipso_doi_value = CIPSO_V4_DOI_UNKNOWN; /* * Values for parsing cipso rules @@ -663,43 +664,60 @@ static const struct file_operations smk_load_ops = { }; /** - * smk_cipso_doi - initialize the CIPSO domain + * smk_cipso_doi - set netlabel maps + * @ndoi: new value for our CIPSO DOI + * @gfp_flags: kmalloc allocation context */ -static void smk_cipso_doi(void) +static int +smk_cipso_doi(u32 ndoi, gfp_t gfp_flags) { - int rc; + int rc = 0; struct cipso_v4_doi *doip; struct netlbl_audit nai; - smk_netlabel_audit_set(&nai); + mutex_lock(&smk_cipso_doi_lock); - rc = netlbl_cfg_map_del(NULL, PF_INET, NULL, NULL, &nai); - if (rc != 0) - printk(KERN_WARNING "%s:%d remove rc = %d\n", - __func__, __LINE__, rc); + if (smk_cipso_doi_value == ndoi) + goto clr_doi_lock; + + smk_netlabel_audit_set(&nai); - doip = kmalloc(sizeof(struct cipso_v4_doi), GFP_KERNEL | __GFP_NOFAIL); + doip = kmalloc(sizeof(struct cipso_v4_doi), gfp_flags); + if (!doip) { + rc = -ENOMEM; + goto clr_doi_lock; + } doip->map.std = NULL; - doip->doi = smk_cipso_doi_value; + doip->doi = ndoi; doip->type = CIPSO_V4_MAP_PASS; doip->tags[0] = CIPSO_V4_TAG_RBITMAP; for (rc = 1; rc < CIPSO_V4_TAG_MAXCNT; rc++) doip->tags[rc] = CIPSO_V4_TAG_INVALID; rc = netlbl_cfg_cipsov4_add(doip, &nai); - if (rc != 0) { - printk(KERN_WARNING "%s:%d cipso add rc = %d\n", - __func__, __LINE__, rc); + if (rc) { kfree(doip); - return; + goto clr_doi_lock; } - rc = netlbl_cfg_cipsov4_map_add(doip->doi, NULL, NULL, NULL, &nai); - if (rc != 0) { - printk(KERN_WARNING "%s:%d map add rc = %d\n", - __func__, __LINE__, rc); - netlbl_cfg_cipsov4_del(doip->doi, &nai); - return; + + if (smk_cipso_doi_value != CIPSO_V4_DOI_UNKNOWN) { + rc = netlbl_cfg_map_del(NULL, PF_INET, NULL, NULL, &nai); + if (rc && rc != -ENOENT) + goto clr_ndoi_def; + + netlbl_cfg_cipsov4_del(smk_cipso_doi_value, &nai); } + + rc = netlbl_cfg_cipsov4_map_add(ndoi, NULL, NULL, NULL, &nai); + if (rc) { + smk_cipso_doi_value = CIPSO_V4_DOI_UNKNOWN; // no default map +clr_ndoi_def: netlbl_cfg_cipsov4_del(ndoi, &nai); + } else + smk_cipso_doi_value = ndoi; + +clr_doi_lock: + mutex_unlock(&smk_cipso_doi_lock); + return rc; } /** @@ -1562,7 +1580,7 @@ static ssize_t smk_read_doi(struct file *filp, char __user *buf, if (*ppos != 0) return 0; - sprintf(temp, "%d", smk_cipso_doi_value); + sprintf(temp, "%lu", (unsigned long)smk_cipso_doi_value); rc = simple_read_from_buffer(buf, count, ppos, temp, strlen(temp)); return rc; @@ -1581,7 +1599,7 @@ static ssize_t smk_write_doi(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { char temp[80]; - int i; + unsigned long u; if (!smack_privileged(CAP_MAC_ADMIN)) return -EPERM; @@ -1594,14 +1612,13 @@ static ssize_t smk_write_doi(struct file *file, const char __user *buf, temp[count] = '\0'; - if (sscanf(temp, "%d", &i) != 1) + if (kstrtoul(temp, 10, &u)) return -EINVAL; - smk_cipso_doi_value = i; - - smk_cipso_doi(); + if (u == CIPSO_V4_DOI_UNKNOWN || u > U32_MAX) + return -EINVAL; - return count; + return smk_cipso_doi(u, GFP_KERNEL) ? : count; } static const struct file_operations smk_doi_ops = { @@ -2982,6 +2999,7 @@ static int __init init_smk_fs(void) { int err; int rc; + struct netlbl_audit nai; if (smack_enabled == 0) return 0; @@ -3000,7 +3018,10 @@ static int __init init_smk_fs(void) } } - smk_cipso_doi(); + smk_netlabel_audit_set(&nai); + (void) netlbl_cfg_map_del(NULL, PF_INET, NULL, NULL, &nai); + (void) smk_cipso_doi(SMACK_CIPSO_DOI_DEFAULT, + GFP_KERNEL | __GFP_NOFAIL); smk_unlbl_ambient(NULL); rc = smack_populate_secattr(&smack_known_floor); diff --git a/sound/core/oss/mixer_oss.c b/sound/core/oss/mixer_oss.c index e839a4bb93f81..aa98caaaea3c5 100644 --- a/sound/core/oss/mixer_oss.c +++ b/sound/core/oss/mixer_oss.c @@ -525,6 +525,8 @@ static void snd_mixer_oss_get_volume1_vol(struct snd_mixer_oss_file *fmixer, if (numid == ID_UNKNOWN) return; guard(rwsem_read)(&card->controls_rwsem); + if (card->shutdown) + return; kctl = snd_ctl_find_numid(card, numid); if (!kctl) return; @@ -558,6 +560,8 @@ static void snd_mixer_oss_get_volume1_sw(struct snd_mixer_oss_file *fmixer, if (numid == ID_UNKNOWN) return; guard(rwsem_read)(&card->controls_rwsem); + if (card->shutdown) + return; kctl = snd_ctl_find_numid(card, numid); if (!kctl) return; @@ -618,6 +622,8 @@ static void snd_mixer_oss_put_volume1_vol(struct snd_mixer_oss_file *fmixer, if (numid == ID_UNKNOWN) return; guard(rwsem_read)(&card->controls_rwsem); + if (card->shutdown) + return; kctl = snd_ctl_find_numid(card, numid); if (!kctl) return; @@ -655,6 +661,8 @@ static void snd_mixer_oss_put_volume1_sw(struct snd_mixer_oss_file *fmixer, if (numid == ID_UNKNOWN) return; guard(rwsem_read)(&card->controls_rwsem); + if (card->shutdown) + return; kctl = snd_ctl_find_numid(card, numid); if (!kctl) return; @@ -792,6 +800,8 @@ static int snd_mixer_oss_get_recsrc2(struct snd_mixer_oss_file *fmixer, unsigned if (uinfo == NULL || uctl == NULL) return -ENOMEM; guard(rwsem_read)(&card->controls_rwsem); + if (card->shutdown) + return -ENODEV; kctl = snd_mixer_oss_test_id(mixer, "Capture Source", 0); if (!kctl) return -ENOENT; @@ -835,6 +845,8 @@ static int snd_mixer_oss_put_recsrc2(struct snd_mixer_oss_file *fmixer, unsigned if (uinfo == NULL || uctl == NULL) return -ENOMEM; guard(rwsem_read)(&card->controls_rwsem); + if (card->shutdown) + return -ENODEV; kctl = snd_mixer_oss_test_id(mixer, "Capture Source", 0); if (!kctl) return -ENOENT; @@ -878,6 +890,8 @@ static int snd_mixer_oss_build_test(struct snd_mixer_oss *mixer, struct slot *sl int err; scoped_guard(rwsem_read, &card->controls_rwsem) { + if (card->shutdown) + return -ENODEV; kcontrol = snd_mixer_oss_test_id(mixer, name, index); if (kcontrol == NULL) return 0; @@ -1002,6 +1016,8 @@ static int snd_mixer_oss_build_input(struct snd_mixer_oss *mixer, if (snd_mixer_oss_build_test_all(mixer, ptr, &slot)) return 0; guard(rwsem_read)(&mixer->card->controls_rwsem); + if (mixer->card->shutdown) + return -ENODEV; kctl = NULL; if (!ptr->index) kctl = snd_mixer_oss_test_id(mixer, "Capture Source", 0); diff --git a/sound/core/pcm.c b/sound/core/pcm.c index 283aac441fa0a..0b512085eb63f 100644 --- a/sound/core/pcm.c +++ b/sound/core/pcm.c @@ -328,13 +328,13 @@ static const char *snd_pcm_oss_format_name(int format) static void snd_pcm_proc_info_read(struct snd_pcm_substream *substream, struct snd_info_buffer *buffer) { - struct snd_pcm_info *info __free(kfree) = NULL; int err; if (! substream) return; - info = kmalloc(sizeof(*info), GFP_KERNEL); + struct snd_pcm_info *info __free(kfree) = + kmalloc(sizeof(*info), GFP_KERNEL); if (!info) return; diff --git a/sound/core/pcm_compat.c b/sound/core/pcm_compat.c index 54eb9bd8eb218..e86f68f1f23c1 100644 --- a/sound/core/pcm_compat.c +++ b/sound/core/pcm_compat.c @@ -235,7 +235,6 @@ static int snd_pcm_ioctl_hw_params_compat(struct snd_pcm_substream *substream, int refine, struct snd_pcm_hw_params32 __user *data32) { - struct snd_pcm_hw_params *data __free(kfree) = NULL; struct snd_pcm_runtime *runtime; int err; @@ -243,7 +242,8 @@ static int snd_pcm_ioctl_hw_params_compat(struct snd_pcm_substream *substream, if (!runtime) return -ENOTTY; - data = kmalloc(sizeof(*data), GFP_KERNEL); + struct snd_pcm_hw_params *data __free(kfree) = + kmalloc(sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; @@ -332,7 +332,6 @@ static int snd_pcm_ioctl_xfern_compat(struct snd_pcm_substream *substream, compat_caddr_t buf; compat_caddr_t __user *bufptr; u32 frames; - void __user **bufs __free(kfree) = NULL; int err, ch, i; if (! substream->runtime) @@ -349,7 +348,9 @@ static int snd_pcm_ioctl_xfern_compat(struct snd_pcm_substream *substream, get_user(frames, &data32->frames)) return -EFAULT; bufptr = compat_ptr(buf); - bufs = kmalloc_array(ch, sizeof(void __user *), GFP_KERNEL); + + void __user **bufs __free(kfree) = + kmalloc_array(ch, sizeof(void __user *), GFP_KERNEL); if (bufs == NULL) return -ENOMEM; for (i = 0; i < ch; i++) { diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c index 932a9bf98cbc0..0a358d94b17c6 100644 --- a/sound/core/pcm_native.c +++ b/sound/core/pcm_native.c @@ -242,10 +242,10 @@ int snd_pcm_info(struct snd_pcm_substream *substream, struct snd_pcm_info *info) int snd_pcm_info_user(struct snd_pcm_substream *substream, struct snd_pcm_info __user * _info) { - struct snd_pcm_info *info __free(kfree) = NULL; int err; + struct snd_pcm_info *info __free(kfree) = + kmalloc(sizeof(*info), GFP_KERNEL); - info = kmalloc(sizeof(*info), GFP_KERNEL); if (! info) return -ENOMEM; err = snd_pcm_info(substream, info); @@ -364,7 +364,6 @@ static int constrain_params_by_rules(struct snd_pcm_substream *substream, struct snd_pcm_hw_constraints *constrs = &substream->runtime->hw_constraints; unsigned int k; - unsigned int *rstamps __free(kfree) = NULL; unsigned int vstamps[SNDRV_PCM_HW_PARAM_LAST_INTERVAL + 1]; unsigned int stamp; struct snd_pcm_hw_rule *r; @@ -380,7 +379,8 @@ static int constrain_params_by_rules(struct snd_pcm_substream *substream, * Each member of 'rstamps' array represents the sequence number of * recent application of corresponding rule. */ - rstamps = kcalloc(constrs->rules_num, sizeof(unsigned int), GFP_KERNEL); + unsigned int *rstamps __free(kfree) = + kcalloc(constrs->rules_num, sizeof(unsigned int), GFP_KERNEL); if (!rstamps) return -ENOMEM; @@ -583,10 +583,10 @@ EXPORT_SYMBOL(snd_pcm_hw_refine); static int snd_pcm_hw_refine_user(struct snd_pcm_substream *substream, struct snd_pcm_hw_params __user * _params) { - struct snd_pcm_hw_params *params __free(kfree) = NULL; int err; + struct snd_pcm_hw_params *params __free(kfree) = + memdup_user(_params, sizeof(*params)); - params = memdup_user(_params, sizeof(*params)); if (IS_ERR(params)) return PTR_ERR(params); @@ -889,10 +889,10 @@ static int snd_pcm_hw_params(struct snd_pcm_substream *substream, static int snd_pcm_hw_params_user(struct snd_pcm_substream *substream, struct snd_pcm_hw_params __user * _params) { - struct snd_pcm_hw_params *params __free(kfree) = NULL; int err; + struct snd_pcm_hw_params *params __free(kfree) = + memdup_user(_params, sizeof(*params)); - params = memdup_user(_params, sizeof(*params)); if (IS_ERR(params)) return PTR_ERR(params); @@ -2267,7 +2267,6 @@ static int snd_pcm_link(struct snd_pcm_substream *substream, int fd) { struct snd_pcm_file *pcm_file; struct snd_pcm_substream *substream1; - struct snd_pcm_group *group __free(kfree) = NULL; struct snd_pcm_group *target_group; bool nonatomic = substream->pcm->nonatomic; CLASS(fd, f)(fd); @@ -2283,7 +2282,8 @@ static int snd_pcm_link(struct snd_pcm_substream *substream, int fd) if (substream == substream1) return -EINVAL; - group = kzalloc(sizeof(*group), GFP_KERNEL); + struct snd_pcm_group *group __free(kfree) = + kzalloc(sizeof(*group), GFP_KERNEL); if (!group) return -ENOMEM; snd_pcm_group_init(group); @@ -3577,7 +3577,6 @@ static ssize_t snd_pcm_readv(struct kiocb *iocb, struct iov_iter *to) struct snd_pcm_runtime *runtime; snd_pcm_sframes_t result; unsigned long i; - void __user **bufs __free(kfree) = NULL; snd_pcm_uframes_t frames; const struct iovec *iov = iter_iov(to); @@ -3596,7 +3595,9 @@ static ssize_t snd_pcm_readv(struct kiocb *iocb, struct iov_iter *to) if (!frame_aligned(runtime, iov->iov_len)) return -EINVAL; frames = bytes_to_samples(runtime, iov->iov_len); - bufs = kmalloc_array(to->nr_segs, sizeof(void *), GFP_KERNEL); + + void __user **bufs __free(kfree) = + kmalloc_array(to->nr_segs, sizeof(void *), GFP_KERNEL); if (bufs == NULL) return -ENOMEM; for (i = 0; i < to->nr_segs; ++i) { @@ -3616,7 +3617,6 @@ static ssize_t snd_pcm_writev(struct kiocb *iocb, struct iov_iter *from) struct snd_pcm_runtime *runtime; snd_pcm_sframes_t result; unsigned long i; - void __user **bufs __free(kfree) = NULL; snd_pcm_uframes_t frames; const struct iovec *iov = iter_iov(from); @@ -3634,7 +3634,9 @@ static ssize_t snd_pcm_writev(struct kiocb *iocb, struct iov_iter *from) !frame_aligned(runtime, iov->iov_len)) return -EINVAL; frames = bytes_to_samples(runtime, iov->iov_len); - bufs = kmalloc_array(from->nr_segs, sizeof(void *), GFP_KERNEL); + + void __user **bufs __free(kfree) = + kmalloc_array(from->nr_segs, sizeof(void *), GFP_KERNEL); if (bufs == NULL) return -ENOMEM; for (i = 0; i < from->nr_segs; ++i) { @@ -4106,15 +4108,15 @@ static void snd_pcm_hw_convert_to_old_params(struct snd_pcm_hw_params_old *opara static int snd_pcm_hw_refine_old_user(struct snd_pcm_substream *substream, struct snd_pcm_hw_params_old __user * _oparams) { - struct snd_pcm_hw_params *params __free(kfree) = NULL; - struct snd_pcm_hw_params_old *oparams __free(kfree) = NULL; int err; - params = kmalloc(sizeof(*params), GFP_KERNEL); + struct snd_pcm_hw_params *params __free(kfree) = + kmalloc(sizeof(*params), GFP_KERNEL); if (!params) return -ENOMEM; - oparams = memdup_user(_oparams, sizeof(*oparams)); + struct snd_pcm_hw_params_old *oparams __free(kfree) = + memdup_user(_oparams, sizeof(*oparams)); if (IS_ERR(oparams)) return PTR_ERR(oparams); snd_pcm_hw_convert_from_old_params(params, oparams); @@ -4135,15 +4137,15 @@ static int snd_pcm_hw_refine_old_user(struct snd_pcm_substream *substream, static int snd_pcm_hw_params_old_user(struct snd_pcm_substream *substream, struct snd_pcm_hw_params_old __user * _oparams) { - struct snd_pcm_hw_params *params __free(kfree) = NULL; - struct snd_pcm_hw_params_old *oparams __free(kfree) = NULL; int err; - params = kmalloc(sizeof(*params), GFP_KERNEL); + struct snd_pcm_hw_params *params __free(kfree) = + kmalloc(sizeof(*params), GFP_KERNEL); if (!params) return -ENOMEM; - oparams = memdup_user(_oparams, sizeof(*oparams)); + struct snd_pcm_hw_params_old *oparams __free(kfree) = + memdup_user(_oparams, sizeof(*oparams)); if (IS_ERR(oparams)) return PTR_ERR(oparams); diff --git a/sound/core/vmaster.c b/sound/core/vmaster.c index c657659b236c4..76cc64245f5df 100644 --- a/sound/core/vmaster.c +++ b/sound/core/vmaster.c @@ -56,10 +56,10 @@ struct link_follower { static int follower_update(struct link_follower *follower) { - struct snd_ctl_elem_value *uctl __free(kfree) = NULL; int err, ch; + struct snd_ctl_elem_value *uctl __free(kfree) = + kzalloc(sizeof(*uctl), GFP_KERNEL); - uctl = kzalloc(sizeof(*uctl), GFP_KERNEL); if (!uctl) return -ENOMEM; uctl->id = follower->follower.id; @@ -74,7 +74,6 @@ static int follower_update(struct link_follower *follower) /* get the follower ctl info and save the initial values */ static int follower_init(struct link_follower *follower) { - struct snd_ctl_elem_info *uinfo __free(kfree) = NULL; int err; if (follower->info.count) { @@ -84,7 +83,8 @@ static int follower_init(struct link_follower *follower) return 0; } - uinfo = kmalloc(sizeof(*uinfo), GFP_KERNEL); + struct snd_ctl_elem_info *uinfo __free(kfree) = + kmalloc(sizeof(*uinfo), GFP_KERNEL); if (!uinfo) return -ENOMEM; uinfo->id = follower->follower.id; @@ -341,9 +341,9 @@ static int master_get(struct snd_kcontrol *kcontrol, static int sync_followers(struct link_master *master, int old_val, int new_val) { struct link_follower *follower; - struct snd_ctl_elem_value *uval __free(kfree) = NULL; + struct snd_ctl_elem_value *uval __free(kfree) = + kmalloc(sizeof(*uval), GFP_KERNEL); - uval = kmalloc(sizeof(*uval), GFP_KERNEL); if (!uval) return -ENOMEM; list_for_each_entry(follower, &master->followers, list) { diff --git a/sound/hda/codecs/conexant.c b/sound/hda/codecs/conexant.c index d6fba74603019..f71123a475464 100644 --- a/sound/hda/codecs/conexant.c +++ b/sound/hda/codecs/conexant.c @@ -299,6 +299,7 @@ enum { CXT_PINCFG_SWS_JS201D, CXT_PINCFG_TOP_SPEAKER, CXT_FIXUP_HP_A_U, + CXT_FIXUP_ACER_SWIFT_HP, }; /* for hda_fixup_thinkpad_acpi() */ @@ -1024,6 +1025,14 @@ static const struct hda_fixup cxt_fixups[] = { .type = HDA_FIXUP_FUNC, .v.func = cxt_fixup_hp_a_u, }, + [CXT_FIXUP_ACER_SWIFT_HP] = { + .type = HDA_FIXUP_PINS, + .v.pins = (const struct hda_pintbl[]) { + { 0x16, 0x0321403f }, /* Headphone */ + { 0x19, 0x40f001f0 }, /* Mic */ + { } + }, + }, }; static const struct hda_quirk cxt5045_fixups[] = { @@ -1073,6 +1082,7 @@ static const struct hda_quirk cxt5066_fixups[] = { SND_PCI_QUIRK(0x1025, 0x0543, "Acer Aspire One 522", CXT_FIXUP_STEREO_DMIC), SND_PCI_QUIRK(0x1025, 0x054c, "Acer Aspire 3830TG", CXT_FIXUP_ASPIRE_DMIC), SND_PCI_QUIRK(0x1025, 0x054f, "Acer Aspire 4830T", CXT_FIXUP_ASPIRE_DMIC), + SND_PCI_QUIRK(0x1025, 0x136d, "Acer Swift SF314", CXT_FIXUP_ACER_SWIFT_HP), SND_PCI_QUIRK(0x103c, 0x8079, "HP EliteBook 840 G3", CXT_FIXUP_HP_DOCK), SND_PCI_QUIRK(0x103c, 0x807C, "HP EliteBook 820 G3", CXT_FIXUP_HP_DOCK), SND_PCI_QUIRK(0x103c, 0x80FD, "HP ProBook 640 G2", CXT_FIXUP_HP_DOCK), @@ -1124,6 +1134,7 @@ static const struct hda_quirk cxt5066_fixups[] = { SND_PCI_QUIRK_VENDOR(0x17aa, "Thinkpad/Ideapad", CXT_FIXUP_LENOVO_XPAD_ACPI), SND_PCI_QUIRK(0x1c06, 0x2011, "Lemote A1004", CXT_PINCFG_LEMOTE_A1004), SND_PCI_QUIRK(0x1c06, 0x2012, "Lemote A1205", CXT_PINCFG_LEMOTE_A1205), + SND_PCI_QUIRK(0x1d05, 0x3012, "MECHREVO Wujie 15X Pro", CXT_FIXUP_HEADSET_MIC), HDA_CODEC_QUIRK(0x2782, 0x12c3, "Sirius Gen1", CXT_PINCFG_TOP_SPEAKER), HDA_CODEC_QUIRK(0x2782, 0x12c5, "Sirius Gen2", CXT_PINCFG_TOP_SPEAKER), {} diff --git a/sound/hda/codecs/hdmi/hdmi.c b/sound/hda/codecs/hdmi/hdmi.c index 111c9b5335afc..c2e3adc7b3c00 100644 --- a/sound/hda/codecs/hdmi/hdmi.c +++ b/sound/hda/codecs/hdmi/hdmi.c @@ -1557,6 +1557,7 @@ static const struct snd_pci_quirk force_connect_list[] = { SND_PCI_QUIRK(0x1043, 0x86ae, "ASUS", 1), /* Z170 PRO */ SND_PCI_QUIRK(0x1043, 0x86c7, "ASUS", 1), /* Z170M PLUS */ SND_PCI_QUIRK(0x1462, 0xec94, "MS-7C94", 1), + SND_PCI_QUIRK(0x1558, 0x14a1, "TUXEDO InfinityBook S 14 Gen6", 1), SND_PCI_QUIRK(0x8086, 0x2060, "Intel NUC5CPYB", 1), SND_PCI_QUIRK(0x8086, 0x2081, "Intel NUC 10", 1), {} diff --git a/sound/hda/codecs/realtek/alc269.c b/sound/hda/codecs/realtek/alc269.c index 2e9efafa732fe..f40e00a578d99 100644 --- a/sound/hda/codecs/realtek/alc269.c +++ b/sound/hda/codecs/realtek/alc269.c @@ -1551,6 +1551,22 @@ static void alc245_fixup_hp_mute_led_v1_coefbit(struct hda_codec *codec, } } +static void alc245_fixup_hp_mute_led_v2_coefbit(struct hda_codec *codec, + const struct hda_fixup *fix, + int action) +{ + struct alc_spec *spec = codec->spec; + + if (action == HDA_FIXUP_ACT_PRE_PROBE) { + spec->mute_led_polarity = 0; + spec->mute_led_coef.idx = 0x0b; + spec->mute_led_coef.mask = 1 << 3; + spec->mute_led_coef.on = 1 << 3; + spec->mute_led_coef.off = 0; + snd_hda_gen_add_mute_led_cdev(codec, coef_mute_led_set); + } +} + /* turn on/off mic-mute LED per capture hook by coef bit */ static int coef_micmute_led_set(struct led_classdev *led_cdev, enum led_brightness brightness) @@ -1630,6 +1646,13 @@ static void alc285_fixup_hp_spectre_x360_mute_led(struct hda_codec *codec, alc285_fixup_hp_gpio_micmute_led(codec, fix, action); } +static void alc245_fixup_hp_envy_x360_mute_led(struct hda_codec *codec, + const struct hda_fixup *fix, int action) +{ + alc245_fixup_hp_mute_led_v1_coefbit(codec, fix, action); + alc245_fixup_hp_gpio_led(codec, fix, action); +} + static void alc236_fixup_hp_mute_led(struct hda_codec *codec, const struct hda_fixup *fix, int action) { @@ -1812,6 +1835,163 @@ static void alc298_samsung_v2_init_amps(struct hda_codec *codec, spec->gen.pcm_playback_hook = alc298_samsung_v2_playback_hook; } +/* LG Gram Style 14: program vendor coef sequence used by HDA-verb workaround */ +struct alc298_lg_gram_style_seq { + unsigned short verb; + unsigned short idx; + unsigned short val; +}; + +static void alc298_lg_gram_style_coef_write(struct hda_codec *codec, + unsigned int verb, + unsigned int idx, + unsigned int val) +{ + snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_COEF_INDEX, 0x23); + snd_hda_codec_write(codec, 0x20, 0, verb, idx); + snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_PROC_COEF, 0x00); + snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_PROC_COEF, val); + snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_PROC_COEF, 0xb011); +} + +static void alc298_lg_gram_style_run_seq(struct hda_codec *codec, + const struct alc298_lg_gram_style_seq *seq, + int seq_size) +{ + int i; + + for (i = 0; i < seq_size; i++) + alc298_lg_gram_style_coef_write(codec, seq[i].verb, + seq[i].idx, seq[i].val); +} + +/* Coef sequences derived from the HDA-verb workaround for this model. */ +static const struct alc298_lg_gram_style_seq alc298_lg_gram_style_preinit_seq[] = { + { 0x420, 0x00, 0x01 }, +}; + +static const struct alc298_lg_gram_style_seq alc298_lg_gram_style_disable_seq[] = { + { 0x423, 0xff, 0x00 }, + { 0x420, 0x3a, 0x80 }, +}; + +static const struct alc298_lg_gram_style_seq alc298_lg_gram_style_enable_seq[] = { + { 0x420, 0x3a, 0x81 }, + { 0x423, 0xff, 0x01 }, +}; + +static const struct alc298_lg_gram_style_seq alc298_lg_gram_style_init_seq_38[] = { + { 0x423, 0xe1, 0x00 }, { 0x420, 0x12, 0x6f }, { 0x420, 0x14, 0x00 }, + { 0x420, 0x1b, 0x01 }, { 0x420, 0x1d, 0x01 }, { 0x420, 0x1f, 0xfe }, + { 0x420, 0x21, 0x00 }, { 0x420, 0x22, 0x10 }, { 0x420, 0x3d, 0x05 }, + { 0x420, 0x3f, 0x03 }, { 0x420, 0x50, 0x2c }, { 0x420, 0x76, 0x0e }, + { 0x420, 0x7c, 0x4a }, { 0x420, 0x81, 0x03 }, { 0x423, 0x99, 0x03 }, + { 0x423, 0xa4, 0xb5 }, { 0x423, 0xa5, 0x01 }, { 0x423, 0xba, 0x94 }, +}; + +static const struct alc298_lg_gram_style_seq alc298_lg_gram_style_init_seq_39[] = { + { 0x423, 0xe1, 0x00 }, { 0x420, 0x12, 0x6f }, { 0x420, 0x14, 0x00 }, + { 0x420, 0x1b, 0x02 }, { 0x420, 0x1d, 0x02 }, { 0x420, 0x1f, 0xfd }, + { 0x420, 0x21, 0x01 }, { 0x420, 0x22, 0x10 }, { 0x420, 0x3d, 0x05 }, + { 0x420, 0x3f, 0x03 }, { 0x420, 0x50, 0x2c }, { 0x420, 0x76, 0x0e }, + { 0x420, 0x7c, 0x4a }, { 0x420, 0x81, 0x03 }, { 0x423, 0x99, 0x03 }, + { 0x423, 0xa4, 0xb5 }, { 0x423, 0xa5, 0x01 }, { 0x423, 0xba, 0x94 }, +}; + +static const struct alc298_lg_gram_style_seq alc298_lg_gram_style_init_seq_3c[] = { + { 0x423, 0xe1, 0x00 }, { 0x420, 0x12, 0x6f }, { 0x420, 0x14, 0x00 }, + { 0x420, 0x1b, 0x01 }, { 0x420, 0x1d, 0x01 }, { 0x420, 0x1f, 0xfe }, + { 0x420, 0x21, 0x00 }, { 0x420, 0x22, 0x10 }, { 0x420, 0x3d, 0x05 }, + { 0x420, 0x3f, 0x03 }, { 0x420, 0x50, 0x2c }, { 0x420, 0x76, 0x0e }, + { 0x420, 0x7c, 0x4a }, { 0x420, 0x81, 0x03 }, { 0x423, 0xba, 0x8d }, +}; + +static const struct alc298_lg_gram_style_seq alc298_lg_gram_style_init_seq_3d[] = { + { 0x423, 0xe1, 0x00 }, { 0x420, 0x12, 0x6f }, { 0x420, 0x14, 0x00 }, + { 0x420, 0x1b, 0x02 }, { 0x420, 0x1d, 0x02 }, { 0x420, 0x1f, 0xfd }, + { 0x420, 0x21, 0x01 }, { 0x420, 0x22, 0x10 }, { 0x420, 0x3d, 0x05 }, + { 0x420, 0x3f, 0x03 }, { 0x420, 0x50, 0x2c }, { 0x420, 0x76, 0x0e }, + { 0x420, 0x7c, 0x4a }, { 0x420, 0x81, 0x03 }, { 0x423, 0xba, 0x8d }, +}; + +struct alc298_lg_gram_style_amp_desc { + unsigned char nid; + const struct alc298_lg_gram_style_seq *init_seq; + int init_seq_size; +}; + +static const struct alc298_lg_gram_style_amp_desc alc298_lg_gram_style_amps[] = { + { 0x38, alc298_lg_gram_style_init_seq_38, + ARRAY_SIZE(alc298_lg_gram_style_init_seq_38) }, + { 0x39, alc298_lg_gram_style_init_seq_39, + ARRAY_SIZE(alc298_lg_gram_style_init_seq_39) }, + { 0x3c, alc298_lg_gram_style_init_seq_3c, + ARRAY_SIZE(alc298_lg_gram_style_init_seq_3c) }, + { 0x3d, alc298_lg_gram_style_init_seq_3d, + ARRAY_SIZE(alc298_lg_gram_style_init_seq_3d) }, +}; + +static void alc298_lg_gram_style_enable_amps(struct hda_codec *codec) +{ + struct alc_spec *spec = codec->spec; + int i; + + for (i = 0; i < spec->num_speaker_amps; i++) { + alc_write_coef_idx(codec, 0x22, alc298_lg_gram_style_amps[i].nid); + alc298_lg_gram_style_run_seq(codec, + alc298_lg_gram_style_enable_seq, + ARRAY_SIZE(alc298_lg_gram_style_enable_seq)); + } +} + +static void alc298_lg_gram_style_disable_amps(struct hda_codec *codec) +{ + struct alc_spec *spec = codec->spec; + int i; + + for (i = 0; i < spec->num_speaker_amps; i++) { + alc_write_coef_idx(codec, 0x22, alc298_lg_gram_style_amps[i].nid); + alc298_lg_gram_style_run_seq(codec, + alc298_lg_gram_style_disable_seq, + ARRAY_SIZE(alc298_lg_gram_style_disable_seq)); + } +} + +static void alc298_lg_gram_style_playback_hook(struct hda_pcm_stream *hinfo, + struct hda_codec *codec, + struct snd_pcm_substream *substream, + int action) +{ + if (action == HDA_GEN_PCM_ACT_OPEN) + alc298_lg_gram_style_enable_amps(codec); + if (action == HDA_GEN_PCM_ACT_CLOSE) + alc298_lg_gram_style_disable_amps(codec); +} + +static void alc298_lg_gram_style_init_amps(struct hda_codec *codec) +{ + struct alc_spec *spec = codec->spec; + int i; + + spec->num_speaker_amps = ARRAY_SIZE(alc298_lg_gram_style_amps); + + for (i = 0; i < spec->num_speaker_amps; i++) { + alc_write_coef_idx(codec, 0x22, alc298_lg_gram_style_amps[i].nid); + alc298_lg_gram_style_run_seq(codec, + alc298_lg_gram_style_preinit_seq, + ARRAY_SIZE(alc298_lg_gram_style_preinit_seq)); + alc298_lg_gram_style_run_seq(codec, + alc298_lg_gram_style_disable_seq, + ARRAY_SIZE(alc298_lg_gram_style_disable_seq)); + alc298_lg_gram_style_run_seq(codec, + alc298_lg_gram_style_amps[i].init_seq, + alc298_lg_gram_style_amps[i].init_seq_size); + alc_write_coef_idx(codec, 0x89, 0x0); + } + + spec->gen.pcm_playback_hook = alc298_lg_gram_style_playback_hook; +} + static void alc298_fixup_samsung_amp_v2_2_amps(struct hda_codec *codec, const struct hda_fixup *fix, int action) { @@ -1826,6 +2006,13 @@ static void alc298_fixup_samsung_amp_v2_4_amps(struct hda_codec *codec, alc298_samsung_v2_init_amps(codec, 4); } +static void alc298_fixup_lg_gram_style_14(struct hda_codec *codec, + const struct hda_fixup *fix, int action) +{ + if (action == HDA_FIXUP_ACT_PROBE) + alc298_lg_gram_style_init_amps(codec); +} + static void gpio2_mic_hotkey_event(struct hda_codec *codec, struct hda_jack_callback *event) { @@ -3371,11 +3558,22 @@ static void alc287_alc1318_playback_pcm_hook(struct hda_pcm_stream *hinfo, struct snd_pcm_substream *substream, int action) { + static const struct coef_fw dis_coefs[] = { + WRITE_COEF(0x24, 0x0013), WRITE_COEF(0x25, 0x0000), WRITE_COEF(0x26, 0xC203), + WRITE_COEF(0x28, 0x0004), WRITE_COEF(0x29, 0xb023), + }; /* Disable AMP silence detection */ + static const struct coef_fw en_coefs[] = { + WRITE_COEF(0x24, 0x0013), WRITE_COEF(0x25, 0x0000), WRITE_COEF(0x26, 0xC203), + WRITE_COEF(0x28, 0x0084), WRITE_COEF(0x29, 0xb023), + }; /* Enable AMP silence detection */ + switch (action) { case HDA_GEN_PCM_ACT_OPEN: + alc_process_coef_fw(codec, dis_coefs); alc_write_coefex_idx(codec, 0x5a, 0x00, 0x954f); /* write gpio3 to high */ break; case HDA_GEN_PCM_ACT_CLOSE: + alc_process_coef_fw(codec, en_coefs); alc_write_coefex_idx(codec, 0x5a, 0x00, 0x554f); /* write gpio3 as default value */ break; } @@ -3579,6 +3777,7 @@ enum { ALC294_FIXUP_ASUS_MIC, ALC294_FIXUP_ASUS_HEADSET_MIC, ALC294_FIXUP_ASUS_I2C_HEADSET_MIC, + ALC294_FIXUP_ASUS_SPI_HEADSET_MIC, ALC294_FIXUP_ASUS_SPK, ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE, ALC285_FIXUP_LENOVO_PC_BEEP_IN_NOISE, @@ -3619,6 +3818,7 @@ enum { ALC285_FIXUP_HP_GPIO_LED, ALC285_FIXUP_HP_MUTE_LED, ALC285_FIXUP_HP_SPECTRE_X360_MUTE_LED, + ALC245_FIXUP_HP_ENVY_X360_MUTE_LED, ALC285_FIXUP_HP_BEEP_MICMUTE_LED, ALC236_FIXUP_HP_MUTE_LED_COEFBIT2, ALC236_FIXUP_HP_GPIO_LED, @@ -3628,6 +3828,7 @@ enum { ALC298_FIXUP_SAMSUNG_AMP, ALC298_FIXUP_SAMSUNG_AMP_V2_2_AMPS, ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS, + ALC298_FIXUP_LG_GRAM_STYLE_14, ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET, ALC256_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET, ALC295_FIXUP_ASUS_MIC_NO_PRESENCE, @@ -3708,6 +3909,7 @@ enum { ALC287_FIXUP_YOGA7_14ARB7_I2C, ALC245_FIXUP_HP_MUTE_LED_COEFBIT, ALC245_FIXUP_HP_MUTE_LED_V1_COEFBIT, + ALC245_FIXUP_HP_MUTE_LED_V2_COEFBIT, ALC245_FIXUP_HP_X360_MUTE_LEDS, ALC287_FIXUP_THINKPAD_I2S_SPK, ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD, @@ -4920,6 +5122,15 @@ static const struct hda_fixup alc269_fixups[] = { .chained = true, .chain_id = ALC287_FIXUP_CS35L41_I2C_2 }, + [ALC294_FIXUP_ASUS_SPI_HEADSET_MIC] = { + .type = HDA_FIXUP_PINS, + .v.pins = (const struct hda_pintbl[]) { + { 0x19, 0x04a11020 }, /* use as headset mic */ + { } + }, + .chained = true, + .chain_id = ALC245_FIXUP_CS35L41_SPI_2 + }, [ALC294_FIXUP_ASUS_SPK] = { .type = HDA_FIXUP_VERBS, .v.verbs = (const struct hda_verb[]) { @@ -5267,6 +5478,10 @@ static const struct hda_fixup alc269_fixups[] = { .type = HDA_FIXUP_FUNC, .v.func = alc285_fixup_hp_spectre_x360_mute_led, }, + [ALC245_FIXUP_HP_ENVY_X360_MUTE_LED] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc245_fixup_hp_envy_x360_mute_led, + }, [ALC285_FIXUP_HP_BEEP_MICMUTE_LED] = { .type = HDA_FIXUP_FUNC, .v.func = alc285_fixup_hp_beep, @@ -5313,6 +5528,10 @@ static const struct hda_fixup alc269_fixups[] = { .type = HDA_FIXUP_FUNC, .v.func = alc298_fixup_samsung_amp_v2_4_amps }, + [ALC298_FIXUP_LG_GRAM_STYLE_14] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc298_fixup_lg_gram_style_14 + }, [ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET] = { .type = HDA_FIXUP_VERBS, .v.verbs = (const struct hda_verb[]) { @@ -6036,6 +6255,10 @@ static const struct hda_fixup alc269_fixups[] = { .type = HDA_FIXUP_FUNC, .v.func = alc245_fixup_hp_mute_led_v1_coefbit, }, + [ALC245_FIXUP_HP_MUTE_LED_V2_COEFBIT] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc245_fixup_hp_mute_led_v2_coefbit, + }, [ALC245_FIXUP_HP_X360_MUTE_LEDS] = { .type = HDA_FIXUP_FUNC, .v.func = alc245_fixup_hp_mute_led_coefbit, @@ -6228,6 +6451,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1025, 0x079b, "Acer Aspire V5-573G", ALC282_FIXUP_ASPIRE_V5_PINS), SND_PCI_QUIRK(0x1025, 0x080d, "Acer Aspire V5-122P", ALC269_FIXUP_ASPIRE_HEADSET_MIC), SND_PCI_QUIRK(0x1025, 0x0840, "Acer Aspire E1", ALC269VB_FIXUP_ASPIRE_E1_COEF), + SND_PCI_QUIRK(0x1025, 0x0943, "Acer Aspire V3-572G", ALC269_FIXUP_ASPIRE_HEADSET_MIC), SND_PCI_QUIRK(0x1025, 0x100c, "Acer Aspire E5-574G", ALC255_FIXUP_ACER_LIMIT_INT_MIC_BOOST), SND_PCI_QUIRK(0x1025, 0x101c, "Acer Veriton N2510G", ALC269_FIXUP_LIFEBOOK), SND_PCI_QUIRK(0x1025, 0x102b, "Acer Aspire C24-860", ALC286_FIXUP_ACER_AIO_MIC_NO_PRESENCE), @@ -6260,6 +6484,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1025, 0x1430, "Acer TravelMate B311R-31", ALC256_FIXUP_ACER_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1025, 0x1466, "Acer Aspire A515-56", ALC255_FIXUP_ACER_HEADPHONE_AND_MIC), SND_PCI_QUIRK(0x1025, 0x1534, "Acer Predator PH315-54", ALC255_FIXUP_ACER_MIC_NO_PRESENCE), + SND_PCI_QUIRK(0x1025, 0x1539, "Acer Nitro 5 AN515-57", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1025, 0x159c, "Acer Nitro 5 AN515-58", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1025, 0x1597, "Acer Nitro 5 AN517-55", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1025, 0x169a, "Acer Swift SFG16", ALC256_FIXUP_ACER_SFG16_MICMUTE_LED), @@ -6506,8 +6731,11 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8895, "HP EliteBook 855 G8 Notebook PC", ALC285_FIXUP_HP_SPEAKERS_MICMUTE_LED), SND_PCI_QUIRK(0x103c, 0x8896, "HP EliteBook 855 G8 Notebook PC", ALC285_FIXUP_HP_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x8898, "HP EliteBook 845 G8 Notebook PC", ALC285_FIXUP_HP_LIMIT_INT_MIC_BOOST), + SND_PCI_QUIRK(0x103c, 0x88b3, "HP ENVY x360 Convertible 15-es0xxx", ALC245_FIXUP_HP_ENVY_X360_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x88d0, "HP Pavilion 15-eh1xxx (mainboard 88D0)", ALC287_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x88d1, "HP Pavilion 15-eh1xxx (mainboard 88D1)", ALC245_FIXUP_HP_MUTE_LED_V1_COEFBIT), SND_PCI_QUIRK(0x103c, 0x88dd, "HP Pavilion 15z-ec200", ALC285_FIXUP_HP_MUTE_LED), + SND_PCI_QUIRK(0x103c, 0x88eb, "HP Victus 16-e0xxx", ALC245_FIXUP_HP_MUTE_LED_V2_COEFBIT), SND_PCI_QUIRK(0x103c, 0x8902, "HP OMEN 16", ALC285_FIXUP_HP_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x890e, "HP 255 G8 Notebook PC", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x8919, "HP Pavilion Aero Laptop 13-be0xxx", ALC287_FIXUP_HP_GPIO_LED), @@ -6811,7 +7039,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x19ce, "ASUS B9450FA", ALC294_FIXUP_ASUS_HPE), SND_PCI_QUIRK(0x1043, 0x19e1, "ASUS UX581LV", ALC295_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1043, 0x1a13, "Asus G73Jw", ALC269_FIXUP_ASUS_G73JW), - SND_PCI_QUIRK(0x1043, 0x1a63, "ASUS UX3405MA", ALC245_FIXUP_CS35L41_SPI_2), + SND_PCI_QUIRK(0x1043, 0x1a63, "ASUS UX3405MA", ALC294_FIXUP_ASUS_SPI_HEADSET_MIC), SND_PCI_QUIRK(0x1043, 0x1a83, "ASUS UM5302LA", ALC294_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x1043, 0x1a8e, "ASUS G712LWS", ALC294_FIXUP_LENOVO_MIC_LOCATION), SND_PCI_QUIRK(0x1043, 0x1a8f, "ASUS UX582ZS", ALC245_FIXUP_CS35L41_SPI_2), @@ -6922,7 +7150,8 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x144d, 0xc109, "Samsung Ativ book 9 (NP900X3G)", ALC269_FIXUP_INV_DMIC), SND_PCI_QUIRK(0x144d, 0xc169, "Samsung Notebook 9 Pen (NP930SBE-K01US)", ALC298_FIXUP_SAMSUNG_AMP), SND_PCI_QUIRK(0x144d, 0xc176, "Samsung Notebook 9 Pro (NP930MBE-K04US)", ALC298_FIXUP_SAMSUNG_AMP), - SND_PCI_QUIRK(0x144d, 0xc189, "Samsung Galaxy Flex Book (NT950QCG-X716)", ALC298_FIXUP_SAMSUNG_AMP), + SND_PCI_QUIRK(0x144d, 0xc188, "Samsung Galaxy Book Flex (NT950QCT-A38A)", ALC298_FIXUP_SAMSUNG_AMP), + SND_PCI_QUIRK(0x144d, 0xc189, "Samsung Galaxy Book Flex (NT950QCG-X716)", ALC298_FIXUP_SAMSUNG_AMP), SND_PCI_QUIRK(0x144d, 0xc18a, "Samsung Galaxy Book Ion (NP930XCJ-K01US)", ALC298_FIXUP_SAMSUNG_AMP), SND_PCI_QUIRK(0x144d, 0xc1a3, "Samsung Galaxy Book Pro (NP935XDB-KC1SE)", ALC298_FIXUP_SAMSUNG_AMP), SND_PCI_QUIRK(0x144d, 0xc1a4, "Samsung Galaxy Book Pro 360 (NT935QBD)", ALC298_FIXUP_SAMSUNG_AMP), @@ -6939,7 +7168,9 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x144d, 0xc872, "Samsung Galaxy Book2 Pro (NP950XEE)", ALC298_FIXUP_SAMSUNG_AMP_V2_2_AMPS), SND_PCI_QUIRK(0x144d, 0xc886, "Samsung Galaxy Book3 Pro (NP964XFG)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), SND_PCI_QUIRK(0x144d, 0xc1ca, "Samsung Galaxy Book3 Pro 360 (NP960QFG)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), + SND_PCI_QUIRK(0x144d, 0xc1cb, "Samsung Galaxy Book3 Pro 360 (NP965QFG)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), SND_PCI_QUIRK(0x144d, 0xc1cc, "Samsung Galaxy Book3 Ultra (NT960XFH)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), + SND_PCI_QUIRK(0x1458, 0x900e, "Gigabyte G5 KF5 (2023)", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1458, 0xfa53, "Gigabyte BXBT-2807", ALC283_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1462, 0xb120, "MSI Cubi MS-B120", ALC283_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1462, 0xb171, "Cubi N 8GL (MS-B171)", ALC283_FIXUP_HEADSET_MIC), @@ -7208,6 +7439,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1854, 0x0488, "LG gram 16 (16Z90R)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), SND_PCI_QUIRK(0x1854, 0x0489, "LG gram 16 (16Z90R-A)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), SND_PCI_QUIRK(0x1854, 0x048a, "LG gram 17 (17ZD90R)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), + SND_PCI_QUIRK(0x1854, 0x0490, "LG Gram Style 14 (14Z90RS)", ALC298_FIXUP_LG_GRAM_STYLE_14), SND_PCI_QUIRK(0x19e5, 0x3204, "Huawei MACH-WX9", ALC256_FIXUP_HUAWEI_MACH_WX9_PINS), SND_PCI_QUIRK(0x19e5, 0x320f, "Huawei WRT-WX9 ", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x19e5, 0x3212, "Huawei KLV-WX9 ", ALC256_FIXUP_ACER_HEADSET_MIC), @@ -7243,6 +7475,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1ee7, 0x2078, "HONOR BRB-X M1010", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1f66, 0x0105, "Ayaneo Portable Game Player", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x2014, 0x800a, "Positivo ARN50", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), + SND_PCI_QUIRK(0x2039, 0x0001, "Inspur S14-G1", ALC295_FIXUP_CHROME_BOOK), SND_PCI_QUIRK(0x2782, 0x0214, "VAIO VJFE-CL", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), SND_PCI_QUIRK(0x2782, 0x0228, "Infinix ZERO BOOK 13", ALC269VB_FIXUP_INFINIX_ZERO_BOOK_13), SND_PCI_QUIRK(0x2782, 0x0232, "CHUWI CoreBook XPro", ALC269VB_FIXUP_CHUWI_COREBOOK_XPRO), diff --git a/sound/hda/codecs/side-codecs/cs35l56_hda.c b/sound/hda/codecs/side-codecs/cs35l56_hda.c index 5bb1c4ebeaf3c..acbacd0766064 100644 --- a/sound/hda/codecs/side-codecs/cs35l56_hda.c +++ b/sound/hda/codecs/side-codecs/cs35l56_hda.c @@ -249,7 +249,7 @@ static int cs35l56_hda_posture_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol) { struct cs35l56_hda *cs35l56 = snd_kcontrol_chip(kcontrol); - unsigned long pos = ucontrol->value.integer.value[0]; + long pos = ucontrol->value.integer.value[0]; bool changed; int ret; diff --git a/sound/hda/codecs/side-codecs/tas2781_hda_spi.c b/sound/hda/codecs/side-codecs/tas2781_hda_spi.c index b9a55672bf15d..488e35dac9524 100644 --- a/sound/hda/codecs/side-codecs/tas2781_hda_spi.c +++ b/sound/hda/codecs/side-codecs/tas2781_hda_spi.c @@ -634,7 +634,7 @@ static void tasdev_fw_ready(const struct firmware *fmw, void *context) struct tasdevice_priv *tas_priv = context; struct tas2781_hda *tas_hda = dev_get_drvdata(tas_priv->dev); struct hda_codec *codec = tas_priv->codec; - int ret, val; + int ret; pm_runtime_get_sync(tas_priv->dev); guard(mutex)(&tas_priv->codec_lock); @@ -673,20 +673,14 @@ static void tasdev_fw_ready(const struct firmware *fmw, void *context) tas_priv->rcabin.profile_cfg_id = 0; tas_priv->fw_state = TASDEVICE_DSP_FW_ALL_OK; - ret = tasdevice_spi_dev_read(tas_priv, tas_priv->index, - TAS2781_REG_CLK_CONFIG, &val); - if (ret < 0) - goto out; - if (val == TAS2781_REG_CLK_CONFIG_RESET) { - ret = tasdevice_prmg_load(tas_priv, 0); - if (ret < 0) { - dev_err(tas_priv->dev, "FW download failed = %d\n", - ret); - goto out; - } - tas_priv->fw_state = TASDEVICE_DSP_FW_ALL_OK; + ret = tasdevice_prmg_load(tas_priv, 0); + if (ret < 0) { + dev_err(tas_priv->dev, "FW download failed = %d\n", ret); + goto out; } + tas_priv->fw_state = TASDEVICE_DSP_FW_ALL_OK; + if (tas_priv->fmw->nr_programs > 0) tas_priv->tasdevice[tas_priv->index].cur_prog = 0; if (tas_priv->fmw->nr_configurations > 0) diff --git a/sound/hda/controllers/intel.c b/sound/hda/controllers/intel.c index a19258c95886c..9306e7a31f02b 100644 --- a/sound/hda/controllers/intel.c +++ b/sound/hda/controllers/intel.c @@ -1751,6 +1751,8 @@ static int default_bdl_pos_adj(struct azx *chip) return 1; case AZX_DRIVER_ZHAOXINHDMI: return 128; + case AZX_DRIVER_NVIDIA: + return 64; default: return 32; } diff --git a/sound/soc/amd/acp/acp-sdw-legacy-mach.c b/sound/soc/amd/acp/acp-sdw-legacy-mach.c index 5a3cfedacbafd..86c534d827448 100644 --- a/sound/soc/amd/acp/acp-sdw-legacy-mach.c +++ b/sound/soc/amd/acp/acp-sdw-legacy-mach.c @@ -95,6 +95,22 @@ static const struct dmi_system_id soc_sdw_quirk_table[] = { }, .driver_data = (void *)(ASOC_SDW_CODEC_SPKR), }, + { + .callback = soc_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_EXACT_MATCH(DMI_PRODUCT_SKU, "21YW"), + }, + .driver_data = (void *)(ASOC_SDW_CODEC_SPKR), + }, + { + .callback = soc_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_EXACT_MATCH(DMI_PRODUCT_SKU, "21YX"), + }, + .driver_data = (void *)(ASOC_SDW_CODEC_SPKR), + }, {} }; diff --git a/sound/soc/amd/yc/acp6x-mach.c b/sound/soc/amd/yc/acp6x-mach.c index c4a4a06528b45..f1a63475100d1 100644 --- a/sound/soc/amd/yc/acp6x-mach.c +++ b/sound/soc/amd/yc/acp6x-mach.c @@ -542,6 +542,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "15NBC1011"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "ASUS EXPERTBOOK PM1503CDA"), + } + }, { .driver_data = &acp6x_card, .matches = { @@ -633,6 +640,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_BOARD_NAME, "8BD6"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "HP"), + DMI_MATCH(DMI_BOARD_NAME, "8EE4"), + } + }, { .driver_data = &acp6x_card, .matches = { @@ -682,7 +696,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_BOARD_NAME, "XyloD5_RBU"), } }, - + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "Vivobook_ASUSLaptop M6501RR_M6501RR"), + } + }, {} }; diff --git a/sound/soc/codecs/aw88261.c b/sound/soc/codecs/aw88261.c index de11ae8dd9d9f..124c0a58d08bd 100644 --- a/sound/soc/codecs/aw88261.c +++ b/sound/soc/codecs/aw88261.c @@ -423,9 +423,10 @@ static int aw88261_dev_reg_update(struct aw88261 *aw88261, if (ret) break; + /* keep all three bits from current hw status */ read_val &= (~AW88261_AMPPD_MASK) | (~AW88261_PWDN_MASK) | (~AW88261_HMUTE_MASK); - reg_val &= (AW88261_AMPPD_MASK | AW88261_PWDN_MASK | AW88261_HMUTE_MASK); + reg_val &= (AW88261_AMPPD_MASK & AW88261_PWDN_MASK & AW88261_HMUTE_MASK); reg_val |= read_val; /* enable uls hmute */ diff --git a/sound/soc/codecs/cs35l45.c b/sound/soc/codecs/cs35l45.c index d4dcdf37bb709..9b1eff4e9bb71 100644 --- a/sound/soc/codecs/cs35l45.c +++ b/sound/soc/codecs/cs35l45.c @@ -455,7 +455,7 @@ static const struct snd_soc_dapm_widget cs35l45_dapm_widgets[] = { SND_SOC_DAPM_AIF_OUT("ASP_TX2", NULL, 1, CS35L45_ASP_ENABLES1, CS35L45_ASP_TX2_EN_SHIFT, 0), SND_SOC_DAPM_AIF_OUT("ASP_TX3", NULL, 2, CS35L45_ASP_ENABLES1, CS35L45_ASP_TX3_EN_SHIFT, 0), SND_SOC_DAPM_AIF_OUT("ASP_TX4", NULL, 3, CS35L45_ASP_ENABLES1, CS35L45_ASP_TX4_EN_SHIFT, 0), - SND_SOC_DAPM_AIF_OUT("ASP_TX5", NULL, 3, CS35L45_ASP_ENABLES1, CS35L45_ASP_TX5_EN_SHIFT, 0), + SND_SOC_DAPM_AIF_OUT("ASP_TX5", NULL, 4, CS35L45_ASP_ENABLES1, CS35L45_ASP_TX5_EN_SHIFT, 0), SND_SOC_DAPM_MUX("ASP_TX1 Source", SND_SOC_NOPM, 0, 0, &cs35l45_asp_muxes[0]), SND_SOC_DAPM_MUX("ASP_TX2 Source", SND_SOC_NOPM, 0, 0, &cs35l45_asp_muxes[1]), diff --git a/sound/soc/codecs/cs42l43-jack.c b/sound/soc/codecs/cs42l43-jack.c index 867e23d4fb8d8..744488f371ea4 100644 --- a/sound/soc/codecs/cs42l43-jack.c +++ b/sound/soc/codecs/cs42l43-jack.c @@ -496,7 +496,23 @@ void cs42l43_bias_sense_timeout(struct work_struct *work) pm_runtime_put_autosuspend(priv->dev); } -static void cs42l43_start_load_detect(struct cs42l43_codec *priv) +static const struct reg_sequence cs42l43_3pole_patch[] = { + { 0x4000, 0x00000055 }, + { 0x4000, 0x000000AA }, + { 0x17420, 0x8500F300 }, + { 0x17424, 0x36003E00 }, + { 0x4000, 0x00000000 }, +}; + +static const struct reg_sequence cs42l43_4pole_patch[] = { + { 0x4000, 0x00000055 }, + { 0x4000, 0x000000AA }, + { 0x17420, 0x7800E600 }, + { 0x17424, 0x36003800 }, + { 0x4000, 0x00000000 }, +}; + +static void cs42l43_start_load_detect(struct cs42l43_codec *priv, bool mic) { struct cs42l43 *cs42l43 = priv->core; @@ -520,6 +536,15 @@ static void cs42l43_start_load_detect(struct cs42l43_codec *priv) dev_err(priv->dev, "Load detect HP power down timed out\n"); } + if (mic) + regmap_multi_reg_write_bypassed(cs42l43->regmap, + cs42l43_4pole_patch, + ARRAY_SIZE(cs42l43_4pole_patch)); + else + regmap_multi_reg_write_bypassed(cs42l43->regmap, + cs42l43_3pole_patch, + ARRAY_SIZE(cs42l43_3pole_patch)); + regmap_update_bits(cs42l43->regmap, CS42L43_BLOCK_EN3, CS42L43_ADC1_EN_MASK | CS42L43_ADC2_EN_MASK, 0); regmap_update_bits(cs42l43->regmap, CS42L43_DACCNFG2, CS42L43_HP_HPF_EN_MASK, 0); @@ -598,7 +623,7 @@ static int cs42l43_run_load_detect(struct cs42l43_codec *priv, bool mic) reinit_completion(&priv->load_detect); - cs42l43_start_load_detect(priv); + cs42l43_start_load_detect(priv, mic); time_left = wait_for_completion_timeout(&priv->load_detect, msecs_to_jiffies(CS42L43_LOAD_TIMEOUT_MS)); cs42l43_stop_load_detect(priv); @@ -622,11 +647,11 @@ static int cs42l43_run_load_detect(struct cs42l43_codec *priv, bool mic) } switch (val & CS42L43_AMP3_RES_DET_MASK) { - case 0x0: // low impedance - case 0x1: // high impedance + case 0x0: // < 22 Ohm impedance + case 0x1: // < 150 Ohm impedance + case 0x2: // < 1000 Ohm impedance return CS42L43_JACK_HEADPHONE; - case 0x2: // lineout - case 0x3: // Open circuit + case 0x3: // > 1000 Ohm impedance return CS42L43_JACK_LINEOUT; default: return -EINVAL; diff --git a/sound/soc/codecs/es8328.c b/sound/soc/codecs/es8328.c index 76159c45e6b52..c0d7ce64b2d96 100644 --- a/sound/soc/codecs/es8328.c +++ b/sound/soc/codecs/es8328.c @@ -756,17 +756,23 @@ static int es8328_resume(struct snd_soc_component *component) es8328->supplies); if (ret) { dev_err(component->dev, "unable to enable regulators\n"); - return ret; + goto err_clk; } regcache_mark_dirty(regmap); ret = regcache_sync(regmap); if (ret) { dev_err(component->dev, "unable to sync regcache\n"); - return ret; + goto err_regulators; } return 0; + +err_regulators: + regulator_bulk_disable(ARRAY_SIZE(es8328->supplies), es8328->supplies); +err_clk: + clk_disable_unprepare(es8328->clk); + return ret; } static int es8328_component_probe(struct snd_soc_component *component) diff --git a/sound/soc/codecs/max98390.c b/sound/soc/codecs/max98390.c index a8a282ff9fc5a..b132ef8c1d2c8 100644 --- a/sound/soc/codecs/max98390.c +++ b/sound/soc/codecs/max98390.c @@ -1073,6 +1073,9 @@ static int max98390_i2c_probe(struct i2c_client *i2c) reset_gpio = devm_gpiod_get_optional(&i2c->dev, "reset", GPIOD_OUT_HIGH); + if (IS_ERR(reset_gpio)) + return dev_err_probe(&i2c->dev, PTR_ERR(reset_gpio), + "Failed to get reset gpio\n"); /* Power on device */ if (reset_gpio) { diff --git a/sound/soc/codecs/nau8821.c b/sound/soc/codecs/nau8821.c index 4fa9a785513e5..dfb9630bffe29 100644 --- a/sound/soc/codecs/nau8821.c +++ b/sound/soc/codecs/nau8821.c @@ -1661,8 +1661,13 @@ int nau8821_enable_jack_detect(struct snd_soc_component *component, int ret; nau8821->jack = jack; + + if (nau8821->jdet_active) + return 0; + /* Initiate jack detection work queue */ INIT_DELAYED_WORK(&nau8821->jdet_work, nau8821_jdet_work); + nau8821->jdet_active = true; ret = devm_request_threaded_irq(nau8821->dev, nau8821->irq, NULL, nau8821_interrupt, IRQF_TRIGGER_LOW | IRQF_ONESHOT, diff --git a/sound/soc/codecs/nau8821.h b/sound/soc/codecs/nau8821.h index 88602923780d8..f9d7cd8cbd211 100644 --- a/sound/soc/codecs/nau8821.h +++ b/sound/soc/codecs/nau8821.h @@ -562,6 +562,7 @@ struct nau8821 { struct snd_soc_dapm_context *dapm; struct snd_soc_jack *jack; struct delayed_work jdet_work; + bool jdet_active; int irq; int clk_id; int micbias_voltage; diff --git a/sound/soc/codecs/rt721-sdca.c b/sound/soc/codecs/rt721-sdca.c index 5f7b505d54147..53cec09d29085 100644 --- a/sound/soc/codecs/rt721-sdca.c +++ b/sound/soc/codecs/rt721-sdca.c @@ -245,12 +245,12 @@ static void rt721_sdca_jack_preset(struct rt721_sdca_priv *rt721) regmap_write(rt721->mbq_regmap, 0x5b10007, 0x2000); regmap_write(rt721->mbq_regmap, 0x5B10017, 0x1b0f); rt_sdca_index_write(rt721->mbq_regmap, RT721_CBJ_CTRL, - RT721_CBJ_A0_GAT_CTRL1, 0x2a02); + RT721_CBJ_A0_GAT_CTRL1, 0x2205); rt_sdca_index_write(rt721->mbq_regmap, RT721_CAP_PORT_CTRL, RT721_HP_AMP_2CH_CAL4, 0xa105); rt_sdca_index_write(rt721->mbq_regmap, RT721_VENDOR_ANA_CTL, RT721_UAJ_TOP_TCON14, 0x3b33); - regmap_write(rt721->mbq_regmap, 0x310400, 0x3023); + regmap_write(rt721->mbq_regmap, 0x310400, 0x3043); rt_sdca_index_write(rt721->mbq_regmap, RT721_VENDOR_ANA_CTL, RT721_UAJ_TOP_TCON14, 0x3f33); rt_sdca_index_write(rt721->mbq_regmap, RT721_VENDOR_ANA_CTL, diff --git a/sound/soc/codecs/wm8962.c b/sound/soc/codecs/wm8962.c index 08c8ec3aeb449..ab8247ae431f8 100644 --- a/sound/soc/codecs/wm8962.c +++ b/sound/soc/codecs/wm8962.c @@ -67,6 +67,8 @@ struct wm8962_priv { struct mutex dsp2_ena_lock; u16 dsp2_ena; + int mic_status; + struct delayed_work mic_work; struct snd_soc_jack *jack; @@ -1760,7 +1762,7 @@ SND_SOC_BYTES("EQR Coefficients", WM8962_EQ24, 18), SOC_SINGLE("3D Switch", WM8962_THREED1, 0, 1, 0), -SND_SOC_BYTES_MASK("3D Coefficients", WM8962_THREED1, 4, WM8962_THREED_ENA), +SND_SOC_BYTES_MASK("3D Coefficients", WM8962_THREED1, 4, WM8962_THREED_ENA | WM8962_ADC_MONOMIX), SOC_SINGLE("DF1 Switch", WM8962_DF1, 0, 1, 0), SND_SOC_BYTES_MASK("DF1 Coefficients", WM8962_DF1, 7, WM8962_DF1_ENA), @@ -3077,8 +3079,16 @@ static void wm8962_mic_work(struct work_struct *work) if (reg & WM8962_MICSHORT_STS) { status |= SND_JACK_BTN_0; irq_pol |= WM8962_MICSCD_IRQ_POL; + + /* Don't report a microphone if it's shorted right after + * plugging in, as this may be a TRS plug in a TRRS socket. + */ + if (!(wm8962->mic_status & WM8962_MICDET_STS)) + status = 0; } + wm8962->mic_status = status; + snd_soc_jack_report(wm8962->jack, status, SND_JACK_MICROPHONE | SND_JACK_BTN_0); diff --git a/sound/soc/fsl/fsl_xcvr.c b/sound/soc/fsl/fsl_xcvr.c index 58db4906a01d5..a268fb81a2f86 100644 --- a/sound/soc/fsl/fsl_xcvr.c +++ b/sound/soc/fsl/fsl_xcvr.c @@ -1323,6 +1323,7 @@ static const struct reg_default fsl_xcvr_phy_reg_defaults[] = { }; static const struct regmap_config fsl_xcvr_regmap_phy_cfg = { + .name = "phy", .reg_bits = 8, .reg_stride = 4, .val_bits = 32, @@ -1335,6 +1336,7 @@ static const struct regmap_config fsl_xcvr_regmap_phy_cfg = { }; static const struct regmap_config fsl_xcvr_regmap_pllv0_cfg = { + .name = "pllv0", .reg_bits = 8, .reg_stride = 4, .val_bits = 32, @@ -1345,6 +1347,7 @@ static const struct regmap_config fsl_xcvr_regmap_pllv0_cfg = { }; static const struct regmap_config fsl_xcvr_regmap_pllv1_cfg = { + .name = "pllv1", .reg_bits = 8, .reg_stride = 4, .val_bits = 32, @@ -1548,28 +1551,24 @@ static int fsl_xcvr_probe(struct platform_device *pdev) xcvr->soc_data = of_device_get_match_data(&pdev->dev); xcvr->ipg_clk = devm_clk_get(dev, "ipg"); - if (IS_ERR(xcvr->ipg_clk)) { - dev_err(dev, "failed to get ipg clock\n"); - return PTR_ERR(xcvr->ipg_clk); - } + if (IS_ERR(xcvr->ipg_clk)) + return dev_err_probe(dev, PTR_ERR(xcvr->ipg_clk), + "failed to get ipg clock\n"); xcvr->phy_clk = devm_clk_get(dev, "phy"); - if (IS_ERR(xcvr->phy_clk)) { - dev_err(dev, "failed to get phy clock\n"); - return PTR_ERR(xcvr->phy_clk); - } + if (IS_ERR(xcvr->phy_clk)) + return dev_err_probe(dev, PTR_ERR(xcvr->phy_clk), + "failed to get phy clock\n"); xcvr->spba_clk = devm_clk_get(dev, "spba"); - if (IS_ERR(xcvr->spba_clk)) { - dev_err(dev, "failed to get spba clock\n"); - return PTR_ERR(xcvr->spba_clk); - } + if (IS_ERR(xcvr->spba_clk)) + return dev_err_probe(dev, PTR_ERR(xcvr->spba_clk), + "failed to get spba clock\n"); xcvr->pll_ipg_clk = devm_clk_get(dev, "pll_ipg"); - if (IS_ERR(xcvr->pll_ipg_clk)) { - dev_err(dev, "failed to get pll_ipg clock\n"); - return PTR_ERR(xcvr->pll_ipg_clk); - } + if (IS_ERR(xcvr->pll_ipg_clk)) + return dev_err_probe(dev, PTR_ERR(xcvr->pll_ipg_clk), + "failed to get pll_ipg clock\n"); fsl_asoc_get_pll_clocks(dev, &xcvr->pll8k_clk, &xcvr->pll11k_clk); @@ -1593,51 +1592,42 @@ static int fsl_xcvr_probe(struct platform_device *pdev) xcvr->regmap = devm_regmap_init_mmio_clk(dev, NULL, regs, &fsl_xcvr_regmap_cfg); - if (IS_ERR(xcvr->regmap)) { - dev_err(dev, "failed to init XCVR regmap: %ld\n", - PTR_ERR(xcvr->regmap)); - return PTR_ERR(xcvr->regmap); - } + if (IS_ERR(xcvr->regmap)) + return dev_err_probe(dev, PTR_ERR(xcvr->regmap), "failed to init XCVR regmap\n"); if (xcvr->soc_data->use_phy) { xcvr->regmap_phy = devm_regmap_init(dev, NULL, xcvr, &fsl_xcvr_regmap_phy_cfg); - if (IS_ERR(xcvr->regmap_phy)) { - dev_err(dev, "failed to init XCVR PHY regmap: %ld\n", - PTR_ERR(xcvr->regmap_phy)); - return PTR_ERR(xcvr->regmap_phy); - } + if (IS_ERR(xcvr->regmap_phy)) + return dev_err_probe(dev, PTR_ERR(xcvr->regmap_phy), + "failed to init XCVR PHY regmap\n"); switch (xcvr->soc_data->pll_ver) { case PLL_MX8MP: xcvr->regmap_pll = devm_regmap_init(dev, NULL, xcvr, &fsl_xcvr_regmap_pllv0_cfg); - if (IS_ERR(xcvr->regmap_pll)) { - dev_err(dev, "failed to init XCVR PLL regmap: %ld\n", - PTR_ERR(xcvr->regmap_pll)); - return PTR_ERR(xcvr->regmap_pll); - } + if (IS_ERR(xcvr->regmap_pll)) + return dev_err_probe(dev, PTR_ERR(xcvr->regmap_pll), + "failed to init XCVR PLL regmap\n"); break; case PLL_MX95: xcvr->regmap_pll = devm_regmap_init(dev, NULL, xcvr, &fsl_xcvr_regmap_pllv1_cfg); - if (IS_ERR(xcvr->regmap_pll)) { - dev_err(dev, "failed to init XCVR PLL regmap: %ld\n", - PTR_ERR(xcvr->regmap_pll)); - return PTR_ERR(xcvr->regmap_pll); - } + if (IS_ERR(xcvr->regmap_pll)) + return dev_err_probe(dev, PTR_ERR(xcvr->regmap_pll), + "failed to init XCVR PLL regmap\n"); break; default: - dev_err(dev, "Error for PLL version %d\n", xcvr->soc_data->pll_ver); - return -EINVAL; + return dev_err_probe(dev, -EINVAL, + "Error for PLL version %d\n", + xcvr->soc_data->pll_ver); } } xcvr->reset = devm_reset_control_get_optional_exclusive(dev, NULL); - if (IS_ERR(xcvr->reset)) { - dev_err(dev, "failed to get XCVR reset control\n"); - return PTR_ERR(xcvr->reset); - } + if (IS_ERR(xcvr->reset)) + return dev_err_probe(dev, PTR_ERR(xcvr->reset), + "failed to get XCVR reset control\n"); /* get IRQs */ irq = platform_get_irq(pdev, 0); @@ -1645,17 +1635,13 @@ static int fsl_xcvr_probe(struct platform_device *pdev) return irq; ret = devm_request_irq(dev, irq, irq0_isr, 0, pdev->name, xcvr); - if (ret) { - dev_err(dev, "failed to claim IRQ0: %i\n", ret); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "failed to claim IRQ0\n"); rx_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rxfifo"); tx_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "txfifo"); - if (!rx_res || !tx_res) { - dev_err(dev, "could not find rxfifo or txfifo resource\n"); - return -EINVAL; - } + if (!rx_res || !tx_res) + return dev_err_probe(dev, -EINVAL, "could not find rxfifo or txfifo resource\n"); xcvr->dma_prms_rx.chan_name = "rx"; xcvr->dma_prms_tx.chan_name = "tx"; xcvr->dma_prms_rx.addr = rx_res->start; @@ -1678,8 +1664,7 @@ static int fsl_xcvr_probe(struct platform_device *pdev) ret = devm_snd_dmaengine_pcm_register(dev, NULL, 0); if (ret) { pm_runtime_disable(dev); - dev_err(dev, "failed to pcm register\n"); - return ret; + return dev_err_probe(dev, ret, "failed to pcm register\n"); } ret = devm_snd_soc_register_component(dev, &fsl_xcvr_comp, diff --git a/sound/soc/fsl/imx-rpmsg.c b/sound/soc/fsl/imx-rpmsg.c index 7cd3aa4c87060..39b2d64eb410d 100644 --- a/sound/soc/fsl/imx-rpmsg.c +++ b/sound/soc/fsl/imx-rpmsg.c @@ -145,7 +145,7 @@ static int imx_rpmsg_probe(struct platform_device *pdev) data->dai.ignore_pmdown_time = 1; data->dai.cpus->dai_name = pdev->dev.platform_data; - cpu_dai = snd_soc_find_dai(data->dai.cpus); + cpu_dai = snd_soc_find_dai_with_mutex(data->dai.cpus); if (!cpu_dai) { ret = -EPROBE_DEFER; goto fail; diff --git a/sound/soc/intel/boards/sof_es8336.c b/sound/soc/intel/boards/sof_es8336.c index 09acd80d23e0f..cf50de5c2edd8 100644 --- a/sound/soc/intel/boards/sof_es8336.c +++ b/sound/soc/intel/boards/sof_es8336.c @@ -332,6 +332,15 @@ static int sof_es8336_quirk_cb(const struct dmi_system_id *id) * if the topology file is modified as well. */ static const struct dmi_system_id sof_es8336_quirk_table[] = { + { + .callback = sof_es8336_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "HUAWEI"), + DMI_MATCH(DMI_PRODUCT_NAME, "BOD-WXX9"), + }, + .driver_data = (void *)(SOF_ES8336_HEADPHONE_GPIO | + SOF_ES8336_ENABLE_DMIC) + }, { .callback = sof_es8336_quirk_cb, .matches = { diff --git a/sound/soc/intel/boards/sof_sdw.c b/sound/soc/intel/boards/sof_sdw.c index 92fac7ed782f7..6c95b1f8fc1a5 100644 --- a/sound/soc/intel/boards/sof_sdw.c +++ b/sound/soc/intel/boards/sof_sdw.c @@ -802,6 +802,7 @@ static const struct snd_pci_quirk sof_sdw_ssid_quirk_table[] = { SND_PCI_QUIRK(0x17aa, 0x2347, "Lenovo P16", SOC_SDW_CODEC_MIC), SND_PCI_QUIRK(0x17aa, 0x2348, "Lenovo P16", SOC_SDW_CODEC_MIC), SND_PCI_QUIRK(0x17aa, 0x2349, "Lenovo P1", SOC_SDW_CODEC_MIC), + SND_PCI_QUIRK(0x17aa, 0x3821, "Lenovo 0x3821", SOC_SDW_SIDECAR_AMPS), {} }; diff --git a/sound/soc/intel/common/soc-acpi-intel-arl-match.c b/sound/soc/intel/common/soc-acpi-intel-arl-match.c index 6bf7a6250ddc3..c952f7d2b2c0e 100644 --- a/sound/soc/intel/common/soc-acpi-intel-arl-match.c +++ b/sound/soc/intel/common/soc-acpi-intel-arl-match.c @@ -45,23 +45,22 @@ static const struct snd_soc_acpi_endpoint spk_3_endpoint = { .group_id = 1, }; -/* - * RT722 is a multi-function codec, three endpoints are created for - * its headset, amp and dmic functions. - */ -static const struct snd_soc_acpi_endpoint rt722_endpoints[] = { +static const struct snd_soc_acpi_endpoint jack_amp_g1_dmic_endpoints[] = { + /* Jack Endpoint */ { .num = 0, .aggregated = 0, .group_position = 0, .group_id = 0, }, + /* Amp Endpoint, work as spk_l_endpoint */ { .num = 1, - .aggregated = 0, + .aggregated = 1, .group_position = 0, - .group_id = 0, + .group_id = 1, }, + /* DMIC Endpoint */ { .num = 2, .aggregated = 0, @@ -229,11 +228,11 @@ static const struct snd_soc_acpi_adr_device rt711_sdca_0_adr[] = { } }; -static const struct snd_soc_acpi_adr_device rt722_0_single_adr[] = { +static const struct snd_soc_acpi_adr_device rt722_0_agg_adr[] = { { .adr = 0x000030025D072201ull, - .num_endpoints = ARRAY_SIZE(rt722_endpoints), - .endpoints = rt722_endpoints, + .num_endpoints = ARRAY_SIZE(jack_amp_g1_dmic_endpoints), + .endpoints = jack_amp_g1_dmic_endpoints, .name_prefix = "rt722" } }; @@ -394,8 +393,8 @@ static const struct snd_soc_acpi_link_adr arl_rt711_l0_rt1316_l3[] = { static const struct snd_soc_acpi_link_adr arl_rt722_l0_rt1320_l2[] = { { .mask = BIT(0), - .num_adr = ARRAY_SIZE(rt722_0_single_adr), - .adr_d = rt722_0_single_adr, + .num_adr = ARRAY_SIZE(rt722_0_agg_adr), + .adr_d = rt722_0_agg_adr, }, { .mask = BIT(2), diff --git a/sound/soc/rockchip/rockchip_i2s_tdm.c b/sound/soc/rockchip/rockchip_i2s_tdm.c index d9a1fab7f4031..e4697ee0addc8 100644 --- a/sound/soc/rockchip/rockchip_i2s_tdm.c +++ b/sound/soc/rockchip/rockchip_i2s_tdm.c @@ -22,6 +22,7 @@ #define DRV_NAME "rockchip-i2s-tdm" +#define DEFAULT_MCLK_FS 256 #define CH_GRP_MAX 4 /* The max channel 8 / 2 */ #define MULTIPLEX_CH_MAX 10 @@ -665,6 +666,15 @@ static int rockchip_i2s_tdm_hw_params(struct snd_pcm_substream *substream, mclk_rate = i2s_tdm->mclk_rx_freq; } + /* + * When the dai/component driver doesn't need to set mclk-fs for a specific + * clock, it can skip the call to set_sysclk() for that clock. + * In that case, simply use the clock rate from the params and multiply it by + * the default mclk-fs value. + */ + if (!mclk_rate) + mclk_rate = DEFAULT_MCLK_FS * params_rate(params); + err = clk_set_rate(mclk, mclk_rate); if (err) return err; diff --git a/sound/soc/sdca/sdca_asoc.c b/sound/soc/sdca/sdca_asoc.c index 892b7c028faea..197a592ec2f16 100644 --- a/sound/soc/sdca/sdca_asoc.c +++ b/sound/soc/sdca/sdca_asoc.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -836,6 +837,48 @@ static int control_limit_kctl(struct device *dev, return 0; } +static int volatile_get_volsw(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + struct snd_soc_component *component = snd_kcontrol_chip(kcontrol); + struct device *dev = component->dev; + int ret; + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) { + dev_err(dev, "failed to resume reading %s: %d\n", + kcontrol->id.name, ret); + return ret; + } + + ret = snd_soc_get_volsw(kcontrol, ucontrol); + + pm_runtime_put(dev); + + return ret; +} + +static int volatile_put_volsw(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + struct snd_soc_component *component = snd_kcontrol_chip(kcontrol); + struct device *dev = component->dev; + int ret; + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) { + dev_err(dev, "failed to resume writing %s: %d\n", + kcontrol->id.name, ret); + return ret; + } + + ret = snd_soc_put_volsw(kcontrol, ucontrol); + + pm_runtime_put(dev); + + return ret; +} + static int populate_control(struct device *dev, struct sdca_function_data *function, struct sdca_entity *entity, @@ -890,8 +933,13 @@ static int populate_control(struct device *dev, (*kctl)->private_value = (unsigned long)mc; (*kctl)->iface = SNDRV_CTL_ELEM_IFACE_MIXER; (*kctl)->info = snd_soc_info_volsw; - (*kctl)->get = snd_soc_get_volsw; - (*kctl)->put = snd_soc_put_volsw; + if (control->is_volatile) { + (*kctl)->get = volatile_get_volsw; + (*kctl)->put = volatile_put_volsw; + } else { + (*kctl)->get = snd_soc_get_volsw; + (*kctl)->put = snd_soc_put_volsw; + } if (readonly_control(control)) (*kctl)->access = SNDRV_CTL_ELEM_ACCESS_READ; @@ -1519,7 +1567,7 @@ static int set_usage(struct device *dev, struct regmap *regmap, unsigned int rate = sdca_range(range, SDCA_USAGE_SAMPLE_RATE, i); unsigned int width = sdca_range(range, SDCA_USAGE_SAMPLE_WIDTH, i); - if ((!rate || rate == target_rate) && width == target_width) { + if ((!rate || rate == target_rate) && (!width || width == target_width)) { unsigned int usage = sdca_range(range, SDCA_USAGE_NUMBER, i); unsigned int reg = SDW_SDCA_CTL(function->desc->adr, entity->id, sel, 0); diff --git a/sound/soc/sdca/sdca_functions.c b/sound/soc/sdca/sdca_functions.c index 19b12564f822e..4417278e39bb1 100644 --- a/sound/soc/sdca/sdca_functions.c +++ b/sound/soc/sdca/sdca_functions.c @@ -779,6 +779,62 @@ find_sdca_control_datatype(const struct sdca_entity *entity, } } +static bool find_sdca_control_volatile(const struct sdca_entity *entity, + const struct sdca_control *control) +{ + switch (control->mode) { + case SDCA_ACCESS_MODE_DC: + return false; + case SDCA_ACCESS_MODE_RO: + case SDCA_ACCESS_MODE_RW1S: + case SDCA_ACCESS_MODE_RW1C: + return true; + default: + break; + } + + switch (SDCA_CTL_TYPE(entity->type, control->sel)) { + case SDCA_CTL_TYPE_S(XU, FDL_CURRENTOWNER): + case SDCA_CTL_TYPE_S(XU, FDL_MESSAGEOFFSET): + case SDCA_CTL_TYPE_S(XU, FDL_MESSAGELENGTH): + case SDCA_CTL_TYPE_S(XU, FDL_STATUS): + case SDCA_CTL_TYPE_S(XU, FDL_HOST_REQUEST): + case SDCA_CTL_TYPE_S(SPE, AUTHTX_CURRENTOWNER): + case SDCA_CTL_TYPE_S(SPE, AUTHTX_MESSAGEOFFSET): + case SDCA_CTL_TYPE_S(SPE, AUTHTX_MESSAGELENGTH): + case SDCA_CTL_TYPE_S(SPE, AUTHRX_CURRENTOWNER): + case SDCA_CTL_TYPE_S(SPE, AUTHRX_MESSAGEOFFSET): + case SDCA_CTL_TYPE_S(SPE, AUTHRX_MESSAGELENGTH): + case SDCA_CTL_TYPE_S(MFPU, AE_CURRENTOWNER): + case SDCA_CTL_TYPE_S(MFPU, AE_MESSAGEOFFSET): + case SDCA_CTL_TYPE_S(MFPU, AE_MESSAGELENGTH): + case SDCA_CTL_TYPE_S(SMPU, HIST_CURRENTOWNER): + case SDCA_CTL_TYPE_S(SMPU, HIST_MESSAGEOFFSET): + case SDCA_CTL_TYPE_S(SMPU, HIST_MESSAGELENGTH): + case SDCA_CTL_TYPE_S(SMPU, DTODTX_CURRENTOWNER): + case SDCA_CTL_TYPE_S(SMPU, DTODTX_MESSAGEOFFSET): + case SDCA_CTL_TYPE_S(SMPU, DTODTX_MESSAGELENGTH): + case SDCA_CTL_TYPE_S(SMPU, DTODRX_CURRENTOWNER): + case SDCA_CTL_TYPE_S(SMPU, DTODRX_MESSAGEOFFSET): + case SDCA_CTL_TYPE_S(SMPU, DTODRX_MESSAGELENGTH): + case SDCA_CTL_TYPE_S(SAPU, DTODTX_CURRENTOWNER): + case SDCA_CTL_TYPE_S(SAPU, DTODTX_MESSAGEOFFSET): + case SDCA_CTL_TYPE_S(SAPU, DTODTX_MESSAGELENGTH): + case SDCA_CTL_TYPE_S(SAPU, DTODRX_CURRENTOWNER): + case SDCA_CTL_TYPE_S(SAPU, DTODRX_MESSAGEOFFSET): + case SDCA_CTL_TYPE_S(SAPU, DTODRX_MESSAGELENGTH): + case SDCA_CTL_TYPE_S(HIDE, HIDTX_CURRENTOWNER): + case SDCA_CTL_TYPE_S(HIDE, HIDTX_MESSAGEOFFSET): + case SDCA_CTL_TYPE_S(HIDE, HIDTX_MESSAGELENGTH): + case SDCA_CTL_TYPE_S(HIDE, HIDRX_CURRENTOWNER): + case SDCA_CTL_TYPE_S(HIDE, HIDRX_MESSAGEOFFSET): + case SDCA_CTL_TYPE_S(HIDE, HIDRX_MESSAGELENGTH): + return true; + default: + return false; + } +} + static int find_sdca_control_range(struct device *dev, struct fwnode_handle *control_node, struct sdca_control_range *range) @@ -854,10 +910,6 @@ static int find_sdca_control_value(struct device *dev, struct sdca_entity *entit return 0; } -/* - * TODO: Add support for -cn- properties, allowing different channels to have - * different defaults etc. - */ static int find_sdca_entity_control(struct device *dev, struct sdca_entity *entity, struct fwnode_handle *control_node, struct sdca_control *control) @@ -931,6 +983,8 @@ static int find_sdca_entity_control(struct device *dev, struct sdca_entity *enti break; } + control->is_volatile = find_sdca_control_volatile(entity, control); + ret = find_sdca_control_range(dev, control_node, &control->range); if (ret) { dev_err(dev, "%s: control %#x: range missing: %d\n", diff --git a/sound/soc/sdca/sdca_interrupts.c b/sound/soc/sdca/sdca_interrupts.c index 79bf3042f57d4..f83413587da5a 100644 --- a/sound/soc/sdca/sdca_interrupts.c +++ b/sound/soc/sdca/sdca_interrupts.c @@ -246,9 +246,9 @@ static int sdca_irq_request_locked(struct device *dev, } /** - * sdca_request_irq - request an individual SDCA interrupt + * sdca_irq_request - request an individual SDCA interrupt * @dev: Pointer to the struct device against which things should be allocated. - * @interrupt_info: Pointer to the interrupt information structure. + * @info: Pointer to the interrupt information structure. * @sdca_irq: SDCA interrupt position. * @name: Name to be given to the IRQ. * @handler: A callback thread function to be called for the IRQ. diff --git a/sound/soc/sdca/sdca_regmap.c b/sound/soc/sdca/sdca_regmap.c index 72f893e00ff50..8fa138fca00ff 100644 --- a/sound/soc/sdca/sdca_regmap.c +++ b/sound/soc/sdca/sdca_regmap.c @@ -147,14 +147,7 @@ bool sdca_regmap_volatile(struct sdca_function_data *function, unsigned int reg) if (!control) return false; - switch (control->mode) { - case SDCA_ACCESS_MODE_RO: - case SDCA_ACCESS_MODE_RW1S: - case SDCA_ACCESS_MODE_RW1C: - return true; - default: - return false; - } + return control->is_volatile; } EXPORT_SYMBOL_NS(sdca_regmap_volatile, "SND_SOC_SDCA"); diff --git a/sound/soc/sof/intel/hda-dai.c b/sound/soc/sof/intel/hda-dai.c index 883d0d3bae9ec..3c742d5351333 100644 --- a/sound/soc/sof/intel/hda-dai.c +++ b/sound/soc/sof/intel/hda-dai.c @@ -70,12 +70,22 @@ static const struct hda_dai_widget_dma_ops * hda_dai_get_ops(struct snd_pcm_substream *substream, struct snd_soc_dai *cpu_dai) { struct snd_soc_dapm_widget *w = snd_soc_dai_get_widget(cpu_dai, substream->stream); - struct snd_sof_widget *swidget = w->dobj.private; + struct snd_sof_widget *swidget; struct snd_sof_dev *sdev; struct snd_sof_dai *sdai; - sdev = widget_to_sdev(w); + /* + * this is unlikely if the topology and the machine driver DAI links match. + * But if there's a missing DAI link in topology, this will prevent a NULL pointer + * dereference later on. + */ + if (!w) { + dev_err(cpu_dai->dev, "%s: widget is NULL\n", __func__); + return NULL; + } + sdev = widget_to_sdev(w); + swidget = w->dobj.private; if (!swidget) { dev_err(sdev->dev, "%s: swidget is NULL\n", __func__); return NULL; diff --git a/sound/soc/sof/ipc4-control.c b/sound/soc/sof/ipc4-control.c index 976a4794d6100..453ed1643b89c 100644 --- a/sound/soc/sof/ipc4-control.c +++ b/sound/soc/sof/ipc4-control.c @@ -66,7 +66,7 @@ static int sof_ipc4_set_get_kcontrol_data(struct snd_sof_control *scontrol, * configuration */ memcpy(scontrol->ipc_control_data, scontrol->old_ipc_control_data, - scontrol->max_size); + scontrol->size); kfree(scontrol->old_ipc_control_data); scontrol->old_ipc_control_data = NULL; /* Send the last known good configuration to firmware */ @@ -412,19 +412,35 @@ static int sof_ipc4_set_get_bytes_data(struct snd_sof_dev *sdev, int ret = 0; /* Send the new data to the firmware only if it is powered up */ - if (set && !pm_runtime_active(sdev->dev)) - return 0; + if (set) { + if (!pm_runtime_active(sdev->dev)) + return 0; + + if (!data->size) { + dev_dbg(sdev->dev, "%s: No data to be sent.\n", + scontrol->name); + return 0; + } + } msg->extension = SOF_IPC4_MOD_EXT_MSG_PARAM_ID(data->type); msg->data_ptr = data->data; - msg->data_size = data->size; + if (set) + msg->data_size = data->size; + else + msg->data_size = scontrol->max_size - sizeof(*data); ret = sof_ipc4_set_get_kcontrol_data(scontrol, set, lock); - if (ret < 0) + if (ret < 0) { dev_err(sdev->dev, "Failed to %s for %s\n", set ? "set bytes update" : "get bytes", scontrol->name); + } else if (!set) { + /* Update the sizes according to the received payload data */ + data->size = msg->data_size; + scontrol->size = sizeof(*cdata) + sizeof(*data) + data->size; + } msg->data_ptr = NULL; msg->data_size = 0; @@ -440,6 +456,7 @@ static int sof_ipc4_bytes_put(struct snd_sof_control *scontrol, struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp); struct sof_abi_hdr *data = cdata->data; size_t size; + int ret; if (scontrol->max_size > sizeof(ucontrol->value.bytes.data)) { dev_err_ratelimited(scomp->dev, @@ -461,9 +478,12 @@ static int sof_ipc4_bytes_put(struct snd_sof_control *scontrol, /* copy from kcontrol */ memcpy(data, ucontrol->value.bytes.data, size); - sof_ipc4_set_get_bytes_data(sdev, scontrol, true, true); + ret = sof_ipc4_set_get_bytes_data(sdev, scontrol, true, true); + if (!ret) + /* Update the cdata size */ + scontrol->size = sizeof(*cdata) + size; - return 0; + return ret; } static int sof_ipc4_bytes_get(struct snd_sof_control *scontrol, @@ -559,7 +579,7 @@ static int sof_ipc4_bytes_ext_put(struct snd_sof_control *scontrol, if (!scontrol->old_ipc_control_data) { /* Create a backup of the current, valid bytes control */ scontrol->old_ipc_control_data = kmemdup(scontrol->ipc_control_data, - scontrol->max_size, GFP_KERNEL); + scontrol->size, GFP_KERNEL); if (!scontrol->old_ipc_control_data) return -ENOMEM; } @@ -567,12 +587,15 @@ static int sof_ipc4_bytes_ext_put(struct snd_sof_control *scontrol, /* Copy the whole binary data which includes the ABI header and the payload */ if (copy_from_user(data, tlvd->tlv, header.length)) { memcpy(scontrol->ipc_control_data, scontrol->old_ipc_control_data, - scontrol->max_size); + scontrol->size); kfree(scontrol->old_ipc_control_data); scontrol->old_ipc_control_data = NULL; return -EFAULT; } + /* Update the cdata size */ + scontrol->size = sizeof(*cdata) + header.length; + return sof_ipc4_set_get_bytes_data(sdev, scontrol, true, true); } diff --git a/sound/soc/sof/ipc4-topology.c b/sound/soc/sof/ipc4-topology.c index 32b628e2fe29b..5ca995acaba2e 100644 --- a/sound/soc/sof/ipc4-topology.c +++ b/sound/soc/sof/ipc4-topology.c @@ -2861,22 +2861,41 @@ static int sof_ipc4_control_load_bytes(struct snd_sof_dev *sdev, struct snd_sof_ struct sof_ipc4_msg *msg; int ret; - if (scontrol->max_size < (sizeof(*control_data) + sizeof(struct sof_abi_hdr))) { - dev_err(sdev->dev, "insufficient size for a bytes control %s: %zu.\n", + /* + * The max_size is coming from topology and indicates the maximum size + * of sof_abi_hdr plus the payload, which excludes the local only + * 'struct sof_ipc4_control_data' + */ + if (scontrol->max_size < sizeof(struct sof_abi_hdr)) { + dev_err(sdev->dev, + "insufficient maximum size for a bytes control %s: %zu.\n", scontrol->name, scontrol->max_size); return -EINVAL; } - if (scontrol->priv_size > scontrol->max_size - sizeof(*control_data)) { - dev_err(sdev->dev, "scontrol %s bytes data size %zu exceeds max %zu.\n", - scontrol->name, scontrol->priv_size, - scontrol->max_size - sizeof(*control_data)); + if (scontrol->priv_size > scontrol->max_size) { + dev_err(sdev->dev, + "bytes control %s initial data size %zu exceeds max %zu.\n", + scontrol->name, scontrol->priv_size, scontrol->max_size); + return -EINVAL; + } + + if (scontrol->priv_size < sizeof(struct sof_abi_hdr)) { + dev_err(sdev->dev, + "bytes control %s initial data size %zu is insufficient.\n", + scontrol->name, scontrol->priv_size); return -EINVAL; } - scontrol->size = sizeof(struct sof_ipc4_control_data) + scontrol->priv_size; + /* + * The used size behind the cdata pointer, which can be smaller than + * the maximum size + */ + scontrol->size = sizeof(*control_data) + scontrol->priv_size; - scontrol->ipc_control_data = kzalloc(scontrol->max_size, GFP_KERNEL); + /* Allocate the cdata: local struct size + maximum payload size */ + scontrol->ipc_control_data = kzalloc(sizeof(*control_data) + scontrol->max_size, + GFP_KERNEL); if (!scontrol->ipc_control_data) return -ENOMEM; diff --git a/sound/soc/sof/ipc4.c b/sound/soc/sof/ipc4.c index a4a090e6724a6..20d723f48fff0 100644 --- a/sound/soc/sof/ipc4.c +++ b/sound/soc/sof/ipc4.c @@ -15,6 +15,7 @@ #include "sof-audio.h" #include "ipc4-fw-reg.h" #include "ipc4-priv.h" +#include "ipc4-topology.h" #include "ipc4-telemetry.h" #include "ops.h" @@ -433,6 +434,23 @@ static int sof_ipc4_tx_msg(struct snd_sof_dev *sdev, void *msg_data, size_t msg_ return ret; } +static bool sof_ipc4_tx_payload_for_get_data(struct sof_ipc4_msg *tx) +{ + /* + * Messages that require TX payload with LARGE_CONFIG_GET. + * The TX payload is placed into the IPC message data section by caller, + * which needs to be copied to temporary buffer since the received data + * will overwrite it. + */ + switch (tx->extension & SOF_IPC4_MOD_EXT_MSG_PARAM_ID_MASK) { + case SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_SWITCH_CONTROL_PARAM_ID): + case SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_ENUM_CONTROL_PARAM_ID): + return true; + default: + return false; + } +} + static int sof_ipc4_set_get_data(struct snd_sof_dev *sdev, void *data, size_t payload_bytes, bool set) { @@ -444,6 +462,8 @@ static int sof_ipc4_set_get_data(struct snd_sof_dev *sdev, void *data, struct sof_ipc4_msg tx = {{ 0 }}; struct sof_ipc4_msg rx = {{ 0 }}; size_t remaining = payload_bytes; + void *tx_payload_for_get = NULL; + size_t tx_data_size = 0; size_t offset = 0; size_t chunk_size; int ret; @@ -469,10 +489,20 @@ static int sof_ipc4_set_get_data(struct snd_sof_dev *sdev, void *data, tx.extension |= SOF_IPC4_MOD_EXT_MSG_FIRST_BLOCK(1); + if (sof_ipc4_tx_payload_for_get_data(&tx)) { + tx_data_size = min(ipc4_msg->data_size, payload_limit); + tx_payload_for_get = kmemdup(ipc4_msg->data_ptr, tx_data_size, + GFP_KERNEL); + if (!tx_payload_for_get) + return -ENOMEM; + } + /* ensure the DSP is in D0i0 before sending IPC */ ret = snd_sof_dsp_set_power_state(sdev, &target_state); - if (ret < 0) + if (ret < 0) { + kfree(tx_payload_for_get); return ret; + } /* Serialise IPC TX */ mutex_lock(&sdev->ipc->tx_mutex); @@ -506,7 +536,15 @@ static int sof_ipc4_set_get_data(struct snd_sof_dev *sdev, void *data, rx.data_size = chunk_size; rx.data_ptr = ipc4_msg->data_ptr + offset; - tx_size = 0; + if (tx_payload_for_get) { + tx_size = tx_data_size; + tx.data_size = tx_size; + tx.data_ptr = tx_payload_for_get; + } else { + tx_size = 0; + tx.data_size = 0; + tx.data_ptr = NULL; + } rx_size = chunk_size; } @@ -553,6 +591,8 @@ static int sof_ipc4_set_get_data(struct snd_sof_dev *sdev, void *data, mutex_unlock(&sdev->ipc->tx_mutex); + kfree(tx_payload_for_get); + return ret; } diff --git a/sound/soc/sunxi/sun50i-dmic.c b/sound/soc/sunxi/sun50i-dmic.c index bab1e29c99887..eddfebe166169 100644 --- a/sound/soc/sunxi/sun50i-dmic.c +++ b/sound/soc/sunxi/sun50i-dmic.c @@ -358,6 +358,9 @@ static int sun50i_dmic_probe(struct platform_device *pdev) host->regmap = devm_regmap_init_mmio(&pdev->dev, base, &sun50i_dmic_regmap_config); + if (IS_ERR(host->regmap)) + return dev_err_probe(&pdev->dev, PTR_ERR(host->regmap), + "failed to initialise regmap\n"); /* Clocks */ host->bus_clk = devm_clk_get(&pdev->dev, "bus"); diff --git a/sound/soc/tegra/tegra210_ahub.c b/sound/soc/tegra/tegra210_ahub.c index 21aeaeba0b107..01d60a74ad1c3 100644 --- a/sound/soc/tegra/tegra210_ahub.c +++ b/sound/soc/tegra/tegra210_ahub.c @@ -2049,6 +2049,61 @@ static const struct snd_soc_component_driver tegra264_ahub_component = { .num_dapm_routes = ARRAY_SIZE(tegra264_ahub_routes), }; +static bool tegra210_ahub_wr_reg(struct device *dev, unsigned int reg) +{ + int part; + + if (reg % TEGRA210_XBAR_RX_STRIDE) + return false; + + for (part = 0; part < TEGRA210_XBAR_UPDATE_MAX_REG; part++) { + switch (reg & ~(part * TEGRA210_XBAR_PART1_RX)) { + case TEGRA210_AXBAR_PART_0_ADMAIF_RX1_0 ... TEGRA210_AXBAR_PART_0_ADMAIF_RX10_0: + case TEGRA210_AXBAR_PART_0_I2S1_RX1_0 ... TEGRA210_AXBAR_PART_0_I2S5_RX1_0: + case TEGRA210_AXBAR_PART_0_SFC1_RX1_0 ... TEGRA210_AXBAR_PART_0_SFC4_RX1_0: + case TEGRA210_AXBAR_PART_0_MIXER1_RX1_0 ... TEGRA210_AXBAR_PART_0_MIXER1_RX10_0: + case TEGRA210_AXBAR_PART_0_SPDIF1_RX1_0 ... TEGRA210_AXBAR_PART_0_SPDIF1_RX2_0: + case TEGRA210_AXBAR_PART_0_AFC1_RX1_0 ... TEGRA210_AXBAR_PART_0_AFC6_RX1_0: + case TEGRA210_AXBAR_PART_0_OPE1_RX1_0 ... TEGRA210_AXBAR_PART_0_OPE2_RX1_0: + case TEGRA210_AXBAR_PART_0_SPKPROT1_RX1_0: + case TEGRA210_AXBAR_PART_0_MVC1_RX1_0 ... TEGRA210_AXBAR_PART_0_MVC2_RX1_0: + case TEGRA210_AXBAR_PART_0_AMX1_RX1_0 ... TEGRA210_AXBAR_PART_0_ADX2_RX1_0: + return true; + default: + break; + } + } + + return false; +} + +static bool tegra186_ahub_wr_reg(struct device *dev, unsigned int reg) +{ + int part; + + if (reg % TEGRA210_XBAR_RX_STRIDE) + return false; + + for (part = 0; part < TEGRA186_XBAR_UPDATE_MAX_REG; part++) { + switch (reg & ~(part * TEGRA210_XBAR_PART1_RX)) { + case TEGRA210_AXBAR_PART_0_ADMAIF_RX1_0 ... TEGRA186_AXBAR_PART_0_I2S6_RX1_0: + case TEGRA210_AXBAR_PART_0_SFC1_RX1_0 ... TEGRA210_AXBAR_PART_0_SFC4_RX1_0: + case TEGRA210_AXBAR_PART_0_MIXER1_RX1_0 ... TEGRA210_AXBAR_PART_0_MIXER1_RX10_0: + case TEGRA186_AXBAR_PART_0_DSPK1_RX1_0 ... TEGRA186_AXBAR_PART_0_DSPK2_RX1_0: + case TEGRA210_AXBAR_PART_0_AFC1_RX1_0 ... TEGRA210_AXBAR_PART_0_AFC6_RX1_0: + case TEGRA210_AXBAR_PART_0_OPE1_RX1_0: + case TEGRA186_AXBAR_PART_0_MVC1_RX1_0 ... TEGRA186_AXBAR_PART_0_MVC2_RX1_0: + case TEGRA186_AXBAR_PART_0_AMX1_RX1_0 ... TEGRA186_AXBAR_PART_0_AMX3_RX4_0: + case TEGRA210_AXBAR_PART_0_ADX1_RX1_0 ... TEGRA186_AXBAR_PART_0_ASRC1_RX7_0: + return true; + default: + break; + } + } + + return false; +} + static bool tegra264_ahub_wr_reg(struct device *dev, unsigned int reg) { int part; @@ -2076,6 +2131,7 @@ static const struct regmap_config tegra210_ahub_regmap_config = { .reg_bits = 32, .val_bits = 32, .reg_stride = 4, + .writeable_reg = tegra210_ahub_wr_reg, .max_register = TEGRA210_MAX_REGISTER_ADDR, .cache_type = REGCACHE_FLAT, }; @@ -2084,6 +2140,7 @@ static const struct regmap_config tegra186_ahub_regmap_config = { .reg_bits = 32, .val_bits = 32, .reg_stride = 4, + .writeable_reg = tegra186_ahub_wr_reg, .max_register = TEGRA186_MAX_REGISTER_ADDR, .cache_type = REGCACHE_FLAT, }; diff --git a/sound/soc/tegra/tegra210_ahub.h b/sound/soc/tegra/tegra210_ahub.h index f355b2cfd19b2..acbe640dd3b57 100644 --- a/sound/soc/tegra/tegra210_ahub.h +++ b/sound/soc/tegra/tegra210_ahub.h @@ -68,6 +68,36 @@ #define TEGRA210_MAX_REGISTER_ADDR (TEGRA210_XBAR_PART2_RX + \ (TEGRA210_XBAR_RX_STRIDE * (TEGRA210_XBAR_AUDIO_RX_COUNT - 1))) +/* AXBAR register offsets */ +#define TEGRA186_AXBAR_PART_0_AMX1_RX1_0 0x120 +#define TEGRA186_AXBAR_PART_0_AMX3_RX4_0 0x14c +#define TEGRA186_AXBAR_PART_0_ASRC1_RX7_0 0x1a8 +#define TEGRA186_AXBAR_PART_0_DSPK1_RX1_0 0xc0 +#define TEGRA186_AXBAR_PART_0_DSPK2_RX1_0 0xc4 +#define TEGRA186_AXBAR_PART_0_I2S6_RX1_0 0x54 +#define TEGRA186_AXBAR_PART_0_MVC1_RX1_0 0x110 +#define TEGRA186_AXBAR_PART_0_MVC2_RX1_0 0x114 +#define TEGRA210_AXBAR_PART_0_ADMAIF_RX10_0 0x24 +#define TEGRA210_AXBAR_PART_0_ADMAIF_RX1_0 0x0 +#define TEGRA210_AXBAR_PART_0_ADX1_RX1_0 0x160 +#define TEGRA210_AXBAR_PART_0_ADX2_RX1_0 0x164 +#define TEGRA210_AXBAR_PART_0_AFC1_RX1_0 0xd0 +#define TEGRA210_AXBAR_PART_0_AFC6_RX1_0 0xe4 +#define TEGRA210_AXBAR_PART_0_AMX1_RX1_0 0x140 +#define TEGRA210_AXBAR_PART_0_I2S1_RX1_0 0x40 +#define TEGRA210_AXBAR_PART_0_I2S5_RX1_0 0x50 +#define TEGRA210_AXBAR_PART_0_MIXER1_RX10_0 0xa4 +#define TEGRA210_AXBAR_PART_0_MIXER1_RX1_0 0x80 +#define TEGRA210_AXBAR_PART_0_MVC1_RX1_0 0x120 +#define TEGRA210_AXBAR_PART_0_MVC2_RX1_0 0x124 +#define TEGRA210_AXBAR_PART_0_OPE1_RX1_0 0x100 +#define TEGRA210_AXBAR_PART_0_OPE2_RX1_0 0x104 +#define TEGRA210_AXBAR_PART_0_SFC1_RX1_0 0x60 +#define TEGRA210_AXBAR_PART_0_SFC4_RX1_0 0x6c +#define TEGRA210_AXBAR_PART_0_SPDIF1_RX1_0 0xc0 +#define TEGRA210_AXBAR_PART_0_SPDIF1_RX2_0 0xc4 +#define TEGRA210_AXBAR_PART_0_SPKPROT1_RX1_0 0x110 + #define MUX_REG(id) (TEGRA210_XBAR_RX_STRIDE * (id)) #define MUX_VALUE(npart, nbit) (1 + (nbit) + (npart) * 32) diff --git a/sound/usb/endpoint.c b/sound/usb/endpoint.c index cc15624ecaffe..eff3329d86b7e 100644 --- a/sound/usb/endpoint.c +++ b/sound/usb/endpoint.c @@ -160,8 +160,8 @@ int snd_usb_endpoint_implicit_feedback_sink(struct snd_usb_endpoint *ep) * This won't be used for implicit feedback which takes the packet size * returned from the sync source */ -static int slave_next_packet_size(struct snd_usb_endpoint *ep, - unsigned int avail) +static int synced_next_packet_size(struct snd_usb_endpoint *ep, + unsigned int avail) { unsigned int phase; int ret; @@ -227,7 +227,7 @@ int snd_usb_endpoint_next_packet_size(struct snd_usb_endpoint *ep, } if (ep->sync_source) - return slave_next_packet_size(ep, avail); + return synced_next_packet_size(ep, avail); else return next_packet_size(ep, avail); } @@ -275,8 +275,8 @@ static inline bool has_tx_length_quirk(struct snd_usb_audio *chip) return chip->quirk_flags & QUIRK_FLAG_TX_LENGTH; } -static void prepare_silent_urb(struct snd_usb_endpoint *ep, - struct snd_urb_ctx *ctx) +static int prepare_silent_urb(struct snd_usb_endpoint *ep, + struct snd_urb_ctx *ctx) { struct urb *urb = ctx->urb; unsigned int offs = 0; @@ -289,28 +289,34 @@ static void prepare_silent_urb(struct snd_usb_endpoint *ep, extra = sizeof(packet_length); for (i = 0; i < ctx->packets; ++i) { - unsigned int offset; - unsigned int length; - int counts; - - counts = snd_usb_endpoint_next_packet_size(ep, ctx, i, 0); - length = counts * ep->stride; /* number of silent bytes */ - offset = offs * ep->stride + extra * i; - urb->iso_frame_desc[i].offset = offset; + int length; + + length = snd_usb_endpoint_next_packet_size(ep, ctx, i, 0); + if (length < 0) + return length; + length *= ep->stride; /* number of silent bytes */ + if (offs + length + extra > ctx->buffer_size) + break; + urb->iso_frame_desc[i].offset = offs; urb->iso_frame_desc[i].length = length + extra; if (extra) { packet_length = cpu_to_le32(length); - memcpy(urb->transfer_buffer + offset, + memcpy(urb->transfer_buffer + offs, &packet_length, sizeof(packet_length)); + offs += extra; } - memset(urb->transfer_buffer + offset + extra, + memset(urb->transfer_buffer + offs, ep->silence_value, length); - offs += counts; + offs += length; } - urb->number_of_packets = ctx->packets; - urb->transfer_buffer_length = offs * ep->stride + ctx->packets * extra; + if (!offs) + return -EPIPE; + + urb->number_of_packets = i; + urb->transfer_buffer_length = offs; ctx->queued = 0; + return 0; } /* @@ -332,8 +338,7 @@ static int prepare_outbound_urb(struct snd_usb_endpoint *ep, if (data_subs && ep->prepare_data_urb) return ep->prepare_data_urb(data_subs, urb, in_stream_lock); /* no data provider, so send silence */ - prepare_silent_urb(ep, ctx); - break; + return prepare_silent_urb(ep, ctx); case SND_USB_ENDPOINT_TYPE_SYNC: if (snd_usb_get_speed(ep->chip->dev) >= USB_SPEED_HIGH) { @@ -481,6 +486,7 @@ int snd_usb_queue_pending_output_urbs(struct snd_usb_endpoint *ep, /* copy over the length information */ if (implicit_fb) { + ctx->packets = packet->packets; for (i = 0; i < packet->packets; i++) ctx->packet_size[i] = packet->packet_size[i]; } @@ -1368,6 +1374,9 @@ int snd_usb_endpoint_set_params(struct snd_usb_audio *chip, return -EINVAL; } + ep->packsize[0] = min(ep->packsize[0], ep->maxframesize); + ep->packsize[1] = min(ep->packsize[1], ep->maxframesize); + /* calculate the frequency in 16.16 format */ ep->freqm = ep->freqn; ep->freqshift = INT_MIN; diff --git a/sound/usb/mixer_s1810c.c b/sound/usb/mixer_s1810c.c index 6e09e074c0e7f..93510aa0dc5ef 100644 --- a/sound/usb/mixer_s1810c.c +++ b/sound/usb/mixer_s1810c.c @@ -82,13 +82,13 @@ * mixer and output but a different set for device. */ struct s1810c_ctl_packet { - u32 a; - u32 b; - u32 fixed1; - u32 fixed2; - u32 c; - u32 d; - u32 e; + __le32 a; + __le32 b; + __le32 fixed1; + __le32 fixed2; + __le32 c; + __le32 d; + __le32 e; }; #define SC1810C_CTL_LINE_SW 0 @@ -118,7 +118,7 @@ struct s1810c_ctl_packet { * being zero and different f1/f2. */ struct s1810c_state_packet { - u32 fields[63]; + __le32 fields[63]; }; #define SC1810C_STATE_48V_SW 58 @@ -140,14 +140,14 @@ snd_s1810c_send_ctl_packet(struct usb_device *dev, u32 a, struct s1810c_ctl_packet pkt = { 0 }; int ret = 0; - pkt.fixed1 = SC1810C_CMD_F1; - pkt.fixed2 = SC1810C_CMD_F2; + pkt.fixed1 = __cpu_to_le32(SC1810C_CMD_F1); + pkt.fixed2 = __cpu_to_le32(SC1810C_CMD_F2); - pkt.a = a; - pkt.b = b; - pkt.c = c; - pkt.d = d; - pkt.e = e; + pkt.a = __cpu_to_le32(a); + pkt.b = __cpu_to_le32(b); + pkt.c = __cpu_to_le32(c); + pkt.d = __cpu_to_le32(d); + pkt.e = __cpu_to_le32(e); ret = snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), SC1810C_CMD_REQ, @@ -176,8 +176,8 @@ snd_sc1810c_get_status_field(struct usb_device *dev, struct s1810c_state_packet pkt_in = { { 0 } }; int ret = 0; - pkt_out.fields[SC1810C_STATE_F1_IDX] = SC1810C_SET_STATE_F1; - pkt_out.fields[SC1810C_STATE_F2_IDX] = SC1810C_SET_STATE_F2; + pkt_out.fields[SC1810C_STATE_F1_IDX] = __cpu_to_le32(SC1810C_SET_STATE_F1); + pkt_out.fields[SC1810C_STATE_F2_IDX] = __cpu_to_le32(SC1810C_SET_STATE_F2); ret = snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), SC1810C_SET_STATE_REQ, SC1810C_SET_STATE_REQTYPE, @@ -197,7 +197,7 @@ snd_sc1810c_get_status_field(struct usb_device *dev, return ret; } - (*field) = pkt_in.fields[field_idx]; + (*field) = __le32_to_cpu(pkt_in.fields[field_idx]); (*seqnum)++; return 0; } diff --git a/sound/usb/mixer_scarlett2.c b/sound/usb/mixer_scarlett2.c index bef8c9e544dd3..380beb7ed4cf8 100644 --- a/sound/usb/mixer_scarlett2.c +++ b/sound/usb/mixer_scarlett2.c @@ -1328,8 +1328,6 @@ struct scarlett2_data { struct snd_kcontrol *mux_ctls[SCARLETT2_MUX_MAX]; struct snd_kcontrol *mix_ctls[SCARLETT2_MIX_MAX]; struct snd_kcontrol *compressor_ctls[SCARLETT2_COMPRESSOR_CTLS_MAX]; - struct snd_kcontrol *precomp_flt_ctls[SCARLETT2_PRECOMP_FLT_CTLS_MAX]; - struct snd_kcontrol *peq_flt_ctls[SCARLETT2_PEQ_FLT_CTLS_MAX]; struct snd_kcontrol *precomp_flt_switch_ctls[SCARLETT2_DSP_SWITCH_MAX]; struct snd_kcontrol *peq_flt_switch_ctls[SCARLETT2_DSP_SWITCH_MAX]; struct snd_kcontrol *direct_monitor_ctl; @@ -3447,7 +3445,6 @@ static int scarlett2_update_autogain(struct usb_mixer_interface *mixer) private->autogain_status[i] = private->num_autogain_status_texts - 1; - for (i = 0; i < SCARLETT2_AG_TARGET_COUNT; i++) if (scarlett2_has_config_item(private, scarlett2_ag_target_configs[i])) { @@ -5372,8 +5369,7 @@ static int scarlett2_update_filter_values(struct usb_mixer_interface *mixer) err = scarlett2_usb_get_config( mixer, SCARLETT2_CONFIG_PEQ_FLT_SWITCH, - info->dsp_input_count * info->peq_flt_count, - private->peq_flt_switch); + info->dsp_input_count, private->peq_flt_switch); if (err < 0) return err; @@ -6546,7 +6542,7 @@ static int scarlett2_add_dsp_ctls(struct usb_mixer_interface *mixer, int i) err = scarlett2_add_new_ctl( mixer, &scarlett2_precomp_flt_ctl, i * info->precomp_flt_count + j, - 1, s, &private->precomp_flt_switch_ctls[j]); + 1, s, NULL); if (err < 0) return err; } @@ -6556,7 +6552,7 @@ static int scarlett2_add_dsp_ctls(struct usb_mixer_interface *mixer, int i) err = scarlett2_add_new_ctl( mixer, &scarlett2_peq_flt_ctl, i * info->peq_flt_count + j, - 1, s, &private->peq_flt_switch_ctls[j]); + 1, s, NULL); if (err < 0) return err; } diff --git a/sound/usb/qcom/qc_audio_offload.c b/sound/usb/qcom/qc_audio_offload.c index cfb30a195364a..297490f0f5874 100644 --- a/sound/usb/qcom/qc_audio_offload.c +++ b/sound/usb/qcom/qc_audio_offload.c @@ -1007,7 +1007,7 @@ static int enable_audio_stream(struct snd_usb_substream *subs, /** * uaudio_transfer_buffer_setup() - fetch and populate xfer buffer params * @subs: usb substream - * @xfer_buf: xfer buf to be allocated + * @xfer_buf_cpu: xfer buf to be allocated * @xfer_buf_len: size of allocation * @mem_info: QMI response info * diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c index 8a646891ebb44..c411005cd4d87 100644 --- a/sound/usb/quirks.c +++ b/sound/usb/quirks.c @@ -2147,6 +2147,8 @@ struct usb_audio_quirk_flags_table { static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { /* Device matches */ + DEVICE_FLG(0x001f, 0x0b21, /* AB13X USB Audio */ + QUIRK_FLAG_FORCE_IFACE_RESET | QUIRK_FLAG_IFACE_DELAY), DEVICE_FLG(0x03f0, 0x654a, /* HP 320 FHD Webcam */ QUIRK_FLAG_GET_SAMPLE_RATE | QUIRK_FLAG_MIC_RES_16), DEVICE_FLG(0x041e, 0x3000, /* Creative SB Extigy */ @@ -2419,7 +2421,7 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { VENDOR_FLG(0x07fd, /* MOTU */ QUIRK_FLAG_VALIDATE_RATES), VENDOR_FLG(0x1235, /* Focusrite Novation */ - QUIRK_FLAG_VALIDATE_RATES), + QUIRK_FLAG_SKIP_IFACE_SETUP), VENDOR_FLG(0x1511, /* AURALiC */ QUIRK_FLAG_DSD_RAW), VENDOR_FLG(0x152a, /* Thesycon devices */ @@ -2501,6 +2503,7 @@ static const char *const snd_usb_audio_quirk_flag_names[] = { QUIRK_STRING_ENTRY(MIC_RES_384), QUIRK_STRING_ENTRY(MIXER_PLAYBACK_MIN_MUTE), QUIRK_STRING_ENTRY(MIXER_CAPTURE_MIN_MUTE), + QUIRK_STRING_ENTRY(SKIP_IFACE_SETUP), NULL }; diff --git a/sound/usb/stream.c b/sound/usb/stream.c index 5c235a5ba7e1b..3b2526964e4b4 100644 --- a/sound/usb/stream.c +++ b/sound/usb/stream.c @@ -1261,6 +1261,9 @@ static int __snd_usb_parse_audio_interface(struct snd_usb_audio *chip, set_iface_first = true; /* try to set the interface... */ + if (chip->quirk_flags & QUIRK_FLAG_SKIP_IFACE_SETUP) + continue; + usb_set_interface(chip->dev, iface_no, 0); if (set_iface_first) usb_set_interface(chip->dev, iface_no, altno); diff --git a/sound/usb/usbaudio.h b/sound/usb/usbaudio.h index 79978cae9799c..085530cf62d92 100644 --- a/sound/usb/usbaudio.h +++ b/sound/usb/usbaudio.h @@ -224,6 +224,10 @@ extern bool snd_usb_skip_validation; * playback value represents muted state instead of minimum audible volume * QUIRK_FLAG_MIXER_CAPTURE_MIN_MUTE * Similar to QUIRK_FLAG_MIXER_PLAYBACK_MIN_MUTE, but for capture streams + * QUIRK_FLAG_SKIP_IFACE_SETUP + * Skip the probe-time interface setup (usb_set_interface, + * init_pitch, init_sample_rate); redundant with + * snd_usb_endpoint_prepare() at stream-open time */ enum { @@ -253,6 +257,7 @@ enum { QUIRK_TYPE_MIC_RES_384 = 23, QUIRK_TYPE_MIXER_PLAYBACK_MIN_MUTE = 24, QUIRK_TYPE_MIXER_CAPTURE_MIN_MUTE = 25, + QUIRK_TYPE_SKIP_IFACE_SETUP = 26, /* Please also edit snd_usb_audio_quirk_flag_names */ }; @@ -284,5 +289,6 @@ enum { #define QUIRK_FLAG_MIC_RES_384 QUIRK_FLAG(MIC_RES_384) #define QUIRK_FLAG_MIXER_PLAYBACK_MIN_MUTE QUIRK_FLAG(MIXER_PLAYBACK_MIN_MUTE) #define QUIRK_FLAG_MIXER_CAPTURE_MIN_MUTE QUIRK_FLAG(MIXER_CAPTURE_MIN_MUTE) +#define QUIRK_FLAG_SKIP_IFACE_SETUP QUIRK_FLAG(SKIP_IFACE_SETUP) #endif /* __USBAUDIO_H */ diff --git a/sound/usb/validate.c b/sound/usb/validate.c index 4bb4893f6e74f..f62b7cc041dc9 100644 --- a/sound/usb/validate.c +++ b/sound/usb/validate.c @@ -281,7 +281,7 @@ static const struct usb_desc_validator audio_validators[] = { /* UAC_VERSION_2, UAC2_SAMPLE_RATE_CONVERTER: not implemented yet */ /* UAC3 */ - FIXED(UAC_VERSION_2, UAC_HEADER, struct uac3_ac_header_descriptor), + FIXED(UAC_VERSION_3, UAC_HEADER, struct uac3_ac_header_descriptor), FIXED(UAC_VERSION_3, UAC_INPUT_TERMINAL, struct uac3_input_terminal_descriptor), FIXED(UAC_VERSION_3, UAC_OUTPUT_TERMINAL, diff --git a/tools/arch/arm64/include/uapi/asm/unistd.h b/tools/arch/arm64/include/uapi/asm/unistd.h index df36f23876e86..9306726337fe0 100644 --- a/tools/arch/arm64/include/uapi/asm/unistd.h +++ b/tools/arch/arm64/include/uapi/asm/unistd.h @@ -1,2 +1,24 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#include +/* + * Copyright (C) 2012 ARM Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#define __ARCH_WANT_RENAMEAT +#define __ARCH_WANT_NEW_STAT +#define __ARCH_WANT_SET_GET_RLIMIT +#define __ARCH_WANT_TIME32_SYSCALLS +#define __ARCH_WANT_MEMFD_SECRET + +#include diff --git a/tools/bpf/bpftool/Makefile b/tools/bpf/bpftool/Makefile index 586d1b2595d16..fd43e5ea63f38 100644 --- a/tools/bpf/bpftool/Makefile +++ b/tools/bpf/bpftool/Makefile @@ -130,8 +130,8 @@ include $(FEATURES_DUMP) endif endif -LIBS = $(LIBBPF) -lelf -lz -lcrypto -LIBS_BOOTSTRAP = $(LIBBPF_BOOTSTRAP) -lelf -lz -lcrypto +LIBS = $(LIBBPF) -lelf -lcrypto -lz +LIBS_BOOTSTRAP = $(LIBBPF_BOOTSTRAP) -lelf -lcrypto -lz ifeq ($(feature-libelf-zstd),1) LIBS += -lzstd diff --git a/tools/bpf/bpftool/net.c b/tools/bpf/bpftool/net.c index cfc6f944f7c33..1a06b0b5eef35 100644 --- a/tools/bpf/bpftool/net.c +++ b/tools/bpf/bpftool/net.c @@ -156,7 +156,7 @@ static int netlink_recv(int sock, __u32 nl_pid, __u32 seq, bool multipart = true; struct nlmsgerr *err; struct nlmsghdr *nh; - char buf[4096]; + char buf[8192]; int len, ret; while (multipart) { @@ -201,6 +201,9 @@ static int netlink_recv(int sock, __u32 nl_pid, __u32 seq, return ret; } } + + if (len) + p_err("Invalid message or trailing data in Netlink response: %d bytes left", len); } ret = 0; done: diff --git a/tools/include/linux/bitfield.h b/tools/include/linux/bitfield.h index 6093fa6db2600..ddf81f24956ba 100644 --- a/tools/include/linux/bitfield.h +++ b/tools/include/linux/bitfield.h @@ -8,6 +8,7 @@ #define _LINUX_BITFIELD_H #include +#include #include /* diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index 6388392f49a0b..53c6624161d79 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -1762,9 +1762,18 @@ static int btf_dump_get_bitfield_value(struct btf_dump *d, __u16 left_shift_bits, right_shift_bits; const __u8 *bytes = data; __u8 nr_copy_bits; + __u8 start_bit, nr_bytes; __u64 num = 0; int i; + /* Calculate how many bytes cover the bitfield */ + start_bit = bits_offset % 8; + nr_bytes = (start_bit + bit_sz + 7) / 8; + + /* Bound check */ + if (data + nr_bytes > d->typed_dump->data_end) + return -E2BIG; + /* Maximum supported bitfield size is 64 bits */ if (t->size > 8) { pr_warn("unexpected bitfield size %d\n", t->size); diff --git a/tools/lib/bpf/linker.c b/tools/lib/bpf/linker.c index 56ae77047bc36..b86b8ae3b6341 100644 --- a/tools/lib/bpf/linker.c +++ b/tools/lib/bpf/linker.c @@ -581,7 +581,7 @@ int bpf_linker__add_buf(struct bpf_linker *linker, void *buf, size_t buf_sz, written = 0; while (written < buf_sz) { - ret = write(fd, buf, buf_sz); + ret = write(fd, buf + written, buf_sz - written); if (ret < 0) { ret = -errno; pr_warn("failed to write '%s': %s\n", filename, errstr(ret)); diff --git a/tools/lib/bpf/netlink.c b/tools/lib/bpf/netlink.c index c997e69d507fe..c9a78fb16f115 100644 --- a/tools/lib/bpf/netlink.c +++ b/tools/lib/bpf/netlink.c @@ -143,7 +143,7 @@ static int libbpf_netlink_recv(int sock, __u32 nl_pid, int seq, struct nlmsghdr *nh; int len, ret; - ret = alloc_iov(&iov, 4096); + ret = alloc_iov(&iov, 8192); if (ret) goto done; @@ -212,6 +212,8 @@ static int libbpf_netlink_recv(int sock, __u32 nl_pid, int seq, } } } + if (len) + pr_warn("Invalid message or trailing data in Netlink response: %d bytes left\n", len); } ret = 0; done: diff --git a/tools/lib/perf/Makefile b/tools/lib/perf/Makefile index 7fbb50b74c00b..5c64122bf5374 100644 --- a/tools/lib/perf/Makefile +++ b/tools/lib/perf/Makefile @@ -51,9 +51,9 @@ INCLUDES = \ -I$(srctree)/tools/include/uapi # Append required CFLAGS +override CFLAGS := $(INCLUDES) $(CFLAGS) override CFLAGS += -g -Werror -Wall override CFLAGS += -fPIC -override CFLAGS += $(INCLUDES) override CFLAGS += -fvisibility=hidden override CFLAGS += $(EXTRA_WARNINGS) override CFLAGS += $(EXTRA_CFLAGS) diff --git a/tools/lib/subcmd/help.c b/tools/lib/subcmd/help.c index ddaeb4eb3e249..db94aa685b73b 100644 --- a/tools/lib/subcmd/help.c +++ b/tools/lib/subcmd/help.c @@ -97,11 +97,13 @@ void exclude_cmds(struct cmdnames *cmds, struct cmdnames *excludes) ei++; } } - if (ci != cj) { - while (ci < cmds->cnt) { - cmds->names[cj++] = cmds->names[ci]; - cmds->names[ci++] = NULL; + while (ci < cmds->cnt) { + if (ci != cj) { + cmds->names[cj] = cmds->names[ci]; + cmds->names[ci] = NULL; } + ci++; + cj++; } for (ci = cj; ci < cmds->cnt; ci++) assert(cmds->names[ci] == NULL); diff --git a/tools/net/sunrpc/xdrgen/generators/__init__.py b/tools/net/sunrpc/xdrgen/generators/__init__.py index b98574a36a4ac..a2eb6652ac900 100644 --- a/tools/net/sunrpc/xdrgen/generators/__init__.py +++ b/tools/net/sunrpc/xdrgen/generators/__init__.py @@ -6,7 +6,7 @@ from jinja2 import Environment, FileSystemLoader, Template from xdr_ast import _XdrAst, Specification, _RpcProgram, _XdrTypeSpecifier -from xdr_ast import public_apis, pass_by_reference, get_header_name +from xdr_ast import public_apis, pass_by_reference, structs, get_header_name from xdr_parse import get_xdr_annotate @@ -22,6 +22,7 @@ def create_jinja2_environment(language: str, xdr_type: str) -> Environment: environment.globals["annotate"] = get_xdr_annotate() environment.globals["public_apis"] = public_apis environment.globals["pass_by_reference"] = pass_by_reference + environment.globals["structs"] = structs return environment case _: raise NotImplementedError("Language not supported") diff --git a/tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 b/tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 index 0b1709cca0d4a..19b219dd276d3 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/program/decoder/argument.j2 @@ -14,7 +14,11 @@ bool {{ program }}_svc_decode_{{ argument }}(struct svc_rqst *rqstp, struct xdr_ {% if argument == 'void' %} return xdrgen_decode_void(xdr); {% else %} +{% if argument in structs %} struct {{ argument }} *argp = rqstp->rq_argp; +{% else %} + {{ argument }} *argp = rqstp->rq_argp; +{% endif %} return xdrgen_decode_{{ argument }}(xdr, argp); {% endif %} diff --git a/tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 b/tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 index 6fc61a5d47b7f..746592cfda562 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/program/encoder/result.j2 @@ -14,8 +14,14 @@ bool {{ program }}_svc_encode_{{ result }}(struct svc_rqst *rqstp, struct xdr_st {% if result == 'void' %} return xdrgen_encode_void(xdr); {% else %} +{% if result in structs %} struct {{ result }} *resp = rqstp->rq_resp; return xdrgen_encode_{{ result }}(xdr, resp); +{% else %} + {{ result }} *resp = rqstp->rq_resp; + + return xdrgen_encode_{{ result }}(xdr, *resp); +{% endif %} {% endif %} } diff --git a/tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 b/tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 index c5518c519854a..df3598c38b2c2 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/source_top/client.j2 @@ -8,6 +8,5 @@ #include #include #include -#include #include diff --git a/tools/objtool/Makefile b/tools/objtool/Makefile index 8c20361dd100e..99d3897e046c6 100644 --- a/tools/objtool/Makefile +++ b/tools/objtool/Makefile @@ -7,6 +7,8 @@ srctree := $(patsubst %/,%,$(dir $(CURDIR))) srctree := $(patsubst %/,%,$(dir $(srctree))) endif +RM ?= rm -f + LIBSUBCMD_DIR = $(srctree)/tools/lib/subcmd/ ifneq ($(OUTPUT),) LIBSUBCMD_OUTPUT = $(abspath $(OUTPUT))/libsubcmd diff --git a/tools/perf/Makefile.perf b/tools/perf/Makefile.perf index 47c906b807ef2..562b9d356d2a2 100644 --- a/tools/perf/Makefile.perf +++ b/tools/perf/Makefile.perf @@ -88,8 +88,6 @@ include ../scripts/utilities.mak # # Define NO_LIBBPF if you do not want BPF support # -# Define NO_LIBCAP if you do not want process capabilities considered by perf -# # Define NO_SDT if you do not want to define SDT event in perf tools, # note that it doesn't disable SDT scanning support. # @@ -253,11 +251,12 @@ else endif # shellcheck is using in tools/perf/tests/Build with option -a/--check-sourced ( -# introduced in v0.4.7) and -S/--severity (introduced in v0.6.0). So make the -# minimal shellcheck version as v0.6.0. +# introduced in v0.4.7) and -S/--severity (introduced in v0.6.0) as well as +# dynamic source inclusions (properly handled since v0.7.2). +# So make the minimal shellcheck version as v0.7.2. ifneq ($(SHELLCHECK),) ifeq ($(shell expr $(shell $(SHELLCHECK) --version | grep version: | \ - sed -e 's/.\+ \([0-9]\+\).\([0-9]\+\).\([0-9]\+\)/\1\2\3/g') \< 060), 1) + sed -e 's/.\+ \([0-9]\+\).\([0-9]\+\).\([0-9]\+\)/\1\2\3/g') \< 072), 1) SHELLCHECK := else SHELLCHECK := $(SHELLCHECK) -s bash -a -S warning diff --git a/tools/perf/arch/loongarch/annotate/instructions.c b/tools/perf/arch/loongarch/annotate/instructions.c index 70262d5f14442..1c3abb43c8d72 100644 --- a/tools/perf/arch/loongarch/annotate/instructions.c +++ b/tools/perf/arch/loongarch/annotate/instructions.c @@ -10,9 +10,7 @@ static int loongarch_call__parse(struct arch *arch, struct ins_operands *ops, st { char *c, *endptr, *tok, *name; struct map *map = ms->map; - struct addr_map_symbol target = { - .ms = { .map = map, }, - }; + struct addr_map_symbol target; c = strchr(ops->raw, '#'); if (c++ == NULL) @@ -38,12 +36,16 @@ static int loongarch_call__parse(struct arch *arch, struct ins_operands *ops, st if (ops->target.name == NULL) return -1; - target.addr = map__objdump_2mem(map, ops->target.addr); + target = (struct addr_map_symbol) { + .ms = { .map = map__get(map), }, + .addr = map__objdump_2mem(map, ops->target.addr), + }; if (maps__find_ams(ms->maps, &target) == 0 && map__rip_2objdump(target.ms.map, map__map_ip(target.ms.map, target.addr)) == ops->target.addr) ops->target.sym = target.ms.sym; + addr_map_symbol__exit(&target); return 0; } @@ -58,7 +60,7 @@ static int loongarch_jump__parse(struct arch *arch, struct ins_operands *ops, st struct map *map = ms->map; struct symbol *sym = ms->sym; struct addr_map_symbol target = { - .ms = { .map = map, }, + .ms = { .map = map__get(map), }, }; const char *c = strchr(ops->raw, '#'); u64 start, end; @@ -90,7 +92,7 @@ static int loongarch_jump__parse(struct arch *arch, struct ins_operands *ops, st } else { ops->target.offset_avail = false; } - + addr_map_symbol__exit(&target); return 0; } diff --git a/tools/perf/arch/s390/annotate/instructions.c b/tools/perf/arch/s390/annotate/instructions.c index c61193f1e0964..626e6d2cbc81a 100644 --- a/tools/perf/arch/s390/annotate/instructions.c +++ b/tools/perf/arch/s390/annotate/instructions.c @@ -6,9 +6,7 @@ static int s390_call__parse(struct arch *arch, struct ins_operands *ops, { char *endptr, *tok, *name; struct map *map = ms->map; - struct addr_map_symbol target = { - .ms = { .map = map, }, - }; + struct addr_map_symbol target; tok = strchr(ops->raw, ','); if (!tok) @@ -36,12 +34,17 @@ static int s390_call__parse(struct arch *arch, struct ins_operands *ops, if (ops->target.name == NULL) return -1; - target.addr = map__objdump_2mem(map, ops->target.addr); + + target = (struct addr_map_symbol) { + .ms = { .map = map__get(map), }, + .addr = map__objdump_2mem(map, ops->target.addr), + }; if (maps__find_ams(ms->maps, &target) == 0 && map__rip_2objdump(target.ms.map, map__map_ip(target.ms.map, target.addr)) == ops->target.addr) ops->target.sym = target.ms.sym; + addr_map_symbol__exit(&target); return 0; } diff --git a/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json b/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json index ff6627a778057..06bbaea159259 100644 --- a/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json +++ b/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json @@ -70,19 +70,19 @@ "EventName": "ls_mab_alloc.load_store_allocations", "EventCode": "0x41", "BriefDescription": "Miss Address Buffer (MAB) entries allocated by a Load-Store (LS) pipe for load-store allocations.", - "UMask": "0x3f" + "UMask": "0x07" }, { "EventName": "ls_mab_alloc.hardware_prefetcher_allocations", "EventCode": "0x41", "BriefDescription": "Miss Address Buffer (MAB) entries allocated by a Load-Store (LS) pipe for hardware prefetcher allocations.", - "UMask": "0x40" + "UMask": "0x08" }, { "EventName": "ls_mab_alloc.all_allocations", "EventCode": "0x41", "BriefDescription": "Miss Address Buffer (MAB) entries allocated by a Load-Store (LS) pipe for all types of allocations.", - "UMask": "0x7f" + "UMask": "0x0f" }, { "EventName": "ls_dmnd_fills_from_sys.local_l2", diff --git a/tools/perf/tests/make b/tools/perf/tests/make index b650ce8864ed5..fa4500c65949a 100644 --- a/tools/perf/tests/make +++ b/tools/perf/tests/make @@ -123,7 +123,7 @@ make_minimal += NO_DEMANGLE=1 NO_LIBELF=1 NO_BACKTRACE=1 make_minimal += NO_LIBNUMA=1 NO_LIBBIONIC=1 NO_LIBDW=1 make_minimal += NO_LIBDW_DWARF_UNWIND=1 NO_AUXTRACE=1 NO_LIBBPF=1 make_minimal += NO_SDT=1 NO_JVMTI=1 NO_LIBZSTD=1 -make_minimal += NO_LIBCAP=1 NO_CAPSTONE=1 +make_minimal += NO_CAPSTONE=1 # $(run) contains all available tests run := make_pure diff --git a/tools/perf/tests/shell/sched.sh b/tools/perf/tests/shell/sched.sh index b9b81eaf856e6..b9637069adb1f 100755 --- a/tools/perf/tests/shell/sched.sh +++ b/tools/perf/tests/shell/sched.sh @@ -53,7 +53,7 @@ start_noploops() { } cleanup_noploops() { - kill "$PID1" "$PID2" + kill "$PID1" "$PID2" || true } test_sched_record() { diff --git a/tools/perf/tests/shell/stat.sh b/tools/perf/tests/shell/stat.sh index 8a100a7f2dc17..d72d16d0b8c46 100755 --- a/tools/perf/tests/shell/stat.sh +++ b/tools/perf/tests/shell/stat.sh @@ -18,7 +18,7 @@ test_default_stat() { test_stat_record_report() { echo "stat record and report test" - if ! perf stat record -o - true | perf stat report -i - 2>&1 | \ + if ! perf stat record -e task-clock -o - true | perf stat report -i - 2>&1 | \ grep -E -q "Performance counter stats for 'pipe':" then echo "stat record and report test [Failed]" @@ -30,7 +30,7 @@ test_stat_record_report() { test_stat_record_script() { echo "stat record and script test" - if ! perf stat record -o - true | perf script -i - 2>&1 | \ + if ! perf stat record -e task-clock -o - true | perf script -i - 2>&1 | \ grep -E -q "CPU[[:space:]]+THREAD[[:space:]]+VAL[[:space:]]+ENA[[:space:]]+RUN[[:space:]]+TIME[[:space:]]+EVENT" then echo "stat record and script test [Failed]" @@ -196,7 +196,7 @@ test_hybrid() { fi # Run default Perf stat - cycles_events=$(perf stat -- true 2>&1 | grep -E "/cycles/[uH]*| cycles[:uH]* " -c) + cycles_events=$(perf stat -a -- sleep 0.1 2>&1 | grep -E "/cpu-cycles/[uH]*| cpu-cycles[:uH]* " | wc -l) # The expectation is that default output will have a cycles events on each # hybrid PMU. In situations with no cycles PMU events, like virtualized, this diff --git a/tools/perf/util/addr2line.c b/tools/perf/util/addr2line.c index f2d94a3272d71..a8b39f4f202b6 100644 --- a/tools/perf/util/addr2line.c +++ b/tools/perf/util/addr2line.c @@ -18,8 +18,8 @@ #define MAX_INLINE_NEST 1024 -/* If addr2line doesn't return data for 1 second then timeout. */ -int addr2line_timeout_ms = 1 * 1000; +/* If addr2line doesn't return data for 5 seconds then timeout. */ +int addr2line_timeout_ms = 5 * 1000; static int filename_split(char *filename, unsigned int *line_nr) { diff --git a/tools/perf/util/annotate.c b/tools/perf/util/annotate.c index 1d6900033b3a0..dc80d922f450d 100644 --- a/tools/perf/util/annotate.c +++ b/tools/perf/util/annotate.c @@ -1031,7 +1031,7 @@ int symbol__annotate(struct map_symbol *ms, struct evsel *evsel, return 0; args.arch = arch; - args.ms = *ms; + args.ms = ms; if (notes->src == NULL) { notes->src = annotated_source__new(); diff --git a/tools/perf/util/capstone.c b/tools/perf/util/capstone.c index be5fd44b1f9dc..2c7feab61b7bf 100644 --- a/tools/perf/util/capstone.c +++ b/tools/perf/util/capstone.c @@ -143,7 +143,7 @@ static void print_capstone_detail(cs_insn *insn, char *buf, size_t len, struct annotate_args *args, u64 addr) { int i; - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct symbol *sym; /* TODO: support more architectures */ @@ -222,7 +222,7 @@ int symbol__disassemble_capstone(const char *filename __maybe_unused, { #ifdef HAVE_LIBCAPSTONE_SUPPORT struct annotation *notes = symbol__annotation(sym); - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct dso *dso = map__dso(map); u64 start = map__rip_2objdump(map, sym->start); u64 offset; @@ -256,7 +256,7 @@ int symbol__disassemble_capstone(const char *filename __maybe_unused, args->line = disasm_buf; args->line_nr = 0; args->fileloc = NULL; - args->ms.sym = sym; + args->ms->sym = sym; dl = disasm_line__new(args); if (dl == NULL) @@ -268,7 +268,7 @@ int symbol__disassemble_capstone(const char *filename __maybe_unused, !strcmp(args->options->disassembler_style, "att")) disassembler_style = true; - if (capstone_init(maps__machine(args->ms.maps), &handle, is_64bit, disassembler_style) < 0) + if (capstone_init(maps__machine(args->ms->maps), &handle, is_64bit, disassembler_style) < 0) goto err; needs_cs_close = true; @@ -345,7 +345,7 @@ int symbol__disassemble_capstone_powerpc(const char *filename __maybe_unused, { #ifdef HAVE_LIBCAPSTONE_SUPPORT struct annotation *notes = symbol__annotation(sym); - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct dso *dso = map__dso(map); struct nscookie nsc; u64 start = map__rip_2objdump(map, sym->start); @@ -382,7 +382,7 @@ int symbol__disassemble_capstone_powerpc(const char *filename __maybe_unused, !strcmp(args->options->disassembler_style, "att")) disassembler_style = true; - if (capstone_init(maps__machine(args->ms.maps), &handle, is_64bit, disassembler_style) < 0) + if (capstone_init(maps__machine(args->ms->maps), &handle, is_64bit, disassembler_style) < 0) goto err; needs_cs_close = true; @@ -408,7 +408,7 @@ int symbol__disassemble_capstone_powerpc(const char *filename __maybe_unused, args->line = disasm_buf; args->line_nr = 0; args->fileloc = NULL; - args->ms.sym = sym; + args->ms->sym = sym; dl = disasm_line__new(args); if (dl == NULL) diff --git a/tools/perf/util/cs-etm.c b/tools/perf/util/cs-etm.c index 30f4bb3e7fa30..06eb1a56430cb 100644 --- a/tools/perf/util/cs-etm.c +++ b/tools/perf/util/cs-etm.c @@ -3089,7 +3089,7 @@ static int cs_etm__queue_aux_fragment(struct perf_session *session, off_t file_o if (aux_offset >= auxtrace_event->offset && aux_offset + aux_size <= auxtrace_event->offset + auxtrace_event->size) { - struct cs_etm_queue *etmq = etm->queues.queue_array[auxtrace_event->idx].priv; + struct cs_etm_queue *etmq = cs_etm__get_queue(etm, auxtrace_event->cpu); /* * If this AUX event was inside this buffer somewhere, create a new auxtrace event @@ -3098,6 +3098,7 @@ static int cs_etm__queue_aux_fragment(struct perf_session *session, off_t file_o auxtrace_fragment.auxtrace = *auxtrace_event; auxtrace_fragment.auxtrace.size = aux_size; auxtrace_fragment.auxtrace.offset = aux_offset; + auxtrace_fragment.auxtrace.idx = etmq->queue_nr; file_offset += aux_offset - auxtrace_event->offset + auxtrace_event->header.size; pr_debug3("CS ETM: Queue buffer size: %#"PRI_lx64" offset: %#"PRI_lx64 diff --git a/tools/perf/util/disasm.c b/tools/perf/util/disasm.c index 50b9433f3f8e6..88706b98b9064 100644 --- a/tools/perf/util/disasm.c +++ b/tools/perf/util/disasm.c @@ -81,7 +81,7 @@ static int arch__grow_instructions(struct arch *arch) if (new_instructions == NULL) return -1; - memcpy(new_instructions, arch->instructions, arch->nr_instructions); + memcpy(new_instructions, arch->instructions, arch->nr_instructions * sizeof(struct ins)); goto out_update_instructions; } @@ -269,9 +269,7 @@ static int call__parse(struct arch *arch, struct ins_operands *ops, struct map_s { char *endptr, *tok, *name; struct map *map = ms->map; - struct addr_map_symbol target = { - .ms = { .map = map, }, - }; + struct addr_map_symbol target; ops->target.addr = strtoull(ops->raw, &endptr, 16); @@ -296,12 +294,16 @@ static int call__parse(struct arch *arch, struct ins_operands *ops, struct map_s if (ops->target.name == NULL) return -1; find_target: - target.addr = map__objdump_2mem(map, ops->target.addr); + target = (struct addr_map_symbol) { + .ms = { .map = map__get(map), }, + .addr = map__objdump_2mem(map, ops->target.addr), + }; if (maps__find_ams(ms->maps, &target) == 0 && map__rip_2objdump(target.ms.map, map__map_ip(target.ms.map, target.addr)) == ops->target.addr) ops->target.sym = target.ms.sym; + addr_map_symbol__exit(&target); return 0; indirect_call: @@ -366,7 +368,7 @@ static int jump__parse(struct arch *arch, struct ins_operands *ops, struct map_s struct map *map = ms->map; struct symbol *sym = ms->sym; struct addr_map_symbol target = { - .ms = { .map = map, }, + .ms = { .map = map__get(map), }, }; const char *c = strchr(ops->raw, ','); u64 start, end; @@ -440,7 +442,7 @@ static int jump__parse(struct arch *arch, struct ins_operands *ops, struct map_s } else { ops->target.offset_avail = false; } - + addr_map_symbol__exit(&target); return 0; } @@ -1046,7 +1048,7 @@ static size_t disasm_line_size(int nr) struct disasm_line *disasm_line__new(struct annotate_args *args) { struct disasm_line *dl = NULL; - struct annotation *notes = symbol__annotation(args->ms.sym); + struct annotation *notes = symbol__annotation(args->ms->sym); int nr = notes->src->nr_events; dl = zalloc(disasm_line_size(nr)); @@ -1064,7 +1066,7 @@ struct disasm_line *disasm_line__new(struct annotate_args *args) } else if (disasm_line__parse(dl->al.line, &dl->ins.name, &dl->ops.raw) < 0) goto out_free_line; - disasm_line__init_ins(dl, args->arch, &args->ms); + disasm_line__init_ins(dl, args->arch, args->ms); } return dl; @@ -1119,7 +1121,7 @@ static int symbol__parse_objdump_line(struct symbol *sym, struct annotate_args *args, char *parsed_line, int *line_nr, char **fileloc) { - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct annotation *notes = symbol__annotation(sym); struct disasm_line *dl; char *tmp; @@ -1151,7 +1153,7 @@ static int symbol__parse_objdump_line(struct symbol *sym, args->line = parsed_line; args->line_nr = *line_nr; args->fileloc = *fileloc; - args->ms.sym = sym; + args->ms->sym = sym; dl = disasm_line__new(args); (*line_nr)++; @@ -1169,12 +1171,14 @@ static int symbol__parse_objdump_line(struct symbol *sym, if (dl->ins.ops && ins__is_call(&dl->ins) && !dl->ops.target.sym) { struct addr_map_symbol target = { .addr = dl->ops.target.addr, - .ms = { .map = map, }, + .ms = { .map = map__get(map), }, }; - if (!maps__find_ams(args->ms.maps, &target) && + if (!maps__find_ams(args->ms->maps, &target) && target.ms.sym->start == target.al_addr) dl->ops.target.sym = target.ms.sym; + + addr_map_symbol__exit(&target); } annotation_line__add(&dl->al, ¬es->src->source); @@ -1338,7 +1342,7 @@ static int symbol__disassemble_raw(char *filename, struct symbol *sym, struct annotate_args *args) { struct annotation *notes = symbol__annotation(sym); - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct dso *dso = map__dso(map); u64 start = map__rip_2objdump(map, sym->start); u64 end = map__rip_2objdump(map, sym->end); @@ -1375,7 +1379,7 @@ static int symbol__disassemble_raw(char *filename, struct symbol *sym, args->line = disasm_buf; args->line_nr = 0; args->fileloc = NULL; - args->ms.sym = sym; + args->ms->sym = sym; dl = disasm_line__new(args); if (dl == NULL) @@ -1501,7 +1505,7 @@ static int symbol__disassemble_objdump(const char *filename, struct symbol *sym, struct annotate_args *args) { struct annotation_options *opts = &annotate_opts; - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct dso *dso = map__dso(map); char *command; FILE *file; @@ -1644,7 +1648,7 @@ static int symbol__disassemble_objdump(const char *filename, struct symbol *sym, int symbol__disassemble(struct symbol *sym, struct annotate_args *args) { struct annotation_options *options = args->options; - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct dso *dso = map__dso(map); char symfs_filename[PATH_MAX]; bool delete_extract = false; diff --git a/tools/perf/util/disasm.h b/tools/perf/util/disasm.h index d2cb555e4a3be..a3ea9d6762816 100644 --- a/tools/perf/util/disasm.h +++ b/tools/perf/util/disasm.h @@ -97,7 +97,7 @@ struct ins_ops { struct annotate_args { struct arch *arch; - struct map_symbol ms; + struct map_symbol *ms; struct annotation_options *options; s64 offset; char *line; diff --git a/tools/perf/util/dso.c b/tools/perf/util/dso.c index 344e689567ee1..dc202d4943721 100644 --- a/tools/perf/util/dso.c +++ b/tools/perf/util/dso.c @@ -111,7 +111,7 @@ bool dso__is_object_file(const struct dso *dso) int dso__read_binary_type_filename(const struct dso *dso, enum dso_binary_type type, - char *root_dir, char *filename, size_t size) + const char *root_dir, char *filename, size_t size) { char build_id_hex[SBUILD_ID_SIZE]; int ret = 0; @@ -563,20 +563,15 @@ char *dso__filename_with_chroot(const struct dso *dso, const char *filename) return filename_with_chroot(nsinfo__pid(dso__nsinfo_const(dso)), filename); } -static int __open_dso(struct dso *dso, struct machine *machine) - EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +static char *dso__get_filename(struct dso *dso, const char *root_dir, + bool *decomp) { - int fd = -EINVAL; - char *root_dir = (char *)""; char *name = malloc(PATH_MAX); - bool decomp = false; - if (!name) - return -ENOMEM; + *decomp = false; - mutex_lock(dso__lock(dso)); - if (machine) - root_dir = machine->root_dir; + if (name == NULL) + return NULL; if (dso__read_binary_type_filename(dso, dso__binary_type(dso), root_dir, name, PATH_MAX)) @@ -601,20 +596,38 @@ static int __open_dso(struct dso *dso, struct machine *machine) size_t len = sizeof(newpath); if (dso__decompress_kmodule_path(dso, name, newpath, len) < 0) { - fd = -(*dso__load_errno(dso)); + errno = *dso__load_errno(dso); goto out; } - decomp = true; + *decomp = true; strcpy(name, newpath); } + return name; + +out: + free(name); + return NULL; +} - fd = do_open(name); +static int __open_dso(struct dso *dso, struct machine *machine) + EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + int fd = -EINVAL; + char *name; + bool decomp = false; + + mutex_lock(dso__lock(dso)); + + name = dso__get_filename(dso, machine ? machine->root_dir : "", &decomp); + if (name) + fd = do_open(name); + else + fd = -errno; if (decomp) unlink(name); -out: mutex_unlock(dso__lock(dso)); free(name); return fd; @@ -1910,3 +1923,23 @@ const u8 *dso__read_symbol(struct dso *dso, const char *symfs_filename, return __dso__read_symbol(dso, symfs_filename, start, len, out_buf, out_buf_len, is_64bit); } + +struct debuginfo *dso__debuginfo(struct dso *dso) +{ + char *name; + bool decomp = false; + struct debuginfo *dinfo = NULL; + + mutex_lock(dso__lock(dso)); + + name = dso__get_filename(dso, "", &decomp); + if (name) + dinfo = debuginfo__new(name); + + if (decomp) + unlink(name); + + mutex_unlock(dso__lock(dso)); + free(name); + return dinfo; +} diff --git a/tools/perf/util/dso.h b/tools/perf/util/dso.h index f8ccb9816b89c..54e470dd07305 100644 --- a/tools/perf/util/dso.h +++ b/tools/perf/util/dso.h @@ -766,7 +766,7 @@ int dso__kernel_module_get_build_id(struct dso *dso, const char *root_dir); char dso__symtab_origin(const struct dso *dso); int dso__read_binary_type_filename(const struct dso *dso, enum dso_binary_type type, - char *root_dir, char *filename, size_t size); + const char *root_dir, char *filename, size_t size); bool is_kernel_module(const char *pathname, int cpumode); bool dso__needs_decompress(struct dso *dso); int dso__decompress_kmodule_fd(struct dso *dso, const char *name); @@ -915,14 +915,7 @@ u64 dso__findnew_global_type(struct dso *dso, u64 addr, u64 offset); bool perf_pid_map_tid(const char *dso_name, int *tid); bool is_perf_pid_map_name(const char *dso_name); -/* - * In the future, we may get debuginfo using build-ID (w/o path). - * Add this helper is for the smooth conversion. - */ -static inline struct debuginfo *dso__debuginfo(struct dso *dso) -{ - return debuginfo__new(dso__long_name(dso)); -} +struct debuginfo *dso__debuginfo(struct dso *dso); const u8 *dso__read_symbol(struct dso *dso, const char *symfs_filename, const struct map *map, const struct symbol *sym, diff --git a/tools/perf/util/evsel_fprintf.c b/tools/perf/util/evsel_fprintf.c index 103984b29b1e1..cbf2dd2dfc6f4 100644 --- a/tools/perf/util/evsel_fprintf.c +++ b/tools/perf/util/evsel_fprintf.c @@ -182,8 +182,12 @@ int sample__fprintf_callchain(struct perf_sample *sample, int left_alignment, if (print_dso && (!sym || !sym->inlined)) printed += map__fprintf_dsoname_dsoff(map, print_dsoff, addr, fp); - if (print_srcline) - printed += map__fprintf_srcline(map, addr, "\n ", fp); + if (print_srcline) { + if (node->srcline) + printed += fprintf(fp, "\n %s", node->srcline); + else + printed += map__fprintf_srcline(map, addr, "\n ", fp); + } if (sym && sym->inlined) printed += fprintf(fp, " (inlined)"); diff --git a/tools/perf/util/libbfd.c b/tools/perf/util/libbfd.c index 6434c2dccd4a4..2324f6846d510 100644 --- a/tools/perf/util/libbfd.c +++ b/tools/perf/util/libbfd.c @@ -494,7 +494,7 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, struct bpf_prog_info_node *info_node; int len = sym->end - sym->start; disassembler_ftype disassemble; - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct perf_bpil *info_linear; struct disassemble_info info; struct dso *dso = map__dso(map); @@ -605,7 +605,7 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, args->line = strdup(srcline); args->line_nr = 0; args->fileloc = NULL; - args->ms.sym = sym; + args->ms->sym = sym; dl = disasm_line__new(args); if (dl) { annotation_line__add(&dl->al, @@ -617,7 +617,7 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, args->line = buf + prev_buf_size; args->line_nr = 0; args->fileloc = NULL; - args->ms.sym = sym; + args->ms->sym = sym; dl = disasm_line__new(args); if (dl) annotation_line__add(&dl->al, ¬es->src->source); diff --git a/tools/perf/util/llvm.c b/tools/perf/util/llvm.c index 2ebf1f5f65bf7..4ada9a10bd93f 100644 --- a/tools/perf/util/llvm.c +++ b/tools/perf/util/llvm.c @@ -118,7 +118,7 @@ int symbol__disassemble_llvm(const char *filename, struct symbol *sym, { #ifdef HAVE_LIBLLVM_SUPPORT struct annotation *notes = symbol__annotation(sym); - struct map *map = args->ms.map; + struct map *map = args->ms->map; struct dso *dso = map__dso(map); u64 start = map__rip_2objdump(map, sym->start); /* Malloc-ed buffer containing instructions read from disk. */ @@ -184,7 +184,7 @@ int symbol__disassemble_llvm(const char *filename, struct symbol *sym, args->line = disasm_buf; args->line_nr = 0; args->fileloc = NULL; - args->ms.sym = sym; + args->ms->sym = sym; dl = disasm_line__new(args); if (dl == NULL) @@ -242,7 +242,7 @@ int symbol__disassemble_llvm(const char *filename, struct symbol *sym, &line_storage_len); args->line_nr = 0; args->fileloc = NULL; - args->ms.sym = sym; + args->ms->sym = sym; llvm_addr2line(filename, pc, &args->fileloc, (unsigned int *)&args->line_nr, false, NULL); diff --git a/tools/perf/util/maps.c b/tools/perf/util/maps.c index 779f6230130af..c51ec159ac769 100644 --- a/tools/perf/util/maps.c +++ b/tools/perf/util/maps.c @@ -676,6 +676,7 @@ int maps__find_ams(struct maps *maps, struct addr_map_symbol *ams) if (ams->addr < map__start(ams->ms.map) || ams->addr >= map__end(ams->ms.map)) { if (maps == NULL) return -1; + map__put(ams->ms.map); ams->ms.map = maps__find(maps, ams->addr); if (ams->ms.map == NULL) return -1; diff --git a/tools/perf/util/symbol-elf.c b/tools/perf/util/symbol-elf.c index 9e820599bab38..9d62386464680 100644 --- a/tools/perf/util/symbol-elf.c +++ b/tools/perf/util/symbol-elf.c @@ -1170,7 +1170,7 @@ int symsrc__init(struct symsrc *ss, struct dso *dso, const char *name, Elf *embedded = read_gnu_debugdata(dso, elf, name, &new_fd); if (!embedded) - goto out_close; + goto out_elf_end; elf_end(elf); close(fd); diff --git a/tools/perf/util/unwind-libdw.c b/tools/perf/util/unwind-libdw.c index ae70fb56a0572..3ff427a49e4c5 100644 --- a/tools/perf/util/unwind-libdw.c +++ b/tools/perf/util/unwind-libdw.c @@ -136,8 +136,8 @@ static int entry(u64 ip, struct unwind_info *ui) } e->ip = ip; - e->ms.maps = al.maps; - e->ms.map = al.map; + e->ms.maps = maps__get(al.maps); + e->ms.map = map__get(al.map); e->ms.sym = al.sym; pr_debug("unwind: %s:ip = 0x%" PRIx64 " (0x%" PRIx64 ")\n", @@ -325,6 +325,9 @@ int unwind__get_entries(unwind_entry_cb_t cb, void *arg, if (err) pr_debug("unwind: failed with '%s'\n", dwfl_errmsg(-1)); + for (i = 0; i < ui->idx; i++) + map_symbol__exit(&ui->entries[i].ms); + dwfl_end(ui->dwfl); free(ui); return 0; diff --git a/tools/power/cpupower/lib/cpuidle.c b/tools/power/cpupower/lib/cpuidle.c index f2c1139adf716..bd857ee7541a7 100644 --- a/tools/power/cpupower/lib/cpuidle.c +++ b/tools/power/cpupower/lib/cpuidle.c @@ -150,6 +150,7 @@ unsigned long long cpuidle_state_get_one_value(unsigned int cpu, if (len == 0) return 0; + errno = 0; value = strtoull(linebuf, &endp, 0); if (endp == linebuf || errno == ERANGE) diff --git a/tools/power/cpupower/utils/cpufreq-info.c b/tools/power/cpupower/utils/cpufreq-info.c index 7d3732f5f2f6f..5fe01e516817e 100644 --- a/tools/power/cpupower/utils/cpufreq-info.c +++ b/tools/power/cpupower/utils/cpufreq-info.c @@ -270,7 +270,7 @@ static int get_freq_hardware(unsigned int cpu, unsigned int human) { unsigned long freq; - if (cpupower_cpu_info.caps & CPUPOWER_CAP_APERF) + if (!(cpupower_cpu_info.caps & CPUPOWER_CAP_APERF)) return -EINVAL; freq = cpufreq_get_freq_hardware(cpu); diff --git a/tools/power/x86/intel-speed-select/isst-config.c b/tools/power/x86/intel-speed-select/isst-config.c index 0ce251b8d4665..a7d54dfd3c68b 100644 --- a/tools/power/x86/intel-speed-select/isst-config.c +++ b/tools/power/x86/intel-speed-select/isst-config.c @@ -950,9 +950,11 @@ int isolate_cpus(struct isst_id *id, int mask_size, cpu_set_t *cpu_mask, int lev ret = write(fd, "member", strlen("member")); if (ret == -1) { printf("Can't update to member\n"); + close(fd); return ret; } + close(fd); return 0; } diff --git a/tools/spi/.gitignore b/tools/spi/.gitignore index 14ddba3d21957..038261b34ed83 100644 --- a/tools/spi/.gitignore +++ b/tools/spi/.gitignore @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only spidev_fdx spidev_test +include/ diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py index 260d8d9aa1db4..2998e1bc088b2 100644 --- a/tools/testing/kunit/kunit_kernel.py +++ b/tools/testing/kunit/kunit_kernel.py @@ -346,8 +346,10 @@ def build_kernel(self, jobs: int, build_dir: str, make_options: Optional[List[st return self.validate_config(build_dir) def run_kernel(self, args: Optional[List[str]]=None, build_dir: str='', filter_glob: str='', filter: str='', filter_action: Optional[str]=None, timeout: Optional[int]=None) -> Iterator[str]: - if not args: - args = [] + # Copy to avoid mutating the caller-supplied list. exec_tests() reuses + # the same args across repeated run_kernel() calls (e.g. --run_isolated), + # so appending to the original would accumulate stale flags on each call. + args = list(args) if args else [] if filter_glob: args.append('kunit.filter_glob=' + filter_glob) if filter: diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index bbba921e0eacb..ed45bac1548d9 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -489,6 +489,32 @@ def fake_start(unused_args, unused_build_dir): with open(kunit_kernel.get_outfile_path(build_dir), 'rt') as outfile: self.assertEqual(outfile.read(), 'hi\nbye\n', msg='Missing some output') + def test_run_kernel_args_not_mutated(self): + """Verify run_kernel() copies args so callers can reuse them.""" + start_calls = [] + + def fake_start(start_args, unused_build_dir): + start_calls.append(list(start_args)) + return subprocess.Popen(['printf', 'KTAP version 1\n'], + text=True, stdout=subprocess.PIPE) + + with tempfile.TemporaryDirectory('') as build_dir: + tree = kunit_kernel.LinuxSourceTree(build_dir, + kunitconfig_paths=[os.devnull]) + with mock.patch.object(tree._ops, 'start', side_effect=fake_start), \ + mock.patch.object(kunit_kernel.subprocess, 'call'): + kernel_args = ['mem=1G'] + for _ in tree.run_kernel(args=kernel_args, build_dir=build_dir, + filter_glob='suite.test1'): + pass + for _ in tree.run_kernel(args=kernel_args, build_dir=build_dir, + filter_glob='suite.test2'): + pass + self.assertEqual(kernel_args, ['mem=1G'], + 'run_kernel() should not modify caller args') + self.assertIn('kunit.filter_glob=suite.test1', start_calls[0]) + self.assertIn('kunit.filter_glob=suite.test2', start_calls[1]) + def test_build_reconfig_no_config(self): with tempfile.TemporaryDirectory('') as build_dir: with open(kunit_kernel.get_kunitconfig_path(build_dir), 'w') as f: diff --git a/tools/testing/selftests/arm64/abi/hwcap.c b/tools/testing/selftests/arm64/abi/hwcap.c index 3b96d090c5ebe..09a326f375e91 100644 --- a/tools/testing/selftests/arm64/abi/hwcap.c +++ b/tools/testing/selftests/arm64/abi/hwcap.c @@ -473,8 +473,8 @@ static void sve2_sigill(void) static void sve2p1_sigill(void) { - /* BFADD Z0.H, Z0.H, Z0.H */ - asm volatile(".inst 0x65000000" : : : "z0"); + /* LD1Q {Z0.Q}, P0/Z, [Z0.D, X0] */ + asm volatile(".inst 0xC400A000" : : : "z0"); } static void sve2p2_sigill(void) diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index e59b2bbf8d920..591e7e77f89ba 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -805,7 +805,7 @@ $(OUTPUT)/test_verifier: test_verifier.c verifier/tests.h $(BPFOBJ) | $(OUTPUT) # Include find_bit.c to compile xskxceiver. EXTRA_SRC := $(TOOLSDIR)/lib/find_bit.c -$(OUTPUT)/xskxceiver: $(EXTRA_SRC) xskxceiver.c xskxceiver.h $(OUTPUT)/network_helpers.o $(OUTPUT)/xsk.o $(OUTPUT)/xsk_xdp_progs.skel.h $(BPFOBJ) | $(OUTPUT) +$(OUTPUT)/xskxceiver: $(EXTRA_SRC) test_xsk.c test_xsk.h xskxceiver.c xskxceiver.h $(OUTPUT)/network_helpers.o $(OUTPUT)/xsk.o $(OUTPUT)/xsk_xdp_progs.skel.h $(BPFOBJ) | $(OUTPUT) $(call msg,BINARY,,$@) $(Q)$(CC) $(CFLAGS) $(filter %.a %.o %.c,$^) $(LDLIBS) -o $@ diff --git a/tools/testing/selftests/bpf/prog_tests/check_mtu.c b/tools/testing/selftests/bpf/prog_tests/check_mtu.c index 2a9a30650350e..65b4512967e74 100644 --- a/tools/testing/selftests/bpf/prog_tests/check_mtu.c +++ b/tools/testing/selftests/bpf/prog_tests/check_mtu.c @@ -153,6 +153,26 @@ static void test_check_mtu_run_tc(struct test_check_mtu *skel, ASSERT_EQ(mtu_result, mtu_expect, "MTU-compare-user"); } +static void test_chk_segs_flag(struct test_check_mtu *skel, __u32 mtu) +{ + int err, prog_fd = bpf_program__fd(skel->progs.tc_chk_segs_flag); + struct __sk_buff skb = { + .gso_size = 10, + }; + LIBBPF_OPTS(bpf_test_run_opts, topts, + .data_in = &pkt_v4, + .data_size_in = sizeof(pkt_v4), + .ctx_in = &skb, + .ctx_size_in = sizeof(skb), + ); + + /* Lower the mtu to test the BPF_MTU_CHK_SEGS */ + SYS_NOFAIL("ip link set dev lo mtu 10"); + err = bpf_prog_test_run_opts(prog_fd, &topts); + SYS_NOFAIL("ip link set dev lo mtu %u", mtu); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, BPF_OK, "retval"); +} static void test_check_mtu_tc(__u32 mtu, __u32 ifindex) { @@ -177,11 +197,12 @@ static void test_check_mtu_tc(__u32 mtu, __u32 ifindex) test_check_mtu_run_tc(skel, skel->progs.tc_minus_delta, mtu); test_check_mtu_run_tc(skel, skel->progs.tc_input_len, mtu); test_check_mtu_run_tc(skel, skel->progs.tc_input_len_exceed, mtu); + test_chk_segs_flag(skel, mtu); cleanup: test_check_mtu__destroy(skel); } -void serial_test_check_mtu(void) +void test_ns_check_mtu(void) { int mtu_lo; diff --git a/tools/testing/selftests/bpf/prog_tests/reg_bounds.c b/tools/testing/selftests/bpf/prog_tests/reg_bounds.c index d93a0c7b1786f..0322f817d07be 100644 --- a/tools/testing/selftests/bpf/prog_tests/reg_bounds.c +++ b/tools/testing/selftests/bpf/prog_tests/reg_bounds.c @@ -2091,7 +2091,7 @@ static struct subtest_case crafted_cases[] = { {U64, S64, {0, 0xffffffffULL}, {0x7fffffff, 0x7fffffff}}, {U64, U32, {0, 0x100000000}, {0, 0}}, - {U64, U32, {0xfffffffe, 0x100000000}, {0x80000000, 0x80000000}}, + {U64, U32, {0xfffffffe, 0x300000000}, {0x80000000, 0x80000000}}, {U64, S32, {0, 0xffffffff00000000ULL}, {0, 0}}, /* these are tricky cases where lower 32 bits allow to tighten 64 diff --git a/tools/testing/selftests/bpf/prog_tests/stacktrace_ips.c b/tools/testing/selftests/bpf/prog_tests/stacktrace_ips.c index c9efdd2a5b18a..c93718dafd9b6 100644 --- a/tools/testing/selftests/bpf/prog_tests/stacktrace_ips.c +++ b/tools/testing/selftests/bpf/prog_tests/stacktrace_ips.c @@ -74,11 +74,20 @@ static void test_stacktrace_ips_kprobe_multi(bool retprobe) load_kallsyms(); - check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 4, - ksym_get_addr("bpf_testmod_stacktrace_test_3"), - ksym_get_addr("bpf_testmod_stacktrace_test_2"), - ksym_get_addr("bpf_testmod_stacktrace_test_1"), - ksym_get_addr("bpf_testmod_test_read")); + if (retprobe) { + check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 4, + ksym_get_addr("bpf_testmod_stacktrace_test_3"), + ksym_get_addr("bpf_testmod_stacktrace_test_2"), + ksym_get_addr("bpf_testmod_stacktrace_test_1"), + ksym_get_addr("bpf_testmod_test_read")); + } else { + check_stacktrace_ips(bpf_map__fd(skel->maps.stackmap), skel->bss->stack_key, 5, + ksym_get_addr("bpf_testmod_stacktrace_test"), + ksym_get_addr("bpf_testmod_stacktrace_test_3"), + ksym_get_addr("bpf_testmod_stacktrace_test_2"), + ksym_get_addr("bpf_testmod_stacktrace_test_1"), + ksym_get_addr("bpf_testmod_test_read")); + } cleanup: stacktrace_ips__destroy(skel); diff --git a/tools/testing/selftests/bpf/prog_tests/wq.c b/tools/testing/selftests/bpf/prog_tests/wq.c index 99e438fe12acd..15ac8e6d17450 100644 --- a/tools/testing/selftests/bpf/prog_tests/wq.c +++ b/tools/testing/selftests/bpf/prog_tests/wq.c @@ -16,12 +16,12 @@ void serial_test_wq(void) /* re-run the success test to check if the timer was actually executed */ wq_skel = wq__open_and_load(); - if (!ASSERT_OK_PTR(wq_skel, "wq_skel_load")) + if (!ASSERT_OK_PTR(wq_skel, "wq__open_and_load")) return; err = wq__attach(wq_skel); if (!ASSERT_OK(err, "wq_attach")) - return; + goto clean_up; prog_fd = bpf_program__fd(wq_skel->progs.test_syscall_array_sleepable); err = bpf_prog_test_run_opts(prog_fd, &topts); @@ -31,6 +31,7 @@ void serial_test_wq(void) usleep(50); /* 10 usecs should be enough, but give it extra */ ASSERT_EQ(wq_skel->bss->ok_sleepable, (1 << 1), "ok_sleepable"); +clean_up: wq__destroy(wq_skel); } diff --git a/tools/testing/selftests/bpf/progs/dmabuf_iter.c b/tools/testing/selftests/bpf/progs/dmabuf_iter.c index 13cdb11fdeb2b..9cbb7442646e5 100644 --- a/tools/testing/selftests/bpf/progs/dmabuf_iter.c +++ b/tools/testing/selftests/bpf/progs/dmabuf_iter.c @@ -48,7 +48,7 @@ int dmabuf_collector(struct bpf_iter__dmabuf *ctx) /* Buffers are not required to be named */ if (pname) { - if (bpf_probe_read_kernel(name, sizeof(name), pname)) + if (bpf_probe_read_kernel_str(name, sizeof(name), pname) < 0) return 1; /* Name strings can be provided by userspace */ diff --git a/tools/testing/selftests/bpf/progs/exceptions_assert.c b/tools/testing/selftests/bpf/progs/exceptions_assert.c index a01c2736890f9..858af5988a38a 100644 --- a/tools/testing/selftests/bpf/progs/exceptions_assert.c +++ b/tools/testing/selftests/bpf/progs/exceptions_assert.c @@ -18,43 +18,43 @@ return *(u64 *)num; \ } -__msg(": R0=0xffffffff80000000") +__msg("R{{.}}=0xffffffff80000000") check_assert(s64, ==, eq_int_min, INT_MIN); -__msg(": R0=0x7fffffff") +__msg("R{{.}}=0x7fffffff") check_assert(s64, ==, eq_int_max, INT_MAX); -__msg(": R0=0") +__msg("R{{.}}=0") check_assert(s64, ==, eq_zero, 0); -__msg(": R0=0x8000000000000000 R1=0x8000000000000000") +__msg("R{{.}}=0x8000000000000000") check_assert(s64, ==, eq_llong_min, LLONG_MIN); -__msg(": R0=0x7fffffffffffffff R1=0x7fffffffffffffff") +__msg("R{{.}}=0x7fffffffffffffff") check_assert(s64, ==, eq_llong_max, LLONG_MAX); -__msg(": R0=scalar(id=1,smax=0x7ffffffe)") +__msg("R{{.}}=scalar(id=1,smax=0x7ffffffe)") check_assert(s64, <, lt_pos, INT_MAX); -__msg(": R0=scalar(id=1,smax=-1,umin=0x8000000000000000,var_off=(0x8000000000000000; 0x7fffffffffffffff))") +__msg("R{{.}}=scalar(id=1,smax=-1,umin=0x8000000000000000,var_off=(0x8000000000000000; 0x7fffffffffffffff))") check_assert(s64, <, lt_zero, 0); -__msg(": R0=scalar(id=1,smax=0xffffffff7fffffff") +__msg("R{{.}}=scalar(id=1,smax=0xffffffff7fffffff") check_assert(s64, <, lt_neg, INT_MIN); -__msg(": R0=scalar(id=1,smax=0x7fffffff)") +__msg("R{{.}}=scalar(id=1,smax=0x7fffffff)") check_assert(s64, <=, le_pos, INT_MAX); -__msg(": R0=scalar(id=1,smax=0)") +__msg("R{{.}}=scalar(id=1,smax=0)") check_assert(s64, <=, le_zero, 0); -__msg(": R0=scalar(id=1,smax=0xffffffff80000000") +__msg("R{{.}}=scalar(id=1,smax=0xffffffff80000000") check_assert(s64, <=, le_neg, INT_MIN); -__msg(": R0=scalar(id=1,smin=umin=0x80000000,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") +__msg("R{{.}}=scalar(id=1,smin=umin=0x80000000,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") check_assert(s64, >, gt_pos, INT_MAX); -__msg(": R0=scalar(id=1,smin=umin=1,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") +__msg("R{{.}}=scalar(id=1,smin=umin=1,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") check_assert(s64, >, gt_zero, 0); -__msg(": R0=scalar(id=1,smin=0xffffffff80000001") +__msg("R{{.}}=scalar(id=1,smin=0xffffffff80000001") check_assert(s64, >, gt_neg, INT_MIN); -__msg(": R0=scalar(id=1,smin=umin=0x7fffffff,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") +__msg("R{{.}}=scalar(id=1,smin=umin=0x7fffffff,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") check_assert(s64, >=, ge_pos, INT_MAX); -__msg(": R0=scalar(id=1,smin=0,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") +__msg("R{{.}}=scalar(id=1,smin=0,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))") check_assert(s64, >=, ge_zero, 0); -__msg(": R0=scalar(id=1,smin=0xffffffff80000000") +__msg("R{{.}}=scalar(id=1,smin=0xffffffff80000000") check_assert(s64, >=, ge_neg, INT_MIN); SEC("?tc") diff --git a/tools/testing/selftests/bpf/progs/test_check_mtu.c b/tools/testing/selftests/bpf/progs/test_check_mtu.c index 2ec1de11a3ae4..7b6b2b342c1de 100644 --- a/tools/testing/selftests/bpf/progs/test_check_mtu.c +++ b/tools/testing/selftests/bpf/progs/test_check_mtu.c @@ -7,6 +7,7 @@ #include #include +#include char _license[] SEC("license") = "GPL"; @@ -288,3 +289,14 @@ int tc_input_len_exceed(struct __sk_buff *ctx) global_bpf_mtu_xdp = mtu_len; return retval; } + +SEC("tc") +int tc_chk_segs_flag(struct __sk_buff *ctx) +{ + __u32 mtu_len = 0; + int err; + + err = bpf_check_mtu(ctx, GLOBAL_USER_IFINDEX, &mtu_len, 0, BPF_MTU_CHK_SEGS); + + return err == -EINVAL ? BPF_OK : BPF_DROP; +} diff --git a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c index c0ce690ddb68a..1fdd85b4b8443 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c +++ b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c @@ -40,6 +40,9 @@ __naked void linked_regs_bpf_k(void) */ "r3 = r10;" "r3 += r0;" + /* Mark r1 and r2 as alive. */ + "r1 = r1;" + "r2 = r2;" "r0 = 0;" "exit;" : @@ -73,6 +76,9 @@ __naked void linked_regs_bpf_x_src(void) */ "r4 = r10;" "r4 += r0;" + /* Mark r1 and r2 as alive. */ + "r1 = r1;" + "r2 = r2;" "r0 = 0;" "exit;" : @@ -106,6 +112,10 @@ __naked void linked_regs_bpf_x_dst(void) */ "r4 = r10;" "r4 += r3;" + /* Mark r1 and r2 as alive. */ + "r0 = r0;" + "r1 = r1;" + "r2 = r2;" "r0 = 0;" "exit;" : @@ -143,6 +153,9 @@ __naked void linked_regs_broken_link(void) */ "r3 = r10;" "r3 += r0;" + /* Mark r1 and r2 as alive. */ + "r1 = r1;" + "r2 = r2;" "r0 = 0;" "exit;" : @@ -156,16 +169,16 @@ __naked void linked_regs_broken_link(void) */ SEC("socket") __success __log_level(2) -__msg("12: (0f) r2 += r1") +__msg("17: (0f) r2 += r1") /* Current state */ -__msg("frame2: last_idx 12 first_idx 11 subseq_idx -1 ") -__msg("frame2: regs=r1 stack= before 11: (bf) r2 = r10") +__msg("frame2: last_idx 17 first_idx 14 subseq_idx -1 ") +__msg("frame2: regs=r1 stack= before 16: (bf) r2 = r10") __msg("frame2: parent state regs=r1 stack=") __msg("frame1: parent state regs= stack=") __msg("frame0: parent state regs= stack=") /* Parent state */ -__msg("frame2: last_idx 10 first_idx 10 subseq_idx 11 ") -__msg("frame2: regs=r1 stack= before 10: (25) if r1 > 0x7 goto pc+0") +__msg("frame2: last_idx 13 first_idx 13 subseq_idx 14 ") +__msg("frame2: regs=r1 stack= before 13: (25) if r1 > 0x7 goto pc+0") __msg("frame2: parent state regs=r1 stack=") /* frame1.r{6,7} are marked because mark_precise_scalar_ids() * looks for all registers with frame2.r1.id in the current state @@ -173,20 +186,20 @@ __msg("frame2: parent state regs=r1 stack=") __msg("frame1: parent state regs=r6,r7 stack=") __msg("frame0: parent state regs=r6 stack=") /* Parent state */ -__msg("frame2: last_idx 8 first_idx 8 subseq_idx 10") -__msg("frame2: regs=r1 stack= before 8: (85) call pc+1") +__msg("frame2: last_idx 9 first_idx 9 subseq_idx 13") +__msg("frame2: regs=r1 stack= before 9: (85) call pc+3") /* frame1.r1 is marked because of backtracking of call instruction */ __msg("frame1: parent state regs=r1,r6,r7 stack=") __msg("frame0: parent state regs=r6 stack=") /* Parent state */ -__msg("frame1: last_idx 7 first_idx 6 subseq_idx 8") -__msg("frame1: regs=r1,r6,r7 stack= before 7: (bf) r7 = r1") -__msg("frame1: regs=r1,r6 stack= before 6: (bf) r6 = r1") +__msg("frame1: last_idx 8 first_idx 7 subseq_idx 9") +__msg("frame1: regs=r1,r6,r7 stack= before 8: (bf) r7 = r1") +__msg("frame1: regs=r1,r6 stack= before 7: (bf) r6 = r1") __msg("frame1: parent state regs=r1 stack=") __msg("frame0: parent state regs=r6 stack=") /* Parent state */ -__msg("frame1: last_idx 4 first_idx 4 subseq_idx 6") -__msg("frame1: regs=r1 stack= before 4: (85) call pc+1") +__msg("frame1: last_idx 4 first_idx 4 subseq_idx 7") +__msg("frame1: regs=r1 stack= before 4: (85) call pc+2") __msg("frame0: parent state regs=r1,r6 stack=") /* Parent state */ __msg("frame0: last_idx 3 first_idx 1 subseq_idx 4") @@ -204,6 +217,7 @@ __naked void precision_many_frames(void) "r1 = r0;" "r6 = r0;" "call precision_many_frames__foo;" + "r6 = r6;" /* mark r6 as live */ "exit;" : : __imm(bpf_ktime_get_ns) @@ -220,6 +234,8 @@ void precision_many_frames__foo(void) "r6 = r1;" "r7 = r1;" "call precision_many_frames__bar;" + "r6 = r6;" /* mark r6 as live */ + "r7 = r7;" /* mark r7 as live */ "exit" ::: __clobber_all); } @@ -229,6 +245,8 @@ void precision_many_frames__bar(void) { asm volatile ( "if r1 > 7 goto +0;" + "r6 = 0;" /* mark r6 as live */ + "r7 = 0;" /* mark r7 as live */ /* force r1 to be precise, this eventually marks: * - bar frame r1 * - foo frame r{1,6,7} @@ -340,6 +358,8 @@ __naked void precision_two_ids(void) "r3 += r7;" /* force r9 to be precise, this also marks r8 */ "r3 += r9;" + "r6 = r6;" /* mark r6 as live */ + "r8 = r8;" /* mark r8 as live */ "exit;" : : __imm(bpf_ktime_get_ns) @@ -353,7 +373,7 @@ __flag(BPF_F_TEST_STATE_FREQ) * collect_linked_regs() can't tie more than 6 registers for a single insn. */ __msg("8: (25) if r0 > 0x7 goto pc+0 ; R0=scalar(id=1") -__msg("9: (bf) r6 = r6 ; R6=scalar(id=2") +__msg("14: (bf) r6 = r6 ; R6=scalar(id=2") /* check that r{0-5} are marked precise after 'if' */ __msg("frame0: regs=r0 stack= before 8: (25) if r0 > 0x7 goto pc+0") __msg("frame0: parent state regs=r0,r1,r2,r3,r4,r5 stack=:") @@ -372,6 +392,12 @@ __naked void linked_regs_too_many_regs(void) "r6 = r0;" /* propagate range for r{0-6} */ "if r0 > 7 goto +0;" + /* keep r{1-5} live */ + "r1 = r1;" + "r2 = r2;" + "r3 = r3;" + "r4 = r4;" + "r5 = r5;" /* make r6 appear in the log */ "r6 = r6;" /* force r0 to be precise, @@ -517,7 +543,7 @@ __naked void check_ids_in_regsafe_2(void) "*(u64*)(r10 - 8) = r1;" /* r9 = pointer to stack */ "r9 = r10;" - "r9 += -8;" + "r9 += -16;" /* r8 = ktime_get_ns() */ "call %[bpf_ktime_get_ns];" "r8 = r0;" @@ -538,6 +564,8 @@ __naked void check_ids_in_regsafe_2(void) "if r7 > 4 goto l2_%=;" /* Access memory at r9[r6] */ "r9 += r6;" + "r9 += r7;" + "r9 += r8;" "r0 = *(u8*)(r9 + 0);" "l2_%=:" "r0 = 0;" diff --git a/tools/testing/selftests/bpf/test_xsk.c b/tools/testing/selftests/bpf/test_xsk.c new file mode 100644 index 0000000000000..55d318c5c5e59 --- /dev/null +++ b/tools/testing/selftests/bpf/test_xsk.c @@ -0,0 +1,2422 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "network_helpers.h" +#include "test_xsk.h" +#include "xsk_xdp_common.h" +#include "xsk_xdp_progs.skel.h" + +#define DEFAULT_BATCH_SIZE 64 +#define MIN_PKT_SIZE 64 +#define MAX_ETH_JUMBO_SIZE 9000 +#define MAX_INTERFACES 2 +#define MAX_TEARDOWN_ITER 10 +#define MAX_TX_BUDGET_DEFAULT 32 +#define PKT_DUMP_NB_TO_PRINT 16 +/* Just to align the data in the packet */ +#define PKT_HDR_SIZE (sizeof(struct ethhdr) + 2) +#define POLL_TMOUT 1000 +#define THREAD_TMOUT 3 +#define UMEM_HEADROOM_TEST_SIZE 128 +#define XSK_DESC__INVALID_OPTION (0xffff) +#define XSK_UMEM__INVALID_FRAME_SIZE (MAX_ETH_JUMBO_SIZE + 1) +#define XSK_UMEM__LARGE_FRAME_SIZE (3 * 1024) +#define XSK_UMEM__MAX_FRAME_SIZE (4 * 1024) + +static const u8 g_mac[ETH_ALEN] = {0x55, 0x44, 0x33, 0x22, 0x11, 0x00}; + +bool opt_verbose; +pthread_barrier_t barr; +pthread_mutex_t pacing_mutex = PTHREAD_MUTEX_INITIALIZER; + +int pkts_in_flight; + +/* The payload is a word consisting of a packet sequence number in the upper + * 16-bits and a intra packet data sequence number in the lower 16 bits. So the 3rd packet's + * 5th word of data will contain the number (2<<16) | 4 as they are numbered from 0. + */ +static void write_payload(void *dest, u32 pkt_nb, u32 start, u32 size) +{ + u32 *ptr = (u32 *)dest, i; + + start /= sizeof(*ptr); + size /= sizeof(*ptr); + for (i = 0; i < size; i++) + ptr[i] = htonl(pkt_nb << 16 | (i + start)); +} + +static void gen_eth_hdr(struct xsk_socket_info *xsk, struct ethhdr *eth_hdr) +{ + memcpy(eth_hdr->h_dest, xsk->dst_mac, ETH_ALEN); + memcpy(eth_hdr->h_source, xsk->src_mac, ETH_ALEN); + eth_hdr->h_proto = htons(ETH_P_LOOPBACK); +} + +static bool is_umem_valid(struct ifobject *ifobj) +{ + return !!ifobj->umem->umem; +} + +static u32 mode_to_xdp_flags(enum test_mode mode) +{ + return (mode == TEST_MODE_SKB) ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE; +} + +static u64 umem_size(struct xsk_umem_info *umem) +{ + return umem->num_frames * umem->frame_size; +} + +int xsk_configure_umem(struct ifobject *ifobj, struct xsk_umem_info *umem, void *buffer, + u64 size) +{ + struct xsk_umem_config cfg = { + .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS, + .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS, + .frame_size = umem->frame_size, + .frame_headroom = umem->frame_headroom, + .flags = XSK_UMEM__DEFAULT_FLAGS + }; + int ret; + + if (umem->fill_size) + cfg.fill_size = umem->fill_size; + + if (umem->comp_size) + cfg.comp_size = umem->comp_size; + + if (umem->unaligned_mode) + cfg.flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG; + + ret = xsk_umem__create(&umem->umem, buffer, size, + &umem->fq, &umem->cq, &cfg); + if (ret) + return ret; + + umem->buffer = buffer; + if (ifobj->shared_umem && ifobj->rx_on) { + umem->base_addr = umem_size(umem); + umem->next_buffer = umem_size(umem); + } + + return 0; +} + +static u64 umem_alloc_buffer(struct xsk_umem_info *umem) +{ + u64 addr; + + addr = umem->next_buffer; + umem->next_buffer += umem->frame_size; + if (umem->next_buffer >= umem->base_addr + umem_size(umem)) + umem->next_buffer = umem->base_addr; + + return addr; +} + +static void umem_reset_alloc(struct xsk_umem_info *umem) +{ + umem->next_buffer = 0; +} + +static void enable_busy_poll(struct xsk_socket_info *xsk) +{ + int sock_opt; + + sock_opt = 1; + if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_PREFER_BUSY_POLL, + (void *)&sock_opt, sizeof(sock_opt)) < 0) + exit_with_error(errno); + + sock_opt = 20; + if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL, + (void *)&sock_opt, sizeof(sock_opt)) < 0) + exit_with_error(errno); + + sock_opt = xsk->batch_size; + if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL_BUDGET, + (void *)&sock_opt, sizeof(sock_opt)) < 0) + exit_with_error(errno); +} + +int xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, + struct ifobject *ifobject, bool shared) +{ + struct xsk_socket_config cfg = {}; + struct xsk_ring_cons *rxr; + struct xsk_ring_prod *txr; + + xsk->umem = umem; + cfg.rx_size = xsk->rxqsize; + cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; + cfg.bind_flags = ifobject->bind_flags; + if (shared) + cfg.bind_flags |= XDP_SHARED_UMEM; + if (ifobject->mtu > MAX_ETH_PKT_SIZE) + cfg.bind_flags |= XDP_USE_SG; + if (umem->comp_size) + cfg.tx_size = umem->comp_size; + if (umem->fill_size) + cfg.rx_size = umem->fill_size; + + txr = ifobject->tx_on ? &xsk->tx : NULL; + rxr = ifobject->rx_on ? &xsk->rx : NULL; + return xsk_socket__create(&xsk->xsk, ifobject->ifindex, 0, umem->umem, rxr, txr, &cfg); +} + +#define MAX_SKB_FRAGS_PATH "/proc/sys/net/core/max_skb_frags" +static unsigned int get_max_skb_frags(void) +{ + unsigned int max_skb_frags = 0; + FILE *file; + + file = fopen(MAX_SKB_FRAGS_PATH, "r"); + if (!file) { + ksft_print_msg("Error opening %s\n", MAX_SKB_FRAGS_PATH); + return 0; + } + + if (fscanf(file, "%u", &max_skb_frags) != 1) + ksft_print_msg("Error reading %s\n", MAX_SKB_FRAGS_PATH); + + fclose(file); + return max_skb_frags; +} + +static int set_ring_size(struct ifobject *ifobj) +{ + int ret; + u32 ctr = 0; + + while (ctr++ < SOCK_RECONF_CTR) { + ret = set_hw_ring_size(ifobj->ifname, &ifobj->ring); + if (!ret) + break; + + /* Retry if it fails */ + if (ctr >= SOCK_RECONF_CTR || errno != EBUSY) + return -errno; + + usleep(USLEEP_MAX); + } + + return ret; +} + +int hw_ring_size_reset(struct ifobject *ifobj) +{ + ifobj->ring.tx_pending = ifobj->set_ring.default_tx; + ifobj->ring.rx_pending = ifobj->set_ring.default_rx; + return set_ring_size(ifobj); +} + +static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, + struct ifobject *ifobj_rx) +{ + u32 i, j; + + for (i = 0; i < MAX_INTERFACES; i++) { + struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; + + ifobj->xsk = &ifobj->xsk_arr[0]; + ifobj->use_poll = false; + ifobj->use_fill_ring = true; + ifobj->release_rx = true; + ifobj->validation_func = NULL; + ifobj->use_metadata = false; + + if (i == 0) { + ifobj->rx_on = false; + ifobj->tx_on = true; + } else { + ifobj->rx_on = true; + ifobj->tx_on = false; + } + + memset(ifobj->umem, 0, sizeof(*ifobj->umem)); + ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS; + ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; + + for (j = 0; j < MAX_SOCKETS; j++) { + memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j])); + ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; + ifobj->xsk_arr[j].batch_size = DEFAULT_BATCH_SIZE; + if (i == 0) + ifobj->xsk_arr[j].pkt_stream = test->tx_pkt_stream_default; + else + ifobj->xsk_arr[j].pkt_stream = test->rx_pkt_stream_default; + + memcpy(ifobj->xsk_arr[j].src_mac, g_mac, ETH_ALEN); + memcpy(ifobj->xsk_arr[j].dst_mac, g_mac, ETH_ALEN); + ifobj->xsk_arr[j].src_mac[5] += ((j * 2) + 0); + ifobj->xsk_arr[j].dst_mac[5] += ((j * 2) + 1); + } + } + + if (ifobj_tx->hw_ring_size_supp) + hw_ring_size_reset(ifobj_tx); + + test->ifobj_tx = ifobj_tx; + test->ifobj_rx = ifobj_rx; + test->current_step = 0; + test->total_steps = 1; + test->nb_sockets = 1; + test->fail = false; + test->set_ring = false; + test->adjust_tail = false; + test->adjust_tail_support = false; + test->mtu = MAX_ETH_PKT_SIZE; + test->xdp_prog_rx = ifobj_rx->xdp_progs->progs.xsk_def_prog; + test->xskmap_rx = ifobj_rx->xdp_progs->maps.xsk; + test->xdp_prog_tx = ifobj_tx->xdp_progs->progs.xsk_def_prog; + test->xskmap_tx = ifobj_tx->xdp_progs->maps.xsk; +} + +void test_init(struct test_spec *test, struct ifobject *ifobj_tx, + struct ifobject *ifobj_rx, enum test_mode mode, + const struct test_spec *test_to_run) +{ + struct pkt_stream *tx_pkt_stream; + struct pkt_stream *rx_pkt_stream; + u32 i; + + tx_pkt_stream = test->tx_pkt_stream_default; + rx_pkt_stream = test->rx_pkt_stream_default; + memset(test, 0, sizeof(*test)); + test->tx_pkt_stream_default = tx_pkt_stream; + test->rx_pkt_stream_default = rx_pkt_stream; + + for (i = 0; i < MAX_INTERFACES; i++) { + struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; + + ifobj->bind_flags = XDP_USE_NEED_WAKEUP; + if (mode == TEST_MODE_ZC) + ifobj->bind_flags |= XDP_ZEROCOPY; + else + ifobj->bind_flags |= XDP_COPY; + } + + memcpy(test->name, test_to_run->name, MAX_TEST_NAME_SIZE); + test->test_func = test_to_run->test_func; + test->mode = mode; + __test_spec_init(test, ifobj_tx, ifobj_rx); +} + +static void test_spec_reset(struct test_spec *test) +{ + __test_spec_init(test, test->ifobj_tx, test->ifobj_rx); +} + +static void test_spec_set_xdp_prog(struct test_spec *test, struct bpf_program *xdp_prog_rx, + struct bpf_program *xdp_prog_tx, struct bpf_map *xskmap_rx, + struct bpf_map *xskmap_tx) +{ + test->xdp_prog_rx = xdp_prog_rx; + test->xdp_prog_tx = xdp_prog_tx; + test->xskmap_rx = xskmap_rx; + test->xskmap_tx = xskmap_tx; +} + +static int test_spec_set_mtu(struct test_spec *test, int mtu) +{ + int err; + + if (test->ifobj_rx->mtu != mtu) { + err = xsk_set_mtu(test->ifobj_rx->ifindex, mtu); + if (err) + return err; + test->ifobj_rx->mtu = mtu; + } + if (test->ifobj_tx->mtu != mtu) { + err = xsk_set_mtu(test->ifobj_tx->ifindex, mtu); + if (err) + return err; + test->ifobj_tx->mtu = mtu; + } + + return 0; +} + +void pkt_stream_reset(struct pkt_stream *pkt_stream) +{ + if (pkt_stream) { + pkt_stream->current_pkt_nb = 0; + pkt_stream->nb_rx_pkts = 0; + } +} + +static struct pkt *pkt_stream_get_next_tx_pkt(struct pkt_stream *pkt_stream) +{ + if (pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts) + return NULL; + + return &pkt_stream->pkts[pkt_stream->current_pkt_nb++]; +} + +static struct pkt *pkt_stream_get_next_rx_pkt(struct pkt_stream *pkt_stream, u32 *pkts_sent) +{ + while (pkt_stream->current_pkt_nb < pkt_stream->nb_pkts) { + (*pkts_sent)++; + if (pkt_stream->pkts[pkt_stream->current_pkt_nb].valid) + return &pkt_stream->pkts[pkt_stream->current_pkt_nb++]; + pkt_stream->current_pkt_nb++; + } + return NULL; +} + +void pkt_stream_delete(struct pkt_stream *pkt_stream) +{ + free(pkt_stream->pkts); + free(pkt_stream); +} + +void pkt_stream_restore_default(struct test_spec *test) +{ + struct pkt_stream *tx_pkt_stream = test->ifobj_tx->xsk->pkt_stream; + struct pkt_stream *rx_pkt_stream = test->ifobj_rx->xsk->pkt_stream; + + if (tx_pkt_stream != test->tx_pkt_stream_default) { + pkt_stream_delete(test->ifobj_tx->xsk->pkt_stream); + test->ifobj_tx->xsk->pkt_stream = test->tx_pkt_stream_default; + } + + if (rx_pkt_stream != test->rx_pkt_stream_default) { + pkt_stream_delete(test->ifobj_rx->xsk->pkt_stream); + test->ifobj_rx->xsk->pkt_stream = test->rx_pkt_stream_default; + } +} + +static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts) +{ + struct pkt_stream *pkt_stream; + + pkt_stream = calloc(1, sizeof(*pkt_stream)); + if (!pkt_stream) + return NULL; + + pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts)); + if (!pkt_stream->pkts) { + free(pkt_stream); + return NULL; + } + + pkt_stream->nb_pkts = nb_pkts; + return pkt_stream; +} + +static u32 pkt_nb_frags(u32 frame_size, struct pkt_stream *pkt_stream, struct pkt *pkt) +{ + u32 nb_frags = 1, next_frag; + + if (!pkt) + return 1; + + if (!pkt_stream->verbatim) { + if (!pkt->valid || !pkt->len) + return 1; + return ceil_u32(pkt->len, frame_size); + } + + /* Search for the end of the packet in verbatim mode */ + if (!pkt_continues(pkt->options) || !pkt->valid) + return nb_frags; + + next_frag = pkt_stream->current_pkt_nb; + pkt++; + while (next_frag++ < pkt_stream->nb_pkts) { + nb_frags++; + if (!pkt_continues(pkt->options) || !pkt->valid) + break; + pkt++; + } + return nb_frags; +} + +static bool set_pkt_valid(int offset, u32 len) +{ + return len <= MAX_ETH_JUMBO_SIZE; +} + +static void pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len) +{ + pkt->offset = offset; + pkt->len = len; + pkt->valid = set_pkt_valid(offset, len); +} + +static void pkt_stream_pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len) +{ + bool prev_pkt_valid = pkt->valid; + + pkt_set(pkt_stream, pkt, offset, len); + pkt_stream->nb_valid_entries += pkt->valid - prev_pkt_valid; +} + +static u32 pkt_get_buffer_len(struct xsk_umem_info *umem, u32 len) +{ + return ceil_u32(len, umem->frame_size) * umem->frame_size; +} + +static struct pkt_stream *__pkt_stream_generate(u32 nb_pkts, u32 pkt_len, u32 nb_start, u32 nb_off) +{ + struct pkt_stream *pkt_stream; + u32 i; + + pkt_stream = __pkt_stream_alloc(nb_pkts); + if (!pkt_stream) + exit_with_error(ENOMEM); + + pkt_stream->nb_pkts = nb_pkts; + pkt_stream->max_pkt_len = pkt_len; + for (i = 0; i < nb_pkts; i++) { + struct pkt *pkt = &pkt_stream->pkts[i]; + + pkt_stream_pkt_set(pkt_stream, pkt, 0, pkt_len); + pkt->pkt_nb = nb_start + i * nb_off; + } + + return pkt_stream; +} + +struct pkt_stream *pkt_stream_generate(u32 nb_pkts, u32 pkt_len) +{ + return __pkt_stream_generate(nb_pkts, pkt_len, 0, 1); +} + +static struct pkt_stream *pkt_stream_clone(struct pkt_stream *pkt_stream) +{ + return pkt_stream_generate(pkt_stream->nb_pkts, pkt_stream->pkts[0].len); +} + +static void pkt_stream_replace_ifobject(struct ifobject *ifobj, u32 nb_pkts, u32 pkt_len) +{ + ifobj->xsk->pkt_stream = pkt_stream_generate(nb_pkts, pkt_len); +} + +static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len) +{ + pkt_stream_replace_ifobject(test->ifobj_tx, nb_pkts, pkt_len); + pkt_stream_replace_ifobject(test->ifobj_rx, nb_pkts, pkt_len); +} + +static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len, + int offset) +{ + struct pkt_stream *pkt_stream; + u32 i; + + pkt_stream = pkt_stream_clone(ifobj->xsk->pkt_stream); + for (i = 1; i < ifobj->xsk->pkt_stream->nb_pkts; i += 2) + pkt_stream_pkt_set(pkt_stream, &pkt_stream->pkts[i], offset, pkt_len); + + ifobj->xsk->pkt_stream = pkt_stream; +} + +static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset) +{ + __pkt_stream_replace_half(test->ifobj_tx, pkt_len, offset); + __pkt_stream_replace_half(test->ifobj_rx, pkt_len, offset); +} + +static void pkt_stream_receive_half(struct test_spec *test) +{ + struct pkt_stream *pkt_stream = test->ifobj_tx->xsk->pkt_stream; + u32 i; + + test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(pkt_stream->nb_pkts, + pkt_stream->pkts[0].len); + pkt_stream = test->ifobj_rx->xsk->pkt_stream; + for (i = 1; i < pkt_stream->nb_pkts; i += 2) + pkt_stream->pkts[i].valid = false; + + pkt_stream->nb_valid_entries /= 2; +} + +static void pkt_stream_even_odd_sequence(struct test_spec *test) +{ + struct pkt_stream *pkt_stream; + u32 i; + + for (i = 0; i < test->nb_sockets; i++) { + pkt_stream = test->ifobj_tx->xsk_arr[i].pkt_stream; + pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2, + pkt_stream->pkts[0].len, i, 2); + test->ifobj_tx->xsk_arr[i].pkt_stream = pkt_stream; + + pkt_stream = test->ifobj_rx->xsk_arr[i].pkt_stream; + pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2, + pkt_stream->pkts[0].len, i, 2); + test->ifobj_rx->xsk_arr[i].pkt_stream = pkt_stream; + } +} + +static u64 pkt_get_addr(struct pkt *pkt, struct xsk_umem_info *umem) +{ + if (!pkt->valid) + return pkt->offset; + return pkt->offset + umem_alloc_buffer(umem); +} + +static void pkt_stream_cancel(struct pkt_stream *pkt_stream) +{ + pkt_stream->current_pkt_nb--; +} + +static void pkt_generate(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, u64 addr, u32 len, + u32 pkt_nb, u32 bytes_written) +{ + void *data = xsk_umem__get_data(umem->buffer, addr); + + if (len < MIN_PKT_SIZE) + return; + + if (!bytes_written) { + gen_eth_hdr(xsk, data); + + len -= PKT_HDR_SIZE; + data += PKT_HDR_SIZE; + } else { + bytes_written -= PKT_HDR_SIZE; + } + + write_payload(data, pkt_nb, bytes_written, len); +} + +static struct pkt_stream *__pkt_stream_generate_custom(struct ifobject *ifobj, struct pkt *frames, + u32 nb_frames, bool verbatim) +{ + u32 i, len = 0, pkt_nb = 0, payload = 0; + struct pkt_stream *pkt_stream; + + pkt_stream = __pkt_stream_alloc(nb_frames); + if (!pkt_stream) + exit_with_error(ENOMEM); + + for (i = 0; i < nb_frames; i++) { + struct pkt *pkt = &pkt_stream->pkts[pkt_nb]; + struct pkt *frame = &frames[i]; + + pkt->offset = frame->offset; + if (verbatim) { + *pkt = *frame; + pkt->pkt_nb = payload; + if (!frame->valid || !pkt_continues(frame->options)) + payload++; + } else { + if (frame->valid) + len += frame->len; + if (frame->valid && pkt_continues(frame->options)) + continue; + + pkt->pkt_nb = pkt_nb; + pkt->len = len; + pkt->valid = frame->valid; + pkt->options = 0; + + len = 0; + } + + print_verbose("offset: %d len: %u valid: %u options: %u pkt_nb: %u\n", + pkt->offset, pkt->len, pkt->valid, pkt->options, pkt->pkt_nb); + + if (pkt->valid && pkt->len > pkt_stream->max_pkt_len) + pkt_stream->max_pkt_len = pkt->len; + + if (pkt->valid) + pkt_stream->nb_valid_entries++; + + pkt_nb++; + } + + pkt_stream->nb_pkts = pkt_nb; + pkt_stream->verbatim = verbatim; + return pkt_stream; +} + +static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts) +{ + struct pkt_stream *pkt_stream; + + pkt_stream = __pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts, true); + test->ifobj_tx->xsk->pkt_stream = pkt_stream; + + pkt_stream = __pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts, false); + test->ifobj_rx->xsk->pkt_stream = pkt_stream; +} + +static void pkt_print_data(u32 *data, u32 cnt) +{ + u32 i; + + for (i = 0; i < cnt; i++) { + u32 seqnum, pkt_nb; + + seqnum = ntohl(*data) & 0xffff; + pkt_nb = ntohl(*data) >> 16; + ksft_print_msg("%u:%u ", pkt_nb, seqnum); + data++; + } +} + +static void pkt_dump(void *pkt, u32 len, bool eth_header) +{ + struct ethhdr *ethhdr = pkt; + u32 i, *data; + + if (eth_header) { + /*extract L2 frame */ + ksft_print_msg("DEBUG>> L2: dst mac: "); + for (i = 0; i < ETH_ALEN; i++) + ksft_print_msg("%02X", ethhdr->h_dest[i]); + + ksft_print_msg("\nDEBUG>> L2: src mac: "); + for (i = 0; i < ETH_ALEN; i++) + ksft_print_msg("%02X", ethhdr->h_source[i]); + + data = pkt + PKT_HDR_SIZE; + } else { + data = pkt; + } + + /*extract L5 frame */ + ksft_print_msg("\nDEBUG>> L5: seqnum: "); + pkt_print_data(data, PKT_DUMP_NB_TO_PRINT); + ksft_print_msg("...."); + if (len > PKT_DUMP_NB_TO_PRINT * sizeof(u32)) { + ksft_print_msg("\n.... "); + pkt_print_data(data + len / sizeof(u32) - PKT_DUMP_NB_TO_PRINT, + PKT_DUMP_NB_TO_PRINT); + } + ksft_print_msg("\n---------------------------------------\n"); +} + +static bool is_offset_correct(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr) +{ + u32 headroom = umem->unaligned_mode ? 0 : umem->frame_headroom; + u32 offset = addr % umem->frame_size, expected_offset; + int pkt_offset = pkt->valid ? pkt->offset : 0; + + if (!umem->unaligned_mode) + pkt_offset = 0; + + expected_offset = (pkt_offset + headroom + XDP_PACKET_HEADROOM) % umem->frame_size; + + if (offset == expected_offset) + return true; + + ksft_print_msg("[%s] expected [%u], got [%u]\n", __func__, expected_offset, offset); + return false; +} + +static bool is_metadata_correct(struct pkt *pkt, void *buffer, u64 addr) +{ + void *data = xsk_umem__get_data(buffer, addr); + struct xdp_info *meta = data - sizeof(struct xdp_info); + + if (meta->count != pkt->pkt_nb) { + ksft_print_msg("[%s] expected meta_count [%d], got meta_count [%llu]\n", + __func__, pkt->pkt_nb, + (unsigned long long)meta->count); + return false; + } + + return true; +} + +static bool is_adjust_tail_supported(struct xsk_xdp_progs *skel_rx) +{ + struct bpf_map *data_map; + int adjust_value = 0; + int key = 0; + int ret; + + data_map = bpf_object__find_map_by_name(skel_rx->obj, "xsk_xdp_.bss"); + if (!data_map || !bpf_map__is_internal(data_map)) { + ksft_print_msg("Error: could not find bss section of XDP program\n"); + exit_with_error(errno); + } + + ret = bpf_map_lookup_elem(bpf_map__fd(data_map), &key, &adjust_value); + if (ret) { + ksft_print_msg("Error: bpf_map_lookup_elem failed with error %d\n", ret); + exit_with_error(errno); + } + + /* Set the 'adjust_value' variable to -EOPNOTSUPP in the XDP program if the adjust_tail + * helper is not supported. Skip the adjust_tail test case in this scenario. + */ + return adjust_value != -EOPNOTSUPP; +} + +static bool is_frag_valid(struct xsk_umem_info *umem, u64 addr, u32 len, u32 expected_pkt_nb, + u32 bytes_processed) +{ + u32 seqnum, pkt_nb, *pkt_data, words_to_end, expected_seqnum; + void *data = xsk_umem__get_data(umem->buffer, addr); + + addr -= umem->base_addr; + + if (addr >= umem->num_frames * umem->frame_size || + addr + len > umem->num_frames * umem->frame_size) { + ksft_print_msg("Frag invalid addr: %llx len: %u\n", + (unsigned long long)addr, len); + return false; + } + if (!umem->unaligned_mode && addr % umem->frame_size + len > umem->frame_size) { + ksft_print_msg("Frag crosses frame boundary addr: %llx len: %u\n", + (unsigned long long)addr, len); + return false; + } + + pkt_data = data; + if (!bytes_processed) { + pkt_data += PKT_HDR_SIZE / sizeof(*pkt_data); + len -= PKT_HDR_SIZE; + } else { + bytes_processed -= PKT_HDR_SIZE; + } + + expected_seqnum = bytes_processed / sizeof(*pkt_data); + seqnum = ntohl(*pkt_data) & 0xffff; + pkt_nb = ntohl(*pkt_data) >> 16; + + if (expected_pkt_nb != pkt_nb) { + ksft_print_msg("[%s] expected pkt_nb [%u], got pkt_nb [%u]\n", + __func__, expected_pkt_nb, pkt_nb); + goto error; + } + if (expected_seqnum != seqnum) { + ksft_print_msg("[%s] expected seqnum at start [%u], got seqnum [%u]\n", + __func__, expected_seqnum, seqnum); + goto error; + } + + words_to_end = len / sizeof(*pkt_data) - 1; + pkt_data += words_to_end; + seqnum = ntohl(*pkt_data) & 0xffff; + expected_seqnum += words_to_end; + if (expected_seqnum != seqnum) { + ksft_print_msg("[%s] expected seqnum at end [%u], got seqnum [%u]\n", + __func__, expected_seqnum, seqnum); + goto error; + } + + return true; + +error: + pkt_dump(data, len, !bytes_processed); + return false; +} + +static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len) +{ + if (pkt->len != len) { + ksft_print_msg("[%s] expected packet length [%d], got length [%d]\n", + __func__, pkt->len, len); + pkt_dump(xsk_umem__get_data(buffer, addr), len, true); + return false; + } + + return true; +} + +static u32 load_value(u32 *counter) +{ + return __atomic_load_n(counter, __ATOMIC_ACQUIRE); +} + +static bool kick_tx_with_check(struct xsk_socket_info *xsk, int *ret) +{ + u32 max_budget = MAX_TX_BUDGET_DEFAULT; + u32 cons, ready_to_send; + int delta; + + cons = load_value(xsk->tx.consumer); + ready_to_send = load_value(xsk->tx.producer) - cons; + *ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); + + delta = load_value(xsk->tx.consumer) - cons; + /* By default, xsk should consume exact @max_budget descs at one + * send in this case where hitting the max budget limit in while + * loop is triggered in __xsk_generic_xmit(). Please make sure that + * the number of descs to be sent is larger than @max_budget, or + * else the tx.consumer will be updated in xskq_cons_peek_desc() + * in time which hides the issue we try to verify. + */ + if (ready_to_send > max_budget && delta != max_budget) + return false; + + return true; +} + +int kick_tx(struct xsk_socket_info *xsk) +{ + int ret; + + if (xsk->check_consumer) { + if (!kick_tx_with_check(xsk, &ret)) + return TEST_FAILURE; + } else { + ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); + } + if (ret >= 0) + return TEST_PASS; + if (errno == ENOBUFS || errno == EAGAIN || errno == EBUSY || errno == ENETDOWN) { + usleep(100); + return TEST_PASS; + } + return TEST_FAILURE; +} + +int kick_rx(struct xsk_socket_info *xsk) +{ + int ret; + + ret = recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL); + if (ret < 0) + return TEST_FAILURE; + + return TEST_PASS; +} + +static int complete_pkts(struct xsk_socket_info *xsk, int batch_size) +{ + unsigned int rcvd; + u32 idx; + int ret; + + if (xsk_ring_prod__needs_wakeup(&xsk->tx)) { + ret = kick_tx(xsk); + if (ret) + return TEST_FAILURE; + } + + rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx); + if (rcvd) { + if (rcvd > xsk->outstanding_tx) { + u64 addr = *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx + rcvd - 1); + + ksft_print_msg("[%s] Too many packets completed\n", __func__); + ksft_print_msg("Last completion address: %llx\n", + (unsigned long long)addr); + return TEST_FAILURE; + } + + xsk_ring_cons__release(&xsk->umem->cq, rcvd); + xsk->outstanding_tx -= rcvd; + } + + return TEST_PASS; +} + +static int __receive_pkts(struct test_spec *test, struct xsk_socket_info *xsk) +{ + u32 frags_processed = 0, nb_frags = 0, pkt_len = 0; + u32 idx_rx = 0, idx_fq = 0, rcvd, pkts_sent = 0; + struct pkt_stream *pkt_stream = xsk->pkt_stream; + struct ifobject *ifobj = test->ifobj_rx; + struct xsk_umem_info *umem = xsk->umem; + struct pollfd fds = { }; + struct pkt *pkt; + u64 first_addr = 0; + int ret; + + fds.fd = xsk_socket__fd(xsk->xsk); + fds.events = POLLIN; + + ret = kick_rx(xsk); + if (ret) + return TEST_FAILURE; + + if (ifobj->use_poll) { + ret = poll(&fds, 1, POLL_TMOUT); + if (ret < 0) + return TEST_FAILURE; + + if (!ret) { + if (!is_umem_valid(test->ifobj_tx)) + return TEST_PASS; + + ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__); + return TEST_CONTINUE; + } + + if (!(fds.revents & POLLIN)) + return TEST_CONTINUE; + } + + rcvd = xsk_ring_cons__peek(&xsk->rx, xsk->batch_size, &idx_rx); + if (!rcvd) + return TEST_CONTINUE; + + if (ifobj->use_fill_ring) { + ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); + while (ret != rcvd) { + if (xsk_ring_prod__needs_wakeup(&umem->fq)) { + ret = poll(&fds, 1, POLL_TMOUT); + if (ret < 0) + return TEST_FAILURE; + } + ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); + } + } + + while (frags_processed < rcvd) { + const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++); + u64 addr = desc->addr, orig; + + orig = xsk_umem__extract_addr(addr); + addr = xsk_umem__add_offset_to_addr(addr); + + if (!nb_frags) { + pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent); + if (!pkt) { + ksft_print_msg("[%s] received too many packets addr: %lx len %u\n", + __func__, addr, desc->len); + return TEST_FAILURE; + } + } + + print_verbose("Rx: addr: %lx len: %u options: %u pkt_nb: %u valid: %u\n", + addr, desc->len, desc->options, pkt->pkt_nb, pkt->valid); + + if (!is_frag_valid(umem, addr, desc->len, pkt->pkt_nb, pkt_len) || + !is_offset_correct(umem, pkt, addr) || (ifobj->use_metadata && + !is_metadata_correct(pkt, umem->buffer, addr))) + return TEST_FAILURE; + + if (!nb_frags++) + first_addr = addr; + frags_processed++; + pkt_len += desc->len; + if (ifobj->use_fill_ring) + *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig; + + if (pkt_continues(desc->options)) + continue; + + /* The complete packet has been received */ + if (!is_pkt_valid(pkt, umem->buffer, first_addr, pkt_len) || + !is_offset_correct(umem, pkt, addr)) + return TEST_FAILURE; + + pkt_stream->nb_rx_pkts++; + nb_frags = 0; + pkt_len = 0; + } + + if (nb_frags) { + /* In the middle of a packet. Start over from beginning of packet. */ + idx_rx -= nb_frags; + xsk_ring_cons__cancel(&xsk->rx, nb_frags); + if (ifobj->use_fill_ring) { + idx_fq -= nb_frags; + xsk_ring_prod__cancel(&umem->fq, nb_frags); + } + frags_processed -= nb_frags; + pkt_stream_cancel(pkt_stream); + pkts_sent--; + } + + if (ifobj->use_fill_ring) + xsk_ring_prod__submit(&umem->fq, frags_processed); + if (ifobj->release_rx) + xsk_ring_cons__release(&xsk->rx, frags_processed); + + pthread_mutex_lock(&pacing_mutex); + pkts_in_flight -= pkts_sent; + pthread_mutex_unlock(&pacing_mutex); + pkts_sent = 0; + + return TEST_CONTINUE; +} + +bool all_packets_received(struct test_spec *test, struct xsk_socket_info *xsk, u32 sock_num, + unsigned long *bitmap) +{ + struct pkt_stream *pkt_stream = xsk->pkt_stream; + + if (!pkt_stream) { + __set_bit(sock_num, bitmap); + return false; + } + + if (pkt_stream->nb_rx_pkts == pkt_stream->nb_valid_entries) { + __set_bit(sock_num, bitmap); + if (bitmap_full(bitmap, test->nb_sockets)) + return true; + } + + return false; +} + +static int receive_pkts(struct test_spec *test) +{ + struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0}; + DECLARE_BITMAP(bitmap, test->nb_sockets); + struct xsk_socket_info *xsk; + u32 sock_num = 0; + int res, ret; + + ret = gettimeofday(&tv_now, NULL); + if (ret) + exit_with_error(errno); + + timeradd(&tv_now, &tv_timeout, &tv_end); + + while (1) { + xsk = &test->ifobj_rx->xsk_arr[sock_num]; + + if ((all_packets_received(test, xsk, sock_num, bitmap))) + break; + + res = __receive_pkts(test, xsk); + if (!(res == TEST_PASS || res == TEST_CONTINUE)) + return res; + + ret = gettimeofday(&tv_now, NULL); + if (ret) + exit_with_error(errno); + + if (timercmp(&tv_now, &tv_end, >)) { + ksft_print_msg("ERROR: [%s] Receive loop timed out\n", __func__); + return TEST_FAILURE; + } + sock_num = (sock_num + 1) % test->nb_sockets; + } + + return TEST_PASS; +} + +static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, bool timeout) +{ + u32 i, idx = 0, valid_pkts = 0, valid_frags = 0, buffer_len; + struct pkt_stream *pkt_stream = xsk->pkt_stream; + struct xsk_umem_info *umem = ifobject->umem; + bool use_poll = ifobject->use_poll; + struct pollfd fds = { }; + int ret; + + buffer_len = pkt_get_buffer_len(umem, pkt_stream->max_pkt_len); + /* pkts_in_flight might be negative if many invalid packets are sent */ + if (pkts_in_flight >= (int)((umem_size(umem) - xsk->batch_size * buffer_len) / + buffer_len)) { + ret = kick_tx(xsk); + if (ret) + return TEST_FAILURE; + return TEST_CONTINUE; + } + + fds.fd = xsk_socket__fd(xsk->xsk); + fds.events = POLLOUT; + + while (xsk_ring_prod__reserve(&xsk->tx, xsk->batch_size, &idx) < xsk->batch_size) { + if (use_poll) { + ret = poll(&fds, 1, POLL_TMOUT); + if (timeout) { + if (ret < 0) { + ksft_print_msg("ERROR: [%s] Poll error %d\n", + __func__, errno); + return TEST_FAILURE; + } + if (ret == 0) + return TEST_PASS; + break; + } + if (ret <= 0) { + ksft_print_msg("ERROR: [%s] Poll error %d\n", + __func__, errno); + return TEST_FAILURE; + } + } + + complete_pkts(xsk, xsk->batch_size); + } + + for (i = 0; i < xsk->batch_size; i++) { + struct pkt *pkt = pkt_stream_get_next_tx_pkt(pkt_stream); + u32 nb_frags_left, nb_frags, bytes_written = 0; + + if (!pkt) + break; + + nb_frags = pkt_nb_frags(umem->frame_size, pkt_stream, pkt); + if (nb_frags > xsk->batch_size - i) { + pkt_stream_cancel(pkt_stream); + xsk_ring_prod__cancel(&xsk->tx, xsk->batch_size - i); + break; + } + nb_frags_left = nb_frags; + + while (nb_frags_left--) { + struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i); + + tx_desc->addr = pkt_get_addr(pkt, ifobject->umem); + if (pkt_stream->verbatim) { + tx_desc->len = pkt->len; + tx_desc->options = pkt->options; + } else if (nb_frags_left) { + tx_desc->len = umem->frame_size; + tx_desc->options = XDP_PKT_CONTD; + } else { + tx_desc->len = pkt->len - bytes_written; + tx_desc->options = 0; + } + if (pkt->valid) + pkt_generate(xsk, umem, tx_desc->addr, tx_desc->len, pkt->pkt_nb, + bytes_written); + bytes_written += tx_desc->len; + + print_verbose("Tx addr: %llx len: %u options: %u pkt_nb: %u\n", + tx_desc->addr, tx_desc->len, tx_desc->options, pkt->pkt_nb); + + if (nb_frags_left) { + i++; + if (pkt_stream->verbatim) + pkt = pkt_stream_get_next_tx_pkt(pkt_stream); + } + } + + if (pkt && pkt->valid) { + valid_pkts++; + valid_frags += nb_frags; + } + } + + pthread_mutex_lock(&pacing_mutex); + pkts_in_flight += valid_pkts; + pthread_mutex_unlock(&pacing_mutex); + + xsk_ring_prod__submit(&xsk->tx, i); + xsk->outstanding_tx += valid_frags; + + if (use_poll) { + ret = poll(&fds, 1, POLL_TMOUT); + if (ret <= 0) { + if (ret == 0 && timeout) + return TEST_PASS; + + ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret); + return TEST_FAILURE; + } + } + + if (!timeout) { + if (complete_pkts(xsk, i)) + return TEST_FAILURE; + + usleep(10); + return TEST_PASS; + } + + return TEST_CONTINUE; +} + +static int wait_for_tx_completion(struct xsk_socket_info *xsk) +{ + struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0}; + int ret; + + ret = gettimeofday(&tv_now, NULL); + if (ret) + exit_with_error(errno); + timeradd(&tv_now, &tv_timeout, &tv_end); + + while (xsk->outstanding_tx) { + ret = gettimeofday(&tv_now, NULL); + if (ret) + exit_with_error(errno); + if (timercmp(&tv_now, &tv_end, >)) { + ksft_print_msg("ERROR: [%s] Transmission loop timed out\n", __func__); + return TEST_FAILURE; + } + + complete_pkts(xsk, xsk->batch_size); + } + + return TEST_PASS; +} + +bool all_packets_sent(struct test_spec *test, unsigned long *bitmap) +{ + return bitmap_full(bitmap, test->nb_sockets); +} + +static int send_pkts(struct test_spec *test, struct ifobject *ifobject) +{ + bool timeout = !is_umem_valid(test->ifobj_rx); + DECLARE_BITMAP(bitmap, test->nb_sockets); + u32 i, ret; + + while (!(all_packets_sent(test, bitmap))) { + for (i = 0; i < test->nb_sockets; i++) { + struct pkt_stream *pkt_stream; + + pkt_stream = ifobject->xsk_arr[i].pkt_stream; + if (!pkt_stream || pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts) { + __set_bit(i, bitmap); + continue; + } + ret = __send_pkts(ifobject, &ifobject->xsk_arr[i], timeout); + if (ret == TEST_CONTINUE && !test->fail) + continue; + + if ((ret || test->fail) && !timeout) + return TEST_FAILURE; + + if (ret == TEST_PASS && timeout) + return ret; + + ret = wait_for_tx_completion(&ifobject->xsk_arr[i]); + if (ret) + return TEST_FAILURE; + } + } + + return TEST_PASS; +} + +static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats) +{ + int fd = xsk_socket__fd(xsk), err; + socklen_t optlen, expected_len; + + optlen = sizeof(*stats); + err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, stats, &optlen); + if (err) { + ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n", + __func__, -err, strerror(-err)); + return TEST_FAILURE; + } + + expected_len = sizeof(struct xdp_statistics); + if (optlen != expected_len) { + ksft_print_msg("[%s] getsockopt optlen error. Expected: %u got: %u\n", + __func__, expected_len, optlen); + return TEST_FAILURE; + } + + return TEST_PASS; +} + +static int validate_rx_dropped(struct ifobject *ifobject) +{ + struct xsk_socket *xsk = ifobject->xsk->xsk; + struct xdp_statistics stats; + int err; + + err = kick_rx(ifobject->xsk); + if (err) + return TEST_FAILURE; + + err = get_xsk_stats(xsk, &stats); + if (err) + return TEST_FAILURE; + + /* The receiver calls getsockopt after receiving the last (valid) + * packet which is not the final packet sent in this test (valid and + * invalid packets are sent in alternating fashion with the final + * packet being invalid). Since the last packet may or may not have + * been dropped already, both outcomes must be allowed. + */ + if (stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 || + stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 - 1) + return TEST_PASS; + + return TEST_FAILURE; +} + +static int validate_rx_full(struct ifobject *ifobject) +{ + struct xsk_socket *xsk = ifobject->xsk->xsk; + struct xdp_statistics stats; + int err; + + usleep(1000); + err = kick_rx(ifobject->xsk); + if (err) + return TEST_FAILURE; + + err = get_xsk_stats(xsk, &stats); + if (err) + return TEST_FAILURE; + + if (stats.rx_ring_full) + return TEST_PASS; + + return TEST_FAILURE; +} + +static int validate_fill_empty(struct ifobject *ifobject) +{ + struct xsk_socket *xsk = ifobject->xsk->xsk; + struct xdp_statistics stats; + int err; + + usleep(1000); + err = kick_rx(ifobject->xsk); + if (err) + return TEST_FAILURE; + + err = get_xsk_stats(xsk, &stats); + if (err) + return TEST_FAILURE; + + if (stats.rx_fill_ring_empty_descs) + return TEST_PASS; + + return TEST_FAILURE; +} + +static int validate_tx_invalid_descs(struct ifobject *ifobject) +{ + struct xsk_socket *xsk = ifobject->xsk->xsk; + int fd = xsk_socket__fd(xsk); + struct xdp_statistics stats; + socklen_t optlen; + int err; + + optlen = sizeof(stats); + err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen); + if (err) { + ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n", + __func__, -err, strerror(-err)); + return TEST_FAILURE; + } + + if (stats.tx_invalid_descs != ifobject->xsk->pkt_stream->nb_pkts / 2) { + ksft_print_msg("[%s] tx_invalid_descs incorrect. Got [%llu] expected [%u]\n", + __func__, + (unsigned long long)stats.tx_invalid_descs, + ifobject->xsk->pkt_stream->nb_pkts); + return TEST_FAILURE; + } + + return TEST_PASS; +} + +static void xsk_configure(struct test_spec *test, struct ifobject *ifobject, + struct xsk_umem_info *umem, bool tx) +{ + int i, ret; + + for (i = 0; i < test->nb_sockets; i++) { + bool shared = (ifobject->shared_umem && tx) ? true : !!i; + u32 ctr = 0; + + while (ctr++ < SOCK_RECONF_CTR) { + ret = xsk_configure_socket(&ifobject->xsk_arr[i], umem, + ifobject, shared); + if (!ret) + break; + + /* Retry if it fails as xsk_socket__create() is asynchronous */ + if (ctr >= SOCK_RECONF_CTR) + exit_with_error(-ret); + usleep(USLEEP_MAX); + } + if (ifobject->busy_poll) + enable_busy_poll(&ifobject->xsk_arr[i]); + } +} + +static void thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject) +{ + xsk_configure(test, ifobject, test->ifobj_rx->umem, true); + ifobject->xsk = &ifobject->xsk_arr[0]; + ifobject->xskmap = test->ifobj_rx->xskmap; + memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info)); + ifobject->umem->base_addr = 0; +} + +static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream, + bool fill_up) +{ + u32 rx_frame_size = umem->frame_size - XDP_PACKET_HEADROOM; + u32 idx = 0, filled = 0, buffers_to_fill, nb_pkts; + int ret; + + if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS) + buffers_to_fill = umem->num_frames; + else + buffers_to_fill = umem->fill_size; + + ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx); + if (ret != buffers_to_fill) + exit_with_error(ENOSPC); + + while (filled < buffers_to_fill) { + struct pkt *pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &nb_pkts); + u64 addr; + u32 i; + + for (i = 0; i < pkt_nb_frags(rx_frame_size, pkt_stream, pkt); i++) { + if (!pkt) { + if (!fill_up) + break; + addr = filled * umem->frame_size + umem->base_addr; + } else if (pkt->offset >= 0) { + addr = pkt->offset % umem->frame_size + umem_alloc_buffer(umem); + } else { + addr = pkt->offset + umem_alloc_buffer(umem); + } + + *xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr; + if (++filled >= buffers_to_fill) + break; + } + } + xsk_ring_prod__submit(&umem->fq, filled); + xsk_ring_prod__cancel(&umem->fq, buffers_to_fill - filled); + + pkt_stream_reset(pkt_stream); + umem_reset_alloc(umem); +} + +static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject) +{ + LIBBPF_OPTS(bpf_xdp_query_opts, opts); + int mmap_flags; + u64 umem_sz; + void *bufs; + int ret; + u32 i; + + umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size; + mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; + + if (ifobject->umem->unaligned_mode) + mmap_flags |= MAP_HUGETLB | MAP_HUGE_2MB; + + if (ifobject->shared_umem) + umem_sz *= 2; + + bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); + if (bufs == MAP_FAILED) + exit_with_error(errno); + + ret = xsk_configure_umem(ifobject, ifobject->umem, bufs, umem_sz); + if (ret) + exit_with_error(-ret); + + xsk_configure(test, ifobject, ifobject->umem, false); + + ifobject->xsk = &ifobject->xsk_arr[0]; + + if (!ifobject->rx_on) + return; + + xsk_populate_fill_ring(ifobject->umem, ifobject->xsk->pkt_stream, ifobject->use_fill_ring); + + for (i = 0; i < test->nb_sockets; i++) { + ifobject->xsk = &ifobject->xsk_arr[i]; + ret = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, i); + if (ret) + exit_with_error(errno); + } +} + +void *worker_testapp_validate_tx(void *arg) +{ + struct test_spec *test = (struct test_spec *)arg; + struct ifobject *ifobject = test->ifobj_tx; + int err; + + if (test->current_step == 1) { + if (!ifobject->shared_umem) + thread_common_ops(test, ifobject); + else + thread_common_ops_tx(test, ifobject); + } + + err = send_pkts(test, ifobject); + + if (!err && ifobject->validation_func) + err = ifobject->validation_func(ifobject); + if (err) + test->fail = true; + + pthread_exit(NULL); +} + +void *worker_testapp_validate_rx(void *arg) +{ + struct test_spec *test = (struct test_spec *)arg; + struct ifobject *ifobject = test->ifobj_rx; + int err; + + if (test->current_step == 1) { + thread_common_ops(test, ifobject); + } else { + xsk_clear_xskmap(ifobject->xskmap); + err = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, 0); + if (err) { + ksft_print_msg("Error: Failed to update xskmap, error %s\n", + strerror(-err)); + exit_with_error(-err); + } + } + + pthread_barrier_wait(&barr); + + err = receive_pkts(test); + + if (!err && ifobject->validation_func) + err = ifobject->validation_func(ifobject); + + if (err) { + if (test->adjust_tail && !is_adjust_tail_supported(ifobject->xdp_progs)) + test->adjust_tail_support = false; + else + test->fail = true; + } + + pthread_exit(NULL); +} + +static void testapp_clean_xsk_umem(struct ifobject *ifobj) +{ + u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size; + + if (ifobj->shared_umem) + umem_sz *= 2; + + umem_sz = ceil_u64(umem_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE; + xsk_umem__delete(ifobj->umem->umem); + munmap(ifobj->umem->buffer, umem_sz); +} + +static void handler(int signum) +{ + pthread_exit(NULL); +} + +static bool xdp_prog_changed_rx(struct test_spec *test) +{ + struct ifobject *ifobj = test->ifobj_rx; + + return ifobj->xdp_prog != test->xdp_prog_rx || ifobj->mode != test->mode; +} + +static bool xdp_prog_changed_tx(struct test_spec *test) +{ + struct ifobject *ifobj = test->ifobj_tx; + + return ifobj->xdp_prog != test->xdp_prog_tx || ifobj->mode != test->mode; +} + +static void xsk_reattach_xdp(struct ifobject *ifobj, struct bpf_program *xdp_prog, + struct bpf_map *xskmap, enum test_mode mode) +{ + int err; + + xsk_detach_xdp_program(ifobj->ifindex, mode_to_xdp_flags(ifobj->mode)); + err = xsk_attach_xdp_program(xdp_prog, ifobj->ifindex, mode_to_xdp_flags(mode)); + if (err) { + ksft_print_msg("Error attaching XDP program\n"); + exit_with_error(-err); + } + + if (ifobj->mode != mode && (mode == TEST_MODE_DRV || mode == TEST_MODE_ZC)) + if (!xsk_is_in_mode(ifobj->ifindex, XDP_FLAGS_DRV_MODE)) { + ksft_print_msg("ERROR: XDP prog not in DRV mode\n"); + exit_with_error(EINVAL); + } + + ifobj->xdp_prog = xdp_prog; + ifobj->xskmap = xskmap; + ifobj->mode = mode; +} + +static void xsk_attach_xdp_progs(struct test_spec *test, struct ifobject *ifobj_rx, + struct ifobject *ifobj_tx) +{ + if (xdp_prog_changed_rx(test)) + xsk_reattach_xdp(ifobj_rx, test->xdp_prog_rx, test->xskmap_rx, test->mode); + + if (!ifobj_tx || ifobj_tx->shared_umem) + return; + + if (xdp_prog_changed_tx(test)) + xsk_reattach_xdp(ifobj_tx, test->xdp_prog_tx, test->xskmap_tx, test->mode); +} + +static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *ifobj1, + struct ifobject *ifobj2) +{ + pthread_t t0, t1; + int err; + + if (test->mtu > MAX_ETH_PKT_SIZE) { + if (test->mode == TEST_MODE_ZC && (!ifobj1->multi_buff_zc_supp || + (ifobj2 && !ifobj2->multi_buff_zc_supp))) { + ksft_print_msg("Multi buffer for zero-copy not supported.\n"); + return TEST_SKIP; + } + if (test->mode != TEST_MODE_ZC && (!ifobj1->multi_buff_supp || + (ifobj2 && !ifobj2->multi_buff_supp))) { + ksft_print_msg("Multi buffer not supported.\n"); + return TEST_SKIP; + } + } + err = test_spec_set_mtu(test, test->mtu); + if (err) { + ksft_print_msg("Error, could not set mtu.\n"); + exit_with_error(err); + } + + if (ifobj2) { + if (pthread_barrier_init(&barr, NULL, 2)) + exit_with_error(errno); + pkt_stream_reset(ifobj2->xsk->pkt_stream); + } + + test->current_step++; + pkt_stream_reset(ifobj1->xsk->pkt_stream); + pkts_in_flight = 0; + + signal(SIGUSR1, handler); + /*Spawn RX thread */ + pthread_create(&t0, NULL, ifobj1->func_ptr, test); + + if (ifobj2) { + pthread_barrier_wait(&barr); + if (pthread_barrier_destroy(&barr)) + exit_with_error(errno); + + /*Spawn TX thread */ + pthread_create(&t1, NULL, ifobj2->func_ptr, test); + + pthread_join(t1, NULL); + } + + if (!ifobj2) + pthread_kill(t0, SIGUSR1); + else + pthread_join(t0, NULL); + + if (test->total_steps == test->current_step || test->fail) { + u32 i; + + if (ifobj2) + for (i = 0; i < test->nb_sockets; i++) + xsk_socket__delete(ifobj2->xsk_arr[i].xsk); + + for (i = 0; i < test->nb_sockets; i++) + xsk_socket__delete(ifobj1->xsk_arr[i].xsk); + + testapp_clean_xsk_umem(ifobj1); + if (ifobj2 && !ifobj2->shared_umem) + testapp_clean_xsk_umem(ifobj2); + } + + return !!test->fail; +} + +static int testapp_validate_traffic(struct test_spec *test) +{ + struct ifobject *ifobj_rx = test->ifobj_rx; + struct ifobject *ifobj_tx = test->ifobj_tx; + + if ((ifobj_rx->umem->unaligned_mode && !ifobj_rx->unaligned_supp) || + (ifobj_tx->umem->unaligned_mode && !ifobj_tx->unaligned_supp)) { + ksft_print_msg("No huge pages present.\n"); + return TEST_SKIP; + } + + if (test->set_ring) { + if (ifobj_tx->hw_ring_size_supp) { + if (set_ring_size(ifobj_tx)) { + ksft_print_msg("Failed to change HW ring size.\n"); + return TEST_FAILURE; + } + } else { + ksft_print_msg("Changing HW ring size not supported.\n"); + return TEST_SKIP; + } + } + + xsk_attach_xdp_progs(test, ifobj_rx, ifobj_tx); + return __testapp_validate_traffic(test, ifobj_rx, ifobj_tx); +} + +static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj) +{ + return __testapp_validate_traffic(test, ifobj, NULL); +} + +int testapp_teardown(struct test_spec *test) +{ + int i; + + for (i = 0; i < MAX_TEARDOWN_ITER; i++) { + if (testapp_validate_traffic(test)) + return TEST_FAILURE; + test_spec_reset(test); + } + + return TEST_PASS; +} + +static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2) +{ + thread_func_t tmp_func_ptr = (*ifobj1)->func_ptr; + struct ifobject *tmp_ifobj = (*ifobj1); + + (*ifobj1)->func_ptr = (*ifobj2)->func_ptr; + (*ifobj2)->func_ptr = tmp_func_ptr; + + *ifobj1 = *ifobj2; + *ifobj2 = tmp_ifobj; +} + +int testapp_bidirectional(struct test_spec *test) +{ + int res; + + test->ifobj_tx->rx_on = true; + test->ifobj_rx->tx_on = true; + test->total_steps = 2; + if (testapp_validate_traffic(test)) + return TEST_FAILURE; + + print_verbose("Switching Tx/Rx direction\n"); + swap_directions(&test->ifobj_rx, &test->ifobj_tx); + res = __testapp_validate_traffic(test, test->ifobj_rx, test->ifobj_tx); + + swap_directions(&test->ifobj_rx, &test->ifobj_tx); + return res; +} + +static int swap_xsk_resources(struct test_spec *test) +{ + int ret; + + test->ifobj_tx->xsk_arr[0].pkt_stream = NULL; + test->ifobj_rx->xsk_arr[0].pkt_stream = NULL; + test->ifobj_tx->xsk_arr[1].pkt_stream = test->tx_pkt_stream_default; + test->ifobj_rx->xsk_arr[1].pkt_stream = test->rx_pkt_stream_default; + test->ifobj_tx->xsk = &test->ifobj_tx->xsk_arr[1]; + test->ifobj_rx->xsk = &test->ifobj_rx->xsk_arr[1]; + + ret = xsk_update_xskmap(test->ifobj_rx->xskmap, test->ifobj_rx->xsk->xsk, 0); + if (ret) + return TEST_FAILURE; + + return TEST_PASS; +} + +int testapp_xdp_prog_cleanup(struct test_spec *test) +{ + test->total_steps = 2; + test->nb_sockets = 2; + if (testapp_validate_traffic(test)) + return TEST_FAILURE; + + if (swap_xsk_resources(test)) + return TEST_FAILURE; + return testapp_validate_traffic(test); +} + +int testapp_headroom(struct test_spec *test) +{ + test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE; + return testapp_validate_traffic(test); +} + +int testapp_stats_rx_dropped(struct test_spec *test) +{ + if (test->mode == TEST_MODE_ZC) { + ksft_print_msg("Can not run RX_DROPPED test for ZC mode\n"); + return TEST_SKIP; + } + + pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0); + test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size - + XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3; + pkt_stream_receive_half(test); + test->ifobj_rx->validation_func = validate_rx_dropped; + return testapp_validate_traffic(test); +} + +int testapp_stats_tx_invalid_descs(struct test_spec *test) +{ + pkt_stream_replace_half(test, XSK_UMEM__INVALID_FRAME_SIZE, 0); + test->ifobj_tx->validation_func = validate_tx_invalid_descs; + return testapp_validate_traffic(test); +} + +int testapp_stats_rx_full(struct test_spec *test) +{ + pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE); + test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE); + + test->ifobj_rx->xsk->rxqsize = DEFAULT_UMEM_BUFFERS; + test->ifobj_rx->release_rx = false; + test->ifobj_rx->validation_func = validate_rx_full; + return testapp_validate_traffic(test); +} + +int testapp_stats_fill_empty(struct test_spec *test) +{ + pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE); + test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE); + + test->ifobj_rx->use_fill_ring = false; + test->ifobj_rx->validation_func = validate_fill_empty; + return testapp_validate_traffic(test); +} + +int testapp_send_receive_unaligned(struct test_spec *test) +{ + test->ifobj_tx->umem->unaligned_mode = true; + test->ifobj_rx->umem->unaligned_mode = true; + /* Let half of the packets straddle a 4K buffer boundary */ + pkt_stream_replace_half(test, MIN_PKT_SIZE, -MIN_PKT_SIZE / 2); + + return testapp_validate_traffic(test); +} + +int testapp_send_receive_unaligned_mb(struct test_spec *test) +{ + test->mtu = MAX_ETH_JUMBO_SIZE; + test->ifobj_tx->umem->unaligned_mode = true; + test->ifobj_rx->umem->unaligned_mode = true; + pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE); + return testapp_validate_traffic(test); +} + +int testapp_single_pkt(struct test_spec *test) +{ + struct pkt pkts[] = {{0, MIN_PKT_SIZE, 0, true}}; + + pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); + return testapp_validate_traffic(test); +} + +int testapp_send_receive_mb(struct test_spec *test) +{ + test->mtu = MAX_ETH_JUMBO_SIZE; + pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE); + + return testapp_validate_traffic(test); +} + +int testapp_invalid_desc_mb(struct test_spec *test) +{ + struct xsk_umem_info *umem = test->ifobj_tx->umem; + u64 umem_size = umem->num_frames * umem->frame_size; + struct pkt pkts[] = { + /* Valid packet for synch to start with */ + {0, MIN_PKT_SIZE, 0, true, 0}, + /* Zero frame len is not legal */ + {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {0, 0, 0, false, 0}, + /* Invalid address in the second frame */ + {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {umem_size, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + /* Invalid len in the middle */ + {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + /* Invalid options in the middle */ + {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XSK_DESC__INVALID_OPTION}, + /* Transmit 2 frags, receive 3 */ + {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, XDP_PKT_CONTD}, + {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, 0}, + /* Middle frame crosses chunk boundary with small length */ + {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {-MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false, 0}, + /* Valid packet for synch so that something is received */ + {0, MIN_PKT_SIZE, 0, true, 0}}; + + if (umem->unaligned_mode) { + /* Crossing a chunk boundary allowed */ + pkts[12].valid = true; + pkts[13].valid = true; + } + + test->mtu = MAX_ETH_JUMBO_SIZE; + pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); + return testapp_validate_traffic(test); +} + +int testapp_invalid_desc(struct test_spec *test) +{ + struct xsk_umem_info *umem = test->ifobj_tx->umem; + u64 umem_size = umem->num_frames * umem->frame_size; + struct pkt pkts[] = { + /* Zero packet address allowed */ + {0, MIN_PKT_SIZE, 0, true}, + /* Allowed packet */ + {0, MIN_PKT_SIZE, 0, true}, + /* Straddling the start of umem */ + {-2, MIN_PKT_SIZE, 0, false}, + /* Packet too large */ + {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false}, + /* Up to end of umem allowed */ + {umem_size - MIN_PKT_SIZE - 2 * umem->frame_size, MIN_PKT_SIZE, 0, true}, + /* After umem ends */ + {umem_size, MIN_PKT_SIZE, 0, false}, + /* Straddle the end of umem */ + {umem_size - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false}, + /* Straddle a 4K boundary */ + {0x1000 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false}, + /* Straddle a 2K boundary */ + {0x800 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, true}, + /* Valid packet for synch so that something is received */ + {0, MIN_PKT_SIZE, 0, true}}; + + if (umem->unaligned_mode) { + /* Crossing a page boundary allowed */ + pkts[7].valid = true; + } + if (umem->frame_size == XSK_UMEM__DEFAULT_FRAME_SIZE / 2) { + /* Crossing a 2K frame size boundary not allowed */ + pkts[8].valid = false; + } + + if (test->ifobj_tx->shared_umem) { + pkts[4].offset += umem_size; + pkts[5].offset += umem_size; + pkts[6].offset += umem_size; + } + + pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); + return testapp_validate_traffic(test); +} + +int testapp_xdp_drop(struct test_spec *test) +{ + struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; + struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; + + test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_drop, skel_tx->progs.xsk_xdp_drop, + skel_rx->maps.xsk, skel_tx->maps.xsk); + + pkt_stream_receive_half(test); + return testapp_validate_traffic(test); +} + +int testapp_xdp_metadata_copy(struct test_spec *test) +{ + struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; + struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; + + test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_populate_metadata, + skel_tx->progs.xsk_xdp_populate_metadata, + skel_rx->maps.xsk, skel_tx->maps.xsk); + test->ifobj_rx->use_metadata = true; + + skel_rx->bss->count = 0; + + return testapp_validate_traffic(test); +} + +int testapp_xdp_shared_umem(struct test_spec *test) +{ + struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; + struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; + + test->total_steps = 1; + test->nb_sockets = 2; + + test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_shared_umem, + skel_tx->progs.xsk_xdp_shared_umem, + skel_rx->maps.xsk, skel_tx->maps.xsk); + + pkt_stream_even_odd_sequence(test); + + return testapp_validate_traffic(test); +} + +int testapp_poll_txq_tmout(struct test_spec *test) +{ + test->ifobj_tx->use_poll = true; + /* create invalid frame by set umem frame_size and pkt length equal to 2048 */ + test->ifobj_tx->umem->frame_size = 2048; + pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048); + return testapp_validate_traffic_single_thread(test, test->ifobj_tx); +} + +int testapp_poll_rxq_tmout(struct test_spec *test) +{ + test->ifobj_rx->use_poll = true; + return testapp_validate_traffic_single_thread(test, test->ifobj_rx); +} + +int testapp_too_many_frags(struct test_spec *test) +{ + struct pkt *pkts; + u32 max_frags, i; + int ret; + + if (test->mode == TEST_MODE_ZC) { + max_frags = test->ifobj_tx->xdp_zc_max_segs; + } else { + max_frags = get_max_skb_frags(); + if (!max_frags) { + ksft_print_msg("Can't get MAX_SKB_FRAGS from system, using default (17)\n"); + max_frags = 17; + } + max_frags += 1; + } + + pkts = calloc(2 * max_frags + 2, sizeof(struct pkt)); + if (!pkts) + return TEST_FAILURE; + + test->mtu = MAX_ETH_JUMBO_SIZE; + + /* Valid packet for synch */ + pkts[0].len = MIN_PKT_SIZE; + pkts[0].valid = true; + + /* One valid packet with the max amount of frags */ + for (i = 1; i < max_frags + 1; i++) { + pkts[i].len = MIN_PKT_SIZE; + pkts[i].options = XDP_PKT_CONTD; + pkts[i].valid = true; + } + pkts[max_frags].options = 0; + + /* An invalid packet with the max amount of frags but signals packet + * continues on the last frag + */ + for (i = max_frags + 1; i < 2 * max_frags + 1; i++) { + pkts[i].len = MIN_PKT_SIZE; + pkts[i].options = XDP_PKT_CONTD; + pkts[i].valid = false; + } + + /* Valid packet for synch */ + pkts[2 * max_frags + 1].len = MIN_PKT_SIZE; + pkts[2 * max_frags + 1].valid = true; + + pkt_stream_generate_custom(test, pkts, 2 * max_frags + 2); + ret = testapp_validate_traffic(test); + + free(pkts); + return ret; +} + +static int xsk_load_xdp_programs(struct ifobject *ifobj) +{ + ifobj->xdp_progs = xsk_xdp_progs__open_and_load(); + if (libbpf_get_error(ifobj->xdp_progs)) + return libbpf_get_error(ifobj->xdp_progs); + + return 0; +} + +/* Simple test */ +static bool hugepages_present(void) +{ + size_t mmap_sz = 2 * DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE; + void *bufs; + + bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, MAP_HUGE_2MB); + if (bufs == MAP_FAILED) + return false; + + mmap_sz = ceil_u64(mmap_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE; + munmap(bufs, mmap_sz); + return true; +} + +void init_iface(struct ifobject *ifobj, thread_func_t func_ptr) +{ + LIBBPF_OPTS(bpf_xdp_query_opts, query_opts); + int err; + + ifobj->func_ptr = func_ptr; + + err = xsk_load_xdp_programs(ifobj); + if (err) { + ksft_print_msg("Error loading XDP program\n"); + exit_with_error(err); + } + + if (hugepages_present()) + ifobj->unaligned_supp = true; + + err = bpf_xdp_query(ifobj->ifindex, XDP_FLAGS_DRV_MODE, &query_opts); + if (err) { + ksft_print_msg("Error querying XDP capabilities\n"); + exit_with_error(-err); + } + if (query_opts.feature_flags & NETDEV_XDP_ACT_RX_SG) + ifobj->multi_buff_supp = true; + if (query_opts.feature_flags & NETDEV_XDP_ACT_XSK_ZEROCOPY) { + if (query_opts.xdp_zc_max_segs > 1) { + ifobj->multi_buff_zc_supp = true; + ifobj->xdp_zc_max_segs = query_opts.xdp_zc_max_segs; + } else { + ifobj->xdp_zc_max_segs = 0; + } + } +} + +int testapp_send_receive(struct test_spec *test) +{ + return testapp_validate_traffic(test); +} + +int testapp_send_receive_2k_frame(struct test_spec *test) +{ + test->ifobj_tx->umem->frame_size = 2048; + test->ifobj_rx->umem->frame_size = 2048; + pkt_stream_replace(test, DEFAULT_PKT_CNT, MIN_PKT_SIZE); + return testapp_validate_traffic(test); +} + +int testapp_poll_rx(struct test_spec *test) +{ + test->ifobj_rx->use_poll = true; + return testapp_validate_traffic(test); +} + +int testapp_poll_tx(struct test_spec *test) +{ + test->ifobj_tx->use_poll = true; + return testapp_validate_traffic(test); +} + +int testapp_aligned_inv_desc(struct test_spec *test) +{ + return testapp_invalid_desc(test); +} + +int testapp_aligned_inv_desc_2k_frame(struct test_spec *test) +{ + test->ifobj_tx->umem->frame_size = 2048; + test->ifobj_rx->umem->frame_size = 2048; + return testapp_invalid_desc(test); +} + +int testapp_unaligned_inv_desc(struct test_spec *test) +{ + test->ifobj_tx->umem->unaligned_mode = true; + test->ifobj_rx->umem->unaligned_mode = true; + return testapp_invalid_desc(test); +} + +int testapp_unaligned_inv_desc_4001_frame(struct test_spec *test) +{ + u64 page_size, umem_size; + + /* Odd frame size so the UMEM doesn't end near a page boundary. */ + test->ifobj_tx->umem->frame_size = 4001; + test->ifobj_rx->umem->frame_size = 4001; + test->ifobj_tx->umem->unaligned_mode = true; + test->ifobj_rx->umem->unaligned_mode = true; + /* This test exists to test descriptors that staddle the end of + * the UMEM but not a page. + */ + page_size = sysconf(_SC_PAGESIZE); + umem_size = test->ifobj_tx->umem->num_frames * test->ifobj_tx->umem->frame_size; + assert(umem_size % page_size > MIN_PKT_SIZE); + assert(umem_size % page_size < page_size - MIN_PKT_SIZE); + + return testapp_invalid_desc(test); +} + +int testapp_aligned_inv_desc_mb(struct test_spec *test) +{ + return testapp_invalid_desc_mb(test); +} + +int testapp_unaligned_inv_desc_mb(struct test_spec *test) +{ + test->ifobj_tx->umem->unaligned_mode = true; + test->ifobj_rx->umem->unaligned_mode = true; + return testapp_invalid_desc_mb(test); +} + +int testapp_xdp_metadata(struct test_spec *test) +{ + return testapp_xdp_metadata_copy(test); +} + +int testapp_xdp_metadata_mb(struct test_spec *test) +{ + test->mtu = MAX_ETH_JUMBO_SIZE; + return testapp_xdp_metadata_copy(test); +} + +int testapp_hw_sw_min_ring_size(struct test_spec *test) +{ + int ret; + + test->set_ring = true; + test->total_steps = 2; + test->ifobj_tx->ring.tx_pending = DEFAULT_BATCH_SIZE; + test->ifobj_tx->ring.rx_pending = DEFAULT_BATCH_SIZE * 2; + test->ifobj_tx->xsk->batch_size = 1; + test->ifobj_rx->xsk->batch_size = 1; + ret = testapp_validate_traffic(test); + if (ret) + return ret; + + /* Set batch size to hw_ring_size - 1 */ + test->ifobj_tx->xsk->batch_size = DEFAULT_BATCH_SIZE - 1; + test->ifobj_rx->xsk->batch_size = DEFAULT_BATCH_SIZE - 1; + return testapp_validate_traffic(test); +} + +int testapp_hw_sw_max_ring_size(struct test_spec *test) +{ + u32 max_descs = XSK_RING_PROD__DEFAULT_NUM_DESCS * 4; + int ret; + + test->set_ring = true; + test->total_steps = 2; + test->ifobj_tx->ring.tx_pending = test->ifobj_tx->ring.tx_max_pending; + test->ifobj_tx->ring.rx_pending = test->ifobj_tx->ring.rx_max_pending; + test->ifobj_rx->umem->num_frames = max_descs; + test->ifobj_rx->umem->fill_size = max_descs; + test->ifobj_rx->umem->comp_size = max_descs; + test->ifobj_tx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; + test->ifobj_rx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; + + ret = testapp_validate_traffic(test); + if (ret) + return ret; + + /* Set batch_size to 8152 for testing, as the ice HW ignores the 3 lowest bits when + * updating the Rx HW tail register. + */ + test->ifobj_tx->xsk->batch_size = test->ifobj_tx->ring.tx_max_pending - 8; + test->ifobj_rx->xsk->batch_size = test->ifobj_tx->ring.tx_max_pending - 8; + pkt_stream_replace(test, max_descs, MIN_PKT_SIZE); + return testapp_validate_traffic(test); +} + +static int testapp_xdp_adjust_tail(struct test_spec *test, int adjust_value) +{ + struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; + struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; + + test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_adjust_tail, + skel_tx->progs.xsk_xdp_adjust_tail, + skel_rx->maps.xsk, skel_tx->maps.xsk); + + skel_rx->bss->adjust_value = adjust_value; + + return testapp_validate_traffic(test); +} + +static int testapp_adjust_tail(struct test_spec *test, u32 value, u32 pkt_len) +{ + int ret; + + test->adjust_tail_support = true; + test->adjust_tail = true; + test->total_steps = 1; + + pkt_stream_replace_ifobject(test->ifobj_tx, DEFAULT_BATCH_SIZE, pkt_len); + pkt_stream_replace_ifobject(test->ifobj_rx, DEFAULT_BATCH_SIZE, pkt_len + value); + + ret = testapp_xdp_adjust_tail(test, value); + if (ret) + return ret; + + if (!test->adjust_tail_support) { + ksft_print_msg("%s %sResize pkt with bpf_xdp_adjust_tail() not supported\n", + mode_string(test), busy_poll_string(test)); + return TEST_SKIP; + } + + return 0; +} + +int testapp_adjust_tail_shrink(struct test_spec *test) +{ + /* Shrink by 4 bytes for testing purpose */ + return testapp_adjust_tail(test, -4, MIN_PKT_SIZE * 2); +} + +int testapp_adjust_tail_shrink_mb(struct test_spec *test) +{ + test->mtu = MAX_ETH_JUMBO_SIZE; + /* Shrink by the frag size */ + return testapp_adjust_tail(test, -XSK_UMEM__MAX_FRAME_SIZE, XSK_UMEM__LARGE_FRAME_SIZE * 2); +} + +int testapp_adjust_tail_grow(struct test_spec *test) +{ + /* Grow by 4 bytes for testing purpose */ + return testapp_adjust_tail(test, 4, MIN_PKT_SIZE * 2); +} + +int testapp_adjust_tail_grow_mb(struct test_spec *test) +{ + test->mtu = MAX_ETH_JUMBO_SIZE; + /* Grow by (frag_size - last_frag_Size) - 1 to stay inside the last fragment */ + return testapp_adjust_tail(test, (XSK_UMEM__MAX_FRAME_SIZE / 2) - 1, + XSK_UMEM__LARGE_FRAME_SIZE * 2); +} + +int testapp_tx_queue_consumer(struct test_spec *test) +{ + int nr_packets; + + if (test->mode == TEST_MODE_ZC) { + ksft_print_msg("Can not run TX_QUEUE_CONSUMER test for ZC mode\n"); + return TEST_SKIP; + } + + nr_packets = MAX_TX_BUDGET_DEFAULT + 1; + pkt_stream_replace(test, nr_packets, MIN_PKT_SIZE); + test->ifobj_tx->xsk->batch_size = nr_packets; + test->ifobj_tx->xsk->check_consumer = true; + + return testapp_validate_traffic(test); +} + +struct ifobject *ifobject_create(void) +{ + struct ifobject *ifobj; + + ifobj = calloc(1, sizeof(struct ifobject)); + if (!ifobj) + return NULL; + + ifobj->xsk_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->xsk_arr)); + if (!ifobj->xsk_arr) + goto out_xsk_arr; + + ifobj->umem = calloc(1, sizeof(*ifobj->umem)); + if (!ifobj->umem) + goto out_umem; + + return ifobj; + +out_umem: + free(ifobj->xsk_arr); +out_xsk_arr: + free(ifobj); + return NULL; +} + +void ifobject_delete(struct ifobject *ifobj) +{ + free(ifobj->umem); + free(ifobj->xsk_arr); + free(ifobj); +} diff --git a/tools/testing/selftests/bpf/test_xsk.h b/tools/testing/selftests/bpf/test_xsk.h new file mode 100644 index 0000000000000..fb546cab39fdf --- /dev/null +++ b/tools/testing/selftests/bpf/test_xsk.h @@ -0,0 +1,297 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef TEST_XSK_H_ +#define TEST_XSK_H_ + +#include +#include + +#include "../kselftest.h" +#include "xsk.h" + +#ifndef SO_PREFER_BUSY_POLL +#define SO_PREFER_BUSY_POLL 69 +#endif + +#ifndef SO_BUSY_POLL_BUDGET +#define SO_BUSY_POLL_BUDGET 70 +#endif + +#define TEST_PASS 0 +#define TEST_FAILURE -1 +#define TEST_CONTINUE 1 +#define TEST_SKIP 2 + +#define DEFAULT_PKT_CNT (4 * 1024) +#define DEFAULT_UMEM_BUFFERS (DEFAULT_PKT_CNT / 4) +#define HUGEPAGE_SIZE (2 * 1024 * 1024) +#define MIN_PKT_SIZE 64 +#define MAX_ETH_PKT_SIZE 1518 +#define MAX_INTERFACE_NAME_CHARS 16 +#define MAX_TEST_NAME_SIZE 48 +#define SOCK_RECONF_CTR 10 +#define USLEEP_MAX 10000 + +extern bool opt_verbose; +#define print_verbose(x...) do { if (opt_verbose) ksft_print_msg(x); } while (0) + +static void __exit_with_error(int error, const char *file, const char *func, int line) +{ + ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error, + strerror(error)); + ksft_exit_xfail(); +} +#define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__) + +static inline u32 ceil_u32(u32 a, u32 b) +{ + return (a + b - 1) / b; +} + +static inline u64 ceil_u64(u64 a, u64 b) +{ + return (a + b - 1) / b; +} + +/* Simple test */ +enum test_mode { + TEST_MODE_SKB, + TEST_MODE_DRV, + TEST_MODE_ZC, + TEST_MODE_ALL +}; + +struct ifobject; +struct test_spec; +typedef int (*validation_func_t)(struct ifobject *ifobj); +typedef void *(*thread_func_t)(void *arg); +typedef int (*test_func_t)(struct test_spec *test); + +struct xsk_socket_info { + struct xsk_ring_cons rx; + struct xsk_ring_prod tx; + struct xsk_umem_info *umem; + struct xsk_socket *xsk; + struct pkt_stream *pkt_stream; + u32 outstanding_tx; + u32 rxqsize; + u32 batch_size; + u8 dst_mac[ETH_ALEN]; + u8 src_mac[ETH_ALEN]; + bool check_consumer; +}; + +int kick_rx(struct xsk_socket_info *xsk); +int kick_tx(struct xsk_socket_info *xsk); + +struct xsk_umem_info { + struct xsk_ring_prod fq; + struct xsk_ring_cons cq; + struct xsk_umem *umem; + u64 next_buffer; + u32 num_frames; + u32 frame_headroom; + void *buffer; + u32 frame_size; + u32 base_addr; + u32 fill_size; + u32 comp_size; + bool unaligned_mode; +}; + +struct set_hw_ring { + u32 default_tx; + u32 default_rx; +}; + +int hw_ring_size_reset(struct ifobject *ifobj); + +struct ifobject { + char ifname[MAX_INTERFACE_NAME_CHARS]; + struct xsk_socket_info *xsk; + struct xsk_socket_info *xsk_arr; + struct xsk_umem_info *umem; + thread_func_t func_ptr; + validation_func_t validation_func; + struct xsk_xdp_progs *xdp_progs; + struct bpf_map *xskmap; + struct bpf_program *xdp_prog; + struct ethtool_ringparam ring; + struct set_hw_ring set_ring; + enum test_mode mode; + int ifindex; + int mtu; + u32 bind_flags; + u32 xdp_zc_max_segs; + bool tx_on; + bool rx_on; + bool use_poll; + bool busy_poll; + bool use_fill_ring; + bool release_rx; + bool shared_umem; + bool use_metadata; + bool unaligned_supp; + bool multi_buff_supp; + bool multi_buff_zc_supp; + bool hw_ring_size_supp; +}; +struct ifobject *ifobject_create(void); +void ifobject_delete(struct ifobject *ifobj); +void init_iface(struct ifobject *ifobj, thread_func_t func_ptr); + +int xsk_configure_umem(struct ifobject *ifobj, struct xsk_umem_info *umem, void *buffer, u64 size); +int xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, + struct ifobject *ifobject, bool shared); + + +struct pkt { + int offset; + u32 len; + u32 pkt_nb; + bool valid; + u16 options; +}; + +struct pkt_stream { + u32 nb_pkts; + u32 current_pkt_nb; + struct pkt *pkts; + u32 max_pkt_len; + u32 nb_rx_pkts; + u32 nb_valid_entries; + bool verbatim; +}; + +static inline bool pkt_continues(u32 options) +{ + return options & XDP_PKT_CONTD; +} + +struct pkt_stream *pkt_stream_generate(u32 nb_pkts, u32 pkt_len); +void pkt_stream_delete(struct pkt_stream *pkt_stream); +void pkt_stream_reset(struct pkt_stream *pkt_stream); +void pkt_stream_restore_default(struct test_spec *test); + +struct test_spec { + struct ifobject *ifobj_tx; + struct ifobject *ifobj_rx; + struct pkt_stream *tx_pkt_stream_default; + struct pkt_stream *rx_pkt_stream_default; + struct bpf_program *xdp_prog_rx; + struct bpf_program *xdp_prog_tx; + struct bpf_map *xskmap_rx; + struct bpf_map *xskmap_tx; + test_func_t test_func; + int mtu; + u16 total_steps; + u16 current_step; + u16 nb_sockets; + bool fail; + bool set_ring; + bool adjust_tail; + bool adjust_tail_support; + enum test_mode mode; + char name[MAX_TEST_NAME_SIZE]; +}; + +#define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : "" +static inline char *mode_string(struct test_spec *test) +{ + switch (test->mode) { + case TEST_MODE_SKB: + return "SKB"; + case TEST_MODE_DRV: + return "DRV"; + case TEST_MODE_ZC: + return "ZC"; + default: + return "BOGUS"; + } +} + +void test_init(struct test_spec *test, struct ifobject *ifobj_tx, + struct ifobject *ifobj_rx, enum test_mode mode, + const struct test_spec *test_to_run); + +int testapp_adjust_tail_grow(struct test_spec *test); +int testapp_adjust_tail_grow_mb(struct test_spec *test); +int testapp_adjust_tail_shrink(struct test_spec *test); +int testapp_adjust_tail_shrink_mb(struct test_spec *test); +int testapp_aligned_inv_desc(struct test_spec *test); +int testapp_aligned_inv_desc_2k_frame(struct test_spec *test); +int testapp_aligned_inv_desc_mb(struct test_spec *test); +int testapp_bidirectional(struct test_spec *test); +int testapp_headroom(struct test_spec *test); +int testapp_hw_sw_max_ring_size(struct test_spec *test); +int testapp_hw_sw_min_ring_size(struct test_spec *test); +int testapp_poll_rx(struct test_spec *test); +int testapp_poll_rxq_tmout(struct test_spec *test); +int testapp_poll_tx(struct test_spec *test); +int testapp_poll_txq_tmout(struct test_spec *test); +int testapp_send_receive(struct test_spec *test); +int testapp_send_receive_2k_frame(struct test_spec *test); +int testapp_send_receive_mb(struct test_spec *test); +int testapp_send_receive_unaligned(struct test_spec *test); +int testapp_send_receive_unaligned_mb(struct test_spec *test); +int testapp_single_pkt(struct test_spec *test); +int testapp_stats_fill_empty(struct test_spec *test); +int testapp_stats_rx_dropped(struct test_spec *test); +int testapp_stats_tx_invalid_descs(struct test_spec *test); +int testapp_stats_rx_full(struct test_spec *test); +int testapp_teardown(struct test_spec *test); +int testapp_too_many_frags(struct test_spec *test); +int testapp_tx_queue_consumer(struct test_spec *test); +int testapp_unaligned_inv_desc(struct test_spec *test); +int testapp_unaligned_inv_desc_4001_frame(struct test_spec *test); +int testapp_unaligned_inv_desc_mb(struct test_spec *test); +int testapp_xdp_drop(struct test_spec *test); +int testapp_xdp_metadata(struct test_spec *test); +int testapp_xdp_metadata_mb(struct test_spec *test); +int testapp_xdp_prog_cleanup(struct test_spec *test); +int testapp_xdp_shared_umem(struct test_spec *test); + +void *worker_testapp_validate_rx(void *arg); +void *worker_testapp_validate_tx(void *arg); + +static const struct test_spec tests[] = { + {.name = "SEND_RECEIVE", .test_func = testapp_send_receive}, + {.name = "SEND_RECEIVE_2K_FRAME", .test_func = testapp_send_receive_2k_frame}, + {.name = "SEND_RECEIVE_SINGLE_PKT", .test_func = testapp_single_pkt}, + {.name = "POLL_RX", .test_func = testapp_poll_rx}, + {.name = "POLL_TX", .test_func = testapp_poll_tx}, + {.name = "POLL_RXQ_FULL", .test_func = testapp_poll_rxq_tmout}, + {.name = "POLL_TXQ_FULL", .test_func = testapp_poll_txq_tmout}, + {.name = "SEND_RECEIVE_UNALIGNED", .test_func = testapp_send_receive_unaligned}, + {.name = "ALIGNED_INV_DESC", .test_func = testapp_aligned_inv_desc}, + {.name = "ALIGNED_INV_DESC_2K_FRAME_SIZE", .test_func = testapp_aligned_inv_desc_2k_frame}, + {.name = "UNALIGNED_INV_DESC", .test_func = testapp_unaligned_inv_desc}, + {.name = "UNALIGNED_INV_DESC_4001_FRAME_SIZE", + .test_func = testapp_unaligned_inv_desc_4001_frame}, + {.name = "UMEM_HEADROOM", .test_func = testapp_headroom}, + {.name = "TEARDOWN", .test_func = testapp_teardown}, + {.name = "BIDIRECTIONAL", .test_func = testapp_bidirectional}, + {.name = "STAT_RX_DROPPED", .test_func = testapp_stats_rx_dropped}, + {.name = "STAT_TX_INVALID", .test_func = testapp_stats_tx_invalid_descs}, + {.name = "STAT_RX_FULL", .test_func = testapp_stats_rx_full}, + {.name = "STAT_FILL_EMPTY", .test_func = testapp_stats_fill_empty}, + {.name = "XDP_PROG_CLEANUP", .test_func = testapp_xdp_prog_cleanup}, + {.name = "XDP_DROP_HALF", .test_func = testapp_xdp_drop}, + {.name = "XDP_SHARED_UMEM", .test_func = testapp_xdp_shared_umem}, + {.name = "XDP_METADATA_COPY", .test_func = testapp_xdp_metadata}, + {.name = "XDP_METADATA_COPY_MULTI_BUFF", .test_func = testapp_xdp_metadata_mb}, + {.name = "SEND_RECEIVE_9K_PACKETS", .test_func = testapp_send_receive_mb}, + {.name = "SEND_RECEIVE_UNALIGNED_9K_PACKETS", + .test_func = testapp_send_receive_unaligned_mb}, + {.name = "ALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_aligned_inv_desc_mb}, + {.name = "UNALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_unaligned_inv_desc_mb}, + {.name = "TOO_MANY_FRAGS", .test_func = testapp_too_many_frags}, + {.name = "HW_SW_MIN_RING_SIZE", .test_func = testapp_hw_sw_min_ring_size}, + {.name = "HW_SW_MAX_RING_SIZE", .test_func = testapp_hw_sw_max_ring_size}, + {.name = "XDP_ADJUST_TAIL_SHRINK", .test_func = testapp_adjust_tail_shrink}, + {.name = "XDP_ADJUST_TAIL_SHRINK_MULTI_BUFF", .test_func = testapp_adjust_tail_shrink_mb}, + {.name = "XDP_ADJUST_TAIL_GROW", .test_func = testapp_adjust_tail_grow}, + {.name = "XDP_ADJUST_TAIL_GROW_MULTI_BUFF", .test_func = testapp_adjust_tail_grow_mb}, + {.name = "TX_QUEUE_CONSUMER", .test_func = testapp_tx_queue_consumer}, + }; + +#endif /* TEST_XSK_H_ */ diff --git a/tools/testing/selftests/bpf/verifier/precise.c b/tools/testing/selftests/bpf/verifier/precise.c index 59a020c356474..ef3ec56672c22 100644 --- a/tools/testing/selftests/bpf/verifier/precise.c +++ b/tools/testing/selftests/bpf/verifier/precise.c @@ -44,9 +44,9 @@ mark_precise: frame0: regs=r2 stack= before 23\ mark_precise: frame0: regs=r2 stack= before 22\ mark_precise: frame0: regs=r2 stack= before 20\ - mark_precise: frame0: parent state regs=r2,r9 stack=:\ + mark_precise: frame0: parent state regs=r2 stack=:\ mark_precise: frame0: last_idx 19 first_idx 10\ - mark_precise: frame0: regs=r2,r9 stack= before 19\ + mark_precise: frame0: regs=r2 stack= before 19\ mark_precise: frame0: regs=r9 stack= before 18\ mark_precise: frame0: regs=r8,r9 stack= before 17\ mark_precise: frame0: regs=r0,r9 stack= before 15\ @@ -107,9 +107,9 @@ mark_precise: frame0: parent state regs=r2 stack=:\ mark_precise: frame0: last_idx 20 first_idx 20\ mark_precise: frame0: regs=r2 stack= before 20\ - mark_precise: frame0: parent state regs=r2,r9 stack=:\ + mark_precise: frame0: parent state regs=r2 stack=:\ mark_precise: frame0: last_idx 19 first_idx 17\ - mark_precise: frame0: regs=r2,r9 stack= before 19\ + mark_precise: frame0: regs=r2 stack= before 19\ mark_precise: frame0: regs=r9 stack= before 18\ mark_precise: frame0: regs=r8,r9 stack= before 17\ mark_precise: frame0: parent state regs= stack=:", diff --git a/tools/testing/selftests/bpf/veristat.c b/tools/testing/selftests/bpf/veristat.c index e962f133250c7..1be1e353d40a7 100644 --- a/tools/testing/selftests/bpf/veristat.c +++ b/tools/testing/selftests/bpf/veristat.c @@ -2580,7 +2580,7 @@ static void output_stats(const struct verif_stats *s, enum resfmt fmt, bool last if (last && fmt == RESFMT_TABLE) { output_header_underlines(); printf("Done. Processed %d files, %d programs. Skipped %d files, %d programs.\n", - env.files_processed, env.files_skipped, env.progs_processed, env.progs_skipped); + env.files_processed, env.progs_processed, env.files_skipped, env.progs_skipped); } } diff --git a/tools/testing/selftests/bpf/xskxceiver.c b/tools/testing/selftests/bpf/xskxceiver.c index 352adc8df2d1c..8e108e3162695 100644 --- a/tools/testing/selftests/bpf/xskxceiver.c +++ b/tools/testing/selftests/bpf/xskxceiver.c @@ -74,31 +74,23 @@ #define _GNU_SOURCE #include #include -#include #include #include #include #include #include -#include #include #include #include #include -#include -#include -#include #include #include #include -#include #include #include -#include -#include #include -#include +#include "test_xsk.h" #include "xsk_xdp_progs.skel.h" #include "xsk.h" #include "xskxceiver.h" @@ -109,181 +101,12 @@ #include -#define MAX_TX_BUDGET_DEFAULT 32 - -static bool opt_verbose; static bool opt_print_tests; static enum test_mode opt_mode = TEST_MODE_ALL; static u32 opt_run_test = RUN_ALL_TESTS; void test__fail(void) { /* for network_helpers.c */ } -static void __exit_with_error(int error, const char *file, const char *func, int line) -{ - ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error, - strerror(error)); - ksft_exit_xfail(); -} - -#define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__) -#define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : "" -static char *mode_string(struct test_spec *test) -{ - switch (test->mode) { - case TEST_MODE_SKB: - return "SKB"; - case TEST_MODE_DRV: - return "DRV"; - case TEST_MODE_ZC: - return "ZC"; - default: - return "BOGUS"; - } -} - -static void report_failure(struct test_spec *test) -{ - if (test->fail) - return; - - ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test), - test->name); - test->fail = true; -} - -/* The payload is a word consisting of a packet sequence number in the upper - * 16-bits and a intra packet data sequence number in the lower 16 bits. So the 3rd packet's - * 5th word of data will contain the number (2<<16) | 4 as they are numbered from 0. - */ -static void write_payload(void *dest, u32 pkt_nb, u32 start, u32 size) -{ - u32 *ptr = (u32 *)dest, i; - - start /= sizeof(*ptr); - size /= sizeof(*ptr); - for (i = 0; i < size; i++) - ptr[i] = htonl(pkt_nb << 16 | (i + start)); -} - -static void gen_eth_hdr(struct xsk_socket_info *xsk, struct ethhdr *eth_hdr) -{ - memcpy(eth_hdr->h_dest, xsk->dst_mac, ETH_ALEN); - memcpy(eth_hdr->h_source, xsk->src_mac, ETH_ALEN); - eth_hdr->h_proto = htons(ETH_P_LOOPBACK); -} - -static bool is_umem_valid(struct ifobject *ifobj) -{ - return !!ifobj->umem->umem; -} - -static u32 mode_to_xdp_flags(enum test_mode mode) -{ - return (mode == TEST_MODE_SKB) ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE; -} - -static u64 umem_size(struct xsk_umem_info *umem) -{ - return umem->num_frames * umem->frame_size; -} - -static int xsk_configure_umem(struct ifobject *ifobj, struct xsk_umem_info *umem, void *buffer, - u64 size) -{ - struct xsk_umem_config cfg = { - .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS, - .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS, - .frame_size = umem->frame_size, - .frame_headroom = umem->frame_headroom, - .flags = XSK_UMEM__DEFAULT_FLAGS - }; - int ret; - - if (umem->fill_size) - cfg.fill_size = umem->fill_size; - - if (umem->comp_size) - cfg.comp_size = umem->comp_size; - - if (umem->unaligned_mode) - cfg.flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG; - - ret = xsk_umem__create(&umem->umem, buffer, size, - &umem->fq, &umem->cq, &cfg); - if (ret) - return ret; - - umem->buffer = buffer; - if (ifobj->shared_umem && ifobj->rx_on) { - umem->base_addr = umem_size(umem); - umem->next_buffer = umem_size(umem); - } - - return 0; -} - -static u64 umem_alloc_buffer(struct xsk_umem_info *umem) -{ - u64 addr; - - addr = umem->next_buffer; - umem->next_buffer += umem->frame_size; - if (umem->next_buffer >= umem->base_addr + umem_size(umem)) - umem->next_buffer = umem->base_addr; - - return addr; -} - -static void umem_reset_alloc(struct xsk_umem_info *umem) -{ - umem->next_buffer = 0; -} - -static void enable_busy_poll(struct xsk_socket_info *xsk) -{ - int sock_opt; - - sock_opt = 1; - if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_PREFER_BUSY_POLL, - (void *)&sock_opt, sizeof(sock_opt)) < 0) - exit_with_error(errno); - - sock_opt = 20; - if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL, - (void *)&sock_opt, sizeof(sock_opt)) < 0) - exit_with_error(errno); - - sock_opt = xsk->batch_size; - if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL_BUDGET, - (void *)&sock_opt, sizeof(sock_opt)) < 0) - exit_with_error(errno); -} - -static int __xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, - struct ifobject *ifobject, bool shared) -{ - struct xsk_socket_config cfg = {}; - struct xsk_ring_cons *rxr; - struct xsk_ring_prod *txr; - - xsk->umem = umem; - cfg.rx_size = xsk->rxqsize; - cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; - cfg.bind_flags = ifobject->bind_flags; - if (shared) - cfg.bind_flags |= XDP_SHARED_UMEM; - if (ifobject->mtu > MAX_ETH_PKT_SIZE) - cfg.bind_flags |= XDP_USE_SG; - if (umem->comp_size) - cfg.tx_size = umem->comp_size; - if (umem->fill_size) - cfg.rx_size = umem->fill_size; - - txr = ifobject->tx_on ? &xsk->tx : NULL; - rxr = ifobject->rx_on ? &xsk->rx : NULL; - return xsk_socket__create(&xsk->xsk, ifobject->ifindex, 0, umem->umem, rxr, txr, &cfg); -} - static bool ifobj_zc_avail(struct ifobject *ifobject) { size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE; @@ -314,7 +137,7 @@ static bool ifobj_zc_avail(struct ifobject *ifobject) ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY; ifobject->rx_on = true; xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; - ret = __xsk_configure_socket(xsk, umem, ifobject, false); + ret = xsk_configure_socket(xsk, umem, ifobject, false); if (!ret) zc_avail = true; @@ -327,25 +150,6 @@ static bool ifobj_zc_avail(struct ifobject *ifobject) return zc_avail; } -#define MAX_SKB_FRAGS_PATH "/proc/sys/net/core/max_skb_frags" -static unsigned int get_max_skb_frags(void) -{ - unsigned int max_skb_frags = 0; - FILE *file; - - file = fopen(MAX_SKB_FRAGS_PATH, "r"); - if (!file) { - ksft_print_msg("Error opening %s\n", MAX_SKB_FRAGS_PATH); - return 0; - } - - if (fscanf(file, "%u", &max_skb_frags) != 1) - ksft_print_msg("Error reading %s\n", MAX_SKB_FRAGS_PATH); - - fclose(file); - return max_skb_frags; -} - static struct option long_options[] = { {"interface", required_argument, 0, 'i'}, {"busy-poll", no_argument, 0, 'b'}, @@ -446,2327 +250,66 @@ static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj } } -static int set_ring_size(struct ifobject *ifobj) -{ - int ret; - u32 ctr = 0; - - while (ctr++ < SOCK_RECONF_CTR) { - ret = set_hw_ring_size(ifobj->ifname, &ifobj->ring); - if (!ret) - break; - - /* Retry if it fails */ - if (ctr >= SOCK_RECONF_CTR || errno != EBUSY) - return -errno; - - usleep(USLEEP_MAX); - } - - return ret; -} - -static int hw_ring_size_reset(struct ifobject *ifobj) -{ - ifobj->ring.tx_pending = ifobj->set_ring.default_tx; - ifobj->ring.rx_pending = ifobj->set_ring.default_rx; - return set_ring_size(ifobj); -} - -static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, - struct ifobject *ifobj_rx) +static void xsk_unload_xdp_programs(struct ifobject *ifobj) { - u32 i, j; - - for (i = 0; i < MAX_INTERFACES; i++) { - struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; - - ifobj->xsk = &ifobj->xsk_arr[0]; - ifobj->use_poll = false; - ifobj->use_fill_ring = true; - ifobj->release_rx = true; - ifobj->validation_func = NULL; - ifobj->use_metadata = false; - - if (i == 0) { - ifobj->rx_on = false; - ifobj->tx_on = true; - } else { - ifobj->rx_on = true; - ifobj->tx_on = false; - } - - memset(ifobj->umem, 0, sizeof(*ifobj->umem)); - ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS; - ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; - - for (j = 0; j < MAX_SOCKETS; j++) { - memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j])); - ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; - ifobj->xsk_arr[j].batch_size = DEFAULT_BATCH_SIZE; - if (i == 0) - ifobj->xsk_arr[j].pkt_stream = test->tx_pkt_stream_default; - else - ifobj->xsk_arr[j].pkt_stream = test->rx_pkt_stream_default; - - memcpy(ifobj->xsk_arr[j].src_mac, g_mac, ETH_ALEN); - memcpy(ifobj->xsk_arr[j].dst_mac, g_mac, ETH_ALEN); - ifobj->xsk_arr[j].src_mac[5] += ((j * 2) + 0); - ifobj->xsk_arr[j].dst_mac[5] += ((j * 2) + 1); - } - } - - if (ifobj_tx->hw_ring_size_supp) - hw_ring_size_reset(ifobj_tx); - - test->ifobj_tx = ifobj_tx; - test->ifobj_rx = ifobj_rx; - test->current_step = 0; - test->total_steps = 1; - test->nb_sockets = 1; - test->fail = false; - test->set_ring = false; - test->adjust_tail = false; - test->adjust_tail_support = false; - test->mtu = MAX_ETH_PKT_SIZE; - test->xdp_prog_rx = ifobj_rx->xdp_progs->progs.xsk_def_prog; - test->xskmap_rx = ifobj_rx->xdp_progs->maps.xsk; - test->xdp_prog_tx = ifobj_tx->xdp_progs->progs.xsk_def_prog; - test->xskmap_tx = ifobj_tx->xdp_progs->maps.xsk; + xsk_xdp_progs__destroy(ifobj->xdp_progs); } -static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, - struct ifobject *ifobj_rx, enum test_mode mode, - const struct test_spec *test_to_run) +static void run_pkt_test(struct test_spec *test) { - struct pkt_stream *tx_pkt_stream; - struct pkt_stream *rx_pkt_stream; - u32 i; - - tx_pkt_stream = test->tx_pkt_stream_default; - rx_pkt_stream = test->rx_pkt_stream_default; - memset(test, 0, sizeof(*test)); - test->tx_pkt_stream_default = tx_pkt_stream; - test->rx_pkt_stream_default = rx_pkt_stream; + int ret; - for (i = 0; i < MAX_INTERFACES; i++) { - struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; + ret = test->test_func(test); - ifobj->bind_flags = XDP_USE_NEED_WAKEUP; - if (mode == TEST_MODE_ZC) - ifobj->bind_flags |= XDP_ZEROCOPY; - else - ifobj->bind_flags |= XDP_COPY; + switch (ret) { + case TEST_PASS: + ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test), + test->name); + break; + case TEST_SKIP: + ksft_test_result_skip("SKIP: %s %s%s\n", mode_string(test), busy_poll_string(test), + test->name); + break; + case TEST_FAILURE: + ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test), + test->name); + break; + default: + ksft_test_result_fail("FAIL: %s %s%s -- Unexpected returned value (%d)\n", + mode_string(test), busy_poll_string(test), test->name, ret); } - strncpy(test->name, test_to_run->name, MAX_TEST_NAME_SIZE); - test->test_func = test_to_run->test_func; - test->mode = mode; - __test_spec_init(test, ifobj_tx, ifobj_rx); -} - -static void test_spec_reset(struct test_spec *test) -{ - __test_spec_init(test, test->ifobj_tx, test->ifobj_rx); + pkt_stream_restore_default(test); } -static void test_spec_set_xdp_prog(struct test_spec *test, struct bpf_program *xdp_prog_rx, - struct bpf_program *xdp_prog_tx, struct bpf_map *xskmap_rx, - struct bpf_map *xskmap_tx) +static bool is_xdp_supported(int ifindex) { - test->xdp_prog_rx = xdp_prog_rx; - test->xdp_prog_tx = xdp_prog_tx; - test->xskmap_rx = xskmap_rx; - test->xskmap_tx = xskmap_tx; -} + int flags = XDP_FLAGS_DRV_MODE; -static int test_spec_set_mtu(struct test_spec *test, int mtu) -{ + LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags); + struct bpf_insn insns[2] = { + BPF_MOV64_IMM(BPF_REG_0, XDP_PASS), + BPF_EXIT_INSN() + }; + int prog_fd, insn_cnt = ARRAY_SIZE(insns); int err; - if (test->ifobj_rx->mtu != mtu) { - err = xsk_set_mtu(test->ifobj_rx->ifindex, mtu); - if (err) - return err; - test->ifobj_rx->mtu = mtu; - } - if (test->ifobj_tx->mtu != mtu) { - err = xsk_set_mtu(test->ifobj_tx->ifindex, mtu); - if (err) - return err; - test->ifobj_tx->mtu = mtu; - } - - return 0; -} - -static void pkt_stream_reset(struct pkt_stream *pkt_stream) -{ - if (pkt_stream) { - pkt_stream->current_pkt_nb = 0; - pkt_stream->nb_rx_pkts = 0; - } -} - -static struct pkt *pkt_stream_get_next_tx_pkt(struct pkt_stream *pkt_stream) -{ - if (pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts) - return NULL; - - return &pkt_stream->pkts[pkt_stream->current_pkt_nb++]; -} - -static struct pkt *pkt_stream_get_next_rx_pkt(struct pkt_stream *pkt_stream, u32 *pkts_sent) -{ - while (pkt_stream->current_pkt_nb < pkt_stream->nb_pkts) { - (*pkts_sent)++; - if (pkt_stream->pkts[pkt_stream->current_pkt_nb].valid) - return &pkt_stream->pkts[pkt_stream->current_pkt_nb++]; - pkt_stream->current_pkt_nb++; - } - return NULL; -} - -static void pkt_stream_delete(struct pkt_stream *pkt_stream) -{ - free(pkt_stream->pkts); - free(pkt_stream); -} - -static void pkt_stream_restore_default(struct test_spec *test) -{ - struct pkt_stream *tx_pkt_stream = test->ifobj_tx->xsk->pkt_stream; - struct pkt_stream *rx_pkt_stream = test->ifobj_rx->xsk->pkt_stream; - - if (tx_pkt_stream != test->tx_pkt_stream_default) { - pkt_stream_delete(test->ifobj_tx->xsk->pkt_stream); - test->ifobj_tx->xsk->pkt_stream = test->tx_pkt_stream_default; - } - - if (rx_pkt_stream != test->rx_pkt_stream_default) { - pkt_stream_delete(test->ifobj_rx->xsk->pkt_stream); - test->ifobj_rx->xsk->pkt_stream = test->rx_pkt_stream_default; - } -} - -static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts) -{ - struct pkt_stream *pkt_stream; - - pkt_stream = calloc(1, sizeof(*pkt_stream)); - if (!pkt_stream) - return NULL; - - pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts)); - if (!pkt_stream->pkts) { - free(pkt_stream); - return NULL; - } - - pkt_stream->nb_pkts = nb_pkts; - return pkt_stream; -} - -static bool pkt_continues(u32 options) -{ - return options & XDP_PKT_CONTD; -} - -static u32 ceil_u32(u32 a, u32 b) -{ - return (a + b - 1) / b; -} - -static u32 pkt_nb_frags(u32 frame_size, struct pkt_stream *pkt_stream, struct pkt *pkt) -{ - u32 nb_frags = 1, next_frag; - - if (!pkt) - return 1; - - if (!pkt_stream->verbatim) { - if (!pkt->valid || !pkt->len) - return 1; - return ceil_u32(pkt->len, frame_size); - } - - /* Search for the end of the packet in verbatim mode */ - if (!pkt_continues(pkt->options)) - return nb_frags; - - next_frag = pkt_stream->current_pkt_nb; - pkt++; - while (next_frag++ < pkt_stream->nb_pkts) { - nb_frags++; - if (!pkt_continues(pkt->options) || !pkt->valid) - break; - pkt++; - } - return nb_frags; -} - -static bool set_pkt_valid(int offset, u32 len) -{ - return len <= MAX_ETH_JUMBO_SIZE; -} - -static void pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len) -{ - pkt->offset = offset; - pkt->len = len; - pkt->valid = set_pkt_valid(offset, len); -} - -static void pkt_stream_pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len) -{ - bool prev_pkt_valid = pkt->valid; - - pkt_set(pkt_stream, pkt, offset, len); - pkt_stream->nb_valid_entries += pkt->valid - prev_pkt_valid; -} - -static u32 pkt_get_buffer_len(struct xsk_umem_info *umem, u32 len) -{ - return ceil_u32(len, umem->frame_size) * umem->frame_size; -} - -static struct pkt_stream *__pkt_stream_generate(u32 nb_pkts, u32 pkt_len, u32 nb_start, u32 nb_off) -{ - struct pkt_stream *pkt_stream; - u32 i; - - pkt_stream = __pkt_stream_alloc(nb_pkts); - if (!pkt_stream) - exit_with_error(ENOMEM); - - pkt_stream->nb_pkts = nb_pkts; - pkt_stream->max_pkt_len = pkt_len; - for (i = 0; i < nb_pkts; i++) { - struct pkt *pkt = &pkt_stream->pkts[i]; - - pkt_stream_pkt_set(pkt_stream, pkt, 0, pkt_len); - pkt->pkt_nb = nb_start + i * nb_off; - } - - return pkt_stream; -} - -static struct pkt_stream *pkt_stream_generate(u32 nb_pkts, u32 pkt_len) -{ - return __pkt_stream_generate(nb_pkts, pkt_len, 0, 1); -} - -static struct pkt_stream *pkt_stream_clone(struct pkt_stream *pkt_stream) -{ - return pkt_stream_generate(pkt_stream->nb_pkts, pkt_stream->pkts[0].len); -} - -static void pkt_stream_replace_ifobject(struct ifobject *ifobj, u32 nb_pkts, u32 pkt_len) -{ - ifobj->xsk->pkt_stream = pkt_stream_generate(nb_pkts, pkt_len); -} - -static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len) -{ - pkt_stream_replace_ifobject(test->ifobj_tx, nb_pkts, pkt_len); - pkt_stream_replace_ifobject(test->ifobj_rx, nb_pkts, pkt_len); -} - -static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len, - int offset) -{ - struct pkt_stream *pkt_stream; - u32 i; - - pkt_stream = pkt_stream_clone(ifobj->xsk->pkt_stream); - for (i = 1; i < ifobj->xsk->pkt_stream->nb_pkts; i += 2) - pkt_stream_pkt_set(pkt_stream, &pkt_stream->pkts[i], offset, pkt_len); - - ifobj->xsk->pkt_stream = pkt_stream; -} - -static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset) -{ - __pkt_stream_replace_half(test->ifobj_tx, pkt_len, offset); - __pkt_stream_replace_half(test->ifobj_rx, pkt_len, offset); -} - -static void pkt_stream_receive_half(struct test_spec *test) -{ - struct pkt_stream *pkt_stream = test->ifobj_tx->xsk->pkt_stream; - u32 i; - - test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(pkt_stream->nb_pkts, - pkt_stream->pkts[0].len); - pkt_stream = test->ifobj_rx->xsk->pkt_stream; - for (i = 1; i < pkt_stream->nb_pkts; i += 2) - pkt_stream->pkts[i].valid = false; - - pkt_stream->nb_valid_entries /= 2; -} - -static void pkt_stream_even_odd_sequence(struct test_spec *test) -{ - struct pkt_stream *pkt_stream; - u32 i; - - for (i = 0; i < test->nb_sockets; i++) { - pkt_stream = test->ifobj_tx->xsk_arr[i].pkt_stream; - pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2, - pkt_stream->pkts[0].len, i, 2); - test->ifobj_tx->xsk_arr[i].pkt_stream = pkt_stream; + prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL); + if (prog_fd < 0) + return false; - pkt_stream = test->ifobj_rx->xsk_arr[i].pkt_stream; - pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2, - pkt_stream->pkts[0].len, i, 2); - test->ifobj_rx->xsk_arr[i].pkt_stream = pkt_stream; + err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL); + if (err) { + close(prog_fd); + return false; } -} - -static u64 pkt_get_addr(struct pkt *pkt, struct xsk_umem_info *umem) -{ - if (!pkt->valid) - return pkt->offset; - return pkt->offset + umem_alloc_buffer(umem); -} -static void pkt_stream_cancel(struct pkt_stream *pkt_stream) -{ - pkt_stream->current_pkt_nb--; -} - -static void pkt_generate(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, u64 addr, u32 len, - u32 pkt_nb, u32 bytes_written) -{ - void *data = xsk_umem__get_data(umem->buffer, addr); - - if (len < MIN_PKT_SIZE) - return; - - if (!bytes_written) { - gen_eth_hdr(xsk, data); - - len -= PKT_HDR_SIZE; - data += PKT_HDR_SIZE; - } else { - bytes_written -= PKT_HDR_SIZE; - } + bpf_xdp_detach(ifindex, flags, NULL); + close(prog_fd); - write_payload(data, pkt_nb, bytes_written, len); + return true; } -static struct pkt_stream *__pkt_stream_generate_custom(struct ifobject *ifobj, struct pkt *frames, - u32 nb_frames, bool verbatim) -{ - u32 i, len = 0, pkt_nb = 0, payload = 0; - struct pkt_stream *pkt_stream; - - pkt_stream = __pkt_stream_alloc(nb_frames); - if (!pkt_stream) - exit_with_error(ENOMEM); - - for (i = 0; i < nb_frames; i++) { - struct pkt *pkt = &pkt_stream->pkts[pkt_nb]; - struct pkt *frame = &frames[i]; - - pkt->offset = frame->offset; - if (verbatim) { - *pkt = *frame; - pkt->pkt_nb = payload; - if (!frame->valid || !pkt_continues(frame->options)) - payload++; - } else { - if (frame->valid) - len += frame->len; - if (frame->valid && pkt_continues(frame->options)) - continue; - - pkt->pkt_nb = pkt_nb; - pkt->len = len; - pkt->valid = frame->valid; - pkt->options = 0; - - len = 0; - } - - print_verbose("offset: %d len: %u valid: %u options: %u pkt_nb: %u\n", - pkt->offset, pkt->len, pkt->valid, pkt->options, pkt->pkt_nb); - - if (pkt->valid && pkt->len > pkt_stream->max_pkt_len) - pkt_stream->max_pkt_len = pkt->len; - - if (pkt->valid) - pkt_stream->nb_valid_entries++; - - pkt_nb++; - } - - pkt_stream->nb_pkts = pkt_nb; - pkt_stream->verbatim = verbatim; - return pkt_stream; -} - -static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts) -{ - struct pkt_stream *pkt_stream; - - pkt_stream = __pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts, true); - test->ifobj_tx->xsk->pkt_stream = pkt_stream; - - pkt_stream = __pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts, false); - test->ifobj_rx->xsk->pkt_stream = pkt_stream; -} - -static void pkt_print_data(u32 *data, u32 cnt) -{ - u32 i; - - for (i = 0; i < cnt; i++) { - u32 seqnum, pkt_nb; - - seqnum = ntohl(*data) & 0xffff; - pkt_nb = ntohl(*data) >> 16; - ksft_print_msg("%u:%u ", pkt_nb, seqnum); - data++; - } -} - -static void pkt_dump(void *pkt, u32 len, bool eth_header) -{ - struct ethhdr *ethhdr = pkt; - u32 i, *data; - - if (eth_header) { - /*extract L2 frame */ - ksft_print_msg("DEBUG>> L2: dst mac: "); - for (i = 0; i < ETH_ALEN; i++) - ksft_print_msg("%02X", ethhdr->h_dest[i]); - - ksft_print_msg("\nDEBUG>> L2: src mac: "); - for (i = 0; i < ETH_ALEN; i++) - ksft_print_msg("%02X", ethhdr->h_source[i]); - - data = pkt + PKT_HDR_SIZE; - } else { - data = pkt; - } - - /*extract L5 frame */ - ksft_print_msg("\nDEBUG>> L5: seqnum: "); - pkt_print_data(data, PKT_DUMP_NB_TO_PRINT); - ksft_print_msg("...."); - if (len > PKT_DUMP_NB_TO_PRINT * sizeof(u32)) { - ksft_print_msg("\n.... "); - pkt_print_data(data + len / sizeof(u32) - PKT_DUMP_NB_TO_PRINT, - PKT_DUMP_NB_TO_PRINT); - } - ksft_print_msg("\n---------------------------------------\n"); -} - -static bool is_offset_correct(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr) -{ - u32 headroom = umem->unaligned_mode ? 0 : umem->frame_headroom; - u32 offset = addr % umem->frame_size, expected_offset; - int pkt_offset = pkt->valid ? pkt->offset : 0; - - if (!umem->unaligned_mode) - pkt_offset = 0; - - expected_offset = (pkt_offset + headroom + XDP_PACKET_HEADROOM) % umem->frame_size; - - if (offset == expected_offset) - return true; - - ksft_print_msg("[%s] expected [%u], got [%u]\n", __func__, expected_offset, offset); - return false; -} - -static bool is_metadata_correct(struct pkt *pkt, void *buffer, u64 addr) -{ - void *data = xsk_umem__get_data(buffer, addr); - struct xdp_info *meta = data - sizeof(struct xdp_info); - - if (meta->count != pkt->pkt_nb) { - ksft_print_msg("[%s] expected meta_count [%d], got meta_count [%llu]\n", - __func__, pkt->pkt_nb, - (unsigned long long)meta->count); - return false; - } - - return true; -} - -static bool is_adjust_tail_supported(struct xsk_xdp_progs *skel_rx) -{ - struct bpf_map *data_map; - int adjust_value = 0; - int key = 0; - int ret; - - data_map = bpf_object__find_map_by_name(skel_rx->obj, "xsk_xdp_.bss"); - if (!data_map || !bpf_map__is_internal(data_map)) { - ksft_print_msg("Error: could not find bss section of XDP program\n"); - exit_with_error(errno); - } - - ret = bpf_map_lookup_elem(bpf_map__fd(data_map), &key, &adjust_value); - if (ret) { - ksft_print_msg("Error: bpf_map_lookup_elem failed with error %d\n", ret); - exit_with_error(errno); - } - - /* Set the 'adjust_value' variable to -EOPNOTSUPP in the XDP program if the adjust_tail - * helper is not supported. Skip the adjust_tail test case in this scenario. - */ - return adjust_value != -EOPNOTSUPP; -} - -static bool is_frag_valid(struct xsk_umem_info *umem, u64 addr, u32 len, u32 expected_pkt_nb, - u32 bytes_processed) -{ - u32 seqnum, pkt_nb, *pkt_data, words_to_end, expected_seqnum; - void *data = xsk_umem__get_data(umem->buffer, addr); - - addr -= umem->base_addr; - - if (addr >= umem->num_frames * umem->frame_size || - addr + len > umem->num_frames * umem->frame_size) { - ksft_print_msg("Frag invalid addr: %llx len: %u\n", - (unsigned long long)addr, len); - return false; - } - if (!umem->unaligned_mode && addr % umem->frame_size + len > umem->frame_size) { - ksft_print_msg("Frag crosses frame boundary addr: %llx len: %u\n", - (unsigned long long)addr, len); - return false; - } - - pkt_data = data; - if (!bytes_processed) { - pkt_data += PKT_HDR_SIZE / sizeof(*pkt_data); - len -= PKT_HDR_SIZE; - } else { - bytes_processed -= PKT_HDR_SIZE; - } - - expected_seqnum = bytes_processed / sizeof(*pkt_data); - seqnum = ntohl(*pkt_data) & 0xffff; - pkt_nb = ntohl(*pkt_data) >> 16; - - if (expected_pkt_nb != pkt_nb) { - ksft_print_msg("[%s] expected pkt_nb [%u], got pkt_nb [%u]\n", - __func__, expected_pkt_nb, pkt_nb); - goto error; - } - if (expected_seqnum != seqnum) { - ksft_print_msg("[%s] expected seqnum at start [%u], got seqnum [%u]\n", - __func__, expected_seqnum, seqnum); - goto error; - } - - words_to_end = len / sizeof(*pkt_data) - 1; - pkt_data += words_to_end; - seqnum = ntohl(*pkt_data) & 0xffff; - expected_seqnum += words_to_end; - if (expected_seqnum != seqnum) { - ksft_print_msg("[%s] expected seqnum at end [%u], got seqnum [%u]\n", - __func__, expected_seqnum, seqnum); - goto error; - } - - return true; - -error: - pkt_dump(data, len, !bytes_processed); - return false; -} - -static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len) -{ - if (pkt->len != len) { - ksft_print_msg("[%s] expected packet length [%d], got length [%d]\n", - __func__, pkt->len, len); - pkt_dump(xsk_umem__get_data(buffer, addr), len, true); - return false; - } - - return true; -} - -static u32 load_value(u32 *counter) -{ - return __atomic_load_n(counter, __ATOMIC_ACQUIRE); -} - -static bool kick_tx_with_check(struct xsk_socket_info *xsk, int *ret) -{ - u32 max_budget = MAX_TX_BUDGET_DEFAULT; - u32 cons, ready_to_send; - int delta; - - cons = load_value(xsk->tx.consumer); - ready_to_send = load_value(xsk->tx.producer) - cons; - *ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); - - delta = load_value(xsk->tx.consumer) - cons; - /* By default, xsk should consume exact @max_budget descs at one - * send in this case where hitting the max budget limit in while - * loop is triggered in __xsk_generic_xmit(). Please make sure that - * the number of descs to be sent is larger than @max_budget, or - * else the tx.consumer will be updated in xskq_cons_peek_desc() - * in time which hides the issue we try to verify. - */ - if (ready_to_send > max_budget && delta != max_budget) - return false; - - return true; -} - -static int kick_tx(struct xsk_socket_info *xsk) -{ - int ret; - - if (xsk->check_consumer) { - if (!kick_tx_with_check(xsk, &ret)) - return TEST_FAILURE; - } else { - ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); - } - if (ret >= 0) - return TEST_PASS; - if (errno == ENOBUFS || errno == EAGAIN || errno == EBUSY || errno == ENETDOWN) { - usleep(100); - return TEST_PASS; - } - return TEST_FAILURE; -} - -static int kick_rx(struct xsk_socket_info *xsk) -{ - int ret; - - ret = recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL); - if (ret < 0) - return TEST_FAILURE; - - return TEST_PASS; -} - -static int complete_pkts(struct xsk_socket_info *xsk, int batch_size) -{ - unsigned int rcvd; - u32 idx; - int ret; - - if (xsk_ring_prod__needs_wakeup(&xsk->tx)) { - ret = kick_tx(xsk); - if (ret) - return TEST_FAILURE; - } - - rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx); - if (rcvd) { - if (rcvd > xsk->outstanding_tx) { - u64 addr = *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx + rcvd - 1); - - ksft_print_msg("[%s] Too many packets completed\n", __func__); - ksft_print_msg("Last completion address: %llx\n", - (unsigned long long)addr); - return TEST_FAILURE; - } - - xsk_ring_cons__release(&xsk->umem->cq, rcvd); - xsk->outstanding_tx -= rcvd; - } - - return TEST_PASS; -} - -static int __receive_pkts(struct test_spec *test, struct xsk_socket_info *xsk) -{ - u32 frags_processed = 0, nb_frags = 0, pkt_len = 0; - u32 idx_rx = 0, idx_fq = 0, rcvd, pkts_sent = 0; - struct pkt_stream *pkt_stream = xsk->pkt_stream; - struct ifobject *ifobj = test->ifobj_rx; - struct xsk_umem_info *umem = xsk->umem; - struct pollfd fds = { }; - struct pkt *pkt; - u64 first_addr = 0; - int ret; - - fds.fd = xsk_socket__fd(xsk->xsk); - fds.events = POLLIN; - - ret = kick_rx(xsk); - if (ret) - return TEST_FAILURE; - - if (ifobj->use_poll) { - ret = poll(&fds, 1, POLL_TMOUT); - if (ret < 0) - return TEST_FAILURE; - - if (!ret) { - if (!is_umem_valid(test->ifobj_tx)) - return TEST_PASS; - - ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__); - return TEST_CONTINUE; - } - - if (!(fds.revents & POLLIN)) - return TEST_CONTINUE; - } - - rcvd = xsk_ring_cons__peek(&xsk->rx, xsk->batch_size, &idx_rx); - if (!rcvd) - return TEST_CONTINUE; - - if (ifobj->use_fill_ring) { - ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); - while (ret != rcvd) { - if (xsk_ring_prod__needs_wakeup(&umem->fq)) { - ret = poll(&fds, 1, POLL_TMOUT); - if (ret < 0) - return TEST_FAILURE; - } - ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); - } - } - - while (frags_processed < rcvd) { - const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++); - u64 addr = desc->addr, orig; - - orig = xsk_umem__extract_addr(addr); - addr = xsk_umem__add_offset_to_addr(addr); - - if (!nb_frags) { - pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent); - if (!pkt) { - ksft_print_msg("[%s] received too many packets addr: %lx len %u\n", - __func__, addr, desc->len); - return TEST_FAILURE; - } - } - - print_verbose("Rx: addr: %lx len: %u options: %u pkt_nb: %u valid: %u\n", - addr, desc->len, desc->options, pkt->pkt_nb, pkt->valid); - - if (!is_frag_valid(umem, addr, desc->len, pkt->pkt_nb, pkt_len) || - !is_offset_correct(umem, pkt, addr) || (ifobj->use_metadata && - !is_metadata_correct(pkt, umem->buffer, addr))) - return TEST_FAILURE; - - if (!nb_frags++) - first_addr = addr; - frags_processed++; - pkt_len += desc->len; - if (ifobj->use_fill_ring) - *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig; - - if (pkt_continues(desc->options)) - continue; - - /* The complete packet has been received */ - if (!is_pkt_valid(pkt, umem->buffer, first_addr, pkt_len) || - !is_offset_correct(umem, pkt, addr)) - return TEST_FAILURE; - - pkt_stream->nb_rx_pkts++; - nb_frags = 0; - pkt_len = 0; - } - - if (nb_frags) { - /* In the middle of a packet. Start over from beginning of packet. */ - idx_rx -= nb_frags; - xsk_ring_cons__cancel(&xsk->rx, nb_frags); - if (ifobj->use_fill_ring) { - idx_fq -= nb_frags; - xsk_ring_prod__cancel(&umem->fq, nb_frags); - } - frags_processed -= nb_frags; - } - - if (ifobj->use_fill_ring) - xsk_ring_prod__submit(&umem->fq, frags_processed); - if (ifobj->release_rx) - xsk_ring_cons__release(&xsk->rx, frags_processed); - - pthread_mutex_lock(&pacing_mutex); - pkts_in_flight -= pkts_sent; - pthread_mutex_unlock(&pacing_mutex); - pkts_sent = 0; - -return TEST_CONTINUE; -} - -bool all_packets_received(struct test_spec *test, struct xsk_socket_info *xsk, u32 sock_num, - unsigned long *bitmap) -{ - struct pkt_stream *pkt_stream = xsk->pkt_stream; - - if (!pkt_stream) { - __set_bit(sock_num, bitmap); - return false; - } - - if (pkt_stream->nb_rx_pkts == pkt_stream->nb_valid_entries) { - __set_bit(sock_num, bitmap); - if (bitmap_full(bitmap, test->nb_sockets)) - return true; - } - - return false; -} - -static int receive_pkts(struct test_spec *test) -{ - struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0}; - DECLARE_BITMAP(bitmap, test->nb_sockets); - struct xsk_socket_info *xsk; - u32 sock_num = 0; - int res, ret; - - ret = gettimeofday(&tv_now, NULL); - if (ret) - exit_with_error(errno); - - timeradd(&tv_now, &tv_timeout, &tv_end); - - while (1) { - xsk = &test->ifobj_rx->xsk_arr[sock_num]; - - if ((all_packets_received(test, xsk, sock_num, bitmap))) - break; - - res = __receive_pkts(test, xsk); - if (!(res == TEST_PASS || res == TEST_CONTINUE)) - return res; - - ret = gettimeofday(&tv_now, NULL); - if (ret) - exit_with_error(errno); - - if (timercmp(&tv_now, &tv_end, >)) { - ksft_print_msg("ERROR: [%s] Receive loop timed out\n", __func__); - return TEST_FAILURE; - } - sock_num = (sock_num + 1) % test->nb_sockets; - } - - return TEST_PASS; -} - -static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, bool timeout) -{ - u32 i, idx = 0, valid_pkts = 0, valid_frags = 0, buffer_len; - struct pkt_stream *pkt_stream = xsk->pkt_stream; - struct xsk_umem_info *umem = ifobject->umem; - bool use_poll = ifobject->use_poll; - struct pollfd fds = { }; - int ret; - - buffer_len = pkt_get_buffer_len(umem, pkt_stream->max_pkt_len); - /* pkts_in_flight might be negative if many invalid packets are sent */ - if (pkts_in_flight >= (int)((umem_size(umem) - xsk->batch_size * buffer_len) / - buffer_len)) { - ret = kick_tx(xsk); - if (ret) - return TEST_FAILURE; - return TEST_CONTINUE; - } - - fds.fd = xsk_socket__fd(xsk->xsk); - fds.events = POLLOUT; - - while (xsk_ring_prod__reserve(&xsk->tx, xsk->batch_size, &idx) < xsk->batch_size) { - if (use_poll) { - ret = poll(&fds, 1, POLL_TMOUT); - if (timeout) { - if (ret < 0) { - ksft_print_msg("ERROR: [%s] Poll error %d\n", - __func__, errno); - return TEST_FAILURE; - } - if (ret == 0) - return TEST_PASS; - break; - } - if (ret <= 0) { - ksft_print_msg("ERROR: [%s] Poll error %d\n", - __func__, errno); - return TEST_FAILURE; - } - } - - complete_pkts(xsk, xsk->batch_size); - } - - for (i = 0; i < xsk->batch_size; i++) { - struct pkt *pkt = pkt_stream_get_next_tx_pkt(pkt_stream); - u32 nb_frags_left, nb_frags, bytes_written = 0; - - if (!pkt) - break; - - nb_frags = pkt_nb_frags(umem->frame_size, pkt_stream, pkt); - if (nb_frags > xsk->batch_size - i) { - pkt_stream_cancel(pkt_stream); - xsk_ring_prod__cancel(&xsk->tx, xsk->batch_size - i); - break; - } - nb_frags_left = nb_frags; - - while (nb_frags_left--) { - struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i); - - tx_desc->addr = pkt_get_addr(pkt, ifobject->umem); - if (pkt_stream->verbatim) { - tx_desc->len = pkt->len; - tx_desc->options = pkt->options; - } else if (nb_frags_left) { - tx_desc->len = umem->frame_size; - tx_desc->options = XDP_PKT_CONTD; - } else { - tx_desc->len = pkt->len - bytes_written; - tx_desc->options = 0; - } - if (pkt->valid) - pkt_generate(xsk, umem, tx_desc->addr, tx_desc->len, pkt->pkt_nb, - bytes_written); - bytes_written += tx_desc->len; - - print_verbose("Tx addr: %llx len: %u options: %u pkt_nb: %u\n", - tx_desc->addr, tx_desc->len, tx_desc->options, pkt->pkt_nb); - - if (nb_frags_left) { - i++; - if (pkt_stream->verbatim) - pkt = pkt_stream_get_next_tx_pkt(pkt_stream); - } - } - - if (pkt && pkt->valid) { - valid_pkts++; - valid_frags += nb_frags; - } - } - - pthread_mutex_lock(&pacing_mutex); - pkts_in_flight += valid_pkts; - pthread_mutex_unlock(&pacing_mutex); - - xsk_ring_prod__submit(&xsk->tx, i); - xsk->outstanding_tx += valid_frags; - - if (use_poll) { - ret = poll(&fds, 1, POLL_TMOUT); - if (ret <= 0) { - if (ret == 0 && timeout) - return TEST_PASS; - - ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret); - return TEST_FAILURE; - } - } - - if (!timeout) { - if (complete_pkts(xsk, i)) - return TEST_FAILURE; - - usleep(10); - return TEST_PASS; - } - - return TEST_CONTINUE; -} - -static int wait_for_tx_completion(struct xsk_socket_info *xsk) -{ - struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0}; - int ret; - - ret = gettimeofday(&tv_now, NULL); - if (ret) - exit_with_error(errno); - timeradd(&tv_now, &tv_timeout, &tv_end); - - while (xsk->outstanding_tx) { - ret = gettimeofday(&tv_now, NULL); - if (ret) - exit_with_error(errno); - if (timercmp(&tv_now, &tv_end, >)) { - ksft_print_msg("ERROR: [%s] Transmission loop timed out\n", __func__); - return TEST_FAILURE; - } - - complete_pkts(xsk, xsk->batch_size); - } - - return TEST_PASS; -} - -bool all_packets_sent(struct test_spec *test, unsigned long *bitmap) -{ - return bitmap_full(bitmap, test->nb_sockets); -} - -static int send_pkts(struct test_spec *test, struct ifobject *ifobject) -{ - bool timeout = !is_umem_valid(test->ifobj_rx); - DECLARE_BITMAP(bitmap, test->nb_sockets); - u32 i, ret; - - while (!(all_packets_sent(test, bitmap))) { - for (i = 0; i < test->nb_sockets; i++) { - struct pkt_stream *pkt_stream; - - pkt_stream = ifobject->xsk_arr[i].pkt_stream; - if (!pkt_stream || pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts) { - __set_bit(i, bitmap); - continue; - } - ret = __send_pkts(ifobject, &ifobject->xsk_arr[i], timeout); - if (ret == TEST_CONTINUE && !test->fail) - continue; - - if ((ret || test->fail) && !timeout) - return TEST_FAILURE; - - if (ret == TEST_PASS && timeout) - return ret; - - ret = wait_for_tx_completion(&ifobject->xsk_arr[i]); - if (ret) - return TEST_FAILURE; - } - } - - return TEST_PASS; -} - -static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats) -{ - int fd = xsk_socket__fd(xsk), err; - socklen_t optlen, expected_len; - - optlen = sizeof(*stats); - err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, stats, &optlen); - if (err) { - ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n", - __func__, -err, strerror(-err)); - return TEST_FAILURE; - } - - expected_len = sizeof(struct xdp_statistics); - if (optlen != expected_len) { - ksft_print_msg("[%s] getsockopt optlen error. Expected: %u got: %u\n", - __func__, expected_len, optlen); - return TEST_FAILURE; - } - - return TEST_PASS; -} - -static int validate_rx_dropped(struct ifobject *ifobject) -{ - struct xsk_socket *xsk = ifobject->xsk->xsk; - struct xdp_statistics stats; - int err; - - err = kick_rx(ifobject->xsk); - if (err) - return TEST_FAILURE; - - err = get_xsk_stats(xsk, &stats); - if (err) - return TEST_FAILURE; - - /* The receiver calls getsockopt after receiving the last (valid) - * packet which is not the final packet sent in this test (valid and - * invalid packets are sent in alternating fashion with the final - * packet being invalid). Since the last packet may or may not have - * been dropped already, both outcomes must be allowed. - */ - if (stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 || - stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 - 1) - return TEST_PASS; - - return TEST_FAILURE; -} - -static int validate_rx_full(struct ifobject *ifobject) -{ - struct xsk_socket *xsk = ifobject->xsk->xsk; - struct xdp_statistics stats; - int err; - - usleep(1000); - err = kick_rx(ifobject->xsk); - if (err) - return TEST_FAILURE; - - err = get_xsk_stats(xsk, &stats); - if (err) - return TEST_FAILURE; - - if (stats.rx_ring_full) - return TEST_PASS; - - return TEST_FAILURE; -} - -static int validate_fill_empty(struct ifobject *ifobject) -{ - struct xsk_socket *xsk = ifobject->xsk->xsk; - struct xdp_statistics stats; - int err; - - usleep(1000); - err = kick_rx(ifobject->xsk); - if (err) - return TEST_FAILURE; - - err = get_xsk_stats(xsk, &stats); - if (err) - return TEST_FAILURE; - - if (stats.rx_fill_ring_empty_descs) - return TEST_PASS; - - return TEST_FAILURE; -} - -static int validate_tx_invalid_descs(struct ifobject *ifobject) -{ - struct xsk_socket *xsk = ifobject->xsk->xsk; - int fd = xsk_socket__fd(xsk); - struct xdp_statistics stats; - socklen_t optlen; - int err; - - optlen = sizeof(stats); - err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen); - if (err) { - ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n", - __func__, -err, strerror(-err)); - return TEST_FAILURE; - } - - if (stats.tx_invalid_descs != ifobject->xsk->pkt_stream->nb_pkts / 2) { - ksft_print_msg("[%s] tx_invalid_descs incorrect. Got [%llu] expected [%u]\n", - __func__, - (unsigned long long)stats.tx_invalid_descs, - ifobject->xsk->pkt_stream->nb_pkts); - return TEST_FAILURE; - } - - return TEST_PASS; -} - -static void xsk_configure_socket(struct test_spec *test, struct ifobject *ifobject, - struct xsk_umem_info *umem, bool tx) -{ - int i, ret; - - for (i = 0; i < test->nb_sockets; i++) { - bool shared = (ifobject->shared_umem && tx) ? true : !!i; - u32 ctr = 0; - - while (ctr++ < SOCK_RECONF_CTR) { - ret = __xsk_configure_socket(&ifobject->xsk_arr[i], umem, - ifobject, shared); - if (!ret) - break; - - /* Retry if it fails as xsk_socket__create() is asynchronous */ - if (ctr >= SOCK_RECONF_CTR) - exit_with_error(-ret); - usleep(USLEEP_MAX); - } - if (ifobject->busy_poll) - enable_busy_poll(&ifobject->xsk_arr[i]); - } -} - -static void thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject) -{ - xsk_configure_socket(test, ifobject, test->ifobj_rx->umem, true); - ifobject->xsk = &ifobject->xsk_arr[0]; - ifobject->xskmap = test->ifobj_rx->xskmap; - memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info)); - ifobject->umem->base_addr = 0; -} - -static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream, - bool fill_up) -{ - u32 rx_frame_size = umem->frame_size - XDP_PACKET_HEADROOM; - u32 idx = 0, filled = 0, buffers_to_fill, nb_pkts; - int ret; - - if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS) - buffers_to_fill = umem->num_frames; - else - buffers_to_fill = umem->fill_size; - - ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx); - if (ret != buffers_to_fill) - exit_with_error(ENOSPC); - - while (filled < buffers_to_fill) { - struct pkt *pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &nb_pkts); - u64 addr; - u32 i; - - for (i = 0; i < pkt_nb_frags(rx_frame_size, pkt_stream, pkt); i++) { - if (!pkt) { - if (!fill_up) - break; - addr = filled * umem->frame_size + umem->base_addr; - } else if (pkt->offset >= 0) { - addr = pkt->offset % umem->frame_size + umem_alloc_buffer(umem); - } else { - addr = pkt->offset + umem_alloc_buffer(umem); - } - - *xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr; - if (++filled >= buffers_to_fill) - break; - } - } - xsk_ring_prod__submit(&umem->fq, filled); - xsk_ring_prod__cancel(&umem->fq, buffers_to_fill - filled); - - pkt_stream_reset(pkt_stream); - umem_reset_alloc(umem); -} - -static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject) -{ - u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size; - int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; - LIBBPF_OPTS(bpf_xdp_query_opts, opts); - void *bufs; - int ret; - u32 i; - - if (ifobject->umem->unaligned_mode) - mmap_flags |= MAP_HUGETLB | MAP_HUGE_2MB; - - if (ifobject->shared_umem) - umem_sz *= 2; - - bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); - if (bufs == MAP_FAILED) - exit_with_error(errno); - - ret = xsk_configure_umem(ifobject, ifobject->umem, bufs, umem_sz); - if (ret) - exit_with_error(-ret); - - xsk_configure_socket(test, ifobject, ifobject->umem, false); - - ifobject->xsk = &ifobject->xsk_arr[0]; - - if (!ifobject->rx_on) - return; - - xsk_populate_fill_ring(ifobject->umem, ifobject->xsk->pkt_stream, ifobject->use_fill_ring); - - for (i = 0; i < test->nb_sockets; i++) { - ifobject->xsk = &ifobject->xsk_arr[i]; - ret = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, i); - if (ret) - exit_with_error(errno); - } -} - -static void *worker_testapp_validate_tx(void *arg) -{ - struct test_spec *test = (struct test_spec *)arg; - struct ifobject *ifobject = test->ifobj_tx; - int err; - - if (test->current_step == 1) { - if (!ifobject->shared_umem) - thread_common_ops(test, ifobject); - else - thread_common_ops_tx(test, ifobject); - } - - err = send_pkts(test, ifobject); - - if (!err && ifobject->validation_func) - err = ifobject->validation_func(ifobject); - if (err) - report_failure(test); - - pthread_exit(NULL); -} - -static void *worker_testapp_validate_rx(void *arg) -{ - struct test_spec *test = (struct test_spec *)arg; - struct ifobject *ifobject = test->ifobj_rx; - int err; - - if (test->current_step == 1) { - thread_common_ops(test, ifobject); - } else { - xsk_clear_xskmap(ifobject->xskmap); - err = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, 0); - if (err) { - ksft_print_msg("Error: Failed to update xskmap, error %s\n", - strerror(-err)); - exit_with_error(-err); - } - } - - pthread_barrier_wait(&barr); - - err = receive_pkts(test); - - if (!err && ifobject->validation_func) - err = ifobject->validation_func(ifobject); - - if (err) { - if (test->adjust_tail && !is_adjust_tail_supported(ifobject->xdp_progs)) - test->adjust_tail_support = false; - else - report_failure(test); - } - - pthread_exit(NULL); -} - -static u64 ceil_u64(u64 a, u64 b) -{ - return (a + b - 1) / b; -} - -static void testapp_clean_xsk_umem(struct ifobject *ifobj) -{ - u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size; - - if (ifobj->shared_umem) - umem_sz *= 2; - - umem_sz = ceil_u64(umem_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE; - xsk_umem__delete(ifobj->umem->umem); - munmap(ifobj->umem->buffer, umem_sz); -} - -static void handler(int signum) -{ - pthread_exit(NULL); -} - -static bool xdp_prog_changed_rx(struct test_spec *test) -{ - struct ifobject *ifobj = test->ifobj_rx; - - return ifobj->xdp_prog != test->xdp_prog_rx || ifobj->mode != test->mode; -} - -static bool xdp_prog_changed_tx(struct test_spec *test) -{ - struct ifobject *ifobj = test->ifobj_tx; - - return ifobj->xdp_prog != test->xdp_prog_tx || ifobj->mode != test->mode; -} - -static void xsk_reattach_xdp(struct ifobject *ifobj, struct bpf_program *xdp_prog, - struct bpf_map *xskmap, enum test_mode mode) -{ - int err; - - xsk_detach_xdp_program(ifobj->ifindex, mode_to_xdp_flags(ifobj->mode)); - err = xsk_attach_xdp_program(xdp_prog, ifobj->ifindex, mode_to_xdp_flags(mode)); - if (err) { - ksft_print_msg("Error attaching XDP program\n"); - exit_with_error(-err); - } - - if (ifobj->mode != mode && (mode == TEST_MODE_DRV || mode == TEST_MODE_ZC)) - if (!xsk_is_in_mode(ifobj->ifindex, XDP_FLAGS_DRV_MODE)) { - ksft_print_msg("ERROR: XDP prog not in DRV mode\n"); - exit_with_error(EINVAL); - } - - ifobj->xdp_prog = xdp_prog; - ifobj->xskmap = xskmap; - ifobj->mode = mode; -} - -static void xsk_attach_xdp_progs(struct test_spec *test, struct ifobject *ifobj_rx, - struct ifobject *ifobj_tx) -{ - if (xdp_prog_changed_rx(test)) - xsk_reattach_xdp(ifobj_rx, test->xdp_prog_rx, test->xskmap_rx, test->mode); - - if (!ifobj_tx || ifobj_tx->shared_umem) - return; - - if (xdp_prog_changed_tx(test)) - xsk_reattach_xdp(ifobj_tx, test->xdp_prog_tx, test->xskmap_tx, test->mode); -} - -static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *ifobj1, - struct ifobject *ifobj2) -{ - pthread_t t0, t1; - int err; - - if (test->mtu > MAX_ETH_PKT_SIZE) { - if (test->mode == TEST_MODE_ZC && (!ifobj1->multi_buff_zc_supp || - (ifobj2 && !ifobj2->multi_buff_zc_supp))) { - ksft_test_result_skip("Multi buffer for zero-copy not supported.\n"); - return TEST_SKIP; - } - if (test->mode != TEST_MODE_ZC && (!ifobj1->multi_buff_supp || - (ifobj2 && !ifobj2->multi_buff_supp))) { - ksft_test_result_skip("Multi buffer not supported.\n"); - return TEST_SKIP; - } - } - err = test_spec_set_mtu(test, test->mtu); - if (err) { - ksft_print_msg("Error, could not set mtu.\n"); - exit_with_error(err); - } - - if (ifobj2) { - if (pthread_barrier_init(&barr, NULL, 2)) - exit_with_error(errno); - pkt_stream_reset(ifobj2->xsk->pkt_stream); - } - - test->current_step++; - pkt_stream_reset(ifobj1->xsk->pkt_stream); - pkts_in_flight = 0; - - signal(SIGUSR1, handler); - /*Spawn RX thread */ - pthread_create(&t0, NULL, ifobj1->func_ptr, test); - - if (ifobj2) { - pthread_barrier_wait(&barr); - if (pthread_barrier_destroy(&barr)) - exit_with_error(errno); - - /*Spawn TX thread */ - pthread_create(&t1, NULL, ifobj2->func_ptr, test); - - pthread_join(t1, NULL); - } - - if (!ifobj2) - pthread_kill(t0, SIGUSR1); - else - pthread_join(t0, NULL); - - if (test->total_steps == test->current_step || test->fail) { - u32 i; - - if (ifobj2) - for (i = 0; i < test->nb_sockets; i++) - xsk_socket__delete(ifobj2->xsk_arr[i].xsk); - - for (i = 0; i < test->nb_sockets; i++) - xsk_socket__delete(ifobj1->xsk_arr[i].xsk); - - testapp_clean_xsk_umem(ifobj1); - if (ifobj2 && !ifobj2->shared_umem) - testapp_clean_xsk_umem(ifobj2); - } - - return !!test->fail; -} - -static int testapp_validate_traffic(struct test_spec *test) -{ - struct ifobject *ifobj_rx = test->ifobj_rx; - struct ifobject *ifobj_tx = test->ifobj_tx; - - if ((ifobj_rx->umem->unaligned_mode && !ifobj_rx->unaligned_supp) || - (ifobj_tx->umem->unaligned_mode && !ifobj_tx->unaligned_supp)) { - ksft_test_result_skip("No huge pages present.\n"); - return TEST_SKIP; - } - - if (test->set_ring) { - if (ifobj_tx->hw_ring_size_supp) { - if (set_ring_size(ifobj_tx)) { - ksft_test_result_skip("Failed to change HW ring size.\n"); - return TEST_FAILURE; - } - } else { - ksft_test_result_skip("Changing HW ring size not supported.\n"); - return TEST_SKIP; - } - } - - xsk_attach_xdp_progs(test, ifobj_rx, ifobj_tx); - return __testapp_validate_traffic(test, ifobj_rx, ifobj_tx); -} - -static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj) -{ - return __testapp_validate_traffic(test, ifobj, NULL); -} - -static int testapp_teardown(struct test_spec *test) -{ - int i; - - for (i = 0; i < MAX_TEARDOWN_ITER; i++) { - if (testapp_validate_traffic(test)) - return TEST_FAILURE; - test_spec_reset(test); - } - - return TEST_PASS; -} - -static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2) -{ - thread_func_t tmp_func_ptr = (*ifobj1)->func_ptr; - struct ifobject *tmp_ifobj = (*ifobj1); - - (*ifobj1)->func_ptr = (*ifobj2)->func_ptr; - (*ifobj2)->func_ptr = tmp_func_ptr; - - *ifobj1 = *ifobj2; - *ifobj2 = tmp_ifobj; -} - -static int testapp_bidirectional(struct test_spec *test) -{ - int res; - - test->ifobj_tx->rx_on = true; - test->ifobj_rx->tx_on = true; - test->total_steps = 2; - if (testapp_validate_traffic(test)) - return TEST_FAILURE; - - print_verbose("Switching Tx/Rx direction\n"); - swap_directions(&test->ifobj_rx, &test->ifobj_tx); - res = __testapp_validate_traffic(test, test->ifobj_rx, test->ifobj_tx); - - swap_directions(&test->ifobj_rx, &test->ifobj_tx); - return res; -} - -static int swap_xsk_resources(struct test_spec *test) -{ - int ret; - - test->ifobj_tx->xsk_arr[0].pkt_stream = NULL; - test->ifobj_rx->xsk_arr[0].pkt_stream = NULL; - test->ifobj_tx->xsk_arr[1].pkt_stream = test->tx_pkt_stream_default; - test->ifobj_rx->xsk_arr[1].pkt_stream = test->rx_pkt_stream_default; - test->ifobj_tx->xsk = &test->ifobj_tx->xsk_arr[1]; - test->ifobj_rx->xsk = &test->ifobj_rx->xsk_arr[1]; - - ret = xsk_update_xskmap(test->ifobj_rx->xskmap, test->ifobj_rx->xsk->xsk, 0); - if (ret) - return TEST_FAILURE; - - return TEST_PASS; -} - -static int testapp_xdp_prog_cleanup(struct test_spec *test) -{ - test->total_steps = 2; - test->nb_sockets = 2; - if (testapp_validate_traffic(test)) - return TEST_FAILURE; - - if (swap_xsk_resources(test)) - return TEST_FAILURE; - return testapp_validate_traffic(test); -} - -static int testapp_headroom(struct test_spec *test) -{ - test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE; - return testapp_validate_traffic(test); -} - -static int testapp_stats_rx_dropped(struct test_spec *test) -{ - if (test->mode == TEST_MODE_ZC) { - ksft_test_result_skip("Can not run RX_DROPPED test for ZC mode\n"); - return TEST_SKIP; - } - - pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0); - test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size - - XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3; - pkt_stream_receive_half(test); - test->ifobj_rx->validation_func = validate_rx_dropped; - return testapp_validate_traffic(test); -} - -static int testapp_stats_tx_invalid_descs(struct test_spec *test) -{ - pkt_stream_replace_half(test, XSK_UMEM__INVALID_FRAME_SIZE, 0); - test->ifobj_tx->validation_func = validate_tx_invalid_descs; - return testapp_validate_traffic(test); -} - -static int testapp_stats_rx_full(struct test_spec *test) -{ - pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE); - test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE); - - test->ifobj_rx->xsk->rxqsize = DEFAULT_UMEM_BUFFERS; - test->ifobj_rx->release_rx = false; - test->ifobj_rx->validation_func = validate_rx_full; - return testapp_validate_traffic(test); -} - -static int testapp_stats_fill_empty(struct test_spec *test) -{ - pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE); - test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE); - - test->ifobj_rx->use_fill_ring = false; - test->ifobj_rx->validation_func = validate_fill_empty; - return testapp_validate_traffic(test); -} - -static int testapp_send_receive_unaligned(struct test_spec *test) -{ - test->ifobj_tx->umem->unaligned_mode = true; - test->ifobj_rx->umem->unaligned_mode = true; - /* Let half of the packets straddle a 4K buffer boundary */ - pkt_stream_replace_half(test, MIN_PKT_SIZE, -MIN_PKT_SIZE / 2); - - return testapp_validate_traffic(test); -} - -static int testapp_send_receive_unaligned_mb(struct test_spec *test) -{ - test->mtu = MAX_ETH_JUMBO_SIZE; - test->ifobj_tx->umem->unaligned_mode = true; - test->ifobj_rx->umem->unaligned_mode = true; - pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE); - return testapp_validate_traffic(test); -} - -static int testapp_single_pkt(struct test_spec *test) -{ - struct pkt pkts[] = {{0, MIN_PKT_SIZE, 0, true}}; - - pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); - return testapp_validate_traffic(test); -} - -static int testapp_send_receive_mb(struct test_spec *test) -{ - test->mtu = MAX_ETH_JUMBO_SIZE; - pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE); - - return testapp_validate_traffic(test); -} - -static int testapp_invalid_desc_mb(struct test_spec *test) -{ - struct xsk_umem_info *umem = test->ifobj_tx->umem; - u64 umem_size = umem->num_frames * umem->frame_size; - struct pkt pkts[] = { - /* Valid packet for synch to start with */ - {0, MIN_PKT_SIZE, 0, true, 0}, - /* Zero frame len is not legal */ - {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, - {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, - {0, 0, 0, false, 0}, - /* Invalid address in the second frame */ - {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, - {umem_size, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, - /* Invalid len in the middle */ - {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, - {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, - /* Invalid options in the middle */ - {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, - {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XSK_DESC__INVALID_OPTION}, - /* Transmit 2 frags, receive 3 */ - {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, XDP_PKT_CONTD}, - {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, 0}, - /* Middle frame crosses chunk boundary with small length */ - {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, - {-MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false, 0}, - /* Valid packet for synch so that something is received */ - {0, MIN_PKT_SIZE, 0, true, 0}}; - - if (umem->unaligned_mode) { - /* Crossing a chunk boundary allowed */ - pkts[12].valid = true; - pkts[13].valid = true; - } - - test->mtu = MAX_ETH_JUMBO_SIZE; - pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); - return testapp_validate_traffic(test); -} - -static int testapp_invalid_desc(struct test_spec *test) -{ - struct xsk_umem_info *umem = test->ifobj_tx->umem; - u64 umem_size = umem->num_frames * umem->frame_size; - struct pkt pkts[] = { - /* Zero packet address allowed */ - {0, MIN_PKT_SIZE, 0, true}, - /* Allowed packet */ - {0, MIN_PKT_SIZE, 0, true}, - /* Straddling the start of umem */ - {-2, MIN_PKT_SIZE, 0, false}, - /* Packet too large */ - {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false}, - /* Up to end of umem allowed */ - {umem_size - MIN_PKT_SIZE - 2 * umem->frame_size, MIN_PKT_SIZE, 0, true}, - /* After umem ends */ - {umem_size, MIN_PKT_SIZE, 0, false}, - /* Straddle the end of umem */ - {umem_size - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false}, - /* Straddle a 4K boundary */ - {0x1000 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false}, - /* Straddle a 2K boundary */ - {0x800 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, true}, - /* Valid packet for synch so that something is received */ - {0, MIN_PKT_SIZE, 0, true}}; - - if (umem->unaligned_mode) { - /* Crossing a page boundary allowed */ - pkts[7].valid = true; - } - if (umem->frame_size == XSK_UMEM__DEFAULT_FRAME_SIZE / 2) { - /* Crossing a 2K frame size boundary not allowed */ - pkts[8].valid = false; - } - - if (test->ifobj_tx->shared_umem) { - pkts[4].offset += umem_size; - pkts[5].offset += umem_size; - pkts[6].offset += umem_size; - } - - pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); - return testapp_validate_traffic(test); -} - -static int testapp_xdp_drop(struct test_spec *test) -{ - struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; - struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; - - test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_drop, skel_tx->progs.xsk_xdp_drop, - skel_rx->maps.xsk, skel_tx->maps.xsk); - - pkt_stream_receive_half(test); - return testapp_validate_traffic(test); -} - -static int testapp_xdp_metadata_copy(struct test_spec *test) -{ - struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; - struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; - - test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_populate_metadata, - skel_tx->progs.xsk_xdp_populate_metadata, - skel_rx->maps.xsk, skel_tx->maps.xsk); - test->ifobj_rx->use_metadata = true; - - skel_rx->bss->count = 0; - - return testapp_validate_traffic(test); -} - -static int testapp_xdp_shared_umem(struct test_spec *test) -{ - struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; - struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; - - test->total_steps = 1; - test->nb_sockets = 2; - - test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_shared_umem, - skel_tx->progs.xsk_xdp_shared_umem, - skel_rx->maps.xsk, skel_tx->maps.xsk); - - pkt_stream_even_odd_sequence(test); - - return testapp_validate_traffic(test); -} - -static int testapp_poll_txq_tmout(struct test_spec *test) -{ - test->ifobj_tx->use_poll = true; - /* create invalid frame by set umem frame_size and pkt length equal to 2048 */ - test->ifobj_tx->umem->frame_size = 2048; - pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048); - return testapp_validate_traffic_single_thread(test, test->ifobj_tx); -} - -static int testapp_poll_rxq_tmout(struct test_spec *test) -{ - test->ifobj_rx->use_poll = true; - return testapp_validate_traffic_single_thread(test, test->ifobj_rx); -} - -static int testapp_too_many_frags(struct test_spec *test) -{ - struct pkt *pkts; - u32 max_frags, i; - int ret; - - if (test->mode == TEST_MODE_ZC) { - max_frags = test->ifobj_tx->xdp_zc_max_segs; - } else { - max_frags = get_max_skb_frags(); - if (!max_frags) { - ksft_print_msg("Couldn't retrieve MAX_SKB_FRAGS from system, using default (17) value\n"); - max_frags = 17; - } - max_frags += 1; - } - - pkts = calloc(2 * max_frags + 2, sizeof(struct pkt)); - if (!pkts) - return TEST_FAILURE; - - test->mtu = MAX_ETH_JUMBO_SIZE; - - /* Valid packet for synch */ - pkts[0].len = MIN_PKT_SIZE; - pkts[0].valid = true; - - /* One valid packet with the max amount of frags */ - for (i = 1; i < max_frags + 1; i++) { - pkts[i].len = MIN_PKT_SIZE; - pkts[i].options = XDP_PKT_CONTD; - pkts[i].valid = true; - } - pkts[max_frags].options = 0; - - /* An invalid packet with the max amount of frags but signals packet - * continues on the last frag - */ - for (i = max_frags + 1; i < 2 * max_frags + 1; i++) { - pkts[i].len = MIN_PKT_SIZE; - pkts[i].options = XDP_PKT_CONTD; - pkts[i].valid = false; - } - - /* Valid packet for synch */ - pkts[2 * max_frags + 1].len = MIN_PKT_SIZE; - pkts[2 * max_frags + 1].valid = true; - - pkt_stream_generate_custom(test, pkts, 2 * max_frags + 2); - ret = testapp_validate_traffic(test); - - free(pkts); - return ret; -} - -static int xsk_load_xdp_programs(struct ifobject *ifobj) -{ - ifobj->xdp_progs = xsk_xdp_progs__open_and_load(); - if (libbpf_get_error(ifobj->xdp_progs)) - return libbpf_get_error(ifobj->xdp_progs); - - return 0; -} - -static void xsk_unload_xdp_programs(struct ifobject *ifobj) -{ - xsk_xdp_progs__destroy(ifobj->xdp_progs); -} - -/* Simple test */ -static bool hugepages_present(void) -{ - size_t mmap_sz = 2 * DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE; - void *bufs; - - bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, MAP_HUGE_2MB); - if (bufs == MAP_FAILED) - return false; - - mmap_sz = ceil_u64(mmap_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE; - munmap(bufs, mmap_sz); - return true; -} - -static void init_iface(struct ifobject *ifobj, thread_func_t func_ptr) -{ - LIBBPF_OPTS(bpf_xdp_query_opts, query_opts); - int err; - - ifobj->func_ptr = func_ptr; - - err = xsk_load_xdp_programs(ifobj); - if (err) { - ksft_print_msg("Error loading XDP program\n"); - exit_with_error(err); - } - - if (hugepages_present()) - ifobj->unaligned_supp = true; - - err = bpf_xdp_query(ifobj->ifindex, XDP_FLAGS_DRV_MODE, &query_opts); - if (err) { - ksft_print_msg("Error querying XDP capabilities\n"); - exit_with_error(-err); - } - if (query_opts.feature_flags & NETDEV_XDP_ACT_RX_SG) - ifobj->multi_buff_supp = true; - if (query_opts.feature_flags & NETDEV_XDP_ACT_XSK_ZEROCOPY) { - if (query_opts.xdp_zc_max_segs > 1) { - ifobj->multi_buff_zc_supp = true; - ifobj->xdp_zc_max_segs = query_opts.xdp_zc_max_segs; - } else { - ifobj->xdp_zc_max_segs = 0; - } - } -} - -static int testapp_send_receive(struct test_spec *test) -{ - return testapp_validate_traffic(test); -} - -static int testapp_send_receive_2k_frame(struct test_spec *test) -{ - test->ifobj_tx->umem->frame_size = 2048; - test->ifobj_rx->umem->frame_size = 2048; - pkt_stream_replace(test, DEFAULT_PKT_CNT, MIN_PKT_SIZE); - return testapp_validate_traffic(test); -} - -static int testapp_poll_rx(struct test_spec *test) -{ - test->ifobj_rx->use_poll = true; - return testapp_validate_traffic(test); -} - -static int testapp_poll_tx(struct test_spec *test) -{ - test->ifobj_tx->use_poll = true; - return testapp_validate_traffic(test); -} - -static int testapp_aligned_inv_desc(struct test_spec *test) -{ - return testapp_invalid_desc(test); -} - -static int testapp_aligned_inv_desc_2k_frame(struct test_spec *test) -{ - test->ifobj_tx->umem->frame_size = 2048; - test->ifobj_rx->umem->frame_size = 2048; - return testapp_invalid_desc(test); -} - -static int testapp_unaligned_inv_desc(struct test_spec *test) -{ - test->ifobj_tx->umem->unaligned_mode = true; - test->ifobj_rx->umem->unaligned_mode = true; - return testapp_invalid_desc(test); -} - -static int testapp_unaligned_inv_desc_4001_frame(struct test_spec *test) -{ - u64 page_size, umem_size; - - /* Odd frame size so the UMEM doesn't end near a page boundary. */ - test->ifobj_tx->umem->frame_size = 4001; - test->ifobj_rx->umem->frame_size = 4001; - test->ifobj_tx->umem->unaligned_mode = true; - test->ifobj_rx->umem->unaligned_mode = true; - /* This test exists to test descriptors that staddle the end of - * the UMEM but not a page. - */ - page_size = sysconf(_SC_PAGESIZE); - umem_size = test->ifobj_tx->umem->num_frames * test->ifobj_tx->umem->frame_size; - assert(umem_size % page_size > MIN_PKT_SIZE); - assert(umem_size % page_size < page_size - MIN_PKT_SIZE); - - return testapp_invalid_desc(test); -} - -static int testapp_aligned_inv_desc_mb(struct test_spec *test) -{ - return testapp_invalid_desc_mb(test); -} - -static int testapp_unaligned_inv_desc_mb(struct test_spec *test) -{ - test->ifobj_tx->umem->unaligned_mode = true; - test->ifobj_rx->umem->unaligned_mode = true; - return testapp_invalid_desc_mb(test); -} - -static int testapp_xdp_metadata(struct test_spec *test) -{ - return testapp_xdp_metadata_copy(test); -} - -static int testapp_xdp_metadata_mb(struct test_spec *test) -{ - test->mtu = MAX_ETH_JUMBO_SIZE; - return testapp_xdp_metadata_copy(test); -} - -static int testapp_hw_sw_min_ring_size(struct test_spec *test) -{ - int ret; - - test->set_ring = true; - test->total_steps = 2; - test->ifobj_tx->ring.tx_pending = DEFAULT_BATCH_SIZE; - test->ifobj_tx->ring.rx_pending = DEFAULT_BATCH_SIZE * 2; - test->ifobj_tx->xsk->batch_size = 1; - test->ifobj_rx->xsk->batch_size = 1; - ret = testapp_validate_traffic(test); - if (ret) - return ret; - - /* Set batch size to hw_ring_size - 1 */ - test->ifobj_tx->xsk->batch_size = DEFAULT_BATCH_SIZE - 1; - test->ifobj_rx->xsk->batch_size = DEFAULT_BATCH_SIZE - 1; - return testapp_validate_traffic(test); -} - -static int testapp_hw_sw_max_ring_size(struct test_spec *test) -{ - u32 max_descs = XSK_RING_PROD__DEFAULT_NUM_DESCS * 4; - int ret; - - test->set_ring = true; - test->total_steps = 2; - test->ifobj_tx->ring.tx_pending = test->ifobj_tx->ring.tx_max_pending; - test->ifobj_tx->ring.rx_pending = test->ifobj_tx->ring.rx_max_pending; - test->ifobj_rx->umem->num_frames = max_descs; - test->ifobj_rx->umem->fill_size = max_descs; - test->ifobj_rx->umem->comp_size = max_descs; - test->ifobj_tx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; - test->ifobj_rx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; - - ret = testapp_validate_traffic(test); - if (ret) - return ret; - - /* Set batch_size to 8152 for testing, as the ice HW ignores the 3 lowest bits when - * updating the Rx HW tail register. - */ - test->ifobj_tx->xsk->batch_size = test->ifobj_tx->ring.tx_max_pending - 8; - test->ifobj_rx->xsk->batch_size = test->ifobj_tx->ring.tx_max_pending - 8; - pkt_stream_replace(test, max_descs, MIN_PKT_SIZE); - return testapp_validate_traffic(test); -} - -static int testapp_xdp_adjust_tail(struct test_spec *test, int adjust_value) -{ - struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; - struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; - - test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_adjust_tail, - skel_tx->progs.xsk_xdp_adjust_tail, - skel_rx->maps.xsk, skel_tx->maps.xsk); - - skel_rx->bss->adjust_value = adjust_value; - - return testapp_validate_traffic(test); -} - -static int testapp_adjust_tail(struct test_spec *test, u32 value, u32 pkt_len) -{ - int ret; - - test->adjust_tail_support = true; - test->adjust_tail = true; - test->total_steps = 1; - - pkt_stream_replace_ifobject(test->ifobj_tx, DEFAULT_BATCH_SIZE, pkt_len); - pkt_stream_replace_ifobject(test->ifobj_rx, DEFAULT_BATCH_SIZE, pkt_len + value); - - ret = testapp_xdp_adjust_tail(test, value); - if (ret) - return ret; - - if (!test->adjust_tail_support) { - ksft_test_result_skip("%s %sResize pkt with bpf_xdp_adjust_tail() not supported\n", - mode_string(test), busy_poll_string(test)); - return TEST_SKIP; - } - - return 0; -} - -static int testapp_adjust_tail_shrink(struct test_spec *test) -{ - /* Shrink by 4 bytes for testing purpose */ - return testapp_adjust_tail(test, -4, MIN_PKT_SIZE * 2); -} - -static int testapp_adjust_tail_shrink_mb(struct test_spec *test) -{ - test->mtu = MAX_ETH_JUMBO_SIZE; - /* Shrink by the frag size */ - return testapp_adjust_tail(test, -XSK_UMEM__MAX_FRAME_SIZE, XSK_UMEM__LARGE_FRAME_SIZE * 2); -} - -static int testapp_adjust_tail_grow(struct test_spec *test) -{ - /* Grow by 4 bytes for testing purpose */ - return testapp_adjust_tail(test, 4, MIN_PKT_SIZE * 2); -} - -static int testapp_adjust_tail_grow_mb(struct test_spec *test) -{ - test->mtu = MAX_ETH_JUMBO_SIZE; - /* Grow by (frag_size - last_frag_Size) - 1 to stay inside the last fragment */ - return testapp_adjust_tail(test, (XSK_UMEM__MAX_FRAME_SIZE / 2) - 1, - XSK_UMEM__LARGE_FRAME_SIZE * 2); -} - -static int testapp_tx_queue_consumer(struct test_spec *test) -{ - int nr_packets; - - if (test->mode == TEST_MODE_ZC) { - ksft_test_result_skip("Can not run TX_QUEUE_CONSUMER test for ZC mode\n"); - return TEST_SKIP; - } - - nr_packets = MAX_TX_BUDGET_DEFAULT + 1; - pkt_stream_replace(test, nr_packets, MIN_PKT_SIZE); - test->ifobj_tx->xsk->batch_size = nr_packets; - test->ifobj_tx->xsk->check_consumer = true; - - return testapp_validate_traffic(test); -} - -static void run_pkt_test(struct test_spec *test) -{ - int ret; - - ret = test->test_func(test); - - if (ret == TEST_PASS) - ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test), - test->name); - pkt_stream_restore_default(test); -} - -static struct ifobject *ifobject_create(void) -{ - struct ifobject *ifobj; - - ifobj = calloc(1, sizeof(struct ifobject)); - if (!ifobj) - return NULL; - - ifobj->xsk_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->xsk_arr)); - if (!ifobj->xsk_arr) - goto out_xsk_arr; - - ifobj->umem = calloc(1, sizeof(*ifobj->umem)); - if (!ifobj->umem) - goto out_umem; - - return ifobj; - -out_umem: - free(ifobj->xsk_arr); -out_xsk_arr: - free(ifobj); - return NULL; -} - -static void ifobject_delete(struct ifobject *ifobj) -{ - free(ifobj->umem); - free(ifobj->xsk_arr); - free(ifobj); -} - -static bool is_xdp_supported(int ifindex) -{ - int flags = XDP_FLAGS_DRV_MODE; - - LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags); - struct bpf_insn insns[2] = { - BPF_MOV64_IMM(BPF_REG_0, XDP_PASS), - BPF_EXIT_INSN() - }; - int prog_fd, insn_cnt = ARRAY_SIZE(insns); - int err; - - prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL); - if (prog_fd < 0) - return false; - - err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL); - if (err) { - close(prog_fd); - return false; - } - - bpf_xdp_detach(ifindex, flags, NULL); - close(prog_fd); - - return true; -} - -static const struct test_spec tests[] = { - {.name = "SEND_RECEIVE", .test_func = testapp_send_receive}, - {.name = "SEND_RECEIVE_2K_FRAME", .test_func = testapp_send_receive_2k_frame}, - {.name = "SEND_RECEIVE_SINGLE_PKT", .test_func = testapp_single_pkt}, - {.name = "POLL_RX", .test_func = testapp_poll_rx}, - {.name = "POLL_TX", .test_func = testapp_poll_tx}, - {.name = "POLL_RXQ_FULL", .test_func = testapp_poll_rxq_tmout}, - {.name = "POLL_TXQ_FULL", .test_func = testapp_poll_txq_tmout}, - {.name = "SEND_RECEIVE_UNALIGNED", .test_func = testapp_send_receive_unaligned}, - {.name = "ALIGNED_INV_DESC", .test_func = testapp_aligned_inv_desc}, - {.name = "ALIGNED_INV_DESC_2K_FRAME_SIZE", .test_func = testapp_aligned_inv_desc_2k_frame}, - {.name = "UNALIGNED_INV_DESC", .test_func = testapp_unaligned_inv_desc}, - {.name = "UNALIGNED_INV_DESC_4001_FRAME_SIZE", - .test_func = testapp_unaligned_inv_desc_4001_frame}, - {.name = "UMEM_HEADROOM", .test_func = testapp_headroom}, - {.name = "TEARDOWN", .test_func = testapp_teardown}, - {.name = "BIDIRECTIONAL", .test_func = testapp_bidirectional}, - {.name = "STAT_RX_DROPPED", .test_func = testapp_stats_rx_dropped}, - {.name = "STAT_TX_INVALID", .test_func = testapp_stats_tx_invalid_descs}, - {.name = "STAT_RX_FULL", .test_func = testapp_stats_rx_full}, - {.name = "STAT_FILL_EMPTY", .test_func = testapp_stats_fill_empty}, - {.name = "XDP_PROG_CLEANUP", .test_func = testapp_xdp_prog_cleanup}, - {.name = "XDP_DROP_HALF", .test_func = testapp_xdp_drop}, - {.name = "XDP_SHARED_UMEM", .test_func = testapp_xdp_shared_umem}, - {.name = "XDP_METADATA_COPY", .test_func = testapp_xdp_metadata}, - {.name = "XDP_METADATA_COPY_MULTI_BUFF", .test_func = testapp_xdp_metadata_mb}, - {.name = "SEND_RECEIVE_9K_PACKETS", .test_func = testapp_send_receive_mb}, - {.name = "SEND_RECEIVE_UNALIGNED_9K_PACKETS", - .test_func = testapp_send_receive_unaligned_mb}, - {.name = "ALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_aligned_inv_desc_mb}, - {.name = "UNALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_unaligned_inv_desc_mb}, - {.name = "TOO_MANY_FRAGS", .test_func = testapp_too_many_frags}, - {.name = "HW_SW_MIN_RING_SIZE", .test_func = testapp_hw_sw_min_ring_size}, - {.name = "HW_SW_MAX_RING_SIZE", .test_func = testapp_hw_sw_max_ring_size}, - {.name = "XDP_ADJUST_TAIL_SHRINK", .test_func = testapp_adjust_tail_shrink}, - {.name = "XDP_ADJUST_TAIL_SHRINK_MULTI_BUFF", .test_func = testapp_adjust_tail_shrink_mb}, - {.name = "XDP_ADJUST_TAIL_GROW", .test_func = testapp_adjust_tail_grow}, - {.name = "XDP_ADJUST_TAIL_GROW_MULTI_BUFF", .test_func = testapp_adjust_tail_grow_mb}, - {.name = "TX_QUEUE_CONSUMER", .test_func = testapp_tx_queue_consumer}, - }; - static void print_tests(void) { u32 i; @@ -2833,7 +376,7 @@ int main(int argc, char **argv) init_iface(ifobj_rx, worker_testapp_validate_rx); init_iface(ifobj_tx, worker_testapp_validate_tx); - test_spec_init(&test, ifobj_tx, ifobj_rx, 0, &tests[0]); + test_init(&test, ifobj_tx, ifobj_rx, 0, &tests[0]); tx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE); rx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE); if (!tx_pkt_stream_default || !rx_pkt_stream_default) @@ -2868,7 +411,7 @@ int main(int argc, char **argv) if (opt_run_test != RUN_ALL_TESTS && j != opt_run_test) continue; - test_spec_init(&test, ifobj_tx, ifobj_rx, i, &tests[j]); + test_init(&test, ifobj_tx, ifobj_rx, i, &tests[j]); run_pkt_test(&test); usleep(USLEEP_MAX); diff --git a/tools/testing/selftests/bpf/xskxceiver.h b/tools/testing/selftests/bpf/xskxceiver.h index 4df3a5d329acf..3ca518df23ada 100644 --- a/tools/testing/selftests/bpf/xskxceiver.h +++ b/tools/testing/selftests/bpf/xskxceiver.h @@ -22,169 +22,13 @@ #define PF_XDP AF_XDP #endif -#ifndef SO_BUSY_POLL_BUDGET -#define SO_BUSY_POLL_BUDGET 70 -#endif - -#ifndef SO_PREFER_BUSY_POLL -#define SO_PREFER_BUSY_POLL 69 -#endif - -#define TEST_PASS 0 -#define TEST_FAILURE -1 -#define TEST_CONTINUE 1 -#define TEST_SKIP 2 -#define MAX_INTERFACES 2 -#define MAX_INTERFACE_NAME_CHARS 16 -#define MAX_TEST_NAME_SIZE 48 #define MAX_TEARDOWN_ITER 10 -#define PKT_HDR_SIZE (sizeof(struct ethhdr) + 2) /* Just to align the data in the packet */ -#define MIN_PKT_SIZE 64 -#define MAX_ETH_PKT_SIZE 1518 #define MAX_ETH_JUMBO_SIZE 9000 -#define USLEEP_MAX 10000 #define SOCK_RECONF_CTR 10 -#define DEFAULT_BATCH_SIZE 64 -#define POLL_TMOUT 1000 -#define THREAD_TMOUT 3 -#define DEFAULT_PKT_CNT (4 * 1024) -#define DEFAULT_UMEM_BUFFERS (DEFAULT_PKT_CNT / 4) #define RX_FULL_RXQSIZE 32 #define UMEM_HEADROOM_TEST_SIZE 128 #define XSK_UMEM__INVALID_FRAME_SIZE (MAX_ETH_JUMBO_SIZE + 1) -#define XSK_UMEM__LARGE_FRAME_SIZE (3 * 1024) -#define XSK_UMEM__MAX_FRAME_SIZE (4 * 1024) -#define XSK_DESC__INVALID_OPTION (0xffff) -#define HUGEPAGE_SIZE (2 * 1024 * 1024) -#define PKT_DUMP_NB_TO_PRINT 16 #define RUN_ALL_TESTS UINT_MAX #define NUM_MAC_ADDRESSES 4 -#define print_verbose(x...) do { if (opt_verbose) ksft_print_msg(x); } while (0) - -enum test_mode { - TEST_MODE_SKB, - TEST_MODE_DRV, - TEST_MODE_ZC, - TEST_MODE_ALL -}; - -struct xsk_umem_info { - struct xsk_ring_prod fq; - struct xsk_ring_cons cq; - struct xsk_umem *umem; - u64 next_buffer; - u32 num_frames; - u32 frame_headroom; - void *buffer; - u32 frame_size; - u32 base_addr; - u32 fill_size; - u32 comp_size; - bool unaligned_mode; -}; - -struct xsk_socket_info { - struct xsk_ring_cons rx; - struct xsk_ring_prod tx; - struct xsk_umem_info *umem; - struct xsk_socket *xsk; - struct pkt_stream *pkt_stream; - u32 outstanding_tx; - u32 rxqsize; - u32 batch_size; - u8 dst_mac[ETH_ALEN]; - u8 src_mac[ETH_ALEN]; - bool check_consumer; -}; - -struct pkt { - int offset; - u32 len; - u32 pkt_nb; - bool valid; - u16 options; -}; - -struct pkt_stream { - u32 nb_pkts; - u32 current_pkt_nb; - struct pkt *pkts; - u32 max_pkt_len; - u32 nb_rx_pkts; - u32 nb_valid_entries; - bool verbatim; -}; - -struct set_hw_ring { - u32 default_tx; - u32 default_rx; -}; - -struct ifobject; -struct test_spec; -typedef int (*validation_func_t)(struct ifobject *ifobj); -typedef void *(*thread_func_t)(void *arg); -typedef int (*test_func_t)(struct test_spec *test); - -struct ifobject { - char ifname[MAX_INTERFACE_NAME_CHARS]; - struct xsk_socket_info *xsk; - struct xsk_socket_info *xsk_arr; - struct xsk_umem_info *umem; - thread_func_t func_ptr; - validation_func_t validation_func; - struct xsk_xdp_progs *xdp_progs; - struct bpf_map *xskmap; - struct bpf_program *xdp_prog; - struct ethtool_ringparam ring; - struct set_hw_ring set_ring; - enum test_mode mode; - int ifindex; - int mtu; - u32 bind_flags; - u32 xdp_zc_max_segs; - bool tx_on; - bool rx_on; - bool use_poll; - bool busy_poll; - bool use_fill_ring; - bool release_rx; - bool shared_umem; - bool use_metadata; - bool unaligned_supp; - bool multi_buff_supp; - bool multi_buff_zc_supp; - bool hw_ring_size_supp; -}; - -struct test_spec { - struct ifobject *ifobj_tx; - struct ifobject *ifobj_rx; - struct pkt_stream *tx_pkt_stream_default; - struct pkt_stream *rx_pkt_stream_default; - struct bpf_program *xdp_prog_rx; - struct bpf_program *xdp_prog_tx; - struct bpf_map *xskmap_rx; - struct bpf_map *xskmap_tx; - test_func_t test_func; - int mtu; - u16 total_steps; - u16 current_step; - u16 nb_sockets; - bool fail; - bool set_ring; - bool adjust_tail; - bool adjust_tail_support; - enum test_mode mode; - char name[MAX_TEST_NAME_SIZE]; -}; - -pthread_barrier_t barr; -pthread_mutex_t pacing_mutex = PTHREAD_MUTEX_INITIALIZER; - -int pkts_in_flight; - -static const u8 g_mac[ETH_ALEN] = {0x55, 0x44, 0x33, 0x22, 0x11, 0x00}; - #endif /* XSKXCEIVER_H_ */ diff --git a/tools/testing/selftests/drivers/net/lib/sh/lib_netcons.sh b/tools/testing/selftests/drivers/net/lib/sh/lib_netcons.sh index 87f89fd92f8c1..64d3941576d5d 100644 --- a/tools/testing/selftests/drivers/net/lib/sh/lib_netcons.sh +++ b/tools/testing/selftests/drivers/net/lib/sh/lib_netcons.sh @@ -247,9 +247,9 @@ function listen_port_and_save_to() { SOCAT_MODE="UDP6-LISTEN" fi - # Just wait for 2 seconds - timeout 2 ip netns exec "${NAMESPACE}" \ - socat "${SOCAT_MODE}":"${PORT}",fork "${OUTPUT}" + # Just wait for 3 seconds + timeout 3 ip netns exec "${NAMESPACE}" \ + socat "${SOCAT_MODE}":"${PORT}",fork "${OUTPUT}" 2> /dev/null } # Only validate that the message arrived properly diff --git a/tools/testing/selftests/drivers/net/mlxsw/tc_restrictions.sh b/tools/testing/selftests/drivers/net/mlxsw/tc_restrictions.sh index 0441a18f098b1..aac8ef490feb8 100755 --- a/tools/testing/selftests/drivers/net/mlxsw/tc_restrictions.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/tc_restrictions.sh @@ -317,7 +317,7 @@ police_limits_test() tc filter add dev $swp1 ingress pref 1 proto ip handle 101 \ flower skip_sw \ - action police rate 0.5kbit burst 1m conform-exceed drop/ok + action police rate 0.5kbit burst 2k conform-exceed drop/ok check_fail $? "Incorrect success to add police action with too low rate" tc filter add dev $swp1 ingress pref 1 proto ip handle 101 \ @@ -327,7 +327,7 @@ police_limits_test() tc filter add dev $swp1 ingress pref 1 proto ip handle 101 \ flower skip_sw \ - action police rate 1.5kbit burst 1m conform-exceed drop/ok + action police rate 1.5kbit burst 2k conform-exceed drop/ok check_err $? "Failed to add police action with low rate" tc filter del dev $swp1 ingress protocol ip pref 1 handle 101 flower diff --git a/tools/testing/selftests/kselftest_harness.h b/tools/testing/selftests/kselftest_harness.h index 3f66e862e83eb..fe162cbfc0912 100644 --- a/tools/testing/selftests/kselftest_harness.h +++ b/tools/testing/selftests/kselftest_harness.h @@ -70,6 +70,15 @@ #include "kselftest.h" +static inline void __kselftest_memset_safe(void *s, int c, size_t n) +{ + if (n > 0) + memset(s, c, n); +} + +#define KSELFTEST_PRIO_TEST_F 20000 +#define KSELFTEST_PRIO_XFAIL 20001 + #define TEST_TIMEOUT_DEFAULT 30 /* Utilities exposed to the test definitions */ @@ -416,7 +425,7 @@ self = mmap(NULL, sizeof(*self), PROT_READ | PROT_WRITE, \ MAP_SHARED | MAP_ANONYMOUS, -1, 0); \ } else { \ - memset(&self_private, 0, sizeof(self_private)); \ + __kselftest_memset_safe(&self_private, 0, sizeof(self_private)); \ self = &self_private; \ } \ } \ @@ -459,7 +468,7 @@ fixture_name##_teardown(_metadata, self, variant); \ } \ static struct __test_metadata *_##fixture_name##_##test_name##_object; \ - static void __attribute__((constructor)) \ + static void __attribute__((constructor(KSELFTEST_PRIO_TEST_F))) \ _register_##fixture_name##_##test_name(void) \ { \ struct __test_metadata *object = mmap(NULL, sizeof(*object), \ @@ -874,7 +883,7 @@ struct __test_xfail { .fixture = &_##fixture_name##_fixture_object, \ .variant = &_##fixture_name##_##variant_name##_object, \ }; \ - static void __attribute__((constructor)) \ + static void __attribute__((constructor(KSELFTEST_PRIO_XFAIL))) \ _register_##fixture_name##_##variant_name##_##test_name##_xfail(void) \ { \ _##fixture_name##_##variant_name##_##test_name##_xfail.test = \ diff --git a/tools/testing/selftests/memfd/memfd_test.c b/tools/testing/selftests/memfd/memfd_test.c index 5b993924cc3f5..2ca07ea7202a5 100644 --- a/tools/testing/selftests/memfd/memfd_test.c +++ b/tools/testing/selftests/memfd/memfd_test.c @@ -18,6 +18,9 @@ #include #include #include +#include +#include +#include #include #include @@ -39,6 +42,20 @@ F_SEAL_EXEC) #define MFD_NOEXEC_SEAL 0x0008U +union semun { + int val; + struct semid_ds *buf; + unsigned short int *array; + struct seminfo *__buf; +}; + +/* + * we use semaphores on nested wait tasks due the use of CLONE_NEWPID: the + * child will be PID 1 and can't send SIGSTOP to themselves due special + * treatment of the init task, so the SIGSTOP/SIGCONT synchronization + * approach can't be used here. + */ +#define SEM_KEY 0xdeadbeef /* * Default is not to test hugetlbfs @@ -1333,8 +1350,22 @@ static int sysctl_nested(void *arg) static int sysctl_nested_wait(void *arg) { - /* Wait for a SIGCONT. */ - kill(getpid(), SIGSTOP); + int sem = semget(SEM_KEY, 1, 0600); + struct sembuf sembuf; + + if (sem < 0) { + perror("semget:"); + abort(); + } + sembuf.sem_num = 0; + sembuf.sem_flg = 0; + sembuf.sem_op = 0; + + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + abort(); + } + return sysctl_nested(arg); } @@ -1355,7 +1386,9 @@ static void test_sysctl_sysctl2_failset(void) static int sysctl_nested_child(void *arg) { - int pid; + int pid, sem; + union semun semun; + struct sembuf sembuf; printf("%s nested sysctl 0\n", memfd_str); sysctl_assert_write("0"); @@ -1389,23 +1422,53 @@ static int sysctl_nested_child(void *arg) test_sysctl_sysctl2_failset); join_thread(pid); + sem = semget(SEM_KEY, 1, IPC_CREAT | 0600); + if (sem < 0) { + perror("semget:"); + return 1; + } + semun.val = 1; + sembuf.sem_op = -1; + sembuf.sem_flg = 0; + sembuf.sem_num = 0; + /* Verify that the rules are actually inherited after fork. */ printf("%s nested sysctl 0 -> 1 after fork\n", memfd_str); sysctl_assert_write("0"); + if (semctl(sem, 0, SETVAL, semun) < 0) { + perror("semctl:"); + return 1; + } + pid = spawn_thread(CLONE_NEWPID, sysctl_nested_wait, test_sysctl_sysctl1_failset); sysctl_assert_write("1"); - kill(pid, SIGCONT); + + /* Allow child to continue */ + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + return 1; + } join_thread(pid); printf("%s nested sysctl 0 -> 2 after fork\n", memfd_str); sysctl_assert_write("0"); + if (semctl(sem, 0, SETVAL, semun) < 0) { + perror("semctl:"); + return 1; + } + pid = spawn_thread(CLONE_NEWPID, sysctl_nested_wait, test_sysctl_sysctl2_failset); sysctl_assert_write("2"); - kill(pid, SIGCONT); + + /* Allow child to continue */ + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + return 1; + } join_thread(pid); /* @@ -1415,28 +1478,62 @@ static int sysctl_nested_child(void *arg) */ printf("%s nested sysctl 2 -> 1 after fork\n", memfd_str); sysctl_assert_write("2"); + + if (semctl(sem, 0, SETVAL, semun) < 0) { + perror("semctl:"); + return 1; + } + pid = spawn_thread(CLONE_NEWPID, sysctl_nested_wait, test_sysctl_sysctl2); sysctl_assert_write("1"); - kill(pid, SIGCONT); + + /* Allow child to continue */ + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + return 1; + } join_thread(pid); printf("%s nested sysctl 2 -> 0 after fork\n", memfd_str); sysctl_assert_write("2"); + + if (semctl(sem, 0, SETVAL, semun) < 0) { + perror("semctl:"); + return 1; + } + pid = spawn_thread(CLONE_NEWPID, sysctl_nested_wait, test_sysctl_sysctl2); sysctl_assert_write("0"); - kill(pid, SIGCONT); + + /* Allow child to continue */ + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + return 1; + } join_thread(pid); printf("%s nested sysctl 1 -> 0 after fork\n", memfd_str); sysctl_assert_write("1"); + + if (semctl(sem, 0, SETVAL, semun) < 0) { + perror("semctl:"); + return 1; + } + pid = spawn_thread(CLONE_NEWPID, sysctl_nested_wait, test_sysctl_sysctl1); sysctl_assert_write("0"); - kill(pid, SIGCONT); + /* Allow child to continue */ + if (semop(sem, &sembuf, 1) < 0) { + perror("semop:"); + return 1; + } join_thread(pid); + semctl(sem, 0, IPC_RMID); + return 0; } diff --git a/tools/testing/selftests/mm/charge_reserved_hugetlb.sh b/tools/testing/selftests/mm/charge_reserved_hugetlb.sh index e1fe16bcbbe88..fa6713892d82d 100755 --- a/tools/testing/selftests/mm/charge_reserved_hugetlb.sh +++ b/tools/testing/selftests/mm/charge_reserved_hugetlb.sh @@ -290,7 +290,7 @@ function run_test() { setup_cgroup "hugetlb_cgroup_test" "$cgroup_limit" "$reservation_limit" mkdir -p /mnt/huge - mount -t hugetlbfs -o pagesize=${MB}M,size=256M none /mnt/huge + mount -t hugetlbfs -o pagesize=${MB}M none /mnt/huge write_hugetlbfs_and_get_usage "hugetlb_cgroup_test" "$size" "$populate" \ "$write" "/mnt/huge/test" "$method" "$private" "$expect_failure" \ @@ -344,7 +344,7 @@ function run_multiple_cgroup_test() { setup_cgroup "hugetlb_cgroup_test2" "$cgroup_limit2" "$reservation_limit2" mkdir -p /mnt/huge - mount -t hugetlbfs -o pagesize=${MB}M,size=256M none /mnt/huge + mount -t hugetlbfs -o pagesize=${MB}M none /mnt/huge write_hugetlbfs_and_get_usage "hugetlb_cgroup_test1" "$size1" \ "$populate1" "$write1" "/mnt/huge/test1" "$method" "$private" \ diff --git a/tools/testing/selftests/mm/cow.c b/tools/testing/selftests/mm/cow.c index 6560c26f47d1f..0df61422467da 100644 --- a/tools/testing/selftests/mm/cow.c +++ b/tools/testing/selftests/mm/cow.c @@ -1612,8 +1612,8 @@ static void run_with_huge_zeropage(non_anon_test_fn fn, const char *desc) * the first sub-page and test if we get another sub-page populated * automatically. */ - FORCE_READ(mem); - FORCE_READ(smem); + FORCE_READ(*mem); + FORCE_READ(*smem); if (!pagemap_is_populated(pagemap_fd, mem + pagesize) || !pagemap_is_populated(pagemap_fd, smem + pagesize)) { ksft_test_result_skip("Did not get THPs populated\n"); @@ -1663,8 +1663,8 @@ static void run_with_memfd(non_anon_test_fn fn, const char *desc) } /* Fault the page in. */ - FORCE_READ(mem); - FORCE_READ(smem); + FORCE_READ(*mem); + FORCE_READ(*smem); fn(mem, smem, pagesize); munmap: @@ -1719,8 +1719,8 @@ static void run_with_tmpfile(non_anon_test_fn fn, const char *desc) } /* Fault the page in. */ - FORCE_READ(mem); - FORCE_READ(smem); + FORCE_READ(*mem); + FORCE_READ(*smem); fn(mem, smem, pagesize); munmap: @@ -1773,8 +1773,8 @@ static void run_with_memfd_hugetlb(non_anon_test_fn fn, const char *desc, } /* Fault the page in. */ - FORCE_READ(mem); - FORCE_READ(smem); + FORCE_READ(*mem); + FORCE_READ(*smem); fn(mem, smem, hugetlbsize); munmap: diff --git a/tools/testing/selftests/net/forwarding/tc_actions.sh b/tools/testing/selftests/net/forwarding/tc_actions.sh index ea89e558672db..86edbc7e2489b 100755 --- a/tools/testing/selftests/net/forwarding/tc_actions.sh +++ b/tools/testing/selftests/net/forwarding/tc_actions.sh @@ -223,7 +223,7 @@ mirred_egress_to_ingress_tcp_test() ip_proto icmp \ action drop - ip vrf exec v$h1 ncat --recv-only -w10 -l -p 12345 -o $mirred_e2i_tf2 & + ip vrf exec v$h1 ncat --recv-only -w10 -l -p 12345 > $mirred_e2i_tf2 & local rpid=$! ip vrf exec v$h1 ncat -w1 --send-only 192.0.2.2 12345 <$mirred_e2i_tf1 wait -n $rpid diff --git a/tools/testing/selftests/net/forwarding/vxlan_bridge_1d.sh b/tools/testing/selftests/net/forwarding/vxlan_bridge_1d.sh index b43816dd998ca..457f41d5e584b 100755 --- a/tools/testing/selftests/net/forwarding/vxlan_bridge_1d.sh +++ b/tools/testing/selftests/net/forwarding/vxlan_bridge_1d.sh @@ -567,6 +567,21 @@ vxlan_encapped_ping_do() local inner_tos=$1; shift local outer_tos=$1; shift + local ipv4hdr=$(: + )"45:"$( : IP version + IHL + )"$inner_tos:"$( : IP TOS + )"00:54:"$( : IP total length + )"99:83:"$( : IP identification + )"40:00:"$( : IP flags + frag off + )"40:"$( : IP TTL + )"01:"$( : IP proto + )"CHECKSUM:"$( : IP header csum + )"c0:00:02:03:"$( : IP saddr: 192.0.2.3 + )"c0:00:02:01"$( : IP daddr: 192.0.2.1 + ) + local checksum=$(payload_template_calc_checksum "$ipv4hdr") + ipv4hdr=$(payload_template_expand_checksum "$ipv4hdr" $checksum) + $MZ $dev -c $count -d 100msec -q \ -b $next_hop_mac -B $dest_ip \ -t udp tos=$outer_tos,sp=23456,dp=$VXPORT,p=$(: @@ -577,16 +592,7 @@ vxlan_encapped_ping_do() )"$dest_mac:"$( : ETH daddr )"$(mac_get w2):"$( : ETH saddr )"08:00:"$( : ETH type - )"45:"$( : IP version + IHL - )"$inner_tos:"$( : IP TOS - )"00:54:"$( : IP total length - )"99:83:"$( : IP identification - )"40:00:"$( : IP flags + frag off - )"40:"$( : IP TTL - )"01:"$( : IP proto - )"00:00:"$( : IP header csum - )"c0:00:02:03:"$( : IP saddr: 192.0.2.3 - )"c0:00:02:01:"$( : IP daddr: 192.0.2.1 + )"$ipv4hdr:"$( : IPv4 header )"08:"$( : ICMP type )"00:"$( : ICMP code )"8b:f2:"$( : ICMP csum diff --git a/tools/testing/selftests/net/forwarding/vxlan_bridge_1d_ipv6.sh b/tools/testing/selftests/net/forwarding/vxlan_bridge_1d_ipv6.sh index a603f7b0a08f0..e642feeada0e7 100755 --- a/tools/testing/selftests/net/forwarding/vxlan_bridge_1d_ipv6.sh +++ b/tools/testing/selftests/net/forwarding/vxlan_bridge_1d_ipv6.sh @@ -695,7 +695,7 @@ vxlan_encapped_ping_do() )"6"$( : IP version )"$inner_tos"$( : Traffic class )"0:00:00:"$( : Flow label - )"00:08:"$( : Payload length + )"00:03:"$( : Payload length )"3a:"$( : Next header )"04:"$( : Hop limit )"$saddr:"$( : IP saddr diff --git a/tools/testing/selftests/net/lib.sh b/tools/testing/selftests/net/lib.sh index f448bafb3f208..d0306b27fe95e 100644 --- a/tools/testing/selftests/net/lib.sh +++ b/tools/testing/selftests/net/lib.sh @@ -576,7 +576,7 @@ ip_link_has_flag() local flag=$1; shift local state=$(ip -j link show "$name" | - jq --arg flag "$flag" 'any(.[].flags.[]; . == $flag)') + jq --arg flag "$flag" 'any(.[].flags[]; . == $flag)') [[ $state == true ]] } diff --git a/tools/testing/selftests/net/mptcp/mptcp_join.sh b/tools/testing/selftests/net/mptcp/mptcp_join.sh index 9a95830005061..8990cd99f4e32 100755 --- a/tools/testing/selftests/net/mptcp/mptcp_join.sh +++ b/tools/testing/selftests/net/mptcp/mptcp_join.sh @@ -103,6 +103,24 @@ CBPF_MPTCP_SUBOPTION_ADD_ADDR="14, 6 0 0 65535, 6 0 0 0" +# IPv4: TCP hdr of 48B, a first suboption of 12B (DACK8), the RM_ADDR suboption +# generated using "nfbpf_compile '(ip[32] & 0xf0) == 0xc0 && ip[53] == 0x0c && +# (ip[66] & 0xf0) == 0x40'" +CBPF_MPTCP_SUBOPTION_RM_ADDR="13, + 48 0 0 0, + 84 0 0 240, + 21 0 9 64, + 48 0 0 32, + 84 0 0 240, + 21 0 6 192, + 48 0 0 53, + 21 0 4 12, + 48 0 0 66, + 84 0 0 240, + 21 0 1 64, + 6 0 0 65535, + 6 0 0 0" + init_partial() { capout=$(mktemp) @@ -2595,6 +2613,19 @@ remove_tests() chk_rst_nr 0 0 fi + # signal+subflow with limits, remove + if reset "remove signal+subflow with limits"; then + pm_nl_set_limits $ns1 0 0 + pm_nl_add_endpoint $ns1 10.0.2.1 flags signal,subflow + pm_nl_set_limits $ns2 0 0 + addr_nr_ns1=-1 speed=slow \ + run_tests $ns1 $ns2 10.0.1.1 + chk_join_nr 0 0 0 + chk_add_nr 1 1 + chk_rm_nr 1 0 invert + chk_rst_nr 0 0 + fi + # addresses remove if reset "remove addresses"; then pm_nl_set_limits $ns1 3 3 @@ -4067,6 +4098,14 @@ endpoint_tests() chk_subflow_nr "after no reject" 3 chk_mptcp_info subflows 2 subflows 2 + # To make sure RM_ADDR are sent over a different subflow, but + # allow the rest to quickly and cleanly close the subflow + local ipt=1 + ip netns exec "${ns2}" ${iptables} -I OUTPUT -s "10.0.1.2" \ + -p tcp -m tcp --tcp-option 30 \ + -m bpf --bytecode \ + "$CBPF_MPTCP_SUBOPTION_RM_ADDR" \ + -j DROP || ipt=0 local i for i in $(seq 3); do pm_nl_del_endpoint $ns2 1 10.0.1.2 @@ -4079,6 +4118,7 @@ endpoint_tests() chk_subflow_nr "after re-add id 0 ($i)" 3 chk_mptcp_info subflows 3 subflows 3 done + [ ${ipt} = 1 ] && ip netns exec "${ns2}" ${iptables} -D OUTPUT 1 mptcp_lib_kill_group_wait $tests_pid @@ -4138,11 +4178,20 @@ endpoint_tests() chk_mptcp_info subflows 2 subflows 2 chk_mptcp_info add_addr_signal 2 add_addr_accepted 2 + # To make sure RM_ADDR are sent over a different subflow, but + # allow the rest to quickly and cleanly close the subflow + local ipt=1 + ip netns exec "${ns1}" ${iptables} -I OUTPUT -s "10.0.1.1" \ + -p tcp -m tcp --tcp-option 30 \ + -m bpf --bytecode \ + "$CBPF_MPTCP_SUBOPTION_RM_ADDR" \ + -j DROP || ipt=0 pm_nl_del_endpoint $ns1 42 10.0.1.1 sleep 0.5 chk_subflow_nr "after delete ID 0" 2 chk_mptcp_info subflows 2 subflows 2 chk_mptcp_info add_addr_signal 2 add_addr_accepted 2 + [ ${ipt} = 1 ] && ip netns exec "${ns1}" ${iptables} -D OUTPUT 1 pm_nl_add_endpoint $ns1 10.0.1.1 id 99 flags signal wait_mpj $ns2 diff --git a/tools/testing/selftests/net/mptcp/simult_flows.sh b/tools/testing/selftests/net/mptcp/simult_flows.sh index 1903e8e84a315..76730e511f92e 100755 --- a/tools/testing/selftests/net/mptcp/simult_flows.sh +++ b/tools/testing/selftests/net/mptcp/simult_flows.sh @@ -233,10 +233,13 @@ run_test() for dev in ns2eth1 ns2eth2; do tc -n $ns2 qdisc del dev $dev root >/dev/null 2>&1 done - tc -n $ns1 qdisc add dev ns1eth1 root netem rate ${rate1}mbit $delay1 - tc -n $ns1 qdisc add dev ns1eth2 root netem rate ${rate2}mbit $delay2 - tc -n $ns2 qdisc add dev ns2eth1 root netem rate ${rate1}mbit $delay1 - tc -n $ns2 qdisc add dev ns2eth2 root netem rate ${rate2}mbit $delay2 + + # keep the queued pkts number low, or the RTT estimator will see + # increasing latency over time. + tc -n $ns1 qdisc add dev ns1eth1 root netem rate ${rate1}mbit $delay1 limit 50 + tc -n $ns1 qdisc add dev ns1eth2 root netem rate ${rate2}mbit $delay2 limit 50 + tc -n $ns2 qdisc add dev ns2eth1 root netem rate ${rate1}mbit $delay1 limit 50 + tc -n $ns2 qdisc add dev ns2eth2 root netem rate ${rate2}mbit $delay2 limit 50 # time is measured in ms, account for transfer size, aggregated link speed # and header overhead (10%) diff --git a/tools/testing/selftests/resctrl/resctrlfs.c b/tools/testing/selftests/resctrl/resctrlfs.c index 195f04c4d1586..b9c1bfb6cc029 100644 --- a/tools/testing/selftests/resctrl/resctrlfs.c +++ b/tools/testing/selftests/resctrl/resctrlfs.c @@ -243,6 +243,16 @@ int snc_nodes_per_l3_cache(void) } snc_mode = cache_cpus / node_cpus; + /* + * On some platforms (e.g. Hygon), + * cache_cpus < node_cpus, the calculated snc_mode is 0. + * + * Set snc_mode = 1 to indicate that SNC mode is not + * supported on the platform. + */ + if (!snc_mode) + snc_mode = 1; + if (snc_mode > 1) ksft_print_msg("SNC-%d mode discovered.\n", snc_mode); } diff --git a/tools/testing/selftests/ublk/kublk.h b/tools/testing/selftests/ublk/kublk.h index 1b8833a400643..39839c711c797 100644 --- a/tools/testing/selftests/ublk/kublk.h +++ b/tools/testing/selftests/ublk/kublk.h @@ -220,7 +220,7 @@ static inline __u64 build_user_data(unsigned tag, unsigned op, unsigned tgt_data, unsigned q_id, unsigned is_target_io) { /* we only have 7 bits to encode q_id */ - _Static_assert(UBLK_MAX_QUEUES_SHIFT <= 7); + _Static_assert(UBLK_MAX_QUEUES_SHIFT <= 7, "UBLK_MAX_QUEUES_SHIFT must be <= 7"); assert(!(tag >> 16) && !(op >> 8) && !(tgt_data >> 16) && !(q_id >> 7)); return tag | (op << 16) | (tgt_data << 24) | diff --git a/tools/tracing/rtla/src/actions.c b/tools/tracing/rtla/src/actions.c index 8945aee58d511..15986505b4376 100644 --- a/tools/tracing/rtla/src/actions.c +++ b/tools/tracing/rtla/src/actions.c @@ -141,6 +141,8 @@ actions_parse(struct actions *self, const char *trigger, const char *tracefn) strcpy(trigger_c, trigger); token = strtok(trigger_c, ","); + if (!token) + return -1; if (strcmp(token, "trace") == 0) type = ACTION_TRACE_OUTPUT;