diff options
Diffstat (limited to 'arch/riscv')
108 files changed, 2345 insertions, 1167 deletions
diff --git a/arch/riscv/Kbuild b/arch/riscv/Kbuild new file mode 100644 index 000000000000..d1d0aa70fdf1 --- /dev/null +++ b/arch/riscv/Kbuild @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-y += kernel/ mm/ net/ diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 59a4727ecd6c..73f029eae0cc 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -12,8 +12,6 @@ config 32BIT config RISCV def_bool y - # even on 32-bit, physical (and DMA) addresses are > 32-bits - select PHYS_ADDR_T_64BIT select OF select OF_EARLY_FLATTREE select OF_IRQ @@ -26,15 +24,20 @@ config RISCV select GENERIC_IRQ_SHOW select GENERIC_PCI_IOMAP select GENERIC_SCHED_CLOCK - select GENERIC_STRNCPY_FROM_USER - select GENERIC_STRNLEN_USER + select GENERIC_STRNCPY_FROM_USER if MMU + select GENERIC_STRNLEN_USER if MMU select GENERIC_SMP_IDLE_THREAD select GENERIC_ATOMIC64 if !64BIT + select GENERIC_IOREMAP select HAVE_ARCH_AUDITSYSCALL + select HAVE_ARCH_SECCOMP_FILTER + select HAVE_ASM_MODVERSIONS select HAVE_MEMBLOCK_NODE_MAP - select HAVE_DMA_CONTIGUOUS + select HAVE_DMA_CONTIGUOUS if MMU select HAVE_FUTEX_CMPXCHG if FUTEX select HAVE_PERF_EVENTS + select HAVE_PERF_REGS + select HAVE_PERF_USER_STACK_DUMP select HAVE_SYSCALL_TRACEPOINTS select IRQ_DOMAIN select SPARSE_IRQ @@ -47,27 +50,82 @@ config RISCV select PCI_DOMAINS_GENERIC if PCI select PCI_MSI if PCI select RISCV_TIMER + select UACCESS_MEMCPY if !MMU select GENERIC_IRQ_MULTI_HANDLER + select GENERIC_ARCH_TOPOLOGY if SMP select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_MMIOWB + select ARCH_HAS_DEBUG_VIRTUAL select HAVE_EBPF_JIT if 64BIT select EDAC_SUPPORT select ARCH_HAS_GIGANTIC_PAGE select ARCH_WANT_HUGE_PMD_SHARE if 64BIT + select SPARSEMEM_STATIC if 32BIT + select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT if MMU + select HAVE_ARCH_MMAP_RND_BITS if MMU + select ARCH_HAS_GCOV_PROFILE_ALL + select HAVE_COPY_THREAD_TLS + select HAVE_ARCH_KASAN if MMU && 64BIT + +config ARCH_MMAP_RND_BITS_MIN + default 18 if 64BIT + default 8 + +# max bits determined by the following formula: +# VA_BITS - PAGE_SHIFT - 3 +config ARCH_MMAP_RND_BITS_MAX + default 24 if 64BIT # SV39 based + default 17 + +# set if we run in machine mode, cleared if we run in supervisor mode +config RISCV_M_MODE + bool + default !MMU + +# set if we are running in S-mode and can use SBI calls +config RISCV_SBI + bool + depends on !RISCV_M_MODE + default y config MMU - def_bool y + bool "MMU-based Paged Memory Management Support" + default y + help + Select if you want MMU-based virtualised addressing space + support by paged memory management. If unsure, say 'Y'. config ZONE_DMA32 bool default y if 64BIT +config VA_BITS + int + default 32 if 32BIT + default 39 if 64BIT + +config PA_BITS + int + default 34 if 32BIT + default 56 if 64BIT + config PAGE_OFFSET hex default 0xC0000000 if 32BIT && MAXPHYSMEM_2GB + default 0x80000000 if 64BIT && !MMU default 0xffffffff80000000 if 64BIT && MAXPHYSMEM_2GB default 0xffffffe000000000 if 64BIT && MAXPHYSMEM_128GB +config ARCH_FLATMEM_ENABLE + def_bool y + +config ARCH_SPARSEMEM_ENABLE + def_bool y + select SPARSEMEM_VMEMMAP_ENABLE + +config ARCH_SELECT_MEMORY_MODEL + def_bool ARCH_SPARSEMEM_ENABLE + config ARCH_WANT_GENERAL_HUGETLB def_bool y @@ -98,7 +156,7 @@ config GENERIC_HWEIGHT def_bool y config FIX_EARLYCON_MEM - def_bool y + def_bool MMU config PGTABLE_LEVELS int @@ -123,17 +181,18 @@ config ARCH_RV32I select GENERIC_LIB_ASHRDI3 select GENERIC_LIB_LSHRDI3 select GENERIC_LIB_UCMPDI2 + select MMU config ARCH_RV64I bool "RV64I" select 64BIT - select ARCH_SUPPORTS_INT128 if GCC_VERSION >= 50000 + select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && GCC_VERSION >= 50000 select HAVE_FUNCTION_TRACER select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FTRACE_MCOUNT_RECORD - select HAVE_DYNAMIC_FTRACE - select HAVE_DYNAMIC_FTRACE_WITH_REGS - select SWIOTLB + select HAVE_DYNAMIC_FTRACE if MMU + select HAVE_DYNAMIC_FTRACE_WITH_REGS if HAVE_DYNAMIC_FTRACE + select SWIOTLB if MMU endchoice @@ -235,6 +294,19 @@ menu "Kernel features" source "kernel/Kconfig.hz" +config SECCOMP + bool "Enable seccomp to safely compute untrusted bytecode" + help + This kernel feature is useful for number crunching applications + that may need to compute untrusted bytecode during their + execution. By using pipes or other transports made available to + the process as file descriptors supporting the read/write + syscalls, it's possible to isolate those applications in + their own address space using seccomp. Once seccomp is + enabled via prctl(PR_SET_SECCOMP), it cannot be disabled + and the task is only allowed to execute a few safe syscalls + defined by each seccomp mode. + endmenu menu "Boot options" diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs index 536c0ef4aee8..d325b67d00df 100644 --- a/arch/riscv/Kconfig.socs +++ b/arch/riscv/Kconfig.socs @@ -1,13 +1,13 @@ menu "SoC selection" config SOC_SIFIVE - bool "SiFive SoCs" - select SERIAL_SIFIVE - select SERIAL_SIFIVE_CONSOLE - select CLK_SIFIVE - select CLK_SIFIVE_FU540_PRCI - select SIFIVE_PLIC - help - This enables support for SiFive SoC platform hardware. + bool "SiFive SoCs" + select SERIAL_SIFIVE if TTY + select SERIAL_SIFIVE_CONSOLE if TTY + select CLK_SIFIVE + select CLK_SIFIVE_FU540_PRCI + select SIFIVE_PLIC + help + This enables support for SiFive SoC platform hardware. endmenu diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile index 7a117be8297c..b9009a2fbaf5 100644 --- a/arch/riscv/Makefile +++ b/arch/riscv/Makefile @@ -52,7 +52,10 @@ ifeq ($(CONFIG_CMODEL_MEDANY),y) KBUILD_CFLAGS += -mcmodel=medany endif ifeq ($(CONFIG_MODULE_SECTIONS),y) - KBUILD_LDFLAGS_MODULE += -T $(srctree)/arch/riscv/kernel/module.lds + KBUILD_LDS_MODULE += $(srctree)/arch/riscv/kernel/module.lds +endif +ifeq ($(CONFIG_PERF_EVENTS),y) + KBUILD_CFLAGS += -fno-omit-frame-pointer endif KBUILD_CFLAGS_MODULE += $(call cc-option,-mno-relax) @@ -72,7 +75,7 @@ KBUILD_IMAGE := $(boot)/Image.gz head-y := arch/riscv/kernel/head.o -core-y += arch/riscv/kernel/ arch/riscv/mm/ arch/riscv/net/ +core-y += arch/riscv/ libs-y += arch/riscv/lib/ @@ -80,13 +83,18 @@ PHONY += vdso_install vdso_install: $(Q)$(MAKE) $(build)=arch/riscv/kernel/vdso $@ -all: Image.gz +ifeq ($(CONFIG_RISCV_M_MODE),y) +KBUILD_IMAGE := $(boot)/loader +else +KBUILD_IMAGE := $(boot)/Image.gz +endif +BOOT_TARGETS := Image Image.gz loader -Image: vmlinux - $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ +all: $(notdir $(KBUILD_IMAGE)) -Image.%: Image +$(BOOT_TARGETS): vmlinux $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ + @$(kecho) ' Kernel: $(boot)/$@ is ready' zinstall install: $(Q)$(MAKE) $(build)=$(boot) $@ diff --git a/arch/riscv/boot/Makefile b/arch/riscv/boot/Makefile index 0990a9fdbe5d..36db8145f9f4 100644 --- a/arch/riscv/boot/Makefile +++ b/arch/riscv/boot/Makefile @@ -16,7 +16,7 @@ OBJCOPYFLAGS_Image :=-O binary -R .note -R .note.gnu.build-id -R .comment -S -targets := Image +targets := Image loader $(obj)/Image: vmlinux FORCE $(call if_changed,objcopy) @@ -24,6 +24,23 @@ $(obj)/Image: vmlinux FORCE $(obj)/Image.gz: $(obj)/Image FORCE $(call if_changed,gzip) +$(obj)/loader.o: $(src)/loader.S $(obj)/Image + +$(obj)/loader: $(obj)/loader.o $(obj)/Image $(obj)/loader.lds FORCE + $(Q)$(LD) -T $(obj)/loader.lds -o $@ $(obj)/loader.o + +$(obj)/Image.bz2: $(obj)/Image FORCE + $(call if_changed,bzip2) + +$(obj)/Image.lz4: $(obj)/Image FORCE + $(call if_changed,lz4) + +$(obj)/Image.lzma: $(obj)/Image FORCE + $(call if_changed,lzma) + +$(obj)/Image.lzo: $(obj)/Image FORCE + $(call if_changed,lzo) + install: $(CONFIG_SHELL) $(srctree)/$(src)/install.sh $(KERNELRELEASE) \ $(obj)/Image System.map "$(INSTALL_PATH)" diff --git a/arch/riscv/boot/dts/sifive/fu540-c000.dtsi b/arch/riscv/boot/dts/sifive/fu540-c000.dtsi index 42b5ec223100..7db861053483 100644 --- a/arch/riscv/boot/dts/sifive/fu540-c000.dtsi +++ b/arch/riscv/boot/dts/sifive/fu540-c000.dtsi @@ -13,6 +13,7 @@ aliases { serial0 = &uart0; serial1 = &uart1; + ethernet0 = ð0; }; chosen { @@ -53,6 +54,7 @@ reg = <1>; riscv,isa = "rv64imafdc"; tlb-split; + next-level-cache = <&l2cache>; cpu1_intc: interrupt-controller { #interrupt-cells = <1>; compatible = "riscv,cpu-intc"; @@ -60,7 +62,6 @@ }; }; cpu2: cpu@2 { - clock-frequency = <0>; compatible = "sifive,u54-mc", "sifive,rocket0", "riscv"; d-cache-block-size = <64>; d-cache-sets = <64>; @@ -77,6 +78,7 @@ reg = <2>; riscv,isa = "rv64imafdc"; tlb-split; + next-level-cache = <&l2cache>; cpu2_intc: interrupt-controller { #interrupt-cells = <1>; compatible = "riscv,cpu-intc"; @@ -84,7 +86,6 @@ }; }; cpu3: cpu@3 { - clock-frequency = <0>; compatible = "sifive,u54-mc", "sifive,rocket0", "riscv"; d-cache-block-size = <64>; d-cache-sets = <64>; @@ -101,6 +102,7 @@ reg = <3>; riscv,isa = "rv64imafdc"; tlb-split; + next-level-cache = <&l2cache>; cpu3_intc: interrupt-controller { #interrupt-cells = <1>; compatible = "riscv,cpu-intc"; @@ -108,7 +110,6 @@ }; }; cpu4: cpu@4 { - clock-frequency = <0>; compatible = "sifive,u54-mc", "sifive,rocket0", "riscv"; d-cache-block-size = <64>; d-cache-sets = <64>; @@ -125,6 +126,7 @@ reg = <4>; riscv,isa = "rv64imafdc"; tlb-split; + next-level-cache = <&l2cache>; cpu4_intc: interrupt-controller { #interrupt-cells = <1>; compatible = "riscv,cpu-intc"; @@ -164,6 +166,13 @@ clocks = <&prci PRCI_CLK_TLCLK>; status = "disabled"; }; + dma: dma@3000000 { + compatible = "sifive,fu540-c000-pdma"; + reg = <0x0 0x3000000 0x0 0x8000>; + interrupt-parent = <&plic0>; + interrupts = <23 24 25 26 27 28 29 30>; + #dma-cells = <1>; + }; uart1: serial@10011000 { compatible = "sifive,fu540-c000-uart", "sifive,uart0"; reg = <0x0 0x10011000 0x0 0x1000>; @@ -230,6 +239,48 @@ #size-cells = <0>; status = "disabled"; }; - + pwm0: pwm@10020000 { + compatible = "sifive,fu540-c000-pwm", "sifive,pwm0"; + reg = <0x0 0x10020000 0x0 0x1000>; + interrupt-parent = <&plic0>; + interrupts = <42 43 44 45>; + clocks = <&prci PRCI_CLK_TLCLK>; + #pwm-cells = <3>; + status = "disabled"; + }; + pwm1: pwm@10021000 { + compatible = "sifive,fu540-c000-pwm", "sifive,pwm0"; + reg = <0x0 0x10021000 0x0 0x1000>; + interrupt-parent = <&plic0>; + interrupts = <46 47 48 49>; + clocks = <&prci PRCI_CLK_TLCLK>; + #pwm-cells = <3>; + status = "disabled"; + }; + l2cache: cache-controller@2010000 { + compatible = "sifive,fu540-c000-ccache", "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-sets = <1024>; + cache-size = <2097152>; + cache-unified; + interrupt-parent = <&plic0>; + interrupts = <1 2 3>; + reg = <0x0 0x2010000 0x0 0x1000>; + }; + gpio: gpio@10060000 { + compatible = "sifive,fu540-c000-gpio", "sifive,gpio0"; + interrupt-parent = <&plic0>; + interrupts = <7>, <8>, <9>, <10>, <11>, <12>, <13>, + <14>, <15>, <16>, <17>, <18>, <19>, <20>, + <21>, <22>; + reg = <0x0 0x10060000 0x0 0x1000>; + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + clocks = <&prci PRCI_CLK_TLCLK>; + status = "disabled"; + }; }; }; diff --git a/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts b/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts index 93d68cbd64fe..609198cb1163 100644 --- a/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts +++ b/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts @@ -13,6 +13,7 @@ compatible = "sifive,hifive-unleashed-a00", "sifive,fu540-c000"; chosen { + stdout-path = "serial0"; }; cpus { @@ -85,3 +86,15 @@ reg = <0>; }; }; + +&pwm0 { + status = "okay"; +}; + +&pwm1 { + status = "okay"; +}; + +&gpio { + status = "okay"; +}; diff --git a/arch/riscv/boot/loader.S b/arch/riscv/boot/loader.S new file mode 100644 index 000000000000..dcf88cf44dc1 --- /dev/null +++ b/arch/riscv/boot/loader.S @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + + .align 4 + .section .payload, "ax", %progbits + .globl _start +_start: + .incbin "arch/riscv/boot/Image" + diff --git a/arch/riscv/boot/loader.lds.S b/arch/riscv/boot/loader.lds.S new file mode 100644 index 000000000000..47a5003c2e28 --- /dev/null +++ b/arch/riscv/boot/loader.lds.S @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include <asm/page.h> + +OUTPUT_ARCH(riscv) +ENTRY(_start) + +SECTIONS +{ + . = PAGE_OFFSET; + + .payload : { + *(.payload) + . = ALIGN(8); + } +} diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig index 3efff552a261..e2ff95cb3390 100644 --- a/arch/riscv/configs/defconfig +++ b/arch/riscv/configs/defconfig @@ -29,6 +29,8 @@ CONFIG_IP_PNP_DHCP=y CONFIG_IP_PNP_BOOTP=y CONFIG_IP_PNP_RARP=y CONFIG_NETLINK_DIAG=y +CONFIG_NET_9P=y +CONFIG_NET_9P_VIRTIO=y CONFIG_PCI=y CONFIG_PCIEPORTBUS=y CONFIG_PCI_HOST_GENERIC=y @@ -39,6 +41,7 @@ CONFIG_BLK_DEV_LOOP=y CONFIG_VIRTIO_BLK=y CONFIG_BLK_DEV_SD=y CONFIG_BLK_DEV_SR=y +CONFIG_SCSI_VIRTIO=y CONFIG_ATA=y CONFIG_SATA_AHCI=y CONFIG_SATA_AHCI_PLATFORM=y @@ -54,6 +57,7 @@ CONFIG_SERIAL_8250_CONSOLE=y CONFIG_SERIAL_OF_PLATFORM=y CONFIG_SERIAL_EARLYCON_RISCV_SBI=y CONFIG_HVC_RISCV_SBI=y +CONFIG_VIRTIO_CONSOLE=y CONFIG_HW_RANDOM=y CONFIG_HW_RANDOM_VIRTIO=y CONFIG_SPI=y @@ -61,6 +65,7 @@ CONFIG_SPI_SIFIVE=y # CONFIG_PTP_1588_CLOCK is not set CONFIG_DRM=y CONFIG_DRM_RADEON=y +CONFIG_DRM_VIRTIO_GPU=y CONFIG_FRAMEBUFFER_CONSOLE=y CONFIG_USB=y CONFIG_USB_XHCI_HCD=y @@ -73,7 +78,12 @@ CONFIG_USB_STORAGE=y CONFIG_USB_UAS=y CONFIG_MMC=y CONFIG_MMC_SPI=y +CONFIG_VIRTIO_PCI=y +CONFIG_VIRTIO_BALLOON=y +CONFIG_VIRTIO_INPUT=y CONFIG_VIRTIO_MMIO=y +CONFIG_RPMSG_CHAR=y +CONFIG_RPMSG_VIRTIO=y CONFIG_EXT4_FS=y CONFIG_EXT4_FS_POSIX_ACL=y CONFIG_AUTOFS4_FS=y @@ -86,7 +96,32 @@ CONFIG_NFS_V4=y CONFIG_NFS_V4_1=y CONFIG_NFS_V4_2=y CONFIG_ROOT_NFS=y +CONFIG_9P_FS=y CONFIG_CRYPTO_USER_API_HASH=y CONFIG_CRYPTO_DEV_VIRTIO=y CONFIG_PRINTK_TIME=y +CONFIG_DEBUG_FS=y +CONFIG_DEBUG_PAGEALLOC=y +CONFIG_DEBUG_VM=y +CONFIG_DEBUG_VM_PGFLAGS=y +CONFIG_DEBUG_MEMORY_INIT=y +CONFIG_DEBUG_PER_CPU_MAPS=y +CONFIG_SOFTLOCKUP_DETECTOR=y +CONFIG_WQ_WATCHDOG=y +CONFIG_SCHED_STACK_END_CHECK=y +CONFIG_DEBUG_TIMEKEEPING=y +CONFIG_DEBUG_RT_MUTEXES=y +CONFIG_DEBUG_SPINLOCK=y +CONFIG_DEBUG_MUTEXES=y +CONFIG_DEBUG_RWSEMS=y +CONFIG_DEBUG_ATOMIC_SLEEP=y +CONFIG_STACKTRACE=y +CONFIG_DEBUG_LIST=y +CONFIG_DEBUG_PLIST=y +CONFIG_DEBUG_SG=y # CONFIG_RCU_TRACE is not set +CONFIG_RCU_EQS_DEBUG=y +CONFIG_DEBUG_BLOCK_EXT_DEVT=y +# CONFIG_FTRACE is not set +# CONFIG_RUNTIME_TESTING_MENU is not set +CONFIG_MEMTEST=y diff --git a/arch/riscv/configs/nommu_virt_defconfig b/arch/riscv/configs/nommu_virt_defconfig new file mode 100644 index 000000000000..cf74e179bf90 --- /dev/null +++ b/arch/riscv/configs/nommu_virt_defconfig @@ -0,0 +1,78 @@ +# CONFIG_CPU_ISOLATION is not set +CONFIG_LOG_BUF_SHIFT=16 +CONFIG_PRINTK_SAFE_LOG_BUF_SHIFT=12 +CONFIG_BLK_DEV_INITRD=y +# CONFIG_RD_BZIP2 is not set +# CONFIG_RD_LZMA is not set +# CONFIG_RD_XZ is not set +# CONFIG_RD_LZO is not set +# CONFIG_RD_LZ4 is not set +CONFIG_CC_OPTIMIZE_FOR_SIZE=y +CONFIG_EXPERT=y +# CONFIG_SYSFS_SYSCALL is not set +# CONFIG_FHANDLE is not set +# CONFIG_BASE_FULL is not set +# CONFIG_EPOLL is not set +# CONFIG_SIGNALFD is not set +# CONFIG_TIMERFD is not set +# CONFIG_EVENTFD is not set +# CONFIG_AIO is not set +# CONFIG_IO_URING is not set +# CONFIG_ADVISE_SYSCALLS is not set +# CONFIG_MEMBARRIER is not set +# CONFIG_KALLSYMS is not set +# CONFIG_VM_EVENT_COUNTERS is not set +# CONFIG_COMPAT_BRK is not set +CONFIG_SLOB=y +# CONFIG_SLAB_MERGE_DEFAULT is not set +# CONFIG_MMU is not set +CONFIG_MAXPHYSMEM_2GB=y +CONFIG_SMP=y +CONFIG_CMDLINE="root=/dev/vda rw earlycon=uart8250,mmio,0x10000000,115200n8 console=ttyS0" +CONFIG_CMDLINE_FORCE=y +# CONFIG_BLK_DEV_BSG is not set +CONFIG_PARTITION_ADVANCED=y +# CONFIG_MSDOS_PARTITION is not set +# CONFIG_EFI_PARTITION is not set +# CONFIG_MQ_IOSCHED_DEADLINE is not set +# CONFIG_MQ_IOSCHED_KYBER is not set +CONFIG_BINFMT_FLAT=y +# CONFIG_COREDUMP is not set +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +# CONFIG_FW_LOADER is not set +# CONFIG_ALLOW_DEV_COREDUMP is not set +CONFIG_VIRTIO_BLK=y +# CONFIG_INPUT_KEYBOARD is not set +# CONFIG_INPUT_MOUSE is not set +# CONFIG_SERIO is not set +# CONFIG_LEGACY_PTYS is not set +# CONFIG_LDISC_AUTOLOAD is not set +# CONFIG_DEVMEM is not set +CONFIG_SERIAL_8250=y +# CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_SERIAL_8250_NR_UARTS=1 +CONFIG_SERIAL_8250_RUNTIME_UARTS=1 +CONFIG_SERIAL_OF_PLATFORM=y +# CONFIG_HW_RANDOM is not set +# CONFIG_HWMON is not set +# CONFIG_LCD_CLASS_DEVICE is not set +# CONFIG_BACKLIGHT_CLASS_DEVICE is not set +# CONFIG_VGA_CONSOLE is not set +# CONFIG_HID is not set +# CONFIG_USB_SUPPORT is not set +CONFIG_VIRTIO_MMIO=y +CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y +CONFIG_SIFIVE_PLIC=y +# CONFIG_VALIDATE_FS_PARSER is not set +CONFIG_EXT2_FS=y +# CONFIG_DNOTIFY is not set +# CONFIG_INOTIFY_USER is not set +# CONFIG_MISC_FILESYSTEMS is not set +CONFIG_LSM="[]" +CONFIG_PRINTK_TIME=y +# CONFIG_SCHED_DEBUG is not set +# CONFIG_RCU_TRACE is not set +# CONFIG_FTRACE is not set +# CONFIG_RUNTIME_TESTING_MENU is not set diff --git a/arch/riscv/configs/rv32_defconfig b/arch/riscv/configs/rv32_defconfig index 7da93e494445..eb519407c841 100644 --- a/arch/riscv/configs/rv32_defconfig +++ b/arch/riscv/configs/rv32_defconfig @@ -29,6 +29,8 @@ CONFIG_IP_PNP_DHCP=y CONFIG_IP_PNP_BOOTP=y CONFIG_IP_PNP_RARP=y CONFIG_NETLINK_DIAG=y +CONFIG_NET_9P=y +CONFIG_NET_9P_VIRTIO=y CONFIG_PCI=y CONFIG_PCIEPORTBUS=y CONFIG_PCI_HOST_GENERIC=y @@ -39,6 +41,7 @@ CONFIG_BLK_DEV_LOOP=y CONFIG_VIRTIO_BLK=y CONFIG_BLK_DEV_SD=y CONFIG_BLK_DEV_SR=y +CONFIG_SCSI_VIRTIO=y CONFIG_ATA=y CONFIG_SATA_AHCI=y CONFIG_SATA_AHCI_PLATFORM=y @@ -54,11 +57,13 @@ CONFIG_SERIAL_8250_CONSOLE=y CONFIG_SERIAL_OF_PLATFORM=y CONFIG_SERIAL_EARLYCON_RISCV_SBI=y CONFIG_HVC_RISCV_SBI=y +CONFIG_VIRTIO_CONSOLE=y CONFIG_HW_RANDOM=y CONFIG_HW_RANDOM_VIRTIO=y # CONFIG_PTP_1588_CLOCK is not set CONFIG_DRM=y CONFIG_DRM_RADEON=y +CONFIG_DRM_VIRTIO_GPU=y CONFIG_FRAMEBUFFER_CONSOLE=y CONFIG_USB=y CONFIG_USB_XHCI_HCD=y @@ -69,7 +74,12 @@ CONFIG_USB_OHCI_HCD=y CONFIG_USB_OHCI_HCD_PLATFORM=y CONFIG_USB_STORAGE=y CONFIG_USB_UAS=y +CONFIG_VIRTIO_PCI=y +CONFIG_VIRTIO_BALLOON=y +CONFIG_VIRTIO_INPUT=y CONFIG_VIRTIO_MMIO=y +CONFIG_RPMSG_CHAR=y +CONFIG_RPMSG_VIRTIO=y CONFIG_SIFIVE_PLIC=y CONFIG_EXT4_FS=y CONFIG_EXT4_FS_POSIX_ACL=y @@ -83,7 +93,32 @@ CONFIG_NFS_V4=y CONFIG_NFS_V4_1=y CONFIG_NFS_V4_2=y CONFIG_ROOT_NFS=y +CONFIG_9P_FS=y CONFIG_CRYPTO_USER_API_HASH=y CONFIG_CRYPTO_DEV_VIRTIO=y CONFIG_PRINTK_TIME=y +CONFIG_DEBUG_FS=y +CONFIG_DEBUG_PAGEALLOC=y +CONFIG_DEBUG_VM=y +CONFIG_DEBUG_VM_PGFLAGS=y +CONFIG_DEBUG_MEMORY_INIT=y +CONFIG_DEBUG_PER_CPU_MAPS=y +CONFIG_SOFTLOCKUP_DETECTOR=y +CONFIG_WQ_WATCHDOG=y +CONFIG_SCHED_STACK_END_CHECK=y +CONFIG_DEBUG_TIMEKEEPING=y +CONFIG_DEBUG_RT_MUTEXES=y +CONFIG_DEBUG_SPINLOCK=y +CONFIG_DEBUG_MUTEXES=y +CONFIG_DEBUG_RWSEMS=y +CONFIG_DEBUG_ATOMIC_SLEEP=y +CONFIG_STACKTRACE=y +CONFIG_DEBUG_LIST=y +CONFIG_DEBUG_PLIST=y +CONFIG_DEBUG_SG=y # CONFIG_RCU_TRACE is not set +CONFIG_RCU_EQS_DEBUG=y +CONFIG_DEBUG_BLOCK_EXT_DEVT=y +# CONFIG_FTRACE is not set +# CONFIG_RUNTIME_TESTING_MENU is not set +CONFIG_MEMTEST=y diff --git a/arch/riscv/include/asm/Kbuild b/arch/riscv/include/asm/Kbuild index 16970f246860..ec0ca8c6ab64 100644 --- a/arch/riscv/include/asm/Kbuild +++ b/arch/riscv/include/asm/Kbuild @@ -7,7 +7,6 @@ generic-y += div64.h generic-y += extable.h generic-y += flat.h generic-y += dma.h -generic-y += dma-contiguous.h generic-y += dma-mapping.h generic-y += emergency-restart.h generic-y += exec.h @@ -22,7 +21,6 @@ generic-y += kvm_para.h generic-y += local.h generic-y += local64.h generic-y += mm-arch-hooks.h -generic-y += msi.h generic-y += percpu.h generic-y += preempt.h generic-y += sections.h diff --git a/arch/riscv/include/asm/asm-prototypes.h b/arch/riscv/include/asm/asm-prototypes.h index c9fecd120d18..27e005fca584 100644 --- a/arch/riscv/include/asm/asm-prototypes.h +++ b/arch/riscv/include/asm/asm-prototypes.h @@ -1,7 +1,12 @@ /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_RISCV_PROTOTYPES_H +#define _ASM_RISCV_PROTOTYPES_H #include <linux/ftrace.h> #include <asm-generic/asm-prototypes.h> +long long __lshrti3(long long a, int b); +long long __ashrti3(long long a, int b); +long long __ashlti3(long long a, int b); + #endif /* _ASM_RISCV_PROTOTYPES_H */ diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h index 5a02b7d50940..9c992a88d858 100644 --- a/arch/riscv/include/asm/asm.h +++ b/arch/riscv/include/asm/asm.h @@ -22,6 +22,7 @@ #define REG_L __REG_SEL(ld, lw) #define REG_S __REG_SEL(sd, sw) +#define REG_SC __REG_SEL(sc.d, sc.w) #define SZREG __REG_SEL(8, 4) #define LGREG __REG_SEL(3, 2) diff --git a/arch/riscv/include/asm/bug.h b/arch/riscv/include/asm/bug.h index 07ceee8b1747..75604fec1b1b 100644 --- a/arch/riscv/include/asm/bug.h +++ b/arch/riscv/include/asm/bug.h @@ -12,7 +12,6 @@ #include <asm/asm.h> -#ifdef CONFIG_GENERIC_BUG #define __INSN_LENGTH_MASK _UL(0x3) #define __INSN_LENGTH_32 _UL(0x3) #define __COMPRESSED_INSN_MASK _UL(0xffff) @@ -20,7 +19,6 @@ #define __BUG_INSN_32 _UL(0x00100073) /* ebreak */ #define __BUG_INSN_16 _UL(0x9002) /* c.ebreak */ -#ifndef __ASSEMBLY__ typedef u32 bug_insn_t; #ifdef CONFIG_GENERIC_BUG_RELATIVE_POINTERS @@ -43,6 +41,7 @@ typedef u32 bug_insn_t; RISCV_SHORT " %2" #endif +#ifdef CONFIG_GENERIC_BUG #define __BUG_FLAGS(flags) \ do { \ __asm__ __volatile__ ( \ @@ -58,14 +57,10 @@ do { \ "i" (flags), \ "i" (sizeof(struct bug_entry))); \ } while (0) - -#endif /* !__ASSEMBLY__ */ #else /* CONFIG_GENERIC_BUG */ -#ifndef __ASSEMBLY__ #define __BUG_FLAGS(flags) do { \ __asm__ __volatile__ ("ebreak\n"); \ } while (0) -#endif /* !__ASSEMBLY__ */ #endif /* CONFIG_GENERIC_BUG */ #define BUG() do { \ @@ -79,15 +74,10 @@ do { \ #include <asm-generic/bug.h> -#ifndef __ASSEMBLY__ - struct pt_regs; struct task_struct; -extern void die(struct pt_regs *regs, const char *str); -extern void do_trap(struct pt_regs *regs, int signo, int code, - unsigned long addr); - -#endif /* !__ASSEMBLY__ */ +void die(struct pt_regs *regs, const char *str); +void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr); #endif /* _ASM_RISCV_BUG_H */ diff --git a/arch/riscv/include/asm/cache.h b/arch/riscv/include/asm/cache.h index bfd523e8f0b2..9b58b104559e 100644 --- a/arch/riscv/include/asm/cache.h +++ b/arch/riscv/include/asm/cache.h @@ -11,4 +11,12 @@ #define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) +/* + * RISC-V requires the stack pointer to be 16-byte aligned, so ensure that + * the flat loader aligns it accordingly. + */ +#ifndef CONFIG_MMU +#define ARCH_SLAB_MINALIGN 16 +#endif + #endif /* _ASM_RISCV_CACHE_H */ diff --git a/arch/riscv/include/asm/clint.h b/arch/riscv/include/asm/clint.h new file mode 100644 index 000000000000..6eaa2eedd694 --- /dev/null +++ b/arch/riscv/include/asm/clint.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_RISCV_CLINT_H +#define _ASM_RISCV_CLINT_H 1 + +#include <linux/io.h> +#include <linux/smp.h> + +#ifdef CONFIG_RISCV_M_MODE +extern u32 __iomem *clint_ipi_base; + +void clint_init_boot_cpu(void); + +static inline void clint_send_ipi_single(unsigned long hartid) +{ + writel(1, clint_ipi_base + hartid); +} + +static inline void clint_send_ipi_mask(const struct cpumask *hartid_mask) +{ + int hartid; + + for_each_cpu(hartid, hartid_mask) + clint_send_ipi_single(hartid); +} + +static inline void clint_clear_ipi(unsigned long hartid) +{ + writel(0, clint_ipi_base + hartid); +} +#else /* CONFIG_RISCV_M_MODE */ +#define clint_init_boot_cpu() do { } while (0) + +/* stubs to for code is only reachable under IS_ENABLED(CONFIG_RISCV_M_MODE): */ +void clint_send_ipi_single(unsigned long hartid); +void clint_send_ipi_mask(const struct cpumask *hartid_mask); +void clint_clear_ipi(unsigned long hartid); +#endif /* CONFIG_RISCV_M_MODE */ + +#endif /* _ASM_RISCV_CLINT_H */ diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h index a18923fa23c8..435b65532e29 100644 --- a/arch/riscv/include/asm/csr.h +++ b/arch/riscv/include/asm/csr.h @@ -11,8 +11,11 @@ /* Status register flags */ #define SR_SIE _AC(0x00000002, UL) /* Supervisor Interrupt Enable */ +#define SR_MIE _AC(0x00000008, UL) /* Machine Interrupt Enable */ #define SR_SPIE _AC(0x00000020, UL) /* Previous Supervisor IE */ +#define SR_MPIE _AC(0x00000080, UL) /* Previous Machine IE */ #define SR_SPP _AC(0x00000100, UL) /* Previously Supervisor */ +#define SR_MPP _AC(0x00001800, UL) /* Previously Machine */ #define SR_SUM _AC(0x00040000, UL) /* Supervisor User Memory Access */ #define SR_FS _AC(0x00006000, UL) /* Floating-point Status */ @@ -44,9 +47,10 @@ #define SATP_MODE SATP_MODE_39 #endif -/* SCAUSE */ -#define SCAUSE_IRQ_FLAG (_AC(1, UL) << (__riscv_xlen - 1)) +/* Exception cause high bit - is an interrupt if set */ +#define CAUSE_IRQ_FLAG (_AC(1, UL) << (__riscv_xlen - 1)) +/* Interrupt causes (minus the high bit) */ #define IRQ_U_SOFT 0 #define IRQ_S_SOFT 1 #define IRQ_M_SOFT 3 @@ -57,6 +61,7 @@ #define IRQ_S_EXT 9 #define IRQ_M_EXT 11 +/* Exception causes */ #define EXC_INST_MISALIGNED 0 #define EXC_INST_ACCESS 1 #define EXC_BREAKPOINT 3 @@ -67,14 +72,14 @@ #define EXC_LOAD_PAGE_FAULT 13 #define EXC_STORE_PAGE_FAULT 15 -/* SIE (Interrupt Enable) and SIP (Interrupt Pending) flags */ -#define SIE_SSIE (_AC(0x1, UL) << IRQ_S_SOFT) -#define SIE_STIE (_AC(0x1, UL) << IRQ_S_TIMER) -#define SIE_SEIE (_AC(0x1, UL) << IRQ_S_EXT) - +/* symbolic CSR names: */ #define CSR_CYCLE 0xc00 #define CSR_TIME 0xc01 #define CSR_INSTRET 0xc02 +#define CSR_CYCLEH 0xc80 +#define CSR_TIMEH 0xc81 +#define CSR_INSTRETH 0xc82 + #define CSR_SSTATUS 0x100 #define CSR_SIE 0x104 #define CSR_STVEC 0x105 @@ -85,9 +90,58 @@ #define CSR_STVAL 0x143 #define CSR_SIP 0x144 #define CSR_SATP 0x180 -#define CSR_CYCLEH 0xc80 -#define CSR_TIMEH 0xc81 -#define CSR_INSTRETH 0xc82 + +#define CSR_MSTATUS 0x300 +#define CSR_MISA 0x301 +#define CSR_MIE 0x304 +#define CSR_MTVEC 0x305 +#define CSR_MSCRATCH 0x340 +#define CSR_MEPC 0x341 +#define CSR_MCAUSE 0x342 +#define CSR_MTVAL 0x343 +#define CSR_MIP 0x344 +#define CSR_MHARTID 0xf14 + +#ifdef CONFIG_RISCV_M_MODE +# define CSR_STATUS CSR_MSTATUS +# define CSR_IE CSR_MIE +# define CSR_TVEC CSR_MTVEC +# define CSR_SCRATCH CSR_MSCRATCH +# define CSR_EPC CSR_MEPC +# define CSR_CAUSE CSR_MCAUSE +# define CSR_TVAL CSR_MTVAL +# define CSR_IP CSR_MIP + +# define SR_IE SR_MIE +# define SR_PIE SR_MPIE +# define SR_PP SR_MPP + +# define RV_IRQ_SOFT IRQ_M_SOFT +# define RV_IRQ_TIMER IRQ_M_TIMER +# define RV_IRQ_EXT IRQ_M_EXT +#else /* CONFIG_RISCV_M_MODE */ +# define CSR_STATUS CSR_SSTATUS +# define CSR_IE CSR_SIE +# define CSR_TVEC CSR_STVEC +# define CSR_SCRATCH CSR_SSCRATCH +# define CSR_EPC CSR_SEPC +# define CSR_CAUSE CSR_SCAUSE +# define CSR_TVAL CSR_STVAL +# define CSR_IP CSR_SIP + +# define SR_IE SR_SIE +# define SR_PIE SR_SPIE +# define SR_PP SR_SPP + +# define RV_IRQ_SOFT IRQ_S_SOFT +# define RV_IRQ_TIMER IRQ_S_TIMER +# define RV_IRQ_EXT IRQ_S_EXT +#endif /* CONFIG_RISCV_M_MODE */ + +/* IE/IP (Supervisor/Machine Interrupt Enable/Pending) flags */ +#define IE_SIE (_AC(0x1, UL) << RV_IRQ_SOFT) +#define IE_TIE (_AC(0x1, UL) << RV_IRQ_TIMER) +#define IE_EIE (_AC(0x1, UL) << RV_IRQ_EXT) #ifndef __ASSEMBLY__ diff --git a/arch/riscv/include/asm/current.h b/arch/riscv/include/asm/current.h index 44dcf7fc15ee..dd973efe5d7c 100644 --- a/arch/riscv/include/asm/current.h +++ b/arch/riscv/include/asm/current.h @@ -7,8 +7,8 @@ */ -#ifndef __ASM_CURRENT_H -#define __ASM_CURRENT_H +#ifndef _ASM_RISCV_CURRENT_H +#define _ASM_RISCV_CURRENT_H #include <linux/bug.h> #include <linux/compiler.h> @@ -34,4 +34,4 @@ static __always_inline struct task_struct *get_current(void) #endif /* __ASSEMBLY__ */ -#endif /* __ASM_CURRENT_H */ +#endif /* _ASM_RISCV_CURRENT_H */ diff --git a/arch/riscv/include/asm/elf.h b/arch/riscv/include/asm/elf.h index ef04084bf0de..d83a4efd052b 100644 --- a/arch/riscv/include/asm/elf.h +++ b/arch/riscv/include/asm/elf.h @@ -56,16 +56,16 @@ extern unsigned long elf_hwcap; */ #define ELF_PLATFORM (NULL) +#ifdef CONFIG_MMU #define ARCH_DLINFO \ do { \ NEW_AUX_ENT(AT_SYSINFO_EHDR, \ (elf_addr_t)current->mm->context.vdso); \ } while (0) - - #define ARCH_HAS_SETUP_ADDITIONAL_PAGES struct linux_binprm; extern int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp); +#endif /* CONFIG_MMU */ #endif /* _ASM_RISCV_ELF_H */ diff --git a/arch/riscv/include/asm/fixmap.h b/arch/riscv/include/asm/fixmap.h index 9c66033c3a54..42d2c42f3cc9 100644 --- a/arch/riscv/include/asm/fixmap.h +++ b/arch/riscv/include/asm/fixmap.h @@ -11,6 +11,7 @@ #include <asm/page.h> #include <asm/pgtable.h> +#ifdef CONFIG_MMU /* * Here we define all the compile-time 'special' virtual addresses. * The point is to have a constant address at compile time, but to @@ -30,10 +31,6 @@ enum fixed_addresses { __end_of_fixed_addresses }; -#define FIXADDR_SIZE (__end_of_fixed_addresses * PAGE_SIZE) -#define FIXADDR_TOP (VMALLOC_START) -#define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE) - #define FIXMAP_PAGE_IO PAGE_KERNEL #define __early_set_fixmap __set_fixmap @@ -46,4 +43,5 @@ extern void __set_fixmap(enum fixed_addresses idx, #include <asm-generic/fixmap.h> +#endif /* CONFIG_MMU */ #endif /* _ASM_RISCV_FIXMAP_H */ diff --git a/arch/riscv/include/asm/ftrace.h b/arch/riscv/include/asm/ftrace.h index c6dcc5291f97..ace8a6e2d11d 100644 --- a/arch/riscv/include/asm/ftrace.h +++ b/arch/riscv/include/asm/ftrace.h @@ -1,6 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (C) 2017 Andes Technology Corporation */ +#ifndef _ASM_RISCV_FTRACE_H +#define _ASM_RISCV_FTRACE_H + /* * The graph frame test is not possible if CONFIG_FRAME_POINTER is not enabled. * Check arch/riscv/kernel/mcount.S for detail. @@ -64,3 +67,5 @@ do { \ */ #define MCOUNT_INSN_SIZE 8 #endif + +#endif /* _ASM_RISCV_FTRACE_H */ diff --git a/arch/riscv/include/asm/futex.h b/arch/riscv/include/asm/futex.h index 4ad6409c4647..fdfaf7f3df7c 100644 --- a/arch/riscv/include/asm/futex.h +++ b/arch/riscv/include/asm/futex.h @@ -4,14 +4,20 @@ * Copyright (c) 2018 Jim Wilson (jimw@sifive.com) */ -#ifndef _ASM_FUTEX_H -#define _ASM_FUTEX_H +#ifndef _ASM_RISCV_FUTEX_H +#define _ASM_RISCV_FUTEX_H #include <linux/futex.h> #include <linux/uaccess.h> #include <linux/errno.h> #include <asm/asm.h> +/* We don't even really need the extable code, but for now keep it simple */ +#ifndef CONFIG_MMU +#define __enable_user_access() do { } while (0) +#define __disable_user_access() do { } while (0) +#endif + #define __futex_atomic_op(insn, ret, oldval, uaddr, oparg) \ { \ uintptr_t tmp; \ @@ -112,4 +118,4 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, return ret; } -#endif /* _ASM_FUTEX_H */ +#endif /* _ASM_RISCV_FUTEX_H */ diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index 7ecb7c6a57b1..1bb0cd04aec3 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -5,8 +5,8 @@ * Copyright (C) 2012 ARM Ltd. * Copyright (C) 2017 SiFive */ -#ifndef __ASM_HWCAP_H -#define __ASM_HWCAP_H +#ifndef _ASM_RISCV_HWCAP_H +#define _ASM_RISCV_HWCAP_H #include <uapi/asm/hwcap.h> @@ -23,4 +23,5 @@ enum { extern unsigned long elf_hwcap; #endif -#endif + +#endif /* _ASM_RISCV_HWCAP_H */ diff --git a/arch/riscv/include/asm/image.h b/arch/riscv/include/asm/image.h index ef28e106f247..e0b319af3681 100644 --- a/arch/riscv/include/asm/image.h +++ b/arch/riscv/include/asm/image.h @@ -1,9 +1,10 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __ASM_IMAGE_H -#define __ASM_IMAGE_H +#ifndef _ASM_RISCV_IMAGE_H +#define _ASM_RISCV_IMAGE_H -#define RISCV_IMAGE_MAGIC "RISCV" +#define RISCV_IMAGE_MAGIC "RISCV\0\0\0" +#define RISCV_IMAGE_MAGIC2 "RSC\x05" #define RISCV_IMAGE_FLAG_BE_SHIFT 0 #define RISCV_IMAGE_FLAG_BE_MASK 0x1 @@ -23,7 +24,7 @@ #define __HEAD_FLAGS (__HEAD_FLAG(BE)) #define RISCV_HEADER_VERSION_MAJOR 0 -#define RISCV_HEADER_VERSION_MINOR 1 +#define RISCV_HEADER_VERSION_MINOR 2 #define RISCV_HEADER_VERSION (RISCV_HEADER_VERSION_MAJOR << 16 | \ RISCV_HEADER_VERSION_MINOR) @@ -39,10 +40,9 @@ * @version: version * @res1: reserved * @res2: reserved - * @magic: Magic number - * @res3: reserved (will be used for additional RISC-V specific - * header) - * @res4: reserved (will be used for PE COFF offset) + * @magic: Magic number (RISC-V specific; deprecated) + * @magic2: Magic number 2 (to match the ARM64 'magic' field pos) + * @res3: reserved (will be used for PE COFF offset) * * The intention is for this header format to be shared between multiple * architectures to avoid a proliferation of image header formats. @@ -58,8 +58,8 @@ struct riscv_image_header { u32 res1; u64 res2; u64 magic; + u32 magic2; u32 res3; - u32 res4; }; #endif /* __ASSEMBLY__ */ -#endif /* __ASM_IMAGE_H */ +#endif /* _ASM_RISCV_IMAGE_H */ diff --git a/arch/riscv/include/asm/io.h b/arch/riscv/include/asm/io.h index fc1189ad3777..0f477206a4ed 100644 --- a/arch/riscv/include/asm/io.h +++ b/arch/riscv/include/asm/io.h @@ -13,153 +13,21 @@ #include <linux/types.h> #include <asm/mmiowb.h> - -extern void __iomem *ioremap(phys_addr_t offset, unsigned long size); - -/* - * The RISC-V ISA doesn't yet specify how to query or modify PMAs, so we can't - * change the properties of memory regions. This should be fixed by the - * upcoming platform spec. - */ -#define ioremap_nocache(addr, size) ioremap((addr), (size)) -#define ioremap_wc(addr, size) ioremap((addr), (size)) -#define ioremap_wt(addr, size) ioremap((addr), (size)) - -extern void iounmap(volatile void __iomem *addr); - -/* Generic IO read/write. These perform native-endian accesses. */ -#define __raw_writeb __raw_writeb -static inline void __raw_writeb(u8 val, volatile void __iomem *addr) -{ - asm volatile("sb %0, 0(%1)" : : "r" (val), "r" (addr)); -} - -#define __raw_writew __raw_writew -static inline void __raw_writew(u16 val, volatile void __iomem *addr) -{ - asm volatile("sh %0, 0(%1)" : : "r" (val), "r" (addr)); -} - -#define __raw_writel __raw_writel -static inline void __raw_writel(u32 val, volatile void __iomem *addr) -{ - asm volatile("sw %0, 0(%1)" : : "r" (val), "r" (addr)); -} - -#ifdef CONFIG_64BIT -#define __raw_writeq __raw_writeq -static inline void __raw_writeq(u64 val, volatile void __iomem *addr) -{ - asm volatile("sd %0, 0(%1)" : : "r" (val), "r" (addr)); -} -#endif - -#define __raw_readb __raw_readb -static inline u8 __raw_readb(const volatile void __iomem *addr) -{ - u8 val; - - asm volatile("lb %0, 0(%1)" : "=r" (val) : "r" (addr)); - return val; -} - -#define __raw_readw __raw_readw -static inline u16 __raw_readw(const volatile void __iomem *addr) -{ - u16 val; - - asm volatile("lh %0, 0(%1)" : "=r" (val) : "r" (addr)); - return val; -} - -#define __raw_readl __raw_readl -static inline u32 __raw_readl(const volatile void __iomem *addr) -{ - u32 val; - - asm volatile("lw %0, 0(%1)" : "=r" (val) : "r" (addr)); - return val; -} - -#ifdef CONFIG_64BIT -#define __raw_readq __raw_readq -static inline u64 __raw_readq(const volatile void __iomem *addr) -{ - u64 val; - - asm volatile("ld %0, 0(%1)" : "=r" (val) : "r" (addr)); - return val; -} -#endif +#include <asm/pgtable.h> /* - * Unordered I/O memory access primitives. These are even more relaxed than - * the relaxed versions, as they don't even order accesses between successive - * operations to the I/O regions. + * MMIO access functions are separated out to break dependency cycles + * when using {read,write}* fns in low-level headers */ -#define readb_cpu(c) ({ u8 __r = __raw_readb(c); __r; }) -#define readw_cpu(c) ({ u16 __r = le16_to_cpu((__force __le16)__raw_readw(c)); __r; }) -#define readl_cpu(c) ({ u32 __r = le32_to_cpu((__force __le32)__raw_readl(c)); __r; }) - -#define writeb_cpu(v,c) ((void)__raw_writeb((v),(c))) -#define writew_cpu(v,c) ((void)__raw_writew((__force u16)cpu_to_le16(v),(c))) -#define writel_cpu(v,c) ((void)__raw_writel((__force u32)cpu_to_le32(v),(c))) - -#ifdef CONFIG_64BIT -#define readq_cpu(c) ({ u64 __r = le64_to_cpu((__force __le64)__raw_readq(c)); __r; }) -#define writeq_cpu(v,c) ((void)__raw_writeq((__force u64)cpu_to_le64(v),(c))) -#endif +#include <asm/mmio.h> /* - * Relaxed I/O memory access primitives. These follow the Device memory - * ordering rules but do not guarantee any ordering relative to Normal memory - * accesses. These are defined to order the indicated access (either a read or - * write) with all other I/O memory accesses. Since the platform specification - * defines that all I/O regions are strongly ordered on channel 2, no explicit - * fences are required to enforce this ordering. + * I/O port access constants. */ -/* FIXME: These are now the same as asm-generic */ -#define __io_rbr() do {} while (0) -#define __io_rar() do {} while (0) -#define __io_rbw() do {} while (0) -#define __io_raw() do {} while (0) - -#define readb_relaxed(c) ({ u8 __v; __io_rbr(); __v = readb_cpu(c); __io_rar(); __v; }) -#define readw_relaxed(c) ({ u16 __v; __io_rbr(); __v = readw_cpu(c); __io_rar(); __v; }) -#define readl_relaxed(c) ({ u32 __v; __io_rbr(); __v = readl_cpu(c); __io_rar(); __v; }) - -#define writeb_relaxed(v,c) ({ __io_rbw(); writeb_cpu((v),(c)); __io_raw(); }) -#define writew_relaxed(v,c) ({ __io_rbw(); writew_cpu((v),(c)); __io_raw(); }) -#define writel_relaxed(v,c) ({ __io_rbw(); writel_cpu((v),(c)); __io_raw(); }) - -#ifdef CONFIG_64BIT -#define readq_relaxed(c) ({ u64 __v; __io_rbr(); __v = readq_cpu(c); __io_rar(); __v; }) -#define writeq_relaxed(v,c) ({ __io_rbw(); writeq_cpu((v),(c)); __io_raw(); }) -#endif - -/* - * I/O memory access primitives. Reads are ordered relative to any - * following Normal memory access. Writes are ordered relative to any prior - * Normal memory access. The memory barriers here are necessary as RISC-V - * doesn't define any ordering between the memory space and the I/O space. - */ -#define __io_br() do {} while (0) -#define __io_ar(v) __asm__ __volatile__ ("fence i,r" : : : "memory"); -#define __io_bw() __asm__ __volatile__ ("fence w,o" : : : "memory"); -#define __io_aw() mmiowb_set_pending() - -#define readb(c) ({ u8 __v; __io_br(); __v = readb_cpu(c); __io_ar(__v); __v; }) -#define readw(c) ({ u16 __v; __io_br(); __v = readw_cpu(c); __io_ar(__v); __v; }) -#define readl(c) ({ u32 __v; __io_br(); __v = readl_cpu(c); __io_ar(__v); __v; }) - -#define writeb(v,c) ({ __io_bw(); writeb_cpu((v),(c)); __io_aw(); }) -#define writew(v,c) ({ __io_bw(); writew_cpu((v),(c)); __io_aw(); }) -#define writel(v,c) ({ __io_bw(); writel_cpu((v),(c)); __io_aw(); }) - -#ifdef CONFIG_64BIT -#define readq(c) ({ u64 __v; __io_br(); __v = readq_cpu(c); __io_ar(__v); __v; }) -#define writeq(v,c) ({ __io_bw(); writeq_cpu((v),(c)); __io_aw(); }) -#endif +#ifdef CONFIG_MMU +#define IO_SPACE_LIMIT (PCI_IO_SIZE - 1) +#define PCI_IOBASE ((void __iomem *)PCI_IO_START) +#endif /* CONFIG_MMU */ /* * Emulation routines for the port-mapped IO space used by some PCI drivers. diff --git a/arch/riscv/include/asm/irq.h b/arch/riscv/include/asm/irq.h index 75576424c0f7..6e1b0e0325eb 100644 --- a/arch/riscv/include/asm/irq.h +++ b/arch/riscv/include/asm/irq.h @@ -7,6 +7,9 @@ #ifndef _ASM_RISCV_IRQ_H #define _ASM_RISCV_IRQ_H +#include <linux/interrupt.h> +#include <linux/linkage.h> + #define NR_IRQS 0 void riscv_timer_interrupt(void); diff --git a/arch/riscv/include/asm/irqflags.h b/arch/riscv/include/asm/irqflags.h index e70f647ce3b7..08d4d6a5b7e9 100644 --- a/arch/riscv/include/asm/irqflags.h +++ b/arch/riscv/include/asm/irqflags.h @@ -13,31 +13,31 @@ /* read interrupt enabled status */ static inline unsigned long arch_local_save_flags(void) { - return csr_read(CSR_SSTATUS); + return csr_read(CSR_STATUS); } /* unconditionally enable interrupts */ static inline void arch_local_irq_enable(void) { - csr_set(CSR_SSTATUS, SR_SIE); + csr_set(CSR_STATUS, SR_IE); } /* unconditionally disable interrupts */ static inline void arch_local_irq_disable(void) { - csr_clear(CSR_SSTATUS, SR_SIE); + csr_clear(CSR_STATUS, SR_IE); } /* get status and disable interrupts */ static inline unsigned long arch_local_irq_save(void) { - return csr_read_clear(CSR_SSTATUS, SR_SIE); + return csr_read_clear(CSR_STATUS, SR_IE); } /* test flags */ static inline int arch_irqs_disabled_flags(unsigned long flags) { - return !(flags & SR_SIE); + return !(flags & SR_IE); } /* test hardware interrupt enable bit */ @@ -49,7 +49,7 @@ static inline int arch_irqs_disabled(void) /* set interrupt enabled status */ static inline void arch_local_irq_restore(unsigned long flags) { - csr_set(CSR_SSTATUS, flags & SR_SIE); + csr_set(CSR_STATUS, flags & SR_IE); } #endif /* _ASM_RISCV_IRQFLAGS_H */ diff --git a/arch/riscv/include/asm/kasan.h b/arch/riscv/include/asm/kasan.h new file mode 100644 index 000000000000..eee6e6588b12 --- /dev/null +++ b/arch/riscv/include/asm/kasan.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2019 Andes Technology Corporation */ + +#ifndef __ASM_KASAN_H +#define __ASM_KASAN_H + +#ifndef __ASSEMBLY__ + +#ifdef CONFIG_KASAN + +#include <asm/pgtable.h> + +#define KASAN_SHADOW_SCALE_SHIFT 3 + +#define KASAN_SHADOW_SIZE (UL(1) << (38 - KASAN_SHADOW_SCALE_SHIFT)) +#define KASAN_SHADOW_START 0xffffffc000000000 /* 2^64 - 2^38 */ +#define KASAN_SHADOW_END (KASAN_SHADOW_START + KASAN_SHADOW_SIZE) + +#define KASAN_SHADOW_OFFSET (KASAN_SHADOW_END - (1ULL << \ + (64 - KASAN_SHADOW_SCALE_SHIFT))) + +void kasan_init(void); +asmlinkage void kasan_early_init(void); + +#endif +#endif +#endif /* __ASM_KASAN_H */ diff --git a/arch/riscv/include/asm/kprobes.h b/arch/riscv/include/asm/kprobes.h index 96e30ef637e8..56a98ea30731 100644 --- a/arch/riscv/include/asm/kprobes.h +++ b/arch/riscv/include/asm/kprobes.h @@ -6,9 +6,9 @@ * Copyright (C) 2017 SiFive */ -#ifndef _RISCV_KPROBES_H -#define _RISCV_KPROBES_H +#ifndef _ASM_RISCV_KPROBES_H +#define _ASM_RISCV_KPROBES_H #include <asm-generic/kprobes.h> -#endif /* _RISCV_KPROBES_H */ +#endif /* _ASM_RISCV_KPROBES_H */ diff --git a/arch/riscv/include/asm/mmio.h b/arch/riscv/include/asm/mmio.h new file mode 100644 index 000000000000..a2c809df2733 --- /dev/null +++ b/arch/riscv/include/asm/mmio.h @@ -0,0 +1,155 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * {read,write}{b,w,l,q} based on arch/arm64/include/asm/io.h + * which was based on arch/arm/include/io.h + * + * Copyright (C) 1996-2000 Russell King + * Copyright (C) 2012 ARM Ltd. + * Copyright (C) 2014 Regents of the University of California + */ + +#ifndef _ASM_RISCV_MMIO_H +#define _ASM_RISCV_MMIO_H + +#include <linux/types.h> +#include <asm/mmiowb.h> + +#ifndef CONFIG_MMU +#define pgprot_noncached(x) (x) +#endif /* CONFIG_MMU */ + +/* Generic IO read/write. These perform native-endian accesses. */ +#define __raw_writeb __raw_writeb +static inline void __raw_writeb(u8 val, volatile void __iomem *addr) +{ + asm volatile("sb %0, 0(%1)" : : "r" (val), "r" (addr)); +} + +#define __raw_writew __raw_writew +static inline void __raw_writew(u16 val, volatile void __iomem *addr) +{ + asm volatile("sh %0, 0(%1)" : : "r" (val), "r" (addr)); +} + +#define __raw_writel __raw_writel +static inline void __raw_writel(u32 val, volatile void __iomem *addr) +{ + asm volatile("sw %0, 0(%1)" : : "r" (val), "r" (addr)); +} + +#ifdef CONFIG_64BIT +#define __raw_writeq __raw_writeq +static inline void __raw_writeq(u64 val, volatile void __iomem *addr) +{ + asm volatile("sd %0, 0(%1)" : : "r" (val), "r" (addr)); +} +#endif + +#define __raw_readb __raw_readb +static inline u8 __raw_readb(const volatile void __iomem *addr) +{ + u8 val; + + asm volatile("lb %0, 0(%1)" : "=r" (val) : "r" (addr)); + return val; +} + +#define __raw_readw __raw_readw +static inline u16 __raw_readw(const volatile void __iomem *addr) +{ + u16 val; + + asm volatile("lh %0, 0(%1)" : "=r" (val) : "r" (addr)); + return val; +} + +#define __raw_readl __raw_readl +static inline u32 __raw_readl(const volatile void __iomem *addr) +{ + u32 val; + + asm volatile("lw %0, 0(%1)" : "=r" (val) : "r" (addr)); + return val; +} + +#ifdef CONFIG_64BIT +#define __raw_readq __raw_readq +static inline u64 __raw_readq(const volatile void __iomem *addr) +{ + u64 val; + + asm volatile("ld %0, 0(%1)" : "=r" (val) : "r" (addr)); + return val; +} +#endif + +/* + * Unordered I/O memory access primitives. These are even more relaxed than + * the relaxed versions, as they don't even order accesses between successive + * operations to the I/O regions. + */ +#define readb_cpu(c) ({ u8 __r = __raw_readb(c); __r; }) +#define readw_cpu(c) ({ u16 __r = le16_to_cpu((__force __le16)__raw_readw(c)); __r; }) +#define readl_cpu(c) ({ u32 __r = le32_to_cpu((__force __le32)__raw_readl(c)); __r; }) + +#define writeb_cpu(v, c) ((void)__raw_writeb((v), (c))) +#define writew_cpu(v, c) ((void)__raw_writew((__force u16)cpu_to_le16(v), (c))) +#define writel_cpu(v, c) ((void)__raw_writel((__force u32)cpu_to_le32(v), (c))) + +#ifdef CONFIG_64BIT +#define readq_cpu(c) ({ u64 __r = le64_to_cpu((__force __le64)__raw_readq(c)); __r; }) +#define writeq_cpu(v, c) ((void)__raw_writeq((__force u64)cpu_to_le64(v), (c))) +#endif + +/* + * Relaxed I/O memory access primitives. These follow the Device memory + * ordering rules but do not guarantee any ordering relative to Normal memory + * accesses. These are defined to order the indicated access (either a read or + * write) with all other I/O memory accesses. Since the platform specification + * defines that all I/O regions are strongly ordered on channel 2, no explicit + * fences are required to enforce this ordering. + */ +/* FIXME: These are now the same as asm-generic */ +#define __io_rbr() do {} while (0) +#define __io_rar() do {} while (0) +#define __io_rbw() do {} while (0) +#define __io_raw() do {} while (0) + +#define readb_relaxed(c) ({ u8 __v; __io_rbr(); __v = readb_cpu(c); __io_rar(); __v; }) +#define readw_relaxed(c) ({ u16 __v; __io_rbr(); __v = readw_cpu(c); __io_rar(); __v; }) +#define readl_relaxed(c) ({ u32 __v; __io_rbr(); __v = readl_cpu(c); __io_rar(); __v; }) + +#define writeb_relaxed(v, c) ({ __io_rbw(); writeb_cpu((v), (c)); __io_raw(); }) +#define writew_relaxed(v, c) ({ __io_rbw(); writew_cpu((v), (c)); __io_raw(); }) +#define writel_relaxed(v, c) ({ __io_rbw(); writel_cpu((v), (c)); __io_raw(); }) + +#ifdef CONFIG_64BIT +#define readq_relaxed(c) ({ u64 __v; __io_rbr(); __v = readq_cpu(c); __io_rar(); __v; }) +#define writeq_relaxed(v, c) ({ __io_rbw(); writeq_cpu((v), (c)); __io_raw(); }) +#endif + +/* + * I/O memory access primitives. Reads are ordered relative to any + * following Normal memory access. Writes are ordered relative to any prior + * Normal memory access. The memory barriers here are necessary as RISC-V + * doesn't define any ordering between the memory space and the I/O space. + */ +#define __io_br() do {} while (0) +#define __io_ar(v) __asm__ __volatile__ ("fence i,r" : : : "memory") +#define __io_bw() __asm__ __volatile__ ("fence w,o" : : : "memory") +#define __io_aw() mmiowb_set_pending() + +#define readb(c) ({ u8 __v; __io_br(); __v = readb_cpu(c); __io_ar(__v); __v; }) +#define readw(c) ({ u16 __v; __io_br(); __v = readw_cpu(c); __io_ar(__v); __v; }) +#define readl(c) ({ u32 __v; __io_br(); __v = readl_cpu(c); __io_ar(__v); __v; }) + +#define writeb(v, c) ({ __io_bw(); writeb_cpu((v), (c)); __io_aw(); }) +#define writew(v, c) ({ __io_bw(); writew_cpu((v), (c)); __io_aw(); }) +#define writel(v, c) ({ __io_bw(); writel_cpu((v), (c)); __io_aw(); }) + +#ifdef CONFIG_64BIT +#define readq(c) ({ u64 __v; __io_br(); __v = readq_cpu(c); __io_ar(__v); __v; }) +#define writeq(v, c) ({ __io_bw(); writeq_cpu((v), (c)); __io_aw(); }) +#endif + +#endif /* _ASM_RISCV_MMIO_H */ diff --git a/arch/riscv/include/asm/mmiowb.h b/arch/riscv/include/asm/mmiowb.h index 5d7e3a2b4e3b..bb4091ff4a21 100644 --- a/arch/riscv/include/asm/mmiowb.h +++ b/arch/riscv/include/asm/mmiowb.h @@ -11,4 +11,4 @@ #include <asm-generic/mmiowb.h> -#endif /* ASM_RISCV_MMIOWB_H */ +#endif /* _ASM_RISCV_MMIOWB_H */ diff --git a/arch/riscv/include/asm/mmu.h b/arch/riscv/include/asm/mmu.h index 151476fb58cb..967eacb01ab5 100644 --- a/arch/riscv/include/asm/mmu.h +++ b/arch/riscv/include/asm/mmu.h @@ -10,6 +10,9 @@ #ifndef __ASSEMBLY__ typedef struct { +#ifndef CONFIG_MMU + unsigned long end_brk; +#endif void *vdso; #ifdef CONFIG_SMP /* A local icache flush is needed before user execution can resume. */ diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h index 707e00a8430b..8ca1930caa44 100644 --- a/arch/riscv/include/asm/page.h +++ b/arch/riscv/include/asm/page.h @@ -88,14 +88,32 @@ typedef struct page *pgtable_t; #define PTE_FMT "%08lx" #endif +#ifdef CONFIG_MMU extern unsigned long va_pa_offset; extern unsigned long pfn_base; +#define ARCH_PFN_OFFSET (pfn_base) +#else +#define va_pa_offset 0 +#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT) +#endif /* CONFIG_MMU */ extern unsigned long max_low_pfn; extern unsigned long min_low_pfn; -#define __pa(x) ((unsigned long)(x) - va_pa_offset) -#define __va(x) ((void *)((unsigned long) (x) + va_pa_offset)) +#define __pa_to_va_nodebug(x) ((void *)((unsigned long) (x) + va_pa_offset)) +#define __va_to_pa_nodebug(x) ((unsigned long)(x) - va_pa_offset) + +#ifdef CONFIG_DEBUG_VIRTUAL +extern phys_addr_t __virt_to_phys(unsigned long x); +extern phys_addr_t __phys_addr_symbol(unsigned long x); +#else +#define __virt_to_phys(x) __va_to_pa_nodebug(x) +#define __phys_addr_symbol(x) __va_to_pa_nodebug(x) +#endif /* CONFIG_DEBUG_VIRTUAL */ + +#define __pa_symbol(x) __phys_addr_symbol(RELOC_HIDE((unsigned long)(x), 0)) +#define __pa(x) __virt_to_phys((unsigned long)(x)) +#define __va(x) ((void *)__pa_to_va_nodebug((phys_addr_t)(x))) #define phys_to_pfn(phys) (PFN_DOWN(phys)) #define pfn_to_phys(pfn) (PFN_PHYS(pfn)) @@ -110,10 +128,10 @@ extern unsigned long min_low_pfn; #define page_to_bus(page) (page_to_phys(page)) #define phys_to_page(paddr) (pfn_to_page(phys_to_pfn(paddr))) +#ifdef CONFIG_FLATMEM #define pfn_valid(pfn) \ - (((pfn) >= pfn_base) && (((pfn)-pfn_base) < max_mapnr)) - -#define ARCH_PFN_OFFSET (pfn_base) + (((pfn) >= ARCH_PFN_OFFSET) && (((pfn) - ARCH_PFN_OFFSET) < max_mapnr)) +#endif #endif /* __ASSEMBLY__ */ diff --git a/arch/riscv/include/asm/pci.h b/arch/riscv/include/asm/pci.h index 5ac8daa1cc36..1c473a1bd986 100644 --- a/arch/riscv/include/asm/pci.h +++ b/arch/riscv/include/asm/pci.h @@ -3,8 +3,8 @@ * Copyright (C) 2016 SiFive */ -#ifndef __ASM_RISCV_PCI_H -#define __ASM_RISCV_PCI_H +#ifndef _ASM_RISCV_PCI_H +#define _ASM_RISCV_PCI_H #include <linux/types.h> #include <linux/slab.h> @@ -34,4 +34,4 @@ static inline int pci_proc_domain(struct pci_bus *bus) } #endif /* CONFIG_PCI */ -#endif /* __ASM_PCI_H */ +#endif /* _ASM_RISCV_PCI_H */ diff --git a/arch/riscv/include/asm/perf_event.h b/arch/riscv/include/asm/perf_event.h index aefbfaa6a781..0234048b12bc 100644 --- a/arch/riscv/include/asm/perf_event.h +++ b/arch/riscv/include/asm/perf_event.h @@ -82,4 +82,8 @@ struct riscv_pmu { int irq; }; +#ifdef CONFIG_PERF_EVENTS +#define perf_arch_bpf_user_pt_regs(regs) (struct user_regs_struct *)regs +#endif + #endif /* _ASM_RISCV_PERF_EVENT_H */ diff --git a/arch/riscv/include/asm/pgalloc.h b/arch/riscv/include/asm/pgalloc.h index 56a67d66f72f..3f601ee8233f 100644 --- a/arch/riscv/include/asm/pgalloc.h +++ b/arch/riscv/include/asm/pgalloc.h @@ -10,6 +10,7 @@ #include <linux/mm.h> #include <asm/tlb.h> +#ifdef CONFIG_MMU #include <asm-generic/pgalloc.h> /* for pte_{alloc,free}_one */ static inline void pmd_populate_kernel(struct mm_struct *mm, @@ -78,12 +79,9 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) #define __pte_free_tlb(tlb, pte, buf) \ do { \ - pgtable_page_dtor(pte); \ + pgtable_pte_page_dtor(pte); \ tlb_remove_page((tlb), pte); \ } while (0) - -static inline void check_pgt_cache(void) -{ -} +#endif /* CONFIG_MMU */ #endif /* _ASM_RISCV_PGALLOC_H */ diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h index 74630989006d..b15f70a1fdfa 100644 --- a/arch/riscv/include/asm/pgtable-64.h +++ b/arch/riscv/include/asm/pgtable-64.h @@ -43,6 +43,13 @@ static inline int pud_bad(pud_t pud) return !pud_present(pud); } +#define pud_leaf pud_leaf +static inline int pud_leaf(pud_t pud) +{ + return pud_present(pud) && + (pud_val(pud) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC)); +} + static inline void set_pud(pud_t *pudp, pud_t pud) { *pudp = pud; @@ -58,6 +65,11 @@ static inline unsigned long pud_page_vaddr(pud_t pud) return (unsigned long)pfn_to_virt(pud_val(pud) >> _PAGE_PFN_SHIFT); } +static inline struct page *pud_page(pud_t pud) +{ + return pfn_to_page(pud_val(pud) >> _PAGE_PFN_SHIFT); +} + #define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)) static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index a364aba23d55..e43041519edd 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -7,6 +7,7 @@ #define _ASM_RISCV_PGTABLE_H #include <linux/mmzone.h> +#include <linux/sizes.h> #include <asm/pgtable-bits.h> @@ -24,6 +25,7 @@ #include <asm/pgtable-32.h> #endif /* CONFIG_64BIT */ +#ifdef CONFIG_MMU /* Number of entries in the page global directory */ #define PTRS_PER_PGD (PAGE_SIZE / sizeof(pgd_t)) /* Number of entries in the page table */ @@ -31,7 +33,6 @@ /* Number of PGD entries that a user-mode program can use */ #define USER_PTRS_PER_PGD (TASK_SIZE / PGDIR_SIZE) -#define FIRST_USER_ADDRESS 0 /* Page protection bits */ #define _PAGE_BASE (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_USER) @@ -61,6 +62,12 @@ #define PAGE_TABLE __pgprot(_PAGE_TABLE) +/* + * The RISC-V ISA doesn't yet specify how to query or modify PMAs, so we can't + * change the properties of memory regions. + */ +#define _PAGE_IOREMAP _PAGE_KERNEL + extern pgd_t swapper_pg_dir[]; /* MAP_PRIVATE permissions: xwr (copy-on-write) */ @@ -83,12 +90,30 @@ extern pgd_t swapper_pg_dir[]; #define __S110 PAGE_SHARED_EXEC #define __S111 PAGE_SHARED_EXEC +#define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1) +#define VMALLOC_END (PAGE_OFFSET - 1) +#define VMALLOC_START (PAGE_OFFSET - VMALLOC_SIZE) + +#define BPF_JIT_REGION_SIZE (SZ_128M) +#define BPF_JIT_REGION_START (PAGE_OFFSET - BPF_JIT_REGION_SIZE) +#define BPF_JIT_REGION_END (VMALLOC_END) + /* - * ZERO_PAGE is a global shared page that is always zero, - * used for zero-mapped memory areas, etc. + * Roughly size the vmemmap space to be large enough to fit enough + * struct pages to map half the virtual address space. Then + * position vmemmap directly below the VMALLOC region. */ -extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; -#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) +#define VMEMMAP_SHIFT \ + (CONFIG_VA_BITS - PAGE_SHIFT - 1 + STRUCT_PAGE_MAX_SHIFT) +#define VMEMMAP_SIZE BIT(VMEMMAP_SHIFT) +#define VMEMMAP_END (VMALLOC_START - 1) +#define VMEMMAP_START (VMALLOC_START - VMEMMAP_SIZE) + +/* + * Define vmemmap for pfn_to_page & page_to_pfn calls. Needed if kernel + * is configured with CONFIG_SPARSEMEM_VMEMMAP enabled. + */ +#define vmemmap ((struct page *)VMEMMAP_START) static inline int pmd_present(pmd_t pmd) { @@ -105,6 +130,13 @@ static inline int pmd_bad(pmd_t pmd) return !pmd_present(pmd); } +#define pmd_leaf pmd_leaf +static inline int pmd_leaf(pmd_t pmd) +{ + return pmd_present(pmd) && + (pmd_val(pmd) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC)); +} + static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) { *pmdp = pmd; @@ -159,10 +191,7 @@ static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot) return __pte((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot)); } -static inline pte_t mk_pte(struct page *page, pgprot_t prot) -{ - return pfn_pte(page_to_pfn(page), prot); -} +#define mk_pte(page, prot) pfn_pte(page_to_pfn(page), prot) #define pte_index(addr) (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) @@ -403,33 +432,53 @@ static inline int ptep_clear_flush_young(struct vm_area_struct *vma, #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) #define __swp_entry_to_pte(x) ((pte_t) { (x).val }) -#ifdef CONFIG_FLATMEM -#define kern_addr_valid(addr) (1) /* FIXME */ -#endif - -extern void *dtb_early_va; -extern void setup_bootmem(void); -extern void paging_init(void); +#define PCI_IO_SIZE SZ_16M +#define PCI_IO_END VMEMMAP_START +#define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE) -static inline void pgtable_cache_init(void) -{ - /* No page table caches to initialize */ -} - -#define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1) -#define VMALLOC_END (PAGE_OFFSET - 1) -#define VMALLOC_START (PAGE_OFFSET - VMALLOC_SIZE) +#define FIXADDR_TOP PCI_IO_START +#ifdef CONFIG_64BIT +#define FIXADDR_SIZE PMD_SIZE +#else +#define FIXADDR_SIZE PGDIR_SIZE +#endif +#define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE) /* - * Task size is 0x4000000000 for RV64 or 0xb800000 for RV32. + * Task size is 0x4000000000 for RV64 or 0x9fc00000 for RV32. * Note that PGDIR_SIZE must evenly divide TASK_SIZE. */ #ifdef CONFIG_64BIT #define TASK_SIZE (PGDIR_SIZE * PTRS_PER_PGD / 2) #else -#define TASK_SIZE VMALLOC_START +#define TASK_SIZE FIXADDR_START #endif +#else /* CONFIG_MMU */ + +#define PAGE_KERNEL __pgprot(0) +#define swapper_pg_dir NULL +#define VMALLOC_START 0 + +#define TASK_SIZE 0xffffffffUL + +#endif /* !CONFIG_MMU */ + +#define kern_addr_valid(addr) (1) /* FIXME */ + +extern void *dtb_early_va; +void setup_bootmem(void); +void paging_init(void); + +#define FIRST_USER_ADDRESS 0 + +/* + * ZERO_PAGE is a global shared page that is always zero, + * used for zero-mapped memory areas, etc. + */ +extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; +#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) + #include <asm-generic/pgtable.h> #endif /* !__ASSEMBLY__ */ diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h index f539149d04c2..3ddb798264f1 100644 --- a/arch/riscv/include/asm/processor.h +++ b/arch/riscv/include/asm/processor.h @@ -42,7 +42,7 @@ struct thread_struct { ((struct pt_regs *)(task_stack_page(tsk) + THREAD_SIZE \ - ALIGN(sizeof(struct pt_regs), STACK_ALIGN))) -#define KSTK_EIP(tsk) (task_pt_regs(tsk)->sepc) +#define KSTK_EIP(tsk) (task_pt_regs(tsk)->epc) #define KSTK_ESP(tsk) (task_pt_regs(tsk)->sp) diff --git a/arch/riscv/include/asm/ptrace.h b/arch/riscv/include/asm/ptrace.h index d48d1e13973c..ee49f80c9533 100644 --- a/arch/riscv/include/asm/ptrace.h +++ b/arch/riscv/include/asm/ptrace.h @@ -12,7 +12,7 @@ #ifndef __ASSEMBLY__ struct pt_regs { - unsigned long sepc; + unsigned long epc; unsigned long ra; unsigned long sp; unsigned long gp; @@ -44,10 +44,10 @@ struct pt_regs { unsigned long t4; unsigned long t5; unsigned long t6; - /* Supervisor CSRs */ - unsigned long sstatus; - unsigned long sbadaddr; - unsigned long scause; + /* Supervisor/Machine CSRs */ + unsigned long status; + unsigned long badaddr; + unsigned long cause; /* a0 value before the syscall */ unsigned long orig_a0; }; @@ -58,18 +58,18 @@ struct pt_regs { #define REG_FMT "%08lx" #endif -#define user_mode(regs) (((regs)->sstatus & SR_SPP) == 0) +#define user_mode(regs) (((regs)->status & SR_PP) == 0) /* Helpers for working with the instruction pointer */ static inline unsigned long instruction_pointer(struct pt_regs *regs) { - return regs->sepc; + return regs->epc; } static inline void instruction_pointer_set(struct pt_regs *regs, unsigned long val) { - regs->sepc = val; + regs->epc = val; } #define profile_pc(regs) instruction_pointer(regs) diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h index 21134b3ef404..2570c1e683d3 100644 --- a/arch/riscv/include/asm/sbi.h +++ b/arch/riscv/include/asm/sbi.h @@ -8,6 +8,7 @@ #include <linux/types.h> +#ifdef CONFIG_RISCV_SBI #define SBI_SET_TIMER 0 #define SBI_CONSOLE_PUTCHAR 1 #define SBI_CONSOLE_GETCHAR 2 @@ -93,5 +94,11 @@ static inline void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, { SBI_CALL_4(SBI_REMOTE_SFENCE_VMA_ASID, hart_mask, start, size, asid); } - -#endif +#else /* CONFIG_RISCV_SBI */ +/* stubs for code that is only reachable under IS_ENABLED(CONFIG_RISCV_SBI): */ +void sbi_set_timer(uint64_t stime_value); +void sbi_clear_ipi(void); +void sbi_send_ipi(const unsigned long *hart_mask); +void sbi_remote_fence_i(const unsigned long *hart_mask); +#endif /* CONFIG_RISCV_SBI */ +#endif /* _ASM_RISCV_SBI_H */ diff --git a/arch/riscv/include/asm/seccomp.h b/arch/riscv/include/asm/seccomp.h new file mode 100644 index 000000000000..bf7744ee3b3d --- /dev/null +++ b/arch/riscv/include/asm/seccomp.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_SECCOMP_H +#define _ASM_SECCOMP_H + +#include <asm/unistd.h> + +#include <asm-generic/seccomp.h> + +#endif /* _ASM_SECCOMP_H */ diff --git a/arch/riscv/include/asm/sifive_l2_cache.h b/arch/riscv/include/asm/sifive_l2_cache.h deleted file mode 100644 index 04f6748fc50b..000000000000 --- a/arch/riscv/include/asm/sifive_l2_cache.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * SiFive L2 Cache Controller header file - * - */ - -#ifndef _ASM_RISCV_SIFIVE_L2_CACHE_H -#define _ASM_RISCV_SIFIVE_L2_CACHE_H - -extern int register_sifive_l2_error_notifier(struct notifier_block *nb); -extern int unregister_sifive_l2_error_notifier(struct notifier_block *nb); - -#define SIFIVE_L2_ERR_TYPE_CE 0 -#define SIFIVE_L2_ERR_TYPE_UE 1 - -#endif /* _ASM_RISCV_SIFIVE_L2_CACHE_H */ diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h index c6ed4d691def..a83451d73a4e 100644 --- a/arch/riscv/include/asm/smp.h +++ b/arch/riscv/include/asm/smp.h @@ -61,11 +61,5 @@ static inline unsigned long cpuid_to_hartid_map(int cpu) return boot_cpu_hartid; } -static inline void riscv_cpuid_to_hartid_mask(const struct cpumask *in, - struct cpumask *out) -{ - cpumask_set_cpu(cpuid_to_hartid_map(0), out); -} - #endif /* CONFIG_SMP */ #endif /* _ASM_RISCV_SMP_H */ diff --git a/arch/riscv/include/asm/sparsemem.h b/arch/riscv/include/asm/sparsemem.h new file mode 100644 index 000000000000..45a7018a8118 --- /dev/null +++ b/arch/riscv/include/asm/sparsemem.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_RISCV_SPARSEMEM_H +#define _ASM_RISCV_SPARSEMEM_H + +#ifdef CONFIG_SPARSEMEM +#define MAX_PHYSMEM_BITS CONFIG_PA_BITS +#define SECTION_SIZE_BITS 27 +#endif /* CONFIG_SPARSEMEM */ + +#endif /* _ASM_RISCV_SPARSEMEM_H */ diff --git a/arch/riscv/include/asm/spinlock_types.h b/arch/riscv/include/asm/spinlock_types.h index 888cbf8e7111..f398e7638dd6 100644 --- a/arch/riscv/include/asm/spinlock_types.h +++ b/arch/riscv/include/asm/spinlock_types.h @@ -22,4 +22,4 @@ typedef struct { #define __ARCH_RW_LOCK_UNLOCKED { 0 } -#endif +#endif /* _ASM_RISCV_SPINLOCK_TYPES_H */ diff --git a/arch/riscv/include/asm/string.h b/arch/riscv/include/asm/string.h index 1b5d44585962..924af13f8555 100644 --- a/arch/riscv/include/asm/string.h +++ b/arch/riscv/include/asm/string.h @@ -11,8 +11,17 @@ #define __HAVE_ARCH_MEMSET extern asmlinkage void *memset(void *, int, size_t); +extern asmlinkage void *__memset(void *, int, size_t); #define __HAVE_ARCH_MEMCPY extern asmlinkage void *memcpy(void *, const void *, size_t); +extern asmlinkage void *__memcpy(void *, const void *, size_t); +/* For those files which don't want to check by kasan. */ +#if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__) + +#define memcpy(dst, src, len) __memcpy(dst, src, len) +#define memset(s, c, n) __memset(s, c, n) + +#endif #endif /* _ASM_RISCV_STRING_H */ diff --git a/arch/riscv/include/asm/switch_to.h b/arch/riscv/include/asm/switch_to.h index f0227bdce0f0..407bcc96a710 100644 --- a/arch/riscv/include/asm/switch_to.h +++ b/arch/riscv/include/asm/switch_to.h @@ -6,6 +6,7 @@ #ifndef _ASM_RISCV_SWITCH_TO_H #define _ASM_RISCV_SWITCH_TO_H +#include <linux/sched/task_stack.h> #include <asm/processor.h> #include <asm/ptrace.h> #include <asm/csr.h> @@ -16,19 +17,19 @@ extern void __fstate_restore(struct task_struct *restore_from); static inline void __fstate_clean(struct pt_regs *regs) { - regs->sstatus = (regs->sstatus & ~SR_FS) | SR_FS_CLEAN; + regs->status = (regs->status & ~SR_FS) | SR_FS_CLEAN; } static inline void fstate_off(struct task_struct *task, struct pt_regs *regs) { - regs->sstatus = (regs->sstatus & ~SR_FS) | SR_FS_OFF; + regs->status = (regs->status & ~SR_FS) | SR_FS_OFF; } static inline void fstate_save(struct task_struct *task, struct pt_regs *regs) { - if ((regs->sstatus & SR_FS) == SR_FS_DIRTY) { + if ((regs->status & SR_FS) == SR_FS_DIRTY) { __fstate_save(task); __fstate_clean(regs); } @@ -37,7 +38,7 @@ static inline void fstate_save(struct task_struct *task, static inline void fstate_restore(struct task_struct *task, struct pt_regs *regs) { - if ((regs->sstatus & SR_FS) != SR_FS_OFF) { + if ((regs->status & SR_FS) != SR_FS_OFF) { __fstate_restore(task); __fstate_clean(regs); } @@ -49,7 +50,7 @@ static inline void __switch_to_aux(struct task_struct *prev, struct pt_regs *regs; regs = task_pt_regs(prev); - if (unlikely(regs->sstatus & SR_SD)) + if (unlikely(regs->status & SR_SD)) fstate_save(prev, regs); fstate_restore(next, task_pt_regs(next)); } diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h index 905372d7eeb8..1dd12a0cbb2b 100644 --- a/arch/riscv/include/asm/thread_info.h +++ b/arch/riscv/include/asm/thread_info.h @@ -75,6 +75,7 @@ struct thread_info { #define TIF_MEMDIE 5 /* is terminating due to OOM killer */ #define TIF_SYSCALL_TRACEPOINT 6 /* syscall tracepoint instrumentation */ #define TIF_SYSCALL_AUDIT 7 /* syscall auditing */ +#define TIF_SECCOMP 8 /* syscall secure computing */ #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) @@ -82,11 +83,13 @@ struct thread_info { #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) +#define _TIF_SECCOMP (1 << TIF_SECCOMP) #define _TIF_WORK_MASK \ (_TIF_NOTIFY_RESUME | _TIF_SIGPENDING | _TIF_NEED_RESCHED) #define _TIF_SYSCALL_WORK \ - (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_TRACEPOINT | _TIF_SYSCALL_AUDIT) + (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_TRACEPOINT | _TIF_SYSCALL_AUDIT | \ + _TIF_SECCOMP) #endif /* _ASM_RISCV_THREAD_INFO_H */ diff --git a/arch/riscv/include/asm/timex.h b/arch/riscv/include/asm/timex.h index 6a703ec9d796..bad2a7c2cda5 100644 --- a/arch/riscv/include/asm/timex.h +++ b/arch/riscv/include/asm/timex.h @@ -6,43 +6,56 @@ #ifndef _ASM_RISCV_TIMEX_H #define _ASM_RISCV_TIMEX_H -#include <asm/param.h> +#include <asm/csr.h> +#include <asm/mmio.h> typedef unsigned long cycles_t; -static inline cycles_t get_cycles_inline(void) -{ - cycles_t n; +extern u64 __iomem *riscv_time_val; +extern u64 __iomem *riscv_time_cmp; - __asm__ __volatile__ ( - "rdtime %0" - : "=r" (n)); - return n; +#ifdef CONFIG_64BIT +#define mmio_get_cycles() readq_relaxed(riscv_time_val) +#else +#define mmio_get_cycles() readl_relaxed(riscv_time_val) +#define mmio_get_cycles_hi() readl_relaxed(((u32 *)riscv_time_val) + 1) +#endif + +static inline cycles_t get_cycles(void) +{ + if (IS_ENABLED(CONFIG_RISCV_SBI)) + return csr_read(CSR_TIME); + return mmio_get_cycles(); } -#define get_cycles get_cycles_inline +#define get_cycles get_cycles #ifdef CONFIG_64BIT -static inline uint64_t get_cycles64(void) +static inline u64 get_cycles64(void) { - return get_cycles(); + return get_cycles(); } -#else -static inline uint64_t get_cycles64(void) +#else /* CONFIG_64BIT */ +static inline u32 get_cycles_hi(void) { - u32 lo, hi, tmp; - __asm__ __volatile__ ( - "1:\n" - "rdtimeh %0\n" - "rdtime %1\n" - "rdtimeh %2\n" - "bne %0, %2, 1b" - : "=&r" (hi), "=&r" (lo), "=&r" (tmp)); + if (IS_ENABLED(CONFIG_RISCV_SBI)) + return csr_read(CSR_TIMEH); + return mmio_get_cycles_hi(); +} + +static inline u64 get_cycles64(void) +{ + u32 hi, lo; + + do { + hi = get_cycles_hi(); + lo = get_cycles(); + } while (hi != get_cycles_hi()); + return ((u64)hi << 32) | lo; } -#endif +#endif /* CONFIG_64BIT */ #define ARCH_HAS_READ_CURRENT_TIMER - static inline int read_current_timer(unsigned long *timer_val) { *timer_val = get_cycles(); diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h index 4d9bbe8438bf..394cfbccdcd9 100644 --- a/arch/riscv/include/asm/tlbflush.h +++ b/arch/riscv/include/asm/tlbflush.h @@ -10,10 +10,7 @@ #include <linux/mm_types.h> #include <asm/smp.h> -/* - * Flush entire local TLB. 'sfence.vma' implicitly fences with the instruction - * cache as well, so a 'fence.i' is not necessary. - */ +#ifdef CONFIG_MMU static inline void local_flush_tlb_all(void) { __asm__ __volatile__ ("sfence.vma" : : : "memory"); @@ -24,8 +21,18 @@ static inline void local_flush_tlb_page(unsigned long addr) { __asm__ __volatile__ ("sfence.vma %0" : : "r" (addr) : "memory"); } - -#ifndef CONFIG_SMP +#else /* CONFIG_MMU */ +#define local_flush_tlb_all() do { } while (0) +#define local_flush_tlb_page(addr) do { } while (0) +#endif /* CONFIG_MMU */ + +#if defined(CONFIG_SMP) && defined(CONFIG_MMU) +void flush_tlb_all(void); +void flush_tlb_mm(struct mm_struct *mm); +void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr); +void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end); +#else /* CONFIG_SMP && CONFIG_MMU */ #define flush_tlb_all() local_flush_tlb_all() #define flush_tlb_page(vma, addr) local_flush_tlb_page(addr) @@ -37,36 +44,7 @@ static inline void flush_tlb_range(struct vm_area_struct *vma, } #define flush_tlb_mm(mm) flush_tlb_all() - -#else /* CONFIG_SMP */ - -#include <asm/sbi.h> - -static inline void remote_sfence_vma(struct cpumask *cmask, unsigned long start, - unsigned long size) -{ - struct cpumask hmask; - - cpumask_clear(&hmask); - riscv_cpuid_to_hartid_mask(cmask, &hmask); - sbi_remote_sfence_vma(hmask.bits, start, size); -} - -#define flush_tlb_all() sbi_remote_sfence_vma(NULL, 0, -1) - -#define flush_tlb_range(vma, start, end) \ - remote_sfence_vma(mm_cpumask((vma)->vm_mm), start, (end) - (start)) - -static inline void flush_tlb_page(struct vm_area_struct *vma, - unsigned long addr) -{ - flush_tlb_range(vma, addr, addr + PAGE_SIZE); -} - -#define flush_tlb_mm(mm) \ - remote_sfence_vma(mm_cpumask(mm), 0, -1) - -#endif /* CONFIG_SMP */ +#endif /* !CONFIG_SMP || !CONFIG_MMU */ /* Flush a range of kernel pages */ static inline void flush_tlb_kernel_range(unsigned long start, diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h index e076437cfafe..f462a183a9c2 100644 --- a/arch/riscv/include/asm/uaccess.h +++ b/arch/riscv/include/asm/uaccess.h @@ -11,6 +11,7 @@ /* * User space memory access functions */ +#ifdef CONFIG_MMU #include <linux/errno.h> #include <linux/compiler.h> #include <linux/thread_info.h> @@ -475,4 +476,7 @@ unsigned long __must_check clear_user(void __user *to, unsigned long n) __ret; \ }) +#else /* CONFIG_MMU */ +#include <asm-generic/uaccess.h> +#endif /* CONFIG_MMU */ #endif /* _ASM_RISCV_UACCESS_H */ diff --git a/arch/riscv/include/asm/vmalloc.h b/arch/riscv/include/asm/vmalloc.h new file mode 100644 index 000000000000..ff9abc00d139 --- /dev/null +++ b/arch/riscv/include/asm/vmalloc.h @@ -0,0 +1,4 @@ +#ifndef _ASM_RISCV_VMALLOC_H +#define _ASM_RISCV_VMALLOC_H + +#endif /* _ASM_RISCV_VMALLOC_H */ diff --git a/arch/riscv/include/uapi/asm/bpf_perf_event.h b/arch/riscv/include/uapi/asm/bpf_perf_event.h new file mode 100644 index 000000000000..6cb1c2823288 --- /dev/null +++ b/arch/riscv/include/uapi/asm/bpf_perf_event.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI__ASM_BPF_PERF_EVENT_H__ +#define _UAPI__ASM_BPF_PERF_EVENT_H__ + +#include <asm/ptrace.h> + +typedef struct user_regs_struct bpf_user_pt_regs_t; + +#endif /* _UAPI__ASM_BPF_PERF_EVENT_H__ */ diff --git a/arch/riscv/include/uapi/asm/elf.h b/arch/riscv/include/uapi/asm/elf.h index 644a00ce6e2e..d696d6610231 100644 --- a/arch/riscv/include/uapi/asm/elf.h +++ b/arch/riscv/include/uapi/asm/elf.h @@ -9,8 +9,8 @@ * (at your option) any later version. */ -#ifndef _UAPI_ASM_ELF_H -#define _UAPI_ASM_ELF_H +#ifndef _UAPI_ASM_RISCV_ELF_H +#define _UAPI_ASM_RISCV_ELF_H #include <asm/ptrace.h> @@ -95,4 +95,4 @@ typedef union __riscv_fp_state elf_fpregset_t; #define R_RISCV_32_PCREL 57 -#endif /* _UAPI_ASM_ELF_H */ +#endif /* _UAPI_ASM_RISCV_ELF_H */ diff --git a/arch/riscv/include/uapi/asm/hwcap.h b/arch/riscv/include/uapi/asm/hwcap.h index 4e7646077056..dee98ee28318 100644 --- a/arch/riscv/include/uapi/asm/hwcap.h +++ b/arch/riscv/include/uapi/asm/hwcap.h @@ -5,8 +5,8 @@ * Copyright (C) 2012 ARM Ltd. * Copyright (C) 2017 SiFive */ -#ifndef __UAPI_ASM_HWCAP_H -#define __UAPI_ASM_HWCAP_H +#ifndef _UAPI_ASM_RISCV_HWCAP_H +#define _UAPI_ASM_RISCV_HWCAP_H /* * Linux saves the floating-point registers according to the ISA Linux is @@ -22,4 +22,4 @@ #define COMPAT_HWCAP_ISA_D (1 << ('D' - 'A')) #define COMPAT_HWCAP_ISA_C (1 << ('C' - 'A')) -#endif +#endif /* _UAPI_ASM_RISCV_HWCAP_H */ diff --git a/arch/riscv/include/uapi/asm/perf_regs.h b/arch/riscv/include/uapi/asm/perf_regs.h new file mode 100644 index 000000000000..196f964bfcb4 --- /dev/null +++ b/arch/riscv/include/uapi/asm/perf_regs.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd. */ + +#ifndef _ASM_RISCV_PERF_REGS_H +#define _ASM_RISCV_PERF_REGS_H + +enum perf_event_riscv_regs { + PERF_REG_RISCV_PC, + PERF_REG_RISCV_RA, + PERF_REG_RISCV_SP, + PERF_REG_RISCV_GP, + PERF_REG_RISCV_TP, + PERF_REG_RISCV_T0, + PERF_REG_RISCV_T1, + PERF_REG_RISCV_T2, + PERF_REG_RISCV_S0, + PERF_REG_RISCV_S1, + PERF_REG_RISCV_A0, + PERF_REG_RISCV_A1, + PERF_REG_RISCV_A2, + PERF_REG_RISCV_A3, + PERF_REG_RISCV_A4, + PERF_REG_RISCV_A5, + PERF_REG_RISCV_A6, + PERF_REG_RISCV_A7, + PERF_REG_RISCV_S2, + PERF_REG_RISCV_S3, + PERF_REG_RISCV_S4, + PERF_REG_RISCV_S5, + PERF_REG_RISCV_S6, + PERF_REG_RISCV_S7, + PERF_REG_RISCV_S8, + PERF_REG_RISCV_S9, + PERF_REG_RISCV_S10, + PERF_REG_RISCV_S11, + PERF_REG_RISCV_T3, + PERF_REG_RISCV_T4, + PERF_REG_RISCV_T5, + PERF_REG_RISCV_T6, + PERF_REG_RISCV_MAX, +}; +#endif /* _ASM_RISCV_PERF_REGS_H */ diff --git a/arch/riscv/include/uapi/asm/ucontext.h b/arch/riscv/include/uapi/asm/ucontext.h index 411dd7b52ed6..44eb993950e5 100644 --- a/arch/riscv/include/uapi/asm/ucontext.h +++ b/arch/riscv/include/uapi/asm/ucontext.h @@ -5,8 +5,8 @@ * * This file was copied from arch/arm64/include/uapi/asm/ucontext.h */ -#ifndef _UAPI__ASM_UCONTEXT_H -#define _UAPI__ASM_UCONTEXT_H +#ifndef _UAPI_ASM_RISCV_UCONTEXT_H +#define _UAPI_ASM_RISCV_UCONTEXT_H #include <linux/types.h> @@ -31,4 +31,4 @@ struct ucontext { struct sigcontext uc_mcontext; }; -#endif /* _UAPI__ASM_UCONTEXT_H */ +#endif /* _UAPI_ASM_RISCV_UCONTEXT_H */ diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index 2420d37d96de..f40205cb9a22 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -25,10 +25,10 @@ obj-y += time.o obj-y += traps.o obj-y += riscv_ksyms.o obj-y += stacktrace.o -obj-y += vdso.o obj-y += cacheinfo.o -obj-y += vdso/ +obj-$(CONFIG_MMU) += vdso.o vdso/ +obj-$(CONFIG_RISCV_M_MODE) += clint.o obj-$(CONFIG_FPU) += fpu.o obj-$(CONFIG_SMP) += smpboot.o obj-$(CONFIG_SMP) += smp.o @@ -38,6 +38,9 @@ obj-$(CONFIG_MODULE_SECTIONS) += module-sections.o obj-$(CONFIG_FUNCTION_TRACER) += mcount.o ftrace.o obj-$(CONFIG_DYNAMIC_FTRACE) += mcount-dyn.o -obj-$(CONFIG_PERF_EVENTS) += perf_event.o +obj-$(CONFIG_PERF_EVENTS) += perf_event.o +obj-$(CONFIG_PERF_EVENTS) += perf_callchain.o +obj-$(CONFIG_HAVE_PERF_REGS) += perf_regs.o +obj-$(CONFIG_RISCV_SBI) += sbi.o clean: diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c index 9f5628c38ac9..07cb9c10de4e 100644 --- a/arch/riscv/kernel/asm-offsets.c +++ b/arch/riscv/kernel/asm-offsets.c @@ -71,7 +71,7 @@ void asm_offsets(void) OFFSET(TASK_THREAD_FCSR, task_struct, thread.fstate.fcsr); DEFINE(PT_SIZE, sizeof(struct pt_regs)); - OFFSET(PT_SEPC, pt_regs, sepc); + OFFSET(PT_EPC, pt_regs, epc); OFFSET(PT_RA, pt_regs, ra); OFFSET(PT_FP, pt_regs, s0); OFFSET(PT_S0, pt_regs, s0); @@ -105,9 +105,9 @@ void asm_offsets(void) OFFSET(PT_T6, pt_regs, t6); OFFSET(PT_GP, pt_regs, gp); OFFSET(PT_ORIG_A0, pt_regs, orig_a0); - OFFSET(PT_SSTATUS, pt_regs, sstatus); - OFFSET(PT_SBADADDR, pt_regs, sbadaddr); - OFFSET(PT_SCAUSE, pt_regs, scause); + OFFSET(PT_STATUS, pt_regs, status); + OFFSET(PT_BADADDR, pt_regs, badaddr); + OFFSET(PT_CAUSE, pt_regs, cause); /* * THREAD_{F,X}* might be larger than a S-type offset can handle, but diff --git a/arch/riscv/kernel/clint.c b/arch/riscv/kernel/clint.c new file mode 100644 index 000000000000..3647980d14c3 --- /dev/null +++ b/arch/riscv/kernel/clint.c @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2019 Christoph Hellwig. + */ + +#include <linux/io.h> +#include <linux/of_address.h> +#include <linux/types.h> +#include <asm/clint.h> +#include <asm/csr.h> +#include <asm/timex.h> +#include <asm/smp.h> + +/* + * This is the layout used by the SiFive clint, which is also shared by the qemu + * virt platform, and the Kendryte KD210 at least. + */ +#define CLINT_IPI_OFF 0 +#define CLINT_TIME_CMP_OFF 0x4000 +#define CLINT_TIME_VAL_OFF 0xbff8 + +u32 __iomem *clint_ipi_base; + +void clint_init_boot_cpu(void) +{ + struct device_node *np; + void __iomem *base; + + np = of_find_compatible_node(NULL, NULL, "riscv,clint0"); + if (!np) { + panic("clint not found"); + return; + } + + base = of_iomap(np, 0); + if (!base) + panic("could not map CLINT"); + + clint_ipi_base = base + CLINT_IPI_OFF; + riscv_time_cmp = base + CLINT_TIME_CMP_OFF; + riscv_time_val = base + CLINT_TIME_VAL_OFF; + + clint_clear_ipi(boot_cpu_hartid); +} diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c index 7da3c6a93abd..40a3c442ac5f 100644 --- a/arch/riscv/kernel/cpu.c +++ b/arch/riscv/kernel/cpu.c @@ -46,51 +46,12 @@ int riscv_of_processor_hartid(struct device_node *node) #ifdef CONFIG_PROC_FS -static void print_isa(struct seq_file *f, const char *orig_isa) +static void print_isa(struct seq_file *f, const char *isa) { - static const char *ext = "mafdcsu"; - const char *isa = orig_isa; - const char *e; - - /* - * Linux doesn't support rv32e or rv128i, and we only support booting - * kernels on harts with the same ISA that the kernel is compiled for. - */ -#if defined(CONFIG_32BIT) - if (strncmp(isa, "rv32i", 5) != 0) - return; -#elif defined(CONFIG_64BIT) - if (strncmp(isa, "rv64i", 5) != 0) - return; -#endif - - /* Print the base ISA, as we already know it's legal. */ + /* Print the entire ISA as it is */ seq_puts(f, "isa\t\t: "); - seq_write(f, isa, 5); - isa += 5; - - /* - * Check the rest of the ISA string for valid extensions, printing those - * we find. RISC-V ISA strings define an order, so we only print the - * extension bits when they're in order. Hide the supervisor (S) - * extension from userspace as it's not accessible from there. - */ - for (e = ext; *e != '\0'; ++e) { - if (isa[0] == e[0]) { - if (isa[0] != 's') - seq_write(f, isa, 1); - - isa++; - } - } + seq_write(f, isa, strlen(isa)); seq_puts(f, "\n"); - - /* - * If we were given an unsupported ISA in the device tree then print - * a bit of info describing what went wrong. - */ - if (isa[0] != '\0') - pr_info("unsupported ISA \"%s\" in device tree\n", orig_isa); } static void print_mmu(struct seq_file *f, const char *mmu_type) diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index b1ade9a49347..a5ad00043104 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -10,6 +10,7 @@ #include <asm/processor.h> #include <asm/hwcap.h> #include <asm/smp.h> +#include <asm/switch_to.h> unsigned long elf_hwcap __read_mostly; #ifdef CONFIG_FPU diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index bc7a56e1ca6f..bad4d85b5e91 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -26,14 +26,14 @@ /* * If coming from userspace, preserve the user thread pointer and load - * the kernel thread pointer. If we came from the kernel, sscratch - * will contain 0, and we should continue on the current TP. + * the kernel thread pointer. If we came from the kernel, the scratch + * register will contain 0, and we should continue on the current TP. */ - csrrw tp, CSR_SSCRATCH, tp + csrrw tp, CSR_SCRATCH, tp bnez tp, _save_context _restore_kernel_tpsp: - csrr tp, CSR_SSCRATCH + csrr tp, CSR_SCRATCH REG_S sp, TASK_TI_KERNEL_SP(tp) _save_context: REG_S sp, TASK_TI_USER_SP(tp) @@ -79,16 +79,16 @@ _save_context: li t0, SR_SUM | SR_FS REG_L s0, TASK_TI_USER_SP(tp) - csrrc s1, CSR_SSTATUS, t0 - csrr s2, CSR_SEPC - csrr s3, CSR_STVAL - csrr s4, CSR_SCAUSE - csrr s5, CSR_SSCRATCH + csrrc s1, CSR_STATUS, t0 + csrr s2, CSR_EPC + csrr s3, CSR_TVAL + csrr s4, CSR_CAUSE + csrr s5, CSR_SCRATCH REG_S s0, PT_SP(sp) - REG_S s1, PT_SSTATUS(sp) - REG_S s2, PT_SEPC(sp) - REG_S s3, PT_SBADADDR(sp) - REG_S s4, PT_SCAUSE(sp) + REG_S s1, PT_STATUS(sp) + REG_S s2, PT_EPC(sp) + REG_S s3, PT_BADADDR(sp) + REG_S s4, PT_CAUSE(sp) REG_S s5, PT_TP(sp) .endm @@ -97,10 +97,29 @@ _save_context: * registers from the stack. */ .macro RESTORE_ALL - REG_L a0, PT_SSTATUS(sp) - REG_L a2, PT_SEPC(sp) - csrw CSR_SSTATUS, a0 - csrw CSR_SEPC, a2 + REG_L a0, PT_STATUS(sp) + /* + * The current load reservation is effectively part of the processor's + * state, in the sense that load reservations cannot be shared between + * different hart contexts. We can't actually save and restore a load + * reservation, so instead here we clear any existing reservation -- + * it's always legal for implementations to clear load reservations at + * any point (as long as the forward progress guarantee is kept, but + * we'll ignore that here). + * + * Dangling load reservations can be the result of taking a trap in the + * middle of an LR/SC sequence, but can also be the result of a taken + * forward branch around an SC -- which is how we implement CAS. As a + * result we need to clear reservations between the last CAS and the + * jump back to the new context. While it is unlikely the store + * completes, implementations are allowed to expand reservations to be + * arbitrarily large. + */ + REG_L a2, PT_EPC(sp) + REG_SC x0, a2, PT_EPC(sp) + + csrw CSR_STATUS, a0 + csrw CSR_EPC, a2 REG_L x1, PT_RA(sp) REG_L x3, PT_GP(sp) @@ -136,7 +155,7 @@ _save_context: REG_L x2, PT_SP(sp) .endm -#if !IS_ENABLED(CONFIG_PREEMPT) +#if !IS_ENABLED(CONFIG_PREEMPTION) .set resume_kernel, restore_all #endif @@ -144,10 +163,10 @@ ENTRY(handle_exception) SAVE_ALL /* - * Set sscratch register to 0, so that if a recursive exception + * Set the scratch register to 0, so that if a recursive exception * occurs, the exception vector knows it came from the kernel */ - csrw CSR_SSCRATCH, x0 + csrw CSR_SCRATCH, x0 /* Load the global pointer */ .option push @@ -166,9 +185,15 @@ ENTRY(handle_exception) move a0, sp /* pt_regs */ tail do_IRQ 1: - /* Exceptions run with interrupts enabled */ - csrs sstatus, SR_SIE + /* + * Exceptions run with interrupts enabled or disabled depending on the + * state of SR_PIE in m/sstatus. + */ + andi t0, s1, SR_PIE + beqz t0, 1f + csrs CSR_STATUS, SR_IE +1: /* Handle syscalls */ li t0, EXC_SYSCALL beq s4, t0, handle_syscall @@ -194,7 +219,7 @@ handle_syscall: * scall instruction on sret */ addi s2, s2, 0x4 - REG_S s2, PT_SEPC(sp) + REG_S s2, PT_EPC(sp) /* Trace syscalls, but only if requested by the user. */ REG_L t0, TASK_TI_FLAGS(tp) andi t0, t0, _TIF_SYSCALL_WORK @@ -203,8 +228,26 @@ check_syscall_nr: /* Check to make sure we don't jump to a bogus syscall number. */ li t0, __NR_syscalls la s0, sys_ni_syscall - /* Syscall number held in a7 */ - bgeu a7, t0, 1f + /* + * The tracer can change syscall number to valid/invalid value. + * We use syscall_set_nr helper in syscall_trace_enter thus we + * cannot trust the current value in a7 and have to reload from + * the current task pt_regs. + */ + REG_L a7, PT_A7(sp) + /* + * Syscall number held in a7. + * If syscall number is above allowed value, redirect to ni_syscall. + */ + bge a7, t0, 1f + /* + * Check if syscall is rejected by tracer or seccomp, i.e., a7 == -1. + * If yes, we pretend it was executed. + */ + li t1, -1 + beq a7, t1, ret_from_syscall_rejected + blt a7, t1, 1f + /* Call syscall */ la s0, sys_call_table slli t0, a7, RISCV_LGPTR add s0, s0, t0 @@ -215,15 +258,27 @@ check_syscall_nr: ret_from_syscall: /* Set user a0 to kernel a0 */ REG_S a0, PT_A0(sp) + /* + * We didn't execute the actual syscall. + * Seccomp already set return value for the current task pt_regs. + * (If it was configured with SECCOMP_RET_ERRNO/TRACE) + */ +ret_from_syscall_rejected: /* Trace syscalls, but only if requested by the user. */ REG_L t0, TASK_TI_FLAGS(tp) andi t0, t0, _TIF_SYSCALL_WORK bnez t0, handle_syscall_trace_exit ret_from_exception: - REG_L s0, PT_SSTATUS(sp) - csrc sstatus, SR_SIE + REG_L s0, PT_STATUS(sp) + csrc CSR_STATUS, SR_IE +#ifdef CONFIG_RISCV_M_MODE + /* the MPP value is too large to be used as an immediate arg for addi */ + li t0, SR_MPP + and s0, s0, t0 +#else andi s0, s0, SR_SPP +#endif bnez s0, resume_kernel resume_userspace: @@ -237,25 +292,28 @@ resume_userspace: REG_S s0, TASK_TI_KERNEL_SP(tp) /* - * Save TP into sscratch, so we can find the kernel data structures - * again. + * Save TP into the scratch register , so we can find the kernel data + * structures again. */ - csrw CSR_SSCRATCH, tp + csrw CSR_SCRATCH, tp restore_all: RESTORE_ALL +#ifdef CONFIG_RISCV_M_MODE + mret +#else sret +#endif -#if IS_ENABLED(CONFIG_PREEMPT) +#if IS_ENABLED(CONFIG_PREEMPTION) resume_kernel: REG_L s0, TASK_TI_PREEMPT_COUNT(tp) bnez s0, restore_all -need_resched: REG_L s0, TASK_TI_FLAGS(tp) andi s0, s0, _TIF_NEED_RESCHED beqz s0, restore_all call preempt_schedule_irq - j need_resched + j restore_all #endif work_pending: @@ -265,7 +323,7 @@ work_pending: bnez s1, work_resched work_notifysig: /* Handle pending signals and notify-resume requests */ - csrs sstatus, SR_SIE /* Enable interrupts for do_notify_resume() */ + csrs CSR_STATUS, SR_IE /* Enable interrupts for do_notify_resume() */ move a0, sp /* pt_regs */ move a1, s0 /* current_thread_info->flags */ tail do_notify_resume @@ -364,6 +422,10 @@ ENTRY(__switch_to) ret ENDPROC(__switch_to) +#ifndef CONFIG_MMU +#define do_page_fault do_trap_unknown +#endif + .section ".rodata" /* Exception vector table */ ENTRY(excp_vect_table) @@ -385,3 +447,10 @@ ENTRY(excp_vect_table) RISCV_PTR do_page_fault /* store page fault */ excp_vect_table_end: END(excp_vect_table) + +#ifndef CONFIG_MMU +ENTRY(__user_rt_sigreturn) + li a7, __NR_rt_sigreturn + scall +END(__user_rt_sigreturn) +#endif diff --git a/arch/riscv/kernel/fpu.S b/arch/riscv/kernel/fpu.S index 1defb0618aff..dd2205473de7 100644 --- a/arch/riscv/kernel/fpu.S +++ b/arch/riscv/kernel/fpu.S @@ -23,7 +23,7 @@ ENTRY(__fstate_save) li a2, TASK_THREAD_F0 add a0, a0, a2 li t1, SR_FS - csrs sstatus, t1 + csrs CSR_STATUS, t1 frcsr t0 fsd f0, TASK_THREAD_F0_F0(a0) fsd f1, TASK_THREAD_F1_F0(a0) @@ -58,7 +58,7 @@ ENTRY(__fstate_save) fsd f30, TASK_THREAD_F30_F0(a0) fsd f31, TASK_THREAD_F31_F0(a0) sw t0, TASK_THREAD_FCSR_F0(a0) - csrc sstatus, t1 + csrc CSR_STATUS, t1 ret ENDPROC(__fstate_save) @@ -67,7 +67,7 @@ ENTRY(__fstate_restore) add a0, a0, a2 li t1, SR_FS lw t0, TASK_THREAD_FCSR_F0(a0) - csrs sstatus, t1 + csrs CSR_STATUS, t1 fld f0, TASK_THREAD_F0_F0(a0) fld f1, TASK_THREAD_F1_F0(a0) fld f2, TASK_THREAD_F2_F0(a0) @@ -101,6 +101,6 @@ ENTRY(__fstate_restore) fld f30, TASK_THREAD_F30_F0(a0) fld f31, TASK_THREAD_F31_F0(a0) fscsr t0 - csrc sstatus, t1 + csrc CSR_STATUS, t1 ret ENDPROC(__fstate_restore) diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c index b94d8db5ddcc..c40fdcdeb950 100644 --- a/arch/riscv/kernel/ftrace.c +++ b/arch/riscv/kernel/ftrace.c @@ -142,7 +142,7 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, */ old = *parent; - if (function_graph_enter(old, self_addr, frame_pointer, parent)) + if (!function_graph_enter(old, self_addr, frame_pointer, parent)) *parent = return_hooker; } diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index 0f1ba17e476f..271860fc2c3f 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -11,6 +11,7 @@ #include <asm/thread_info.h> #include <asm/page.h> #include <asm/csr.h> +#include <asm/hwcap.h> #include <asm/image.h> __INIT @@ -39,16 +40,30 @@ ENTRY(_start) .word RISCV_HEADER_VERSION .word 0 .dword 0 - .asciz RISCV_IMAGE_MAGIC - .word 0 + .ascii RISCV_IMAGE_MAGIC .balign 4 + .ascii RISCV_IMAGE_MAGIC2 .word 0 .global _start_kernel _start_kernel: /* Mask all interrupts */ - csrw CSR_SIE, zero - csrw CSR_SIP, zero + csrw CSR_IE, zero + csrw CSR_IP, zero + +#ifdef CONFIG_RISCV_M_MODE + /* flush the instruction cache */ + fence.i + + /* Reset all registers except ra, a0, a1 */ + call reset_regs + + /* + * The hartid in a0 is expected later on, and we have no firmware + * to hand it to us. + */ + csrr a0, CSR_MHARTID +#endif /* CONFIG_RISCV_M_MODE */ /* Load the global pointer */ .option push @@ -61,7 +76,14 @@ _start_kernel: * floating point in kernel space */ li t0, SR_FS - csrc sstatus, t0 + csrc CSR_STATUS, t0 + +#ifdef CONFIG_SMP + li t0, CONFIG_NR_CPUS + blt a0, t0, .Lgood_cores + tail .Lsecondary_park +.Lgood_cores: +#endif /* Pick one hart to run the main boot sequence */ la a3, hart_lottery @@ -89,18 +111,24 @@ clear_bss_done: la sp, init_thread_union + THREAD_SIZE mv a0, s1 call setup_vm +#ifdef CONFIG_MMU la a0, early_pg_dir call relocate +#endif /* CONFIG_MMU */ /* Restore C environment */ la tp, init_task sw zero, TASK_TI_CPU(tp) la sp, init_thread_union + THREAD_SIZE +#ifdef CONFIG_KASAN + call kasan_early_init +#endif /* Start the kernel */ call parse_dtb tail start_kernel +#ifdef CONFIG_MMU relocate: /* Relocate return address */ li a1, PAGE_OFFSET @@ -111,7 +139,7 @@ relocate: /* Point stvec to virtual address of intruction after satp write */ la a2, 1f add a2, a2, a1 - csrw CSR_STVEC, a2 + csrw CSR_TVEC, a2 /* Compute satp for kernel page tables, but don't load it yet */ srl a2, a0, PAGE_SHIFT @@ -133,7 +161,7 @@ relocate: 1: /* Set trap vector to spin forever to help debug */ la a0, .Lsecondary_park - csrw CSR_STVEC, a0 + csrw CSR_TVEC, a0 /* Reload the global pointer */ .option push @@ -151,15 +179,13 @@ relocate: sfence.vma ret +#endif /* CONFIG_MMU */ .Lsecondary_start: #ifdef CONFIG_SMP - li a1, CONFIG_NR_CPUS - bgeu a0, a1, .Lsecondary_park - /* Set trap vector to spin forever to help debug */ la a3, .Lsecondary_park - csrw CSR_STVEC, a3 + csrw CSR_TVEC, a3 slli a3, a0, LGREG la a1, __cpu_up_stack_pointer @@ -179,19 +205,102 @@ relocate: beqz tp, .Lwait_for_cpu_up fence +#ifdef CONFIG_MMU /* Enable virtual memory and relocate to virtual address */ la a0, swapper_pg_dir call relocate +#endif tail smp_callin #endif +END(_start) + +#ifdef CONFIG_RISCV_M_MODE +ENTRY(reset_regs) + li sp, 0 + li gp, 0 + li tp, 0 + li t0, 0 + li t1, 0 + li t2, 0 + li s0, 0 + li s1, 0 + li a2, 0 + li a3, 0 + li a4, 0 + li a5, 0 + li a6, 0 + li a7, 0 + li s2, 0 + li s3, 0 + li s4, 0 + li s5, 0 + li s6, 0 + li s7, 0 + li s8, 0 + li s9, 0 + li s10, 0 + li s11, 0 + li t3, 0 + li t4, 0 + li t5, 0 + li t6, 0 + csrw CSR_SCRATCH, 0 + +#ifdef CONFIG_FPU + csrr t0, CSR_MISA + andi t0, t0, (COMPAT_HWCAP_ISA_F | COMPAT_HWCAP_ISA_D) + beqz t0, .Lreset_regs_done + + li t1, SR_FS + csrs CSR_STATUS, t1 + fmv.s.x f0, zero + fmv.s.x f1, zero + fmv.s.x f2, zero + fmv.s.x f3, zero + fmv.s.x f4, zero + fmv.s.x f5, zero + fmv.s.x f6, zero + fmv.s.x f7, zero + fmv.s.x f8, zero + fmv.s.x f9, zero + fmv.s.x f10, zero + fmv.s.x f11, zero + fmv.s.x f12, zero + fmv.s.x f13, zero + fmv.s.x f14, zero + fmv.s.x f15, zero + fmv.s.x f16, zero + fmv.s.x f17, zero + fmv.s.x f18, zero + fmv.s.x f19, zero + fmv.s.x f20, zero + fmv.s.x f21, zero + fmv.s.x f22, zero + fmv.s.x f23, zero + fmv.s.x f24, zero + fmv.s.x f25, zero + fmv.s.x f26, zero + fmv.s.x f27, zero + fmv.s.x f28, zero + fmv.s.x f29, zero + fmv.s.x f30, zero + fmv.s.x f31, zero + csrw fcsr, 0 + /* note that the caller must clear SR_FS */ +#endif /* CONFIG_FPU */ +.Lreset_regs_done: + ret +END(reset_regs) +#endif /* CONFIG_RISCV_M_MODE */ + +.section ".text", "ax",@progbits .align 2 .Lsecondary_park: /* We lack SMP support or have too many harts, so park this hart */ wfi j .Lsecondary_park -END(_start) __PAGE_ALIGNED_BSS /* Empty zero page */ diff --git a/arch/riscv/kernel/head.h b/arch/riscv/kernel/head.h new file mode 100644 index 000000000000..105fb0496b24 --- /dev/null +++ b/arch/riscv/kernel/head.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2019 SiFive, Inc. + */ +#ifndef __ASM_HEAD_H +#define __ASM_HEAD_H + +#include <linux/linkage.h> +#include <linux/init.h> + +extern atomic_t hart_lottery; + +asmlinkage void do_page_fault(struct pt_regs *regs); +asmlinkage void __init setup_vm(uintptr_t dtb_pa); + +extern void *__cpu_up_stack_pointer[]; +extern void *__cpu_up_task_pointer[]; + +void __init parse_dtb(void); + +#endif /* __ASM_HEAD_H */ diff --git a/arch/riscv/kernel/irq.c b/arch/riscv/kernel/irq.c index 6d8659388c49..345c4f2eba13 100644 --- a/arch/riscv/kernel/irq.c +++ b/arch/riscv/kernel/irq.c @@ -11,30 +11,23 @@ #include <linux/seq_file.h> #include <asm/smp.h> -/* - * Possible interrupt causes: - */ -#define INTERRUPT_CAUSE_SOFTWARE IRQ_S_SOFT -#define INTERRUPT_CAUSE_TIMER IRQ_S_TIMER -#define INTERRUPT_CAUSE_EXTERNAL IRQ_S_EXT - int arch_show_interrupts(struct seq_file *p, int prec) { show_ipi_stats(p, prec); return 0; } -asmlinkage void __irq_entry do_IRQ(struct pt_regs *regs) +asmlinkage __visible void __irq_entry do_IRQ(struct pt_regs *regs) { struct pt_regs *old_regs = set_irq_regs(regs); irq_enter(); - switch (regs->scause & ~SCAUSE_IRQ_FLAG) { - case INTERRUPT_CAUSE_TIMER: + switch (regs->cause & ~CAUSE_IRQ_FLAG) { + case RV_IRQ_TIMER: riscv_timer_interrupt(); break; #ifdef CONFIG_SMP - case INTERRUPT_CAUSE_SOFTWARE: + case RV_IRQ_SOFT: /* * We only use software interrupts to pass IPIs, so if a non-SMP * system gets one, then we don't know what to do. @@ -42,11 +35,11 @@ asmlinkage void __irq_entry do_IRQ(struct pt_regs *regs) riscv_software_interrupt(); break; #endif - case INTERRUPT_CAUSE_EXTERNAL: + case RV_IRQ_EXT: handle_arch_irq(regs); break; default: - pr_alert("unexpected interrupt cause 0x%lx", regs->scause); + pr_alert("unexpected interrupt cause 0x%lx", regs->cause); BUG(); } irq_exit(); diff --git a/arch/riscv/kernel/module-sections.c b/arch/riscv/kernel/module-sections.c index c9ae48333114..e264e59e596e 100644 --- a/arch/riscv/kernel/module-sections.c +++ b/arch/riscv/kernel/module-sections.c @@ -8,6 +8,7 @@ #include <linux/elf.h> #include <linux/kernel.h> #include <linux/module.h> +#include <linux/moduleloader.h> unsigned long module_emit_got_entry(struct module *mod, unsigned long val) { diff --git a/arch/riscv/kernel/module.c b/arch/riscv/kernel/module.c index 70bb94ae61c5..b7401858d872 100644 --- a/arch/riscv/kernel/module.c +++ b/arch/riscv/kernel/module.c @@ -315,8 +315,8 @@ int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, /* Ignore unresolved weak symbol */ if (ELF_ST_BIND(sym->st_info) == STB_WEAK) continue; - pr_warning("%s: Unknown symbol %s\n", - me->name, strtab + sym->st_name); + pr_warn("%s: Unknown symbol %s\n", + me->name, strtab + sym->st_name); return -ENOENT; } diff --git a/arch/riscv/kernel/perf_callchain.c b/arch/riscv/kernel/perf_callchain.c new file mode 100644 index 000000000000..cf190197a22f --- /dev/null +++ b/arch/riscv/kernel/perf_callchain.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd. */ + +#include <linux/perf_event.h> +#include <linux/uaccess.h> + +/* Kernel callchain */ +struct stackframe { + unsigned long fp; + unsigned long ra; +}; + +/* + * Get the return address for a single stackframe and return a pointer to the + * next frame tail. + */ +static unsigned long user_backtrace(struct perf_callchain_entry_ctx *entry, + unsigned long fp, unsigned long reg_ra) +{ + struct stackframe buftail; + unsigned long ra = 0; + unsigned long *user_frame_tail = + (unsigned long *)(fp - sizeof(struct stackframe)); + + /* Check accessibility of one struct frame_tail beyond */ + if (!access_ok(user_frame_tail, sizeof(buftail))) + return 0; + if (__copy_from_user_inatomic(&buftail, user_frame_tail, + sizeof(buftail))) + return 0; + + if (reg_ra != 0) + ra = reg_ra; + else + ra = buftail.ra; + + fp = buftail.fp; + if (ra != 0) + perf_callchain_store(entry, ra); + else + return 0; + + return fp; +} + +/* + * This will be called when the target is in user mode + * This function will only be called when we use + * "PERF_SAMPLE_CALLCHAIN" in + * kernel/events/core.c:perf_prepare_sample() + * + * How to trigger perf_callchain_[user/kernel] : + * $ perf record -e cpu-clock --call-graph fp ./program + * $ perf report --call-graph + * + * On RISC-V platform, the program being sampled and the C library + * need to be compiled with -fno-omit-frame-pointer, otherwise + * the user stack will not contain function frame. + */ +void perf_callchain_user(struct perf_callchain_entry_ctx *entry, + struct pt_regs *regs) +{ + unsigned long fp = 0; + + /* RISC-V does not support perf in guest mode. */ + if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) + return; + + fp = regs->s0; + perf_callchain_store(entry, regs->epc); + + fp = user_backtrace(entry, fp, regs->ra); + while (fp && !(fp & 0x3) && entry->nr < entry->max_stack) + fp = user_backtrace(entry, fp, 0); +} + +bool fill_callchain(unsigned long pc, void *entry) +{ + return perf_callchain_store(entry, pc); +} + +void notrace walk_stackframe(struct task_struct *task, + struct pt_regs *regs, bool (*fn)(unsigned long, void *), void *arg); +void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, + struct pt_regs *regs) +{ + /* RISC-V does not support perf in guest mode. */ + if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { + pr_warn("RISC-V does not support perf in guest mode!"); + return; + } + + walk_stackframe(NULL, regs, fill_callchain, entry); +} diff --git a/arch/riscv/kernel/perf_regs.c b/arch/riscv/kernel/perf_regs.c new file mode 100644 index 000000000000..04a38fbeb9c7 --- /dev/null +++ b/arch/riscv/kernel/perf_regs.c @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd. */ + +#include <linux/errno.h> +#include <linux/kernel.h> +#include <linux/perf_event.h> +#include <linux/bug.h> +#include <asm/perf_regs.h> +#include <asm/ptrace.h> + +u64 perf_reg_value(struct pt_regs *regs, int idx) +{ + if (WARN_ON_ONCE((u32)idx >= PERF_REG_RISCV_MAX)) + return 0; + + return ((unsigned long *)regs)[idx]; +} + +#define REG_RESERVED (~((1ULL << PERF_REG_RISCV_MAX) - 1)) + +int perf_reg_validate(u64 mask) +{ + if (!mask || mask & REG_RESERVED) + return -EINVAL; + + return 0; +} + +u64 perf_reg_abi(struct task_struct *task) +{ +#if __riscv_xlen == 64 + return PERF_SAMPLE_REGS_ABI_64; +#else + return PERF_SAMPLE_REGS_ABI_32; +#endif +} + +void perf_get_regs_user(struct perf_regs *regs_user, + struct pt_regs *regs, + struct pt_regs *regs_user_copy) +{ + regs_user->regs = task_pt_regs(current); + regs_user->abi = perf_reg_abi(current); +} diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c index fb3a082362eb..817cf7b0974c 100644 --- a/arch/riscv/kernel/process.c +++ b/arch/riscv/kernel/process.c @@ -7,6 +7,7 @@ * Copyright (C) 2017 SiFive */ +#include <linux/cpu.h> #include <linux/kernel.h> #include <linux/sched.h> #include <linux/sched/task_stack.h> @@ -19,6 +20,7 @@ #include <asm/csr.h> #include <asm/string.h> #include <asm/switch_to.h> +#include <asm/thread_info.h> extern asmlinkage void ret_from_fork(void); extern asmlinkage void ret_from_kernel_thread(void); @@ -33,8 +35,8 @@ void show_regs(struct pt_regs *regs) { show_regs_print_info(KERN_DEFAULT); - pr_cont("sepc: " REG_FMT " ra : " REG_FMT " sp : " REG_FMT "\n", - regs->sepc, regs->ra, regs->sp); + pr_cont("epc: " REG_FMT " ra : " REG_FMT " sp : " REG_FMT "\n", + regs->epc, regs->ra, regs->sp); pr_cont(" gp : " REG_FMT " tp : " REG_FMT " t0 : " REG_FMT "\n", regs->gp, regs->tp, regs->t0); pr_cont(" t1 : " REG_FMT " t2 : " REG_FMT " s0 : " REG_FMT "\n", @@ -56,23 +58,23 @@ void show_regs(struct pt_regs *regs) pr_cont(" t5 : " REG_FMT " t6 : " REG_FMT "\n", regs->t5, regs->t6); - pr_cont("sstatus: " REG_FMT " sbadaddr: " REG_FMT " scause: " REG_FMT "\n", - regs->sstatus, regs->sbadaddr, regs->scause); + pr_cont("status: " REG_FMT " badaddr: " REG_FMT " cause: " REG_FMT "\n", + regs->status, regs->badaddr, regs->cause); } void start_thread(struct pt_regs *regs, unsigned long pc, unsigned long sp) { - regs->sstatus = SR_SPIE; + regs->status = SR_PIE; if (has_fpu) { - regs->sstatus |= SR_FS_INITIAL; + regs->status |= SR_FS_INITIAL; /* * Restore the initial value to the FP register * before starting the user program. */ fstate_restore(current, regs); } - regs->sepc = pc; + regs->epc = pc; regs->sp = sp; set_fs(USER_DS); } @@ -97,8 +99,8 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) return 0; } -int copy_thread(unsigned long clone_flags, unsigned long usp, - unsigned long arg, struct task_struct *p) +int copy_thread_tls(unsigned long clone_flags, unsigned long usp, + unsigned long arg, struct task_struct *p, unsigned long tls) { struct pt_regs *childregs = task_pt_regs(p); @@ -108,7 +110,8 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, const register unsigned long gp __asm__ ("gp"); memset(childregs, 0, sizeof(struct pt_regs)); childregs->gp = gp; - childregs->sstatus = SR_SPP | SR_SPIE; /* Supervisor, irqs on */ + /* Supervisor/Machine, irqs on: */ + childregs->status = SR_PP | SR_PIE; p->thread.ra = (unsigned long)ret_from_kernel_thread; p->thread.s[0] = usp; /* fn */ @@ -118,7 +121,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, if (usp) /* User fork */ childregs->sp = usp; if (clone_flags & CLONE_SETTLS) - childregs->tp = childregs->a5; + childregs->tp = tls; childregs->a0 = 0; /* Return value of fork() */ p->thread.ra = (unsigned long)ret_from_fork; } diff --git a/arch/riscv/kernel/ptrace.c b/arch/riscv/kernel/ptrace.c index 368751438366..407464201b91 100644 --- a/arch/riscv/kernel/ptrace.c +++ b/arch/riscv/kernel/ptrace.c @@ -148,12 +148,22 @@ long arch_ptrace(struct task_struct *child, long request, * Allows PTRACE_SYSCALL to work. These are called from entry.S in * {handle,ret_from}_syscall. */ -void do_syscall_trace_enter(struct pt_regs *regs) +__visible void do_syscall_trace_enter(struct pt_regs *regs) { if (test_thread_flag(TIF_SYSCALL_TRACE)) if (tracehook_report_syscall_entry(regs)) syscall_set_nr(current, regs, -1); + /* + * Do the secure computing after ptrace; failures should be fast. + * If this fails we might have return value in a0 from seccomp + * (via SECCOMP_RET_ERRNO/TRACE). + */ + if (secure_computing() == -1) { + syscall_set_nr(current, regs, -1); + return; + } + #ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS if (test_thread_flag(TIF_SYSCALL_TRACEPOINT)) trace_sys_enter(regs, syscall_get_nr(current, regs)); @@ -162,7 +172,7 @@ void do_syscall_trace_enter(struct pt_regs *regs) audit_syscall_entry(regs->a7, regs->a0, regs->a1, regs->a2, regs->a3); } -void do_syscall_trace_exit(struct pt_regs *regs) +__visible void do_syscall_trace_exit(struct pt_regs *regs) { audit_syscall_exit(regs); diff --git a/arch/riscv/kernel/reset.c b/arch/riscv/kernel/reset.c index d0fe623bfb8f..ee5878d968cc 100644 --- a/arch/riscv/kernel/reset.c +++ b/arch/riscv/kernel/reset.c @@ -4,12 +4,12 @@ */ #include <linux/reboot.h> -#include <asm/sbi.h> +#include <linux/pm.h> static void default_power_off(void) { - sbi_shutdown(); - while (1); + while (1) + wait_for_interrupt(); } void (*pm_power_off)(void) = default_power_off; diff --git a/arch/riscv/kernel/riscv_ksyms.c b/arch/riscv/kernel/riscv_ksyms.c index 4800cf703186..450492e1cb4e 100644 --- a/arch/riscv/kernel/riscv_ksyms.c +++ b/arch/riscv/kernel/riscv_ksyms.c @@ -9,8 +9,7 @@ /* * Assembly functions that may be used (directly or indirectly) by modules */ -EXPORT_SYMBOL(__clear_user); -EXPORT_SYMBOL(__asm_copy_to_user); -EXPORT_SYMBOL(__asm_copy_from_user); EXPORT_SYMBOL(memset); EXPORT_SYMBOL(memcpy); +EXPORT_SYMBOL(__memset); +EXPORT_SYMBOL(__memcpy); diff --git a/arch/riscv/kernel/sbi.c b/arch/riscv/kernel/sbi.c new file mode 100644 index 000000000000..f6c7c3e82d28 --- /dev/null +++ b/arch/riscv/kernel/sbi.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <linux/init.h> +#include <linux/pm.h> +#include <asm/sbi.h> + +static void sbi_power_off(void) +{ + sbi_shutdown(); +} + +static int __init sbi_init(void) +{ + pm_power_off = sbi_power_off; + return 0; +} +early_initcall(sbi_init); diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index a990a6cb184f..0a6d415b0a5a 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -17,12 +17,16 @@ #include <linux/sched/task.h> #include <linux/swiotlb.h> +#include <asm/clint.h> #include <asm/setup.h> #include <asm/sections.h> #include <asm/pgtable.h> #include <asm/smp.h> #include <asm/tlbflush.h> #include <asm/thread_info.h> +#include <asm/kasan.h> + +#include "head.h" #ifdef CONFIG_DUMMY_CONSOLE struct screen_info screen_info = { @@ -65,17 +69,18 @@ void __init setup_arch(char **cmdline_p) setup_bootmem(); paging_init(); unflatten_device_tree(); + clint_init_boot_cpu(); #ifdef CONFIG_SWIOTLB swiotlb_init(1); #endif -#ifdef CONFIG_SMP - setup_smp(); +#ifdef CONFIG_KASAN + kasan_init(); #endif -#ifdef CONFIG_DUMMY_CONSOLE - conswitchp = &dummy_con; +#ifdef CONFIG_SMP + setup_smp(); #endif riscv_fill_hwcap(); diff --git a/arch/riscv/kernel/signal.c b/arch/riscv/kernel/signal.c index b14d7647d800..17ba190e84a5 100644 --- a/arch/riscv/kernel/signal.c +++ b/arch/riscv/kernel/signal.c @@ -17,16 +17,21 @@ #include <asm/switch_to.h> #include <asm/csr.h> +extern u32 __user_rt_sigreturn[2]; + #define DEBUG_SIG 0 struct rt_sigframe { struct siginfo info; struct ucontext uc; +#ifndef CONFIG_MMU + u32 sigreturn_code[2]; +#endif }; #ifdef CONFIG_FPU static long restore_fp_state(struct pt_regs *regs, - union __riscv_fp_state *sc_fpregs) + union __riscv_fp_state __user *sc_fpregs) { long err; struct __riscv_d_ext_state __user *state = &sc_fpregs->d; @@ -53,7 +58,7 @@ static long restore_fp_state(struct pt_regs *regs, } static long save_fp_state(struct pt_regs *regs, - union __riscv_fp_state *sc_fpregs) + union __riscv_fp_state __user *sc_fpregs) { long err; struct __riscv_d_ext_state __user *state = &sc_fpregs->d; @@ -124,7 +129,7 @@ badframe: pr_info_ratelimited( "%s[%d]: bad frame in %s: frame=%p pc=%p sp=%p\n", task->comm, task_pid_nr(task), __func__, - frame, (void *)regs->sepc, (void *)regs->sp); + frame, (void *)regs->epc, (void *)regs->sp); } force_sig(SIGSEGV); return 0; @@ -166,7 +171,6 @@ static inline void __user *get_sigframe(struct ksignal *ksig, return (void __user *)sp; } - static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) { @@ -189,8 +193,19 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, return -EFAULT; /* Set up to return from userspace. */ +#ifdef CONFIG_MMU regs->ra = (unsigned long)VDSO_SYMBOL( current->mm->context.vdso, rt_sigreturn); +#else + /* + * For the nommu case we don't have a VDSO. Instead we push two + * instructions to call the rt_sigreturn syscall onto the user stack. + */ + if (copy_to_user(&frame->sigreturn_code, __user_rt_sigreturn, + sizeof(frame->sigreturn_code))) + return -EFAULT; + regs->ra = (unsigned long)&frame->sigreturn_code; +#endif /* CONFIG_MMU */ /* * Set up registers for signal handler. @@ -199,7 +214,7 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, * We always pass siginfo and mcontext, regardless of SA_SIGINFO, * since some things rely on this (e.g. glibc's debug/segfault.c). */ - regs->sepc = (unsigned long)ksig->ka.sa.sa_handler; + regs->epc = (unsigned long)ksig->ka.sa.sa_handler; regs->sp = (unsigned long)frame; regs->a0 = ksig->sig; /* a0: signal number */ regs->a1 = (unsigned long)(&frame->info); /* a1: siginfo pointer */ @@ -208,7 +223,7 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, #if DEBUG_SIG pr_info("SIG deliver (%s:%d): sig=%d pc=%p ra=%p sp=%p\n", current->comm, task_pid_nr(current), ksig->sig, - (void *)regs->sepc, (void *)regs->ra, frame); + (void *)regs->epc, (void *)regs->ra, frame); #endif return 0; @@ -220,10 +235,9 @@ static void handle_signal(struct ksignal *ksig, struct pt_regs *regs) int ret; /* Are we from a system call? */ - if (regs->scause == EXC_SYSCALL) { + if (regs->cause == EXC_SYSCALL) { /* Avoid additional syscall restarting via ret_from_exception */ - regs->scause = -1UL; - + regs->cause = -1UL; /* If so, check system call restarting.. */ switch (regs->a0) { case -ERESTART_RESTARTBLOCK: @@ -239,7 +253,7 @@ static void handle_signal(struct ksignal *ksig, struct pt_regs *regs) /* fallthrough */ case -ERESTARTNOINTR: regs->a0 = regs->orig_a0; - regs->sepc -= 0x4; + regs->epc -= 0x4; break; } } @@ -261,9 +275,9 @@ static void do_signal(struct pt_regs *regs) } /* Did we come from a system call? */ - if (regs->scause == EXC_SYSCALL) { + if (regs->cause == EXC_SYSCALL) { /* Avoid additional syscall restarting via ret_from_exception */ - regs->scause = -1UL; + regs->cause = -1UL; /* Restart the system call - no handlers present */ switch (regs->a0) { @@ -271,12 +285,12 @@ static void do_signal(struct pt_regs *regs) case -ERESTARTSYS: case -ERESTARTNOINTR: regs->a0 = regs->orig_a0; - regs->sepc -= 0x4; + regs->epc -= 0x4; break; case -ERESTART_RESTARTBLOCK: regs->a0 = regs->orig_a0; regs->a7 = __NR_restart_syscall; - regs->sepc -= 0x4; + regs->epc -= 0x4; break; } } @@ -292,8 +306,8 @@ static void do_signal(struct pt_regs *regs) * notification of userspace execution resumption * - triggered by the _TIF_WORK_MASK flags */ -asmlinkage void do_notify_resume(struct pt_regs *regs, - unsigned long thread_info_flags) +asmlinkage __visible void do_notify_resume(struct pt_regs *regs, + unsigned long thread_info_flags) { /* Handle pending signal delivery */ if (thread_info_flags & _TIF_SIGPENDING) diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index 5a9834503a2f..eb878abcaaf8 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -8,12 +8,15 @@ * Copyright (C) 2017 SiFive */ +#include <linux/cpu.h> #include <linux/interrupt.h> +#include <linux/profile.h> #include <linux/smp.h> #include <linux/sched.h> #include <linux/seq_file.h> #include <linux/delay.h> +#include <asm/clint.h> #include <asm/sbi.h> #include <asm/tlbflush.h> #include <asm/cacheflush.h> @@ -56,6 +59,7 @@ void riscv_cpuid_to_hartid_mask(const struct cpumask *in, struct cpumask *out) { int cpu; + cpumask_clear(out); for_each_cpu(cpu, in) cpumask_set_cpu(cpuid_to_hartid_map(cpu), out); } @@ -78,13 +82,51 @@ static void ipi_stop(void) wait_for_interrupt(); } +static void send_ipi_mask(const struct cpumask *mask, enum ipi_message_type op) +{ + struct cpumask hartid_mask; + int cpu; + + smp_mb__before_atomic(); + for_each_cpu(cpu, mask) + set_bit(op, &ipi_data[cpu].bits); + smp_mb__after_atomic(); + + riscv_cpuid_to_hartid_mask(mask, &hartid_mask); + if (IS_ENABLED(CONFIG_RISCV_SBI)) + sbi_send_ipi(cpumask_bits(&hartid_mask)); + else + clint_send_ipi_mask(&hartid_mask); +} + +static void send_ipi_single(int cpu, enum ipi_message_type op) +{ + int hartid = cpuid_to_hartid_map(cpu); + + smp_mb__before_atomic(); + set_bit(op, &ipi_data[cpu].bits); + smp_mb__after_atomic(); + + if (IS_ENABLED(CONFIG_RISCV_SBI)) + sbi_send_ipi(cpumask_bits(cpumask_of(hartid))); + else + clint_send_ipi_single(hartid); +} + +static inline void clear_ipi(void) +{ + if (IS_ENABLED(CONFIG_RISCV_SBI)) + csr_clear(CSR_IP, IE_SIE); + else + clint_clear_ipi(cpuid_to_hartid_map(smp_processor_id())); +} + void riscv_software_interrupt(void) { unsigned long *pending_ipis = &ipi_data[smp_processor_id()].bits; unsigned long *stats = ipi_data[smp_processor_id()].stats; - /* Clear pending IPI */ - csr_clear(CSR_SIP, SIE_SSIE); + clear_ipi(); while (true) { unsigned long ops; @@ -118,23 +160,6 @@ void riscv_software_interrupt(void) } } -static void -send_ipi_message(const struct cpumask *to_whom, enum ipi_message_type operation) -{ - int cpuid, hartid; - struct cpumask hartid_mask; - - cpumask_clear(&hartid_mask); - mb(); - for_each_cpu(cpuid, to_whom) { - set_bit(operation, &ipi_data[cpuid].bits); - hartid = cpuid_to_hartid_map(cpuid); - cpumask_set_cpu(hartid, &hartid_mask); - } - mb(); - sbi_send_ipi(cpumask_bits(&hartid_mask)); -} - static const char * const ipi_names[] = { [IPI_RESCHEDULE] = "Rescheduling interrupts", [IPI_CALL_FUNC] = "Function call interrupts", @@ -156,12 +181,12 @@ void show_ipi_stats(struct seq_file *p, int prec) void arch_send_call_function_ipi_mask(struct cpumask *mask) { - send_ipi_message(mask, IPI_CALL_FUNC); + send_ipi_mask(mask, IPI_CALL_FUNC); } void arch_send_call_function_single_ipi(int cpu) { - send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC); + send_ipi_single(cpu, IPI_CALL_FUNC); } void smp_send_stop(void) @@ -176,7 +201,7 @@ void smp_send_stop(void) if (system_state <= SYSTEM_RUNNING) pr_crit("SMP: stopping secondary CPUs\n"); - send_ipi_message(&mask, IPI_CPU_STOP); + send_ipi_mask(&mask, IPI_CPU_STOP); } /* Wait up to one second for other CPUs to stop */ @@ -191,6 +216,6 @@ void smp_send_stop(void) void smp_send_reschedule(int cpu) { - send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE); + send_ipi_single(cpu, IPI_RESCHEDULE); } - +EXPORT_SYMBOL_GPL(smp_send_reschedule); diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c index 7462a44304fe..8bc01f0ca73b 100644 --- a/arch/riscv/kernel/smpboot.c +++ b/arch/riscv/kernel/smpboot.c @@ -8,6 +8,7 @@ * Copyright (C) 2017 SiFive */ +#include <linux/arch_topology.h> #include <linux/module.h> #include <linux/init.h> #include <linux/kernel.h> @@ -23,11 +24,15 @@ #include <linux/of.h> #include <linux/sched/task_stack.h> #include <linux/sched/mm.h> +#include <asm/clint.h> #include <asm/irq.h> #include <asm/mmu_context.h> #include <asm/tlbflush.h> #include <asm/sections.h> #include <asm/sbi.h> +#include <asm/smp.h> + +#include "head.h" void *__cpu_up_stack_pointer[NR_CPUS]; void *__cpu_up_task_pointer[NR_CPUS]; @@ -35,6 +40,7 @@ static DECLARE_COMPLETION(cpu_running); void __init smp_prepare_boot_cpu(void) { + init_cpu_topology(); } void __init smp_prepare_cpus(unsigned int max_cpus) @@ -128,16 +134,20 @@ void __init smp_cpus_done(unsigned int max_cpus) /* * C entry point for a secondary processor. */ -asmlinkage void __init smp_callin(void) +asmlinkage __visible void __init smp_callin(void) { struct mm_struct *mm = &init_mm; + if (!IS_ENABLED(CONFIG_RISCV_SBI)) + clint_clear_ipi(cpuid_to_hartid_map(smp_processor_id())); + /* All kernel threads share the same mm context. */ mmgrab(mm); current->active_mm = mm; trap_init(); notify_cpu_starting(smp_processor_id()); + update_siblings_masks(smp_processor_id()); set_cpu_online(smp_processor_id(), 1); /* * Remote TLB flushes are ignored while the CPU is offline, so emit diff --git a/arch/riscv/kernel/stacktrace.c b/arch/riscv/kernel/stacktrace.c index f15642715d1a..0940681d2f68 100644 --- a/arch/riscv/kernel/stacktrace.c +++ b/arch/riscv/kernel/stacktrace.c @@ -19,8 +19,8 @@ struct stackframe { unsigned long ra; }; -static void notrace walk_stackframe(struct task_struct *task, - struct pt_regs *regs, bool (*fn)(unsigned long, void *), void *arg) +void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, + bool (*fn)(unsigned long, void *), void *arg) { unsigned long fp, sp, pc; diff --git a/arch/riscv/kernel/syscall_table.c b/arch/riscv/kernel/syscall_table.c index e5dd52d8f633..f1ead9df96ca 100644 --- a/arch/riscv/kernel/syscall_table.c +++ b/arch/riscv/kernel/syscall_table.c @@ -8,6 +8,7 @@ #include <linux/syscalls.h> #include <asm-generic/syscalls.h> #include <asm/vdso.h> +#include <asm/syscall.h> #undef __SYSCALL #define __SYSCALL(nr, call) [nr] = (call), diff --git a/arch/riscv/kernel/time.c b/arch/riscv/kernel/time.c index 541a2b885814..6a53c02e9c73 100644 --- a/arch/riscv/kernel/time.c +++ b/arch/riscv/kernel/time.c @@ -7,8 +7,10 @@ #include <linux/clocksource.h> #include <linux/delay.h> #include <asm/sbi.h> +#include <asm/processor.h> unsigned long riscv_timebase; +EXPORT_SYMBOL_GPL(riscv_timebase); void __init time_init(void) { diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index 424eb72d56b1..f4cad5163bf2 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -3,6 +3,7 @@ * Copyright (C) 2012 Regents of the University of California */ +#include <linux/cpu.h> #include <linux/kernel.h> #include <linux/init.h> #include <linux/sched.h> @@ -40,7 +41,7 @@ void die(struct pt_regs *regs, const char *str) print_modules(); show_regs(regs); - ret = notify_die(DIE_OOPS, str, regs, 0, regs->scause, SIGSEGV); + ret = notify_die(DIE_OOPS, str, regs, 0, regs->cause, SIGSEGV); bust_spinlocks(0); add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); @@ -83,9 +84,9 @@ static void do_trap_error(struct pt_regs *regs, int signo, int code, } #define DO_ERROR_INFO(name, signo, code, str) \ -asmlinkage void name(struct pt_regs *regs) \ +asmlinkage __visible void name(struct pt_regs *regs) \ { \ - do_trap_error(regs, signo, code, regs->sepc, "Oops - " str); \ + do_trap_error(regs, signo, code, regs->epc, "Oops - " str); \ } DO_ERROR_INFO(do_trap_unknown, @@ -111,7 +112,6 @@ DO_ERROR_INFO(do_trap_ecall_s, DO_ERROR_INFO(do_trap_ecall_m, SIGILL, ILL_ILLTRP, "environment call from M-mode"); -#ifdef CONFIG_GENERIC_BUG static inline unsigned long get_break_insn_length(unsigned long pc) { bug_insn_t insn; @@ -120,28 +120,15 @@ static inline unsigned long get_break_insn_length(unsigned long pc) return 0; return (((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) ? 4UL : 2UL); } -#endif /* CONFIG_GENERIC_BUG */ -asmlinkage void do_trap_break(struct pt_regs *regs) +asmlinkage __visible void do_trap_break(struct pt_regs *regs) { -#ifdef CONFIG_GENERIC_BUG - if (!user_mode(regs)) { - enum bug_trap_type type; - - type = report_bug(regs->sepc, regs); - switch (type) { - case BUG_TRAP_TYPE_NONE: - break; - case BUG_TRAP_TYPE_WARN: - regs->sepc += get_break_insn_length(regs->sepc); - break; - case BUG_TRAP_TYPE_BUG: - die(regs, "Kernel BUG"); - } - } -#endif /* CONFIG_GENERIC_BUG */ - - force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)(regs->sepc)); + if (user_mode(regs)) + force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->epc); + else if (report_bug(regs->epc, regs) == BUG_TRAP_TYPE_WARN) + regs->epc += get_break_insn_length(regs->epc); + else + die(regs, "Kernel BUG"); } #ifdef CONFIG_GENERIC_BUG @@ -166,9 +153,9 @@ void __init trap_init(void) * Set sup0 scratch register to 0, indicating to exception vector * that we are presently executing in the kernel */ - csr_write(CSR_SSCRATCH, 0); + csr_write(CSR_SCRATCH, 0); /* Set the exception vector address */ - csr_write(CSR_STVEC, &handle_exception); + csr_write(CSR_TVEC, &handle_exception); /* Enable all interrupts */ - csr_write(CSR_SIE, -1); + csr_write(CSR_IE, -1); } diff --git a/arch/riscv/kernel/vdso.c b/arch/riscv/kernel/vdso.c index c9c21e0d5641..484d95a70907 100644 --- a/arch/riscv/kernel/vdso.c +++ b/arch/riscv/kernel/vdso.c @@ -6,6 +6,7 @@ * Copyright (C) 2015 Regents of the University of California */ +#include <linux/elf.h> #include <linux/mm.h> #include <linux/slab.h> #include <linux/binfmts.h> @@ -25,7 +26,7 @@ static union { struct vdso_data data; u8 page[PAGE_SIZE]; } vdso_data_store __page_aligned_data; -struct vdso_data *vdso_data = &vdso_data_store.data; +static struct vdso_data *vdso_data = &vdso_data_store.data; static int __init vdso_init(void) { diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index 49a5852fd07d..33b16f4212f7 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -58,7 +58,8 @@ quiet_cmd_vdsold = VDSOLD $@ cmd_vdsold = $(CC) $(KBUILD_CFLAGS) $(call cc-option, -no-pie) -nostdlib -nostartfiles $(SYSCFLAGS_$(@F)) \ -Wl,-T,$(filter-out FORCE,$^) -o $@.tmp && \ $(CROSS_COMPILE)objcopy \ - $(patsubst %, -G __vdso_%, $(vdso-syms)) $@.tmp $@ + $(patsubst %, -G __vdso_%, $(vdso-syms)) $@.tmp $@ && \ + rm $@.tmp # install commands for the unstripped file quiet_cmd_vdso_install = INSTALL $@ diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S index 23cd1a9e52a1..1e0193ded420 100644 --- a/arch/riscv/kernel/vmlinux.lds.S +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -46,18 +46,19 @@ SECTIONS KPROBES_TEXT ENTRY_TEXT IRQENTRY_TEXT + SOFTIRQENTRY_TEXT *(.fixup) _etext = .; } /* Start of data section */ _sdata = .; - RO_DATA_SECTION(L1_CACHE_BYTES) + RO_DATA(L1_CACHE_BYTES) .srodata : { *(.srodata*) } - RW_DATA_SECTION(L1_CACHE_BYTES, PAGE_SIZE, THREAD_SIZE) + RW_DATA(L1_CACHE_BYTES, PAGE_SIZE, THREAD_SIZE) .sdata : { __global_pointer$ = . + 0x800; *(.sdata*) @@ -69,7 +70,6 @@ SECTIONS BSS_SECTION(PAGE_SIZE, PAGE_SIZE, 0) EXCEPTION_TABLE(0x10) - NOTES .rel.dyn : { *(.rel.dyn*) diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile index 267feaa10f6a..47e7a8204460 100644 --- a/arch/riscv/lib/Makefile +++ b/arch/riscv/lib/Makefile @@ -1,7 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only -lib-y += delay.o -lib-y += memcpy.o -lib-y += memset.o -lib-y += uaccess.o - -lib-$(CONFIG_64BIT) += tishift.o +lib-y += delay.o +lib-y += memcpy.o +lib-y += memset.o +lib-$(CONFIG_MMU) += uaccess.o +lib-$(CONFIG_64BIT) += tishift.o diff --git a/arch/riscv/lib/memcpy.S b/arch/riscv/lib/memcpy.S index b4c477846e91..51ab716253fa 100644 --- a/arch/riscv/lib/memcpy.S +++ b/arch/riscv/lib/memcpy.S @@ -7,7 +7,8 @@ #include <asm/asm.h> /* void *memcpy(void *, const void *, size_t) */ -ENTRY(memcpy) +ENTRY(__memcpy) +WEAK(memcpy) move t6, a0 /* Preserve return value */ /* Defer to byte-oriented copy for small sizes */ @@ -104,4 +105,4 @@ ENTRY(memcpy) bltu a1, a3, 5b 6: ret -END(memcpy) +END(__memcpy) diff --git a/arch/riscv/lib/memset.S b/arch/riscv/lib/memset.S index 5a7386b47175..34c5360c6705 100644 --- a/arch/riscv/lib/memset.S +++ b/arch/riscv/lib/memset.S @@ -8,7 +8,8 @@ #include <asm/asm.h> /* void *memset(void *, int, size_t) */ -ENTRY(memset) +ENTRY(__memset) +WEAK(memset) move t0, a0 /* Preserve return value */ /* Defer to byte-oriented fill for small sizes */ @@ -109,4 +110,4 @@ ENTRY(memset) bltu t0, a3, 5b 6: ret -END(memset) +END(__memset) diff --git a/arch/riscv/lib/tishift.S b/arch/riscv/lib/tishift.S index 15f9d54c7db6..ef90075c4b0a 100644 --- a/arch/riscv/lib/tishift.S +++ b/arch/riscv/lib/tishift.S @@ -4,34 +4,73 @@ */ #include <linux/linkage.h> +#include <asm-generic/export.h> -ENTRY(__lshrti3) +SYM_FUNC_START(__lshrti3) beqz a2, .L1 li a5,64 sub a5,a5,a2 - addi sp,sp,-16 sext.w a4,a5 blez a5, .L2 sext.w a2,a2 - sll a4,a1,a4 srl a0,a0,a2 - srl a1,a1,a2 + sll a4,a1,a4 + srl a2,a1,a2 or a0,a0,a4 - sd a1,8(sp) - sd a0,0(sp) - ld a0,0(sp) - ld a1,8(sp) - addi sp,sp,16 - ret + mv a1,a2 .L1: ret .L2: - negw a4,a4 - srl a1,a1,a4 - sd a1,0(sp) - sd zero,8(sp) - ld a0,0(sp) - ld a1,8(sp) - addi sp,sp,16 + negw a0,a4 + li a2,0 + srl a0,a1,a0 + mv a1,a2 + ret +SYM_FUNC_END(__lshrti3) +EXPORT_SYMBOL(__lshrti3) + +SYM_FUNC_START(__ashrti3) + beqz a2, .L3 + li a5,64 + sub a5,a5,a2 + sext.w a4,a5 + blez a5, .L4 + sext.w a2,a2 + srl a0,a0,a2 + sll a4,a1,a4 + sra a2,a1,a2 + or a0,a0,a4 + mv a1,a2 +.L3: + ret +.L4: + negw a0,a4 + srai a2,a1,0x3f + sra a0,a1,a0 + mv a1,a2 + ret +SYM_FUNC_END(__ashrti3) +EXPORT_SYMBOL(__ashrti3) + +SYM_FUNC_START(__ashlti3) + beqz a2, .L5 + li a5,64 + sub a5,a5,a2 + sext.w a4,a5 + blez a5, .L6 + sext.w a2,a2 + sll a1,a1,a2 + srl a4,a0,a4 + sll a2,a0,a2 + or a1,a1,a4 + mv a0,a2 +.L5: + ret +.L6: + negw a1,a4 + li a2,0 + sll a1,a0,a1 + mv a0,a2 ret -ENDPROC(__lshrti3) +SYM_FUNC_END(__ashlti3) +EXPORT_SYMBOL(__ashlti3) diff --git a/arch/riscv/lib/uaccess.S b/arch/riscv/lib/uaccess.S index 399e6f0c2d98..f29d2ba2c0a6 100644 --- a/arch/riscv/lib/uaccess.S +++ b/arch/riscv/lib/uaccess.S @@ -1,4 +1,5 @@ #include <linux/linkage.h> +#include <asm-generic/export.h> #include <asm/asm.h> #include <asm/csr.h> @@ -18,7 +19,7 @@ ENTRY(__asm_copy_from_user) /* Enable access to user memory */ li t6, SR_SUM - csrs sstatus, t6 + csrs CSR_STATUS, t6 add a3, a1, a2 /* Use word-oriented copy only if low-order bits match */ @@ -47,7 +48,7 @@ ENTRY(__asm_copy_from_user) 3: /* Disable access to user memory */ - csrc sstatus, t6 + csrc CSR_STATUS, t6 li a0, 0 ret 4: /* Edge case: unalignment */ @@ -66,13 +67,15 @@ ENTRY(__asm_copy_from_user) j 3b ENDPROC(__asm_copy_to_user) ENDPROC(__asm_copy_from_user) +EXPORT_SYMBOL(__asm_copy_to_user) +EXPORT_SYMBOL(__asm_copy_from_user) ENTRY(__clear_user) /* Enable access to user memory */ li t6, SR_SUM - csrs sstatus, t6 + csrs CSR_STATUS, t6 add a3, a0, a1 addi t0, a0, SZREG-1 @@ -94,7 +97,7 @@ ENTRY(__clear_user) 3: /* Disable access to user memory */ - csrc sstatus, t6 + csrc CSR_STATUS, t6 li a0, 0 ret 4: /* Edge case: unalignment */ @@ -108,17 +111,18 @@ ENTRY(__clear_user) bltu a0, a3, 5b j 3b ENDPROC(__clear_user) +EXPORT_SYMBOL(__clear_user) .section .fixup,"ax" .balign 4 /* Fixup code for __copy_user(10) and __clear_user(11) */ 10: /* Disable access to user memory */ - csrs sstatus, t6 + csrs CSR_STATUS, t6 mv a0, a2 ret 11: - csrs sstatus, t6 + csrs CSR_STATUS, t6 mv a0, a1 ret .previous diff --git a/arch/riscv/mm/Makefile b/arch/riscv/mm/Makefile index 74055e1d6f21..50b7af58c566 100644 --- a/arch/riscv/mm/Makefile +++ b/arch/riscv/mm/Makefile @@ -6,11 +6,20 @@ CFLAGS_REMOVE_init.o = -pg endif obj-y += init.o -obj-y += fault.o obj-y += extable.o -obj-y += ioremap.o +obj-$(CONFIG_MMU) += fault.o obj-y += cacheflush.o obj-y += context.o -obj-y += sifive_l2_cache.o +ifeq ($(CONFIG_MMU),y) +obj-$(CONFIG_SMP) += tlbflush.o +endif obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o +obj-$(CONFIG_KASAN) += kasan_init.o + +ifdef CONFIG_KASAN +KASAN_SANITIZE_kasan_init.o := n +KASAN_SANITIZE_init.o := n +endif + +obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o diff --git a/arch/riscv/mm/cacheflush.c b/arch/riscv/mm/cacheflush.c index 9ebcff8ba263..8930ab7278e6 100644 --- a/arch/riscv/mm/cacheflush.c +++ b/arch/riscv/mm/cacheflush.c @@ -10,10 +10,19 @@ #include <asm/sbi.h> +static void ipi_remote_fence_i(void *info) +{ + return local_flush_icache_all(); +} + void flush_icache_all(void) { - sbi_remote_fence_i(NULL); + if (IS_ENABLED(CONFIG_RISCV_SBI)) + sbi_remote_fence_i(NULL); + else + on_each_cpu(ipi_remote_fence_i, NULL, 1); } +EXPORT_SYMBOL(flush_icache_all); /* * Performs an icache flush for the given MM context. RISC-V has no direct @@ -28,7 +37,7 @@ void flush_icache_all(void) void flush_icache_mm(struct mm_struct *mm, bool local) { unsigned int cpu; - cpumask_t others, hmask, *mask; + cpumask_t others, *mask; preempt_disable(); @@ -46,11 +55,7 @@ void flush_icache_mm(struct mm_struct *mm, bool local) */ cpumask_andnot(&others, mm_cpumask(mm), cpumask_of(cpu)); local |= cpumask_empty(&others); - if (mm != current->active_mm || !local) { - cpumask_clear(&hmask); - riscv_cpuid_to_hartid_mask(&others, &hmask); - sbi_remote_fence_i(hmask.bits); - } else { + if (mm == current->active_mm && local) { /* * It's assumed that at least one strongly ordered operation is * performed on this hart between setting a hart's cpumask bit @@ -60,6 +65,13 @@ void flush_icache_mm(struct mm_struct *mm, bool local) * with flush_icache_deferred(). */ smp_mb(); + } else if (IS_ENABLED(CONFIG_RISCV_SBI)) { + cpumask_t hartid_mask; + + riscv_cpuid_to_hartid_mask(&others, &hartid_mask); + sbi_remote_fence_i(cpumask_bits(&hartid_mask)); + } else { + on_each_cpu_mask(&others, ipi_remote_fence_i, NULL, 1); } preempt_enable(); @@ -67,6 +79,7 @@ void flush_icache_mm(struct mm_struct *mm, bool local) #endif /* CONFIG_SMP */ +#ifdef CONFIG_MMU void flush_icache_pte(pte_t pte) { struct page *page = pte_page(pte); @@ -74,3 +87,4 @@ void flush_icache_pte(pte_t pte) if (!test_and_set_bit(PG_dcache_clean, &page->flags)) flush_icache_all(); } +#endif /* CONFIG_MMU */ diff --git a/arch/riscv/mm/context.c b/arch/riscv/mm/context.c index 89ceb3cbe218..613ec81a8979 100644 --- a/arch/riscv/mm/context.c +++ b/arch/riscv/mm/context.c @@ -7,6 +7,7 @@ #include <linux/mm.h> #include <asm/tlbflush.h> #include <asm/cacheflush.h> +#include <asm/mmu_context.h> /* * When necessary, performs a deferred icache flush for the given MM context, @@ -57,13 +58,10 @@ void switch_mm(struct mm_struct *prev, struct mm_struct *next, cpumask_clear_cpu(cpu, mm_cpumask(prev)); cpumask_set_cpu(cpu, mm_cpumask(next)); - /* - * Use the old spbtr name instead of using the current satp - * name to support binutils 2.29 which doesn't know about the - * privileged ISA 1.10 yet. - */ - csr_write(sptbr, virt_to_pfn(next->pgd) | SATP_MODE); +#ifdef CONFIG_MMU + csr_write(CSR_SATP, virt_to_pfn(next->pgd) | SATP_MODE); local_flush_tlb_all(); +#endif flush_icache_deferred(next); } diff --git a/arch/riscv/mm/extable.c b/arch/riscv/mm/extable.c index 7aed9178d365..2fc729422151 100644 --- a/arch/riscv/mm/extable.c +++ b/arch/riscv/mm/extable.c @@ -15,9 +15,9 @@ int fixup_exception(struct pt_regs *regs) { const struct exception_table_entry *fixup; - fixup = search_exception_tables(regs->sepc); + fixup = search_exception_tables(regs->epc); if (fixup) { - regs->sepc = fixup->fixup; + regs->epc = fixup->fixup; return 1; } return 0; diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c index 96add1427a75..cf7248e07f43 100644 --- a/arch/riscv/mm/fault.c +++ b/arch/riscv/mm/fault.c @@ -18,6 +18,8 @@ #include <asm/ptrace.h> #include <asm/tlbflush.h> +#include "../kernel/head.h" + /* * This routine handles page faults. It determines the address and the * problem, and then passes it off to one of the appropriate routines. @@ -32,8 +34,8 @@ asmlinkage void do_page_fault(struct pt_regs *regs) int code = SEGV_MAPERR; vm_fault_t fault; - cause = regs->scause; - addr = regs->sbadaddr; + cause = regs->cause; + addr = regs->badaddr; tsk = current; mm = tsk->mm; @@ -51,7 +53,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs) goto vmalloc_fault; /* Enable interrupts if they were enabled in the parent context. */ - if (likely(regs->sstatus & SR_SPIE)) + if (likely(regs->status & SR_PIE)) local_irq_enable(); /* diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index 42bf939693d3..965a8cf4829c 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -11,6 +11,7 @@ #include <linux/swap.h> #include <linux/sizes.h> #include <linux/of_fdt.h> +#include <linux/libfdt.h> #include <asm/fixmap.h> #include <asm/tlbflush.h> @@ -18,11 +19,14 @@ #include <asm/pgtable.h> #include <asm/io.h> +#include "../kernel/head.h" + unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss; EXPORT_SYMBOL(empty_zero_page); extern char _start[]; +void *dtb_early_va; static void __init zone_sizes_init(void) { @@ -37,11 +41,42 @@ static void __init zone_sizes_init(void) free_area_init_nodes(max_zone_pfns); } -void setup_zero_page(void) +static void setup_zero_page(void) { memset((void *)empty_zero_page, 0, PAGE_SIZE); } +#ifdef CONFIG_DEBUG_VM +static inline void print_mlk(char *name, unsigned long b, unsigned long t) +{ + pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld kB)\n", name, b, t, + (((t) - (b)) >> 10)); +} + +static inline void print_mlm(char *name, unsigned long b, unsigned long t) +{ + pr_notice("%12s : 0x%08lx - 0x%08lx (%4ld MB)\n", name, b, t, + (((t) - (b)) >> 20)); +} + +static void print_vm_layout(void) +{ + pr_notice("Virtual kernel memory layout:\n"); + print_mlk("fixmap", (unsigned long)FIXADDR_START, + (unsigned long)FIXADDR_TOP); + print_mlm("pci io", (unsigned long)PCI_IO_START, + (unsigned long)PCI_IO_END); + print_mlm("vmemmap", (unsigned long)VMEMMAP_START, + (unsigned long)VMEMMAP_END); + print_mlm("vmalloc", (unsigned long)VMALLOC_START, + (unsigned long)VMALLOC_END); + print_mlm("lowmem", (unsigned long)PAGE_OFFSET, + (unsigned long)high_memory); +} +#else +static void print_vm_layout(void) { } +#endif /* CONFIG_DEBUG_VM */ + void __init mem_init(void) { #ifdef CONFIG_FLATMEM @@ -52,6 +87,7 @@ void __init mem_init(void) memblock_free_all(); mem_init_print_info(NULL); + print_vm_layout(); } #ifdef CONFIG_BLK_DEV_INITRD @@ -63,13 +99,13 @@ static void __init setup_initrd(void) pr_info("initrd not found or empty"); goto disable; } - if (__pa(initrd_end) > PFN_PHYS(max_low_pfn)) { + if (__pa_symbol(initrd_end) > PFN_PHYS(max_low_pfn)) { pr_err("initrd extends beyond end of memory"); goto disable; } size = initrd_end - initrd_start; - memblock_reserve(__pa(initrd_start), size); + memblock_reserve(__pa_symbol(initrd_start), size); initrd_below_start_ok = 1; pr_info("Initial ramdisk at: 0x%p (%lu bytes)\n", @@ -82,12 +118,14 @@ disable: } #endif /* CONFIG_BLK_DEV_INITRD */ +static phys_addr_t dtb_early_pa __initdata; + void __init setup_bootmem(void) { struct memblock_region *reg; phys_addr_t mem_size = 0; - phys_addr_t vmlinux_end = __pa(&_end); - phys_addr_t vmlinux_start = __pa(&_start); + phys_addr_t vmlinux_end = __pa_symbol(&_end); + phys_addr_t vmlinux_start = __pa_symbol(&_start); /* Find the memory region containing the kernel */ for_each_memblock(memory, reg) { @@ -117,7 +155,12 @@ void __init setup_bootmem(void) setup_initrd(); #endif /* CONFIG_BLK_DEV_INITRD */ - early_init_fdt_reserve_self(); + /* + * Avoid using early_init_fdt_reserve_self() since __pa() does + * not work for DTB pointers that are fixmap addresses + */ + memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va)); + early_init_fdt_scan_reserved_mem(); memblock_allow_resize(); memblock_dump_all(); @@ -132,12 +175,12 @@ void __init setup_bootmem(void) } } +#ifdef CONFIG_MMU unsigned long va_pa_offset; EXPORT_SYMBOL(va_pa_offset); unsigned long pfn_base; EXPORT_SYMBOL(pfn_base); -void *dtb_early_va; pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss; pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss; pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss; @@ -263,7 +306,6 @@ static void __init create_pmd_mapping(pmd_t *pmdp, #define get_pgd_next_virt(__pa) get_pmd_virt(__pa) #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ create_pmd_mapping(__nextp, __va, __pa, __sz, __prot) -#define PTE_PARENT_SIZE PMD_SIZE #define fixmap_pgd_next fixmap_pmd #else #define pgd_next_t pte_t @@ -271,7 +313,6 @@ static void __init create_pmd_mapping(pmd_t *pmdp, #define get_pgd_next_virt(__pa) get_pte_virt(__pa) #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ create_pte_mapping(__nextp, __va, __pa, __sz, __prot) -#define PTE_PARENT_SIZE PGDIR_SIZE #define fixmap_pgd_next fixmap_pte #endif @@ -304,14 +345,11 @@ static void __init create_pgd_mapping(pgd_t *pgdp, static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) { - uintptr_t map_size = PAGE_SIZE; + /* Upgrade to PMD_SIZE mappings whenever possible */ + if ((base & (PMD_SIZE - 1)) || (size & (PMD_SIZE - 1))) + return PAGE_SIZE; - /* Upgrade to PMD/PGDIR mappings whenever possible */ - if (!(base & (PTE_PARENT_SIZE - 1)) && - !(size & (PTE_PARENT_SIZE - 1))) - map_size = PTE_PARENT_SIZE; - - return map_size; + return PMD_SIZE; } /* @@ -329,8 +367,7 @@ static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) */ #ifndef __riscv_cmodel_medany -#error "setup_vm() is called from head.S before relocate so it should " - "not use absolute addressing." +#error "setup_vm() is called from head.S before relocate so it should not use absolute addressing." #endif asmlinkage void __init setup_vm(uintptr_t dtb_pa) @@ -393,6 +430,8 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) /* Save pointer to DTB for early FDT parsing */ dtb_early_va = (void *)fix_to_virt(FIX_FDT) + (dtb_pa & ~PAGE_MASK); + /* Save physical address for memblock reservation */ + dtb_early_pa = dtb_pa; } static void __init setup_vm_final(void) @@ -406,7 +445,7 @@ static void __init setup_vm_final(void) /* Setup swapper PGD for fixmap */ create_pgd_mapping(swapper_pg_dir, FIXADDR_START, - __pa(fixmap_pgd_next), + __pa_symbol(fixmap_pgd_next), PGDIR_SIZE, PAGE_TABLE); /* Map all memory banks */ @@ -435,13 +474,33 @@ static void __init setup_vm_final(void) clear_fixmap(FIX_PMD); /* Move to swapper page table */ - csr_write(sptbr, PFN_DOWN(__pa(swapper_pg_dir)) | SATP_MODE); + csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | SATP_MODE); local_flush_tlb_all(); } +#else +asmlinkage void __init setup_vm(uintptr_t dtb_pa) +{ + dtb_early_va = (void *)dtb_pa; +} + +static inline void setup_vm_final(void) +{ +} +#endif /* CONFIG_MMU */ void __init paging_init(void) { setup_vm_final(); + memblocks_present(); + sparse_init(); setup_zero_page(); zone_sizes_init(); } + +#ifdef CONFIG_SPARSEMEM_VMEMMAP +int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, + struct vmem_altmap *altmap) +{ + return vmemmap_populate_basepages(start, end, node); +} +#endif diff --git a/arch/riscv/mm/ioremap.c b/arch/riscv/mm/ioremap.c deleted file mode 100644 index ac621ddb45c0..000000000000 --- a/arch/riscv/mm/ioremap.c +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * (C) Copyright 1995 1996 Linus Torvalds - * (C) Copyright 2012 Regents of the University of California - */ - -#include <linux/export.h> -#include <linux/mm.h> -#include <linux/vmalloc.h> -#include <linux/io.h> - -#include <asm/pgtable.h> - -/* - * Remap an arbitrary physical address space into the kernel virtual - * address space. Needed when the kernel wants to access high addresses - * directly. - * - * NOTE! We need to allow non-page-aligned mappings too: we will obviously - * have to convert them into an offset in a page-aligned mapping, but the - * caller shouldn't need to know that small detail. - */ -static void __iomem *__ioremap_caller(phys_addr_t addr, size_t size, - pgprot_t prot, void *caller) -{ - phys_addr_t last_addr; - unsigned long offset, vaddr; - struct vm_struct *area; - - /* Disallow wrap-around or zero size */ - last_addr = addr + size - 1; - if (!size || last_addr < addr) - return NULL; - - /* Page-align mappings */ - offset = addr & (~PAGE_MASK); - addr -= offset; - size = PAGE_ALIGN(size + offset); - - area = get_vm_area_caller(size, VM_IOREMAP, caller); - if (!area) - return NULL; - vaddr = (unsigned long)area->addr; - - if (ioremap_page_range(vaddr, vaddr + size, addr, prot)) { - free_vm_area(area); - return NULL; - } - - return (void __iomem *)(vaddr + offset); -} - -/* - * ioremap - map bus memory into CPU space - * @offset: bus address of the memory - * @size: size of the resource to map - * - * ioremap performs a platform specific sequence of operations to - * make bus memory CPU accessible via the readb/readw/readl/writeb/ - * writew/writel functions and the other mmio helpers. The returned - * address is not guaranteed to be usable directly as a virtual - * address. - * - * Must be freed with iounmap. - */ -void __iomem *ioremap(phys_addr_t offset, unsigned long size) -{ - return __ioremap_caller(offset, size, PAGE_KERNEL, - __builtin_return_address(0)); -} -EXPORT_SYMBOL(ioremap); - - -/** - * iounmap - Free a IO remapping - * @addr: virtual address from ioremap_* - * - * Caller must ensure there is only one unmapping for the same pointer. - */ -void iounmap(volatile void __iomem *addr) -{ - vunmap((void *)((unsigned long)addr & PAGE_MASK)); -} -EXPORT_SYMBOL(iounmap); diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c new file mode 100644 index 000000000000..f0cc86040587 --- /dev/null +++ b/arch/riscv/mm/kasan_init.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2019 Andes Technology Corporation + +#include <linux/pfn.h> +#include <linux/init_task.h> +#include <linux/kasan.h> +#include <linux/kernel.h> +#include <linux/memblock.h> +#include <asm/tlbflush.h> +#include <asm/pgtable.h> +#include <asm/fixmap.h> + +extern pgd_t early_pg_dir[PTRS_PER_PGD]; +asmlinkage void __init kasan_early_init(void) +{ + uintptr_t i; + pgd_t *pgd = early_pg_dir + pgd_index(KASAN_SHADOW_START); + + for (i = 0; i < PTRS_PER_PTE; ++i) + set_pte(kasan_early_shadow_pte + i, + mk_pte(virt_to_page(kasan_early_shadow_page), + PAGE_KERNEL)); + + for (i = 0; i < PTRS_PER_PMD; ++i) + set_pmd(kasan_early_shadow_pmd + i, + pfn_pmd(PFN_DOWN(__pa((uintptr_t)kasan_early_shadow_pte)), + __pgprot(_PAGE_TABLE))); + + for (i = KASAN_SHADOW_START; i < KASAN_SHADOW_END; + i += PGDIR_SIZE, ++pgd) + set_pgd(pgd, + pfn_pgd(PFN_DOWN(__pa(((uintptr_t)kasan_early_shadow_pmd))), + __pgprot(_PAGE_TABLE))); + + /* init for swapper_pg_dir */ + pgd = pgd_offset_k(KASAN_SHADOW_START); + + for (i = KASAN_SHADOW_START; i < KASAN_SHADOW_END; + i += PGDIR_SIZE, ++pgd) + set_pgd(pgd, + pfn_pgd(PFN_DOWN(__pa(((uintptr_t)kasan_early_shadow_pmd))), + __pgprot(_PAGE_TABLE))); + + flush_tlb_all(); +} + +static void __init populate(void *start, void *end) +{ + unsigned long i; + unsigned long vaddr = (unsigned long)start & PAGE_MASK; + unsigned long vend = PAGE_ALIGN((unsigned long)end); + unsigned long n_pages = (vend - vaddr) / PAGE_SIZE; + unsigned long n_pmds = + (n_pages % PTRS_PER_PTE) ? n_pages / PTRS_PER_PTE + 1 : + n_pages / PTRS_PER_PTE; + pgd_t *pgd = pgd_offset_k(vaddr); + pmd_t *pmd = memblock_alloc(n_pmds * sizeof(pmd_t), PAGE_SIZE); + pte_t *pte = memblock_alloc(n_pages * sizeof(pte_t), PAGE_SIZE); + + for (i = 0; i < n_pages; i++) { + phys_addr_t phys = memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); + + set_pte(pte + i, pfn_pte(PHYS_PFN(phys), PAGE_KERNEL)); + } + + for (i = 0; i < n_pmds; ++pgd, i += PTRS_PER_PMD) + set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(((uintptr_t)(pmd + i)))), + __pgprot(_PAGE_TABLE))); + + for (i = 0; i < n_pages; ++pmd, i += PTRS_PER_PTE) + set_pmd(pmd, pfn_pmd(PFN_DOWN(__pa((uintptr_t)(pte + i))), + __pgprot(_PAGE_TABLE))); + + flush_tlb_all(); + memset(start, 0, end - start); +} + +void __init kasan_init(void) +{ + struct memblock_region *reg; + unsigned long i; + + kasan_populate_early_shadow((void *)KASAN_SHADOW_START, + (void *)kasan_mem_to_shadow((void *)VMALLOC_END)); + + for_each_memblock(memory, reg) { + void *start = (void *)__va(reg->base); + void *end = (void *)__va(reg->base + reg->size); + + if (start >= end) + break; + + populate(kasan_mem_to_shadow(start), + kasan_mem_to_shadow(end)); + }; + + for (i = 0; i < PTRS_PER_PTE; i++) + set_pte(&kasan_early_shadow_pte[i], + mk_pte(virt_to_page(kasan_early_shadow_page), + __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_ACCESSED))); + + memset(kasan_early_shadow_page, 0, PAGE_SIZE); + init_task.kasan_depth = 0; +} diff --git a/arch/riscv/mm/physaddr.c b/arch/riscv/mm/physaddr.c new file mode 100644 index 000000000000..e8e4dcd39fed --- /dev/null +++ b/arch/riscv/mm/physaddr.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/types.h> +#include <linux/mmdebug.h> +#include <linux/mm.h> +#include <asm/page.h> +#include <asm/sections.h> + +phys_addr_t __virt_to_phys(unsigned long x) +{ + phys_addr_t y = x - PAGE_OFFSET; + + /* + * Boundary checking aginst the kernel linear mapping space. + */ + WARN(y >= KERN_VIRT_SIZE, + "virt_to_phys used for non-linear address: %pK (%pS)\n", + (void *)x, (void *)x); + + return __va_to_pa_nodebug(x); +} +EXPORT_SYMBOL(__virt_to_phys); + +phys_addr_t __phys_addr_symbol(unsigned long x) +{ + unsigned long kernel_start = (unsigned long)PAGE_OFFSET; + unsigned long kernel_end = (unsigned long)_end; + + /* + * Boundary checking aginst the kernel image mapping. + * __pa_symbol should only be used on kernel symbol addresses. + */ + VIRTUAL_BUG_ON(x < kernel_start || x > kernel_end); + + return __va_to_pa_nodebug(x); +} +EXPORT_SYMBOL(__phys_addr_symbol); diff --git a/arch/riscv/mm/sifive_l2_cache.c b/arch/riscv/mm/sifive_l2_cache.c deleted file mode 100644 index 2e637ad71c05..000000000000 --- a/arch/riscv/mm/sifive_l2_cache.c +++ /dev/null @@ -1,178 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * SiFive L2 cache controller Driver - * - * Copyright (C) 2018-2019 SiFive, Inc. - * - */ -#include <linux/debugfs.h> -#include <linux/interrupt.h> -#include <linux/of_irq.h> -#include <linux/of_address.h> -#include <asm/sifive_l2_cache.h> - -#define SIFIVE_L2_DIRECCFIX_LOW 0x100 -#define SIFIVE_L2_DIRECCFIX_HIGH 0x104 -#define SIFIVE_L2_DIRECCFIX_COUNT 0x108 - -#define SIFIVE_L2_DATECCFIX_LOW 0x140 -#define SIFIVE_L2_DATECCFIX_HIGH 0x144 -#define SIFIVE_L2_DATECCFIX_COUNT 0x148 - -#define SIFIVE_L2_DATECCFAIL_LOW 0x160 -#define SIFIVE_L2_DATECCFAIL_HIGH 0x164 -#define SIFIVE_L2_DATECCFAIL_COUNT 0x168 - -#define SIFIVE_L2_CONFIG 0x00 -#define SIFIVE_L2_WAYENABLE 0x08 -#define SIFIVE_L2_ECCINJECTERR 0x40 - -#define SIFIVE_L2_MAX_ECCINTR 3 - -static void __iomem *l2_base; -static int g_irq[SIFIVE_L2_MAX_ECCINTR]; - -enum { - DIR_CORR = 0, - DATA_CORR, - DATA_UNCORR, -}; - -#ifdef CONFIG_DEBUG_FS -static struct dentry *sifive_test; - -static ssize_t l2_write(struct file *file, const char __user *data, - size_t count, loff_t *ppos) -{ - unsigned int val; - - if (kstrtouint_from_user(data, count, 0, &val)) - return -EINVAL; - if ((val >= 0 && val < 0xFF) || (val >= 0x10000 && val < 0x100FF)) - writel(val, l2_base + SIFIVE_L2_ECCINJECTERR); - else - return -EINVAL; - return count; -} - -static const struct file_operations l2_fops = { - .owner = THIS_MODULE, - .open = simple_open, - .write = l2_write -}; - -static void setup_sifive_debug(void) -{ - sifive_test = debugfs_create_dir("sifive_l2_cache", NULL); - - debugfs_create_file("sifive_debug_inject_error", 0200, - sifive_test, NULL, &l2_fops); -} -#endif - -static void l2_config_read(void) -{ - u32 regval, val; - - regval = readl(l2_base + SIFIVE_L2_CONFIG); - val = regval & 0xFF; - pr_info("L2CACHE: No. of Banks in the cache: %d\n", val); - val = (regval & 0xFF00) >> 8; - pr_info("L2CACHE: No. of ways per bank: %d\n", val); - val = (regval & 0xFF0000) >> 16; - pr_info("L2CACHE: Sets per bank: %llu\n", (uint64_t)1 << val); - val = (regval & 0xFF000000) >> 24; - pr_info("L2CACHE: Bytes per cache block: %llu\n", (uint64_t)1 << val); - - regval = readl(l2_base + SIFIVE_L2_WAYENABLE); - pr_info("L2CACHE: Index of the largest way enabled: %d\n", regval); -} - -static const struct of_device_id sifive_l2_ids[] = { - { .compatible = "sifive,fu540-c000-ccache" }, - { /* end of table */ }, -}; - -static ATOMIC_NOTIFIER_HEAD(l2_err_chain); - -int register_sifive_l2_error_notifier(struct notifier_block *nb) -{ - return atomic_notifier_chain_register(&l2_err_chain, nb); -} -EXPORT_SYMBOL_GPL(register_sifive_l2_error_notifier); - -int unregister_sifive_l2_error_notifier(struct notifier_block *nb) -{ - return atomic_notifier_chain_unregister(&l2_err_chain, nb); -} -EXPORT_SYMBOL_GPL(unregister_sifive_l2_error_notifier); - -static irqreturn_t l2_int_handler(int irq, void *device) -{ - unsigned int add_h, add_l; - - if (irq == g_irq[DIR_CORR]) { - add_h = readl(l2_base + SIFIVE_L2_DIRECCFIX_HIGH); - add_l = readl(l2_base + SIFIVE_L2_DIRECCFIX_LOW); - pr_err("L2CACHE: DirError @ 0x%08X.%08X\n", add_h, add_l); - /* Reading this register clears the DirError interrupt sig */ - readl(l2_base + SIFIVE_L2_DIRECCFIX_COUNT); - atomic_notifier_call_chain(&l2_err_chain, SIFIVE_L2_ERR_TYPE_CE, - "DirECCFix"); - } - if (irq == g_irq[DATA_CORR]) { - add_h = readl(l2_base + SIFIVE_L2_DATECCFIX_HIGH); - add_l = readl(l2_base + SIFIVE_L2_DATECCFIX_LOW); - pr_err("L2CACHE: DataError @ 0x%08X.%08X\n", add_h, add_l); - /* Reading this register clears the DataError interrupt sig */ - readl(l2_base + SIFIVE_L2_DATECCFIX_COUNT); - atomic_notifier_call_chain(&l2_err_chain, SIFIVE_L2_ERR_TYPE_CE, - "DatECCFix"); - } - if (irq == g_irq[DATA_UNCORR]) { - add_h = readl(l2_base + SIFIVE_L2_DATECCFAIL_HIGH); - add_l = readl(l2_base + SIFIVE_L2_DATECCFAIL_LOW); - pr_err("L2CACHE: DataFail @ 0x%08X.%08X\n", add_h, add_l); - /* Reading this register clears the DataFail interrupt sig */ - readl(l2_base + SIFIVE_L2_DATECCFAIL_COUNT); - atomic_notifier_call_chain(&l2_err_chain, SIFIVE_L2_ERR_TYPE_UE, - "DatECCFail"); - } - - return IRQ_HANDLED; -} - -int __init sifive_l2_init(void) -{ - struct device_node *np; - struct resource res; - int i, rc; - - np = of_find_matching_node(NULL, sifive_l2_ids); - if (!np) - return -ENODEV; - - if (of_address_to_resource(np, 0, &res)) - return -ENODEV; - - l2_base = ioremap(res.start, resource_size(&res)); - if (!l2_base) - return -ENOMEM; - - for (i = 0; i < SIFIVE_L2_MAX_ECCINTR; i++) { - g_irq[i] = irq_of_parse_and_map(np, i); - rc = request_irq(g_irq[i], l2_int_handler, 0, "l2_ecc", NULL); - if (rc) { - pr_err("L2CACHE: Could not request IRQ %d\n", g_irq[i]); - return rc; - } - } - - l2_config_read(); - -#ifdef CONFIG_DEBUG_FS - setup_sifive_debug(); -#endif - return 0; -} -device_initcall(sifive_l2_init); diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c new file mode 100644 index 000000000000..720b443c4528 --- /dev/null +++ b/arch/riscv/mm/tlbflush.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/mm.h> +#include <linux/smp.h> +#include <linux/sched.h> +#include <asm/sbi.h> + +void flush_tlb_all(void) +{ + sbi_remote_sfence_vma(NULL, 0, -1); +} + +/* + * This function must not be called with cmask being null. + * Kernel may panic if cmask is NULL. + */ +static void __sbi_tlb_flush_range(struct cpumask *cmask, unsigned long start, + unsigned long size) +{ + struct cpumask hmask; + unsigned int cpuid; + + if (cpumask_empty(cmask)) + return; + + cpuid = get_cpu(); + + if (cpumask_any_but(cmask, cpuid) >= nr_cpu_ids) { + /* local cpu is the only cpu present in cpumask */ + if (size <= PAGE_SIZE) + local_flush_tlb_page(start); + else + local_flush_tlb_all(); + } else { + riscv_cpuid_to_hartid_mask(cmask, &hmask); + sbi_remote_sfence_vma(cpumask_bits(&hmask), start, size); + } + + put_cpu(); +} + +void flush_tlb_mm(struct mm_struct *mm) +{ + __sbi_tlb_flush_range(mm_cpumask(mm), 0, -1); +} + +void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) +{ + __sbi_tlb_flush_range(mm_cpumask(vma->vm_mm), addr, PAGE_SIZE); +} + +void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end) +{ + __sbi_tlb_flush_range(mm_cpumask(vma->vm_mm), start, end - start); +} diff --git a/arch/riscv/net/bpf_jit_comp.c b/arch/riscv/net/bpf_jit_comp.c index 5451ef3845f2..483f4ad7f4dc 100644 --- a/arch/riscv/net/bpf_jit_comp.c +++ b/arch/riscv/net/bpf_jit_comp.c @@ -120,6 +120,11 @@ static bool seen_reg(int reg, struct rv_jit_context *ctx) return false; } +static void mark_fp(struct rv_jit_context *ctx) +{ + __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags); +} + static void mark_call(struct rv_jit_context *ctx) { __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); @@ -456,6 +461,11 @@ static u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f); } +static u32 rv_auipc(u8 rd, u32 imm31_12) +{ + return rv_u_insn(imm31_12, rd, 0x17); +} + static bool is_12b_int(s64 val) { return -(1 << 11) <= val && val < (1 << 11); @@ -479,27 +489,7 @@ static bool is_32b_int(s64 val) static int is_12b_check(int off, int insn) { if (!is_12b_int(off)) { - pr_err("bpf-jit: insn=%d offset=%d not supported yet!\n", - insn, (int)off); - return -1; - } - return 0; -} - -static int is_13b_check(int off, int insn) -{ - if (!is_13b_int(off)) { - pr_err("bpf-jit: insn=%d offset=%d not supported yet!\n", - insn, (int)off); - return -1; - } - return 0; -} - -static int is_21b_check(int off, int insn) -{ - if (!is_21b_int(off)) { - pr_err("bpf-jit: insn=%d offset=%d not supported yet!\n", + pr_err("bpf-jit: insn=%d 12b < offset=%d not supported yet!\n", insn, (int)off); return -1; } @@ -545,10 +535,13 @@ static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx) emit(rv_addi(rd, rd, lower), ctx); } -static int rv_offset(int bpf_to, int bpf_from, struct rv_jit_context *ctx) +static int rv_offset(int insn, int off, struct rv_jit_context *ctx) { - int from = ctx->offset[bpf_from] - 1, to = ctx->offset[bpf_to]; + int from, to; + off++; /* BPF branch is from PC+1, RV is from PC */ + from = (insn > 0) ? ctx->offset[insn - 1] : 0; + to = (insn + off > 0) ? ctx->offset[insn + off - 1] : 0; return (to - from) << 2; } @@ -559,7 +552,7 @@ static int epilogue_offset(struct rv_jit_context *ctx) return (to - from) << 2; } -static void __build_epilogue(u8 reg, struct rv_jit_context *ctx) +static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) { int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; @@ -596,8 +589,114 @@ static void __build_epilogue(u8 reg, struct rv_jit_context *ctx) emit(rv_addi(RV_REG_SP, RV_REG_SP, stack_adjust), ctx); /* Set return value. */ - emit(rv_addi(RV_REG_A0, RV_REG_A5, 0), ctx); - emit(rv_jalr(RV_REG_ZERO, reg, 0), ctx); + if (!is_tail_call) + emit(rv_addi(RV_REG_A0, RV_REG_A5, 0), ctx); + emit(rv_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, + is_tail_call ? 4 : 0), /* skip TCC init */ + ctx); +} + +/* return -1 or inverted cond */ +static int invert_bpf_cond(u8 cond) +{ + switch (cond) { + case BPF_JEQ: + return BPF_JNE; + case BPF_JGT: + return BPF_JLE; + case BPF_JLT: + return BPF_JGE; + case BPF_JGE: + return BPF_JLT; + case BPF_JLE: + return BPF_JGT; + case BPF_JNE: + return BPF_JEQ; + case BPF_JSGT: + return BPF_JSLE; + case BPF_JSLT: + return BPF_JSGE; + case BPF_JSGE: + return BPF_JSLT; + case BPF_JSLE: + return BPF_JSGT; + } + return -1; +} + +static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff, + struct rv_jit_context *ctx) +{ + switch (cond) { + case BPF_JEQ: + emit(rv_beq(rd, rs, rvoff >> 1), ctx); + return; + case BPF_JGT: + emit(rv_bltu(rs, rd, rvoff >> 1), ctx); + return; + case BPF_JLT: + emit(rv_bltu(rd, rs, rvoff >> 1), ctx); + return; + case BPF_JGE: + emit(rv_bgeu(rd, rs, rvoff >> 1), ctx); + return; + case BPF_JLE: + emit(rv_bgeu(rs, rd, rvoff >> 1), ctx); + return; + case BPF_JNE: + emit(rv_bne(rd, rs, rvoff >> 1), ctx); + return; + case BPF_JSGT: + emit(rv_blt(rs, rd, rvoff >> 1), ctx); + return; + case BPF_JSLT: + emit(rv_blt(rd, rs, rvoff >> 1), ctx); + return; + case BPF_JSGE: + emit(rv_bge(rd, rs, rvoff >> 1), ctx); + return; + case BPF_JSLE: + emit(rv_bge(rs, rd, rvoff >> 1), ctx); + } +} + +static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff, + struct rv_jit_context *ctx) +{ + s64 upper, lower; + + if (is_13b_int(rvoff)) { + emit_bcc(cond, rd, rs, rvoff, ctx); + return; + } + + /* Adjust for jal */ + rvoff -= 4; + + /* Transform, e.g.: + * bne rd,rs,foo + * to + * beq rd,rs,<.L1> + * (auipc foo) + * jal(r) foo + * .L1 + */ + cond = invert_bpf_cond(cond); + if (is_21b_int(rvoff)) { + emit_bcc(cond, rd, rs, 8, ctx); + emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx); + return; + } + + /* 32b No need for an additional rvoff adjustment, since we + * get that from the auipc at PC', where PC = PC' + 4. + */ + upper = (rvoff + (1 << 11)) >> 12; + lower = rvoff & 0xfff; + + emit_bcc(cond, rd, rs, 12, ctx); + emit(rv_auipc(RV_REG_T1, upper), ctx); + emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx); } static void emit_zext_32(u8 reg, struct rv_jit_context *ctx) @@ -627,18 +726,14 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) return -1; emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx); off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; - if (is_13b_check(off, insn)) - return -1; - emit(rv_bgeu(RV_REG_A2, RV_REG_T1, off >> 1), ctx); + emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx); - /* if (--TCC < 0) + /* if (TCC-- < 0) * goto out; */ emit(rv_addi(RV_REG_T1, tcc, -1), ctx); off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; - if (is_13b_check(off, insn)) - return -1; - emit(rv_blt(RV_REG_T1, RV_REG_ZERO, off >> 1), ctx); + emit_branch(BPF_JSLT, tcc, RV_REG_ZERO, off, ctx); /* prog = array->ptrs[index]; * if (!prog) @@ -651,18 +746,15 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) return -1; emit(rv_ld(RV_REG_T2, off, RV_REG_T2), ctx); off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; - if (is_13b_check(off, insn)) - return -1; - emit(rv_beq(RV_REG_T2, RV_REG_ZERO, off >> 1), ctx); + emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); /* goto *(prog->bpf_func + 4); */ off = offsetof(struct bpf_prog, bpf_func); if (is_12b_check(off, insn)) return -1; emit(rv_ld(RV_REG_T3, off, RV_REG_T2), ctx); - emit(rv_addi(RV_REG_T3, RV_REG_T3, 4), ctx); emit(rv_addi(RV_REG_TCC, RV_REG_T1, 0), ctx); - __build_epilogue(RV_REG_T3, ctx); + __build_epilogue(true, ctx); return 0; } @@ -687,13 +779,6 @@ static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn, *rs = bpf_to_rv_reg(insn->src_reg, ctx); } -static int rv_offset_check(int *rvoff, s16 off, int insn, - struct rv_jit_context *ctx) -{ - *rvoff = rv_offset(insn + off, insn, ctx); - return is_13b_check(*rvoff, insn); -} - static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) { emit(rv_addi(RV_REG_T2, *rd, 0), ctx); @@ -726,13 +811,57 @@ static void emit_sext_32_rd(u8 *rd, struct rv_jit_context *ctx) *rd = RV_REG_T2; } +static void emit_jump_and_link(u8 rd, s64 rvoff, bool force_jalr, + struct rv_jit_context *ctx) +{ + s64 upper, lower; + + if (rvoff && is_21b_int(rvoff) && !force_jalr) { + emit(rv_jal(rd, rvoff >> 1), ctx); + return; + } + + upper = (rvoff + (1 << 11)) >> 12; + lower = rvoff & 0xfff; + emit(rv_auipc(RV_REG_T1, upper), ctx); + emit(rv_jalr(rd, RV_REG_T1, lower), ctx); +} + +static bool is_signed_bpf_cond(u8 cond) +{ + return cond == BPF_JSGT || cond == BPF_JSLT || + cond == BPF_JSGE || cond == BPF_JSLE; +} + +static int emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx) +{ + s64 off = 0; + u64 ip; + u8 rd; + + if (addr && ctx->insns) { + ip = (u64)(long)(ctx->insns + ctx->ninsns); + off = addr - ip; + if (!is_32b_int(off)) { + pr_err("bpf-jit: target call addr %pK is out of range\n", + (void *)addr); + return -ERANGE; + } + } + + emit_jump_and_link(RV_REG_RA, off, !fixed, ctx); + rd = bpf_to_rv_reg(BPF_REG_0, ctx); + emit(rv_addi(rd, RV_REG_A0, 0), ctx); + return 0; +} + static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, bool extra_pass) { bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || BPF_CLASS(insn->code) == BPF_JMP; + int s, e, rvoff, i = insn - ctx->prog->insnsi; struct bpf_prog_aux *aux = ctx->prog->aux; - int rvoff, i = insn - ctx->prog->insnsi; u8 rd = -1, rs = -1, code = insn->code; s16 off = insn->off; s32 imm = insn->imm; @@ -1000,214 +1129,110 @@ out_be: /* JUMP off */ case BPF_JMP | BPF_JA: - rvoff = rv_offset(i + off, i, ctx); - if (!is_21b_int(rvoff)) { - pr_err("bpf-jit: insn=%d offset=%d not supported yet!\n", - i, rvoff); - return -1; - } - - emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx); + rvoff = rv_offset(i, off, ctx); + emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx); break; /* IF (dst COND src) JUMP off */ case BPF_JMP | BPF_JEQ | BPF_X: case BPF_JMP32 | BPF_JEQ | BPF_X: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - if (!is64) - emit_zext_32_rd_rs(&rd, &rs, ctx); - emit(rv_beq(rd, rs, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JGT | BPF_X: case BPF_JMP32 | BPF_JGT | BPF_X: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - if (!is64) - emit_zext_32_rd_rs(&rd, &rs, ctx); - emit(rv_bltu(rs, rd, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JLT | BPF_X: case BPF_JMP32 | BPF_JLT | BPF_X: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - if (!is64) - emit_zext_32_rd_rs(&rd, &rs, ctx); - emit(rv_bltu(rd, rs, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JGE | BPF_X: case BPF_JMP32 | BPF_JGE | BPF_X: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - if (!is64) - emit_zext_32_rd_rs(&rd, &rs, ctx); - emit(rv_bgeu(rd, rs, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JLE | BPF_X: case BPF_JMP32 | BPF_JLE | BPF_X: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - if (!is64) - emit_zext_32_rd_rs(&rd, &rs, ctx); - emit(rv_bgeu(rs, rd, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JNE | BPF_X: case BPF_JMP32 | BPF_JNE | BPF_X: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - if (!is64) - emit_zext_32_rd_rs(&rd, &rs, ctx); - emit(rv_bne(rd, rs, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JSGT | BPF_X: case BPF_JMP32 | BPF_JSGT | BPF_X: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - if (!is64) - emit_sext_32_rd_rs(&rd, &rs, ctx); - emit(rv_blt(rs, rd, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JSLT | BPF_X: case BPF_JMP32 | BPF_JSLT | BPF_X: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - if (!is64) - emit_sext_32_rd_rs(&rd, &rs, ctx); - emit(rv_blt(rd, rs, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JSGE | BPF_X: case BPF_JMP32 | BPF_JSGE | BPF_X: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - if (!is64) - emit_sext_32_rd_rs(&rd, &rs, ctx); - emit(rv_bge(rd, rs, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JSLE | BPF_X: case BPF_JMP32 | BPF_JSLE | BPF_X: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - if (!is64) - emit_sext_32_rd_rs(&rd, &rs, ctx); - emit(rv_bge(rs, rd, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JSET | BPF_X: case BPF_JMP32 | BPF_JSET | BPF_X: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - if (!is64) - emit_zext_32_rd_rs(&rd, &rs, ctx); - emit(rv_and(RV_REG_T1, rd, rs), ctx); - emit(rv_bne(RV_REG_T1, RV_REG_ZERO, rvoff >> 1), ctx); + rvoff = rv_offset(i, off, ctx); + if (!is64) { + s = ctx->ninsns; + if (is_signed_bpf_cond(BPF_OP(code))) + emit_sext_32_rd_rs(&rd, &rs, ctx); + else + emit_zext_32_rd_rs(&rd, &rs, ctx); + e = ctx->ninsns; + + /* Adjust for extra insns */ + rvoff -= (e - s) << 2; + } + + if (BPF_OP(code) == BPF_JSET) { + /* Adjust for and */ + rvoff -= 4; + emit(rv_and(RV_REG_T1, rd, rs), ctx); + emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, + ctx); + } else { + emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); + } break; /* IF (dst COND imm) JUMP off */ case BPF_JMP | BPF_JEQ | BPF_K: case BPF_JMP32 | BPF_JEQ | BPF_K: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - emit_imm(RV_REG_T1, imm, ctx); - if (!is64) - emit_zext_32_rd_t1(&rd, ctx); - emit(rv_beq(rd, RV_REG_T1, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JGT | BPF_K: case BPF_JMP32 | BPF_JGT | BPF_K: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - emit_imm(RV_REG_T1, imm, ctx); - if (!is64) - emit_zext_32_rd_t1(&rd, ctx); - emit(rv_bltu(RV_REG_T1, rd, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JLT | BPF_K: case BPF_JMP32 | BPF_JLT | BPF_K: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - emit_imm(RV_REG_T1, imm, ctx); - if (!is64) - emit_zext_32_rd_t1(&rd, ctx); - emit(rv_bltu(rd, RV_REG_T1, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JGE | BPF_K: case BPF_JMP32 | BPF_JGE | BPF_K: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - emit_imm(RV_REG_T1, imm, ctx); - if (!is64) - emit_zext_32_rd_t1(&rd, ctx); - emit(rv_bgeu(rd, RV_REG_T1, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JLE | BPF_K: case BPF_JMP32 | BPF_JLE | BPF_K: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - emit_imm(RV_REG_T1, imm, ctx); - if (!is64) - emit_zext_32_rd_t1(&rd, ctx); - emit(rv_bgeu(RV_REG_T1, rd, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JNE | BPF_K: case BPF_JMP32 | BPF_JNE | BPF_K: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - emit_imm(RV_REG_T1, imm, ctx); - if (!is64) - emit_zext_32_rd_t1(&rd, ctx); - emit(rv_bne(rd, RV_REG_T1, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JSGT | BPF_K: case BPF_JMP32 | BPF_JSGT | BPF_K: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - emit_imm(RV_REG_T1, imm, ctx); - if (!is64) - emit_sext_32_rd(&rd, ctx); - emit(rv_blt(RV_REG_T1, rd, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JSLT | BPF_K: case BPF_JMP32 | BPF_JSLT | BPF_K: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - emit_imm(RV_REG_T1, imm, ctx); - if (!is64) - emit_sext_32_rd(&rd, ctx); - emit(rv_blt(rd, RV_REG_T1, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JSGE | BPF_K: case BPF_JMP32 | BPF_JSGE | BPF_K: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - emit_imm(RV_REG_T1, imm, ctx); - if (!is64) - emit_sext_32_rd(&rd, ctx); - emit(rv_bge(rd, RV_REG_T1, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JSLE | BPF_K: case BPF_JMP32 | BPF_JSLE | BPF_K: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - emit_imm(RV_REG_T1, imm, ctx); - if (!is64) - emit_sext_32_rd(&rd, ctx); - emit(rv_bge(RV_REG_T1, rd, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JSET | BPF_K: case BPF_JMP32 | BPF_JSET | BPF_K: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; + rvoff = rv_offset(i, off, ctx); + s = ctx->ninsns; emit_imm(RV_REG_T1, imm, ctx); - if (!is64) - emit_zext_32_rd_t1(&rd, ctx); - emit(rv_and(RV_REG_T1, rd, RV_REG_T1), ctx); - emit(rv_bne(RV_REG_T1, RV_REG_ZERO, rvoff >> 1), ctx); + if (!is64) { + if (is_signed_bpf_cond(BPF_OP(code))) + emit_sext_32_rd(&rd, ctx); + else + emit_zext_32_rd_t1(&rd, ctx); + } + e = ctx->ninsns; + + /* Adjust for extra insns */ + rvoff -= (e - s) << 2; + + if (BPF_OP(code) == BPF_JSET) { + /* Adjust for and */ + rvoff -= 4; + emit(rv_and(RV_REG_T1, rd, RV_REG_T1), ctx); + emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, + ctx); + } else { + emit_branch(BPF_OP(code), rd, RV_REG_T1, rvoff, ctx); + } break; /* function call */ case BPF_JMP | BPF_CALL: { bool fixed; - int i, ret; + int ret; u64 addr; mark_call(ctx); @@ -1215,20 +1240,9 @@ out_be: &fixed); if (ret < 0) return ret; - if (fixed) { - emit_imm(RV_REG_T1, addr, ctx); - } else { - i = ctx->ninsns; - emit_imm(RV_REG_T1, addr, ctx); - for (i = ctx->ninsns - i; i < 8; i++) { - /* nop */ - emit(rv_addi(RV_REG_ZERO, RV_REG_ZERO, 0), - ctx); - } - } - emit(rv_jalr(RV_REG_RA, RV_REG_T1, 0), ctx); - rd = bpf_to_rv_reg(BPF_REG_0, ctx); - emit(rv_addi(rd, RV_REG_A0, 0), ctx); + ret = emit_call(fixed, addr, ctx); + if (ret) + return ret; break; } /* tail call */ @@ -1243,9 +1257,7 @@ out_be: break; rvoff = epilogue_offset(ctx); - if (is_21b_check(rvoff, i)) - return -1; - emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx); + emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx); break; /* dst = imm64 */ @@ -1426,6 +1438,10 @@ static void build_prologue(struct rv_jit_context *ctx) { int stack_adjust = 0, store_offset, bpf_stack_adjust; + bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); + if (bpf_stack_adjust) + mark_fp(ctx); + if (seen_reg(RV_REG_RA, ctx)) stack_adjust += 8; stack_adjust += 8; /* RV_REG_FP */ @@ -1443,7 +1459,6 @@ static void build_prologue(struct rv_jit_context *ctx) stack_adjust += 8; stack_adjust = round_up(stack_adjust, 16); - bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); stack_adjust += bpf_stack_adjust; store_offset = stack_adjust - 8; @@ -1502,10 +1517,10 @@ static void build_prologue(struct rv_jit_context *ctx) static void build_epilogue(struct rv_jit_context *ctx) { - __build_epilogue(RV_REG_RA, ctx); + __build_epilogue(false, ctx); } -static int build_body(struct rv_jit_context *ctx, bool extra_pass) +static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset) { const struct bpf_prog *prog = ctx->prog; int i; @@ -1517,12 +1532,12 @@ static int build_body(struct rv_jit_context *ctx, bool extra_pass) ret = emit_insn(insn, ctx, extra_pass); if (ret > 0) { i++; - if (ctx->insns == NULL) - ctx->offset[i] = ctx->ninsns; + if (offset) + offset[i] = ctx->ninsns; continue; } - if (ctx->insns == NULL) - ctx->offset[i] = ctx->ninsns; + if (offset) + offset[i] = ctx->ninsns; if (ret) return ret; } @@ -1548,9 +1563,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) { bool tmp_blinded = false, extra_pass = false; struct bpf_prog *tmp, *orig_prog = prog; + int pass = 0, prev_ninsns = 0, i; struct rv_jit_data *jit_data; + unsigned int image_size = 0; struct rv_jit_context *ctx; - unsigned int image_size; if (!prog->jit_requested) return orig_prog; @@ -1587,33 +1603,59 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) prog = orig_prog; goto out_offset; } + for (i = 0; i < prog->len; i++) { + prev_ninsns += 32; + ctx->offset[i] = prev_ninsns; + } - /* First pass generates the ctx->offset, but does not emit an image. */ - if (build_body(ctx, extra_pass)) { - prog = orig_prog; - goto out_offset; + for (i = 0; i < 16; i++) { + pass++; + ctx->ninsns = 0; + if (build_body(ctx, extra_pass, ctx->offset)) { + prog = orig_prog; + goto out_offset; + } + build_prologue(ctx); + ctx->epilogue_offset = ctx->ninsns; + build_epilogue(ctx); + + if (ctx->ninsns == prev_ninsns) { + if (jit_data->header) + break; + + image_size = sizeof(u32) * ctx->ninsns; + jit_data->header = + bpf_jit_binary_alloc(image_size, + &jit_data->image, + sizeof(u32), + bpf_fill_ill_insns); + if (!jit_data->header) { + prog = orig_prog; + goto out_offset; + } + + ctx->insns = (u32 *)jit_data->image; + /* Now, when the image is allocated, the image + * can potentially shrink more (auipc/jalr -> + * jal). + */ + } + prev_ninsns = ctx->ninsns; } - build_prologue(ctx); - ctx->epilogue_offset = ctx->ninsns; - build_epilogue(ctx); - /* Allocate image, now that we know the size. */ - image_size = sizeof(u32) * ctx->ninsns; - jit_data->header = bpf_jit_binary_alloc(image_size, &jit_data->image, - sizeof(u32), - bpf_fill_ill_insns); - if (!jit_data->header) { + if (i == 16) { + pr_err("bpf-jit: image did not converge in <%d passes!\n", i); + bpf_jit_binary_free(jit_data->header); prog = orig_prog; goto out_offset; } - /* Second, real pass, that acutally emits the image. */ - ctx->insns = (u32 *)jit_data->image; skip_init_ctx: + pass++; ctx->ninsns = 0; build_prologue(ctx); - if (build_body(ctx, extra_pass)) { + if (build_body(ctx, extra_pass, NULL)) { bpf_jit_binary_free(jit_data->header); prog = orig_prog; goto out_offset; @@ -1621,7 +1663,7 @@ skip_init_ctx: build_epilogue(ctx); if (bpf_jit_enable > 1) - bpf_jit_dump(prog->len, image_size, 2, ctx->insns); + bpf_jit_dump(prog->len, image_size, pass, ctx->insns); prog->bpf_func = (void *)ctx->insns; prog->jited = 1; @@ -1641,3 +1683,16 @@ out: tmp : orig_prog); return prog; } + +void *bpf_jit_alloc_exec(unsigned long size) +{ + return __vmalloc_node_range(size, PAGE_SIZE, BPF_JIT_REGION_START, + BPF_JIT_REGION_END, GFP_KERNEL, + PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE, + __builtin_return_address(0)); +} + +void bpf_jit_free_exec(void *addr) +{ + return vfree(addr); +} |
