[hwasan] Fix inline instrumentation.

This patch changes hwasan inline instrumentation:

Fixes address untagging for shadow address calculation (use 0xFF instead of 0x00 for the top byte).
Emits brk instruction instead of hlt for the kernel and user space.
Use 0x900 instead of 0x100 for brk immediate (0x100 - 0x800 are unavailable in the kernel).
Fixes and adds appropriate tests.

Patch by Andrey Konovalov.

Differential Revision: https://reviews.llvm.org/D43135

llvm-svn: 325711

11 files changed, 86 insertions, 50 deletions

diff --git a/compiler-rt/lib/hwasan/hwasan.cc b/compiler-rt/lib/hwasan/hwasan.cc
index f565e7858af..b2aadc4c856 100644
--- a/compiler-rt/lib/hwasan/hwasan.cc
+++ b/compiler-rt/lib/hwasan/hwasan.cc
@@ -84,7 +84,7 @@ static void InitializeFlags() {
     cf.check_printf = false;
     cf.intercept_tls_get_addr = true;
     cf.exitcode = 99;
-    cf.handle_sigill = kHandleSignalExclusive;
+    cf.handle_sigtrap = kHandleSignalExclusive;
     OverrideCommonFlags(cf);
   }
 
@@ -240,9 +240,9 @@ void __sanitizer_unaligned_store64(uu64 *p, u64 x) {
 
 template<unsigned X>
 __attribute__((always_inline))
-static void SigIll() {
+static void SigTrap() {
 #if defined(__aarch64__)
-  asm("hlt %0\n\t" ::"n"(X));
+  asm("brk %0\n\t" ::"n"(X));
 #elif defined(__x86_64__) || defined(__i386__)
   asm("ud2\n\t");
 #else
@@ -261,7 +261,7 @@ __attribute__((always_inline, nodebug)) static void CheckAddress(uptr p) {
   uptr ptr_raw = p & ~kAddressTagMask;
   tag_t mem_tag = *(tag_t *)MEM_TO_SHADOW(ptr_raw);
   if (UNLIKELY(ptr_tag != mem_tag)) {
-    SigIll<0x100 + 0x20 * (EA == ErrorAction::Recover) +
+    SigTrap<0x900 + 0x20 * (EA == ErrorAction::Recover) +
            0x10 * (AT == AccessType::Store) + LogSize>();
     if (EA == ErrorAction::Abort) __builtin_unreachable();
   }
@@ -277,7 +277,7 @@ __attribute__((always_inline, nodebug)) static void CheckAddressSized(uptr p,
   tag_t *shadow_last = (tag_t *)MEM_TO_SHADOW(ptr_raw + sz - 1);
   for (tag_t *t = shadow_first; t <= shadow_last; ++t)
     if (UNLIKELY(ptr_tag != *t)) {
-      SigIll<0x100 + 0x20 * (EA == ErrorAction::Recover) +
+      SigTrap<0x900 + 0x20 * (EA == ErrorAction::Recover) +
              0x10 * (AT == AccessType::Store) + 0xf>();
       if (EA == ErrorAction::Abort) __builtin_unreachable();
     }
diff --git a/compiler-rt/lib/hwasan/hwasan_linux.cc b/compiler-rt/lib/hwasan/hwasan_linux.cc
index f660272c46e..9d3e4272e75 100644
--- a/compiler-rt/lib/hwasan/hwasan_linux.cc
+++ b/compiler-rt/lib/hwasan/hwasan_linux.cc
@@ -188,7 +188,7 @@ struct AccessInfo {
 
 #if defined(__aarch64__)
 static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
-  // Access type is encoded in HLT immediate as 0x1XY,
+  // Access type is encoded in BRK immediate as 0x9XY,
   // where X&1 is 1 for store, 0 for load,
   // and X&2 is 1 if the error is recoverable.
   // Valid values of Y are 0 to 4, which are interpreted as log2(access_size),
@@ -197,7 +197,7 @@ static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
   AccessInfo ai;
   uptr pc = (uptr)info->si_addr;
   unsigned code = ((*(u32 *)pc) >> 5) & 0xffff;
-  if ((code & 0xff00) != 0x100)
+  if ((code & 0xff00) != 0x900)
     return AccessInfo{0, 0, false, false}; // Not ours.
   bool is_store = code & 0x10;
   bool recover = code & 0x20;
@@ -221,7 +221,7 @@ static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
 }
 #endif
 
-static bool HwasanOnSIGILL(int signo, siginfo_t *info, ucontext_t *uc) {
+static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) {
   SignalContext sig{info, uc};
   AccessInfo ai = GetAccessInfo(info, uc);
   if (!ai.is_store && !ai.is_load)
@@ -251,8 +251,8 @@ static void OnStackUnwind(const SignalContext &sig, const void *,
 
 void HwasanOnDeadlySignal(int signo, void *info, void *context) {
   // Probably a tag mismatch.
-  if (signo == SIGILL)
-    if (HwasanOnSIGILL(signo, (siginfo_t *)info, (ucontext_t*)context))
+  if (signo == SIGTRAP)
+    if (HwasanOnSIGTRAP(signo, (siginfo_t *)info, (ucontext_t*)context))
       return;
 
   HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr);
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc b/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc
index cf04b18b62e..e231a8a30b5 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_flags.inc
@@ -93,6 +93,8 @@ COMMON_FLAG(HandleSignalMode, handle_abort, kHandleSignalNo,
             COMMON_FLAG_HANDLE_SIGNAL_HELP(SIGABRT))
 COMMON_FLAG(HandleSignalMode, handle_sigill, kHandleSignalNo,
             COMMON_FLAG_HANDLE_SIGNAL_HELP(SIGILL))
+COMMON_FLAG(HandleSignalMode, handle_sigtrap, kHandleSignalNo,
+            COMMON_FLAG_HANDLE_SIGNAL_HELP(SIGTRAP))
 COMMON_FLAG(HandleSignalMode, handle_sigfpe, kHandleSignalYes,
             COMMON_FLAG_HANDLE_SIGNAL_HELP(SIGFPE))
 #undef COMMON_FLAG_HANDLE_SIGNAL_HELP
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc
index 66b05130cd7..4cea03f7fe2 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc
@@ -1618,6 +1618,8 @@ static HandleSignalMode GetHandleSignalModeImpl(int signum) {
       return common_flags()->handle_abort;
     case SIGILL:
       return common_flags()->handle_sigill;
+    case SIGTRAP:
+      return common_flags()->handle_sigtrap;
     case SIGFPE:
       return common_flags()->handle_sigfpe;
     case SIGSEGV:
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc
index 1489cdc9ee1..8fa61318a76 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cc
@@ -435,6 +435,8 @@ static HandleSignalMode GetHandleSignalModeImpl(int signum) {
       return common_flags()->handle_abort;
     case SIGILL:
       return common_flags()->handle_sigill;
+    case SIGTRAP:
+      return common_flags()->handle_sigtrap;
     case SIGFPE:
       return common_flags()->handle_sigfpe;
     case SIGSEGV:
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc b/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc
index db41cad6c31..73b65a71c9b 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc
@@ -218,6 +218,7 @@ void InstallDeadlySignalHandlers(SignalHandlerType handler) {
   MaybeInstallSigaction(SIGABRT, handler);
   MaybeInstallSigaction(SIGFPE, handler);
   MaybeInstallSigaction(SIGILL, handler);
+  MaybeInstallSigaction(SIGTRAP, handler);
 }
 
 bool SignalContext::IsStackOverflow() const {
diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index cc8abb8a60a..21541fc9029 100644
--- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -134,6 +134,7 @@ public:
   bool isInterestingAlloca(const AllocaInst &AI);
   bool tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag);
   Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag);
+  Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong);
   bool instrumentStack(SmallVectorImpl<AllocaInst *> &Allocas,
                        SmallVectorImpl<Instruction *> &RetVec);
   Value *getNextTagWithCall(IRBuilder<> &IRB);
@@ -291,9 +292,7 @@ void HWAddressSanitizer::instrumentMemAccessInline(Value *PtrLong, bool IsWrite,
                                                    Instruction *InsertBefore) {
   IRBuilder<> IRB(InsertBefore);
   Value *PtrTag = IRB.CreateTrunc(IRB.CreateLShr(PtrLong, kPointerTagShift), IRB.getInt8Ty());
-  Value *AddrLong =
-      IRB.CreateAnd(PtrLong, ConstantInt::get(PtrLong->getType(),
-                                              ~(0xFFULL << kPointerTagShift)));
+  Value *AddrLong = untagPointer(IRB, PtrLong);
   Value *ShadowLong = IRB.CreateLShr(AddrLong, kShadowScale);
   if (ClMappingOffset)
     ShadowLong = IRB.CreateAdd(
@@ -311,8 +310,8 @@ void HWAddressSanitizer::instrumentMemAccessInline(Value *PtrLong, bool IsWrite,
   // The signal handler will find the data address in x0.
   InlineAsm *Asm = InlineAsm::get(
       FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
-      "hlt #" +
-          itostr(0x100 + Recover * 0x20 + IsWrite * 0x10 + AccessSizeIndex),
+      "brk #" +
+          itostr(0x900 + Recover * 0x20 + IsWrite * 0x10 + AccessSizeIndex),
       "{x0}",
       /*hasSideEffects=*/true);
   IRB.CreateCall(Asm, PtrLong);
@@ -461,6 +460,21 @@ Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong
   return IRB.CreateIntToPtr(TaggedPtrLong, Ty);
 }
 
+// Remove tag from an address.
+Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) {
+  Value *UntaggedPtrLong;
+  if (ClEnableKhwasan) {
+    // Kernel addresses have 0xFF in the most significant byte.
+    UntaggedPtrLong = IRB.CreateOr(PtrLong,
+        ConstantInt::get(PtrLong->getType(), 0xFFULL << kPointerTagShift));
+  } else {
+    // Userspace addresses have 0x00.
+    UntaggedPtrLong = IRB.CreateAnd(PtrLong,
+        ConstantInt::get(PtrLong->getType(), ~(0xFFULL << kPointerTagShift)));
+  }
+  return UntaggedPtrLong;
+}
+
 bool HWAddressSanitizer::instrumentStack(
     SmallVectorImpl<AllocaInst *> &Allocas,
     SmallVectorImpl<Instruction *> &RetVec) {
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/atomic.ll b/llvm/test/Instrumentation/HWAddressSanitizer/atomic.ll
index 5492fda61bb..e6f7c2df23f 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/atomic.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/atomic.ll
@@ -8,7 +8,7 @@ target triple = "aarch64--linux-android"
 define void @atomicrmw(i64* %ptr) sanitize_hwaddress {
 ; CHECK-LABEL: @atomicrmw(
 ; CHECK: lshr i64 %[[A:[^ ]*]], 56
-; CHECK: call void asm sideeffect "hlt #275", "{x0}"(i64 %[[A]])
+; CHECK: call void asm sideeffect "brk #2323", "{x0}"(i64 %[[A]])
 ; CHECK: atomicrmw add i64* %ptr, i64 1 seq_cst
 ; CHECK: ret void
 
@@ -20,7 +20,7 @@ entry:
 define void @cmpxchg(i64* %ptr, i64 %compare_to, i64 %new_value) sanitize_hwaddress {
 ; CHECK-LABEL: @cmpxchg(
 ; CHECK: lshr i64 %[[A:[^ ]*]], 56
-; CHECK: call void asm sideeffect "hlt #275", "{x0}"(i64 %[[A]])
+; CHECK: call void asm sideeffect "brk #2323", "{x0}"(i64 %[[A]])
 ; CHECK: cmpxchg i64* %ptr, i64 %compare_to, i64 %new_value seq_cst seq_cst
 ; CHECK: ret void
 
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll b/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll
index 8c99a15b9ed..353d4afd72f 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll
@@ -18,9 +18,9 @@ define i8 @test_load8(i8* %a) sanitize_hwaddress {
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; ABORT: call void asm sideeffect "hlt #256", "{x0}"(i64 %[[A]])
+; ABORT: call void asm sideeffect "brk #2304", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
-; RECOVER: call void asm sideeffect "hlt #288", "{x0}"(i64 %[[A]])
+; RECOVER: call void asm sideeffect "brk #2336", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: %[[G:[^ ]*]] = load i8, i8* %a, align 4
@@ -43,9 +43,9 @@ define i16 @test_load16(i16* %a) sanitize_hwaddress {
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; ABORT: call void asm sideeffect "hlt #257", "{x0}"(i64 %[[A]])
+; ABORT: call void asm sideeffect "brk #2305", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
-; RECOVER: call void asm sideeffect "hlt #289", "{x0}"(i64 %[[A]])
+; RECOVER: call void asm sideeffect "brk #2337", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: %[[G:[^ ]*]] = load i16, i16* %a, align 4
@@ -68,9 +68,9 @@ define i32 @test_load32(i32* %a) sanitize_hwaddress {
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; ABORT: call void asm sideeffect "hlt #258", "{x0}"(i64 %[[A]])
+; ABORT: call void asm sideeffect "brk #2306", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
-; RECOVER: call void asm sideeffect "hlt #290", "{x0}"(i64 %[[A]])
+; RECOVER: call void asm sideeffect "brk #2338", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: %[[G:[^ ]*]] = load i32, i32* %a, align 4
@@ -93,9 +93,9 @@ define i64 @test_load64(i64* %a) sanitize_hwaddress {
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; ABORT: call void asm sideeffect "hlt #259", "{x0}"(i64 %[[A]])
+; ABORT: call void asm sideeffect "brk #2307", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
-; RECOVER: call void asm sideeffect "hlt #291", "{x0}"(i64 %[[A]])
+; RECOVER: call void asm sideeffect "brk #2339", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: %[[G:[^ ]*]] = load i64, i64* %a, align 8
@@ -118,9 +118,9 @@ define i128 @test_load128(i128* %a) sanitize_hwaddress {
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; ABORT: call void asm sideeffect "hlt #260", "{x0}"(i64 %[[A]])
+; ABORT: call void asm sideeffect "brk #2308", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
-; RECOVER: call void asm sideeffect "hlt #292", "{x0}"(i64 %[[A]])
+; RECOVER: call void asm sideeffect "brk #2340", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: %[[G:[^ ]*]] = load i128, i128* %a, align 16
@@ -156,9 +156,9 @@ define void @test_store8(i8* %a, i8 %b) sanitize_hwaddress {
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; ABORT: call void asm sideeffect "hlt #272", "{x0}"(i64 %[[A]])
+; ABORT: call void asm sideeffect "brk #2320", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
-; RECOVER: call void asm sideeffect "hlt #304", "{x0}"(i64 %[[A]])
+; RECOVER: call void asm sideeffect "brk #2352", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: store i8 %b, i8* %a, align 4
@@ -181,9 +181,9 @@ define void @test_store16(i16* %a, i16 %b) sanitize_hwaddress {
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; ABORT: call void asm sideeffect "hlt #273", "{x0}"(i64 %[[A]])
+; ABORT: call void asm sideeffect "brk #2321", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
-; RECOVER: call void asm sideeffect "hlt #305", "{x0}"(i64 %[[A]])
+; RECOVER: call void asm sideeffect "brk #2353", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: store i16 %b, i16* %a, align 4
@@ -206,9 +206,9 @@ define void @test_store32(i32* %a, i32 %b) sanitize_hwaddress {
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; ABORT: call void asm sideeffect "hlt #274", "{x0}"(i64 %[[A]])
+; ABORT: call void asm sideeffect "brk #2322", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
-; RECOVER: call void asm sideeffect "hlt #306", "{x0}"(i64 %[[A]])
+; RECOVER: call void asm sideeffect "brk #2354", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: store i32 %b, i32* %a, align 4
@@ -231,9 +231,9 @@ define void @test_store64(i64* %a, i64 %b) sanitize_hwaddress {
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; ABORT: call void asm sideeffect "hlt #275", "{x0}"(i64 %[[A]])
+; ABORT: call void asm sideeffect "brk #2323", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
-; RECOVER: call void asm sideeffect "hlt #307", "{x0}"(i64 %[[A]])
+; RECOVER: call void asm sideeffect "brk #2355", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: store i64 %b, i64* %a, align 8
@@ -256,9 +256,9 @@ define void @test_store128(i128* %a, i128 %b) sanitize_hwaddress {
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
 ; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; ABORT: call void asm sideeffect "hlt #276", "{x0}"(i64 %[[A]])
+; ABORT: call void asm sideeffect "brk #2324", "{x0}"(i64 %[[A]])
 ; ABORT: unreachable
-; RECOVER: call void asm sideeffect "hlt #308", "{x0}"(i64 %[[A]])
+; RECOVER: call void asm sideeffect "brk #2356", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
 ; CHECK: store i128 %b, i128* %a, align 16
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/kernel-alloca.ll b/llvm/test/Instrumentation/HWAddressSanitizer/kernel-alloca.ll
index 6e56919960a..8d31aec21cd 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/kernel-alloca.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/kernel-alloca.ll
@@ -1,4 +1,4 @@
-; Test basic address sanitizer instrumentation.
+; Test kernel hwasan instrumentation for alloca.
 ;
 ; RUN: opt < %s -hwasan -hwasan-kernel=1 -S | FileCheck %s
 
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/kernel.ll b/llvm/test/Instrumentation/HWAddressSanitizer/kernel.ll
index d6919aad21c..43e60be01f4 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/kernel.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/kernel.ll
@@ -1,27 +1,42 @@
 ; Test kernel hwasan instrumentation.
 ;
-; RUN: opt < %s -hwasan -hwasan-kernel=1 -S | FileCheck %s --allow-empty --check-prefixes=KERNEL
-; RUN: opt < %s -hwasan -hwasan-mapping-offset=12345678 -S | FileCheck %s  --check-prefixes=OFFSET
+; RUN: opt < %s -hwasan -hwasan-kernel=1 -S | FileCheck %s --allow-empty --check-prefixes=INIT
+; RUN: opt < %s -hwasan -hwasan-kernel=1 -S | FileCheck %s  --check-prefixes=CHECK,NOOFFSET
+; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-mapping-offset=12345678 -S | FileCheck %s  --check-prefixes=CHECK,OFFSET
+; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=0 -S | FileCheck %s  --check-prefixes=CHECK,NOOFFSET,ABORT
+; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=1 -S | FileCheck %s  --check-prefixes=CHECK,NOOFFSET,RECOVER
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
 
 define i8 @test_load(i8* %a) sanitize_hwaddress {
-; OFFSET-LABEL: @test_load(
-; OFFSET: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
-; OFFSET: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; OFFSET: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; OFFSET: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; OFFSET: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; CHECK-LABEL: @test_load(
+; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
+; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; CHECK: %[[C:[^ ]*]] = or i64 %[[A]], -72057594037927936
+; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+
+; NOOFFSET: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+
 ; OFFSET: %[[D1:[^ ]*]] = add i64 %[[D]], 12345678
 ; OFFSET: %[[E:[^ ]*]] = inttoptr i64 %[[D1]] to i8*
-; OFFSET: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
-; OFFSET: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; OFFSET: br i1 %[[F]],
+
+; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
+; ABORT: call void asm sideeffect "brk #2304", "{x0}"(i64 %[[A]])
+; ABORT: unreachable
+; RECOVER: call void asm sideeffect "brk #2336", "{x0}"(i64 %[[A]])
+; RECOVER: br label
+
+; CHECK: %[[G:[^ ]*]] = load i8, i8* %a, align 4
+; CHECK: ret i8 %[[G]]
 
 entry:
   %b = load i8, i8* %a, align 4
   ret i8 %b
 }
 
-; KERNEL-NOT: call void @__hwasan_init
+; INIT-NOT: call void @__hwasan_init