[llvm-mc] Ignore opcode size prefix in 64-bit CALL disassembly

This is a fix for disassembling unusual instruction sequences in 64-bit
mode w.r.t the CALL rel16 instruction. It might be desirable to move the
check somewhere else, but it essentially mimics the special case
handling with JCXZ in 16-bit mode.

The current behavior accepts the opcode size prefix and causes the
call's immediate to stop disassembling after 2 bytes. When debugging
sequences of instructions with this pattern, the disassembler output
becomes extremely unreliable and essentially useless (if you jump midway
into what lldb thinks is a unified instruction, you'll lose %rip). So we
ignore the prefix and consume all 4 bytes when disassembling a 64-bit
mode binary.

Note: in Vol. 2A 3-99 the Intel spec states that CALL rel16 is N.S. N.S.
is defined as:

    Indicates an instruction syntax that requires an address override
    prefix in 64-bit mode and is not supported. Using an address
    override prefix in 64-bit mode may result in model-specific
    execution behavior. (Vol. 2A 3-7)

Since 0x66 is an operand override prefix we should be OK (although we
may want to warn about 0x67 prefixes to 0xe8). On the CPUs I tested
with, they all ignore the 0x66 prefix in 64-bit mode.

Patch by Matthew Barney!

Differential Revision: http://reviews.llvm.org/D9573

llvm-svn: 246038

2 files changed, 152 insertions, 0 deletions

diff --git a/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.cpp b/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.cpp
index f73fa75f888..e7a1d7dad98 100644
--- a/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.cpp
+++ b/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.cpp
@@ -980,6 +980,47 @@ static int getID(struct InternalInstruction* insn, const void *miiArg) {
       insn->opcode == 0xE3)
     attrMask ^= ATTR_ADSIZE;
 
+  /*
+   * In 64-bit mode all f64 superscripted opcodes ignore opcode size prefix
+   * CALL/JMP/JCC instructions need to ignore 0x66 and consume 4 bytes
+   */
+
+  if (insn->mode == MODE_64BIT &&
+      isPrefixAtLocation(insn, 0x66, insn->necessaryPrefixLocation)) {
+    switch (insn->opcode) {
+    case 0xE8:
+    case 0xE9:
+      if (insn->opcodeType ==
+          ONEBYTE) { // breaks psubsb and other mmx instructions otherwise
+        attrMask ^= ATTR_OPSIZE;
+        insn->immediateSize = 4;
+        insn->displacementSize = 4;
+      }
+      break;
+    case 0x82:
+    case 0x83:
+    case 0x84:
+    case 0x85:
+    case 0x86:
+    case 0x87:
+    case 0x88:
+    case 0x89:
+    case 0x8A:
+    case 0x8B:
+    case 0x8C:
+    case 0x8D:
+    case 0x8E:
+    case 0x8F:
+      if (insn->opcodeType ==
+          TWOBYTE) { // breaks lea and three byte ops otherwise
+        attrMask ^= ATTR_OPSIZE;
+        insn->immediateSize = 4;
+        insn->displacementSize = 4; // otherwise not sign extended
+      }
+      break;
+    }
+  }
+
   if (getIDWithAttrMask(&instructionID, insn, attrMask))
     return -1;
 
diff --git a/llvm/test/MC/Disassembler/X86/x86-64.txt b/llvm/test/MC/Disassembler/X86/x86-64.txt
index 065b2a57c84..13e36df002a 100644
--- a/llvm/test/MC/Disassembler/X86/x86-64.txt
+++ b/llvm/test/MC/Disassembler/X86/x86-64.txt
@@ -302,6 +302,117 @@
 # CHECK: movq %rax, 1515870810
 0x67, 0x48 0xa3 0x5a 0x5a 0x5a 0x5a
 
+# CHECK: callq -32769
+0x66 0xe8 0xff 0x7f 0xff 0xff
+
+# CHECK: callq -32769
+0x66 0x66 0x48 0xe8 0xff 0x7f 0xff 0xff
+
+# CHECK: jmp -32769
+0xe9 0xff 0x7f 0xff 0xff
+
+# CHECK: jmp -32769
+0x66 0xe9 0xff 0x7f 0xff 0xff
+
+# CHECK: jmp -32769
+0x66 0x66 0x48 0xe9 0xff 0x7f 0xff 0xff
+
+# CHECK: jb -32769
+0x0f 0x82 0xff 0x7f 0xff 0xff
+
+# CHECK: jb -32769
+0x66 0x0f 0x82 0xff 0x7f 0xff 0xff
+
+# CHECK: jae -32769
+0x0f 0x83 0xff 0x7f 0xff 0xff
+
+# CHECK: jae -32769
+0x66 0x0f 0x83 0xff 0x7f 0xff 0xff
+
+# CHECK: je -32769
+0x0f 0x84 0xff 0x7f 0xff 0xff
+
+# CHECK: je -32769
+0x66 0x0f 0x84 0xff 0x7f 0xff 0xff
+
+# CHECK: jne -32769
+0x0f 0x85 0xff 0x7f 0xff 0xff
+
+# CHECK: jne -32769
+0x66 0x0f 0x85 0xff 0x7f 0xff 0xff
+
+# CHECK: jbe -32769
+0x0f 0x86 0xff 0x7f 0xff 0xff
+
+# CHECK: jbe -32769
+0x66 0x0f 0x86 0xff 0x7f 0xff 0xff
+
+# CHECK: ja -32769
+0x0f 0x87 0xff 0x7f 0xff 0xff
+
+# CHECK: ja -32769
+0x66 0x0f 0x87 0xff 0x7f 0xff 0xff
+
+# CHECK: js -32769
+0x0f 0x88 0xff 0x7f 0xff 0xff
+
+# CHECK: js -32769
+0x66 0x0f 0x88 0xff 0x7f 0xff 0xff
+
+# CHECK: jns -32769
+0x0f 0x89 0xff 0x7f 0xff 0xff
+
+# CHECK: jns -32769
+0x66 0x0f 0x89 0xff 0x7f 0xff 0xff
+
+# CHECK: jp -32769
+0x0f 0x8a 0xff 0x7f 0xff 0xff
+
+# CHECK: jp -32769
+0x66 0x0f 0x8a 0xff 0x7f 0xff 0xff
+
+# CHECK: jnp -32769
+0x0f 0x8b 0xff 0x7f 0xff 0xff
+
+# CHECK: jnp -32769
+0x66 0x0f 0x8b 0xff 0x7f 0xff 0xff
+
+# CHECK: jl -32769
+0x0f 0x8c 0xff 0x7f 0xff 0xff
+
+# CHECK: jl -32769
+0x66 0x0f 0x8c 0xff 0x7f 0xff 0xff
+
+# CHECK: jge -32769
+0x0f 0x8d 0xff 0x7f 0xff 0xff
+
+# CHECK: jge -32769
+0x66 0x0f 0x8d 0xff 0x7f 0xff 0xff
+
+# CHECK: jle -32769
+0x0f 0x8e 0xff 0x7f 0xff 0xff
+
+# CHECK: jle -32769
+0x66 0x0f 0x8e 0xff 0x7f 0xff 0xff
+
+# CHECK: jg -32769
+0x0f 0x8f 0xff 0x7f 0xff 0xff
+
+# CHECK: jg -32769
+0x66 0x0f 0x8f 0xff 0x7f 0xff 0xff
+
+# CHECK: lcallw	*-32769(%rip)
+0x66 0xff 0x1d 0xff 0x7f 0xff 0xff
+
+# CHECK: ljmpw	*-32769(%rip)
+0x66 0xff 0x2d 0xff 0x7f 0xff 0xff
+
+# CHECK: psubsb	(%rdx), %mm3
+0x0f 0xe8 0x1a
+
+# CHECK: psubsb	(%rdx), %xmm3
+0x66 0x0f 0xe8 0x1a
+
 # CHECK: addq 255(%rip), %rbx
 0x49, 0x03, 0x1d, 0xff, 0x00, 0x00, 0x00