COFF: Implement subsystem inference.

llvm-svn: 238668

11 files changed, 103 insertions, 40 deletions

diff --git a/lld/COFF/Config.h b/lld/COFF/Config.h
index 677a87083b0..e0b6b20e377 100644
--- a/lld/COFF/Config.h
+++ b/lld/COFF/Config.h
@@ -25,7 +25,7 @@ class Configuration {
 public:
   llvm::COFF::MachineTypes MachineType = llvm::COFF::IMAGE_FILE_MACHINE_AMD64;
   bool Verbose = false;
-  WindowsSubsystem Subsystem = llvm::COFF::IMAGE_SUBSYSTEM_WINDOWS_CUI;
+  WindowsSubsystem Subsystem = llvm::COFF::IMAGE_SUBSYSTEM_UNKNOWN;
   std::string EntryName;
 
   uint64_t ImageBase = 0x140000000;
diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index 5b0ce1c92d3..8673fe004f6 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -16,6 +16,7 @@
 #include "lld/Core/Error.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringSwitch.h"
 #include "llvm/Option/Arg.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/Option.h"
@@ -26,6 +27,9 @@
 #include <memory>
 
 using namespace llvm;
+using llvm::COFF::IMAGE_SUBSYSTEM_UNKNOWN;
+using llvm::COFF::IMAGE_SUBSYSTEM_WINDOWS_CUI;
+using llvm::COFF::IMAGE_SUBSYSTEM_WINDOWS_GUI;
 
 namespace lld {
 namespace coff {
@@ -200,6 +204,22 @@ bool link(int Argc, const char *Argv[]) {
   if (Symtab.reportRemainingUndefines())
     return false;
 
+  // Windows specific -- if no /subsystem is given, we need to infer
+  // that from entry point name.
+  if (Config->Subsystem == IMAGE_SUBSYSTEM_UNKNOWN) {
+    Config->Subsystem =
+      StringSwitch<WindowsSubsystem>(Config->EntryName)
+          .Case("mainCRTStartup", IMAGE_SUBSYSTEM_WINDOWS_CUI)
+          .Case("wmainCRTStartup", IMAGE_SUBSYSTEM_WINDOWS_CUI)
+          .Case("WinMainCRTStartup", IMAGE_SUBSYSTEM_WINDOWS_GUI)
+          .Case("wWinMainCRTStartup", IMAGE_SUBSYSTEM_WINDOWS_GUI)
+          .Default(IMAGE_SUBSYSTEM_UNKNOWN);
+    if (Config->Subsystem == IMAGE_SUBSYSTEM_UNKNOWN) {
+      llvm::errs() << "subsystem must be defined\n";
+      return false;
+    }
+  }
+
   // Write the result.
   Writer Out(&Symtab);
   if (auto EC = Out.write(getOutputPath(Args.get()))) {
diff --git a/lld/test/COFF/base.test b/lld/test/COFF/base.test
index 02535dfdde5..b1e3917c8b2 100644
--- a/lld/test/COFF/base.test
+++ b/lld/test/COFF/base.test
@@ -1,10 +1,11 @@
-# RUN: lld -flavor link2 /entry:main /out:%t.exe %p/Inputs/ret42.obj
+# RUN: lld -flavor link2 /entry:main /out:%t.exe /subsystem:console \
+# RUN:   %p/Inputs/ret42.obj
 # RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=DEFAULT %s
 
 DEFAULT: ImageBase: 0x140000000
 
-# RUN: lld -flavor link2 /entry:main /out:%t.exe /base:0x280000000 \
-# RUN:   %p/Inputs/ret42.obj
+# RUN: lld -flavor link2 /entry:main /out:%t.exe /subsystem:console \
+# RUN:   /base:0x280000000 %p/Inputs/ret42.obj
 # RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=BASE %s
 
 BASE: ImageBase: 0x280000000
diff --git a/lld/test/COFF/driver.test b/lld/test/COFF/driver.test
index 4d3242757a4..f29f6785c55 100644
--- a/lld/test/COFF/driver.test
+++ b/lld/test/COFF/driver.test
@@ -1,5 +1,3 @@
 # RUN: not lld -flavor link2 nosuchfile.obj >& %t.log
 # RUN: FileCheck -check-prefix=MISSING %s < %t.log
 MISSING: nosuchfile.obj: {{[Nn]}}o such file or directory
-
-# RUN: lld -flavor link2 /entry:main /out:%t.exe %p/Inputs/ret42.obj
diff --git a/lld/test/COFF/heap.test b/lld/test/COFF/heap.test
index dbd5ffdcdd1..c3b1d96b284 100644
--- a/lld/test/COFF/heap.test
+++ b/lld/test/COFF/heap.test
@@ -1,18 +1,19 @@
-# RUN: lld -flavor link2 /entry:main /out:%t.exe %p/Inputs/ret42.obj
+# RUN: lld -flavor link2 /entry:main /subsystem:console /out:%t.exe \
+# RUN:   %p/Inputs/ret42.obj
 # RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=DEFAULT %s
 
 DEFAULT: SizeOfHeapReserve: 1048576
 DEFAULT: SizeOfHeapCommit: 4096
 
-# RUN: lld -flavor link2 /entry:main /out:%t.exe /heap:0x3000 \
-# RUN:   %p/Inputs/ret42.obj
+# RUN: lld -flavor link2 /entry:main /subsystem:console /out:%t.exe \
+# RUN:   /heap:0x3000 %p/Inputs/ret42.obj
 # RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=CHECK1 %s
 
 CHECK1: SizeOfHeapReserve: 12288
 CHECK1: SizeOfHeapCommit: 4096
 
-# RUN: lld -flavor link2 /entry:main /out:%t.exe /heap:0x5000,0x3000 \
-# RUN:   %p/Inputs/ret42.obj
+# RUN: lld -flavor link2 /entry:main /subsystem:console /out:%t.exe \
+# RUN:   /heap:0x5000,0x3000 %p/Inputs/ret42.obj
 # RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=CHECK2 %s
 
 CHECK2: SizeOfHeapReserve: 20480
diff --git a/lld/test/COFF/imports.test b/lld/test/COFF/imports.test
index 7b21b4cc2fa..1bc358ef488 100644
--- a/lld/test/COFF/imports.test
+++ b/lld/test/COFF/imports.test
@@ -1,7 +1,7 @@
 # Verify that the lld can handle .lib files and emit .idata sections.
 #
-# RUN: lld -flavor link2 /out:%t.exe /entry:main %p/Inputs/hello64.obj \
-# RUN:   %p/Inputs/std64.lib
+# RUN: lld -flavor link2 /out:%t.exe /entry:main /subsystem:console \
+# RUN:   %p/Inputs/hello64.obj %p/Inputs/std64.lib
 # RUN: llvm-objdump -d %t.exe | FileCheck -check-prefix=TEXT %s
 # RUN: llvm-readobj -coff-imports %t.exe | FileCheck -check-prefix=IMPORT %s
 
diff --git a/lld/test/COFF/machine.test b/lld/test/COFF/machine.test
index e5107d91561..39b3f9610cf 100644
--- a/lld/test/COFF/machine.test
+++ b/lld/test/COFF/machine.test
@@ -1,20 +1,20 @@
-# RUN: lld -flavor link2 /entry:main /machine:x64 /out:%t.exe \
-# RUN:   %p/Inputs/ret42.obj
+# RUN: lld -flavor link2 /entry:main /subsystem:console /machine:x64 \
+# RUN:   /out:%t.exe %p/Inputs/ret42.obj
 # RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=AMD64 %s
-# RUN: lld -flavor link2 /entry:main /machine:x64 /out:%t.exe \
-# RUN:   %p/Inputs/ret42.lib
+# RUN: lld -flavor link2 /entry:main /subsystem:console /machine:x64 \
+# RUN:   /out:%t.exe %p/Inputs/ret42.lib
 # RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=AMD64 %s
-# RUN: lld -flavor link2 /entry:main /out:%t.exe \
+# RUN: lld -flavor link2 /entry:main /subsystem:console /out:%t.exe \
 # RUN:   %p/Inputs/ret42.obj
 # RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=AMD64 %s
-# RUN: lld -flavor link2 /entry:main /out:%t.exe \
+# RUN: lld -flavor link2 /entry:main /subsystem:console /out:%t.exe \
 # RUN:   %p/Inputs/ret42.lib
 # RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=AMD64 %s
 
 AMD64: Machine: IMAGE_FILE_MACHINE_AMD64
 
-# RUN: lld -flavor link2 /entry:main /machine:x86 /out:%t.exe \
-# RUN:   %p/Inputs/ret42.obj
+# RUN: lld -flavor link2 /entry:main /subsystem:console /machine:x86 \
+# RUN:   /out:%t.exe %p/Inputs/ret42.obj
 # RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=I386 %s
 
 I386: Machine: IMAGE_FILE_MACHINE_I386
diff --git a/lld/test/COFF/stack.test b/lld/test/COFF/stack.test
index a6eff6246c2..5d55cef95cd 100644
--- a/lld/test/COFF/stack.test
+++ b/lld/test/COFF/stack.test
@@ -1,18 +1,19 @@
-# RUN: lld -flavor link2 /entry:main /out:%t.exe %p/Inputs/ret42.obj
+# RUN: lld -flavor link2 /entry:main /subsystem:console /out:%t.exe \
+# RUN:   %p/Inputs/ret42.obj
 # RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=DEFAULT %s
 
 DEFAULT: SizeOfStackReserve: 1048576
 DEFAULT: SizeOfStackCommit: 4096
 
-# RUN: lld -flavor link2 /entry:main /out:%t.exe /stack:0x3000 \
-# RUN:   %p/Inputs/ret42.obj
+# RUN: lld -flavor link2 /entry:main /subsystem:console /out:%t.exe \
+# RUN:   /stack:0x3000 %p/Inputs/ret42.obj
 # RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=CHECK1 %s
 
 CHECK1: SizeOfStackReserve: 12288
 CHECK1: SizeOfStackCommit: 4096
 
-# RUN: lld -flavor link2 /entry:main /out:%t.exe /stack:0x5000,0x3000 \
-# RUN:   %p/Inputs/ret42.obj
+# RUN: lld -flavor link2 /entry:main /subsystem:console /out:%t.exe \
+# RUN:   /stack:0x5000,0x3000 %p/Inputs/ret42.obj
 # RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=CHECK2 %s
 
 CHECK2: SizeOfStackReserve: 20480
diff --git a/lld/test/COFF/subsystem-inference.test b/lld/test/COFF/subsystem-inference.test
new file mode 100644
index 00000000000..4edf5d45f3c
--- /dev/null
+++ b/lld/test/COFF/subsystem-inference.test
@@ -0,0 +1,50 @@
+# RUN: sed -e s/ENTRYNAME/mainCRTStartup/ %s | yaml2obj > %t.obj
+# RUN: lld -flavor link2 /out:%t.exe %t.obj
+# RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=MAIN %s
+
+# RUN: sed s/ENTRYNAME/wmainCRTStartup/ %s | yaml2obj > %t.obj
+# RUN: lld -flavor link2 /out:%t.exe %t.obj
+# RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=WMAIN %s
+
+# RUN: sed s/ENTRYNAME/WinMainCRTStartup/ %s | yaml2obj > %t.obj
+# RUN: lld -flavor link2 /out:%t.exe %t.obj
+# RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=WINMAIN %s
+
+# RUN: sed s/ENTRYNAME/wWinMainCRTStartup/ %s | yaml2obj > %t.obj
+# RUN: lld -flavor link2 /out:%t.exe %t.obj
+# RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=WWINMAIN %s
+
+# MAIN:     Subsystem: IMAGE_SUBSYSTEM_WINDOWS_CUI
+# WMAIN:    Subsystem: IMAGE_SUBSYSTEM_WINDOWS_CUI
+# WINMAIN:  Subsystem: IMAGE_SUBSYSTEM_WINDOWS_GUI
+# WWINMAIN: Subsystem: IMAGE_SUBSYSTEM_WINDOWS_GUI
+
+---
+header:
+  Machine:         IMAGE_FILE_MACHINE_AMD64
+  Characteristics: []
+sections:
+  - Name:            .text
+    Characteristics: [ IMAGE_SCN_CNT_CODE, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ ]
+    Alignment:       4
+    SectionData:     B82A000000C3
+symbols:
+  - Name:            .text
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+    SectionDefinition:
+      Length:          6
+      NumberOfRelocations: 0
+      NumberOfLinenumbers: 0
+      CheckSum:        0
+      Number:          0
+  - Name:            ENTRYNAME
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+...
diff --git a/lld/test/COFF/subsystem.test b/lld/test/COFF/subsystem.test
index 4dc92a23737..8a7c7676861 100644
--- a/lld/test/COFF/subsystem.test
+++ b/lld/test/COFF/subsystem.test
@@ -1,12 +1,3 @@
-# RUN: lld -flavor link2 /entry:main /out:%t.exe %p/Inputs/ret42.obj
-# RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=DEFAULT %s
-
-DEFAULT: MajorOperatingSystemVersion: 6
-DEFAULT: MinorOperatingSystemVersion: 0
-DEFAULT: MajorSubsystemVersion: 6
-DEFAULT: MinorSubsystemVersion: 0
-DEFAULT: Subsystem: IMAGE_SUBSYSTEM_WINDOWS_CUI
-
 # RUN: lld -flavor link2 /entry:main /out:%t.exe /subsystem:windows \
 # RUN:   %p/Inputs/ret42.obj
 # RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=CHECK1 %s
diff --git a/lld/test/COFF/version.test b/lld/test/COFF/version.test
index c5c494ef4ba..f4a80f2d9ba 100644
--- a/lld/test/COFF/version.test
+++ b/lld/test/COFF/version.test
@@ -1,18 +1,19 @@
-# RUN: lld -flavor link2 /entry:main /out:%t.exe %p/Inputs/ret42.obj
+# RUN: lld -flavor link2 /entry:main /subsystem:console /out:%t.exe \
+# RUN:   %p/Inputs/ret42.obj
 # RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=DEFAULT %s
 
 DEFAULT: MajorImageVersion: 0
 DEFAULT: MinorImageVersion: 0
 
-# RUN: lld -flavor link2 /entry:main /out:%t.exe /version:11 \
-# RUN:   %p/Inputs/ret42.obj
+# RUN: lld -flavor link2 /entry:main /subsystem:console /out:%t.exe \
+# RUN:   /version:11 %p/Inputs/ret42.obj
 # RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=CHECK1 %s
 
 CHECK1: MajorImageVersion: 11
 CHECK1: MinorImageVersion: 0
 
-# RUN: lld -flavor link2 /entry:main /out:%t.exe /version:11.22 \
-# RUN:   %p/Inputs/ret42.obj
+# RUN: lld -flavor link2 /entry:main /subsystem:console /out:%t.exe \
+# RUN:   /version:11.22 %p/Inputs/ret42.obj
 # RUN: llvm-readobj -file-headers %t.exe | FileCheck -check-prefix=CHECK2 %s
 
 CHECK2: MajorImageVersion: 11