[libc] Add a TableGen based header generator.

Summary:
* The Python header generator has been removed.
* Docs giving a highlevel overview of the header gen scheme have been
  added.

Reviewers: phosek, abrachet

Subscribers: mgorny, MaskRay, tschuett, libc-commits

Tags: #libc-project

Differential Revision: https://reviews.llvm.org/D70197

11 files changed, 667 insertions, 188 deletions

diff --git a/libc/utils/CMakeLists.txt b/libc/utils/CMakeLists.txt
new file mode 100644
index 00000000000..472d08c97d1
--- /dev/null
+++ b/libc/utils/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(HdrGen)
diff --git a/libc/utils/HdrGen/CMakeLists.txt b/libc/utils/HdrGen/CMakeLists.txt
new file mode 100644
index 00000000000..95b9a8952eb
--- /dev/null
+++ b/libc/utils/HdrGen/CMakeLists.txt
@@ -0,0 +1,10 @@
+add_tablegen(libc-hdrgen llvm-libc
+  Command.h
+  Generator.cpp
+  Generator.h
+  IncludeFileCommand.cpp
+  IncludeFileCommand.h
+  Main.cpp
+  PublicAPICommand.cpp
+  PublicAPICommand.h
+)
diff --git a/libc/utils/HdrGen/Command.h b/libc/utils/HdrGen/Command.h
new file mode 100644
index 00000000000..759e4315ff9
--- /dev/null
+++ b/libc/utils/HdrGen/Command.h
@@ -0,0 +1,52 @@
+//===-------- Base class for header generation commands ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_UTILS_HDRGEN_COMMAND_H
+#define LLVM_LIBC_UTILS_HDRGEN_COMMAND_H
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/Support/SourceMgr.h"
+
+#include <cstdlib>
+
+namespace llvm {
+
+class raw_ostream;
+class RecordKeeper;
+
+} // namespace llvm
+
+namespace llvm_libc {
+
+typedef llvm::SmallVector<llvm::StringRef, 4> ArgVector;
+
+class Command {
+public:
+  class ErrorReporter {
+    llvm::SMLoc Loc;
+    const llvm::SourceMgr &SrcMgr;
+
+  public:
+    ErrorReporter(llvm::SMLoc L, llvm::SourceMgr &SM) : Loc(L), SrcMgr(SM) {}
+
+    void printFatalError(llvm::Twine Msg) const {
+      SrcMgr.PrintMessage(Loc, llvm::SourceMgr::DK_Error, Msg);
+      std::exit(1);
+    }
+  };
+
+  virtual void run(llvm::raw_ostream &OS, const ArgVector &Args,
+                   llvm::StringRef StdHeader, llvm::RecordKeeper &Records,
+                   const ErrorReporter &Reporter) const = 0;
+};
+
+} // namespace llvm_libc
+
+#endif // LLVM_LIBC_UTILS_HDRGEN_COMMAND_H
diff --git a/libc/utils/HdrGen/Generator.cpp b/libc/utils/HdrGen/Generator.cpp
new file mode 100644
index 00000000000..996f3bf2699
--- /dev/null
+++ b/libc/utils/HdrGen/Generator.cpp
@@ -0,0 +1,119 @@
+//===---- Implementation of the main header generation class -----*- C++ -*===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Generator.h"
+
+#include "IncludeFileCommand.h"
+#include "PublicAPICommand.h"
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <cstdlib>
+#include <memory>
+
+static const char CommandPrefix[] = "%%";
+static const size_t CommandPrefixSize = llvm::StringRef(CommandPrefix).size();
+
+static const char CommentPrefix[] = "<!>";
+
+static const char ParamNamePrefix[] = "${";
+static const size_t ParamNamePrefixSize =
+    llvm::StringRef(ParamNamePrefix).size();
+static const char ParamNameSuffix[] = "}";
+static const size_t ParamNameSuffixSize =
+    llvm::StringRef(ParamNameSuffix).size();
+
+namespace llvm_libc {
+
+Command *Generator::getCommandHandler(llvm::StringRef CommandName) {
+  if (CommandName == IncludeFileCommand::Name) {
+    if (!IncludeFileCmd)
+      IncludeFileCmd = std::make_unique<IncludeFileCommand>();
+    return IncludeFileCmd.get();
+  } else if (CommandName == PublicAPICommand::Name) {
+    if (!PublicAPICmd)
+      PublicAPICmd = std::make_unique<PublicAPICommand>();
+    return PublicAPICmd.get();
+  } else {
+    return nullptr;
+  }
+}
+
+void Generator::parseCommandArgs(llvm::StringRef ArgStr, ArgVector &Args) {
+  if (!ArgStr.contains(',') && ArgStr.trim(' ').trim('\t').size() == 0) {
+    // If it is just space between the parenthesis
+    return;
+  }
+
+  ArgStr.split(Args, ",");
+  for (llvm::StringRef &A : Args) {
+    A = A.trim(' ');
+    if (A.startswith(ParamNamePrefix) && A.endswith(ParamNameSuffix)) {
+      A = A.drop_front(ParamNamePrefixSize).drop_back(ParamNameSuffixSize);
+      A = ArgMap[A];
+    }
+  }
+}
+
+void Generator::generate(llvm::raw_ostream &OS, llvm::RecordKeeper &Records) {
+  auto DefFileBuffer = llvm::MemoryBuffer::getFile(HeaderDefFile);
+  if (!DefFileBuffer) {
+    llvm::errs() << "Unable to open " << HeaderDefFile << ".\n";
+    std::exit(1);
+  }
+  llvm::SourceMgr SrcMgr;
+  unsigned DefFileID = SrcMgr.AddNewSourceBuffer(
+      std::move(DefFileBuffer.get()), llvm::SMLoc::getFromPointer(nullptr));
+
+  llvm::StringRef Content = SrcMgr.getMemoryBuffer(DefFileID)->getBuffer();
+  while (true) {
+    std::pair<llvm::StringRef, llvm::StringRef> P = Content.split('\n');
+    Content = P.second;
+
+    llvm::StringRef Line = P.first.trim(' ');
+    if (Line.startswith(CommandPrefix)) {
+      Line = Line.drop_front(CommandPrefixSize);
+
+      P = Line.split("(");
+      if (P.second.empty() || P.second[P.second.size() - 1] != ')') {
+        SrcMgr.PrintMessage(llvm::SMLoc::getFromPointer(P.second.data()),
+                            llvm::SourceMgr::DK_Error,
+                            "Command argument list should begin with '(' "
+                            "and end with ')'.");
+        std::exit(1);
+      }
+      llvm::StringRef CommandName = P.first;
+      Command *Cmd = getCommandHandler(CommandName);
+      if (Cmd == nullptr) {
+        SrcMgr.PrintMessage(llvm::SMLoc::getFromPointer(CommandName.data()),
+                            llvm::SourceMgr::DK_Error,
+                            "Unknown command '%%" + CommandName + "'.");
+        std::exit(1);
+      }
+
+      llvm::StringRef ArgStr = P.second.drop_back(1);
+      ArgVector Args;
+      parseCommandArgs(ArgStr, Args);
+
+      Command::ErrorReporter Reporter(
+          llvm::SMLoc::getFromPointer(CommandName.data()), SrcMgr);
+      Cmd->run(OS, Args, StdHeader, Records, Reporter);
+    } else if (!Line.startswith(CommentPrefix)) {
+      // There is no comment or command on this line so we just write it as is.
+      OS << P.first << "\n";
+    }
+
+    if (P.second.empty())
+      break;
+  }
+}
+
+} // namespace llvm_libc
diff --git a/libc/utils/HdrGen/Generator.h b/libc/utils/HdrGen/Generator.h
new file mode 100644
index 00000000000..84197c491d0
--- /dev/null
+++ b/libc/utils/HdrGen/Generator.h
@@ -0,0 +1,56 @@
+//===------------- - The main header generation class -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_UTILS_HDRGEN_GENERATOR_H
+#define LLVM_LIBC_UTILS_HDRGEN_GENERATOR_H
+
+#include "Command.h"
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+
+namespace llvm {
+
+class raw_ostream;
+class RecordKeeper;
+
+} // namespace llvm
+
+namespace llvm_libc {
+
+class Command;
+
+class Generator {
+  llvm::StringRef HeaderDefFile;
+  llvm::StringRef StdHeader;
+  std::unordered_map<std::string, std::string> &ArgMap;
+
+  std::unique_ptr<Command> IncludeFileCmd;
+  std::unique_ptr<Command> PublicAPICmd;
+
+  Command *getCommandHandler(llvm::StringRef CommandName);
+
+  void parseCommandArgs(llvm::StringRef ArgStr, ArgVector &Args);
+
+  void printError(llvm::StringRef Msg);
+
+public:
+  Generator(const std::string &DefFile, const std::string &Header,
+            std::unordered_map<std::string, std::string> &Map)
+      : HeaderDefFile(DefFile), StdHeader(Header), ArgMap(Map) {}
+
+  void generate(llvm::raw_ostream &OS, llvm::RecordKeeper &Records);
+};
+
+} // namespace llvm_libc
+
+#endif // LLVM_LIBC_UTILS_HDRGEN_GENERATOR_H
diff --git a/libc/utils/HdrGen/IncludeFileCommand.cpp b/libc/utils/HdrGen/IncludeFileCommand.cpp
new file mode 100644
index 00000000000..bffaa0f91db
--- /dev/null
+++ b/libc/utils/HdrGen/IncludeFileCommand.cpp
@@ -0,0 +1,50 @@
+//===----------- Implementation of IncludeFileCommand -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "IncludeFileCommand.h"
+
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/SourceMgr.h"
+
+#include <cstdlib>
+
+namespace llvm_libc {
+
+const char IncludeFileCommand::Name[] = "include_file";
+
+void IncludeFileCommand::run(llvm::raw_ostream &OS, const ArgVector &Args,
+                             llvm::StringRef StdHeader,
+                             llvm::RecordKeeper &Records,
+                             const Command::ErrorReporter &Reporter) const {
+  if (Args.size() != 1) {
+    Reporter.printFatalError(
+        "%%include_file command takes exactly 1 argument.");
+  }
+
+  llvm::StringRef IncludeFile = Args[0];
+  auto Buffer = llvm::MemoryBuffer::getFileAsStream(IncludeFile);
+  if (!Buffer)
+    Reporter.printFatalError(llvm::StringRef("Unable to open ") + IncludeFile);
+
+  llvm::StringRef Content = Buffer.get()->getBuffer();
+
+  // If the included file has %%begin() command listed, then we want to write
+  // only the content after the begin command.
+  // TODO: The way the content is split below does not allow space within the
+  // the parentheses and, before and after the command. This probably is too
+  // strict and should be relaxed.
+  auto P = Content.split("\n%%begin()\n");
+  if (P.second.empty()) {
+    // There was no %%begin in the content.
+    OS << P.first;
+  } else {
+    OS << P.second;
+  }
+}
+
+} // namespace llvm_libc
diff --git a/libc/utils/HdrGen/IncludeFileCommand.h b/libc/utils/HdrGen/IncludeFileCommand.h
new file mode 100644
index 00000000000..d8875e31c0e
--- /dev/null
+++ b/libc/utils/HdrGen/IncludeFileCommand.h
@@ -0,0 +1,32 @@
+//===-------- Class which implements the %%include_file command -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_UTILS_HDRGEN_INCLUDE_COMMAND_H
+#define LLVM_LIBC_UTILS_HDRGEN_INCLUDE_COMMAND_H
+
+#include "Command.h"
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+
+#include <string>
+
+namespace llvm_libc {
+
+class IncludeFileCommand : public Command {
+public:
+  static const char Name[];
+
+  void run(llvm::raw_ostream &OS, const ArgVector &Args,
+           llvm::StringRef StdHeader, llvm::RecordKeeper &Records,
+           const Command::ErrorReporter &Reporter) const override;
+};
+
+} // namespace llvm_libc
+
+#endif // LLVM_LIBC_UTILS_HDRGEN_INCLUDE_COMMAND_H
diff --git a/libc/utils/HdrGen/Main.cpp b/libc/utils/HdrGen/Main.cpp
new file mode 100644
index 00000000000..13ff5baa0e9
--- /dev/null
+++ b/libc/utils/HdrGen/Main.cpp
@@ -0,0 +1,56 @@
+//===---------------- "main" function of libc-hdrgen ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Generator.h"
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/TableGen/Main.h"
+
+#include <string>
+#include <unordered_map>
+
+namespace {
+
+llvm::cl::opt<std::string>
+    HeaderDefFile("def", llvm::cl::desc("Path to the .h.def file."),
+                  llvm::cl::value_desc("<filename>"), llvm::cl::Required);
+llvm::cl::opt<std::string> StandardHeader(
+    "header",
+    llvm::cl::desc("The standard header file which is to be generated."),
+    llvm::cl::value_desc("<header file>"));
+llvm::cl::list<std::string> ReplacementValues(
+    "args", llvm::cl::desc("Command seperated <argument name>=<value> pairs."),
+    llvm::cl::value_desc("<name=value>[,name=value]"));
+
+void ParseArgValuePairs(std::unordered_map<std::string, std::string> &Map) {
+  for (std::string &R : ReplacementValues) {
+    auto Pair = llvm::StringRef(R).split('=');
+    Map[Pair.first] = Pair.second;
+  }
+}
+
+} // anonymous namespace
+
+namespace llvm_libc {
+
+bool HeaderGeneratorMain(llvm::raw_ostream &OS, llvm::RecordKeeper &Records) {
+  std::unordered_map<std::string, std::string> ArgMap;
+  ParseArgValuePairs(ArgMap);
+  Generator G(HeaderDefFile, StandardHeader, ArgMap);
+  G.generate(OS, Records);
+
+  return false;
+}
+
+} // namespace llvm_libc
+
+int main(int argc, char *argv[]) {
+  llvm::cl::ParseCommandLineOptions(argc, argv);
+  return TableGenMain(argv[0], &llvm_libc::HeaderGeneratorMain);
+}
diff --git a/libc/utils/HdrGen/PublicAPICommand.cpp b/libc/utils/HdrGen/PublicAPICommand.cpp
new file mode 100644
index 00000000000..95b58b1daa0
--- /dev/null
+++ b/libc/utils/HdrGen/PublicAPICommand.cpp
@@ -0,0 +1,255 @@
+//===--------------- Implementation of PublicAPICommand ----------*-C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "PublicAPICommand.h"
+
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
+
+static const char NamedTypeClassName[] = "NamedType";
+static const char PtrTypeClassName[] = "PtrType";
+static const char RestrictedPtrTypeClassName[] = "RestrictedPtrType";
+static const char ConstTypeClassName[] = "ConstType";
+static const char StructTypeClassName[] = "Struct";
+
+static const char StandardSpecClassName[] = "StandardSpec";
+static const char PublicAPIClassName[] = "PublicAPI";
+
+static bool isa(llvm::Record *Def, llvm::Record *TypeClass) {
+  llvm::RecordRecTy *RecordType = Def->getType();
+  llvm::ArrayRef<llvm::Record *> Classes = RecordType->getClasses();
+  // We want exact types. That is, we don't want the classes listed in
+  // spec.td to be subclassed. Hence, we do not want the record |Def|
+  // to be of more than one class type..
+  if (Classes.size() != 1)
+    return false;
+  return Classes[0] == TypeClass;
+}
+
+// Text blocks for macro definitions and type decls can be indented to
+// suit the surrounding tablegen listing. We need to dedent such blocks
+// before writing them out.
+static void dedentAndWrite(llvm::StringRef Text, llvm::raw_ostream &OS) {
+  llvm::SmallVector<llvm::StringRef, 10> Lines;
+  llvm::SplitString(Text, Lines, "\n");
+  size_t shortest_indent = 1024;
+  for (llvm::StringRef L : Lines) {
+    llvm::StringRef Indent = L.take_while([](char c) { return c == ' '; });
+    size_t IndentSize = Indent.size();
+    if (Indent.size() == L.size()) {
+      // Line is all spaces so no point noting the indent.
+      continue;
+    }
+    if (IndentSize < shortest_indent)
+      shortest_indent = IndentSize;
+  }
+  for (llvm::StringRef L : Lines) {
+    if (L.size() >= shortest_indent)
+      OS << L.drop_front(shortest_indent) << '\n';
+  }
+}
+
+class APIGenerator {
+  llvm::StringRef StdHeader;
+
+  // TableGen classes in spec.td.
+  llvm::Record *NamedTypeClass;
+  llvm::Record *PtrTypeClass;
+  llvm::Record *RestrictedPtrTypeClass;
+  llvm::Record *ConstTypeClass;
+  llvm::Record *StructClass;
+  llvm::Record *StandardSpecClass;
+  llvm::Record *PublicAPIClass;
+
+  using NameToRecordMapping = std::unordered_map<std::string, llvm::Record *>;
+  using NameSet = std::unordered_set<std::string>;
+
+  // Mapping from names to records defining them.
+  NameToRecordMapping MacroSpecMap;
+  NameToRecordMapping TypeSpecMap;
+  NameToRecordMapping FunctionSpecMap;
+  NameToRecordMapping MacroDefsMap;
+  NameToRecordMapping TypeDeclsMap;
+
+  NameSet Structs;
+  NameSet Functions;
+
+  bool isaNamedType(llvm::Record *Def) { return isa(Def, NamedTypeClass); }
+
+  bool isaStructType(llvm::Record *Def) { return isa(Def, StructClass); }
+
+  bool isaPtrType(llvm::Record *Def) { return isa(Def, PtrTypeClass); }
+
+  bool isaConstType(llvm::Record *Def) { return isa(Def, ConstTypeClass); }
+
+  bool isaRestrictedPtrType(llvm::Record *Def) {
+    return isa(Def, RestrictedPtrTypeClass);
+  }
+
+  bool isaStandardSpec(llvm::Record *Def) {
+    return isa(Def, StandardSpecClass);
+  }
+
+  bool isaPublicAPI(llvm::Record *Def) { return isa(Def, PublicAPIClass); }
+
+  std::string getTypeAsString(llvm::Record *TypeRecord) {
+    if (isaNamedType(TypeRecord) || isaStructType(TypeRecord)) {
+      return TypeRecord->getValueAsString("Name");
+    } else if (isaPtrType(TypeRecord)) {
+      return getTypeAsString(TypeRecord->getValueAsDef("PointeeType")) + " *";
+    } else if (isaConstType(TypeRecord)) {
+      return std::string("const ") +
+             getTypeAsString(TypeRecord->getValueAsDef("UnqualifiedType"));
+    } else if (isaRestrictedPtrType(TypeRecord)) {
+      return getTypeAsString(TypeRecord->getValueAsDef("PointeeType")) +
+             " *__restrict";
+    } else {
+      llvm::PrintFatalError(TypeRecord->getLoc(), "Invalid type.\n");
+    }
+  }
+
+  void indexStandardSpecDef(llvm::Record *StandardSpec) {
+    auto HeaderSpecList = StandardSpec->getValueAsListOfDefs("Headers");
+    for (llvm::Record *HeaderSpec : HeaderSpecList) {
+      if (HeaderSpec->getValueAsString("Name") == StdHeader) {
+        auto MacroSpecList = HeaderSpec->getValueAsListOfDefs("Macros");
+        // TODO: Trigger a fatal error on duplicate specs.
+        for (llvm::Record *MacroSpec : MacroSpecList)
+          MacroSpecMap[MacroSpec->getValueAsString("Name")] = MacroSpec;
+
+        auto TypeSpecList = HeaderSpec->getValueAsListOfDefs("Types");
+        for (llvm::Record *TypeSpec : TypeSpecList)
+          TypeSpecMap[TypeSpec->getValueAsString("Name")] = TypeSpec;
+
+        auto FunctionSpecList = HeaderSpec->getValueAsListOfDefs("Functions");
+        for (llvm::Record *FunctionSpec : FunctionSpecList) {
+          FunctionSpecMap[FunctionSpec->getValueAsString("Name")] =
+              FunctionSpec;
+        }
+      }
+    }
+  }
+
+  void indexPublicAPIDef(llvm::Record *PublicAPI) {
+    // While indexing the public API, we do not check if any of the entities
+    // requested is from an included standard. Such a check is done while
+    // generating the API.
+    auto MacroDefList = PublicAPI->getValueAsListOfDefs("Macros");
+    for (llvm::Record *MacroDef : MacroDefList)
+      MacroDefsMap[MacroDef->getValueAsString("Name")] = MacroDef;
+
+    auto TypeDeclList = PublicAPI->getValueAsListOfDefs("TypeDeclarations");
+    for (llvm::Record *TypeDecl : TypeDeclList)
+      TypeDeclsMap[TypeDecl->getValueAsString("Name")] = TypeDecl;
+
+    auto StructList = PublicAPI->getValueAsListOfStrings("Structs");
+    for (llvm::StringRef StructName : StructList)
+      Structs.insert(StructName);
+
+    auto FunctionList = PublicAPI->getValueAsListOfStrings("Functions");
+    for (llvm::StringRef FunctionName : FunctionList)
+      Functions.insert(FunctionName);
+  }
+
+  void index(llvm::RecordKeeper &Records) {
+    NamedTypeClass = Records.getClass(NamedTypeClassName);
+    PtrTypeClass = Records.getClass(PtrTypeClassName);
+    RestrictedPtrTypeClass = Records.getClass(RestrictedPtrTypeClassName);
+    StructClass = Records.getClass(StructTypeClassName);
+    ConstTypeClass = Records.getClass(ConstTypeClassName);
+    StandardSpecClass = Records.getClass(StandardSpecClassName);
+    PublicAPIClass = Records.getClass(PublicAPIClassName);
+
+    const auto &DefsMap = Records.getDefs();
+    for (auto &Pair : DefsMap) {
+      llvm::Record *Def = Pair.second.get();
+      if (isaStandardSpec(Def))
+        indexStandardSpecDef(Def);
+      if (isaPublicAPI(Def)) {
+        if (Def->getValueAsString("HeaderName") == StdHeader)
+          indexPublicAPIDef(Def);
+      }
+    }
+  }
+
+public:
+  APIGenerator(llvm::StringRef Header, llvm::RecordKeeper &Records)
+      : StdHeader(Header) {
+    index(Records);
+  }
+
+  void write(llvm::raw_ostream &OS) {
+    for (auto &Pair : MacroDefsMap) {
+      const std::string &Name = Pair.first;
+      if (MacroSpecMap.find(Name) == MacroSpecMap.end())
+        llvm::PrintFatalError(Name + " not found in any standard spec.\n");
+
+      llvm::Record *MacroDef = Pair.second;
+      dedentAndWrite(MacroDef->getValueAsString("Defn"), OS);
+
+      OS << '\n';
+    }
+
+    for (auto &Pair : TypeDeclsMap) {
+      const std::string &Name = Pair.first;
+      if (TypeSpecMap.find(Name) == TypeSpecMap.end())
+        llvm::PrintFatalError(Name + " not found in any standard spec.\n");
+
+      llvm::Record *TypeDecl = Pair.second;
+      dedentAndWrite(TypeDecl->getValueAsString("Decl"), OS);
+
+      OS << '\n';
+    }
+
+    OS << "__BEGIN_C_DECLS\n\n";
+    for (auto &Name : Functions) {
+      if (FunctionSpecMap.find(Name) == FunctionSpecMap.end())
+        llvm::PrintFatalError(Name + " not found in any standard spec.\n");
+
+      llvm::Record *FunctionSpec = FunctionSpecMap[Name];
+      llvm::Record *RetValSpec = FunctionSpec->getValueAsDef("Return");
+      llvm::Record *ReturnType = RetValSpec->getValueAsDef("ReturnType");
+
+      OS << getTypeAsString(ReturnType) << " " << Name << "(";
+
+      auto ArgsList = FunctionSpec->getValueAsListOfDefs("Args");
+      for (size_t i = 0; i < ArgsList.size(); ++i) {
+        llvm::Record *ArgType = ArgsList[i]->getValueAsDef("ArgType");
+        OS << getTypeAsString(ArgType);
+        if (i < ArgsList.size() - 1)
+          OS << ", ";
+      }
+
+      OS << ");\n\n";
+    }
+    OS << "__END_C_DECLS\n";
+  }
+};
+
+namespace llvm_libc {
+
+void writePublicAPI(llvm::raw_ostream &OS, llvm::RecordKeeper &Records) {}
+
+const char PublicAPICommand::Name[] = "public_api";
+
+void PublicAPICommand::run(llvm::raw_ostream &OS, const ArgVector &Args,
+                           llvm::StringRef StdHeader,
+                           llvm::RecordKeeper &Records,
+                           const Command::ErrorReporter &Reporter) const {
+  if (Args.size() != 0) {
+    Reporter.printFatalError("public_api command does not take any arguments.");
+  }
+
+  APIGenerator G(StdHeader, Records);
+  G.write(OS);
+}
+
+} // namespace llvm_libc
diff --git a/libc/utils/HdrGen/PublicAPICommand.h b/libc/utils/HdrGen/PublicAPICommand.h
new file mode 100644
index 00000000000..253cd13f676
--- /dev/null
+++ b/libc/utils/HdrGen/PublicAPICommand.h
@@ -0,0 +1,36 @@
+//===---------- Implementation of PublicAPICommand --------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Command.h"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+
+namespace llvm {
+
+class raw_ostream;
+class Record;
+class RecordKeeper;
+
+} // namespace llvm
+
+namespace llvm_libc {
+
+class PublicAPICommand : public Command {
+public:
+  static const char Name[];
+
+  void run(llvm::raw_ostream &OS, const ArgVector &Args,
+           llvm::StringRef StdHeader, llvm::RecordKeeper &Records,
+           const Command::ErrorReporter &Reporter) const override;
+};
+
+} // namespace llvm_libc
diff --git a/libc/utils/build_scripts/gen_hdr.py b/libc/utils/build_scripts/gen_hdr.py
deleted file mode 100755
index 54ee0063a02..00000000000
--- a/libc/utils/build_scripts/gen_hdr.py
+++ /dev/null
@@ -1,188 +0,0 @@
-#! /usr/bin/python
-#===---------------- Script to generate header files ----------------------===#
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https:#llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-#===-----------------------------------------------------------------------===#
-#
-# This script takes a .h.def file and generates a .h header file.
-# See docs/header_generation.md for more information.
-#
-#===-----------------------------------------------------------------------===#
-
-import argparse
-import contextlib
-import os
-import sys
-
-COMMAND_PREFIX = "%%"
-COMMENT_PREFIX = "<!>"
-
-BEGIN_COMMAND = "begin"
-COMMENT_COMMAND = "comment"
-INCLUDE_FILE_COMMAND = "include_file"
-
-
-class _Location(object):
-    def __init__(self, filename, line_number):
-        self.filename = filename
-        self.line_number = line_number
-
-    def __str__(self):
-        return "%s:%s" % (self.filename, self.line_number)
-
-
-@contextlib.contextmanager
-def output_stream_manager(filename):
-    if filename is None:
-        try:
-            yield sys.stdout
-        finally:
-            pass
-    else:
-        output_stream = open(filename, "w")
-        try:
-            yield output_stream
-        finally:
-            output_stream.close()
-
-
-def _parse_command(loc, line):
-    open_paren = line.find("(")
-    if open_paren < 0 or line[-1] != ")":
-        return _fatal_error(loc, "Incorrect header generation command syntax.")
-    command_name = line[len(COMMAND_PREFIX):open_paren]
-    args = line[open_paren + 1:-1].split(",")
-    args = [a.strip() for a in args]
-    if len(args) == 1 and not args[0]:
-        # There are no args, so we will make the args list an empty list.
-        args = []
-    return command_name.strip(), args
-
-
-def _is_named_arg(token):
-    if token.startswith("${") and token.endswith("}"):
-        return True
-    else:
-        return False
-
-
-def _get_arg_name(token):
-    return token[2:-1]
-
-
-def _fatal_error(loc, msg):
-    sys.exit("ERROR:%s: %s" % (loc, msg))
-
-
-def _is_begin_command(line):
-    if line.startswith(COMMAND_PREFIX + BEGIN_COMMAND):
-        return True
-
-
-def include_file_command(out_stream, loc, args, values):
-    if len(args) != 1:
-        _fatal_error(loc, "`%%include_file` command takes exactly one "
-                     "argument. %d given." % len(args))
-    include_file_path = args[0]
-    if _is_named_arg(include_file_path):
-        arg_name = _get_arg_name(include_file_path)
-        include_file_path = values.get(arg_name)
-        if not include_file_path:
-            _fatal_error(
-                loc,
-                "No value specified for argument '%s'." % arg_name)
-        if not os.path.exists(include_file_path):
-            _fatal_error(
-                loc,
-                "Include file %s not found." % include_file_path)
-    with open(include_file_path, "r") as include_file:
-        begin = False
-        for line in include_file.readlines():
-            line = line.strip()
-            if _is_begin_command(line):
-                # Parse the command to make sure there are no errors.
-                command_name, args = _parse_command(loc, line)
-                if args:
-                    _fatal_error(loc, "Begin command does not take any args.")
-                begin = True
-                # Skip the line on which %%begin() is listed.
-                continue
-            if begin:
-                out_stream.write(line + "\n")
-
-
-def begin_command(out_stream, loc, args, values):
-    # "begin" command can only occur in a file included with %%include_file
-    # command. It is not a replacement command. Hence, we just fail with
-    # a fatal error.
-    _fatal_error(loc, "Begin command cannot be listed in an input file.")
-
-
-# Mapping from a command name to its implementation function.
-REPLACEMENT_COMMANDS = {
-    INCLUDE_FILE_COMMAND: include_file_command,
-    BEGIN_COMMAND: begin_command,
-}
-
-
-def apply_replacement_command(out_stream, loc, line, values):
-    if not line.startswith(COMMAND_PREFIX):
-        # This line is not a replacement command.
-        return line
-    command_name, args = _parse_command(loc, line)
-    command = REPLACEMENT_COMMANDS.get(command_name.strip())
-    if not command:
-        _fatal_error(loc, "Unknown replacement command `%`", command_name)
-    command(out_stream, loc, args, values)
-
-
-def parse_options():
-    parser = argparse.ArgumentParser(
-        description="Script to generate header files from .def files.")
-    parser.add_argument("def_file", metavar="DEF_FILE",
-                        help="Path to the .def file.")
-    parser.add_argument("--args", "-P", nargs= "*", default=[],
-                        help="NAME=VALUE pairs for command arguments in the "
-                             "input .def file.")
-    # The output file argument is optional. If not specified, the generated
-    # header file content will be written to stdout.
-    parser.add_argument("--out-file", "-o",
-                        help="Path to the generated header file. Defaults to "
-                             "stdout")
-    opts = parser.parse_args()
-    if not all(["=" in arg for arg in opts.args]):
-        # We want all args to be specified in the form "name=value".
-        _fatal_error(
-            __file__ + ":" + "[command line]",
-            "Command arguments should be listed in the form NAME=VALUE")
-    return opts
-
-
-def main():
-    opts = parse_options()
-    arg_values = {}
-    for name_value_pair in opts.args:
-        name, value = name_value_pair.split("=")
-        arg_values[name] = value
-    with open(opts.def_file, "r") as def_file:
-        loc = _Location(opts.def_file, 0)
-        with output_stream_manager(opts.out_file) as out_stream:
-            for line in def_file:
-                loc.line_number += 1
-                line = line.strip()
-                if line.startswith(COMMAND_PREFIX):
-                    replacement_text = apply_replacement_command(
-                        out_stream, loc, line, arg_values)
-                    out_stream.write("\n")
-                elif line.startswith(COMMENT_PREFIX):
-                    # Ignore comment line
-                    continue
-                else:
-                    out_stream.write(line + "\n")
-
-
-if __name__ == "__main__":
-    main()