[randstruct] Also randomize composite function pointer structs

[kees]

Check for struct members that are structs filled only with function pointers by recursively examining it. Since the lamba IsFunctionPointerOrForwardDecl cannot call itself directly, move it into a helper function, EntirelyFunctionPointers, so it can be called from the lambda.

Add test for composite function pointer structs getting automatically randomized.

Add more tests for validating automatic randomization vs explicitly annotated with "randomize_layout", and excluded with "no_randomize_layout".

Reorder the "should we randomize?" "if" statement to check for enablement before checking for Record details.

Fixes #138355

[llvmbot]

@llvm/pr-subscribers-clang

Author: Kees Cook (kees)

Changes

Check for struct members that are structs filled only with function pointers by recursively examining it. Since the lamba IsFunctionPointerOrForwardDecl cannot call itself directly, move it into a helper function, EntirelyFunctionPointers, so it can be called from the lambda.

Add test for composite function pointer structs getting automatically randomized.

Add more tests for validating automatic randomization vs explicitly annotated with "randomize_layout", and excluded with "no_randomize_layout".

Reorder the "should we randomize?" "if" statement to check for enablement before checking for Record details.

Fixes #138355

---

Full diff: https://github.com/llvm/llvm-project/pull/138385.diff

3 Files Affected:

• (modified) clang/include/clang/Sema/Sema.h (+2)
• (modified) clang/lib/Sema/SemaDecl.cpp (+40-31)
• (modified) clang/test/Sema/init-randomized-struct.c (+46)

diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 14f9304b99030..1de7aa7a762bb 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -6280,6 +6280,8 @@ class Sema final : public SemaBase {
   void setupImplicitSpecialMemberType(CXXMethodDecl *SpecialMem,
                                       QualType ResultTy,
                                       ArrayRef<QualType> Args);
+  // Helper for ActOnFields to check for all function pointer members.
+  bool EntirelyFunctionPointers(const RecordDecl *Record);
 
   // A cache representing if we've fully checked the various comparison category
   // types stored in ASTContext. The bit-index corresponds to the integer value
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index a3285e8f6f5a2..39eeb900c782f 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -19270,6 +19270,43 @@ static void ComputeSpecialMemberFunctionsEligiblity(Sema &S,
                      CXXSpecialMemberKind::MoveAssignment);
 }
 
+bool Sema::EntirelyFunctionPointers(const RecordDecl *Record) {
+  // Check to see if a FieldDecl is a pointer to a function.
+  auto IsFunctionPointerOrForwardDecl = [&](const Decl *D) {
+    const FieldDecl *FD = dyn_cast<FieldDecl>(D);
+    if (!FD) {
+      // Check whether this is a forward declaration that was inserted by
+      // Clang. This happens when a non-forward declared / defined type is
+      // used, e.g.:
+      //
+      //   struct foo {
+      //     struct bar *(*f)();
+      //     struct bar *(*g)();
+      //   };
+      //
+      // "struct bar" shows up in the decl AST as a "RecordDecl" with an
+      // incomplete definition.
+      if (const auto *TD = dyn_cast<TagDecl>(D))
+        return !TD->isCompleteDefinition();
+      return false;
+    }
+    QualType FieldType = FD->getType().getDesugaredType(Context);
+    if (isa<PointerType>(FieldType)) {
+      QualType PointeeType = cast<PointerType>(FieldType)->getPointeeType();
+      return PointeeType.getDesugaredType(Context)->isFunctionType();
+    }
+    // If a member is a struct entirely of function pointers, that counts too.
+    if (const RecordType *RT = FieldType->getAs<RecordType>()) {
+      const RecordDecl *Record = RT->getDecl();
+      if (Record->isStruct() && EntirelyFunctionPointers(Record))
+        return true;
+    }
+    return false;
+  };
+
+  return llvm::all_of(Record->decls(), IsFunctionPointerOrForwardDecl);
+}
+
 void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl,
                        ArrayRef<Decl *> Fields, SourceLocation LBrac,
                        SourceLocation RBrac,
@@ -19607,41 +19644,13 @@ void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl,
     // Handle attributes before checking the layout.
     ProcessDeclAttributeList(S, Record, Attrs);
 
-    // Check to see if a FieldDecl is a pointer to a function.
-    auto IsFunctionPointerOrForwardDecl = [&](const Decl *D) {
-      const FieldDecl *FD = dyn_cast<FieldDecl>(D);
-      if (!FD) {
-        // Check whether this is a forward declaration that was inserted by
-        // Clang. This happens when a non-forward declared / defined type is
-        // used, e.g.:
-        //
-        //   struct foo {
-        //     struct bar *(*f)();
-        //     struct bar *(*g)();
-        //   };
-        //
-        // "struct bar" shows up in the decl AST as a "RecordDecl" with an
-        // incomplete definition.
-        if (const auto *TD = dyn_cast<TagDecl>(D))
-          return !TD->isCompleteDefinition();
-        return false;
-      }
-      QualType FieldType = FD->getType().getDesugaredType(Context);
-      if (isa<PointerType>(FieldType)) {
-        QualType PointeeType = cast<PointerType>(FieldType)->getPointeeType();
-        return PointeeType.getDesugaredType(Context)->isFunctionType();
-      }
-      return false;
-    };
-
     // Maybe randomize the record's decls. We automatically randomize a record
     // of function pointers, unless it has the "no_randomize_layout" attribute.
-    if (!getLangOpts().CPlusPlus &&
+    if (!getLangOpts().CPlusPlus && !getLangOpts().RandstructSeed.empty() &&
+        !Record->isRandomized() && !Record->isUnion() &&
         (Record->hasAttr<RandomizeLayoutAttr>() ||
          (!Record->hasAttr<NoRandomizeLayoutAttr>() &&
-          llvm::all_of(Record->decls(), IsFunctionPointerOrForwardDecl))) &&
-        !Record->isUnion() && !getLangOpts().RandstructSeed.empty() &&
-        !Record->isRandomized()) {
+          EntirelyFunctionPointers(Record)))) {
       SmallVector<Decl *, 32> NewDeclOrdering;
       if (randstruct::randomizeStructureLayout(Context, Record,
                                                NewDeclOrdering))
diff --git a/clang/test/Sema/init-randomized-struct.c b/clang/test/Sema/init-randomized-struct.c
index d421597fa522f..8adc91ac58de7 100644
--- a/clang/test/Sema/init-randomized-struct.c
+++ b/clang/test/Sema/init-randomized-struct.c
@@ -75,3 +75,49 @@ struct enum_decl_test {
 } __attribute__((randomize_layout));
 
 struct enum_decl_test t13 = { BORK }; // Okay
+
+struct mixed {
+  int a;
+  short b;
+  unsigned c;
+  char d;
+} __attribute__((randomize_layout));
+
+struct mixed t14 = { 7 }; // expected-error {{a randomized struct can only be initialized with a designated initializer}}
+struct mixed t15 = { .b = 8 }; // Okay
+
+// This should be autodetected as randomized.
+struct funcs {
+  func_ptr a;
+  func_ptr b;
+  func_ptr c;
+  func_ptr d;
+};
+
+struct funcs t16 = { .c = foo }; // Okay
+struct funcs t17 = { foo }; // expected-error {{a randomized struct can only be initialized with a designated initializer}}
+
+// This should be forced off.
+struct funcs_unshuffled {
+  func_ptr a;
+  func_ptr b;
+  func_ptr c;
+  func_ptr d;
+} __attribute__((no_randomize_layout));
+
+struct funcs_unshuffled t18 = { .d = foo }; // Okay
+struct funcs_unshuffled t19 = { foo }; // Okay
+
+// This is still all function pointers.
+// https://github.com/llvm/llvm-project/issues/138355
+struct funcs_composite {
+  func_ptr a;
+  func_ptr b;
+  struct funcs inner;
+  func_ptr c;
+  func_ptr d;
+};
+
+struct funcs_composite t20 = { .a = foo }; // Okay
+struct funcs_composite t21 = { .inner.c = foo }; // Okay
+struct funcs_composite t22 = { foo }; // expected-error {{a randomized struct can only be initialized with a designated initializer}}

[bwendling]

Looks fine. Though I can't shake the feeling that randomization should happen in the ActOnTagFinishDefinition method.