Add static branch prediction to type checks

Annotate generated type checks with static branch prediction
optimizing well-formed messages.

* utils.c (WriteBogusDefines): Define `mig_unlikely' if not defined.
* server.c: Use `mig_unlikely' in generated type checks.
* user.c: Likewise.

3 files changed, 34 insertions, 23 deletions

diff --git a/server.c b/server.c
index bcdeb0b..56ccfc7 100644
--- a/server.c
+++ b/server.c
@@ -402,16 +402,16 @@ WriteCheckHead(FILE *file, const routine_t *rt)
 	fprintf(file, "\tmsgh_simple = !(In0P->Head.msgh_bits & MACH_MSGH_BITS_COMPLEX);\n");
 
     if (rt->rtNumRequestVar > 0)
-	fprintf(file, "\tif ((msgh_size < %d)",
+	fprintf(file, "\tif (mig_unlikely ((msgh_size < %d)",
 		rt->rtRequestSize);
     else
-	fprintf(file, "\tif ((In0P->Head.msgh_size != %d)",
+	fprintf(file, "\tif (mig_unlikely ((In0P->Head.msgh_size != %d)",
 		rt->rtRequestSize);
 
     if (rt->rtSimpleCheckRequest)
 	fprintf(file, " ||\n\t    %s(In0P->Head.msgh_bits & MACH_MSGH_BITS_COMPLEX)",
 		rt->rtSimpleReceiveRequest ? "" : "!");
-    fprintf(file, ")\n");
+    fprintf(file, "))\n");
     WriteMsgError(file, "MIG_BAD_ARGUMENTS");
     fprintf(file, "#endif\t/* TypeCheck */\n");
     fprintf(file, "\n");
@@ -429,7 +429,7 @@ WriteTypeCheck(FILE *file, const argument_t *arg)
 		arg->argRequestPos, arg->argTTName, arg->argVarName);
     else
     {
-	fprintf(file, "\tif (");
+	fprintf(file, "\tif (mig_unlikely (");
 	if (!it->itIndefinite) {
 	    fprintf(file, "(In%dP->%s%s.msgt_inline != %s) ||\n\t    ",
 		arg->argRequestPos, arg->argTTName,
@@ -457,7 +457,7 @@ WriteTypeCheck(FILE *file, const argument_t *arg)
 		    arg->argRequestPos, arg->argTTName,
 		    arg->argLongForm ? "l" : "",
 		    it->itNumber);
-	fprintf(file, "\t    (In%dP->%s.msgt%s_size != %d))\n",
+	fprintf(file, "\t    (In%dP->%s.msgt%s_size != %d)))\n",
 		arg->argRequestPos, arg->argTTName,
 		arg->argLongForm ? "l" : "",
 		it->itSize);
@@ -515,9 +515,10 @@ WriteCheckMsgSize(FILE *file, const argument_t *arg)
     if (arg->argRequestPos == rt->rtMaxRequestPos)
     {
 	fprintf(file, "#if\tTypeCheck\n");
-	fprintf(file, "\tif (msgh_size != %d + (", rt->rtRequestSize);
+	fprintf(file, "\tif (mig_unlikely (msgh_size != %d + (",
+		rt->rtRequestSize);
 	WriteCheckArgSize(file, arg);
-	fprintf(file, "))\n");
+	fprintf(file, ")))\n");
 
 	WriteMsgError(file, "MIG_BAD_ARGUMENTS");
 	fprintf(file, "#endif\t/* TypeCheck */\n");
@@ -545,10 +546,12 @@ WriteCheckMsgSize(FILE *file, const argument_t *arg)
 	   it won't underflow. */
 
 	if (LastVarArg)
-	    fprintf(file, "\tif (msgh_size != %d + msgh_size_delta)\n",
+	    fprintf(file,
+		    "\tif (mig_unlikely (msgh_size != %d + msgh_size_delta))\n",
 		rt->rtRequestSize);
 	else
-	    fprintf(file, "\tif (msgh_size < %d + msgh_size_delta)\n",
+	    fprintf(file,
+		    "\tif (mig_unlikely (msgh_size < %d + msgh_size_delta))\n",
 		rt->rtRequestSize);
 	WriteMsgError(file, "MIG_BAD_ARGUMENTS");
 
diff --git a/user.c b/user.c
index f4a6cd5..5451a03 100644
--- a/user.c
+++ b/user.c
@@ -686,7 +686,7 @@ WriteRequestArgs(FILE *file, const routine_t *rt)
 static void
 WriteCheckIdentity(FILE *file, const routine_t *rt)
 {
-    fprintf(file, "\tif (OutP->Head.msgh_id != %d) {\n",
+    fprintf(file, "\tif (mig_unlikely (OutP->Head.msgh_id != %d)) {\n",
 	    rt->rtNumber + SubsystemBase + 100);
     fprintf(file, "\t\tif (OutP->Head.msgh_id == MACH_NOTIFY_SEND_ONCE)\n\t");
     WriteMsgError(file, rt, "MIG_SERVER_DIED");
@@ -708,16 +708,17 @@ WriteCheckIdentity(FILE *file, const routine_t *rt)
 	    fprintf(file, "\tmsgh_size = OutP->Head.msgh_size;\n\n");
 
 	fprintf(file,
-	    "\tif ((OutP->Head.msgh_bits & MACH_MSGH_BITS_COMPLEX) ||\n");
+	    "\tif (mig_unlikely ("
+	    "(OutP->Head.msgh_bits & MACH_MSGH_BITS_COMPLEX) ||\n");
 	if (rt->rtNoReplyArgs)
-	    fprintf(file, "\t    (OutP->Head.msgh_size != %d))\n",
+	    fprintf(file, "\t    (OutP->Head.msgh_size != %d)))\n",
 			rt->rtReplySize);
 	else {
 	    fprintf(file, "\t    ((msgh_size %s %d) &&\n",
 		(rt->rtNumReplyVar > 0) ? "<" : "!=",
 		rt->rtReplySize);
 	    fprintf(file, "\t     ((msgh_size != sizeof(mig_reply_header_t)) ||\n");
-	    fprintf(file, "\t      (OutP->RetCode == KERN_SUCCESS))))\n");
+	    fprintf(file, "\t      (OutP->RetCode == KERN_SUCCESS)))))\n");
 	}
     }
     else {
@@ -727,7 +728,7 @@ WriteCheckIdentity(FILE *file, const routine_t *rt)
 	fprintf(file, "\tmsgh_simple = !(OutP->Head.msgh_bits & MACH_MSGH_BITS_COMPLEX);\n");
 	fprintf(file, "\n");
 
-	fprintf(file, "\tif (((msgh_size %s %d)",
+	fprintf(file, "\tif (mig_unlikely (((msgh_size %s %d)",
 		(rt->rtNumReplyVar > 0) ? "<" : "!=",
 		rt->rtReplySize);
 
@@ -740,7 +741,7 @@ WriteCheckIdentity(FILE *file, const routine_t *rt)
 
 	fprintf(file, "\t    ((msgh_size != sizeof(mig_reply_header_t)) ||\n");
 	fprintf(file, "\t     !msgh_simple ||\n");
-	fprintf(file, "\t     (OutP->RetCode == KERN_SUCCESS)))\n");
+	fprintf(file, "\t     (OutP->RetCode == KERN_SUCCESS))))\n");
     }
     WriteMsgError(file, rt, "MIG_TYPE_ERROR");
     fprintf(file, "#endif\t/* TypeCheck */\n");
@@ -778,7 +779,7 @@ WriteTypeCheck(FILE *file, const argument_t *arg)
     }
     else
     {
-	fprintf(file, "\tif (");
+	fprintf(file, "\tif (mig_unlikely (");
 	if (!it->itIndefinite) {
 	    fprintf(file, "(OutP->%s%s.msgt_inline != %s) ||\n\t    ",
 		arg->argTTName,
@@ -806,7 +807,7 @@ WriteTypeCheck(FILE *file, const argument_t *arg)
 		    arg->argTTName,
 		    arg->argLongForm ? "l" : "",
 		    it->itNumber);
-	fprintf(file, "\t    (OutP->%s.msgt%s_size != %d))\n",
+	fprintf(file, "\t    (OutP->%s.msgt%s_size != %d)))\n",
 		arg->argTTName,
 		arg->argLongForm ? "l" : "",
 		it->itSize);
@@ -861,10 +862,10 @@ WriteCheckMsgSize(FILE *file, const argument_t *arg)
     if (arg->argReplyPos == rt->rtMaxReplyPos)
     {
 	fprintf(file, "#if\tTypeCheck\n");
-	fprintf(file, "\tif (msgh_size != %d + (",
+	fprintf(file, "\tif (mig_unlikely (msgh_size != %d + (",
 		rt->rtReplySize);
 	WriteCheckArgSize(file, arg);
-	fprintf(file, "))\n");
+	fprintf(file, ")))\n");
 
 	WriteMsgError(file, rt, "MIG_TYPE_ERROR");
 	fprintf(file, "#endif\t/* TypeCheck */\n");
@@ -892,10 +893,12 @@ WriteCheckMsgSize(FILE *file, const argument_t *arg)
 	   it won't underflow. */
 
 	if (LastVarArg)
-	    fprintf(file, "\tif (msgh_size != %d + msgh_size_delta)\n",
+	    fprintf(file,
+		"\tif (mig_unlikely (msgh_size != %d + msgh_size_delta))\n",
 		rt->rtReplySize);
 	else
-	    fprintf(file, "\tif (msgh_size < %d + msgh_size_delta)\n",
+	    fprintf(file,
+		"\tif (mig_unlikely (msgh_size < %d + msgh_size_delta))\n",
 		rt->rtReplySize);
 	WriteMsgError(file, rt, "MIG_TYPE_ERROR");
 
diff --git a/utils.c b/utils.c
index 6c42bc3..396e743 100644
--- a/utils.c
+++ b/utils.c
@@ -60,6 +60,11 @@ WriteBogusDefines(FILE *file)
     fprintf(file, "#endif\n");
     fprintf(file, "\n");
 
+    fprintf(file, "#ifndef\tmig_unlikely\n");
+    fprintf(file, "#define\tmig_unlikely(X)\t__builtin_expect (!! (X), 0)\n");
+    fprintf(file, "#endif\n");
+    fprintf(file, "\n");
+
     fprintf(file, "#ifndef\tTypeCheck\n");
     fprintf(file, "#define\tTypeCheck 1\n");
     fprintf(file, "#endif\n");
@@ -70,11 +75,11 @@ WriteBogusDefines(FILE *file)
     fprintf(file, "#endif\n");
     fprintf(file, "\n");
 
-    fprintf(file, "#define BAD_TYPECHECK(type, check) ({\\\n");
+    fprintf(file, "#define BAD_TYPECHECK(type, check) mig_unlikely (({\\\n");
     fprintf(file,
 	    "  union { mach_msg_type_t t; unsigned32_t w; } _t, _c;\\\n");
     fprintf(file,
-	    "  _t.t = *(type); _c.t = *(check); _t.w != _c.w; })\n");
+	    "  _t.t = *(type); _c.t = *(check);_t.w != _c.w; }))\n");
 }
 
 void