x86_64: add 64-bit syscall entry point

While theoretically we could still use the same call gate as for
32-bit userspace, it doesn't seem very common, and gcc seems to not
encode properly the instruction. Instead we use syscall/sysret as
other kernels (e.g. XNU,Linux). This version still has some
limitations, but should be enough to start working on the 64-bit user
space.

* i386/i386/i386asm.sym: add more constants to fill pcb->iss
* i386/i386/ldt.c: configure 64-bit syscall entry point. We can just
  check for the SEP bit as MSR are always available on x86_64.
* i386/i386/ldt.h: swap CS/DS segments order if !USER32 as required by
  sysret
* i386/i386/locore.h: add syscall64 prototype
* i386/i386/msr.h: add MSR definitions and C read/write helpers
* i386/include/mach/i386/syscall_sw.h: remove old BSD_TRAP
* x86_64/Makefrag.am: selectively install syscall_sw.h depending on
  USER32
* x86_64/include/syscall_sw.h: add entry point template from user
  space
* x86_64/locore.S: implement syscall64 entry point and use it when a
  64-bit user-space is configured
Message-Id: <20230419194703.410575-4-luca@orpolo.org>

6 files changed, 95 insertions, 14 deletions

diff --git a/i386/i386/i386asm.sym b/i386/i386/i386asm.sym
index 8317db6c..1b9b40bb 100644
--- a/i386/i386/i386asm.sym
+++ b/i386/i386/i386asm.sym
@@ -52,6 +52,8 @@ expr	CALL_SINGLE_FUNCTION_BASE
 
 offset	ApicLocalUnit		lu	apic_id		APIC_ID
 
+offset	pcb			pcb	iss
+
 offset	thread			th	pcb
 offset	thread			th	task
 offset	thread			th	recover
@@ -82,16 +84,29 @@ size	i386_kernel_state	iks
 
 size	i386_exception_link	iel
 
+offset	i386_saved_state	r	gs
+offset	i386_saved_state	r	fs
 offset	i386_saved_state	r	cs
 offset	i386_saved_state	r	uesp
 offset	i386_saved_state	r	eax
+offset	i386_saved_state	r	ebx
+offset	i386_saved_state	r	ecx
+offset	i386_saved_state	r	edx
+offset	i386_saved_state	r	ebp
 offset	i386_saved_state	r	trapno
 offset	i386_saved_state	r	err
 offset	i386_saved_state	r	efl		R_EFLAGS
 offset	i386_saved_state	r	eip
 offset	i386_saved_state	r	cr2
 offset	i386_saved_state	r	edi
+offset	i386_saved_state	r	esi
 #ifdef __x86_64__
+offset	i386_saved_state	r	r8
+offset	i386_saved_state	r	r9
+offset	i386_saved_state	r	r10
+offset	i386_saved_state	r	r12
+offset	i386_saved_state	r	r13
+offset	i386_saved_state	r	r14
 offset	i386_saved_state	r	r15
 #endif
 
diff --git a/i386/i386/ldt.c b/i386/i386/ldt.c
index b86a0e3c..4d7ec19a 100644
--- a/i386/i386/ldt.c
+++ b/i386/i386/ldt.c
@@ -31,6 +31,7 @@
 #include <mach/xen.h>
 
 #include <intel/pmap.h>
+#include <kern/debug.h>
 
 #include "vm_param.h"
 #include "seg.h"
@@ -38,6 +39,7 @@
 #include "ldt.h"
 #include "locore.h"
 #include "mp_desc.h"
+#include "msr.h"
 
 #ifdef	MACH_PV_DESCRIPTORS
 /* It is actually defined in xen_boothdr.S */
@@ -65,10 +67,22 @@ ldt_fill(struct real_descriptor *myldt, struct real_descriptor *mygdt)
 			        ACC_PL_K|ACC_LDT, 0);
 #endif	/* MACH_PV_DESCRIPTORS */
 
-	/* Initialize the 32bit LDT descriptors.  */
+	/* Initialize the syscall entry point */
+#if defined(__x86_64__) && ! defined(USER32)
+        if (!CPU_HAS_FEATURE(CPU_FEATURE_SEP))
+            panic("syscall support is missing on 64 bit");
+        /* Enable 64-bit syscalls */
+        wrmsr(MSR_REG_EFER, rdmsr(MSR_REG_EFER) | MSR_EFER_SCE);
+        wrmsr(MSR_REG_LSTAR, (vm_offset_t)syscall64);
+        wrmsr(MSR_REG_STAR, ((((long)USER_CS - 16) << 16) | (long)KERNEL_CS) << 32);
+        wrmsr(MSR_REG_FMASK, 0);  // ?
+#else /* defined(__x86_64__) && ! defined(USER32) */
 	fill_ldt_gate(myldt, USER_SCALL,
 		      (vm_offset_t)&syscall, KERNEL_CS,
 		      ACC_PL_U|ACC_CALL_GATE, 0);
+#endif /* defined(__x86_64__) && ! defined(USER32) */
+
+	/* Initialize the 32bit LDT descriptors.  */
 	fill_ldt_descriptor(myldt, USER_CS,
 			    VM_MIN_USER_ADDRESS,
 			    VM_MAX_USER_ADDRESS-VM_MIN_USER_ADDRESS-4096,
diff --git a/i386/i386/ldt.h b/i386/i386/ldt.h
index b15f11a5..51867f47 100644
--- a/i386/i386/ldt.h
+++ b/i386/i386/ldt.h
@@ -43,11 +43,16 @@
  * User descriptors for Mach - 32-bit flat address space
  */
 #define	USER_SCALL	0x07		/* system call gate */
-#ifdef __x86_64__
+#if defined(__x86_64__) && ! defined(USER32)
 /* Call gate needs two entries */
-#endif
+
+/* The sysret instruction puts some constraints on the user segment indexes */
+#define	USER_CS		0x1f		/* user code segment */
+#define	USER_DS		0x17		/* user data segment */
+#else
 #define	USER_CS		0x17		/* user code segment */
 #define	USER_DS		0x1f		/* user data segment */
+#endif
 
 #define	LDTSZ		4
 
diff --git a/i386/i386/locore.h b/i386/i386/locore.h
index a8807dbf..4388ea28 100644
--- a/i386/i386/locore.h
+++ b/i386/i386/locore.h
@@ -57,6 +57,7 @@ extern int inst_fetch (int eip, int cs);
 extern void cpu_shutdown (void);
 
 extern int syscall (void);
+extern int syscall64 (void);
 
 extern unsigned int cpu_features[2];
 
diff --git a/i386/i386/msr.h b/i386/i386/msr.h
new file mode 100644
index 00000000..8f09b80b
--- /dev/null
+++ b/i386/i386/msr.h
@@ -0,0 +1,56 @@
+/*
+ *  Copyright (C) 2023 Free Software Foundation
+ *
+ * This program is free software ; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation ; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY ; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with the program ; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#ifndef _MACHINE_MSR_H_
+#define _MACHINE_MSR_H_
+
+#define MSR_REG_EFER  0xC0000080
+#define MSR_REG_STAR  0xC0000081
+#define MSR_REG_LSTAR 0xC0000082
+#define MSR_REG_CSTAR 0xC0000083
+#define MSR_REG_FMASK 0xC0000084
+#define MSR_REG_FSBASE 0xC0000100
+#define MSR_REG_GSBASE 0xC0000101
+
+#define MSR_EFER_SCE  0x00000001
+
+#ifndef __ASSEMBLER__
+
+static inline void wrmsr(uint32_t regaddr, uint64_t value)
+{
+  uint32_t low = (uint32_t) value, high = ((uint32_t) (value >> 32));
+  asm volatile("wrmsr"
+               :
+               : "c" (regaddr), "a" (low), "d" (high)
+               : "memory"  /* wrmsr may cause a read from memory, so
+                            * make the compiler flush any changes */
+               );
+}
+
+static inline uint64_t rdmsr(uint32_t regaddr)
+{
+  uint32_t low, high;
+  asm volatile("rdmsr"
+               : "=a" (low), "=d" (high)
+               : "c" (regaddr)
+               );
+  return ((uint64_t)high << 32) | low;
+}
+#endif /* __ASSEMBLER__ */
+
+#endif /* _MACHINE_MSR_H_ */
diff --git a/i386/include/mach/i386/syscall_sw.h b/i386/include/mach/i386/syscall_sw.h
index 86f6ff2f..9eeb2939 100644
--- a/i386/include/mach/i386/syscall_sw.h
+++ b/i386/include/mach/i386/syscall_sw.h
@@ -29,21 +29,11 @@
 
 #include <mach/machine/asm.h>
 
-#if BSD_TRAP
-#define kernel_trap(trap_name,trap_number,number_args) \
+#define kernel_trap(trap_name,trap_number,number_args)  \
 ENTRY(trap_name) \
 	movl	$ trap_number,%eax; \
 	SVC; \
-	jb LCL(cerror); \
 	ret; \
 END(trap_name)
-#else
-#define kernel_trap(trap_name,trap_number,number_args) \
-ENTRY(trap_name) \
-	movl	$ trap_number,%eax; \
-	SVC; \
-	ret; \
-END(trap_name)
-#endif
 
 #endif	/* _MACH_I386_SYSCALL_SW_H_ */