Merge pull request #6 from retrage/handle-svc-imm

Add option to build syscall table

Author: retrage

.github/workflows/build.yaml

@@ -22,6 +22,7 @@ jobs:
         env:
           CC: aarch64-linux-gnu-gcc
           FULL_CONTEXT: 0
+          USE_SYSCALL_TABLE: 0
           SYSCALL_RECORD: 0
         run: |
           make clean
@@ -32,6 +33,7 @@ jobs:
         env:
           CC: aarch64-linux-gnu-gcc
           FULL_CONTEXT: 1
+          USE_SYSCALL_TABLE: 1
           SYSCALL_RECORD: 1
         run: |
           make clean

Makefile

@@ -21,6 +21,10 @@ ifeq ($(FULL_CONTEXT), 1)
 CFLAGS += -DFULL_CONTEXT
 endif
 
+ifeq ($(USE_SYSCALL_TABLE), 1)
+CFLAGS += -DUSE_SYSCALL_TABLE
+endif
+
 ifeq ($(SYSCALL_RECORD), 1)
 CFLAGS += -DSUPPLEMENTAL__SYSCALL_RECORD
 endif

main.c

@@ -46,7 +46,6 @@ static void bm_increment(size_t syscall_nr) {
 }
 #endif /* SUPPLEMENTAL__SYSCALL_RECORD */
 
-extern void syscall_addr(void);
 extern void do_rt_sigreturn(void);
 extern long enter_syscall(int64_t, int64_t, int64_t, int64_t, int64_t, int64_t,
                           int64_t, int64_t);
@@ -93,6 +92,13 @@ extern void asm_syscall_hook(void);
 #define PUSH_CONTEXT(reg, size) "sub " #reg ", " #reg ", " STR(size) " \n\t"
 #define POP_CONTEXT(reg, size) "add " #reg ", " #reg ", " STR(size) " \n\t"
 
+#ifdef USE_SYSCALL_TABLE
+void *syscall_table = NULL;
+size_t syscall_table_size = 0;
+#else
+extern void syscall_addr(void);
+#endif /* !USE_SYSCALL_TABLE */
+
 void ____asm_impl(void) {
   /*
    * enter_syscall triggers a kernel-space system call
@@ -106,6 +112,17 @@ void ____asm_impl(void) {
    * @param	a8	return address (x7)
    * @return		return value (x0)
    */
+#ifdef USE_SYSCALL_TABLE
+  asm volatile(
+      ".extern syscall_table \n\t"
+      ".globl enter_syscall \n\t"
+      "enter_syscall: \n\t"
+      "mov x8, x6 \n\t"
+      "ldr x6, =syscall_table \n\t"
+      "ldr x6, [x6] \n\t"
+      "add x6, x6, xzr, lsl #3 \n\t"
+      "br x6 \n\t");
+#else
   asm volatile(
       ".globl enter_syscall \n\t"
       "enter_syscall: \n\t"
@@ -114,6 +131,7 @@ void ____asm_impl(void) {
       "syscall_addr: \n\t"
       "svc #0 \n\t"
       "ret \n\t");
+#endif /* !USE_SYSCALL_TABLE */
 
   /*
    * asm_syscall_hook is the address where the
@@ -267,13 +285,24 @@ static inline uint32_t gen_br(uint8_t rn) {
   return insn;
 }
 
+static inline uint32_t gen_ret(void) { return 0xd65f03c0; }
+
+static inline uint32_t gen_svc(uint16_t imm) {
+  return 0xd4000001 | ((uint32_t)imm << 5);
+}
+
 static inline bool is_svc(uint32_t insn) {
   return (insn & 0xffe0000f) == 0xd4000001;
 }
 
+static inline uint16_t get_svc_imm(uint32_t insn) {
+  return (uint16_t)((insn >> 5) & 0xffff);
+}
+
 struct records_entry {
   uintptr_t *records;
-  size_t records_size;
+  uint16_t *imms;
+  size_t records_size_max;
   size_t count;
   uintptr_t reachable_range_min;
   uintptr_t reachable_range_max;
@@ -287,33 +316,54 @@ LIST_HEAD(records_head, records_entry) head;
 #define PAGE_SIZE (0x1000)
 #endif
 
+#define INITIAL_RECORDS_SIZE (PAGE_SIZE / sizeof(uintptr_t))
+
 static const size_t jump_code_size = 5;
+
+#ifdef USE_SYSCALL_TABLE
+static const size_t svc_entry_size = 2;
+static const size_t svc_gate_size = 6;
+#else
 static const size_t svc_gate_size = 5;
+#endif /* !USE_SYSCALL_TABLE */
 
 static void init_records(struct records_entry *entry) {
   assert(entry != NULL);
   entry->trampoline = NULL;
   entry->reachable_range_min = 0;
   entry->reachable_range_max = UINT64_MAX;
   entry->count = 0;
-  entry->records_size = PAGE_SIZE;
-  entry->records = malloc(entry->records_size);
+  entry->records_size_max = INITIAL_RECORDS_SIZE;
+  entry->records = malloc(entry->records_size_max * sizeof(uintptr_t));
   assert(entry->records != NULL);
+  entry->imms = malloc(entry->records_size_max * sizeof(uint16_t));
+  assert(entry->imms != NULL);
 }
 
 __attribute__((unused)) static void dump_records(struct records_entry *entry) {
   assert(entry != NULL);
   fprintf(stderr, "reachable_range: [0x%016lx-0x%016lx]\n",
           entry->reachable_range_min, entry->reachable_range_max);
   fprintf(stderr, "count: %ld\n", entry->count);
-  fprintf(stderr, "records_size: 0x%lx\n", entry->records_size);
+  fprintf(stderr, "records_size_max: 0x%lx\n", entry->records_size_max);
   for (size_t i = 0; i < entry->count; i++) {
     uintptr_t record = entry->records[i];
     fprintf(stderr, "record[%ld]: 0x%016lx %c%c%c\n", i, (record & ~0x3),
             (record & 0x2) ? 'r' : '-', (record & 0x1) ? 'w' : '-', 'x');
   }
 }
 
+static inline bool should_hook(uintptr_t addr) {
+#ifdef USE_SYSCALL_TABLE
+  return (addr != (uintptr_t)do_rt_sigreturn) &&
+         (addr < (uintptr_t)syscall_table ||
+          addr >= (uintptr_t)syscall_table + (uintptr_t)syscall_table_size);
+#else
+  return (addr != (uintptr_t)do_rt_sigreturn) &&
+         (addr != (uintptr_t)syscall_addr);
+#endif /* !USE_SYSCALL_TABLE */
+}
+
 /* find svc using pattern matching */
 static void record_svc(char *code, size_t code_size, int mem_prot) {
   /* add PROT_READ to read the code */
@@ -327,13 +377,7 @@ static void record_svc(char *code, size_t code_size, int mem_prot) {
     }
     uintptr_t addr = (uintptr_t)ptr;
     assert((addr & 0x3ULL) == 0);
-    if ((addr == (uintptr_t)syscall_addr) ||
-        (addr == (uintptr_t)do_rt_sigreturn)) {
-      /*
-       * skip the syscall replacement for
-       * our system call hook (enter_syscall)
-       * so that it can issue system calls.
-       */
+    if (!should_hook(addr)) {
       continue;
     }
 
@@ -357,11 +401,16 @@ static void record_svc(char *code, size_t code_size, int mem_prot) {
     /* Embed mem prot info in the last two bits */
     uintptr_t record = addr | (has_r ? (1 << 1) : 0) | (has_w ? (1 << 0) : 0);
     entry->records[entry->count] = record;
+    entry->imms[entry->count] = get_svc_imm(*ptr);
     entry->count += 1;
-    if (entry->count * sizeof(uintptr_t) >= entry->records_size) {
-      entry->records_size *= 2;
-      entry->records = realloc(entry->records, entry->records_size);
+    if (entry->count >= entry->records_size_max) {
+      entry->records_size_max *= 2;
+      entry->records =
+          realloc(entry->records, entry->records_size_max * sizeof(uintptr_t));
       assert(entry->records != NULL);
+      entry->imms =
+          realloc(entry->imms, entry->records_size_max * sizeof(uint16_t));
+      assert(entry->imms != NULL);
     }
 
     entry->reachable_range_min = range_min;
@@ -484,6 +533,36 @@ static void rewrite_code(void) {
   }
 }
 
+#ifdef USE_SYSCALL_TABLE
+/* Create a system call table for every svc #imm */
+/* NOTE: Although Linux does not use the #imm in svc instructions, some OSes
+ * such as NetBSD and Windows use it to store the system call number. To support
+ * such systems, we create a system call table for every svc #imm.
+ */
+static void setup_syscall_table(void) {
+  const size_t nr_svc = UINT16_MAX + 1; /* 0x10000, as #imm is 16-bit */
+  const size_t svc_table_size =
+      align_up(sizeof(uint32_t) * svc_entry_size * nr_svc, PAGE_SIZE);
+  void *svc_table = mmap(NULL, svc_table_size, PROT_READ | PROT_WRITE,
+                         MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+  assert(svc_table != MAP_FAILED);
+
+  uint32_t *code = (uint32_t *)svc_table;
+  size_t off = 0;
+  for (size_t i = 0; i < nr_svc; i++) {
+    assert(off == i * svc_entry_size);
+    code[off++] = gen_svc(i); /* svc #i */
+    code[off++] = gen_ret();  /* ret */
+    assert(off - i * svc_entry_size == svc_entry_size);
+  }
+
+  syscall_table = svc_table;
+  syscall_table_size = svc_table_size;
+
+  assert(!mprotect(svc_table, svc_table_size, PROT_EXEC));
+}
+#endif /* USE_SYSCALL_TABLE */
+
 static void setup_trampoline(void) {
   struct records_entry *entry = NULL;
 
@@ -495,7 +574,7 @@ static void setup_trampoline(void) {
     assert(range_max > 0);
     assert(range_max - range_min >= PAGE_SIZE);
 
-    assert(entry->count * sizeof(uintptr_t) <= entry->records_size);
+    assert(entry->count <= entry->records_size_max);
 
     const size_t mem_size = align_up(
         jump_code_size + svc_gate_size * sizeof(uint32_t) * entry->count,
@@ -555,6 +634,9 @@ static void setup_trampoline(void) {
       /*
        * put 'gate' code for each svc instruction
        *
+       * #ifdef USE_SYSCALL_TABLE
+       * movz x6, (#imm & 0xffff)
+       * #endif
        * movz x14, (#return_pc & 0xffff)
        * movk x14, ((#return_pc >> 16) & 0xffff), lsl 16
        * movk x14, ((#return_pc >> 32) & 0xffff), lsl 32
@@ -565,6 +647,11 @@ static void setup_trampoline(void) {
       const size_t gate_off = off;
       assert(gate_off == jump_code_size + svc_gate_size * i);
 
+#ifdef USE_SYSCALL_TABLE
+      const uint16_t imm = entry->imms[i];
+      code[off++] = gen_movz(6, (imm >> 0) & 0xffff, 0);
+#endif /* USE_SYSCALL_TABLE */
+
       const uintptr_t return_pc = (entry->records[i] & ~0x3) + sizeof(uint32_t);
       code[off++] = gen_movz(14, (return_pc >> 0) & 0xffff, 0);
       code[off++] = gen_movk(14, (return_pc >> 16) & 0xffff, 16);
@@ -627,6 +714,9 @@ __attribute__((constructor(0xffff))) static void __svc_hook_init(void) {
   bm_init();
 #endif /* SUPPLEMENTAL__SYSCALL_RECORD */
   scan_code();
+#ifdef USE_SYSCALL_TABLE
+  setup_syscall_table();
+#endif /* USE_SYSCALL_TABLE */
   setup_trampoline();
   rewrite_code();
   load_hook_lib();