diff --git a/src/hooks/syscalls_hc.c b/src/hooks/syscalls_hc.c index da29f0e..087e4d9 100644 --- a/src/hooks/syscalls_hc.c +++ b/src/hooks/syscalls_hc.c @@ -24,6 +24,8 @@ /* [ADD] Global counter to allow early exit if no hooks exist */ atomic_t global_syscall_hook_count = ATOMIC_INIT(0); +atomic_t global_syscall_enter_hook_count = ATOMIC_INIT(0); +atomic_t global_syscall_return_hook_count = ATOMIC_INIT(0); /* Global hash tables for organizing hooks */ struct hlist_head syscall_hook_table[1024]; /* Main table indexed by hook pointer */ @@ -49,6 +51,102 @@ EXPORT_SYMBOL(syscall_hook_lock); // Replace mutex with spinlock which is safe for atomic contexts DEFINE_SPINLOCK(syscall_hc_lock); // Keep commented out unless hypercall needs external locking +struct syscall_name_cache_entry { + const char *raw_name; + const char *normalized_name; + u32 normalized_hash; + u16 normalized_len; + bool is_sendto; + struct hlist_node hlist; +}; + +struct syscall_name_info { + const char *normalized_name; + u32 normalized_hash; + u16 normalized_len; + bool is_sendto; +}; + +#define SYSCALL_NAME_PTR_CACHE_BITS 8 +#define SYSCALL_NAME_PTR_CACHE_MAX 1024 + +static DEFINE_HASHTABLE(syscall_name_ptr_cache, SYSCALL_NAME_PTR_CACHE_BITS); +static DEFINE_SPINLOCK(syscall_name_ptr_cache_lock); +static atomic_t syscall_name_ptr_cache_count = ATOMIC_INIT(0); + +static inline struct syscall_name_info get_syscall_name_info(const char *syscall_name) +{ + struct syscall_name_cache_entry *entry; + struct syscall_name_cache_entry *existing; + struct syscall_name_info info = { + .normalized_name = NULL, + .normalized_hash = 0, + .normalized_len = 0, + .is_sendto = false, + }; + const char *normalized_name; + unsigned long key; + + if (!syscall_name) { + return info; + } + + key = (unsigned long)syscall_name; + + rcu_read_lock(); + hash_for_each_possible_rcu(syscall_name_ptr_cache, entry, hlist, key) { + if (entry->raw_name == syscall_name) { + info.normalized_name = entry->normalized_name; + info.normalized_hash = entry->normalized_hash; + info.normalized_len = entry->normalized_len; + info.is_sendto = entry->is_sendto; + rcu_read_unlock(); + return info; + } + } + rcu_read_unlock(); + + normalized_name = normalize_syscall_name(syscall_name); + info.normalized_name = normalized_name; + info.normalized_hash = syscall_normalized_name_hash(normalized_name); + info.normalized_len = normalized_name ? strlen(normalized_name) : 0; + info.is_sendto = normalized_name && strcmp(normalized_name, "sendto") == 0; + + if (atomic_read(&syscall_name_ptr_cache_count) >= SYSCALL_NAME_PTR_CACHE_MAX) { + return info; + } + + entry = kzalloc(sizeof(*entry), GFP_ATOMIC); + if (!entry) { + return info; + } + + entry->raw_name = syscall_name; + entry->normalized_name = info.normalized_name; + entry->normalized_hash = info.normalized_hash; + entry->normalized_len = info.normalized_len; + entry->is_sendto = info.is_sendto; + + spin_lock(&syscall_name_ptr_cache_lock); + hash_for_each_possible(syscall_name_ptr_cache, existing, hlist, key) { + if (existing->raw_name == syscall_name) { + spin_unlock(&syscall_name_ptr_cache_lock); + kfree(entry); + return info; + } + } + if (atomic_read(&syscall_name_ptr_cache_count) >= SYSCALL_NAME_PTR_CACHE_MAX) { + spin_unlock(&syscall_name_ptr_cache_lock); + kfree(entry); + return info; + } + hash_add_rcu(syscall_name_ptr_cache, &entry->hlist, key); + atomic_inc(&syscall_name_ptr_cache_count); + spin_unlock(&syscall_name_ptr_cache_lock); + + return info; +} + /* Function to print syscall information */ void print_syscall_info(const struct syscall_event *sc, const char *prefix); void print_syscall_info(const struct syscall_event *sc, const char *prefix) { @@ -187,7 +285,9 @@ static inline bool value_matches_filter(long value, const struct value_filter *f } // Change hook_matches_syscall and hook_matches_syscall_return to take struct kernel_syscall_hook * -static inline bool hook_matches_syscall(struct kernel_syscall_hook *hook, const char *syscall_name, +static inline bool hook_matches_syscall(struct kernel_syscall_hook *hook, + const char *syscall_name, u32 syscall_name_hash, + u16 syscall_name_len, int argc, const unsigned long args[]) { if (!hook->hook.enabled){ @@ -197,7 +297,13 @@ static inline bool hook_matches_syscall(struct kernel_syscall_hook *hook, const return true; } if (syscall_name && hook->hook.name[0] != '\0') { - if (strcmp(hook->normalized_name, syscall_name) != 0) { + if (hook->normalized_name_hash != syscall_name_hash) { + return false; + } + if (hook->normalized_name_len != syscall_name_len) { + return false; + } + if (unlikely(strcmp(hook->normalized_name, syscall_name) != 0)) { return false; } } else if (hook->hook.name[0] != '\0') { @@ -284,10 +390,11 @@ static inline bool hook_matches_syscall(struct kernel_syscall_hook *hook, const } static inline bool hook_matches_syscall_return(struct kernel_syscall_hook *hook, const char *syscall_name, - int argc, const unsigned long args[], long retval) + u32 syscall_name_hash, u16 syscall_name_len, int argc, + const unsigned long args[], long retval) { struct value_filter *f; - if (!hook_matches_syscall(hook, syscall_name, argc, args)){ + if (!hook_matches_syscall(hook, syscall_name, syscall_name_hash, syscall_name_len, argc, args)){ return false; } f = &hook->hook.retval_filter; @@ -319,6 +426,7 @@ static inline void capture_syscall_event(struct syscall_event *dest, static long process_syscall_hooks( bool is_entry, const char *syscall_name, + const struct syscall_name_info *cached_name_info, int argc, const unsigned long args[], igloo_syscall_setter_t setter_func, @@ -337,22 +445,31 @@ static long process_syscall_hooks( long modified_ret = orig_ret; long skip_ret_val_local = 0; bool skip_syscall = false; + struct syscall_name_info name_info; const char *normsc; u32 name_hash; + u16 name_len; int i; - if (atomic_read(&global_syscall_hook_count) == 0) { + if (atomic_read(is_entry ? &global_syscall_enter_hook_count : + &global_syscall_return_hook_count) == 0) { return is_entry ? 0 : orig_ret; } - normsc = normalize_syscall_name(syscall_name); - name_hash = syscall_normalized_name_hash(normsc); + if (cached_name_info) { + name_info = *cached_name_info; + } else { + name_info = get_syscall_name_info(syscall_name); + } + normsc = name_info.normalized_name; + name_hash = name_info.normalized_hash; + name_len = name_info.normalized_len; rcu_read_lock(); // 1. Check the "match all syscalls" list first if (!hlist_empty(&syscall_all_hooks)) { hlist_for_each_entry_rcu(hook, &syscall_all_hooks, name_hlist) { bool matches = is_entry ? - (hook->hook.on_enter && hook_matches_syscall(hook, normsc, argc, args)) : - (hook->hook.on_return && hook_matches_syscall_return(hook, normsc, argc, args, modified_ret)); + (hook->hook.on_enter && hook_matches_syscall(hook, normsc, name_hash, name_len, argc, args)) : + (hook->hook.on_return && hook_matches_syscall_return(hook, normsc, name_hash, name_len, argc, args, modified_ret)); if (matches) { if (match_count < IGLOO_STACK_BATCH_SIZE) { if (unlikely(!template_initialized)) { @@ -370,8 +487,8 @@ static long process_syscall_hooks( if (normsc) { hash_for_each_possible_rcu(syscall_name_table, hook, name_hlist, name_hash) { bool matches = is_entry ? - (hook->hook.on_enter && hook_matches_syscall(hook, normsc, argc, args)) : - (hook->hook.on_return && hook_matches_syscall_return(hook, normsc, argc, args, modified_ret)); + (hook->hook.on_enter && hook_matches_syscall(hook, normsc, name_hash, name_len, argc, args)) : + (hook->hook.on_return && hook_matches_syscall_return(hook, normsc, name_hash, name_len, argc, args, modified_ret)); if (matches) { if (match_count < IGLOO_STACK_BATCH_SIZE) { if (unlikely(!template_initialized)) { @@ -412,8 +529,8 @@ static long process_syscall_hooks( if (!hlist_empty(&syscall_all_hooks)) { hlist_for_each_entry_rcu(hook, &syscall_all_hooks, name_hlist) { bool matches = is_entry ? - (hook->hook.on_enter && hook_matches_syscall(hook, syscall_name, argc, args)) : - (hook->hook.on_return && hook_matches_syscall_return(hook, syscall_name, argc, args, modified_ret)); + (hook->hook.on_enter && hook_matches_syscall(hook, normsc, name_hash, name_len, argc, args)) : + (hook->hook.on_return && hook_matches_syscall_return(hook, normsc, name_hash, name_len, argc, args, modified_ret)); if (matches && match_count < capacity) { capture_syscall_event(&batch[match_count], hook, &template_event, is_entry, modified_ret); @@ -423,11 +540,11 @@ static long process_syscall_hooks( } // Re-scan named list - if (syscall_name) { + if (normsc) { hash_for_each_possible_rcu(syscall_name_table, hook, name_hlist, name_hash) { bool matches = is_entry ? - (hook->hook.on_enter && hook_matches_syscall(hook, normsc, argc, args)) : - (hook->hook.on_return && hook_matches_syscall_return(hook, normsc, argc, args, modified_ret)); + (hook->hook.on_enter && hook_matches_syscall(hook, normsc, name_hash, name_len, argc, args)) : + (hook->hook.on_return && hook_matches_syscall_return(hook, normsc, name_hash, name_len, argc, args, modified_ret)); if (matches && match_count < capacity) { capture_syscall_event(&batch[match_count], hook, &template_event, is_entry, modified_ret); @@ -494,7 +611,8 @@ static long process_syscall_hooks( static bool syscall_entry_handler(const char *syscall_name, long *skip_ret_val, int argc, const unsigned long args[], igloo_syscall_setter_t setter_func) { int i; - unsigned long safe_args[IGLOO_SYSCALL_MAXARGS] = {0}; + unsigned long safe_args[IGLOO_SYSCALL_MAXARGS]; + struct syscall_name_info name_info; check_portal_interrupt(); if (!args || !skip_ret_val) { return 0; @@ -502,10 +620,19 @@ static bool syscall_entry_handler(const char *syscall_name, long *skip_ret_val, if (current->flags & PF_KTHREAD) { return 0; } + if (atomic_read(&global_syscall_enter_hook_count) == 0 && + !portalcall_fastpath_is_enabled()) { + return 0; + } for (i = 0; i < IGLOO_SYSCALL_MAXARGS && i < argc; i++) { safe_args[i] = args[i]; } - return process_syscall_hooks(true, syscall_name, argc, safe_args, setter_func, skip_ret_val, 0); + name_info = get_syscall_name_info(syscall_name); + if (portalcall_fastpath_should_skip(name_info.is_sendto, argc, safe_args)) { + *skip_ret_val = 0; + return 1; + } + return process_syscall_hooks(true, syscall_name, &name_info, argc, safe_args, setter_func, skip_ret_val, 0); } // Return handler for system calls @@ -517,7 +644,10 @@ static long syscall_ret_handler(const char *syscall_name, long orig_ret, int arg if (current->flags & PF_KTHREAD) { return orig_ret; } - return process_syscall_hooks(false, syscall_name, argc, args, NULL, NULL, orig_ret); + if (atomic_read(&global_syscall_return_hook_count) == 0) { + return orig_ret; + } + return process_syscall_hooks(false, syscall_name, NULL, argc, args, NULL, NULL, orig_ret); } int syscalls_hc_init(void) { diff --git a/src/hooks/syscalls_hc.h b/src/hooks/syscalls_hc.h index feb5d91..6e1523e 100644 --- a/src/hooks/syscalls_hc.h +++ b/src/hooks/syscalls_hc.h @@ -84,6 +84,8 @@ struct kernel_syscall_hook { bool in_use; /* Whether this slot is used */ struct rcu_head rcu; /* For RCU freeing */ char normalized_name[SYSCALL_NAME_MAX_LEN]; /* Cached normalized syscall name */ + u32 normalized_name_hash; /* Cached full hash of normalized_name */ + u16 normalized_name_len; /* Cached length of normalized_name */ struct work_struct unregister_work; }; @@ -91,6 +93,12 @@ struct kernel_syscall_hook { extern struct hlist_head syscall_hook_table[1024]; extern spinlock_t syscall_hook_lock; +bool portalcall_fastpath_register_magic(u32 user_magic); +void portalcall_fastpath_set_enabled(bool enabled); +bool portalcall_fastpath_is_enabled(void); +bool portalcall_fastpath_should_skip(bool is_sendto, + int argc, + const unsigned long args[]); /* Unregister a syscall hook using its pointer */ int unregister_syscall_hook(struct kernel_syscall_hook *hook_ptr); @@ -289,4 +297,4 @@ static inline u32 syscall_normalized_name_hash(const char *str) return full_name_hash(NULL, str, strlen(str)); } -#endif /* _SYSCALLS_HC_H */ \ No newline at end of file +#endif /* _SYSCALLS_HC_H */ diff --git a/src/portal/portal.c b/src/portal/portal.c index ef40b10..a082b54 100644 --- a/src/portal/portal.c +++ b/src/portal/portal.c @@ -4,7 +4,6 @@ #include #include #include "portal_op_list.h" -#include "ehypercall.h" uint64_t portal_interrupt = 0; diff --git a/src/portal/portal_op_list.h b/src/portal/portal_op_list.h index 574f128..918603f 100644 --- a/src/portal/portal_op_list.h +++ b/src/portal/portal_op_list.h @@ -21,6 +21,8 @@ X(unregister_uprobe, UNREGISTER_UPROBE) \ X(register_syscall_hook, REGISTER_SYSCALL_HOOK) \ X(unregister_syscall_hook, UNREGISTER_SYSCALL_HOOK) \ + X(register_portalcall_magic, REGISTER_PORTALCALL_MAGIC) \ + X(set_portalcall_fastpath, SET_PORTALCALL_FASTPATH) \ X(register_signal_hook, REGISTER_SIGNAL_HOOK) \ X(unregister_signal_hook, UNREGISTER_SIGNAL_HOOK) \ X(ffi_exec, FFI_EXEC) \ diff --git a/src/portal/portal_syscall.c b/src/portal/portal_syscall.c index 0bb2091..4143fea 100644 --- a/src/portal/portal_syscall.c +++ b/src/portal/portal_syscall.c @@ -12,6 +12,8 @@ extern struct hlist_head syscall_name_table[1024]; extern struct hlist_head syscall_all_hooks; extern spinlock_t syscall_hook_lock; extern atomic_t global_syscall_hook_count; +extern atomic_t global_syscall_enter_hook_count; +extern atomic_t global_syscall_return_hook_count; // Using normalize_syscall_name and syscall_name_hash from syscalls_hc.h @@ -45,8 +47,14 @@ void handle_op_register_syscall_hook(portal_region *mem_region) normalize_syscall_name(kernel_hook->hook.name), SYSCALL_NAME_MAX_LEN - 1); kernel_hook->normalized_name[SYSCALL_NAME_MAX_LEN - 1] = '\0'; + kernel_hook->normalized_name_hash = + syscall_normalized_name_hash(kernel_hook->normalized_name); + kernel_hook->normalized_name_len = + strnlen(kernel_hook->normalized_name, SYSCALL_NAME_MAX_LEN); } else { kernel_hook->normalized_name[0] = '\0'; + kernel_hook->normalized_name_hash = 0; + kernel_hook->normalized_name_len = 0; } // Add to the main hook table indexed by pointer address @@ -65,6 +73,12 @@ void handle_op_register_syscall_hook(portal_region *mem_region) spin_unlock(&syscall_hook_lock); atomic_inc(&global_syscall_hook_count); + if (kernel_hook->hook.on_enter) { + atomic_inc(&global_syscall_enter_hook_count); + } + if (kernel_hook->hook.on_return) { + atomic_inc(&global_syscall_return_hook_count); + } // Return the hook's memory address in the size field mem_region->header.size = (unsigned long)kernel_hook; @@ -93,6 +107,12 @@ static void unregister_syscall_deferred(struct work_struct *work) // 3. Decrement global count atomic_dec(&global_syscall_hook_count); + if (hook_ptr->hook.on_enter) { + atomic_dec(&global_syscall_enter_hook_count); + } + if (hook_ptr->hook.on_return) { + atomic_dec(&global_syscall_return_hook_count); + } for (i = 0; i < IGLOO_SYSCALL_MAXARGS; i++) { if (hook_ptr->hook.arg_filters[i].pattern) { @@ -127,4 +147,4 @@ void handle_op_unregister_syscall_hook(portal_region *mem_region) // Return success immediately mem_region->header.op = HYPER_RESP_READ_OK; -} \ No newline at end of file +} diff --git a/src/portal/portalcall_fastpath.c b/src/portal/portalcall_fastpath.c new file mode 100644 index 0000000..fcc6c7b --- /dev/null +++ b/src/portal/portalcall_fastpath.c @@ -0,0 +1,108 @@ +#include "portal_internal.h" +#include +#include +#include + +#define PORTALCALL_SYSCALL_MAGIC 0xc1d1e1f1UL + +struct portalcall_magic_entry { + u32 magic; + struct hlist_node node; + struct rcu_head rcu; +}; + +static DEFINE_HASHTABLE(portalcall_magic_table, 8); +static DEFINE_SPINLOCK(portalcall_magic_lock); +static bool portalcall_fastpath_enabled; + +bool portalcall_fastpath_register_magic(u32 user_magic) +{ + struct portalcall_magic_entry *entry; + + spin_lock(&portalcall_magic_lock); + hash_for_each_possible(portalcall_magic_table, entry, node, user_magic) { + if (entry->magic == user_magic) { + spin_unlock(&portalcall_magic_lock); + return true; + } + } + + entry = kzalloc(sizeof(*entry), GFP_ATOMIC); + if (!entry) { + spin_unlock(&portalcall_magic_lock); + return false; + } + + entry->magic = user_magic; + hash_add_rcu(portalcall_magic_table, &entry->node, user_magic); + spin_unlock(&portalcall_magic_lock); + return true; +} +EXPORT_SYMBOL(portalcall_fastpath_register_magic); + +void portalcall_fastpath_set_enabled(bool enabled) +{ + WRITE_ONCE(portalcall_fastpath_enabled, enabled); +} +EXPORT_SYMBOL(portalcall_fastpath_set_enabled); + +bool portalcall_fastpath_is_enabled(void) +{ + return READ_ONCE(portalcall_fastpath_enabled); +} +EXPORT_SYMBOL(portalcall_fastpath_is_enabled); + +static bool portalcall_magic_is_registered(u32 user_magic) +{ + struct portalcall_magic_entry *entry; + bool registered = false; + + rcu_read_lock(); + hash_for_each_possible_rcu(portalcall_magic_table, entry, node, user_magic) { + if (entry->magic == user_magic) { + registered = true; + break; + } + } + rcu_read_unlock(); + + return registered; +} + +bool portalcall_fastpath_should_skip(bool is_sendto, + int argc, + const unsigned long args[]) +{ + unsigned long magic; + unsigned long user_magic; + + if (!READ_ONCE(portalcall_fastpath_enabled) || !is_sendto || + !args || argc < 2) { + return false; + } + + magic = *(unsigned long *)args[0]; + if ((magic & 0xffffffffUL) != PORTALCALL_SYSCALL_MAGIC) { + return false; + } + + user_magic = *(unsigned long *)args[1]; + return !portalcall_magic_is_registered((u32)user_magic); +} +EXPORT_SYMBOL(portalcall_fastpath_should_skip); + +void handle_op_register_portalcall_magic(portal_region *mem_region) +{ + if (!portalcall_fastpath_register_magic((u32)mem_region->header.addr)) { + portalcall_fastpath_set_enabled(false); + mem_region->header.op = HYPER_RESP_WRITE_FAIL; + return; + } + mem_region->header.op = HYPER_RESP_WRITE_OK; +} + +void handle_op_set_portalcall_fastpath(portal_region *mem_region) +{ + portalcall_fastpath_set_enabled(mem_region->header.addr != 0); + mem_region->header.op = HYPER_RESP_WRITE_OK; +}