From c3e8e2a1a449dd5556f45abde8f023aab8e3f5f2 Mon Sep 17 00:00:00 2001 From: Samuel J. Greear Date: Mon, 8 Feb 2010 15:18:46 -0700 Subject: [PATCH 3/4] Convert sf_buf hash table to a red-black tree --- sys/kern/kern_sfbuf.c | 91 +++++++++++++++++++++++------------------------- sys/sys/sfbuf.h | 5 ++- 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/sys/kern/kern_sfbuf.c b/sys/kern/kern_sfbuf.c index 24f77ff..4630ba3 100644 --- a/sys/kern/kern_sfbuf.c +++ b/sys/kern/kern_sfbuf.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -46,19 +47,11 @@ static void sf_buf_init(void *arg); SYSINIT(sock_sf, SI_BOOT2_MACHDEP, SI_ORDER_ANY, sf_buf_init, NULL) -LIST_HEAD(sf_buf_list, sf_buf); - SYSCTL_INT(_kern_ipc, OID_AUTO, nsfbufs, CTLFLAG_RD, &nsfbufs, 0, "Initial number of sf_bufs allocated by the system"); SYSCTL_INT(_kern_ipc, OID_AUTO, maxsfbufs, CTLFLAG_RW, &maxsfbufs, 0, "Maximum number of sf_bufs available to the system"); -/* - * A hash table of active sendfile(2) buffers - */ -static struct sf_buf_list *sf_buf_hashtable; -static u_long sf_buf_hashmask; - static TAILQ_HEAD(, sf_buf) sf_buf_freelist; static u_int sf_buf_alloc_want; @@ -75,16 +68,21 @@ static int nsffree; SYSCTL_INT(_kern_ipc, OID_AUTO, nsffree, CTLFLAG_RD, &nsffree, 0, "Number of free sf_bufs available to the system"); -static __inline -int -sf_buf_hash(vm_page_t m) +static int +sf_m_compare(struct sf_buf *buf1, struct sf_buf *buf2) { - int hv; + if (buf1->m < buf2->m) + return (-1); + if (buf1->m > buf2->m) + return (1); - hv = ((int)(intptr_t)m / sizeof(vm_page_t)) + ((int)(intptr_t)m >> 12); - return(hv & sf_buf_hashmask); + return (0); } +RB_HEAD(sf_buf_tree, sf_buf) sf_rb_head; +RB_PROTOTYPE(sf_buf_tree, sf_buf, rb_entry, sf_m_compare); +RB_GENERATE(sf_buf_tree, sf_buf, rb_entry, sf_m_compare); + /* * Allocate a pool of sf_bufs (sendfile(2) or "super-fast" if you prefer. :-)) */ @@ -93,8 +91,8 @@ sf_buf_init(void *arg) { int i; - sf_buf_hashtable = hashinit(nsfbufs, M_TEMP, &sf_buf_hashmask); spin_init(&sf_buf_spin); + RB_INIT(&sf_rb_head); TAILQ_INIT(&sf_buf_freelist); sf_base = kmem_alloc_nofault(&kernel_map, nsfbufs * PAGE_SIZE); sf_bufs = kmalloc(nsfbufs * sizeof(struct sf_buf), M_TEMP, @@ -114,8 +112,7 @@ sf_buf_init(void *arg) struct sf_buf * sf_buf_alloc(struct vm_page *m, int flags) { - struct sf_buf_list *hash_chain; - struct sf_buf *sf; + struct sf_buf *sf, key; globaldata_t gd; int error; int pflags; @@ -123,35 +120,35 @@ sf_buf_alloc(struct vm_page *m, int flags) gd = mycpu; spin_lock_wr(&sf_buf_spin); crit_enter(); - hash_chain = &sf_buf_hashtable[sf_buf_hash(m)]; - LIST_FOREACH(sf, hash_chain, list_entry) { - if (sf->m == m) { - /* - * cache hit - * - * We must invalidate the TLB entry based on whether - * it need only be valid on the local cpu (SFB_CPUPRIVATE), - * or on all cpus. This is conditionalized and in - * most cases no system-wide invalidation should be - * needed. - * - * Note: we do not remove the entry from the freelist - * on the 0->1 transition. - */ - ++sf->refcnt; - if ((flags & SFB_CPUPRIVATE) && sfbuf_quick) { - if ((sf->cpumask & gd->gd_cpumask) == 0) { - pmap_kenter_sync_quick(sf->kva); - sf->cpumask |= gd->gd_cpumask; - } - } else { - if (sf->cpumask != (cpumask_t)-1) { - pmap_kenter_sync(sf->kva); - sf->cpumask = (cpumask_t)-1; - } + + key.m = m; + sf = RB_FIND(sf_buf_tree, &sf_rb_head, &key); + if (sf != NULL) { + /* + * cache hit + * + * We must invalidate the TLB entry based on whether + * it need only be valid on the local cpu (SFB_CPUPRIVATE), + * or on all cpus. This is conditionalized and in + * most cases no system-wide invalidation should be + * needed. + * + * Note: we do not remove the entry from the freelist + * on the 0->1 transition. + */ + ++sf->refcnt; + if ((flags & SFB_CPUPRIVATE) && sfbuf_quick) { + if ((sf->cpumask & gd->gd_cpumask) == 0) { + pmap_kenter_sync_quick(sf->kva); + sf->cpumask |= gd->gd_cpumask; + } + } else { + if (sf->cpumask != (cpumask_t)-1) { + pmap_kenter_sync(sf->kva); + sf->cpumask = (cpumask_t)-1; } - goto done; /* found existing mapping */ } + goto done; /* found existing mapping */ } /* @@ -190,9 +187,9 @@ sf_buf_alloc(struct vm_page *m, int flags) break; } } - if (sf->m != NULL) /* remove previous mapping from hash table */ - LIST_REMOVE(sf, list_entry); - LIST_INSERT_HEAD(hash_chain, sf, list_entry); + if (sf->m != NULL) /* remove previous mapping from rbtree */ + RB_REMOVE(sf_buf_tree, &sf_rb_head, sf); + RB_INSERT(sf_buf_tree, &sf_rb_head, sf); sf->refcnt = 1; sf->m = m; if ((flags & SFB_CPUPRIVATE) && sfbuf_quick) { diff --git a/sys/sys/sfbuf.h b/sys/sys/sfbuf.h index 4db8d07..127a275 100644 --- a/sys/sys/sfbuf.h +++ b/sys/sys/sfbuf.h @@ -29,6 +29,9 @@ #ifndef _SFBUF_H_ #define _SFBUF_H_ +#ifndef _SYS_TREE_H_ +#include +#endif #ifndef _SYS_TYPES_H_ #include #endif @@ -42,7 +45,7 @@ #endif struct sf_buf { - LIST_ENTRY(sf_buf) list_entry; /* hash chain of active buffers */ + RB_ENTRY(sf_buf) rb_entry; /* rbtree of active buffers */ TAILQ_ENTRY(sf_buf) free_entry; /* list of free buffers */ struct vm_page *m; /* currently mapped page */ vm_offset_t kva; /* va of mapping */ -- 1.6.4