From 7b07dcebb85789a7c3cd08c8088bacac2ea728e8 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Fri, 19 Aug 2011 12:04:30 -0400 Subject: [PATCH] namecache: limit the number of delegations to prevent delegations from claiming all available slots in the attribute cache, nfs41_name_cache_insert() now returns ERROR_TOO_MANY_OPEN_FILES when the number of delegated entries reaches 50% of the cache capacity. see comment 'delegations and cache feedback' in name_cache.c for details when nfs41_open() sees this error from nfs41_name_cache_insert(), it calls new function nfs41_client_delegation_return_lru() to return the least-recently-used delegation and, if successful, loops back to nfs41_name_cache_insert(). nfs41_client_delegation_return_lru() returns NFS4ERR_BADHANDLE if all delegations are currently in use (associated with an existing open), in which case nfs41_open() returns the newly-granted delegation Signed-off-by: Casey Bodley --- daemon/delegation.c | 48 ++++++++++++++++++++++++++++++-- daemon/delegation.h | 5 ++++ daemon/name_cache.c | 51 +++++++++++++++++++++++++++++++--- daemon/nfs41_ops.c | 68 +++++++++++++++++++++++++++++++-------------- 4 files changed, 145 insertions(+), 27 deletions(-) diff --git a/daemon/delegation.c b/daemon/delegation.c index 6d4571a..d4865d5 100644 --- a/daemon/delegation.c +++ b/daemon/delegation.c @@ -301,7 +301,6 @@ out: return status; } - /* open delegation */ int nfs41_delegation_granted( IN nfs41_session *session, @@ -320,8 +319,10 @@ int nfs41_delegation_granted( delegation->type != OPEN_DELEGATE_WRITE) goto out; - if (delegation->recalled) + if (delegation->recalled) { + status = NFS4ERR_DELEG_REVOKED; goto out_return; + } /* allocate the delegation state */ status = delegation_create(parent, file, delegation, &state); @@ -398,8 +399,13 @@ static int delegation_find( EnterCriticalSection(&client->state.lock); entry = list_search(&client->state.delegations, value, cmp); if (entry) { + /* return a reference to the delegation */ *deleg_out = deleg_entry(entry); nfs41_delegation_ref(*deleg_out); + + /* move to the 'most recently used' end of the list */ + list_remove(entry); + list_add_tail(&client->state.delegations, entry); status = NFS4_OK; } LeaveCriticalSection(&client->state.lock); @@ -718,6 +724,7 @@ out_deleg: goto out; } + void nfs41_client_delegation_free( IN nfs41_client *client) { @@ -790,3 +797,40 @@ int nfs41_client_delegation_recovery( out: return status; } + + +int nfs41_client_delegation_return_lru( + IN nfs41_client *client) +{ + struct list_entry *entry; + nfs41_delegation_state *state = NULL; + int status = NFS4ERR_BADHANDLE; + + /* starting from the least recently opened, find a delegation + * that's not 'in use' and return it */ + EnterCriticalSection(&client->state.lock); + list_for_each(entry, &client->state.delegations) { + state = deleg_entry(entry); + + /* skip if it's currently in use for an open; note that ref_count + * can't go from 1 to 2 without holding client->state.lock */ + if (state->ref_count > 1) + continue; + + AcquireSRWLockExclusive(&state->lock); + if (state->status == DELEGATION_GRANTED) { + /* start returning the delegation */ + state->status = DELEGATION_RETURNING; + status = NFS4ERR_DELEG_REVOKED; + } + ReleaseSRWLockExclusive(&state->lock); + + if (status == NFS4ERR_DELEG_REVOKED) + break; + } + LeaveCriticalSection(&client->state.lock); + + if (status == NFS4ERR_DELEG_REVOKED) + status = delegation_return(client, state, FALSE, TRUE); + return status; +} diff --git a/daemon/delegation.h b/daemon/delegation.h index 0e82397..438c226 100644 --- a/daemon/delegation.h +++ b/daemon/delegation.h @@ -101,4 +101,9 @@ int nfs41_delegation_recall( int nfs41_client_delegation_recovery( IN nfs41_client *client); +/* attempt to return the least recently used delegation; + * fails with NFS4ERR_BADHANDLE if all delegations are in use */ +int nfs41_client_delegation_return_lru( + IN nfs41_client *client); + #endif /* DELEGATION_H */ diff --git a/daemon/name_cache.c b/daemon/name_cache.c index 7d7dfcf..eba4b37 100644 --- a/daemon/name_cache.c +++ b/daemon/name_cache.c @@ -53,8 +53,38 @@ enum { * lookups over the wire. a name cache entry is negative when its attributes * pointer is NULL. negative entries are created by three functions: * nfs41_name_cache_remove(), _insert() when called with NULL for the fh and - * attributes, and _rename() for the source entry + * attributes, and _rename() for the source entry */ + +/* delegations and cache feedback + * + * delegations provide a guarantee that no links or attributes will change + * without notice. the name cache takes advantage of this by preventing + * delegated entries from being removed on NAME_CACHE_EXPIRATION, though + * they're still removed when a parent is invalidated. the attribute cache + * holds an extra reference on delegated entries to prevent their removal + * entirely, until the delegation is returned. + * this extra reference presents a problem when the number of delegations + * approaches the maximum number of attribute cache entries. when there are + * not enough available entries to store the parent directories, every lookup + * results in a name cache miss, and cache performance degrades significantly. + * the solution is to provide feedback via nfs41_name_cache_insert() when + * delegations reach a certain percent of the cache capacity. the error code + * ERROR_TOO_MANY_OPEN_FILES, chosen arbitrarily for this case, instructs the + * caller to return an outstanding delegation before caching a new one. */ +static __inline bool_t is_delegation( + IN enum open_delegation_type4 type) +{ + return type == OPEN_DELEGATE_READ || type == OPEN_DELEGATE_WRITE; +} + +static __inline bool_t deleg_entry_limit( + IN uint32_t delegations, + IN uint32_t max_entries) +{ + /* limit the number of delegations to 50% of the cache capacity */ + return delegations >= max_entries / 2; +} /* attribute cache */ @@ -283,7 +313,7 @@ static void attr_cache_update( } } - if (delegation == OPEN_DELEGATE_READ || delegation == OPEN_DELEGATE_WRITE) + if (is_delegation(delegation)) entry->delegated = TRUE; } @@ -335,6 +365,7 @@ struct nfs41_name_cache { struct list_entry exp_entries; /* list of entries by expiry */ uint32_t expiration; uint32_t entries; + uint32_t delegations; uint32_t max_entries; SRWLOCK lock; }; @@ -468,8 +499,10 @@ static int name_cache_entry_update( attr_cache_update(entry->attributes, info, delegation); /* hold a reference as long as we have the delegation */ - if (delegation == OPEN_DELEGATE_READ || delegation == OPEN_DELEGATE_WRITE) + if (is_delegation(delegation)) { attr_cache_entry_ref(&cache->attributes, entry->attributes); + cache->delegations++; + } /* keep the entry from expiring */ if (entry->attributes->delegated) @@ -865,6 +898,13 @@ int nfs41_name_cache_insert( goto out_unlock; } + /* limit the number of delegations to prevent attr cache starvation */ + if (is_delegation(delegation) && deleg_entry_limit( + cache->delegations, cache->max_entries)) { + status = ERROR_TOO_MANY_OPEN_FILES; + goto out_unlock; + } + /* an empty path or component implies the root entry */ if (path == NULL || name == NULL || name->len == 0) { /* create the root entry if it doesn't exist */ @@ -906,13 +946,15 @@ out_unlock: return status; out_err_deleg: - if (delegation == OPEN_DELEGATE_READ || delegation == OPEN_DELEGATE_WRITE) { + if (is_delegation(delegation)) { /* we still need a reference to the attributes for the delegation */ struct attr_cache_entry *attributes; status = attr_cache_find_or_create(&cache->attributes, info->fileid, &attributes); if (status == NO_ERROR) attr_cache_update(attributes, info, delegation); + else + status = ERROR_TOO_MANY_OPEN_FILES; } goto out_unlock; } @@ -958,6 +1000,7 @@ int nfs41_name_cache_delegreturn( /* release the reference from name_cache_entry_update() */ attributes->delegated = FALSE; attr_cache_entry_deref(&cache->attributes, attributes); + cache->delegations--; status = NO_ERROR; out_unlock: diff --git a/daemon/nfs41_ops.c b/daemon/nfs41_ops.c index 2dda52e..eff4eef 100644 --- a/daemon/nfs41_ops.c +++ b/daemon/nfs41_ops.c @@ -34,6 +34,7 @@ #include "nfs41_compound.h" #include "nfs41_xdr.h" #include "name_cache.h" +#include "delegation.h" #include "daemon_debug.h" #include "util.h" @@ -329,6 +330,47 @@ static void open_delegation_return( delegation->type = OPEN_DELEGATE_NONE; } +static void open_update_cache( + IN nfs41_session *session, + IN nfs41_path_fh *parent, + IN nfs41_path_fh *file, + IN open_delegation4 *delegation, + IN change_info4 *changeinfo, + IN nfs41_getattr_res *dir_attrs, + IN nfs41_getattr_res *file_attrs) +{ + struct nfs41_name_cache *cache = session_name_cache(session); + uint32_t status; + + /* update the attributes of the parent directory */ + memcpy(&dir_attrs->info->attrmask, &dir_attrs->obj_attributes.attrmask, + sizeof(bitmap4)); + nfs41_attr_cache_update(cache, parent->fh.fileid, dir_attrs->info); + + /* add the file handle and attributes to the name cache */ + memcpy(&file_attrs->info->attrmask, &file_attrs->obj_attributes.attrmask, + sizeof(bitmap4)); +retry_cache_insert: + AcquireSRWLockShared(&file->path->lock); + status = nfs41_name_cache_insert(cache, file->path->path, &file->name, + &file->fh, file_attrs->info, changeinfo, delegation->type); + ReleaseSRWLockShared(&file->path->lock); + + if (status == ERROR_TOO_MANY_OPEN_FILES) { + /* the cache won't accept any more delegations; ask the client to + * return a delegation to free up a slot in the attribute cache */ + status = nfs41_client_delegation_return_lru(session->client); + if (status == NFS4_OK) + goto retry_cache_insert; + } + + if (status) { + /* if we can't make room in the cache, return this + * delegation immediately to free resources on the server */ + open_delegation_return(session, file, delegation); + } +} + int nfs41_open( IN nfs41_session *session, IN nfs41_path_fh *parent, @@ -345,7 +387,7 @@ int nfs41_open( OUT open_delegation4 *delegation, OUT OPTIONAL nfs41_file_info *info) { - int status, attr_status; + int status; nfs41_compound compound; nfs_argop4 argops[8]; nfs_resop4 resops[8]; @@ -473,28 +515,12 @@ int nfs41_open( if (status) goto out; - /* update the attributes of the parent directory */ - memcpy(&dir_info.attrmask, &pgetattr_res.obj_attributes.attrmask, - sizeof(bitmap4)); - nfs41_attr_cache_update(session_name_cache(session), - parent->fh.fileid, &dir_info); - - /* add the file handle and attributes to the name cache */ - memcpy(&info->attrmask, &getattr_res.obj_attributes.attrmask, - sizeof(bitmap4)); - AcquireSRWLockShared(&file->path->lock); - attr_status = nfs41_name_cache_insert(session_name_cache(session), - file->path->path, &file->name, &file->fh, - info, &open_res.resok4.cinfo, delegation->type); - ReleaseSRWLockShared(&file->path->lock); - - /* if we fail to cache the attributes, return the delegation - * immediately to free resources on the server */ - if (attr_status) - open_delegation_return(session, file, delegation); - if (create == OPEN4_CREATE) nfs41_superblock_space_changed(file->fh.superblock); + + /* update the name/attr cache with the results */ + open_update_cache(session, parent, file, delegation, + &open_res.resok4.cinfo, &pgetattr_res, &getattr_res); out: return status; }