namecache: limit the number of delegations

to prevent delegations from claiming all available slots in the attribute cache, nfs41_name_cache_insert() now returns ERROR_TOO_MANY_OPEN_FILES when the number of delegated entries reaches 50% of the cache capacity.  see comment 'delegations and cache feedback' in name_cache.c for details

when nfs41_open() sees this error from nfs41_name_cache_insert(), it calls new function nfs41_client_delegation_return_lru() to return the least-recently-used delegation and, if successful, loops back to nfs41_name_cache_insert().  nfs41_client_delegation_return_lru() returns NFS4ERR_BADHANDLE if all delegations are currently in use (associated with an existing open), in which case nfs41_open() returns the newly-granted delegation

Signed-off-by: Casey Bodley <cbodley@citi.umich.edu>

daemon/delegation.c

@@ -301,7 +301,6 @@ out:
     return status;
 }
 
-
 /* open delegation */
 int nfs41_delegation_granted(
     IN nfs41_session *session,
@@ -320,8 +319,10 @@ int nfs41_delegation_granted(
         delegation->type != OPEN_DELEGATE_WRITE)
         goto out;
 
-    if (delegation->recalled)
+    if (delegation->recalled) {
+        status = NFS4ERR_DELEG_REVOKED;
         goto out_return;
+    }
 
     /* allocate the delegation state */
     status = delegation_create(parent, file, delegation, &state);
@@ -398,8 +399,13 @@ static int delegation_find(
     EnterCriticalSection(&client->state.lock);
     entry = list_search(&client->state.delegations, value, cmp);
     if (entry) {
+        /* return a reference to the delegation */
         *deleg_out = deleg_entry(entry);
         nfs41_delegation_ref(*deleg_out);
+
+        /* move to the 'most recently used' end of the list */
+        list_remove(entry);
+        list_add_tail(&client->state.delegations, entry);
         status = NFS4_OK;
     }
     LeaveCriticalSection(&client->state.lock);
@@ -718,6 +724,7 @@ out_deleg:
     goto out;
 }
 
+
 void nfs41_client_delegation_free(
     IN nfs41_client *client)
 {
@@ -790,3 +797,40 @@ int nfs41_client_delegation_recovery(
 out:
     return status;
 }
+
+
+int nfs41_client_delegation_return_lru(
+    IN nfs41_client *client)
+{
+    struct list_entry *entry;
+    nfs41_delegation_state *state = NULL;
+    int status = NFS4ERR_BADHANDLE;
+
+    /* starting from the least recently opened, find a delegation
+     * that's not 'in use' and return it */
+    EnterCriticalSection(&client->state.lock);
+    list_for_each(entry, &client->state.delegations) {
+        state = deleg_entry(entry);
+
+        /* skip if it's currently in use for an open; note that ref_count
+         * can't go from 1 to 2 without holding client->state.lock */
+        if (state->ref_count > 1)
+            continue;
+
+        AcquireSRWLockExclusive(&state->lock);
+        if (state->status == DELEGATION_GRANTED) {
+            /* start returning the delegation */
+            state->status = DELEGATION_RETURNING;
+            status = NFS4ERR_DELEG_REVOKED;
+        }
+        ReleaseSRWLockExclusive(&state->lock);
+
+        if (status == NFS4ERR_DELEG_REVOKED)
+            break;
+    }
+    LeaveCriticalSection(&client->state.lock);
+
+    if (status == NFS4ERR_DELEG_REVOKED)
+        status = delegation_return(client, state, FALSE, TRUE);
+    return status;
+}

daemon/delegation.h

@@ -101,4 +101,9 @@ int nfs41_delegation_recall(
 int nfs41_client_delegation_recovery(
     IN nfs41_client *client);
 
+/* attempt to return the least recently used delegation;
+ * fails with NFS4ERR_BADHANDLE if all delegations are in use */
+int nfs41_client_delegation_return_lru(
+    IN nfs41_client *client);
+
 #endif /* DELEGATION_H */

daemon/name_cache.c

@@ -53,8 +53,38 @@ enum {
  * lookups over the wire.  a name cache entry is negative when its attributes
  * pointer is NULL.  negative entries are created by three functions:
  * nfs41_name_cache_remove(), _insert() when called with NULL for the fh and
- * attributes, and _rename() for the source entry
+ * attributes, and _rename() for the source entry */
+
+/* delegations and cache feedback
+ *
+ *   delegations provide a guarantee that no links or attributes will change
+ * without notice.  the name cache takes advantage of this by preventing
+ * delegated entries from being removed on NAME_CACHE_EXPIRATION, though
+ * they're still removed when a parent is invalidated.  the attribute cache
+ * holds an extra reference on delegated entries to prevent their removal
+ * entirely, until the delegation is returned.
+ *   this extra reference presents a problem when the number of delegations
+ * approaches the maximum number of attribute cache entries.  when there are
+ * not enough available entries to store the parent directories, every lookup
+ * results in a name cache miss, and cache performance degrades significantly.
+ *   the solution is to provide feedback via nfs41_name_cache_insert() when
+ * delegations reach a certain percent of the cache capacity.  the error code
+ * ERROR_TOO_MANY_OPEN_FILES, chosen arbitrarily for this case, instructs the
+ * caller to return an outstanding delegation before caching a new one.
  */
+static __inline bool_t is_delegation(
+    IN enum open_delegation_type4 type)
+{
+    return type == OPEN_DELEGATE_READ || type == OPEN_DELEGATE_WRITE;
+}
+
+static __inline bool_t deleg_entry_limit(
+    IN uint32_t delegations,
+    IN uint32_t max_entries)
+{
+    /* limit the number of delegations to 50% of the cache capacity */
+    return delegations >= max_entries / 2;
+}
 
 
 /* attribute cache */
@@ -283,7 +313,7 @@ static void attr_cache_update(
         }
     }
 
-    if (delegation == OPEN_DELEGATE_READ || delegation == OPEN_DELEGATE_WRITE)
+    if (is_delegation(delegation))
         entry->delegated = TRUE;
 }
 
@@ -335,6 +365,7 @@ struct nfs41_name_cache {
     struct list_entry       exp_entries; /* list of entries by expiry */
     uint32_t                expiration;
     uint32_t                entries;
+    uint32_t                delegations;
     uint32_t                max_entries;
     SRWLOCK                 lock;
 };
@@ -468,8 +499,10 @@ static int name_cache_entry_update(
             attr_cache_update(entry->attributes, info, delegation);
 
             /* hold a reference as long as we have the delegation */
-            if (delegation == OPEN_DELEGATE_READ || delegation == OPEN_DELEGATE_WRITE)
+            if (is_delegation(delegation)) {
                 attr_cache_entry_ref(&cache->attributes, entry->attributes);
+                cache->delegations++;
+            }
 
             /* keep the entry from expiring */
             if (entry->attributes->delegated)
@@ -865,6 +898,13 @@ int nfs41_name_cache_insert(
         goto out_unlock;
     }
 
+    /* limit the number of delegations to prevent attr cache starvation */
+    if (is_delegation(delegation) && deleg_entry_limit(
+        cache->delegations, cache->max_entries)) {
+        status = ERROR_TOO_MANY_OPEN_FILES;
+        goto out_unlock;
+    }
+
     /* an empty path or component implies the root entry */
     if (path == NULL || name == NULL || name->len == 0) {
         /* create the root entry if it doesn't exist */
@@ -906,13 +946,15 @@ out_unlock:
     return status;
 
 out_err_deleg:
-    if (delegation == OPEN_DELEGATE_READ || delegation == OPEN_DELEGATE_WRITE) {
+    if (is_delegation(delegation)) {
         /* we still need a reference to the attributes for the delegation */
         struct attr_cache_entry *attributes;
         status = attr_cache_find_or_create(&cache->attributes,
             info->fileid, &attributes);
         if (status == NO_ERROR)
             attr_cache_update(attributes, info, delegation);
+        else
+            status = ERROR_TOO_MANY_OPEN_FILES;
     }
     goto out_unlock;
 }
@@ -958,6 +1000,7 @@ int nfs41_name_cache_delegreturn(
     /* release the reference from name_cache_entry_update() */
     attributes->delegated = FALSE;
     attr_cache_entry_deref(&cache->attributes, attributes);
+    cache->delegations--;
     status = NO_ERROR;
 
 out_unlock:

daemon/nfs41_ops.c

@@ -34,6 +34,7 @@
 #include "nfs41_compound.h"
 #include "nfs41_xdr.h"
 #include "name_cache.h"
+#include "delegation.h"
 #include "daemon_debug.h"
 #include "util.h"
 
@@ -329,6 +330,47 @@ static void open_delegation_return(
     delegation->type = OPEN_DELEGATE_NONE;
 }
 
+static void open_update_cache(
+    IN nfs41_session *session,
+    IN nfs41_path_fh *parent,
+    IN nfs41_path_fh *file,
+    IN open_delegation4 *delegation,
+    IN change_info4 *changeinfo,
+    IN nfs41_getattr_res *dir_attrs,
+    IN nfs41_getattr_res *file_attrs)
+{
+    struct nfs41_name_cache *cache = session_name_cache(session);
+    uint32_t status;
+    
+    /* update the attributes of the parent directory */
+    memcpy(&dir_attrs->info->attrmask, &dir_attrs->obj_attributes.attrmask,
+        sizeof(bitmap4));
+    nfs41_attr_cache_update(cache, parent->fh.fileid, dir_attrs->info);
+
+    /* add the file handle and attributes to the name cache */
+    memcpy(&file_attrs->info->attrmask, &file_attrs->obj_attributes.attrmask,
+        sizeof(bitmap4));
+retry_cache_insert:
+    AcquireSRWLockShared(&file->path->lock);
+    status = nfs41_name_cache_insert(cache, file->path->path, &file->name,
+        &file->fh, file_attrs->info, changeinfo, delegation->type);
+    ReleaseSRWLockShared(&file->path->lock);
+
+    if (status == ERROR_TOO_MANY_OPEN_FILES) {
+        /* the cache won't accept any more delegations; ask the client to
+         * return a delegation to free up a slot in the attribute cache */
+        status = nfs41_client_delegation_return_lru(session->client);
+        if (status == NFS4_OK)
+            goto retry_cache_insert;
+    }
+
+    if (status) {
+        /* if we can't make room in the cache, return this
+         * delegation immediately to free resources on the server */
+        open_delegation_return(session, file, delegation);
+    }
+}
+
 int nfs41_open(
     IN nfs41_session *session,
     IN nfs41_path_fh *parent,
@@ -345,7 +387,7 @@ int nfs41_open(
     OUT open_delegation4 *delegation,
     OUT OPTIONAL nfs41_file_info *info)
 {
-    int status, attr_status;
+    int status;
     nfs41_compound compound;
     nfs_argop4 argops[8];
     nfs_resop4 resops[8];
@@ -473,28 +515,12 @@ int nfs41_open(
     if (status)
         goto out;
 
-    /* update the attributes of the parent directory */
-    memcpy(&dir_info.attrmask, &pgetattr_res.obj_attributes.attrmask,
-        sizeof(bitmap4));
-    nfs41_attr_cache_update(session_name_cache(session),
-        parent->fh.fileid, &dir_info);
-
-    /* add the file handle and attributes to the name cache */
-    memcpy(&info->attrmask, &getattr_res.obj_attributes.attrmask,
-        sizeof(bitmap4));
-    AcquireSRWLockShared(&file->path->lock);
-    attr_status = nfs41_name_cache_insert(session_name_cache(session),
-        file->path->path, &file->name, &file->fh,
-        info, &open_res.resok4.cinfo, delegation->type);
-    ReleaseSRWLockShared(&file->path->lock);
-
-    /* if we fail to cache the attributes, return the delegation
-     * immediately to free resources on the server */
-    if (attr_status)
-        open_delegation_return(session, file, delegation);
-
     if (create == OPEN4_CREATE)
         nfs41_superblock_space_changed(file->fh.superblock);
+
+    /* update the name/attr cache with the results */
+    open_update_cache(session, parent, file, delegation,
+        &open_res.resok4.cinfo, &pgetattr_res, &getattr_res);
 out:
     return status;
 }