recovery: handle delegation.recalled on reclaim

while the server is required to grant us delegations we reclaim through CLAIM_PREVIOUS, it may set the flag recalled=TRUE if it's not ready to grant the delegation.  the client is then responsible for flushing modified state to the server and returning the delegation

new function nfs41_client_delegation_recovery() cleans up after delegation recovery by a) returning any delegations flagged as recalled, and b) 'forgetting' any delegations that we failed to reclaim.  this function is called under the client's state recovery lock, directly after open and delegation state is recovered

added 'try_recovery' argument to delegation_return(), allowing it to be called during client state recovery.  split out the code that removes the delegation from the client and its opens into delegation_remove(), which is what nfs41_client_delegation_recovery() uses to 'forget' a delegation

Signed-off-by: Casey Bodley <cbodley@citi.umich.edu>

daemon/delegation.c

@@ -121,41 +121,19 @@ static nfs41_open_state* deleg_open_find(
     return open;
 }
 
-#pragma warning (disable : 4706) /* assignment within conditional expression */
-
-static void delegation_return(
+static void delegation_remove(
     IN nfs41_client *client,
-    IN nfs41_delegation_state *deleg,
-    IN bool_t truncate)
+    IN nfs41_delegation_state *deleg)
 {
-    stateid_arg stateid;
     struct list_entry *entry;
-    nfs41_open_state *open;
-
-    /* recover opens associated with the delegation */
-    while (open = deleg_open_find(&client->state, deleg)) {
-        nfs41_delegation_to_open(open, TRUE);
-        nfs41_open_state_deref(open);
-    }
-
-    /* TODO: flush data and metadata before returning delegation */
-
-    /* return the delegation */
-    stateid.type = STATEID_DELEG_FILE;
-    stateid.open = NULL;
-    stateid.delegation = deleg;
-    AcquireSRWLockShared(&deleg->lock);
-    memcpy(&stateid.stateid, &deleg->state.stateid, sizeof(stateid4));
-    ReleaseSRWLockShared(&deleg->lock);
-
-    nfs41_delegreturn(client->session, &deleg->file, &stateid, TRUE);
 
     /* remove from the client's list */
     EnterCriticalSection(&client->state.lock);
     list_remove(&deleg->client_entry);
 
+    /* remove from each associated open */
     list_for_each(entry, &client->state.opens) {
-        open = open_entry(entry);
+        nfs41_open_state *open = open_entry(entry);
         AcquireSRWLockExclusive(&open->lock);
         if (open->delegation.state == deleg) {
             /* drop the delegation reference */
@@ -176,6 +154,45 @@ static void delegation_return(
     nfs41_delegation_deref(deleg);
 }
 
+#pragma warning (disable : 4706) /* assignment within conditional expression */
+
+static int delegation_return(
+    IN nfs41_client *client,
+    IN nfs41_delegation_state *deleg,
+    IN bool_t truncate,
+    IN bool_t try_recovery)
+{
+    stateid_arg stateid;
+    int status = NFS4_OK;
+
+    /* recover opens associated with the delegation */
+    nfs41_open_state *open;
+    while (open = deleg_open_find(&client->state, deleg)) {
+        status = nfs41_delegation_to_open(open, try_recovery);
+        nfs41_open_state_deref(open);
+
+        if (status == NFS4ERR_BADSESSION)
+            goto out;
+    }
+
+    /* return the delegation */
+    stateid.type = STATEID_DELEG_FILE;
+    stateid.open = NULL;
+    stateid.delegation = deleg;
+    AcquireSRWLockShared(&deleg->lock);
+    memcpy(&stateid.stateid, &deleg->state.stateid, sizeof(stateid4));
+    ReleaseSRWLockShared(&deleg->lock);
+
+    status = nfs41_delegreturn(client->session,
+        &deleg->file, &stateid, try_recovery);
+    if (status == NFS4ERR_BADSESSION)
+        goto out;
+
+    delegation_remove(client, deleg);
+out:
+    return status;
+}
+
 
 /* open delegation */
 int nfs41_delegation_granted(
@@ -371,7 +388,7 @@ out:
     return status;
 
 out_return:
-    delegation_return(client, deleg, create == OPEN4_CREATE);
+    delegation_return(client, deleg, create == OPEN4_CREATE, TRUE);
 
 out_deleg:
     nfs41_delegation_deref(deleg);
@@ -483,7 +500,7 @@ int nfs41_delegation_return(
     ReleaseSRWLockExclusive(&deleg->lock);
 
     if (status == NFS4ERR_DELEG_REVOKED) {
-        delegation_return(client, deleg, truncate);
+        delegation_return(client, deleg, truncate, TRUE);
         status = NFS4_OK;
     }
 
@@ -505,7 +522,7 @@ static unsigned int WINAPI delegation_recall_thread(void *args)
 {
     struct recall_thread_args *recall = (struct recall_thread_args*)args;
 
-    delegation_return(recall->client, recall->delegation, recall->truncate);
+    delegation_return(recall->client, recall->delegation, recall->truncate, TRUE);
 
     /* clean up thread arguments */
     nfs41_delegation_deref(recall->delegation);
@@ -604,3 +621,55 @@ void nfs41_client_delegation_free(
     }
     LeaveCriticalSection(&client->state.lock);
 }
+
+
+static int delegation_recovery_status(
+    IN nfs41_delegation_state *deleg)
+{
+    int status = NFS4_OK;
+
+    AcquireSRWLockExclusive(&deleg->lock);
+    if (deleg->status == DELEGATION_GRANTED) {
+        if (deleg->revoked) {
+            deleg->status = DELEGATION_RETURNED;
+            status = NFS4ERR_BADHANDLE;
+        } else if (deleg->state.recalled) {
+            deleg->status = DELEGATION_RETURNING;
+            status = NFS4ERR_DELEG_REVOKED;
+        }
+    }
+    ReleaseSRWLockExclusive(&deleg->lock);
+    return status;
+}
+
+int nfs41_client_delegation_recovery(
+    IN nfs41_client *client)
+{
+    struct list_entry *entry, *tmp;
+    nfs41_delegation_state *deleg;
+    int status = NFS4_OK;
+
+    list_for_each_tmp(entry, tmp, &client->state.delegations) {
+        deleg = list_container(entry, nfs41_delegation_state, client_entry);
+
+        status = delegation_recovery_status(deleg);
+        switch (status) {
+        case NFS4ERR_DELEG_REVOKED:
+            /* the delegation was reclaimed, but flagged as recalled;
+             * return it with try_recovery=FALSE */
+            status = delegation_return(client, deleg, FALSE, FALSE);
+            break;
+
+        case NFS4ERR_BADHANDLE:
+            /* reclaim failed, so we have no delegation state on the server;
+             * 'forget' the delegation without trying to return it */
+            delegation_remove(client, deleg);
+            status = NFS4_OK;
+            break;
+        }
+
+        if (status == NFS4ERR_BADSESSION)
+            break;
+    }
+    return status;
+}

daemon/delegation.h

@@ -92,4 +92,10 @@ int nfs41_delegation_recall(
     IN const stateid4 *stateid,
     IN bool_t truncate);
 
+
+/* after client state recovery, return any 'recalled' delegations;
+ * must be called under the client's state lock */
+int nfs41_client_delegation_recovery(
+    IN nfs41_client *client);
+
 #endif /* DELEGATION_H */

daemon/nfs41_xdr.c

@@ -1997,12 +1997,10 @@ static bool_t decode_open_read_delegation4(
     XDR *xdr,
     open_delegation4 *delegation)
 {
-    bool_t tmp_bool;
-
     if (!xdr_stateid4(xdr, &delegation->stateid))
         return FALSE;
 
-    if (!xdr_bool(xdr, &tmp_bool))
+    if (!xdr_bool(xdr, &delegation->recalled))
         return FALSE;
 
     return xdr_nfsace4(xdr, &delegation->permissions);
@@ -2053,12 +2051,10 @@ static bool_t decode_open_write_delegation4(
     XDR *xdr,
     open_delegation4 *delegation)
 {
-    bool_t tmp_bool;
-
     if (!xdr_stateid4(xdr, &delegation->stateid))
         return FALSE;
 
-    if (!xdr_bool(xdr, &tmp_bool))
+    if (!xdr_bool(xdr, &delegation->recalled))
         return FALSE;
 
     if (!decode_space_limit4(xdr))

daemon/recovery.c

@@ -354,7 +354,8 @@ int nfs41_recover_client_state(
         }
     }
 
-    /* TODO: return any delegations that were reclaimed as 'recalled' */
+    /* return any delegations that were reclaimed as 'recalled' */
+    status = nfs41_client_delegation_recovery(client);
 unlock:
     LeaveCriticalSection(&state->lock);