recovery: reboot recovery of delegation stateids

recover_client_state() flags all delegations as revoked before starting open state recovery.  if recover_open() finds that its delegation is revoked, it attempts to recover it using CLAIM_PREVIOUS.  if its delegation has already been reclaimed by another open, it can skip reclaiming the open stateid (provided it has no byte-range locks to reclaim).  after all opens have been reclaimed, any delegations still marked 'revoked' are passed to recover_delegation().  recover_delegation() also uses CLAIM_PREVIOUS (or CLAIM_NULL outside of the grace period) to reclaim the delegation, but has to throw away the open stateid with CLOSE

added a try_recovery argument to nfs41_delegreturn() and nfs41_delegation_granted(), so it can be called by recover_open() if OPEN grants an unexpected open

Signed-off-by: Casey Bodley <cbodley@citi.umich.edu>

daemon/delegation.c

@@ -32,6 +32,7 @@
 
 /* allocation and reference counting */
 static int delegation_create(
+    IN const nfs41_path_fh *parent,
     IN const nfs41_path_fh *file,
     IN const open_delegation4 *delegation,
     OUT nfs41_delegation_state **deleg_out)
@@ -46,9 +47,15 @@ static int delegation_create(
     }
 
     memcpy(&state->state, delegation, sizeof(open_delegation4));
+
     abs_path_copy(&state->path, file->path);
     path_fh_init(&state->file, &state->path);
     fh_copy(&state->file.fh, &file->fh);
+    path_fh_init(&state->parent, &state->path);
+    last_component(state->path.path, state->file.name.name,
+        &state->parent.name);
+    fh_copy(&state->parent.fh, &parent->fh);
+
     list_init(&state->client_entry);
     state->status = DELEGATION_GRANTED;
     InitializeSRWLock(&state->lock);
@@ -132,7 +139,8 @@ static void delegation_return(
 
     /* TODO: flush data and metadata before returning delegation */
 
-    nfs41_delegreturn(client->session, &deleg->file, &deleg->state.stateid);
+    nfs41_delegreturn(client->session, &deleg->file,
+        &deleg->state.stateid, TRUE);
 
     /* remove from the client's list */
     EnterCriticalSection(&client->state.lock);
@@ -164,21 +172,25 @@ static void delegation_return(
 /* open delegation */
 int nfs41_delegation_granted(
     IN nfs41_session *session,
+    IN nfs41_path_fh *parent,
     IN nfs41_path_fh *file,
     IN open_delegation4 *delegation,
+    IN bool_t try_recovery,
     OUT nfs41_delegation_state **deleg_out)
 {
     nfs41_client *client = session->client;
     nfs41_delegation_state *state;
     int status = NO_ERROR;
 
-    if (delegation->recalled ||
+    if (delegation->type != OPEN_DELEGATE_READ &&
-        delegation->type == OPEN_DELEGATE_NONE ||
+        delegation->type != OPEN_DELEGATE_WRITE)
-        delegation->type == OPEN_DELEGATE_NONE_EXT)
         goto out;
 
+    if (delegation->recalled)
+        goto out_return;
+
     /* allocate the delegation state */
-    status = delegation_create(file, delegation, &state);
+    status = delegation_create(parent, file, delegation, &state);
     if (status)
         goto out_return;
 
@@ -194,7 +206,7 @@ out:
     return status;
 
 out_return: /* return the delegation on failure */
-    nfs41_delegreturn(session, file, &delegation->stateid);
+    nfs41_delegreturn(session, file, &delegation->stateid, try_recovery);
     goto out;
 }
 
@@ -439,7 +451,7 @@ int nfs41_delegation_return(
         }
     } else {
         /* the delegation is being returned, wait for it to finish */
-        while (deleg->status != DELEGATION_RETURNING)
+        while (deleg->status != DELEGATION_RETURNED)
             SleepConditionVariableSRW(&deleg->cond, &deleg->lock, INFINITE, 0);
         status = NFS4ERR_BADHANDLE;
     }

daemon/delegation.h

@@ -45,8 +45,10 @@ void nfs41_client_delegation_free(
 /* open delegation */
 int nfs41_delegation_granted(
     IN nfs41_session *session,
+    IN nfs41_path_fh *parent,
     IN nfs41_path_fh *file,
     IN open_delegation4 *delegation,
+    IN bool_t try_recovery,
     OUT nfs41_delegation_state **deleg_out);
 
 int nfs41_delegate_open(

daemon/nfs41.h

@@ -89,6 +89,7 @@ enum delegation_status {
 typedef struct __nfs41_delegation_state {
     open_delegation4 state;
     nfs41_abs_path path;
+    nfs41_path_fh parent;
     nfs41_path_fh file;
     struct list_entry client_entry; /* entry in nfs41_client.delegations */
     LONG ref_count;
@@ -96,6 +97,8 @@ typedef struct __nfs41_delegation_state {
     enum delegation_status status;
     SRWLOCK lock;
     CONDITION_VARIABLE cond;
+
+    bool_t revoked; /* for recovery, accessed under client.state.lock */
 } nfs41_delegation_state;
 
 typedef struct __nfs41_lock_state {

daemon/nfs41_ops.c

@@ -1530,7 +1530,8 @@ out:
 int nfs41_delegreturn(
     IN nfs41_session *session,
     IN nfs41_path_fh *file,
-    IN stateid4 *stateid)
+    IN stateid4 *stateid,
+    IN bool_t try_recovery)
 {
     int status;
     nfs41_compound compound;
@@ -1557,7 +1558,7 @@ int nfs41_delegreturn(
     compound_add_op(&compound, OP_DELEGRETURN, &dr_args, &dr_res);
     dr_args.stateid = stateid;
 
-    status = compound_encode_send_decode(session, &compound, TRUE);
+    status = compound_encode_send_decode(session, &compound, try_recovery);
     if (status)
         goto out;
 

daemon/nfs41_ops.h

@@ -1154,7 +1154,8 @@ int nfs41_access(
 int nfs41_delegreturn(
     IN nfs41_session *session,
     IN nfs41_path_fh *file,
-    IN stateid4 *stateid);
+    IN stateid4 *stateid,
+    IN bool_t try_recovery);
 
 enum nfsstat4 nfs41_fs_locations(
     IN nfs41_session *session,

daemon/open.c

@@ -209,8 +209,8 @@ static int do_open(
         goto out;
 
     /* allocate delegation state and register it with the client */
-    nfs41_delegation_granted(state->session,
+    nfs41_delegation_granted(state->session, &state->parent,
-        &state->file, &delegation, &deleg_state);
+        &state->file, &delegation, TRUE, &deleg_state);
 
     AcquireSRWLockExclusive(&state->lock);
     /* update the stateid */

daemon/recovery.c

@@ -21,7 +21,10 @@
  * such damages.
  */
 
+#include <time.h>
+
 #include "recovery.h"
+#include "delegation.h"
 #include "nfs41_callback.h"
 #include "nfs41_compound.h"
 #include "nfs41_ops.h"
@@ -63,44 +66,131 @@ void nfs41_recovery_finish(
     LeaveCriticalSection(&client->recovery.lock);
 }
 
+
+/* client state recovery for server reboot or lease expiration */
+static int recover_open_grace(
+    IN nfs41_session *session,
+    IN nfs41_path_fh *parent,
+    IN nfs41_path_fh *file,
+    IN state_owner4 *owner,
+    IN uint32_t access,
+    IN uint32_t deny,
+    IN enum open_delegation_type4 delegate_type,
+    OUT stateid4 *stateid,
+    OUT open_delegation4 *delegation)
+{
+    /* reclaim the open stateid with CLAIM_PREVIOUS */
+    open_claim4 claim;
+    claim.claim = CLAIM_PREVIOUS;
+    claim.u.prev.delegate_type = delegate_type;
+
+    return nfs41_open(session, parent, file, owner,
+        &claim, access, deny, OPEN4_NOCREATE, 0, 0, FALSE,
+        stateid, delegation, NULL);
+}
+
+static int recover_open_no_grace(
+    IN nfs41_session *session,
+    IN nfs41_path_fh *parent,
+    IN nfs41_path_fh *file,
+    IN state_owner4 *owner,
+    IN uint32_t access,
+    IN uint32_t deny,
+    IN enum open_delegation_type4 delegate_type,
+    OUT stateid4 *stateid,
+    OUT open_delegation4 *delegation)
+{
+    open_claim4 claim;
+
+    /* TODO: try CLAIM_DELEGATE_PREV / CLAIM_DELEG_PREV_FH first */
+
+    /* attempt out-of-grace recovery with CLAIM_NULL */
+    claim.claim = CLAIM_NULL;
+    claim.u.null.filename = &file->name;
+
+    /* ask nicely for the delegation we had */
+    if (delegate_type == OPEN_DELEGATE_READ)
+        access |= OPEN4_SHARE_ACCESS_WANT_READ_DELEG;
+    else if (delegate_type == OPEN_DELEGATE_WRITE)
+        access |= OPEN4_SHARE_ACCESS_WANT_WRITE_DELEG;
+
+    return nfs41_open(session, parent, file, owner,
+        &claim, access, deny, OPEN4_NOCREATE, 0, 0, FALSE,
+        stateid, delegation, NULL);
+}
+
 static int recover_open(
     IN nfs41_session *session,
     IN nfs41_open_state *open,
     IN OUT bool_t *grace)
 {
-    open_claim4 claim;
+    open_delegation4 delegation = { 0 };
-    open_delegation4 delegation;
     stateid4 stateid;
-    int status;
+    enum open_delegation_type4 delegate_type = OPEN_DELEGATE_NONE;
+    int status = NFS4ERR_BADHANDLE;
 
-    /* reclaim the open stateid */
+    /* check for an associated delegation */
-    claim.claim = CLAIM_PREVIOUS;
+    AcquireSRWLockExclusive(&open->lock);
-    claim.u.prev.delegate_type = OPEN_DELEGATE_NONE;
+    if (open->delegation.state) {
+        nfs41_delegation_state *deleg = open->delegation.state;
+        if (deleg->revoked) {
+            /* reclaim the delegation along with the open */
+            AcquireSRWLockShared(&deleg->lock);
+            delegate_type = deleg->state.type;
+            ReleaseSRWLockShared(&deleg->lock);
+        } else if (deleg->state.recalled) {
+            /* we'll need an open stateid regardless */
+        } else if (list_empty(&open->locks.list)) {
+            /* if there are locks, we need an open stateid to
+             * reclaim them; otherwise, the open can be delegated */
+            open->do_close = FALSE;
+            status = NFS4_OK;
+        }
+    }
+    ReleaseSRWLockExclusive(&open->lock);
+
+    if (status == NFS4_OK) /* use existing delegation */
+        goto out;
 
     if (*grace)
-        status = nfs41_open(session, &open->parent, &open->file,
+        status = recover_open_grace(session, &open->parent, &open->file,
-            &open->owner, &claim, open->share_access, open->share_deny,
+            &open->owner, open->share_access, open->share_deny,
-            OPEN4_NOCREATE, 0, 0, FALSE, &stateid, &delegation, NULL);
+            delegate_type, &stateid, &delegation);
     else
         status = NFS4ERR_NO_GRACE;
 
     if (status == NFS4ERR_NO_GRACE) {
         *grace = FALSE;
-        /* attempt out-of-grace recovery with CLAIM_NULL */
+        status = recover_open_no_grace(session, &open->parent, &open->file,
-        claim.claim = CLAIM_NULL;
+            &open->owner, open->share_access, open->share_deny,
-        claim.u.null.filename = &open->file.name;
+            delegate_type, &stateid, &delegation);
-
-        status = nfs41_open(session, &open->parent, &open->file,
-            &open->owner, &claim, open->share_access, open->share_deny,
-            OPEN4_NOCREATE, 0, 0, FALSE, &stateid, &delegation, NULL);
     }
     if (status)
         goto out;
 
-    /* update the open stateid on success */
     AcquireSRWLockExclusive(&open->lock);
+    /* update the open stateid */
     memcpy(&open->stateid, &stateid, sizeof(stateid4));
-    open->layout = NULL;
+    open->do_close = TRUE;
+
+    if (open->delegation.state) {
+        nfs41_delegation_state *deleg = open->delegation.state;
+        if (deleg->revoked) {
+            /* update delegation state */
+            AcquireSRWLockExclusive(&deleg->lock);
+            if (delegation.type != OPEN_DELEGATE_READ &&
+                delegation.type != OPEN_DELEGATE_WRITE) {
+                eprintf("recover_open() got delegation type %u, "
+                    "expected %u\n", delegation.type, deleg->state.type);
+            } else {
+                memcpy(&deleg->state, &delegation, sizeof(open_delegation4));
+                deleg->revoked = FALSE;
+            }
+            ReleaseSRWLockExclusive(&deleg->lock);
+        }
+    } else /* granted a new delegation? */
+        nfs41_delegation_granted(session, &open->parent, &open->file,
+            &delegation, FALSE, &open->delegation.state);
     ReleaseSRWLockExclusive(&open->lock);
 out:
     return status;
@@ -155,6 +245,69 @@ static int recover_locks(
     return status;
 }
 
+/* delegation recovery via OPEN (requires corresponding CLOSE) */
+static int recover_delegation_open(
+    IN nfs41_session *session,
+    IN nfs41_delegation_state *deleg,
+    IN OUT bool_t *grace)
+{
+    state_owner4 owner;
+    open_delegation4 delegation = { 0 };
+    stateid_arg stateid;
+    uint32_t access = OPEN4_SHARE_ACCESS_READ;
+    uint32_t deny = OPEN4_SHARE_DENY_NONE;
+    enum open_delegation_type4 delegate_type = OPEN_DELEGATE_NONE;
+    int status = NFS4_OK;
+
+    /* choose the desired access mode based on delegation type */
+    AcquireSRWLockShared(&deleg->lock);
+    delegate_type = deleg->state.type;
+    if (delegate_type == OPEN_DELEGATE_WRITE)
+        access |= OPEN4_SHARE_ACCESS_WRITE | OPEN4_SHARE_ACCESS_WANT_WRITE_DELEG;
+    else
+        access |= OPEN4_SHARE_ACCESS_WANT_READ_DELEG;
+    ReleaseSRWLockShared(&deleg->lock);
+
+    /* construct a temporary open owner by concatenating the time
+     * in seconds with the delegation pointer */
+    time((time_t*)owner.owner);
+    memcpy(owner.owner + sizeof(time_t), deleg, sizeof(deleg));
+    owner.owner_len = sizeof(time_t) + sizeof(deleg);
+
+    if (*grace)
+        status = recover_open_grace(session, &deleg->parent, &deleg->file,
+            &owner, access, deny, delegate_type, &stateid.stateid, &delegation);
+    else
+        status = NFS4ERR_NO_GRACE;
+
+    if (status == NFS4ERR_NO_GRACE) {
+        *grace = FALSE;
+        status = recover_open_no_grace(session, &deleg->parent, &deleg->file,
+            &owner, access, deny, delegate_type, &stateid.stateid, &delegation);
+    }
+    if (status)
+        goto out;
+
+    /* update delegation state */
+    AcquireSRWLockExclusive(&deleg->lock);
+    if (delegation.type != OPEN_DELEGATE_READ &&
+        delegation.type != OPEN_DELEGATE_WRITE) {
+        eprintf("recover_delegation_open() got delegation type %u, "
+            "expected %u\n", delegation.type, deleg->state.type);
+    } else {
+        memcpy(&deleg->state, &delegation, sizeof(open_delegation4));
+        deleg->revoked = FALSE;
+    }
+    ReleaseSRWLockExclusive(&deleg->lock);
+
+    /* send CLOSE to free the open stateid */
+    stateid.open = NULL;
+    stateid.type = STATEID_OPEN;
+    nfs41_close(session, &deleg->file, &stateid);
+out:
+    return status;
+}
+
 int nfs41_recover_client_state(
     IN nfs41_session *session,
     IN nfs41_client *client)
@@ -164,19 +317,43 @@ int nfs41_recover_client_state(
     struct client_state *state = &session->client->state;
     struct list_entry *entry;
     nfs41_open_state *open;
+    nfs41_delegation_state *deleg;
     bool_t grace = TRUE;
     int status = NFS4_OK;
 
-    /* recover each of the client's opens */
     EnterCriticalSection(&state->lock);
+
+    /* flag all delegations as revoked until successful recovery;
+     * recover_open() and recover_delegation_open() will only ask
+     * for delegations when revoked = TRUE */
+    list_for_each(entry, &state->delegations) {
+        deleg = list_container(entry, nfs41_delegation_state, client_entry);
+        deleg->revoked = TRUE;
+    }
+
+    /* recover each of the client's opens and associated delegations */
     list_for_each(entry, &state->opens) {
         open = list_container(entry, nfs41_open_state, client_entry);
         status = recover_open(session, open, &grace);
         if (status == NFS4_OK)
             status = recover_locks(session, open, &grace);
         if (status == NFS4ERR_BADSESSION)
-            break;
+            goto unlock;
     }
+
+    /* recover delegations that weren't associated with any opens */
+    list_for_each(entry, &state->delegations) {
+        deleg = list_container(entry, nfs41_delegation_state, client_entry);
+        if (deleg->revoked) {
+            /* TODO: try WANT_DELEGATION (server support is optional) */
+            status = recover_delegation_open(session, deleg, &grace);
+            if (status == NFS4ERR_BADSESSION)
+                goto unlock;
+        }
+    }
+
+    /* TODO: return any delegations that were reclaimed as 'recalled' */
+unlock:
     LeaveCriticalSection(&state->lock);
 
     /* revoke all of the client's layouts */