recovery: recover from delegation stateid errors

delegation stateid arguments to DELEGRETURN and OPEN are now stateid_arg, for use with recover_stateid_delegation().  added a nfs41_delegation_state pointer to stateid_arg, for when a delegation stateid is used in the absence of nfs41_open_state (DELEGRETURN, SETATTR)

recovery during a call to nfs41_delegation_to_open() requires special attention; recover_stateid_delegation() has to handle the case where recover_open() already reclaimed the open stateid.  it does this by returning BAD_STATEID instead of retrying the OPEN (which would generate yet another open stateid)

Signed-off-by: Casey Bodley <cbodley@citi.umich.edu>

daemon/delegation.c

@@ -128,6 +128,7 @@ static void delegation_return(
     IN nfs41_delegation_state *deleg,
     IN bool_t truncate)
 {
+    stateid_arg stateid;
     struct list_entry *entry;
     nfs41_open_state *open;
 
@@ -139,8 +140,15 @@ static void delegation_return(
 
     /* TODO: flush data and metadata before returning delegation */
 
-    nfs41_delegreturn(client->session, &deleg->file,
+    /* return the delegation */
-        &deleg->state.stateid, TRUE);
+    stateid.type = STATEID_DELEG_FILE;
+    stateid.open = NULL;
+    stateid.delegation = deleg;
+    AcquireSRWLockShared(&deleg->lock);
+    memcpy(&stateid.stateid, &deleg->state.stateid, sizeof(stateid4));
+    ReleaseSRWLockShared(&deleg->lock);
+
+    nfs41_delegreturn(client->session, &deleg->file, &stateid, TRUE);
 
     /* remove from the client's list */
     EnterCriticalSection(&client->state.lock);
@@ -178,6 +186,7 @@ int nfs41_delegation_granted(
     IN bool_t try_recovery,
     OUT nfs41_delegation_state **deleg_out)
 {
+    stateid_arg stateid;
     nfs41_client *client = session->client;
     nfs41_delegation_state *state;
     int status = NO_ERROR;
@@ -206,7 +215,11 @@ out:
     return status;
 
 out_return: /* return the delegation on failure */
-    nfs41_delegreturn(session, file, &delegation->stateid, try_recovery);
+    memcpy(&stateid.stateid, &delegation->stateid, sizeof(stateid4));
+    stateid.type = STATEID_DELEG_FILE;
+    stateid.open = NULL;
+    stateid.delegation = NULL;
+    nfs41_delegreturn(session, file, &stateid, try_recovery);
     goto out;
 }
 
@@ -332,6 +345,7 @@ int nfs41_delegate_open(
     } else if (create == OPEN4_CREATE) {
         /* copy the stateid for SETATTR */
         stateid.open = NULL;
+        stateid.delegation = deleg;
         stateid.type = STATEID_DELEG_FILE;
         memcpy(&stateid.stateid, &deleg->state.stateid, sizeof(stateid4));
     }
@@ -370,7 +384,8 @@ int nfs41_delegation_to_open(
 {
     open_delegation4 ignore;
     open_claim4 claim;
-    stateid4 deleg_stateid, open_stateid = { 0 };
+    stateid4 open_stateid = { 0 };
+    stateid_arg deleg_stateid;
     int status = NFS4_OK;
 
     AcquireSRWLockExclusive(&open->lock);
@@ -387,12 +402,16 @@ int nfs41_delegation_to_open(
             SleepConditionVariableSRW(&open->delegation.cond, &open->lock,
                 INFINITE, 0);
         } while (open->delegation.reclaim);
-        goto out_unlock;
+        if (open->do_close)
+            goto out_unlock;
     }
     open->delegation.reclaim = 1;
 
     AcquireSRWLockShared(&open->delegation.state->lock);
-    memcpy(&deleg_stateid, &open->delegation.state->state.stateid,
+    deleg_stateid.open = open;
+    deleg_stateid.delegation = NULL;
+    deleg_stateid.type = STATEID_DELEG_FILE;
+    memcpy(&deleg_stateid.stateid, &open->delegation.state->state.stateid,
         sizeof(stateid4));
     ReleaseSRWLockShared(&open->delegation.state->lock);
 
@@ -406,20 +425,26 @@ int nfs41_delegation_to_open(
     status = nfs41_open(open->session, &open->parent, &open->file,
         &open->owner, &claim, open->share_access, open->share_deny,
         OPEN4_NOCREATE, 0, 0, try_recovery, &open_stateid, &ignore, NULL);
-    if (status)
-        eprintf("nfs41_delegation_to_open(%p) failed with %s\n",
-            open, nfs_error_string(status));
 
     AcquireSRWLockExclusive(&open->lock);
-    /* save the new open stateid */
+    if (status == NFS4_OK) {
-    memcpy(&open->stateid, &open_stateid, sizeof(stateid4));
+        /* save the new open stateid */
+        memcpy(&open->stateid, &open_stateid, sizeof(stateid4));
+        open->do_close = 1;
+    } else if (status == NFS4ERR_BAD_STATEID && open->do_close) {
+        /* something triggered client state recovery, and the open stateid
+         * has already been reclaimed; see recover_stateid_delegation() */
+        status = NFS4_OK;
+    }
     open->delegation.reclaim = 0;
-    open->do_close = 1;
 
     /* signal anyone waiting on the open stateid */
     WakeAllConditionVariable(&open->delegation.cond);
 out_unlock:
     ReleaseSRWLockExclusive(&open->lock);
+    if (status)
+        eprintf("nfs41_delegation_to_open(%p) failed with %s\n",
+            open, nfs_error_string(status));
     return status;
 }
 

daemon/lock.c

@@ -39,6 +39,7 @@ static void lock_stateid_arg(
     OUT stateid_arg *arg)
 {
     arg->open = state;
+    arg->delegation = NULL;
 
     /* open_to_lock_owner4 requires an open stateid; if we
      * have a delegation, convert it to an open stateid */

daemon/nfs41_ops.c

@@ -1530,7 +1530,7 @@ out:
 int nfs41_delegreturn(
     IN nfs41_session *session,
     IN nfs41_path_fh *file,
-    IN stateid4 *stateid,
+    IN stateid_arg *stateid,
     IN bool_t try_recovery)
 {
     int status;

daemon/nfs41_ops.h

@@ -287,6 +287,7 @@ typedef struct __stateid_arg {
     stateid4                stateid;
     enum stateid_type       type;
     nfs41_open_state        *open;
+    nfs41_delegation_state  *delegation;
 } stateid_arg;
 
 
@@ -374,7 +375,7 @@ typedef struct __nfs41_create_res {
 
 /* OP_DELEGRETURN */
 typedef struct __nfs41_delegreturn_args {
-    stateid4                *stateid;
+    stateid_arg             *stateid;
 } nfs41_delegreturn_args;
 
 typedef struct __nfs41_delegreturn_res {
@@ -624,12 +625,12 @@ typedef struct __open_claim4 {
         } prev;
     /* case CLAIM_DELEGATE_CUR: */
         struct __open_claim_deleg_cur {
-            stateid4        *delegate_stateid;
+            stateid_arg     *delegate_stateid;
             nfs41_component *name;
         } deleg_cur;
     /* case CLAIM_DELEG_CUR_FH: */
         struct __open_claim_deleg_cur_fh {
-            stateid4        *delegate_stateid;
+            stateid_arg     *delegate_stateid;
         } deleg_cur_fh;
     /* case CLAIM_DELEGATE_PREV: */
         struct __open_claim_deleg_prev {
@@ -1154,7 +1155,7 @@ int nfs41_access(
 int nfs41_delegreturn(
     IN nfs41_session *session,
     IN nfs41_path_fh *file,
-    IN stateid4 *stateid,
+    IN stateid_arg *stateid,
     IN bool_t try_recovery);
 
 enum nfsstat4 nfs41_fs_locations(

daemon/nfs41_xdr.c

@@ -1644,7 +1644,7 @@ static bool_t encode_op_delegreturn(
     if (unexpected_op(argop->op, OP_DELEGRETURN))
         return FALSE;
 
-    return xdr_stateid4(xdr, args->stateid);
+    return xdr_stateid4(xdr, &args->stateid->stateid);
 }
 
 static bool_t decode_op_delegreturn(
@@ -1930,9 +1930,11 @@ static bool_t encode_open_claim4(
         return TRUE; /* use current file handle */
     case CLAIM_DELEGATE_CUR:
         return encode_claim_deleg_cur(xdr,
-            oc->u.deleg_cur.delegate_stateid, oc->u.deleg_cur.name);
+            &oc->u.deleg_cur.delegate_stateid->stateid,
+            oc->u.deleg_cur.name);
     case CLAIM_DELEG_CUR_FH:
-        return xdr_stateid4(xdr, oc->u.deleg_cur_fh.delegate_stateid);
+        return xdr_stateid4(xdr,
+            &oc->u.deleg_cur_fh.delegate_stateid->stateid);
     case CLAIM_DELEGATE_PREV:
         return encode_component(xdr, oc->u.deleg_prev.filename);
     case CLAIM_DELEG_PREV_FH:

daemon/open.c

@@ -127,6 +127,7 @@ void nfs41_open_stateid_arg(
     OUT stateid_arg *arg)
 {
     arg->open = state;
+    arg->delegation = NULL;
 
     AcquireSRWLockShared(&state->lock);
 
@@ -659,6 +660,7 @@ static void cancel_open(IN nfs41_upcall *upcall)
     if (state->do_close) {
         stateid_arg stateid;
         stateid.open = state;
+        stateid.delegation = NULL;
         stateid.type = STATEID_OPEN;
         memcpy(&stateid.stateid, &state->stateid, sizeof(stateid4));
 
@@ -742,6 +744,7 @@ static int handle_close(nfs41_upcall *upcall)
     if (state->do_close) {
         stateid_arg stateid;
         stateid.open = state;
+        stateid.delegation = NULL;
         stateid.type = STATEID_OPEN;
         memcpy(&stateid.stateid, &state->stateid, sizeof(stateid4));
 

daemon/recovery.c

@@ -212,6 +212,7 @@ static int recover_locks(
     memcpy(&stateid.stateid, &open->stateid, sizeof(stateid4));
     stateid.type = STATEID_OPEN;
     stateid.open = open;
+    stateid.delegation = NULL;
 
     /* recover any locks for this open */
     list_for_each(entry, &open->locks.list) {
@@ -302,6 +303,7 @@ static int recover_delegation_open(
 
     /* send CLOSE to free the open stateid */
     stateid.open = NULL;
+    stateid.delegation = NULL;
     stateid.type = STATEID_OPEN;
     nfs41_close(session, &deleg->file, &stateid);
 out:
@@ -418,13 +420,56 @@ static bool_t recover_stateid_lock(
     return retry;
 }
 
+static bool_t recover_stateid_delegation(
+    IN nfs_argop4 *argop,
+    IN stateid_arg *stateid)
+{
+    bool_t retry = FALSE;
+
+    if (stateid->open) {
+        /* if the source stateid is different, update and retry */
+        AcquireSRWLockShared(&stateid->open->lock);
+        if (argop->op == OP_OPEN && stateid->open->do_close) {
+            /* for nfs41_delegation_to_open(); if we've already reclaimed
+            * an open stateid, just fail this OPEN with BAD_STATEID */
+        } else if (stateid->open->delegation.state) {
+            nfs41_delegation_state *deleg = stateid->open->delegation.state;
+            stateid4 *source = &deleg->state.stateid;
+            AcquireSRWLockShared(&deleg->lock);
+            if (memcmp(&stateid->stateid, source, sizeof(stateid4))) {
+                memcpy(&stateid->stateid, source, sizeof(stateid4));
+                retry = TRUE;
+            }
+            ReleaseSRWLockShared(&deleg->lock);
+        }
+        ReleaseSRWLockShared(&stateid->open->lock);
+    } else if (stateid->delegation) {
+        nfs41_delegation_state *deleg = stateid->delegation;
+        stateid4 *source = &deleg->state.stateid;
+        AcquireSRWLockShared(&deleg->lock);
+        if (memcmp(&stateid->stateid, source, sizeof(stateid4))) {
+            memcpy(&stateid->stateid, source, sizeof(stateid4));
+            retry = TRUE;
+        }
+        ReleaseSRWLockShared(&deleg->lock);
+    }
+    return retry;
+}
+
 bool_t nfs41_recover_stateid(
     IN nfs41_session *session,
     IN nfs_argop4 *argop)
 {
     stateid_arg *stateid = NULL;
 
-    if (argop->op == OP_CLOSE) {
+    /* get the stateid_arg from the operation's arguments */
+    if (argop->op == OP_OPEN) {
+        nfs41_op_open_args *open = (nfs41_op_open_args*)argop->arg;
+        if (open->claim->claim == CLAIM_DELEGATE_CUR)
+            stateid = open->claim->u.deleg_cur.delegate_stateid;
+        else if (open->claim->claim == CLAIM_DELEG_CUR_FH)
+            stateid = open->claim->u.deleg_cur_fh.delegate_stateid;
+    } else if (argop->op == OP_CLOSE) {
         nfs41_op_close_args *close = (nfs41_op_close_args*)argop->arg;
         stateid = close->stateid;
     } else if (argop->op == OP_READ) {
@@ -448,6 +493,9 @@ bool_t nfs41_recover_stateid(
     } else if (argop->op == OP_LAYOUTGET) {
         pnfs_layoutget_args *lget = (pnfs_layoutget_args*)argop->arg;
         stateid = lget->stateid;
+    } else if (argop->op == OP_DELEGRETURN) {
+        nfs41_delegreturn_args *dr = (nfs41_delegreturn_args*)argop->arg;
+        stateid = dr->stateid;
     }
     if (stateid == NULL)
         return FALSE;
@@ -466,6 +514,9 @@ bool_t nfs41_recover_stateid(
     case STATEID_LOCK:
         return recover_stateid_lock(argop, stateid);
 
+    case STATEID_DELEG_FILE:
+        return recover_stateid_delegation(argop, stateid);
+
     default:
         eprintf("%s can't recover stateid type %u\n",
             nfs_opnum_to_string(argop->op), stateid->type);