data cache invalidation on delegation recalls

when we receive a delegation recall, we need to make a downcall
to the kernel and change a buffering/caching policy on this file.

on each open and close we pass an srv_open pointer to the nfsd to
keep in case it receives a cb_recall. we store srv_open in the
delegation state if we got a delegation.

daemon/delegation.c

@@ -30,6 +30,9 @@
 #include "util.h"
 #include "daemon_debug.h"
 
+#include <devioctl.h>
+#include "nfs41_driver.h" /* for making downcall to invalidate cache */
+#include "util.h"
 
 #define DGLVL 2 /* dprintf level for delegation logging */
 
@@ -273,6 +276,32 @@ static int delegation_return(
     nfs41_open_state *open;
     int status;
 
+    if (deleg->srv_open) {
+        /* make an upcall to the kernel: invalide data cache */
+        HANDLE pipe;
+        unsigned char inbuf[sizeof(HANDLE)], *buffer = inbuf; 
+        DWORD inbuf_len = sizeof(HANDLE), outbuf_len, dstatus;
+        uint32_t length;
+        dprintf(1, "delegation_return: making a downcall for srv_open=%x\n", 
+            deleg->srv_open);
+        pipe = CreateFile(NFS41_USER_DEVICE_NAME_A, GENERIC_READ|GENERIC_WRITE, 
+                FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_EXISTING, 0, NULL);
+        if (pipe == INVALID_HANDLE_VALUE) {
+            eprintf("delegation_return: Unable to open downcall pipe %d\n", 
+                GetLastError());
+            goto out_downcall;
+        }
+        length = inbuf_len;
+        safe_write(&buffer, &length, &deleg->srv_open, sizeof(HANDLE));
+
+        dstatus = DeviceIoControl(pipe, IOCTL_NFS41_INVALCACHE, inbuf, inbuf_len,
+            NULL, 0, (LPDWORD)&outbuf_len, NULL);
+        if (!dstatus)
+            eprintf("IOCTL_NFS41_INVALCACHE failed %d\n", GetLastError());
+        CloseHandle(pipe);
+    }
+out_downcall:
+
     /* recover opens and locks associated with the delegation */
     while (open = deleg_open_find(&client->state, deleg)) {
         status = nfs41_delegation_to_open(open, try_recovery);
@@ -481,6 +510,11 @@ int nfs41_delegate_open(
         stateid.type = STATEID_DELEG_FILE;
         memcpy(&stateid.stateid, &deleg->state.stateid, sizeof(stateid4));
     }
+    if (!status) {
+        dprintf(1, "nfs41_delegate_open: updating srv_open from %x to %x\n", 
+            deleg->srv_open, state->srv_open);
+        deleg->srv_open = state->srv_open;
+    }
     ReleaseSRWLockExclusive(&deleg->lock);
 
     if (status == NFS4ERR_DELEG_REVOKED)
@@ -581,6 +615,21 @@ out_unlock:
     return status;
 }
 
+void nfs41_delegation_remove_srvopen(
+    IN nfs41_session *session,
+    IN nfs41_path_fh *file)
+{
+    nfs41_delegation_state *deleg = NULL;
+
+    /* find a delegation for this file */
+    if (delegation_find(session->client, &file->fh, deleg_fh_cmp, &deleg) || !deleg)
+        return;
+    dprintf(1, "nfs41_delegation_remove_srvopen: removing reference to "
+        "srv_open=%x\n", deleg->srv_open);
+    AcquireSRWLockExclusive(&deleg->lock);
+    deleg->srv_open = NULL;
+    ReleaseSRWLockExclusive(&deleg->lock);
+}
 
 /* synchronous delegation return */
 #ifdef DELEGATION_RETURN_ON_CONFLICT

daemon/delegation.h

@@ -64,6 +64,9 @@ int nfs41_delegation_to_open(
     IN nfs41_open_state *open,
     IN bool_t try_recovery);
 
+void nfs41_delegation_remove_srvopen(
+    IN nfs41_session *session,
+    IN nfs41_path_fh *file);
 
 /* synchronous delegation return */
 #ifdef DELEGATION_RETURN_ON_CONFLICT

daemon/nfs41.h

@@ -102,6 +102,8 @@ typedef struct __nfs41_delegation_state {
     CONDITION_VARIABLE cond;
 
     bool_t revoked; /* for recovery, accessed under client.state.lock */
+
+    HANDLE srv_open; /* for rdbss cache invalidation */
 } nfs41_delegation_state;
 
 typedef struct __nfs41_lock_state {
@@ -148,6 +150,8 @@ typedef struct __nfs41_open_state {
         struct list_entry list;
         uint32_t counter;
     } locks;
+
+    HANDLE srv_open; /* for data cache invalidation */
 } nfs41_open_state;
 
 typedef struct __nfs41_rpc_clnt {

daemon/open.c

@@ -211,6 +211,11 @@ static int do_open(
     /* allocate delegation state and register it with the client */
     nfs41_delegation_granted(state->session, &state->parent,
         &state->file, &delegation, TRUE, &deleg_state);
+    if (deleg_state) {
+        deleg_state->srv_open = state->srv_open;
+        dprintf(1, "do_open: received delegation: saving srv_open = %x\n", 
+            state->srv_open);
+    }
 
     AcquireSRWLockExclusive(&state->lock);
     /* update the stateid */
@@ -270,12 +275,15 @@ static int parse_open(unsigned char *buffer, uint32_t length, nfs41_upcall *upca
     if (status) goto out;
     status = safe_read(&buffer, &length, &args->mode, sizeof(DWORD));
     if (status) goto out;
+    status = safe_read(&buffer, &length, &args->srv_open, sizeof(HANDLE));
+    if (status) goto out;
 
     dprintf(1, "parsing NFS41_OPEN: filename='%s' access mask=%d "
         "access mode=%d\n\tfile attrs=0x%x create attrs=0x%x "
-        "(kernel) disposition=%d\n\topen_owner_id=%d mode=%o\n", 
-        args->path, args->access_mask, args->access_mode, args->file_attrs,
-        args->create_opts, args->disposition, args->open_owner_id, args->mode);
+        "(kernel) disposition=%d\n\topen_owner_id=%d mode=%o "
+        "srv_open=%x\n", args->path, args->access_mask, args->access_mode, 
+        args->file_attrs, args->create_opts, args->disposition, 
+        args->open_owner_id, args->mode, args->srv_open);
     print_disposition(2, args->disposition);
     print_access_mask(2, args->access_mask);
     print_share_mode(2, args->access_mode);
@@ -432,6 +440,7 @@ static int handle_open(nfs41_upcall *upcall)
             args->open_owner_id, status);
         goto out;
     }
+    state->srv_open = args->srv_open;
 
     // first check if windows told us it's a directory
     if (args->create_opts & FILE_DIRECTORY_FILE)
@@ -700,6 +709,8 @@ static int parse_close(unsigned char *buffer, uint32_t length, nfs41_upcall *upc
 
     status = safe_read(&buffer, &length, &args->remove, sizeof(BOOLEAN));
     if (status) goto out;
+    status = safe_read(&buffer, &length, &args->srv_open, sizeof(HANDLE));
+    if (status) goto out;
     if (args->remove) {
         status = get_name(&buffer, &length, &args->path);
         if (status) goto out;
@@ -707,8 +718,9 @@ static int parse_close(unsigned char *buffer, uint32_t length, nfs41_upcall *upc
         if (status) goto out;
     }
 
-    dprintf(1, "parsing NFS41_CLOSE: remove=%d renamed=%d filename='%s'\n",
-        args->remove, args->renamed, args->remove ? args->path : "");
+    dprintf(1, "parsing NFS41_CLOSE: remove=%d srv_open=%x renamed=%d "
+        "filename='%s'\n", args->remove, args->srv_open, args->renamed, 
+        args->remove ? args->path : "");
 out:
     return status;
 }
@@ -742,6 +754,9 @@ static int handle_close(nfs41_upcall *upcall)
     if (state->type == NF4REG)
         pnfs_layout_state_close(state->session, state, args->remove);
 
+    if (state->srv_open == args->srv_open)
+        nfs41_delegation_remove_srvopen(state->session, &state->file);
+
     if (args->remove) {
         nfs41_component *name = &state->file.name;
 

daemon/upcall.h

@@ -54,11 +54,13 @@ typedef struct __open_upcall_args {
     LONG open_owner_id;
     DWORD mode;
     LONGLONG changeattr;
+    HANDLE srv_open;
     BOOLEAN created;
     BOOLEAN symlink_embedded;
 } open_upcall_args;
 
 typedef struct __close_upcall_args {
+    HANDLE srv_open;
     const char *path;
     BOOLEAN remove;
     BOOLEAN renamed;