data cache invalidation on delegation recalls

when we receive a delegation recall, we need to make a downcall
to the kernel and change a buffering/caching policy on this file.

on each open and close we pass an srv_open pointer to the nfsd to
keep in case it receives a cb_recall. we store srv_open in the
delegation state if we got a delegation.

daemon/delegation.c

@@ -30,6 +30,9 @@
 #include "util.h"
 #include "daemon_debug.h"
 
+#include <devioctl.h>
+#include "nfs41_driver.h" /* for making downcall to invalidate cache */
+#include "util.h"
 
 #define DGLVL 2 /* dprintf level for delegation logging */
 
@@ -273,6 +276,32 @@ static int delegation_return(
     nfs41_open_state *open;
     int status;
 
+    if (deleg->srv_open) {
+        /* make an upcall to the kernel: invalide data cache */
+        HANDLE pipe;
+        unsigned char inbuf[sizeof(HANDLE)], *buffer = inbuf; 
+        DWORD inbuf_len = sizeof(HANDLE), outbuf_len, dstatus;
+        uint32_t length;
+        dprintf(1, "delegation_return: making a downcall for srv_open=%x\n", 
+            deleg->srv_open);
+        pipe = CreateFile(NFS41_USER_DEVICE_NAME_A, GENERIC_READ|GENERIC_WRITE, 
+                FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_EXISTING, 0, NULL);
+        if (pipe == INVALID_HANDLE_VALUE) {
+            eprintf("delegation_return: Unable to open downcall pipe %d\n", 
+                GetLastError());
+            goto out_downcall;
+        }
+        length = inbuf_len;
+        safe_write(&buffer, &length, &deleg->srv_open, sizeof(HANDLE));
+
+        dstatus = DeviceIoControl(pipe, IOCTL_NFS41_INVALCACHE, inbuf, inbuf_len,
+            NULL, 0, (LPDWORD)&outbuf_len, NULL);
+        if (!dstatus)
+            eprintf("IOCTL_NFS41_INVALCACHE failed %d\n", GetLastError());
+        CloseHandle(pipe);
+    }
+out_downcall:
+
     /* recover opens and locks associated with the delegation */
     while (open = deleg_open_find(&client->state, deleg)) {
         status = nfs41_delegation_to_open(open, try_recovery);
@@ -481,6 +510,11 @@ int nfs41_delegate_open(
         stateid.type = STATEID_DELEG_FILE;
         memcpy(&stateid.stateid, &deleg->state.stateid, sizeof(stateid4));
     }
+    if (!status) {
+        dprintf(1, "nfs41_delegate_open: updating srv_open from %x to %x\n", 
+            deleg->srv_open, state->srv_open);
+        deleg->srv_open = state->srv_open;
+    }
     ReleaseSRWLockExclusive(&deleg->lock);
 
     if (status == NFS4ERR_DELEG_REVOKED)
@@ -581,6 +615,21 @@ out_unlock:
     return status;
 }
 
+void nfs41_delegation_remove_srvopen(
+    IN nfs41_session *session,
+    IN nfs41_path_fh *file)
+{
+    nfs41_delegation_state *deleg = NULL;
+
+    /* find a delegation for this file */
+    if (delegation_find(session->client, &file->fh, deleg_fh_cmp, &deleg) || !deleg)
+        return;
+    dprintf(1, "nfs41_delegation_remove_srvopen: removing reference to "
+        "srv_open=%x\n", deleg->srv_open);
+    AcquireSRWLockExclusive(&deleg->lock);
+    deleg->srv_open = NULL;
+    ReleaseSRWLockExclusive(&deleg->lock);
+}
 
 /* synchronous delegation return */
 #ifdef DELEGATION_RETURN_ON_CONFLICT

daemon/delegation.h

@@ -64,6 +64,9 @@ int nfs41_delegation_to_open(
     IN nfs41_open_state *open,
     IN bool_t try_recovery);
 
+void nfs41_delegation_remove_srvopen(
+    IN nfs41_session *session,
+    IN nfs41_path_fh *file);
 
 /* synchronous delegation return */
 #ifdef DELEGATION_RETURN_ON_CONFLICT

daemon/nfs41.h

@@ -102,6 +102,8 @@ typedef struct __nfs41_delegation_state {
     CONDITION_VARIABLE cond;
 
     bool_t revoked; /* for recovery, accessed under client.state.lock */
+
+    HANDLE srv_open; /* for rdbss cache invalidation */
 } nfs41_delegation_state;
 
 typedef struct __nfs41_lock_state {
@@ -148,6 +150,8 @@ typedef struct __nfs41_open_state {
         struct list_entry list;
         uint32_t counter;
     } locks;
+
+    HANDLE srv_open; /* for data cache invalidation */
 } nfs41_open_state;
 
 typedef struct __nfs41_rpc_clnt {

daemon/open.c

@@ -211,6 +211,11 @@ static int do_open(
     /* allocate delegation state and register it with the client */
     nfs41_delegation_granted(state->session, &state->parent,
         &state->file, &delegation, TRUE, &deleg_state);
+    if (deleg_state) {
+        deleg_state->srv_open = state->srv_open;
+        dprintf(1, "do_open: received delegation: saving srv_open = %x\n", 
+            state->srv_open);
+    }
 
     AcquireSRWLockExclusive(&state->lock);
     /* update the stateid */
@@ -270,12 +275,15 @@ static int parse_open(unsigned char *buffer, uint32_t length, nfs41_upcall *upca
     if (status) goto out;
     status = safe_read(&buffer, &length, &args->mode, sizeof(DWORD));
     if (status) goto out;
+    status = safe_read(&buffer, &length, &args->srv_open, sizeof(HANDLE));
+    if (status) goto out;
 
     dprintf(1, "parsing NFS41_OPEN: filename='%s' access mask=%d "
         "access mode=%d\n\tfile attrs=0x%x create attrs=0x%x "
-        "(kernel) disposition=%d\n\topen_owner_id=%d mode=%o\n", 
+        "(kernel) disposition=%d\n\topen_owner_id=%d mode=%o "
-        args->path, args->access_mask, args->access_mode, args->file_attrs,
+        "srv_open=%x\n", args->path, args->access_mask, args->access_mode, 
-        args->create_opts, args->disposition, args->open_owner_id, args->mode);
+        args->file_attrs, args->create_opts, args->disposition, 
+        args->open_owner_id, args->mode, args->srv_open);
     print_disposition(2, args->disposition);
     print_access_mask(2, args->access_mask);
     print_share_mode(2, args->access_mode);
@@ -432,6 +440,7 @@ static int handle_open(nfs41_upcall *upcall)
             args->open_owner_id, status);
         goto out;
     }
+    state->srv_open = args->srv_open;
 
     // first check if windows told us it's a directory
     if (args->create_opts & FILE_DIRECTORY_FILE)
@@ -700,6 +709,8 @@ static int parse_close(unsigned char *buffer, uint32_t length, nfs41_upcall *upc
 
     status = safe_read(&buffer, &length, &args->remove, sizeof(BOOLEAN));
     if (status) goto out;
+    status = safe_read(&buffer, &length, &args->srv_open, sizeof(HANDLE));
+    if (status) goto out;
     if (args->remove) {
         status = get_name(&buffer, &length, &args->path);
         if (status) goto out;
@@ -707,8 +718,9 @@ static int parse_close(unsigned char *buffer, uint32_t length, nfs41_upcall *upc
         if (status) goto out;
     }
 
-    dprintf(1, "parsing NFS41_CLOSE: remove=%d renamed=%d filename='%s'\n",
+    dprintf(1, "parsing NFS41_CLOSE: remove=%d srv_open=%x renamed=%d "
-        args->remove, args->renamed, args->remove ? args->path : "");
+        "filename='%s'\n", args->remove, args->srv_open, args->renamed, 
+        args->remove ? args->path : "");
 out:
     return status;
 }
@@ -742,6 +754,9 @@ static int handle_close(nfs41_upcall *upcall)
     if (state->type == NF4REG)
         pnfs_layout_state_close(state->session, state, args->remove);
 
+    if (state->srv_open == args->srv_open)
+        nfs41_delegation_remove_srvopen(state->session, &state->file);
+
     if (args->remove) {
         nfs41_component *name = &state->file.name;
 

daemon/upcall.h

@@ -54,11 +54,13 @@ typedef struct __open_upcall_args {
     LONG open_owner_id;
     DWORD mode;
     LONGLONG changeattr;
+    HANDLE srv_open;
     BOOLEAN created;
     BOOLEAN symlink_embedded;
 } open_upcall_args;
 
 typedef struct __close_upcall_args {
+    HANDLE srv_open;
     const char *path;
     BOOLEAN remove;
     BOOLEAN renamed;

sys/nfs41_driver.c

@@ -174,10 +174,12 @@ typedef struct _updowncall_entry {
             LONG open_owner_id;
             DWORD mode;
             LONGLONG changeattr;
+            HANDLE srv_open;
             BOOLEAN symlink_embedded;
         } Open;
         struct {
             PUNICODE_STRING filename;
+            HANDLE srv_open;
             BOOLEAN remove;
             BOOLEAN renamed;
         } Close;
@@ -615,7 +617,7 @@ NTSTATUS marshal_nfs41_open(
     else 
         tmp += *len;
     header_len = *len + length_as_ansi(entry->u.Open.filename) +
-        5 * sizeof(ULONG) + sizeof(LONG) + sizeof(DWORD);
+        5 * sizeof(ULONG) + sizeof(LONG) + sizeof(DWORD) + sizeof(HANDLE);
     if (header_len > buf_len) { 
         status = STATUS_INSUFFICIENT_RESOURCES;
         goto out;
@@ -638,14 +640,17 @@ NTSTATUS marshal_nfs41_open(
         sizeof(entry->u.Open.open_owner_id));
     tmp += sizeof(entry->u.Open.open_owner_id);
     RtlCopyMemory(tmp, &entry->u.Open.mode, sizeof(DWORD));
+    tmp += sizeof(DWORD);
+    RtlCopyMemory(tmp, &entry->u.Open.srv_open, sizeof(HANDLE));
 
     *len = header_len;
 
     DbgP("marshal_nfs41_open: mask=0x%x mode=0x%x attrs=0x%x opts=0x%x "
-         "dispo=0x%x open_owner_id=0x%x mode=%o\n", entry->u.Open.access_mask, 
+         "dispo=0x%x open_owner_id=0x%x mode=%o srv_open=%p\n", 
-         entry->u.Open.access_mode, entry->u.Open.attrs, 
+         entry->u.Open.access_mask, entry->u.Open.access_mode, 
-         entry->u.Open.copts, entry->u.Open.disp,
+         entry->u.Open.attrs, entry->u.Open.copts, entry->u.Open.disp,
-         entry->u.Open.open_owner_id, entry->u.Open.mode); 
+         entry->u.Open.open_owner_id, entry->u.Open.mode,
+         entry->u.Open.srv_open); 
 out:
     DbgEx();
     return status;
@@ -812,7 +817,7 @@ NTSTATUS marshal_nfs41_close(
         tmp += *len;
 
 
-    header_len = *len + sizeof(BOOLEAN);
+    header_len = *len + sizeof(BOOLEAN) + sizeof(HANDLE);
     if (entry->u.Close.remove)
         header_len += length_as_ansi(entry->u.Close.filename) +
             sizeof(BOOLEAN);
@@ -822,8 +827,10 @@ NTSTATUS marshal_nfs41_close(
         goto out;
     }
     RtlCopyMemory(tmp, &entry->u.Close.remove, sizeof(BOOLEAN));
+    tmp += sizeof(BOOLEAN);
+    RtlCopyMemory(tmp, &entry->u.Close.srv_open, sizeof(HANDLE));
     if (entry->u.Close.remove) {
-        tmp += sizeof(BOOLEAN);
+        tmp += sizeof(HANDLE);
         status = marshall_unicode_as_ansi(&tmp, entry->u.Close.filename);
         if (status) goto out;
         RtlCopyMemory(tmp, &entry->u.Close.renamed, sizeof(BOOLEAN));
@@ -831,8 +838,9 @@ NTSTATUS marshal_nfs41_close(
 
     *len = header_len;
 
-    DbgP("marshal_nfs41_close: remove=%d renamed=%d filename=%wZ\n", 
+    DbgP("marshal_nfs41_close: remove=%d srv_open=%p renamed=%d "
-         entry->u.Close.remove, entry->u.Close.renamed, entry->u.Close.filename);
+        "filename=%wZ\n", entry->u.Close.remove, entry->u.Close.srv_open, 
+        entry->u.Close.renamed, entry->u.Close.filename);
 out:
     DbgEx();
     return status;
@@ -1229,7 +1237,17 @@ NTSTATUS nfs41_invalidate_cache (
     IN PRX_CONTEXT RxContext)
 {
     NTSTATUS status = STATUS_SUCCESS;
+    PLOWIO_CONTEXT LowIoContext = &RxContext->LowIoContext;
+    unsigned char *buf = LowIoContext->ParamsFor.IoCtl.pInputBuffer;
+    ULONG flag = DISABLE_CACHING;
+    PMRX_SRV_OPEN srv_open;
+
     DbgEn();
+    RtlCopyMemory(&srv_open, buf, sizeof(HANDLE));
+
+    DbgP("nfs41_invalidate_cache: received srv_open=%p\n", srv_open);
+    if (MmIsAddressValid(srv_open))
+        RxChangeBufferingState((PSRV_OPEN)srv_open, ULongToPtr(flag), 1);
     DbgEx();
     return status;
 }
@@ -3253,6 +3271,7 @@ NTSTATUS nfs41_Create(
     entry->u.Open.attrs = params.FileAttributes;
     entry->u.Open.disp = params.Disposition;
     entry->u.Open.copts = params.CreateOptions;
+    entry->u.Open.srv_open = SrvOpen;
     if (isDataAccess(params.DesiredAccess))
         entry->u.Open.open_owner_id = InterlockedIncrement(&open_owner_id);
     // if we are creating a file check if nfsv3attributes were passed in
@@ -3581,6 +3600,7 @@ NTSTATUS nfs41_CloseSrvOpen(
         pNetRootContext->nfs41d_version, &entry);
     if (status)
         goto out;
+    entry->u.Close.srv_open = SrvOpen;
     if (!RxContext->pFcb->OpenCount) {
         entry->u.Close.remove = nfs41_fcb->StandardInfo.DeletePending;
         entry->u.Close.renamed = nfs41_fcb->Renamed;