Add AddWith() to pass options, allow controlling Windows buffer size (#521)

This is similar to Add(), except that you can pass options. Ideally this
should just be Add(), but that's not a compatible change, so we're stuck
with this until we do a v2.

There are quite a few enhancements that depend on *some* way to pass
options; as an example I added WithBufferSize() to control the buffer
size for (see #72) for the Windows backend, because that one is fairly
trivial to implement:

	w, err := fsnotify.NewWatcher()
	err = w.AddWith("/path", fsnotify.WithBufferSize(65536*4))

Some other options we might want to add:

	err = w.AddWith("/path",
		fsnotify.WithEvents(fsnotify.Open | fsnotify.Close),  // Filter events
		fsnotify.WithPoll(time.Second),                       // Use poll watcher
		fsnotify.WithFanotify(),                              // Prefer fanotify on Linux
		fsnotify.WithFollowLinks(true),                       // Control symlink follow behaviour
		fsnotify.WithDebounce(100*time.Milliseconds),         // Automatically debounce duplicate events
		fsnotify.WithRetry(1*time.Second, 1*time.Minute),     // Retry every second if the path disappears for a minute
	)

These are just some ideas, nothing fixed here yet. Some of these options
are likely to change once I get around to actually working on it.

This uses "functional options" so we can add more later. Options are
passed to Add() rather than the Watcher itself, so the behaviour can be
modified for every watch, rather than being global. This way you can do
things like watch /nfs-drive with a poll backend, and use the regular OS
backend for ~/dir, without having to create two watchers.

This upgrades fairly nicely to v2 where we rename AddWith() to Add():

	err = w.Add("/path",
		fsnotify.WithBufferSize(65536*4),
		fsnotify.WithEvents(fsnotify.Open | fsnotify.Close))

Folks will just have to s/fsnotify.AddWith/fsnotify.Add/, without having
to change all the option names. Plus having a consistent prefix
autocompletes nicely in editors.

Fixes #72

diff --git a/backend_fen.go b/backend_fen.go
index 95fbe99a9bf7e511cf82476db2383d1ece53389e..1f4dcf778d6b0dff6bbb1157af786fcd7ca7d623 100644 (file)
--- a/backend_fen.go
+++ b/backend_fen.go
@@ -105,7 +105,7 @@ type Watcher struct {
        // [ErrEventOverflow] is used to indicate there are too many events:
        //
        //  - inotify: there are too many queued events (fs.inotify.max_queued_events sysctl)
-       //  - windows: The buffer size is too small.
+       //  - windows: The buffer size is too small; [WithBufferSize] can be used to increase it.
        //  - kqueue, fen: not used.
        Errors chan error
 
@@ -195,6 +195,8 @@ func (w *Watcher) Close() error {
 //
 // Returns [ErrClosed] if [Watcher.Close] was called.
 //
+// See [AddWith] for a version that allows adding options.
+//
 // # Watching directories
 //
 // All files in a directory are monitored, including new files that are created
@@ -212,7 +214,16 @@ func (w *Watcher) Close() error {
 //
 // Instead, watch the parent directory and use Event.Name to filter out files
 // you're not interested in. There is an example of this in [cmd/fsnotify/file.go].
-func (w *Watcher) Add(name string) error {
+func (w *Watcher) Add(name string) error { return w.AddWith(name) }
+
+// AddWith is like [Add], but allows adding options. When using Add() the
+// defaults described below are used.
+//
+// Possible options are:
+//
+//   - [WithBufferSize] sets the buffer size for the Windows backend; no-op on
+//     other platforms. The default is 64K (65536 bytes).
+func (w *Watcher) AddWith(name string, opts ...addOpt) error {
        if w.isClosed() {
                return ErrClosed
        }
@@ -220,6 +231,8 @@ func (w *Watcher) Add(name string) error {
                return nil
        }
 
+       _ = getOptions(opts...)
+
        // Currently we resolve symlinks that were explicitly requested to be
        // watched. Otherwise we would use LStat here.
        stat, err := os.Stat(name)

diff --git a/backend_inotify.go b/backend_inotify.go
index a96c3adf349f6903127163a20b56c5a5174fa178..16ac36e127edabf12fe3119072d5e7cf8e9f8f56 100644 (file)
--- a/backend_inotify.go
+++ b/backend_inotify.go
@@ -108,7 +108,7 @@ type Watcher struct {
        // [ErrEventOverflow] is used to indicate there are too many events:
        //
        //  - inotify: there are too many queued events (fs.inotify.max_queued_events sysctl)
-       //  - windows: The buffer size is too small.
+       //  - windows: The buffer size is too small; [WithBufferSize] can be used to increase it.
        //  - kqueue, fen: not used.
        Errors chan error
 
@@ -217,6 +217,8 @@ func (w *Watcher) Close() error {
 //
 // Returns [ErrClosed] if [Watcher.Close] was called.
 //
+// See [AddWith] for a version that allows adding options.
+//
 // # Watching directories
 //
 // All files in a directory are monitored, including new files that are created
@@ -234,12 +236,23 @@ func (w *Watcher) Close() error {
 //
 // Instead, watch the parent directory and use Event.Name to filter out files
 // you're not interested in. There is an example of this in [cmd/fsnotify/file.go].
-func (w *Watcher) Add(name string) error {
-       name = filepath.Clean(name)
+func (w *Watcher) Add(name string) error { return w.AddWith(name) }
+
+// AddWith is like [Add], but allows adding options. When using Add() the
+// defaults described below are used.
+//
+// Possible options are:
+//
+//   - [WithBufferSize] sets the buffer size for the Windows backend; no-op on
+//     other platforms. The default is 64K (65536 bytes).
+func (w *Watcher) AddWith(name string, opts ...addOpt) error {
        if w.isClosed() {
                return ErrClosed
        }
 
+       name = filepath.Clean(name)
+       _ = getOptions(opts...)
+
        var flags uint32 = unix.IN_MOVED_TO | unix.IN_MOVED_FROM |
                unix.IN_CREATE | unix.IN_ATTRIB | unix.IN_MODIFY |
                unix.IN_MOVE_SELF | unix.IN_DELETE | unix.IN_DELETE_SELF

diff --git a/backend_kqueue.go b/backend_kqueue.go
index cbe4778b245d92ed45b419444650e3336670cad4..f9a0951fed939f808413c9f361f35c65559a799b 100644 (file)
--- a/backend_kqueue.go
+++ b/backend_kqueue.go
@@ -106,7 +106,7 @@ type Watcher struct {
        // [ErrEventOverflow] is used to indicate there are too many events:
        //
        //  - inotify: there are too many queued events (fs.inotify.max_queued_events sysctl)
-       //  - windows: The buffer size is too small.
+       //  - windows: The buffer size is too small; [WithBufferSize] can be used to increase it.
        //  - kqueue, fen: not used.
        Errors chan error
 
@@ -249,6 +249,8 @@ func (w *Watcher) Close() error {
 //
 // Returns [ErrClosed] if [Watcher.Close] was called.
 //
+// See [AddWith] for a version that allows adding options.
+//
 // # Watching directories
 //
 // All files in a directory are monitored, including new files that are created
@@ -266,7 +268,18 @@ func (w *Watcher) Close() error {
 //
 // Instead, watch the parent directory and use Event.Name to filter out files
 // you're not interested in. There is an example of this in [cmd/fsnotify/file.go].
-func (w *Watcher) Add(name string) error {
+func (w *Watcher) Add(name string) error { return w.AddWith(name) }
+
+// AddWith is like [Add], but allows adding options. When using Add() the
+// defaults described below are used.
+//
+// Possible options are:
+//
+//   - [WithBufferSize] sets the buffer size for the Windows backend; no-op on
+//     other platforms. The default is 64K (65536 bytes).
+func (w *Watcher) AddWith(name string, opts ...addOpt) error {
+       _ = getOptions(opts...)
+
        w.mu.Lock()
        w.userWatches[name] = struct{}{}
        w.mu.Unlock()

diff --git a/backend_other.go b/backend_other.go
index fc6bacae54ba91bd977c56796c007cdac10ca0a4..60d57053e534dbaa68e4e3a6fc7325f7e17e811b 100644 (file)
--- a/backend_other.go
+++ b/backend_other.go
@@ -100,7 +100,7 @@ type Watcher struct {
        // [ErrEventOverflow] is used to indicate there are too many events:
        //
        //  - inotify: there are too many queued events (fs.inotify.max_queued_events sysctl)
-       //  - windows: The buffer size is too small.
+       //  - windows: The buffer size is too small; [WithBufferSize] can be used to increase it.
        //  - kqueue, fen: not used.
        Errors chan error
 }
@@ -133,6 +133,8 @@ func (w *Watcher) WatchList() []string { return nil }
 //
 // Returns [ErrClosed] if [Watcher.Close] was called.
 //
+// See [AddWith] for a version that allows adding options.
+//
 // # Watching directories
 //
 // All files in a directory are monitored, including new files that are created
@@ -152,6 +154,17 @@ func (w *Watcher) WatchList() []string { return nil }
 // you're not interested in. There is an example of this in [cmd/fsnotify/file.go].
 func (w *Watcher) Add(name string) error { return nil }
 
+// AddWith is like [Add], but allows adding options. When using Add() the
+// defaults described below are used.
+//
+// Possible options are:
+//
+//   - [WithBufferSize] sets the buffer size for the Windows backend; no-op on
+//     other platforms. The default is 64K (65536 bytes).
+func (w *Watcher) AddWith(name string, opts ...addOpt) error {
+       return nil
+}
+
 // Remove stops monitoring the path for changes.
 //
 // Directories are always removed non-recursively. For example, if you added

diff --git a/backend_windows.go b/backend_windows.go
index ba5e95817a3f0ba9b238a4dbbda9df0df319e0bb..e99139e7980b02b3543fd8ef74553381f2bda636 100644 (file)
--- a/backend_windows.go
+++ b/backend_windows.go
@@ -109,7 +109,7 @@ type Watcher struct {
        // [ErrEventOverflow] is used to indicate there are too many events:
        //
        //  - inotify: there are too many queued events (fs.inotify.max_queued_events sysctl)
-       //  - windows: The buffer size is too small.
+       //  - windows: The buffer size is too small; [WithBufferSize] can be used to increase it.
        //  - kqueue, fen: not used.
        Errors chan error
 
@@ -204,6 +204,8 @@ func (w *Watcher) Close() error {
 //
 // Returns [ErrClosed] if [Watcher.Close] was called.
 //
+// See [AddWith] for a version that allows adding options.
+//
 // # Watching directories
 //
 // All files in a directory are monitored, including new files that are created
@@ -221,16 +223,31 @@ func (w *Watcher) Close() error {
 //
 // Instead, watch the parent directory and use Event.Name to filter out files
 // you're not interested in. There is an example of this in [cmd/fsnotify/file.go].
-func (w *Watcher) Add(name string) error {
+func (w *Watcher) Add(name string) error { return w.AddWith(name) }
+
+// AddWith is like [Add], but allows adding options. When using Add() the
+// defaults described below are used.
+//
+// Possible options are:
+//
+//   - [WithBufferSize] sets the buffer size for the Windows backend; no-op on
+//     other platforms. The default is 64K (65536 bytes).
+func (w *Watcher) AddWith(name string, opts ...addOpt) error {
        if w.isClosed() {
                return ErrClosed
        }
 
+       with := getOptions(opts...)
+       if with.bufsize < 4096 {
+               return fmt.Errorf("fsnotify.WithBufferSize: buffer size cannot be smaller than 4096 bytes")
+       }
+
        in := &input{
-               op:    opAddWatch,
-               path:  filepath.Clean(name),
-               flags: sysFSALLEVENTS,
-               reply: make(chan error),
+               op:      opAddWatch,
+               path:    filepath.Clean(name),
+               flags:   sysFSALLEVENTS,
+               reply:   make(chan error),
+               bufsize: with.bufsize,
        }
        w.input <- in
        if err := w.wakeupReader(); err != nil {
@@ -329,10 +346,11 @@ const (
 )
 
 type input struct {
-       op    int
-       path  string
-       flags uint32
-       reply chan error
+       op      int
+       path    string
+       flags   uint32
+       bufsize int
+       reply   chan error
 }
 
 type inode struct {
@@ -348,7 +366,7 @@ type watch struct {
        mask   uint64            // Directory itself is being watched with these notify flags
        names  map[string]uint64 // Map of names being watched and their notify flags
        rename string            // Remembers the old name while renaming a file
-       buf    [65536]byte       // 64K buffer
+       buf    []byte            // buffer, allocated later
 }
 
 type (
@@ -421,7 +439,7 @@ func (m watchMap) set(ino *inode, watch *watch) {
 }
 
 // Must run within the I/O thread.
-func (w *Watcher) addWatch(pathname string, flags uint64) error {
+func (w *Watcher) addWatch(pathname string, flags uint64, bufsize int) error {
        dir, err := w.getDir(pathname)
        if err != nil {
                return err
@@ -444,6 +462,7 @@ func (w *Watcher) addWatch(pathname string, flags uint64) error {
                        ino:   ino,
                        path:  dir,
                        names: make(map[string]uint64),
+                       buf:   make([]byte, bufsize),
                }
                w.mu.Lock()
                w.watches.set(ino, watchEntry)
@@ -543,8 +562,11 @@ func (w *Watcher) startRead(watch *watch) error {
                return nil
        }
 
-       rdErr := windows.ReadDirectoryChanges(watch.ino.handle, &watch.buf[0],
-               uint32(unsafe.Sizeof(watch.buf)), false, mask, nil, &watch.ov, 0)
+       // We need to pass the array, rather than the slice.
+       hdr := (*reflect.SliceHeader)(unsafe.Pointer(&watch.buf))
+       rdErr := windows.ReadDirectoryChanges(watch.ino.handle,
+               (*byte)(unsafe.Pointer(hdr.Data)), uint32(hdr.Len),
+               false, mask, nil, &watch.ov, 0)
        if rdErr != nil {
                err := os.NewSyscallError("ReadDirectoryChanges", rdErr)
                if rdErr == windows.ERROR_ACCESS_DENIED && watch.mask&provisional == 0 {
@@ -603,7 +625,7 @@ func (w *Watcher) readEvents() {
                        case in := <-w.input:
                                switch in.op {
                                case opAddWatch:
-                                       in.reply <- w.addWatch(in.path, uint64(in.flags))
+                                       in.reply <- w.addWatch(in.path, uint64(in.flags), in.bufsize)
                                case opRemoveWatch:
                                        in.reply <- w.remWatch(in.path)
                                }

diff --git a/fsnotify.go b/fsnotify.go
index 4f7f445fb1a4cf5a7a49ac615abe0bea169e991e..142169da68ae97b8b87402f8a6164f0456a3ac75 100644 (file)
--- a/fsnotify.go
+++ b/fsnotify.go
@@ -101,3 +101,33 @@ func (e Event) Has(op Op) bool { return e.Op.Has(op) }
 func (e Event) String() string {
        return fmt.Sprintf("%-13s %q", e.Op.String(), e.Name)
 }
+
+type (
+       addOpt   func(opt *withOpts)
+       withOpts struct {
+               bufsize int
+       }
+)
+
+var defaultOpts = withOpts{
+       bufsize: 65536, // 64K
+}
+
+func getOptions(opts ...addOpt) withOpts {
+       with := defaultOpts
+       for _, o := range opts {
+               o(&with)
+       }
+       return with
+}
+
+// WithBufferSize sets the buffer size for the Windows backend. This is a no-op
+// for other backends.
+//
+// The default value is 64K (65536 bytes) which is the highest value that works
+// on all filesystems and should be enough for most applications, but if you
+// have a large burst of events it may not be enough. You can increase it if
+// you're hitting "queue or buffer overflow" errors ([ErrEventOverflow]).
+func WithBufferSize(bytes int) addOpt {
+       return func(opt *withOpts) { opt.bufsize = bytes }
+}

diff --git a/mkdoc.zsh b/mkdoc.zsh
index 19aa1edf5041709d5eecea29bccfef7c15a90b56..02216e74171f4b29384560e9fa8356b6150e9b2c 100755 (executable)
--- a/mkdoc.zsh
+++ b/mkdoc.zsh
@@ -80,6 +80,8 @@ add=$(<<EOF
 //
 // Returns [ErrClosed] if [Watcher.Close] was called.
 //
+// See [AddWith] for a version that allows adding options.
+//
 // # Watching directories
 //
 // All files in a directory are monitored, including new files that are created
@@ -100,6 +102,17 @@ add=$(<<EOF
 EOF
 )
 
+addwith=$(<<EOF
+// AddWith is like [Add], but allows adding options. When using Add() the
+// defaults described below are used.
+//
+// Possible options are:
+//
+//   - [WithBufferSize] sets the buffer size for the Windows backend; no-op on
+//     other platforms. The default is 64K (65536 bytes).
+EOF
+)
+
 remove=$(<<EOF
 // Remove stops monitoring the path for changes.
 //
@@ -166,7 +179,7 @@ errors=$(<<EOF
        // [ErrEventOverflow] is used to indicate there are too many events:
        //
        //  - inotify: there are too many queued events (fs.inotify.max_queued_events sysctl)
-       //  - windows: The buffer size is too small.
+       //  - windows: The buffer size is too small; [WithBufferSize] can be used to increase it.
        //  - kqueue, fen: not used.
 EOF
 )
@@ -202,6 +215,7 @@ set-cmt() {
 set-cmt '^type Watcher struct '             $watcher
 set-cmt '^func NewWatcher('                 $new
 set-cmt '^func (w \*Watcher) Add('          $add
+set-cmt '^func (w \*Watcher) AddWith('      $addwith
 set-cmt '^func (w \*Watcher) Remove('       $remove
 set-cmt '^func (w \*Watcher) Close('        $close
 set-cmt '^func (w \*Watcher) WatchList('    $watchlist