From 2f03f8dd4224a68dfd86028b24680f9fd9673194 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Ludwig?= Date: Tue, 19 May 2020 10:10:15 +0200 Subject: [PATCH 1/9] Move Posix event loop implementations to their own package. --- source/eventcore/core.d | 6 +++--- source/eventcore/drivers/posix/{ => loop}/epoll.d | 2 +- source/eventcore/drivers/posix/{ => loop}/kqueue.d | 2 +- source/eventcore/drivers/posix/{ => loop}/select.d | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) rename source/eventcore/drivers/posix/{ => loop}/epoll.d (98%) rename source/eventcore/drivers/posix/{ => loop}/kqueue.d (98%) rename source/eventcore/drivers/posix/{ => loop}/select.d (98%) diff --git a/source/eventcore/core.d b/source/eventcore/core.d index db4dc5f..44d8754 100644 --- a/source/eventcore/core.d +++ b/source/eventcore/core.d @@ -2,9 +2,9 @@ module eventcore.core; public import eventcore.driver; -import eventcore.drivers.posix.select; -import eventcore.drivers.posix.epoll; -import eventcore.drivers.posix.kqueue; +import eventcore.drivers.posix.loop.select; +import eventcore.drivers.posix.loop.epoll; +import eventcore.drivers.posix.loop.kqueue; import eventcore.drivers.libasync; import eventcore.drivers.winapi.driver; import eventcore.internal.utils : mallocT, freeT; diff --git a/source/eventcore/drivers/posix/epoll.d b/source/eventcore/drivers/posix/loop/epoll.d similarity index 98% rename from source/eventcore/drivers/posix/epoll.d rename to source/eventcore/drivers/posix/loop/epoll.d index b6a5767..672476c 100644 --- a/source/eventcore/drivers/posix/epoll.d +++ b/source/eventcore/drivers/posix/loop/epoll.d @@ -4,7 +4,7 @@ Epoll is an efficient API for asynchronous I/O on Linux, suitable for large numbers of concurrently open sockets. */ -module eventcore.drivers.posix.epoll; +module eventcore.drivers.posix.loop.epoll; @safe @nogc nothrow: version (linux): diff --git a/source/eventcore/drivers/posix/kqueue.d b/source/eventcore/drivers/posix/loop/kqueue.d similarity index 98% rename from source/eventcore/drivers/posix/kqueue.d rename to source/eventcore/drivers/posix/loop/kqueue.d index 64b15a6..14aa2a3 100644 --- a/source/eventcore/drivers/posix/kqueue.d +++ b/source/eventcore/drivers/posix/loop/kqueue.d @@ -4,7 +4,7 @@ Kqueue is an efficient API for asynchronous I/O on BSD flavors, including OS X/macOS, suitable for large numbers of concurrently open sockets. */ -module eventcore.drivers.posix.kqueue; +module eventcore.drivers.posix.loop.kqueue; @safe: /*@nogc:*/ nothrow: version (FreeBSD) enum have_kqueue = true; diff --git a/source/eventcore/drivers/posix/select.d b/source/eventcore/drivers/posix/loop/select.d similarity index 98% rename from source/eventcore/drivers/posix/select.d rename to source/eventcore/drivers/posix/loop/select.d index dabc491..ee9d39b 100644 --- a/source/eventcore/drivers/posix/select.d +++ b/source/eventcore/drivers/posix/loop/select.d @@ -5,7 +5,7 @@ Windows. It has a good performance for small numbers of cuncurrently open files/sockets, but is not suited for larger amounts. */ -module eventcore.drivers.posix.select; +module eventcore.drivers.posix.loop.select; @safe: /*@nogc:*/ nothrow: public import eventcore.drivers.posix.driver; From 1f9a99a8055c92dc27ee99e1a9680fbb1c752190 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Ludwig?= Date: Tue, 19 May 2020 22:52:23 +0200 Subject: [PATCH 2/9] Fix potential access of an invalid waiter queue. --- source/eventcore/drivers/posix/events.d | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/source/eventcore/drivers/posix/events.d b/source/eventcore/drivers/posix/events.d index 791907b..00631c3 100644 --- a/source/eventcore/drivers/posix/events.d +++ b/source/eventcore/drivers/posix/events.d @@ -114,6 +114,10 @@ final class PosixEventDriverEvents(Loop : PosixEventLoop, Sockets : EventDriverS { if (!isValid(event)) return; + // make sure the event stays alive until all waiters have been notified. + addRef(event); + scope (exit) releaseRef(event); + auto slot = getSlot(event); if (notify_all) { //log("emitting only for this thread (%s waiters)", m_fds[event].waiters.length); @@ -167,9 +171,12 @@ final class PosixEventDriverEvents(Loop : PosixEventLoop, Sockets : EventDriverS trigger(event, cnt > 0); } } else { - private void onSocketData(DatagramSocketFD s, IOStatus, size_t, scope RefAddress) + private void onSocketData(DatagramSocketFD s, IOStatus st, size_t, scope RefAddress) @nogc { - m_sockets.receiveNoGC(s, m_buf, IOMode.once, &onSocketData); + // avoid infinite recursion in case of errors + if (st == IOStatus.ok) + m_sockets.receiveNoGC(s, m_buf, IOMode.once, &onSocketData); + try { EventID evt = m_sockets.userData!EventID(s); scope doit = { From aa3659fca61f162d726ebe38aa7993df27f1b3d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Ludwig?= Date: Tue, 19 May 2020 22:53:29 +0200 Subject: [PATCH 3/9] Handle redundant cancelReceive calls gracefully and clean up the callback. --- source/eventcore/drivers/posix/sockets.d | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/source/eventcore/drivers/posix/sockets.d b/source/eventcore/drivers/posix/sockets.d index fe6aaea..1f8557a 100644 --- a/source/eventcore/drivers/posix/sockets.d +++ b/source/eventcore/drivers/posix/sockets.d @@ -866,9 +866,12 @@ final class PosixEventDriverSockets(Loop : PosixEventLoop) : EventDriverSockets @nogc { if (!isValid(socket)) return; - assert(m_loop.m_fds[socket].datagramSocket.readCallback !is null, "Cancelling read when there is no read in progress."); + auto slot = () @trusted { return &m_loop.m_fds[socket].datagramSocket(); } (); + if (slot.readCallback is null) return; + m_loop.setNotifyCallback!(EventType.read)(socket, null); - m_loop.m_fds[socket].datagramSocket.readBuffer = null; + slot.readCallback = null; + slot.readBuffer = null; } private void onDgramRead(FD fd) @@ -894,7 +897,9 @@ final class PosixEventDriverSockets(Loop : PosixEventLoop) : EventDriverSockets auto l = lockHandle(socket); m_loop.setNotifyCallback!(EventType.read)(socket, null); scope src_addrc = new RefAddress(() @trusted { return cast(sockaddr*)&src_addr; } (), src_addr.sizeof); - () @trusted { return cast(DatagramIOCallback)slot.readCallback; } ()(socket, IOStatus.ok, ret, src_addrc); + auto cb = () @trusted { return cast(DatagramIOCallback)slot.readCallback; } (); + slot.readCallback = null; + cb(socket, IOStatus.ok, ret, src_addrc); } void send(DatagramSocketFD socket, const(ubyte)[] buffer, IOMode mode, Address target_address, DatagramIOCallback on_send_finish) From 4b1afa8d6c444e5b54a8d43c137a6a819b37fb42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Ludwig?= Date: Tue, 19 May 2020 22:52:44 +0200 Subject: [PATCH 4/9] Allow to inherit from the kqueue loop implementation. --- source/eventcore/drivers/posix/loop/kqueue.d | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/source/eventcore/drivers/posix/loop/kqueue.d b/source/eventcore/drivers/posix/loop/kqueue.d index 14aa2a3..4a0c8ad 100644 --- a/source/eventcore/drivers/posix/loop/kqueue.d +++ b/source/eventcore/drivers/posix/loop/kqueue.d @@ -31,8 +31,16 @@ import core.sys.linux.epoll; alias KqueueEventDriver = PosixEventDriver!KqueueEventLoop; -final class KqueueEventLoop : PosixEventLoop { - private { +final class KqueueEventLoop : KqueueEventLoopBase { + override bool doProcessEvents(Duration timeout) + @trusted { + return doProcessEventsBase(timeout); + } +} + + +abstract class KqueueEventLoopBase : PosixEventLoop { + protected { int m_queue; size_t m_changeCount = 0; kevent_t[100] m_changes; @@ -45,8 +53,8 @@ final class KqueueEventLoop : PosixEventLoop { assert(m_queue >= 0, "Failed to create kqueue."); } - override bool doProcessEvents(Duration timeout) - @trusted { + protected bool doProcessEventsBase(Duration timeout) + @trusted nothrow { import std.algorithm : min; //assert(Fiber.getThis() is null, "processEvents may not be called from within a fiber!"); From e28450f9f5b9a80c3f5a19b7078307b95c3c1245 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Ludwig?= Date: Tue, 19 May 2020 10:42:25 +0200 Subject: [PATCH 5/9] Implement a CFRunLoop based event loop. This enables efficient integration of the kqueue based I/O processing with Apple OS based UI apps. On top of that, an FSEvent based directory watcher can now be implemented to replace the inefficient generic watcher that is used on macOS right now. --- dub.sdl | 6 ++ source/eventcore/core.d | 4 +- .../eventcore/drivers/posix/loop/cfrunloop.d | 65 +++++++++++++++++++ source/eventcore/internal/corefoundation.d | 35 ++++++++++ 4 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 source/eventcore/drivers/posix/loop/cfrunloop.d create mode 100644 source/eventcore/internal/corefoundation.d diff --git a/dub.sdl b/dub.sdl index 5abd12e..2217d0c 100644 --- a/dub.sdl +++ b/dub.sdl @@ -23,6 +23,12 @@ configuration "epoll-gaia" { versions "EventcoreEpollDriver" } +configuration "cfrunloop" { + platforms "osx" + versions "EventcoreCFRunLoopDriver" + lflags "-framework" "CoreFoundation" +} + configuration "kqueue" { platforms "osx" "freebsd" versions "EventcoreKqueueDriver" diff --git a/source/eventcore/core.d b/source/eventcore/core.d index 44d8754..c1447ba 100644 --- a/source/eventcore/core.d +++ b/source/eventcore/core.d @@ -2,14 +2,16 @@ module eventcore.core; public import eventcore.driver; -import eventcore.drivers.posix.loop.select; +import eventcore.drivers.posix.loop.cfrunloop; import eventcore.drivers.posix.loop.epoll; import eventcore.drivers.posix.loop.kqueue; +import eventcore.drivers.posix.loop.select; import eventcore.drivers.libasync; import eventcore.drivers.winapi.driver; import eventcore.internal.utils : mallocT, freeT; version (EventcoreEpollDriver) alias NativeEventDriver = EpollEventDriver; +else version (EventcoreCFRunLoopDriver) alias NativeEventDriver = CFRunLoopEventDriver; else version (EventcoreKqueueDriver) alias NativeEventDriver = KqueueEventDriver; else version (EventcoreWinAPIDriver) alias NativeEventDriver = WinAPIEventDriver; else version (EventcoreLibasyncDriver) alias NativeEventDriver = LibasyncEventDriver; diff --git a/source/eventcore/drivers/posix/loop/cfrunloop.d b/source/eventcore/drivers/posix/loop/cfrunloop.d new file mode 100644 index 0000000..9dca204 --- /dev/null +++ b/source/eventcore/drivers/posix/loop/cfrunloop.d @@ -0,0 +1,65 @@ +/** + `CFRunLoop` based event loop for macOS UI compatible operation. +*/ +module eventcore.drivers.posix.loop.cfrunloop; +@safe: /*@nogc:*/ nothrow: + +version (EventcoreCFRunLoopDriver): + +import eventcore.drivers.posix.loop.kqueue; +import eventcore.internal.corefoundation; +import eventcore.internal.utils; +import core.time; + + +alias CFRunLoopEventDriver = PosixEventDriver!CFRunLoopEventLoop; + +final class CFRunLoopEventLoop : KqueueEventLoopBase { +@safe nothrow: + private { + CFFileDescriptorRef m_kqueueDescriptor; + CFRunLoopSourceRef m_kqueueSource; + } + + this() + @trusted @nogc { + super(); + + CFFileDescriptorContext ctx; + ctx.info = cast(void*)this; + + m_kqueueDescriptor = CFFileDescriptorCreate(kCFAllocatorDefault, + m_queue, false, &processKqueue, &ctx); + + CFFileDescriptorEnableCallBacks(m_kqueueDescriptor, CFOptionFlags.kCFFileDescriptorReadCallBack); + m_kqueueSource = CFFileDescriptorCreateRunLoopSource(kCFAllocatorDefault, m_kqueueDescriptor, 0); + CFRunLoopAddSource(CFRunLoopGetMain(), m_kqueueSource, kCFRunLoopDefaultMode); + } + + override bool doProcessEvents(Duration timeout) + @trusted { + // submit changes and process pending events + auto kres = doProcessEventsBase(0.seconds); + + CFTimeInterval to = kres ? 0.0 : 1e-7 * timeout.total!"hnsecs"; + auto res = CFRunLoopRunInMode(kCFRunLoopDefaultMode, to, true); + return kres || res == CFRunLoopRunResult.kCFRunLoopRunHandledSource; + } + + override void dispose() + { + () @trusted { + CFRelease(m_kqueueSource); + CFRelease(m_kqueueDescriptor); + } (); + super.dispose(); + } + + private static extern(C) void processKqueue(CFFileDescriptorRef fdref, + CFOptionFlags callBackTypes, void* info) + { + auto this_ = () @trusted { return cast(CFRunLoopEventLoop)info; } (); + auto res = this_.doProcessEventsBase(0.seconds); + () @trusted { CFFileDescriptorEnableCallBacks(this_.m_kqueueDescriptor, CFOptionFlags.kCFFileDescriptorReadCallBack); } (); + } +} diff --git a/source/eventcore/internal/corefoundation.d b/source/eventcore/internal/corefoundation.d new file mode 100644 index 0000000..5545da0 --- /dev/null +++ b/source/eventcore/internal/corefoundation.d @@ -0,0 +1,35 @@ +module eventcore.internal.corefoundation; + +version (Darwin): + +extern(C): + +static if (!is(typeof(CFRelease))) { + alias CFTypeRef = const(void)*; + alias CFTypeRef CFAllocatorRef; + extern const CFAllocatorRef kCFAllocatorDefault; + CFTypeRef CFRetain(CFTypeRef cf); + void CFRelease(CFTypeRef cf); +} + +static if (!is(typeof(CFRunLoop))) { + alias CFRunLoopMode = CFStringRef; + struct __CFRunLoop; + alias CFRunLoopRef = __CFRunLoop*; + struct __CFRunLoopSource; + alias CFRunLoopSourceRef = __CFRunLoopSource*; + + alias CFTimeInterval = double; + alias Boolean = bool; + + extern const CFStringRef kCFRunLoopDefaultMode; + extern const CFStringRef kCFRunLoopCommonModes; + + void CFRunLoopAddSource(CFRunLoopRef rl, CFRunLoopSourceRef source, CFRunLoopMode mode); + CFRunLoopRunResult CFRunLoopRunInMode(CFRunLoopMode mode, CFTimeInterval seconds, Boolean returnAfterSourceHandled); +} + +static if (!is(CFFileDescriptor)) { + alias FSEventStreamRef = x; + FSEventStreamRef FSEventStreamCreate(CFAllocatorRef allocator, FSEventStreamCallback callback, FSEventStreamContext *context, CFArrayRef pathsToWatch, FSEventStreamEventId sinceWhen, CFTimeInterval latency, FSEventStreamCreateFlags flags); +} From 6e5fb79c6284b00d8f13fc0deaa8b3512d16e840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Ludwig?= Date: Fri, 22 May 2020 09:56:35 +0200 Subject: [PATCH 6/9] Work around platform differences in the directory watcher test. - Drops all events that happen before any changes are made (FSEventStream reports events from the past to a certain degree) - Tests the same file twice in a row, since FSEventStream might coalesce events and report bogus changes - Allows "modified" instead of "added" events, because FSEventStream does not allow to distinguish in some cases --- tests/0-dirwatcher-rec.d | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/tests/0-dirwatcher-rec.d b/tests/0-dirwatcher-rec.d index a52eab2..d6b9538 100644 --- a/tests/0-dirwatcher-rec.d +++ b/tests/0-dirwatcher-rec.d @@ -35,13 +35,15 @@ void main() // test non-recursive watcher watcher = eventDriver.watchers.watchDirectory(testDir, false, toDelegate(&testCallback)); assert(watcher != WatcherID.invalid); - Thread.sleep(400.msecs); // some watcher implementations need time to initialize + // some watcher implementations need time to initialize or report past events + dropChanges(2000.msecs); testFile( "file1.dat"); testFile( "file2.dat"); testFile( "dira/file1.dat", false); testCreateDir("dirb"); testFile( "dirb/file1.dat", false); testRemoveDir("dirb"); + testFile( "file1.dat"); eventDriver.watchers.releaseRef(watcher); testFile( "file1.dat", false); testRemoveDir("dira", false); @@ -50,7 +52,8 @@ void main() // test recursive watcher watcher = eventDriver.watchers.watchDirectory(testDir, true, toDelegate(&testCallback)); assert(watcher != WatcherID.invalid); - Thread.sleep(400.msecs); // some watcher implementations need time to initialize + // some watcher implementations need time to initialize or report past events + dropChanges(2000.msecs); testFile( "file1.dat"); testFile( "file2.dat"); testFile( "dira/file1.dat"); @@ -58,6 +61,7 @@ void main() testFile( "dirb/file1.dat"); testRename( "dirb", "dirc"); testFile( "dirc/file2.dat"); + testFile( "file1.dat"); eventDriver.watchers.releaseRef(watcher); testFile( "file1.dat", false); testFile( "dira/file1.dat", false); @@ -78,11 +82,30 @@ void testCallback(WatcherID w, in ref FileChange ch) pendingChanges ~= ch; } +void dropChanges(Duration dur) +{ + auto starttime = MonoTime.currTime(); + auto remdur = dur; + while (remdur > 0.msecs) { + auto er = eventDriver.core.processEvents(remdur); + switch (er) { + default: assert(false, format("Unexpected event loop exit code: %s", er)); + case ExitReason.idle: break; + case ExitReason.timeout: break; + case ExitReason.outOfWaiters: + assert(false, "No watcher left, but expected change."); + } + remdur = dur - (MonoTime.currTime() - starttime); + } + + pendingChanges = null; +} + void expectChange(FileChange ch, bool expect_change) { auto starttime = MonoTime.currTime(); again: while (!pendingChanges.length) { - auto er = eventDriver.core.processEvents(10.msecs); + auto er = eventDriver.core.processEvents(100.msecs); switch (er) { default: assert(false, format("Unexpected event loop exit code: %s", er)); case ExitReason.idle: break; @@ -123,9 +146,12 @@ void expectChange(FileChange ch, bool expect_change) if (pch.kind == FileChangeKind.modified && (pch.name == "dira" || pch.name == "dirb")) goto again; - // test all field excep the isDir one, which does not work on all systems - assert(pch.kind == ch.kind && pch.baseDirectory == ch.baseDirectory && - pch.directory == ch.directory && pch.name == ch.name, + // test all field except the isDir one, which does not work on all systems + // we allow "modified" instead of "added" here, as the FSEvents based watcher + // has strange results on the CI VM + assert((pch.kind == ch.kind || pch.kind == FileChangeKind.modified && ch.kind == FileChangeKind.added) + && pch.baseDirectory == ch.baseDirectory + && pch.directory == ch.directory && pch.name == ch.name, format("Unexpected change: %s vs %s", pch, ch)); } From 0cdfe793afc24856e0d113bb52f1e120c4ac4b32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Ludwig?= Date: Fri, 22 May 2020 09:58:08 +0200 Subject: [PATCH 7/9] Implement an FSEvents based watcher for macOS. Provides efficient and low latency directory watching on macOS when using the CFRunLoop based event loop. --- dub.sdl | 1 + source/eventcore/drivers/posix/driver.d | 2 +- source/eventcore/drivers/posix/watchers.d | 193 +++++++++++++++++++-- source/eventcore/internal/corefoundation.d | 118 +++++++++++-- source/eventcore/internal/coreservices.d | 100 +++++++++++ 5 files changed, 387 insertions(+), 27 deletions(-) create mode 100644 source/eventcore/internal/coreservices.d diff --git a/dub.sdl b/dub.sdl index 2217d0c..40d503a 100644 --- a/dub.sdl +++ b/dub.sdl @@ -27,6 +27,7 @@ configuration "cfrunloop" { platforms "osx" versions "EventcoreCFRunLoopDriver" lflags "-framework" "CoreFoundation" + lflags "-framework" "CoreServices" } configuration "kqueue" { diff --git a/source/eventcore/drivers/posix/driver.d b/source/eventcore/drivers/posix/driver.d index 1997689..2ac4e62 100644 --- a/source/eventcore/drivers/posix/driver.d +++ b/source/eventcore/drivers/posix/driver.d @@ -54,7 +54,7 @@ final class PosixEventDriver(Loop : PosixEventLoop) : EventDriver { version (Posix) alias PipeDriver = PosixEventDriverPipes!Loop; else alias PipeDriver = DummyEventDriverPipes!Loop; version (linux) alias WatcherDriver = InotifyEventDriverWatchers!EventsDriver; - //else version (OSX) alias WatcherDriver = FSEventsEventDriverWatchers!EventsDriver; + else version (EventcoreCFRunLoopDriver) alias WatcherDriver = FSEventsEventDriverWatchers!EventsDriver; else alias WatcherDriver = PollEventDriverWatchers!EventsDriver; version (Posix) alias ProcessDriver = PosixEventDriverProcesses!Loop; else alias ProcessDriver = DummyEventDriverProcesses!Loop; diff --git a/source/eventcore/drivers/posix/watchers.d b/source/eventcore/drivers/posix/watchers.d index 30d7233..4ad8ce3 100644 --- a/source/eventcore/drivers/posix/watchers.d +++ b/source/eventcore/drivers/posix/watchers.d @@ -196,51 +196,214 @@ final class InotifyEventDriverWatchers(Events : EventDriverEvents) : EventDriver } } -version (OSX) +version (darwin) final class FSEventsEventDriverWatchers(Events : EventDriverEvents) : EventDriverWatchers { @safe: /*@nogc:*/ nothrow: - private Events m_events; + import eventcore.internal.corefoundation; + import eventcore.internal.coreservices; + import std.string : toStringz; + + private { + static struct WatcherSlot { + FSEventStreamRef stream; + string path; + string fullPath; + FileChangesCallback callback; + WatcherID id; + int refCount = 1; + bool recursive; + FSEventStreamEventId lastEvent; + ubyte[16 * size_t.sizeof] userData; + DataInitializer userDataDestructor; + } + //HashMap!(void*, WatcherSlot) m_watches; + WatcherSlot[WatcherID] m_watches; + WatcherID[void*] m_streamMap; + Events m_events; + size_t m_handleCounter = 1; + uint m_validationCounter; + } this(Events events) { m_events = events; } final override WatcherID watchDirectory(string path, bool recursive, FileChangesCallback on_change) - { - /*FSEventStreamCreate - FSEventStreamScheduleWithRunLoop - FSEventStreamStart*/ - assert(false, "TODO!"); + @trusted { + import std.path : absolutePath; + + FSEventStreamContext ctx; + ctx.info = () @trusted { return cast(void*)this; } (); + + string abspath; + try abspath = absolutePath(path); + catch (Exception e) assert(false, e.msg); + + if (m_handleCounter == 0) { + m_handleCounter++; + m_validationCounter++; + } + auto id = WatcherID(cast(size_t)m_handleCounter++, m_validationCounter); + + WatcherSlot slot = { + path: path, + fullPath: abspath, + callback: on_change, + id: id, + recursive: recursive, + lastEvent: kFSEventStreamEventIdSinceNow + }; + + startStream(slot, kFSEventStreamEventIdSinceNow); + + m_events.loop.m_waiterCount++; + m_watches[id] = slot; + return id; } final override bool isValid(WatcherID handle) const { - return false; + return !!(handle in m_watches); } final override void addRef(WatcherID descriptor) { if (!isValid(descriptor)) return; - assert(false, "TODO!"); + auto slot = descriptor in m_watches; + slot.refCount++; } final override bool releaseRef(WatcherID descriptor) { if (!isValid(descriptor)) return true; - /*FSEventStreamStop - FSEventStreamUnscheduleFromRunLoop - FSEventStreamInvalidate - FSEventStreamRelease*/ - assert(false, "TODO!"); + auto slot = descriptor in m_watches; + if (!--slot.refCount) { + destroyStream(slot.stream); + m_watches.remove(descriptor); + m_events.loop.m_waiterCount--; + return false; + } + + return true; } final protected override void* rawUserData(WatcherID descriptor, size_t size, DataInitializer initialize, DataInitializer destroy) @system { if (!isValid(descriptor)) return null; - return m_loop.rawUserDataImpl(descriptor, size, initialize, destroy); + auto slot = descriptor in m_watches; + + if (size > WatcherSlot.userData.length) assert(false); + if (!slot.userDataDestructor) { + initialize(slot.userData.ptr); + slot.userDataDestructor = destroy; + } + return slot.userData.ptr; } + private static extern(C) void onFSEvent(ConstFSEventStreamRef streamRef, + void* clientCallBackInfo, size_t numEvents, void* eventPaths, + const(FSEventStreamEventFlags)* eventFlags, + const(FSEventStreamEventId)* eventIds) + { + import std.conv : to; + import std.path : asRelativePath, baseName, dirName; + + if (!numEvents) return; + + auto this_ = () @trusted { return cast(FSEventsEventDriverWatchers)clientCallBackInfo; } (); + auto h = () @trusted { return cast(void*)streamRef; } (); + auto ps = h in this_.m_streamMap; + if (!ps) return; + auto id = *ps; + auto slot = id in this_.m_watches; + + auto patharr = () @trusted { return (cast(const(char)**)eventPaths)[0 .. numEvents]; } (); + auto flagsarr = () @trusted { return eventFlags[0 .. numEvents]; } (); + auto idarr = () @trusted { return eventIds[0 .. numEvents]; } (); + + // A new stream needs to be created after every change, because events + // get coalesced per file (event flags get or'ed together) and it becomes + // impossible to determine the actual event + this_.startStream(*slot, idarr[$-1]); + + foreach (i; 0 .. numEvents) { + auto pathstr = () @trusted { return to!string(patharr[i]); } (); + auto f = flagsarr[i]; + + string rp; + try rp = pathstr.asRelativePath(slot.fullPath).to!string; + catch (Exception e) assert(false, e.msg); + + if (rp == "." || rp == "") continue; + + FileChange ch; + ch.baseDirectory = slot.path; + ch.directory = dirName(rp); + ch.name = baseName(rp); + + if (ch.directory == ".") ch.directory = ""; + + if (!slot.recursive && ch.directory != "") continue; + + void emit(FileChangeKind k) + { + ch.kind = k; + slot.callback(id, ch); + } + + import std.file : exists; + bool does_exist = exists(pathstr); + + // The order of tests is important to properly lower the more + // complex flags system to the three event types provided by + // eventcore + if (f & kFSEventStreamEventFlagItemRenamed) { + if (f & kFSEventStreamEventFlagItemCreated) + emit(FileChangeKind.removed); + else emit(FileChangeKind.added); + } else if (f & kFSEventStreamEventFlagItemRemoved && !does_exist) { + emit(FileChangeKind.removed); + } else if (f & kFSEventStreamEventFlagItemModified && does_exist) { + emit(FileChangeKind.modified); + } else if (f & kFSEventStreamEventFlagItemCreated && does_exist) { + emit(FileChangeKind.added); + } + } + } + + private void startStream(ref WatcherSlot slot, FSEventStreamEventId since_when) + @trusted { + if (slot.stream) { + destroyStream(slot.stream); + slot.stream = null; + } + + FSEventStreamContext ctx; + ctx.info = () @trusted { return cast(void*)this; } (); + + auto pstr = CFStringCreateWithBytes(null, + cast(const(ubyte)*)slot.path.ptr, slot.path.length, + kCFStringEncodingUTF8, false); + scope (exit) CFRelease(pstr); + auto paths = CFArrayCreate(null, cast(const(void)**)&pstr, 1, null); + scope (exit) CFRelease(paths); + + slot.stream = FSEventStreamCreate(null, &onFSEvent, () @trusted { return &ctx; } (), + paths, since_when, 0.1, kFSEventStreamCreateFlagFileEvents|kFSEventStreamCreateFlagNoDefer); + FSEventStreamScheduleWithRunLoop(slot.stream, CFRunLoopGetMain(), kCFRunLoopDefaultMode); + FSEventStreamStart(slot.stream); + + m_streamMap[cast(void*)slot.stream] = slot.id; + } + + private void destroyStream(FSEventStreamRef stream) + @trusted { + FSEventStreamStop(stream); + FSEventStreamInvalidate(stream); + FSEventStreamRelease(stream); + m_streamMap.remove(cast(void*)stream); + } } diff --git a/source/eventcore/internal/corefoundation.d b/source/eventcore/internal/corefoundation.d index 5545da0..b893d0e 100644 --- a/source/eventcore/internal/corefoundation.d +++ b/source/eventcore/internal/corefoundation.d @@ -1,18 +1,79 @@ module eventcore.internal.corefoundation; -version (Darwin): +version (darwin): -extern(C): +nothrow extern(C): -static if (!is(typeof(CFRelease))) { +static if (!is(CFTypeRef)) { alias CFTypeRef = const(void)*; alias CFTypeRef CFAllocatorRef; extern const CFAllocatorRef kCFAllocatorDefault; - CFTypeRef CFRetain(CFTypeRef cf); - void CFRelease(CFTypeRef cf); + + CFTypeRef CFRetain(CFTypeRef cf) @nogc; + void CFRelease(CFTypeRef cf) @nogc; + + struct __CFString; + alias CFStringRef = const(__CFString)*; + + alias Boolean = ubyte; + alias UniChar = ushort; + alias CFTypeID = size_t; + alias CFIndex = sizediff_t; + + alias CFAllocatorRetainCallBack = const(void)* function(const void *info); + alias CFAllocatorReleaseCallBack = void function(const void *info); + alias CFAllocatorCopyDescriptionCallBack = CFStringRef function(const void *info); } -static if (!is(typeof(CFRunLoop))) { +static if (!is(CFArrayRef)) { + struct __CFArray; + alias CFArrayRef = const(__CFArray)*; + + alias CFArrayRetainCallBack = const(void)* function(CFAllocatorRef allocator, const(void)* value); + alias CFArrayReleaseCallBack = void function(CFAllocatorRef allocator, const(void)* value); + alias CFArrayCopyDescriptionCallBack = CFStringRef function(const(void)* value); + alias CFArrayEqualCallBack = Boolean function(const(void)* value1, const(void)* value2); + + struct CFArrayCallBacks { + CFIndex version_; + CFArrayRetainCallBack retain; + CFArrayReleaseCallBack release; + CFArrayCopyDescriptionCallBack copyDescription; + CFArrayEqualCallBack equal; + } + + CFArrayRef CFArrayCreate(CFAllocatorRef allocator, const(void)** values, CFIndex numValues, const(CFArrayCallBacks)* callBacks); +} + +static if (!is(CFStringEncoding)) { + alias CFStringEncoding = uint; + + enum kCFStringEncodingInvalidId = 0xffffffffU; + enum { + kCFStringEncodingMacRoman = 0, + kCFStringEncodingWindowsLatin1 = 0x0500, + kCFStringEncodingISOLatin1 = 0x0201, + kCFStringEncodingNextStepLatin = 0x0B01, + kCFStringEncodingASCII = 0x0600, + kCFStringEncodingUnicode = 0x0100, + kCFStringEncodingUTF8 = 0x08000100, + kCFStringEncodingNonLossyASCII = 0x0BFF, + + kCFStringEncodingUTF16 = 0x0100, + kCFStringEncodingUTF16BE = 0x10000100, + kCFStringEncodingUTF16LE = 0x14000100, + + kCFStringEncodingUTF32 = 0x0c000100, + kCFStringEncodingUTF32BE = 0x18000100, + kCFStringEncodingUTF32LE = 0x1c000100 + } + + CFStringRef CFStringCreateWithCString(CFAllocatorRef alloc, const(char)* cStr, CFStringEncoding encoding); + CFStringRef CFStringCreateWithBytes(CFAllocatorRef alloc, const(ubyte)* bytes, CFIndex numBytes, CFStringEncoding encoding, Boolean isExternalRepresentation); + CFStringRef CFStringCreateWithCharacters(CFAllocatorRef alloc, const(UniChar)* chars, CFIndex numChars); +} + +static if (!is(CFRunLoopRef)) { alias CFRunLoopMode = CFStringRef; struct __CFRunLoop; alias CFRunLoopRef = __CFRunLoop*; @@ -20,16 +81,51 @@ static if (!is(typeof(CFRunLoop))) { alias CFRunLoopSourceRef = __CFRunLoopSource*; alias CFTimeInterval = double; - alias Boolean = bool; + + enum CFRunLoopRunResult : int { + kCFRunLoopRunFinished = 1, + kCFRunLoopRunStopped = 2, + kCFRunLoopRunTimedOut = 3, + kCFRunLoopRunHandledSource = 4 + } extern const CFStringRef kCFRunLoopDefaultMode; extern const CFStringRef kCFRunLoopCommonModes; - void CFRunLoopAddSource(CFRunLoopRef rl, CFRunLoopSourceRef source, CFRunLoopMode mode); + CFRunLoopRef CFRunLoopGetMain() @nogc; + void CFRunLoopAddSource(CFRunLoopRef rl, CFRunLoopSourceRef source, CFRunLoopMode mode) @nogc; CFRunLoopRunResult CFRunLoopRunInMode(CFRunLoopMode mode, CFTimeInterval seconds, Boolean returnAfterSourceHandled); } -static if (!is(CFFileDescriptor)) { - alias FSEventStreamRef = x; - FSEventStreamRef FSEventStreamCreate(CFAllocatorRef allocator, FSEventStreamCallback callback, FSEventStreamContext *context, CFArrayRef pathsToWatch, FSEventStreamEventId sinceWhen, CFTimeInterval latency, FSEventStreamCreateFlags flags); +static if (!is(CFFileDescriptorRef)) { + alias CFFileDescriptorNativeDescriptor = int; + + struct __CFFileDescriptor; + alias CFFileDescriptorRef = __CFFileDescriptor*; + + /* Callback Reason Types */ + enum CFOptionFlags { + kCFFileDescriptorReadCallBack = 1UL << 0, + kCFFileDescriptorWriteCallBack = 1UL << 1 + } + + alias CFFileDescriptorCallBack = void function(CFFileDescriptorRef f, CFOptionFlags callBackTypes, void* info); + + struct CFFileDescriptorContext { + CFIndex version_; + void* info; + void* function(void *info) retain; + void function(void *info) release; + CFStringRef function(void *info) copyDescription; + } + + CFTypeID CFFileDescriptorGetTypeID() @nogc; + CFFileDescriptorRef CFFileDescriptorCreate(CFAllocatorRef allocator, CFFileDescriptorNativeDescriptor fd, Boolean closeOnInvalidate, CFFileDescriptorCallBack callout, const(CFFileDescriptorContext)* context) @nogc; + CFFileDescriptorNativeDescriptor CFFileDescriptorGetNativeDescriptor(CFFileDescriptorRef f) @nogc; + void CFFileDescriptorGetContext(CFFileDescriptorRef f, CFFileDescriptorContext* context) @nogc; + void CFFileDescriptorEnableCallBacks(CFFileDescriptorRef f, CFOptionFlags callBackTypes) @nogc; + void CFFileDescriptorDisableCallBacks(CFFileDescriptorRef f, CFOptionFlags callBackTypes) @nogc; + void CFFileDescriptorInvalidate(CFFileDescriptorRef f) @nogc; + Boolean CFFileDescriptorIsValid(CFFileDescriptorRef f) @nogc; + CFRunLoopSourceRef CFFileDescriptorCreateRunLoopSource(CFAllocatorRef allocator, CFFileDescriptorRef f, CFIndex order) @nogc; } diff --git a/source/eventcore/internal/coreservices.d b/source/eventcore/internal/coreservices.d new file mode 100644 index 0000000..f68a168 --- /dev/null +++ b/source/eventcore/internal/coreservices.d @@ -0,0 +1,100 @@ +module eventcore.internal.coreservices; + +import eventcore.internal.corefoundation; + +version (darwin): + +nothrow extern(C): + +static if (!is(FSEventStreamRef)) { + alias FSEventStreamCreateFlags = uint; + + enum { + kFSEventStreamCreateFlagNone = 0x00000000, + kFSEventStreamCreateFlagUseCFTypes = 0x00000001, + kFSEventStreamCreateFlagNoDefer = 0x00000002, + kFSEventStreamCreateFlagWatchRoot = 0x00000004, + kFSEventStreamCreateFlagIgnoreSelf = 0x00000008, + kFSEventStreamCreateFlagFileEvents = 0x00000010, + kFSEventStreamCreateFlagMarkSelf = 0x00000020, + kFSEventStreamCreateFlagUseExtendedData = 0x00000040 + } + + //#define kFSEventStreamEventExtendedDataPathKey CFSTR("path") + //#define kFSEventStreamEventExtendedFileIDKey CFSTR("fileID") + + alias FSEventStreamEventFlags = uint; + + enum { + kFSEventStreamEventFlagNone = 0x00000000, + kFSEventStreamEventFlagMustScanSubDirs = 0x00000001, + kFSEventStreamEventFlagUserDropped = 0x00000002, + kFSEventStreamEventFlagKernelDropped = 0x00000004, + kFSEventStreamEventFlagEventIdsWrapped = 0x00000008, + kFSEventStreamEventFlagHistoryDone = 0x00000010, + kFSEventStreamEventFlagRootChanged = 0x00000020, + kFSEventStreamEventFlagMount = 0x00000040, + kFSEventStreamEventFlagUnmount = 0x00000080, + kFSEventStreamEventFlagItemCreated = 0x00000100, + kFSEventStreamEventFlagItemRemoved = 0x00000200, + kFSEventStreamEventFlagItemInodeMetaMod = 0x00000400, + kFSEventStreamEventFlagItemRenamed = 0x00000800, + kFSEventStreamEventFlagItemModified = 0x00001000, + kFSEventStreamEventFlagItemFinderInfoMod = 0x00002000, + kFSEventStreamEventFlagItemChangeOwner = 0x00004000, + kFSEventStreamEventFlagItemXattrMod = 0x00008000, + kFSEventStreamEventFlagItemIsFile = 0x00010000, + kFSEventStreamEventFlagItemIsDir = 0x00020000, + kFSEventStreamEventFlagItemIsSymlink = 0x00040000, + kFSEventStreamEventFlagOwnEvent = 0x00080000, + kFSEventStreamEventFlagItemIsHardlink = 0x00100000, + kFSEventStreamEventFlagItemIsLastHardlink = 0x00200000, + kFSEventStreamEventFlagItemCloned = 0x00400000 + } + + alias FSEventStreamEventId = ulong; + + enum kFSEventStreamEventIdSinceNow = 0xFFFFFFFFFFFFFFFFUL; + + + struct __FSEventStream; + alias FSEventStreamRef = __FSEventStream*; + alias ConstFSEventStreamRef = const(__FSEventStream)*; + + struct FSEventStreamContext { + CFIndex version_; + void* info; + CFAllocatorRetainCallBack retain; + CFAllocatorReleaseCallBack release; + CFAllocatorCopyDescriptionCallBack copyDescription; + } + + alias FSEventStreamCallback = void function(ConstFSEventStreamRef streamRef, + void* clientCallBackInfo, size_t numEvents, void* eventPaths, + const(FSEventStreamEventFlags)* eventFlags, + const(FSEventStreamEventId)* eventIds); + + FSEventStreamRef FSEventStreamCreate(CFAllocatorRef allocator, + FSEventStreamCallback callback, FSEventStreamContext* context, + CFArrayRef pathsToWatch, FSEventStreamEventId sinceWhen, + CFTimeInterval latency, FSEventStreamCreateFlags flags); + + void FSEventStreamRetain(FSEventStreamRef streamRef); + void FSEventStreamRelease(FSEventStreamRef streamRef); + + void FSEventStreamScheduleWithRunLoop(FSEventStreamRef streamRef, + CFRunLoopRef runLoop, CFStringRef runLoopMode); + + void FSEventStreamUnscheduleFromRunLoop(FSEventStreamRef streamRef, + CFRunLoopRef runLoop, CFStringRef runLoopMode); + + void FSEventStreamInvalidate(FSEventStreamRef streamRef); + + Boolean FSEventStreamStart(FSEventStreamRef streamRef); + + FSEventStreamEventId FSEventStreamFlushAsync(FSEventStreamRef streamRef); + + void FSEventStreamFlushSync(FSEventStreamRef streamRef); + + void FSEventStreamStop(FSEventStreamRef streamRef); +} From 49b2aff0c2ec409fca00d6f7a95952d184611b76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Ludwig?= Date: Fri, 22 May 2020 11:02:43 +0200 Subject: [PATCH 8/9] Move loop modules back to the posix package to work around compiler errors. Older DMD frontend versions don't handle "package" protection within derived classes correctly. --- source/eventcore/core.d | 8 ++++---- source/eventcore/drivers/posix/{loop => }/cfrunloop.d | 4 ++-- source/eventcore/drivers/posix/{loop => }/epoll.d | 2 +- source/eventcore/drivers/posix/{loop => }/kqueue.d | 2 +- source/eventcore/drivers/posix/{loop => }/select.d | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) rename source/eventcore/drivers/posix/{loop => }/cfrunloop.d (95%) rename source/eventcore/drivers/posix/{loop => }/epoll.d (98%) rename source/eventcore/drivers/posix/{loop => }/kqueue.d (98%) rename source/eventcore/drivers/posix/{loop => }/select.d (98%) diff --git a/source/eventcore/core.d b/source/eventcore/core.d index c1447ba..e25c5b2 100644 --- a/source/eventcore/core.d +++ b/source/eventcore/core.d @@ -2,10 +2,10 @@ module eventcore.core; public import eventcore.driver; -import eventcore.drivers.posix.loop.cfrunloop; -import eventcore.drivers.posix.loop.epoll; -import eventcore.drivers.posix.loop.kqueue; -import eventcore.drivers.posix.loop.select; +import eventcore.drivers.posix.cfrunloop; +import eventcore.drivers.posix.epoll; +import eventcore.drivers.posix.kqueue; +import eventcore.drivers.posix.select; import eventcore.drivers.libasync; import eventcore.drivers.winapi.driver; import eventcore.internal.utils : mallocT, freeT; diff --git a/source/eventcore/drivers/posix/loop/cfrunloop.d b/source/eventcore/drivers/posix/cfrunloop.d similarity index 95% rename from source/eventcore/drivers/posix/loop/cfrunloop.d rename to source/eventcore/drivers/posix/cfrunloop.d index 9dca204..d4a48dc 100644 --- a/source/eventcore/drivers/posix/loop/cfrunloop.d +++ b/source/eventcore/drivers/posix/cfrunloop.d @@ -1,12 +1,12 @@ /** `CFRunLoop` based event loop for macOS UI compatible operation. */ -module eventcore.drivers.posix.loop.cfrunloop; +module eventcore.drivers.posix.cfrunloop; @safe: /*@nogc:*/ nothrow: version (EventcoreCFRunLoopDriver): -import eventcore.drivers.posix.loop.kqueue; +import eventcore.drivers.posix.kqueue; import eventcore.internal.corefoundation; import eventcore.internal.utils; import core.time; diff --git a/source/eventcore/drivers/posix/loop/epoll.d b/source/eventcore/drivers/posix/epoll.d similarity index 98% rename from source/eventcore/drivers/posix/loop/epoll.d rename to source/eventcore/drivers/posix/epoll.d index 672476c..b6a5767 100644 --- a/source/eventcore/drivers/posix/loop/epoll.d +++ b/source/eventcore/drivers/posix/epoll.d @@ -4,7 +4,7 @@ Epoll is an efficient API for asynchronous I/O on Linux, suitable for large numbers of concurrently open sockets. */ -module eventcore.drivers.posix.loop.epoll; +module eventcore.drivers.posix.epoll; @safe @nogc nothrow: version (linux): diff --git a/source/eventcore/drivers/posix/loop/kqueue.d b/source/eventcore/drivers/posix/kqueue.d similarity index 98% rename from source/eventcore/drivers/posix/loop/kqueue.d rename to source/eventcore/drivers/posix/kqueue.d index 4a0c8ad..3692e83 100644 --- a/source/eventcore/drivers/posix/loop/kqueue.d +++ b/source/eventcore/drivers/posix/kqueue.d @@ -4,7 +4,7 @@ Kqueue is an efficient API for asynchronous I/O on BSD flavors, including OS X/macOS, suitable for large numbers of concurrently open sockets. */ -module eventcore.drivers.posix.loop.kqueue; +module eventcore.drivers.posix.kqueue; @safe: /*@nogc:*/ nothrow: version (FreeBSD) enum have_kqueue = true; diff --git a/source/eventcore/drivers/posix/loop/select.d b/source/eventcore/drivers/posix/select.d similarity index 98% rename from source/eventcore/drivers/posix/loop/select.d rename to source/eventcore/drivers/posix/select.d index ee9d39b..dabc491 100644 --- a/source/eventcore/drivers/posix/loop/select.d +++ b/source/eventcore/drivers/posix/select.d @@ -5,7 +5,7 @@ Windows. It has a good performance for small numbers of cuncurrently open files/sockets, but is not suited for larger amounts. */ -module eventcore.drivers.posix.loop.select; +module eventcore.drivers.posix.select; @safe: /*@nogc:*/ nothrow: public import eventcore.drivers.posix.driver; From 8e84af14836332bb0a28869580f68866fb1d00d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Ludwig?= Date: Fri, 22 May 2020 17:30:57 +0200 Subject: [PATCH 9/9] Test CFRunLoop on macOS. --- .travis.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 45f0901..7c6062d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,18 +26,22 @@ d: env: - CONFIG=select - CONFIG=epoll - - CONFIG=kqueue + - CONFIG=cfrunloop - CONFIG=libasync matrix: allow_failures: - d: dmd-beta + include: + - os: osx + d: ldc + env: CONFIG=kqueue exclude: - env: CONFIG=libasync - os: osx env: CONFIG=epoll - os: linux - env: CONFIG=kqueue + env: CONFIG=cfrunloop - os: osx d: dmd-beta