Rework the listDirectory implementation.

- Directly uses OS facilities instead of Phobos to avoid string processing overhead and to enable fast skipping of non-directories
- Introduces a DirectoryListMode, similar to SpanMode
- Uses low-overhead channels to reduce the communication overhead between the calling thread and the worker thread that calls the OS
- Adds FileInfo.path to properly support the new recursive directory iteration schemes

dub.sdl

@@ -13,9 +13,11 @@ mainSourceFile "source/vibe/appmain.d"
 
 configuration "winapi" {
 	subConfiguration "eventcore" "winapi"
+	versions "Windows7"
 }
 configuration "winapi-optlink" {
 	subConfiguration "eventcore" "winapi-optlink"
+	versions "Windows7"
 }
 configuration "epoll" {
 	subConfiguration "eventcore" "epoll"
@@ -28,9 +30,11 @@ configuration "kqueue" {
 }
 configuration "select" {
 	subConfiguration "eventcore" "select"
+	versions "Windows7" platform="win"
 }
 configuration "select-optlink" {
 	subConfiguration "eventcore" "select-optlink"
+	versions "Windows7" platform="win"
 }
 configuration "libasync" {
 	subConfiguration "eventcore" "libasync"

source/vibe/core/file.d

@@ -357,55 +357,61 @@ void createDirectory(string path, Flag!"recursive" recursive = No.recursive)
 /**
 	Enumerates all files in the specified directory.
 */
-void listDirectory(NativePath path, scope bool delegate(FileInfo info) @safe del)
+void listDirectory(NativePath path, DirectoryListMode mode,
+	scope bool delegate(FileInfo info) @safe del)
 {
-	listDirectory(path.toNativeString, del);
+	import vibe.core.channel : ChannelConfig, ChannelPriority, createChannel;
-}
+	import vibe.core.core : runWorkerTask;
-/// ditto
-void listDirectory(string path, scope bool delegate(FileInfo info) @safe del)
-{
-	import vibe.core.core : runWorkerTaskH;
-	import vibe.core.channel : Channel, createChannel;
 
-	struct S {
+	ChannelConfig cc;
-		FileInfo info;
+	cc.priority = ChannelPriority.overhead;
-		string error;
-	}
 
-	auto ch = createChannel!S();
+	ListDirectoryRequest req;
-	TaskSettings ts;
+	req.path = path;
-	ts.priority = 10 * Task.basePriority;
+	req.channel = createChannel!ListDirectoryData(cc);
-	auto t = runWorkerTaskH(ioTaskSettings, (string path, Channel!S ch) nothrow {
+	req.spanMode = mode;
-		scope (exit) ch.close();
-		try {
-			foreach (DirEntry ent; dirEntries(path, SpanMode.shallow)) {
-				auto nfo = makeFileInfo(ent);
-				try ch.put(S(nfo, null));
-				catch (Exception e) break; // channel got closed
-			}
-		} catch (Exception e) {
-			try ch.put(S(FileInfo.init, e.msg.length ? e.msg : "Failed to iterate directory"));
-			catch (Exception e) {} // channel got closed
-		}
-	}, path, ch);
 
-	scope (exit) {
+	runWorkerTask(ioTaskSettings, &performListDirectory, req);
-		t.interrupt();
-		t.joinUninterruptible();
-	}
 
-	S itm;
+	ListDirectoryData itm;
-	while (ch.tryConsumeOne(itm)) {
+	while (req.channel.tryConsumeOne(itm)) {
 		if (itm.error.length)
 			throw new Exception(itm.error);
 
 		if (!del(itm.info)) {
-			ch.close();
+			req.channel.close();
+			// makes sure that the directory handle is closed before returning
+			while (!req.channel.empty) req.channel.tryConsumeOne(itm);
 			break;
 		}
 	}
 }
 /// ditto
+void listDirectory(string path, DirectoryListMode mode,
+	scope bool delegate(FileInfo info) @safe del)
+{
+	listDirectory(NativePath(path), mode, del);
+}
+void listDirectory(NativePath path, scope bool delegate(FileInfo info) @safe del)
+{
+	listDirectory(path, DirectoryListMode.shallow, del);
+}
+/// ditto
+void listDirectory(string path, scope bool delegate(FileInfo info) @safe del)
+{
+	listDirectory(path, DirectoryListMode.shallow, del);
+}
+/// ditto
+void listDirectory(NativePath path, DirectoryListMode mode, scope bool delegate(FileInfo info) @system del)
+@system {
+	listDirectory(path, mode, (nfo) @trusted => del(nfo));
+}
+/// ditto
+void listDirectory(string path, DirectoryListMode mode, scope bool delegate(FileInfo info) @system del)
+@system {
+	listDirectory(path, mode, (nfo) @trusted => del(nfo));
+}
+/// ditto
 void listDirectory(NativePath path, scope bool delegate(FileInfo info) @system del)
 @system {
 	listDirectory(path, (nfo) @trusted => del(nfo));
@@ -416,11 +422,12 @@ void listDirectory(string path, scope bool delegate(FileInfo info) @system del)
 	listDirectory(path, (nfo) @trusted => del(nfo));
 }
 /// ditto
-int delegate(scope int delegate(ref FileInfo)) iterateDirectory(NativePath path)
+int delegate(scope int delegate(ref FileInfo)) iterateDirectory(NativePath path,
+	DirectoryListMode mode = DirectoryListMode.shallow)
 {
 	int iterator(scope int delegate(ref FileInfo) del){
 		int ret = 0;
-		listDirectory(path, (fi){
+		listDirectory(path, mode, (fi) {
 			ret = del(fi);
 			return ret == 0;
 		});
@@ -429,9 +436,10 @@ int delegate(scope int delegate(ref FileInfo)) iterateDirectory(NativePath path)
 	return &iterator;
 }
 /// ditto
-int delegate(scope int delegate(ref FileInfo)) iterateDirectory(string path)
+int delegate(scope int delegate(ref FileInfo)) iterateDirectory(string path,
+	DirectoryListMode mode = DirectoryListMode.shallow)
 {
-	return iterateDirectory(NativePath(path));
+	return iterateDirectory(NativePath(path), mode);
 }
 
 /**
@@ -462,6 +470,9 @@ struct FileInfo {
 	/// Name of the file (not including the path)
 	string name;
 
+	/// The directory containing the file
+	NativePath directory;
+
 	/// Size of the file (zero for directories)
 	ulong size;
 
@@ -502,6 +513,16 @@ enum FileMode {
 	append = FileOpenMode.append
 }
 
+enum DirectoryListMode {
+	/// Only iterate the directory itself
+	shallow,
+	/// Iterate recursively (depth-first, pre-order)
+	recursive,
+	/// Iterate only directories recursively (depth-first, pre-order)
+	recursiveDirectories,
+}
+
+
 /**
 	Accesses the contents of a file as a stream.
 */
@@ -822,11 +843,11 @@ private FileInfo makeFileInfo(DirEntry ent)
 
 	FileInfo ret;
 	string fullname = ent.name;
-	if (ent.name.length) {
+	if (fullname.length) {
 		if (ent.name[$-1].among('/', '\\'))
 			fullname = ent.name[0 .. $-1];
 		ret.name = baseName(fullname);
-		if (ret.name.length == 0) ret.name = fullname;
+		ret.directory = NativePath.fromTrustedString(dirName(fullname));
 	}
 
 	try {
@@ -923,4 +944,173 @@ private auto performInWorker(C, ARGS...)(C callable, auto ref ARGS args)
 	}
 }
 
+private void performListDirectory(ListDirectoryRequest req)
+@trusted nothrow {
+	scope (exit) req.channel.close();
+
+	auto dirs_only = req.spanMode == DirectoryListMode.recursiveDirectories;
+
+	bool scanRec(NativePath path)
+	{
+		import std.algorithm.comparison : among;
+		import std.algorithm.searching : countUntil;
+
+		version (Windows) {
+			import core.sys.windows.windows : FILE_ATTRIBUTE_DIRECTORY,
+				FILE_ATTRIBUTE_DEVICE, FILE_ATTRIBUTE_HIDDEN,
+				FILE_ATTRIBUTE_REPARSE_POINT, FINDEX_INFO_LEVELS, FINDEX_SEARCH_OPS,
+				INVALID_HANDLE_VALUE, WIN32_FIND_DATAW,
+				FindFirstFileExW, FindNextFileW, FindClose;
+			import std.conv : to;
+			import std.utf : toUTF16z;
+			import std.windows.syserror : wenforce;
+
+			static immutable timebase = SysTime(DateTime(1601, 1, 1), UTC());
+
+			WIN32_FIND_DATAW fd;
+			FINDEX_INFO_LEVELS lvl;
+			static if (is(typeof(FINDEX_INFO_LEVELS.FindExInfoBasic)))
+				lvl = FINDEX_INFO_LEVELS.FindExInfoBasic;
+			else lvl = cast(FINDEX_INFO_LEVELS)1;
+			auto fh = FindFirstFileExW((path.toString ~ "\\*").toUTF16z,
+				lvl, &fd, dirs_only ? FINDEX_SEARCH_OPS.FindExSearchLimitToDirectories
+					: FINDEX_SEARCH_OPS.FindExSearchNameMatch,
+				null, 2/*FIND_FIRST_EX_LARGE_FETCH*/);
+			wenforce(fh != INVALID_HANDLE_VALUE, path.toString);
+			scope (exit) FindClose(fh);
+			do {
+				// skip non-directories if requested
+				if (dirs_only && !(fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY))
+					continue;
+
+				FileInfo fi;
+				auto zi = fd.cFileName[].representation.countUntil(0);
+				if (zi < 0) zi = fd.cFileName.length;
+				if (fd.cFileName[0 .. zi].among("."w, ".."w))
+					continue;
+				fi.name = fd.cFileName[0 .. zi].to!string;
+				fi.directory = path;
+				fi.size = (ulong(fd.nFileSizeHigh) << 32) + fd.nFileSizeLow;
+				fi.timeModified = timebase + hnsecs((ulong(fd.ftLastWriteTime.dwHighDateTime) << 32) + fd.ftLastWriteTime.dwLowDateTime);
+				fi.timeCreated = timebase + hnsecs((ulong(fd.ftCreationTime.dwHighDateTime) << 32) + fd.ftCreationTime.dwLowDateTime);
+				fi.isSymlink = !!(fd.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT);
+				fi.isDirectory = !!(fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY);
+				fi.isFile = !fi.isDirectory && !(fd.dwFileAttributes & FILE_ATTRIBUTE_DEVICE);
+				fi.hidden = !!(fd.dwFileAttributes & FILE_ATTRIBUTE_HIDDEN);
+
+				try req.channel.put(ListDirectoryData(fi, null));
+				catch (Exception e) return false; // channel got closed
+
+				if (req.spanMode != DirectoryListMode.shallow && fi.isDirectory) {
+					if (fi.isSymlink && !req.followSymlinks)
+						continue;
+					try {
+						if (!scanRec(path ~ NativePath.Segment2(fi.name)))
+							return false;
+					} catch (Exception e) {}
+				}
+			} while (FindNextFileW(fh, &fd));
+		} else {
+			import core.sys.posix.dirent : DT_DIR, DT_LNK, DT_UNKNOWN,
+				dirent, opendir, closedir, readdir;
+			import std.string : toStringz;
+
+			static immutable timebase = SysTime(DateTime(1970, 1, 1), UTC());
+
+			auto dir = opendir(path.toString.toStringz);
+			errnoEnforce(dir !is null, path.toString);
+			scope (exit) closedir(dir);
+
+			auto dfd = dirfd(dir);
+
+			dirent* de;
+			while ((de = readdir(dir)) !is null) {
+				// skip non-directories early, if possible
+				if (dirs_only && !de.d_type.among(DT_DIR, DT_LNK, DT_UNKNOWN))
+					continue;
+
+				FileInfo fi;
+				auto zi = de.d_name[].countUntil(0);
+				if (zi < 0) zi = de.d_name.length;
+				if (de.d_name[0 .. zi].among(".", ".."))
+					continue;
+
+				fi.name = de.d_name[0 .. zi].idup;
+				fi.directory = path;
+
+				stat_t st;
+				if (fstatat(dfd, fi.name.toStringz, &st, AT_SYMLINK_NOFOLLOW) != 0)
+					continue;
+
+				fi.isSymlink = S_ISLNK(st.st_mode);
+
+				// apart from the symlink flag, get the rest of the information from the link target
+				if (fi.isSymlink) fstatat(dfd, fi.name.toStringz, &st, 0);
+
+				fi.size = st.st_size;
+				fi.timeModified = timebase + st.st_mtime.seconds + (st.st_mtimensec / 100).hnsecs;
+				fi.timeCreated = timebase + st.st_ctime.seconds + (st.st_ctimensec / 100).hnsecs;
+				fi.isDirectory = S_ISDIR(st.st_mode);
+				fi.isFile = S_ISREG(st.st_mode);
+				fi.hidden = de.d_name[0] == '.';
+
+				// skip non-directories if requested
+				if (dirs_only && !fi.isDirectory)
+					continue;
+
+				try req.channel.put(ListDirectoryData(fi, null));
+				catch (Exception e) return false; // channel got closed
+
+				if (req.spanMode != DirectoryListMode.shallow && fi.isDirectory) {
+					if (fi.isSymlink && !req.followSymlinks)
+						continue;
+					try {
+						if (!scanRec(path ~ NativePath.Segment2(fi.name)))
+							return false;
+					} catch (Exception e) {}
+				}
+			}
+		}
+
+		return true;
+	}
+
+	try scanRec(req.path);
+	catch (Exception e) {
+		logException(e, "goo");
+		try req.channel.put(ListDirectoryData(FileInfo.init, e.msg.length ? e.msg : "Failed to iterate directory"));
+		catch (Exception e2) {} // channel got closed
+	}
+}
+
+version (Posix) {
+	import core.sys.posix.dirent : DIR;
+	import core.sys.posix.sys.stat : stat;
+	extern(C) @safe nothrow @nogc {
+		static if (!is(typeof(dirfd)))
+			 int dirfd(DIR*);
+		static if (!is(typeof(fstatat)))
+			int fstatat(int dirfd, const(char)* pathname, stat_t *statbuf, int flags);
+	}
+
+	version (darwin) {
+		static if (!is(typeof(AT_SYMLINK_NOFOLLOW)))
+			enum AT_SYMLINK_NOFOLLOW = 0x0020;
+	}
+}
+
 private immutable TaskSettings ioTaskSettings = { priority: 20 * Task.basePriority };
+
+private struct ListDirectoryData {
+	FileInfo info;
+	string error;
+}
+
+private struct ListDirectoryRequest {
+	import vibe.core.channel : Channel;
+
+	NativePath path;
+	DirectoryListMode spanMode;
+	Channel!ListDirectoryData channel;
+	bool followSymlinks;
+}