vibe-core/source/vibe/core/file.d
Sönke Ludwig d9e545bf60 Fix getFileInfo's exception handling and reduce overhead of the background work.
Instead of asyncWork, now uses a worker task directly and signals the finalization of the result using message passing. This avoids the roundtrip required to return the task handle, as well as the heap allocated result buffer of Future!T.
2019-08-25 12:02:08 +02:00

909 lines
23 KiB
D

/**
File handling functions and types.
Copyright: © 2012-2019 RejectedSoftware e.K.
License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
Authors: Sönke Ludwig
*/
module vibe.core.file;
import eventcore.core : NativeEventDriver, eventDriver;
import eventcore.driver;
import vibe.core.internal.release;
import vibe.core.log;
import vibe.core.path;
import vibe.core.stream;
import vibe.internal.async : asyncAwait, asyncAwaitUninterruptible;
import core.stdc.stdio;
import core.sys.posix.unistd;
import core.sys.posix.fcntl;
import core.sys.posix.sys.stat;
import core.time;
import std.conv : octal;
import std.datetime;
import std.exception;
import std.file;
import std.path;
import std.string;
import std.typecons : Flag, No;
version(Posix){
private extern(C) int mkstemps(char* templ, int suffixlen);
}
@safe:
/**
Opens a file stream with the specified mode.
*/
FileStream openFile(NativePath path, FileMode mode = FileMode.read)
{
auto fil = eventDriver.files.open(path.toNativeString(), cast(FileOpenMode)mode);
enforce(fil != FileFD.invalid, "Failed to open file '"~path.toNativeString~"'");
return FileStream(fil, path, mode);
}
/// ditto
FileStream openFile(string path, FileMode mode = FileMode.read)
{
return openFile(NativePath(path), mode);
}
/**
Read a whole file into a buffer.
If the supplied buffer is large enough, it will be used to store the
contents of the file. Otherwise, a new buffer will be allocated.
Params:
path = The path of the file to read
buffer = An optional buffer to use for storing the file contents
*/
ubyte[] readFile(NativePath path, ubyte[] buffer = null, size_t max_size = size_t.max)
{
auto fil = openFile(path);
scope (exit) fil.close();
enforce(fil.size <= max_size, "File is too big.");
auto sz = cast(size_t)fil.size;
auto ret = sz <= buffer.length ? buffer[0 .. sz] : new ubyte[sz];
fil.read(ret);
return ret;
}
/// ditto
ubyte[] readFile(string path, ubyte[] buffer = null, size_t max_size = size_t.max)
{
return readFile(NativePath(path), buffer, max_size);
}
/**
Write a whole file at once.
*/
void writeFile(NativePath path, in ubyte[] contents)
{
auto fil = openFile(path, FileMode.createTrunc);
scope (exit) fil.close();
fil.write(contents);
}
/// ditto
void writeFile(string path, in ubyte[] contents)
{
writeFile(NativePath(path), contents);
}
/**
Convenience function to append to a file.
*/
void appendToFile(NativePath path, string data) {
auto fil = openFile(path, FileMode.append);
scope(exit) fil.close();
fil.write(data);
}
/// ditto
void appendToFile(string path, string data)
{
appendToFile(NativePath(path), data);
}
/**
Read a whole UTF-8 encoded file into a string.
The resulting string will be sanitized and will have the
optional byte order mark (BOM) removed.
*/
string readFileUTF8(NativePath path)
{
import vibe.internal.string;
return stripUTF8Bom(sanitizeUTF8(readFile(path)));
}
/// ditto
string readFileUTF8(string path)
{
return readFileUTF8(NativePath(path));
}
/**
Write a string into a UTF-8 encoded file.
The file will have a byte order mark (BOM) prepended.
*/
void writeFileUTF8(NativePath path, string contents)
{
static immutable ubyte[] bom = [0xEF, 0xBB, 0xBF];
auto fil = openFile(path, FileMode.createTrunc);
scope (exit) fil.close();
fil.write(bom);
fil.write(contents);
}
/**
Creates and opens a temporary file for writing.
*/
FileStream createTempFile(string suffix = null)
{
version(Windows){
import std.conv : to;
string tmpname;
() @trusted {
auto fn = tmpnam(null);
enforce(fn !is null, "Failed to generate temporary name.");
tmpname = to!string(fn);
} ();
if (tmpname.startsWith("\\")) tmpname = tmpname[1 .. $];
tmpname ~= suffix;
return openFile(tmpname, FileMode.createTrunc);
} else {
enum pattern ="/tmp/vtmp.XXXXXX";
scope templ = new char[pattern.length+suffix.length+1];
templ[0 .. pattern.length] = pattern;
templ[pattern.length .. $-1] = (suffix)[];
templ[$-1] = '\0';
assert(suffix.length <= int.max);
auto fd = () @trusted { return mkstemps(templ.ptr, cast(int)suffix.length); } ();
enforce(fd >= 0, "Failed to create temporary file.");
auto efd = eventDriver.files.adopt(fd);
return FileStream(efd, NativePath(templ[0 .. $-1].idup), FileMode.createTrunc);
}
}
/**
Moves or renames a file.
Params:
from = Path to the file/directory to move/rename.
to = The target path
copy_fallback = Determines if copy/remove should be used in case of the
source and destination path pointing to different devices.
*/
void moveFile(NativePath from, NativePath to, bool copy_fallback = false)
{
moveFile(from.toNativeString(), to.toNativeString(), copy_fallback);
}
/// ditto
void moveFile(string from, string to, bool copy_fallback = false)
{
if (!copy_fallback) {
std.file.rename(from, to);
} else {
try {
std.file.rename(from, to);
} catch (FileException e) {
copyFile(from, to);
std.file.remove(from);
}
}
}
/**
Copies a file.
Note that attributes and time stamps are currently not retained.
Params:
from = Path of the source file
to = Path for the destination file
overwrite = If true, any file existing at the destination path will be
overwritten. If this is false, an exception will be thrown should
a file already exist at the destination path.
Throws:
An Exception if the copy operation fails for some reason.
*/
void copyFile(NativePath from, NativePath to, bool overwrite = false)
{
DirEntry info;
static if (__VERSION__ < 2078) {
() @trusted {
info = DirEntry(from.toString);
enforce(info.isFile, "The source path is not a file and cannot be copied.");
} ();
} else {
info = DirEntry(from.toString);
enforce(info.isFile, "The source path is not a file and cannot be copied.");
}
{
auto src = openFile(from, FileMode.read);
scope(exit) src.close();
enforce(overwrite || !existsFile(to), "Destination file already exists.");
auto dst = openFile(to, FileMode.createTrunc);
scope(exit) dst.close();
dst.write(src);
}
// TODO: also retain creation time on windows
static if (__VERSION__ < 2078) {
() @trusted {
setTimes(to.toString, info.timeLastAccessed, info.timeLastModified);
setAttributes(to.toString, info.attributes);
} ();
} else {
setTimes(to.toString, info.timeLastAccessed, info.timeLastModified);
setAttributes(to.toString, info.attributes);
}
}
/// ditto
void copyFile(string from, string to)
{
copyFile(NativePath(from), NativePath(to));
}
/**
Removes a file
*/
void removeFile(NativePath path)
{
removeFile(path.toNativeString());
}
/// ditto
void removeFile(string path)
{
std.file.remove(path);
}
/**
Checks if a file exists
*/
bool existsFile(NativePath path) nothrow
{
return existsFile(path.toNativeString());
}
/// ditto
bool existsFile(string path) nothrow
{
// This was *annotated* nothrow in 2.067.
static if (__VERSION__ < 2067)
scope(failure) assert(0, "Error: existsFile should never throw");
try return performInWorker((string p) => std.file.exists(p), path);
catch (Exception e) {
logDebug("Failed to determine file existence for '%s': %s", path, e.msg);
return false;
}
}
/** Stores information about the specified file/directory into 'info'
Throws: A `FileException` is thrown if the file does not exist.
*/
FileInfo getFileInfo(NativePath path)
@trusted {
return getFileInfo(path.toNativeString);
}
/// ditto
FileInfo getFileInfo(string path)
{
import std.typecons : tuple;
auto ret = performInWorker((string p) {
try {
auto ent = DirEntry(p);
return tuple(makeFileInfo(ent), "");
} catch (Exception e) {
return tuple(FileInfo.init, e.msg.length ? e.msg : "Failed to get file information");
}
}, path);
if (ret[1].length) throw new Exception(ret[1]);
return ret[0];
}
/**
Creates a new directory.
*/
void createDirectory(NativePath path)
{
createDirectory(path.toNativeString);
}
/// ditto
void createDirectory(string path, Flag!"recursive" recursive = No.recursive)
{
auto fail = performInWorker((string p, bool rec) {
try {
if (rec) mkdirRecurse(p);
else mkdir(p);
} catch (Exception e) {
return e.msg.length ? e.msg : "Failed to create directory.";
}
return null;
}, path, !!recursive);
if (fail) throw new Exception(fail);
}
/**
Enumerates all files in the specified directory.
*/
void listDirectory(NativePath path, scope bool delegate(FileInfo info) @safe del)
{
listDirectory(path.toNativeString, del);
}
/// ditto
void listDirectory(string path, scope bool delegate(FileInfo info) @safe del)
{
import vibe.core.core : runWorkerTaskH;
import vibe.core.channel : Channel, createChannel;
struct S {
FileInfo info;
string error;
}
auto ch = createChannel!S();
runWorkerTaskH((string path, Channel!S ch) nothrow {
scope (exit) ch.close();
try {
foreach (DirEntry ent; dirEntries(path, SpanMode.shallow)) {
auto nfo = makeFileInfo(ent);
try ch.put(S(nfo, null));
catch (Exception e) break; // channel got closed
}
} catch (Exception e) {
try ch.put(S(FileInfo.init, e.msg.length ? e.msg : "Failed to iterate directory"));
catch (Exception e) {} // channel got closed
}
}, path, ch);
S itm;
while (ch.tryConsumeOne(itm)) {
if (itm.error.length)
throw new Exception(itm.error);
if (!del(itm.info)) {
ch.close();
break;
}
}
}
/// ditto
void listDirectory(NativePath path, scope bool delegate(FileInfo info) @system del)
@system {
listDirectory(path, (nfo) @trusted => del(nfo));
}
/// ditto
void listDirectory(string path, scope bool delegate(FileInfo info) @system del)
@system {
listDirectory(path, (nfo) @trusted => del(nfo));
}
/// ditto
int delegate(scope int delegate(ref FileInfo)) iterateDirectory(NativePath path)
{
int iterator(scope int delegate(ref FileInfo) del){
int ret = 0;
listDirectory(path, (fi){
ret = del(fi);
return ret == 0;
});
return ret;
}
return &iterator;
}
/// ditto
int delegate(scope int delegate(ref FileInfo)) iterateDirectory(string path)
{
return iterateDirectory(NativePath(path));
}
/**
Starts watching a directory for changes.
*/
DirectoryWatcher watchDirectory(NativePath path, bool recursive = true)
{
return DirectoryWatcher(path, recursive);
}
// ditto
DirectoryWatcher watchDirectory(string path, bool recursive = true)
{
return watchDirectory(NativePath(path), recursive);
}
/**
Returns the current working directory.
*/
NativePath getWorkingDirectory()
{
return NativePath(() @trusted { return std.file.getcwd(); } ());
}
/** Contains general information about a file.
*/
struct FileInfo {
/// Name of the file (not including the path)
string name;
/// Size of the file (zero for directories)
ulong size;
/// Time of the last modification
SysTime timeModified;
/// Time of creation (not available on all operating systems/file systems)
SysTime timeCreated;
/// True if this is a symlink to an actual file
bool isSymlink;
/// True if this is a directory or a symlink pointing to a directory
bool isDirectory;
/// True if this is a file. On POSIX if both isFile and isDirectory are false it is a special file.
bool isFile;
/** True if the file's hidden attribute is set.
On systems that don't support a hidden attribute, any file starting with
a single dot will be treated as hidden.
*/
bool hidden;
}
/**
Specifies how a file is manipulated on disk.
*/
enum FileMode {
/// The file is opened read-only.
read = FileOpenMode.read,
/// The file is opened for read-write random access.
readWrite = FileOpenMode.readWrite,
/// The file is truncated if it exists or created otherwise and then opened for read-write access.
createTrunc = FileOpenMode.createTrunc,
/// The file is opened for appending data to it and created if it does not exist.
append = FileOpenMode.append
}
/**
Accesses the contents of a file as a stream.
*/
struct FileStream {
@safe:
private struct CTX {
NativePath path;
ulong size;
FileMode mode;
ulong ptr;
shared(NativeEventDriver) driver;
}
private {
FileFD m_fd;
CTX* m_ctx;
}
private this(FileFD fd, NativePath path, FileMode mode)
{
assert(fd != FileFD.invalid, "Constructing FileStream from invalid file descriptor.");
m_fd = fd;
m_ctx = new CTX; // TODO: use FD custom storage
m_ctx.path = path;
m_ctx.mode = mode;
m_ctx.size = eventDriver.files.getSize(fd);
m_ctx.driver = () @trusted { return cast(shared)eventDriver; } ();
if (mode == FileMode.append)
m_ctx.ptr = m_ctx.size;
}
this(this)
{
if (m_fd != FileFD.invalid)
eventDriver.files.addRef(m_fd);
}
~this()
{
if (m_fd != FileFD.invalid)
releaseHandle!"files"(m_fd, m_ctx.driver);
}
@property int fd() { return cast(int)m_fd; }
/// The path of the file.
@property NativePath path() const { return ctx.path; }
/// Determines if the file stream is still open
@property bool isOpen() const { return m_fd != FileFD.invalid; }
@property ulong size() const nothrow { return ctx.size; }
@property bool readable() const nothrow { return ctx.mode != FileMode.append; }
@property bool writable() const nothrow { return ctx.mode != FileMode.read; }
bool opCast(T)() if (is (T == bool)) { return m_fd != FileFD.invalid; }
void takeOwnershipOfFD()
{
assert(false, "TODO!");
}
void seek(ulong offset)
{
enforce(ctx.mode != FileMode.append, "File opened for appending, not random access. Cannot seek.");
ctx.ptr = offset;
}
ulong tell() nothrow { return ctx.ptr; }
void truncate(ulong size)
{
enforce(ctx.mode != FileMode.append, "File opened for appending, not random access. Cannot truncate.");
auto res = asyncAwaitUninterruptible!(FileIOCallback,
cb => eventDriver.files.truncate(m_fd, size, cb)
);
enforce(res[1] == IOStatus.ok, "Failed to resize file.");
m_ctx.size = size;
}
/// Closes the file handle.
void close()
{
if (m_fd != FileFD.init) {
eventDriver.files.close(m_fd); // FIXME: may leave dangling references!
releaseHandle!"files"(m_fd, m_ctx.driver);
m_fd = FileFD.init;
m_ctx = null;
}
}
@property bool empty() const { assert(this.readable); return ctx.ptr >= ctx.size; }
@property ulong leastSize() const { assert(this.readable); return ctx.size - ctx.ptr; }
@property bool dataAvailableForRead() { return true; }
const(ubyte)[] peek()
{
return null;
}
size_t read(ubyte[] dst, IOMode mode)
{
auto res = asyncAwait!(FileIOCallback,
cb => eventDriver.files.read(m_fd, ctx.ptr, dst, mode, cb),
cb => eventDriver.files.cancelRead(m_fd)
);
ctx.ptr += res[2];
enforce(res[1] == IOStatus.ok, "Failed to read data from disk.");
return res[2];
}
void read(ubyte[] dst)
{
auto ret = read(dst, IOMode.all);
assert(ret == dst.length, "File.read returned less data than requested for IOMode.all.");
}
size_t write(in ubyte[] bytes, IOMode mode)
{
auto res = asyncAwait!(FileIOCallback,
cb => eventDriver.files.write(m_fd, ctx.ptr, bytes, mode, cb),
cb => eventDriver.files.cancelWrite(m_fd)
);
ctx.ptr += res[2];
if (ctx.ptr > ctx.size) ctx.size = ctx.ptr;
enforce(res[1] == IOStatus.ok, "Failed to write data to disk.");
return res[2];
}
void write(in ubyte[] bytes)
{
write(bytes, IOMode.all);
}
void write(in char[] bytes)
{
write(cast(const(ubyte)[])bytes);
}
void write(InputStream)(InputStream stream, ulong nbytes = ulong.max)
if (isInputStream!InputStream)
{
writeDefault(this, stream, nbytes);
}
void flush()
{
assert(this.writable);
}
void finalize()
{
flush();
}
private inout(CTX)* ctx() inout nothrow { return m_ctx; }
}
mixin validateRandomAccessStream!FileStream;
private void writeDefault(OutputStream, InputStream)(ref OutputStream dst, InputStream stream, ulong nbytes = ulong.max)
if (isOutputStream!OutputStream && isInputStream!InputStream)
{
import vibe.internal.allocator : theAllocator, make, dispose;
import std.algorithm.comparison : min;
static struct Buffer { ubyte[64*1024] bytes = void; }
auto bufferobj = () @trusted { return theAllocator.make!Buffer(); } ();
scope (exit) () @trusted { theAllocator.dispose(bufferobj); } ();
auto buffer = bufferobj.bytes[];
//logTrace("default write %d bytes, empty=%s", nbytes, stream.empty);
if (nbytes == ulong.max) {
while (!stream.empty) {
size_t chunk = min(stream.leastSize, buffer.length);
assert(chunk > 0, "leastSize returned zero for non-empty stream.");
//logTrace("read pipe chunk %d", chunk);
stream.read(buffer[0 .. chunk]);
dst.write(buffer[0 .. chunk]);
}
} else {
while (nbytes > 0) {
size_t chunk = min(nbytes, buffer.length);
//logTrace("read pipe chunk %d", chunk);
stream.read(buffer[0 .. chunk]);
dst.write(buffer[0 .. chunk]);
nbytes -= chunk;
}
}
}
/**
Interface for directory watcher implementations.
Directory watchers monitor the contents of a directory (wither recursively or non-recursively)
for changes, such as file additions, deletions or modifications.
*/
struct DirectoryWatcher { // TODO: avoid all those heap allocations!
import std.array : Appender, appender;
import vibe.core.sync : LocalManualEvent, createManualEvent;
@safe:
private static struct Context {
NativePath path;
bool recursive;
Appender!(DirectoryChange[]) changes;
LocalManualEvent changeEvent;
shared(NativeEventDriver) driver;
void onChange(WatcherID, in ref FileChange change)
nothrow {
DirectoryChangeType ct;
final switch (change.kind) {
case FileChangeKind.added: ct = DirectoryChangeType.added; break;
case FileChangeKind.removed: ct = DirectoryChangeType.removed; break;
case FileChangeKind.modified: ct = DirectoryChangeType.modified; break;
}
static if (is(typeof(change.baseDirectory))) {
// eventcore 0.8.23 and up
this.changes ~= DirectoryChange(ct, NativePath.fromTrustedString(change.baseDirectory) ~ NativePath.fromTrustedString(change.directory) ~ NativePath.fromTrustedString(change.name.idup));
} else {
this.changes ~= DirectoryChange(ct, NativePath.fromTrustedString(change.directory) ~ NativePath.fromTrustedString(change.name.idup));
}
this.changeEvent.emit();
}
}
private {
WatcherID m_watcher;
Context* m_context;
}
private this(NativePath path, bool recursive)
{
m_context = new Context; // FIME: avoid GC allocation (use FD user data slot)
m_context.changeEvent = createManualEvent();
m_watcher = eventDriver.watchers.watchDirectory(path.toNativeString, recursive, &m_context.onChange);
enforce(m_watcher != WatcherID.invalid, "Failed to watch directory.");
m_context.path = path;
m_context.recursive = recursive;
m_context.changes = appender!(DirectoryChange[]);
m_context.driver = () @trusted { return cast(shared)eventDriver; } ();
}
this(this) nothrow { if (m_watcher != WatcherID.invalid) eventDriver.watchers.addRef(m_watcher); }
~this()
nothrow {
if (m_watcher != WatcherID.invalid)
releaseHandle!"watchers"(m_watcher, m_context.driver);
}
/// The path of the watched directory
@property NativePath path() const nothrow { return m_context.path; }
/// Indicates if the directory is watched recursively
@property bool recursive() const nothrow { return m_context.recursive; }
/** Fills the destination array with all changes that occurred since the last call.
The function will block until either directory changes have occurred or until the
timeout has elapsed. Specifying a negative duration will cause the function to
wait without a timeout.
Params:
dst = The destination array to which the changes will be appended
timeout = Optional timeout for the read operation. A value of
`Duration.max` will wait indefinitely.
Returns:
If the call completed successfully, true is returned.
*/
bool readChanges(ref DirectoryChange[] dst, Duration timeout = Duration.max)
{
if (timeout == Duration.max) {
while (!m_context.changes.data.length)
m_context.changeEvent.wait(Duration.max, m_context.changeEvent.emitCount);
} else {
MonoTime now = MonoTime.currTime();
MonoTime final_time = now + timeout;
while (!m_context.changes.data.length) {
m_context.changeEvent.wait(final_time - now, m_context.changeEvent.emitCount);
now = MonoTime.currTime();
if (now >= final_time) break;
}
if (!m_context.changes.data.length) return false;
}
dst = m_context.changes.data;
m_context.changes = appender!(DirectoryChange[]);
return true;
}
}
/** Specifies the kind of change in a watched directory.
*/
enum DirectoryChangeType {
/// A file or directory was added
added,
/// A file or directory was deleted
removed,
/// A file or directory was modified
modified
}
/** Describes a single change in a watched directory.
*/
struct DirectoryChange {
/// The type of change
DirectoryChangeType type;
/// Path of the file/directory that was changed
NativePath path;
}
private FileInfo makeFileInfo(DirEntry ent)
@trusted nothrow {
import std.algorithm.comparison : among;
FileInfo ret;
string fullname = ent.name;
if (ent.name.length) {
if (ent.name[$-1].among('/', '\\'))
fullname = ent.name[0 .. $-1];
ret.name = baseName(fullname);
if (ret.name.length == 0) ret.name = fullname;
}
try {
ret.isFile = ent.isFile;
ret.isDirectory = ent.isDir;
ret.isSymlink = ent.isSymlink;
ret.timeModified = ent.timeLastModified;
version(Windows) ret.timeCreated = ent.timeCreated;
else ret.timeCreated = ent.timeLastModified;
ret.size = ent.size;
} catch (Exception e) {
logDebug("Failed to get information for file '%s': %s", fullname, e.msg);
}
version (Windows) {
import core.sys.windows.windows : FILE_ATTRIBUTE_HIDDEN;
ret.hidden = (ent.attributes & FILE_ATTRIBUTE_HIDDEN) != 0;
}
else ret.hidden = ret.name.length > 1 && ret.name[0] == '.' && ret.name != "..";
return ret;
}
version (Windows) {} else unittest {
void test(string name_in, string name_out, bool hidden) {
auto de = DirEntry(name_in);
assert(makeFileInfo(de).hidden == hidden);
assert(makeFileInfo(de).name == name_out);
}
void testCreate(string name_in, string name_out, bool hidden)
{
if (name_in.endsWith("/"))
createDirectory(name_in);
else writeFileUTF8(NativePath(name_in), name_in);
scope (exit) removeFile(name_in);
test(name_in, name_out, hidden);
}
test(".", ".", false);
test("..", "..", false);
testCreate(".test_foo", ".test_foo", true);
test("./", ".", false);
testCreate(".test_foo/", ".test_foo", true);
test("/", "", false);
}
unittest {
auto name = "toAppend.txt";
scope(exit) removeFile(name);
{
auto handle = openFile(name, FileMode.createTrunc);
handle.write("create,");
assert(handle.tell() == "create,".length);
handle.close();
}
{
auto handle = openFile(name, FileMode.append);
handle.write(" then append");
assert(handle.tell() == "create, then append".length);
handle.close();
}
assert(readFile(name) == "create, then append");
}
private auto performInWorker(C, ARGS...)(C callable, auto ref ARGS args)
{
version (none) {
import vibe.core.concurrency : asyncWork;
return asyncWork(callable, args).getResult();
} else {
import vibe.core.core : runWorkerTask;
import core.atomic : atomicFence;
import std.concurrency : Tid, send, receiveOnly, thisTid;
struct R {}
alias RET = typeof(callable(args));
shared(RET) ret;
runWorkerTask((shared(RET)* r, Tid caller, C c, ref ARGS a) nothrow {
*() @trusted { return cast(RET*)r; } () = c(a);
// Just as a precaution, because ManualEvent is not well defined in
// terms of fence semantics
atomicFence();
try caller.send(R.init);
catch (Exception e) assert(false, e.msg);
}, () @trusted { return &ret; } (), thisTid, callable, args);
() @trusted { receiveOnly!R(); } ();
atomicFence();
return ret;
}
}