add System/NativeFile

This commit is contained in:
falsycat 2022-06-04 10:27:06 +09:00
parent 50b84e9b65
commit 5e3c053fdd
11 changed files with 762 additions and 4 deletions

View File

@ -45,6 +45,10 @@ target_sources(nf7
nf7.hh
common/aggregate_command.hh
common/async_buffer.hh
common/async_buffer_adaptor.hh
common/buffer.hh
common/conditional_queue.hh
common/dir.hh
common/dir_item.hh
common/file_ref.hh
@ -58,6 +62,7 @@ target_sources(nf7
common/history.hh
common/lambda.hh
common/memento.hh
common/native_file.hh
common/node.hh
common/node_link_store.hh
common/logger.hh
@ -68,11 +73,14 @@ target_sources(nf7
common/wait_queue.hh
common/yas.hh
$<$<PLATFORM_ID:Linux>:common/native_file_unix.cc>
file/luajit_context.cc
file/node_network.cc
file/system_dir.cc
file/system_imgui_config.cc
file/system_logger.cc
file/system_native_file.cc
)
target_link_libraries(nf7
PRIVATE

38
common/async_buffer.hh Normal file
View File

@ -0,0 +1,38 @@
#pragma once
#include <cstdint>
#include <exception>
#include <future>
#include <memory>
#include "nf7.hh"
#include "common/buffer.hh"
#include "common/lock.hh"
namespace nf7 {
class AsyncBuffer : public nf7::File::Interface, public nf7::Lock::Resource {
public:
using IOException = Buffer::IOException;
AsyncBuffer() = default;
AsyncBuffer(const AsyncBuffer&) = delete;
AsyncBuffer(AsyncBuffer&&) = delete;
AsyncBuffer& operator=(const AsyncBuffer&) = delete;
AsyncBuffer& operator=(AsyncBuffer&&) = delete;
virtual std::future<size_t> Read(size_t offset, uint8_t* buf, size_t size) noexcept = 0;
virtual std::future<size_t> Write(size_t offset, const uint8_t* buf, size_t size) noexcept = 0;
virtual std::future<size_t> Truncate(size_t) noexcept = 0;
virtual std::future<size_t> size() const noexcept = 0;
virtual Buffer::Flags flags() const noexcept = 0;
protected:
using nf7::Lock::Resource::OnLock;
using nf7::Lock::Resource::OnUnlock;
};
} // namespace nf7

View File

@ -0,0 +1,107 @@
#pragma once
#include <functional>
#include <future>
#include <memory>
#include <mutex>
#include <utility>
#include "nf7.hh"
#include "common/async_buffer.hh"
#include "common/buffer.hh"
#include "common/queue.hh"
namespace nf7 {
class AsyncBufferAdaptor : public nf7::AsyncBuffer {
public:
AsyncBufferAdaptor(const std::shared_ptr<nf7::Context>& ctx,
const std::shared_ptr<nf7::Buffer>& buf) noexcept :
data_(std::make_shared<Data>()) {
data_->ctx = ctx;
data_->buf = buf;
}
std::future<size_t> Read(size_t offset, uint8_t* ptr, size_t size) noexcept override {
return ExecWithPromise<size_t>(
[buf = data_->buf, offset, ptr, size]() {
return buf->Read(offset, ptr, size);
});
}
std::future<size_t> Write(size_t offset, const uint8_t* ptr, size_t size) noexcept override {
return ExecWithPromise<size_t>(
[buf = data_->buf, offset, ptr, size]() {
return buf->Write(offset, ptr, size);
});
}
std::future<size_t> Truncate(size_t size) noexcept override {
return ExecWithPromise<size_t>(
[buf = data_->buf, size]() { return buf->Truncate(size); });
}
std::future<size_t> size() const noexcept override {
return const_cast<AsyncBufferAdaptor&>(*this).
ExecWithPromise<size_t>(
[buf = data_->buf]() { return buf->size(); });
}
Buffer::Flags flags() const noexcept override {
return data_->buf->flags();
}
protected:
void OnLock() noexcept override {
Exec([buf = data_->buf]() { return buf->Lock(); });
}
void OnUnlock() noexcept override {
Exec([buf = data_->buf]() { return buf->Unlock(); });
}
private:
struct Data {
std::shared_ptr<nf7::Context> ctx;
std::shared_ptr<nf7::Buffer> buf;
std::mutex mtx;
bool working = false;
nf7::Queue<std::function<void()>> q;
};
std::shared_ptr<Data> data_;
template <typename R>
std::future<R> ExecWithPromise(std::function<R()>&& f) noexcept {
auto pro = std::make_shared<std::promise<R>>();
auto task = [pro, f = std::move(f)]() {
try {
pro->set_value(f());
} catch (...) {
pro->set_exception(std::current_exception());
}
};
Exec(std::move(task));
return pro->get_future();
}
void Exec(std::function<void()>&& f) noexcept {
data_->q.Push(std::move(f));
std::unique_lock<std::mutex> k(data_->mtx);
if (!std::exchange(data_->working, true)) {
data_->ctx->env().ExecAsync(
data_->ctx, [data = data_]() { Handle(data); });
}
}
static void Handle(const std::shared_ptr<Data>& data) noexcept {
std::unique_lock<std::mutex> k(data->mtx);
if (auto task = data->q.Pop()) {
k.unlock();
(*task)();
data->ctx->env().ExecAsync(data->ctx, [data]() { Handle(data); });
} else {
data->working = false;
}
}
};
} // namespace nf7

40
common/buffer.hh Normal file
View File

@ -0,0 +1,40 @@
#pragma once
#include <cstdint>
namespace nf7 {
class Buffer {
public:
enum Flag : uint8_t {
kRead = 1 << 0,
kWrite = 1 << 1,
};
using Flags = uint8_t;
class IOException;
Buffer() = default;
virtual ~Buffer() = default;
Buffer(const Buffer&) = delete;
Buffer(Buffer&&) = delete;
Buffer& operator=(const Buffer&) = delete;
Buffer& operator=(Buffer&&) = delete;
virtual void Lock() = 0;
virtual void Unlock() = 0;
virtual size_t Read(size_t offset, uint8_t* buf, size_t size) = 0;
virtual size_t Write(size_t offset, const uint8_t* buf, size_t size) = 0;
virtual size_t Truncate(size_t size) = 0;
virtual size_t size() const = 0;
virtual Flags flags() const noexcept = 0;
};
class Buffer::IOException : public nf7::Exception {
public:
using Exception::Exception;
};
} // namespace nf7

View File

@ -0,0 +1,60 @@
#pragma once
#include <chrono>
#include <memory>
#include <functional>
#include <future>
#include "common/queue.hh"
namespace nf7 {
class ConditionalQueue final :
public nf7::Queue<std::function<bool(void)>> {
public:
ConditionalQueue() = default;
template <typename T>
void Push(std::future<T>&& fu, auto&& f) {
auto fu_ptr = std::make_shared<std::future<T>>(std::move(fu));
auto task = [fu_ptr = std::move(fu_ptr), f = std::move(f)]() mutable {
if (fu_ptr->wait_for(std::chrono::seconds(0)) != std::future_status::ready) {
return false;
}
f(std::move(*fu_ptr));
return true;
};
Queue<std::function<bool(void)>>::Push(std::move(task));
}
template <typename T>
void Push(std::shared_future<T> fu, auto&& f) {
auto task = [fu, f = std::move(f)]() mutable {
if (fu.wait_for(std::chrono::seconds(0)) != std::future_status::ready) {
return false;
}
f(std::move(fu));
return true;
};
Queue<std::function<bool(void)>>::Push(std::move(task));
}
void Push(const std::shared_ptr<nf7::Lock>& k, std::function<void(const std::shared_ptr<nf7::Lock>&)>&& f) {
auto task = [k, f = std::move(f)]() {
if (!k->acquired() && !k->cancelled()) {
return false;
}
f(k);
return true;
};
Queue<std::function<bool(void)>>::Push(std::move(task));
}
bool PopAndExec() noexcept {
if (auto task = Pop()) {
if ((*task)()) return true;
Interrupt(std::move(*task));
}
return false;
}
};
} // namespace nf7

96
common/lock.hh Normal file
View File

@ -0,0 +1,96 @@
#pragma once
#include <algorithm>
#include <deque>
#include <memory>
#include <utility>
#include <vector>
#include "nf7.hh"
namespace nf7 {
class Lock {
public:
class Resource;
Lock() = default;
Lock(Resource& res, bool ex) noexcept : res_(&res), ex_(ex) {
}
inline ~Lock() noexcept;
Lock(const Lock&) = delete;
Lock(Lock&&) = delete;
Lock& operator=(const Lock&) = delete;
Lock& operator=(Lock&&) = delete;
bool cancelled() const noexcept { return !res_; }
bool acquired() const noexcept { return acquired_; }
private:
Resource* res_ = nullptr;
bool ex_ = false;
bool acquired_ = false;
};
class Lock::Resource {
public:
friend Lock;
Resource() = default;
virtual ~Resource() noexcept {
if (auto lock = lock_.lock()) {
lock->res_ = nullptr;
}
for (auto lock : plocks_) {
lock->res_ = nullptr;
}
}
Resource(const Resource&) = delete;
Resource(Resource&&) = delete;
Resource& operator=(const Resource&) = delete;
Resource& operator=(Resource&&) = delete;
std::shared_ptr<Lock> Acquire(bool ex) noexcept {
if (auto ret = TryAcquire(ex)) return ret;
if (!ex && !plocks_.empty() && !plocks_.back()->ex_) {
return plocks_.back();
}
plocks_.push_back(std::make_shared<Lock>(*this, ex));
return plocks_.back();
}
std::shared_ptr<Lock> TryAcquire(bool ex) noexcept {
if (!lock_.expired()) return nullptr;
auto ret = std::make_shared<Lock>(*this, ex);
ret->acquired_ = true;
lock_ = ret;
OnLock();
return ret;
}
protected:
virtual void OnLock() noexcept { }
virtual void OnUnlock() noexcept { }
private:
std::weak_ptr<Lock> lock_;
std::deque<std::shared_ptr<Lock>> plocks_;
};
Lock::~Lock() noexcept {
if (!res_) return;
if (res_->plocks_.empty()) {
res_->OnUnlock();
return;
}
auto next = std::move(res_->plocks_.front());
res_->plocks_.pop_front();
res_->lock_ = next;
next->acquired_ = true;
}
} // namespace nf7

62
common/native_file.hh Normal file
View File

@ -0,0 +1,62 @@
#pragma once
#include <cstdint>
#include <filesystem>
#include <optional>
#include <string>
#include <string_view>
#include "nf7.hh"
#include "common/buffer.hh"
namespace nf7 {
class NativeFile final : public nf7::Buffer, public nf7::Context {
public:
enum Flag : uint8_t {
kCreateIf = 1 << 0,
kExclusive = 1 << 1,
kTrunc = 1 << 2,
};
using Flags = uint8_t;
NativeFile() = delete;
NativeFile(nf7::File& f,
const std::filesystem::path& path,
Buffer::Flags flags,
Flags nflags) noexcept :
Context(f.env(), f.id()), path_(path), flags_(flags), nflags_(nflags) {
}
NativeFile(const NativeFile&) = delete;
NativeFile(NativeFile&&) = delete;
NativeFile& operator=(const NativeFile&) = delete;
NativeFile& operator=(NativeFile&&) = delete;
void Lock() override;
void Unlock() override;
size_t Read(size_t offset, uint8_t* buf, size_t size) override;
size_t Write(size_t offset, const uint8_t* buf, size_t size) override;
size_t Truncate(size_t size) override;
size_t size() const override;
Buffer::Flags flags() const noexcept override {
return flags_;
}
void CleanUp() noexcept override;
void Abort() noexcept override;
size_t GetMemoryUsage() const noexcept override;
std::string GetDescription() const noexcept override;
private:
const std::filesystem::path path_;
const Buffer::Flags flags_;
const NativeFile::Flags nflags_;
std::optional<uint64_t> handle_;
};
} // namespace nf7

130
common/native_file_unix.cc Normal file
View File

@ -0,0 +1,130 @@
#include "common/native_file.hh"
extern "C" {
#include <fcntl.h>
#include <sys/file.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
}
#include <thread>
namespace nf7 {
void NativeFile::Lock() {
if (handle_) {
throw nf7::Buffer::IOException("already locked");
}
int flags = 0;
if ((flags_ & nf7::Buffer::kRead) && (flags_ & nf7::Buffer::kWrite)) {
flags |= O_RDWR;
} else if (flags_ & nf7::Buffer::kRead) {
flags |= O_RDONLY;
} else if (flags_ & nf7::Buffer::kWrite) {
flags |= O_WRONLY;
}
if (nflags_ & kCreateIf) flags |= O_CREAT;
if (nflags_ & kTrunc) flags |= O_TRUNC;
int fd = open(path_.string().c_str(), flags, 0600);
if (fd < 0) {
throw nf7::Buffer::IOException("open failure");
}
handle_ = static_cast<uint64_t>(fd);
if (nflags_ & kExclusive) {
if (flock(fd, LOCK_EX) != 0) {
close(fd);
throw nf7::Buffer::IOException("flock failure");
}
}
}
void NativeFile::Unlock() {
if (!handle_) {
throw nf7::Buffer::IOException("not locked yet");
}
const auto fd = static_cast<int>(*handle_);
if (nflags_ & kExclusive) {
if (flock(fd, LOCK_UN) != 0) {
close(fd);
throw nf7::Buffer::IOException("flock failure");
}
}
if (close(fd) == -1) {
throw nf7::Buffer::IOException("close failure");
}
handle_ = std::nullopt;
}
size_t NativeFile::Read(size_t offset, uint8_t* buf, size_t size) {
if (!handle_) {
throw nf7::Buffer::IOException("not locked yet");
}
const auto fd = static_cast<int>(*handle_);
const auto off = static_cast<off_t>(offset);
if (lseek(fd, off, SEEK_SET) == off-1) {
throw nf7::Buffer::IOException("lseek failure");
}
const auto ret = read(fd, buf, size);
if (ret == -1) {
throw nf7::Buffer::IOException("read failure");
}
return static_cast<size_t>(ret);
}
size_t NativeFile::Write(size_t offset, const uint8_t* buf, size_t size) {
if (!handle_) {
throw nf7::Buffer::IOException("not locked yet");
}
const auto fd = static_cast<int>(*handle_);
const auto off = static_cast<off_t>(offset);
if (lseek(fd, off, SEEK_SET) == off-1) {
throw nf7::Buffer::IOException("lseek failure");
}
const auto ret = write(fd, buf, size);
if (ret == -1) {
throw nf7::Buffer::IOException("write failure");
}
return static_cast<size_t>(ret);
}
size_t NativeFile::Truncate(size_t size) {
if (!handle_) {
throw nf7::Buffer::IOException("not locked yet");
}
const auto fd = static_cast<int>(*handle_);
if (ftruncate(fd, static_cast<off_t>(size)) == 0) {
throw nf7::Buffer::IOException("ftruncate failure");
}
return size;
}
size_t NativeFile::size() const {
if (!handle_) {
throw nf7::Buffer::IOException("not locked yet");
}
const auto fd = static_cast<int>(*handle_);
const auto ret = lseek(fd, 0, SEEK_END);
if (ret == -1) {
throw nf7::Buffer::IOException("lseek failure");
}
return static_cast<size_t>(ret);
}
void NativeFile::CleanUp() noexcept {
}
void NativeFile::Abort() noexcept {
}
size_t NativeFile::GetMemoryUsage() const noexcept {
return 0;
}
std::string NativeFile::GetDescription() const noexcept {
if (!handle_) {
return "unix file descriptor: "+path_.string();
} else {
return "unix file descriptor (active): "+path_.string();
}
}
} // namespace nf7

View File

@ -13,15 +13,19 @@ template <typename T>
class Queue {
public:
Queue() = default;
Queue(const Queue&) = default;
Queue(Queue&&) = default;
Queue& operator=(const Queue&) = default;
Queue& operator=(Queue&&) = default;
Queue(const Queue&) = delete;
Queue(Queue&&) = delete;
Queue& operator=(const Queue&) = delete;
Queue& operator=(Queue&&) = delete;
void Push(T&& task) noexcept {
std::unique_lock<std::mutex> _(mtx_);
tasks_.push_back(std::move(task));
}
void Interrupt(T&& task) noexcept {
std::unique_lock<std::mutex> _(mtx_);
tasks_.push_front(std::move(task));
}
std::optional<T> Pop() noexcept {
std::unique_lock<std::mutex> k(mtx_);
if (tasks_.empty()) return std::nullopt;
@ -31,6 +35,11 @@ class Queue {
return ret;
}
void Clear() noexcept {
std::unique_lock<std::mutex> k(mtx_);
tasks_.clear();
}
protected:
std::mutex mtx_;

View File

@ -0,0 +1,33 @@
#pragma once
#include <filesystem>
#include <string>
#include <yas/serialize.hpp>
#include <yas/types/std/string.hpp>
namespace yas::detail {
template <size_t F>
struct serializer<
type_prop::not_a_fundamental,
ser_case::use_internal_serializer,
F,
std::filesystem::path> {
public:
template <typename Archive>
static Archive& save(Archive& ar, const std::filesystem::path& p) {
ar(p.generic_string());
return ar;
}
template <typename Archive>
static Archive& load(Archive& ar, std::filesystem::path& p) {
std::string str;
ar(str);
p = std::filesystem::path(str).lexically_normal();
return ar;
}
};
} // namespace yas::detail

175
file/system_native_file.cc Normal file
View File

@ -0,0 +1,175 @@
#include <chrono>
#include <functional>
#include <future>
#include <memory>
#include <mutex>
#include <string>
#include <string_view>
#include <typeinfo>
#include <utility>
#include <vector>
#include <imgui.h>
#include <imgui_stdlib.h>
#include <yas/serialize.hpp>
#include <yas/types/std/chrono.hpp>
#include <yas/types/std/string.hpp>
#include "nf7.hh"
#include "common/async_buffer.hh"
#include "common/async_buffer_adaptor.hh"
#include "common/dir_item.hh"
#include "common/generic_context.hh"
#include "common/generic_type_info.hh"
#include "common/gui_window.hh"
#include "common/native_file.hh"
#include "common/ptr_selector.hh"
#include "common/queue.hh"
#include "common/yas_std_filesystem.hh"
namespace nf7 {
namespace {
class NativeFile final : public File,
public nf7::DirItem {
public:
static inline const GenericTypeInfo<NativeFile> kType = {
"System/NativeFile", {"AsyncBuffer", "DirItem"}};
NativeFile(Env& env, const std::filesystem::path& path = "", std::string_view mode = "") noexcept :
File(kType, env), DirItem(DirItem::kMenu | DirItem::kTooltip),
npath_(path), mode_(mode) {
}
NativeFile(Env& env, Deserializer& ar) : NativeFile(env) {
ar(npath_, mode_, lastmod_);
}
void Serialize(Serializer& ar) const noexcept override {
ar(npath_, mode_, lastmod_);
}
std::unique_ptr<File> Clone(Env& env) const noexcept override {
return std::make_unique<NativeFile>(env, npath_, mode_);
}
void Update() noexcept override;
void UpdateMenu() noexcept override;
void UpdateTooltip() noexcept override;
void Handle(const Event& ev) noexcept override {
switch (ev.type) {
case Event::kAdd:
Reset();
return;
case Event::kRemove:
buf_ = std::nullopt;
return;
default:
return;
}
}
File::Interface* interface(const std::type_info& t) noexcept override {
return InterfaceSelector<nf7::AsyncBuffer, nf7::DirItem>(t).Select(this, &*buf_);
}
private:
std::optional<nf7::AsyncBufferAdaptor> buf_;
const char* popup_ = nullptr;
// persistent params
std::filesystem::path npath_;
std::string mode_;
std::filesystem::file_time_type lastmod_;
void Reset() noexcept {
bool exlock = false;
nf7::Buffer::Flags flags = 0;
for (auto c : mode_) {
if (c == 'x') exlock = true;
flags |=
c == 'r'? nf7::Buffer::kRead:
c == 'w'? nf7::Buffer::kWrite: 0;
}
auto buf = std::make_shared<
nf7::NativeFile>(*this, env().npath()/npath_, flags, exlock);
buf_.emplace(buf, buf);
}
void Touch() noexcept {
env().Handle({.id = id(), .type = Event::kUpdate,});
}
};
void NativeFile::Update() noexcept {
// file update check
try {
const auto lastmod = std::filesystem::last_write_time(env().npath()/npath_);
if (std::exchange(lastmod_, lastmod) < lastmod) {
Touch();
}
} catch (std::filesystem::filesystem_error&) {
}
if (const auto popup = std::exchange(popup_, nullptr)) {
ImGui::OpenPopup(popup);
}
if (ImGui::BeginPopup("ConfigPopup")) {
static std::string path;
static bool flag_exlock;
static bool flag_readable;
static bool flag_writeable;
ImGui::TextUnformatted("System/NativeFile: config");
if (ImGui::IsWindowAppearing()) {
path = npath_.generic_string();
flag_exlock = mode_.find('x') != std::string::npos;
flag_readable = mode_.find('r') != std::string::npos;
flag_writeable = mode_.find('w') != std::string::npos;
}
ImGui::InputText("path", &path);
if (ImGui::IsItemHovered()) {
ImGui::SetTooltip(
"path to the native file system (base: '%s')",
env().npath().generic_string().c_str());
}
ImGui::Checkbox("exclusive lock", &flag_exlock);
ImGui::Checkbox("readable", &flag_readable);
ImGui::Checkbox("writeable", &flag_writeable);
if (ImGui::Button("ok")) {
ImGui::CloseCurrentPopup();
npath_ = path;
mode_ = "";
if (flag_exlock) mode_ += 'x';
if (flag_readable) mode_ += 'r';
if (flag_writeable) mode_ += 'w';
auto ctx = std::make_shared<nf7::GenericContext>(env(), id());
ctx->description() = "resetting native file handle";
env().ExecMain(ctx, [this]() { Reset(); Touch(); });
}
if (!std::filesystem::exists(env().npath()/path)) {
ImGui::Bullet(); ImGui::TextUnformatted("target file seems to be missing...");
}
ImGui::EndPopup();
}
}
void NativeFile::UpdateMenu() noexcept {
if (ImGui::MenuItem("config")) {
popup_ = "ConfigPopup";
}
}
void NativeFile::UpdateTooltip() noexcept {
ImGui::Text("basepath: %s", env().npath().generic_string().c_str());
ImGui::Text("path : %s", npath_.generic_string().c_str());
ImGui::Text("mode : %s", mode_.c_str());
}
}
} // namespace nf7