add System/NativeFile

CMakeLists.txt

@@ -45,6 +45,7 @@ target_sources(nf7
     nf7.hh
 
     common/aggregate_command.hh
+    common/buffer.hh
     common/dir.hh
     common/dir_item.hh
     common/file_ref.hh
@@ -58,6 +59,7 @@ target_sources(nf7
     common/history.hh
     common/lambda.hh
     common/memento.hh
+    common/native_file.hh
     common/node.hh
     common/node_link_store.hh
     common/logger.hh
@@ -68,11 +70,14 @@ target_sources(nf7
     common/wait_queue.hh
     common/yas.hh
 
+    $<$<PLATFORM_ID:Linux>:common/native_file_unix.cc>
+
     file/luajit_context.cc
     file/node_network.cc
     file/system_dir.cc
     file/system_imgui_config.cc
     file/system_logger.cc
+    file/system_native_file.cc
 )
 target_link_libraries(nf7
   PRIVATE

common/buffer.hh

@@ -0,0 +1,91 @@
+#pragma once
+
+#include <cstdint>
+#include <exception>
+#include <future>
+#include <memory>
+
+#include <yas/serialize.hpp>
+#include <yas/types/std/string.hpp>
+
+#include "nf7.hh"
+
+
+namespace nf7 {
+
+class Buffer : public nf7::File::Interface {
+ public:
+  enum Flag : uint8_t {
+    kRead  = 1 << 0,
+    kWrite = 1 << 1,
+  };
+  using Flags = uint8_t;
+
+  // ReadLock and WriteLock are still alive even after Buffer is deleted
+  class ReadLock;
+  class WriteLock;
+
+  class LockException;
+  class IOException;
+
+  Buffer() = default;
+  Buffer(const Buffer&) = delete;
+  Buffer(Buffer&&) = delete;
+  Buffer& operator=(const Buffer&) = delete;
+  Buffer& operator=(Buffer&&) = delete;
+
+  virtual inline std::future<std::unique_ptr<ReadLock>> LockForRead() noexcept;
+  virtual inline std::future<std::unique_ptr<WriteLock>> LockForWrite() noexcept;
+
+  virtual size_t size() const noexcept = 0;
+  virtual Flags flags() const noexcept = 0;
+};
+
+class Buffer::ReadLock {
+ public:
+  ReadLock() = default;
+  virtual ~ReadLock() = default;
+  ReadLock(const ReadLock&) = delete;
+  ReadLock(ReadLock&&) = delete;
+  ReadLock& operator=(const ReadLock&) = delete;
+  ReadLock& operator=(ReadLock&&) = delete;
+
+  virtual std::future<size_t> Read(
+      size_t offset, uint8_t* buf, size_t size) noexcept = 0;
+};
+class Buffer::WriteLock {
+ public:
+  WriteLock() = default;
+  virtual ~WriteLock() = default;
+  WriteLock(const WriteLock&) = delete;
+  WriteLock(WriteLock&&) = delete;
+  WriteLock& operator=(const WriteLock&) = delete;
+  WriteLock& operator=(WriteLock&&) = delete;
+
+  virtual std::future<size_t> Write(
+      size_t offset, const uint8_t* buf, size_t size) noexcept = 0;
+
+  virtual std::future<size_t> Truncate(size_t) noexcept = 0;
+};
+
+class Buffer::LockException : public nf7::Exception {
+ public:
+  using Exception::Exception;
+};
+class Buffer::IOException : public nf7::Exception {
+ public:
+  using Exception::Exception;
+};
+
+std::future<std::unique_ptr<Buffer::ReadLock>> Buffer::LockForRead() noexcept {
+  std::promise<std::unique_ptr<Buffer::ReadLock>> pro;
+  pro.set_exception(std::make_exception_ptr<LockException>({"Buffer is not readable"}));
+  return pro.get_future();
+}
+std::future<std::unique_ptr<Buffer::WriteLock>> Buffer::LockForWrite() noexcept {
+  std::promise<std::unique_ptr<Buffer::WriteLock>> pro;
+  pro.set_exception(std::make_exception_ptr<LockException>({"Buffer is not writeable"}));
+  return pro.get_future();
+}
+
+}  // namespace nf7

common/native_file.hh

@@ -0,0 +1,50 @@
+#pragma once
+
+#include <cstdint>
+#include <filesystem>
+#include <string>
+#include <string_view>
+
+#include "nf7.hh"
+
+
+namespace nf7 {
+
+class NativeFile final : public nf7::Context {
+ public:
+  enum Flag : uint8_t {
+    kRead   = 1 << 0,
+    kWrite  = 1 << 1,
+    kExLock = 1 << 2,
+  };
+  using Flags = uint8_t;
+
+  NativeFile() = delete;
+  NativeFile(Env& env, File::Id id, const std::filesystem::path& path, Flags flags) noexcept :
+      Context(env, id), path_(path), flags_(flags) {
+  }
+  NativeFile(const NativeFile&) = delete;
+  NativeFile(NativeFile&&) = delete;
+  NativeFile& operator=(const NativeFile&) = delete;
+  NativeFile& operator=(NativeFile&&) = delete;
+
+  void CleanUp() noexcept override;
+  void Abort() noexcept override;
+
+  void Lock();
+  void Unlock();
+  size_t Read(size_t offset, uint8_t* buf, size_t size);
+  size_t Write(size_t offset, const uint8_t* buf, size_t size);
+  size_t Truncate(size_t size);
+
+  size_t GetMemoryUsage() const noexcept override;
+  std::string GetDescription() const noexcept override;
+
+ private:
+  const std::filesystem::path path_;
+  const Flags flags_;
+
+  uint64_t handle_;
+};
+
+}  // namespace nf7

common/native_file_unix.cc

@@ -0,0 +1,40 @@
+#include "common/native_file.hh"
+
+
+namespace nf7 {
+
+void NativeFile::CleanUp() noexcept {
+}
+void NativeFile::Abort() noexcept {
+}
+
+void NativeFile::Lock() {
+}
+void NativeFile::Unlock() {
+}
+
+size_t NativeFile::Read(size_t offset, uint8_t* buf, size_t size) {
+  (void) offset;
+  (void) buf;
+  (void) size;
+  return 0;
+}
+size_t NativeFile::Write(size_t offset, const uint8_t* buf, size_t size) {
+  (void) offset;
+  (void) buf;
+  (void) size;
+  return 0;
+}
+size_t NativeFile::Truncate(size_t size) {
+  (void) size;
+  return 0;
+}
+
+size_t NativeFile::GetMemoryUsage() const noexcept {
+  return 0;
+}
+std::string NativeFile::GetDescription() const noexcept {
+  return "";
+}
+
+}  // namespace nf7

common/promise_queue.hh

@@ -0,0 +1,50 @@
+#pragma once
+
+#include <chrono>
+#include <memory>
+#include <functional>
+#include <future>
+
+#include "common/queue.hh"
+
+
+namespace nf7 {
+
+class PromiseQueue final :
+    public nf7::Queue<std::function<bool(void)>> {
+ public:
+  PromiseQueue() = default;
+
+  template <typename T>
+  void Push(std::future<T>&& fu, auto&& f) {
+    auto fu_ptr = std::make_shared<std::future<T>>(std::move(fu));
+    auto task   = [fu_ptr = std::move(fu_ptr), f = std::move(f)]() mutable {
+      if (fu_ptr->wait_for(std::chrono::seconds(0)) != std::future_status::ready) {
+        return false;
+      }
+      f(std::move(*fu_ptr));
+      return true;
+    };
+    Queue<std::function<bool(void)>>::Push(std::move(task));
+  }
+  template <typename T>
+  void Push(std::shared_future<T> fu, auto&& f) {
+    auto task = [fu, f = std::move(f)]() mutable {
+      if (fu.wait_for(std::chrono::seconds(0)) != std::future_status::ready) {
+        return false;
+      }
+      f(std::move(fu));
+      return true;
+    };
+    Queue<std::function<bool(void)>>::Push(std::move(task));
+  }
+  bool PopAndExec() noexcept {
+    if (auto task = Pop()) {
+      if ((*task)()) return true;
+      Interrupt(std::move(*task));
+    }
+    return false;
+  }
+};
+
+}  // namespace nf7

common/queue.hh

@@ -13,15 +13,19 @@ template <typename T>
 class Queue {
  public:
   Queue() = default;
-  Queue(const Queue&) = default;
-  Queue(Queue&&) = default;
-  Queue& operator=(const Queue&) = default;
-  Queue& operator=(Queue&&) = default;
+  Queue(const Queue&) = delete;
+  Queue(Queue&&) = delete;
+  Queue& operator=(const Queue&) = delete;
+  Queue& operator=(Queue&&) = delete;
 
   void Push(T&& task) noexcept {
     std::unique_lock<std::mutex> _(mtx_);
     tasks_.push_back(std::move(task));
   }
+  void Interrupt(T&& task) noexcept {
+    std::unique_lock<std::mutex> _(mtx_);
+    tasks_.push_front(std::move(task));
+  }
   std::optional<T> Pop() noexcept {
     std::unique_lock<std::mutex> k(mtx_);
     if (tasks_.empty()) return std::nullopt;
@@ -31,6 +35,11 @@ class Queue {
     return ret;
   }
 
+  void Clear() noexcept {
+    std::unique_lock<std::mutex> k(mtx_);
+    tasks_.clear();
+  }
+
  protected:
   std::mutex mtx_;
 

common/yas_std_filesystem.hh

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <filesystem>
+#include <string>
+
+#include <yas/serialize.hpp>
+#include <yas/types/std/string.hpp>
+
+
+namespace yas::detail {
+
+template <size_t F>
+struct serializer<
+    type_prop::not_a_fundamental,
+    ser_case::use_internal_serializer,
+    F,
+    std::filesystem::path> {
+ public:
+  template <typename Archive>
+  static Archive& save(Archive& ar, const std::filesystem::path& p) {
+    ar(p.generic_string());
+    return ar;
+  }
+  template <typename Archive>
+  static Archive& load(Archive& ar, std::filesystem::path& p) {
+    std::string str;
+    ar(str);
+    p = std::filesystem::path(str).lexically_normal();
+    return ar;
+  }
+};
+
+}  // namespace yas::detail

file/system_native_file.cc

@@ -0,0 +1,419 @@
+#include <functional>
+#include <future>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <string_view>
+#include <typeinfo>
+#include <utility>
+
+#include <imgui.h>
+#include <imgui_stdlib.h>
+#include <yas/serialize.hpp>
+#include <yas/types/std/string.hpp>
+
+#include "nf7.hh"
+
+#include "common/buffer.hh"
+#include "common/dir_item.hh"
+#include "common/generic_context.hh"
+#include "common/generic_type_info.hh"
+#include "common/gui_window.hh"
+#include "common/logger_pool.hh"
+#include "common/native_file.hh"
+#include "common/promise_queue.hh"
+#include "common/ptr_selector.hh"
+#include "common/queue.hh"
+#include "common/yas_std_filesystem.hh"
+
+
+namespace nf7 {
+namespace {
+
+class BufferAdaptor final : public nf7::Buffer {
+ public:
+  class Lock;
+
+  template <typename T>
+  using PromisePtr = std::promise<std::unique_ptr<T>>;
+  template <typename T>
+  using SharedPromisePtr = std::shared_ptr<PromisePtr<T>>;
+
+  using LockTask = std::function<bool(void)>;
+  using Task     = std::function<void(void)>;
+
+  BufferAdaptor(nf7::Env& env) noexcept : env_(&env)  {
+  }
+
+  std::future<std::unique_ptr<nf7::Buffer::ReadLock>> LockForRead() noexcept override {
+    auto pro = std::make_shared<PromisePtr<nf7::Buffer::ReadLock>>();
+    LockForRead(pro);
+    return pro->get_future();
+  }
+  void LockForRead(const SharedPromisePtr<nf7::Buffer::ReadLock>& pro) noexcept {
+    if (!(flags_ & nf7::Buffer::kRead)) {
+      pro->set_exception(std::make_exception_ptr<LockException>({"not readable"}));
+      return;
+    }
+
+    if (exclusive_ && lock_) {
+      auto task = [self = self.lock(), pro]() mutable {
+        if (self->exclusive_ && self->lock_) return false;
+        self->LockForRead(pro);
+        return true;
+      };
+      lockq_.Push(std::move(task));
+      return;
+    }
+    CreateLock(pro);
+  }
+
+  std::future<std::unique_ptr<nf7::Buffer::WriteLock>> LockForWrite() noexcept override {
+    auto pro = std::make_shared<PromisePtr<nf7::Buffer::WriteLock>>();
+    LockForWrite(pro);
+    return pro->get_future();
+  }
+  void LockForWrite(const SharedPromisePtr<nf7::Buffer::WriteLock>& pro) noexcept {
+    if (!(flags_ & nf7::Buffer::kWrite)) {
+      pro->set_exception(std::make_exception_ptr<LockException>({"not writeable"}));
+      return;
+    }
+
+    if (lock_) {
+      auto task = [self = self.lock(), pro]() {
+        if (self->lock_) return false;
+        self->LockForWrite(pro);
+        return true;
+      };
+      lockq_.Push(std::move(task));
+      return;
+    }
+    exclusive_ = true;
+    CreateLock(pro);
+  }
+
+  void Reset(const std::filesystem::path& path, std::string_view mode) noexcept {
+    impl_ = nullptr;
+    path_ = path;
+
+    flags_ = 0;
+    for (auto c : mode) {
+      flags_ |=
+          c == 'r'? nf7::Buffer::kRead:
+          c == 'w'? nf7::Buffer::kWrite: 0;
+      impl_flags_ |=
+          c == 'r'? nf7::NativeFile::kRead:
+          c == 'w'? nf7::NativeFile::kWrite:
+          c == 'x'? nf7::NativeFile::kExLock: 0;
+    }
+  }
+
+  const std::filesystem::path& path() const noexcept { return path_; }
+  Flags flags() const noexcept override { return flags_; }
+  size_t size() const noexcept override
+  try {
+    return std::filesystem::file_size(path_);
+  } catch (std::filesystem::filesystem_error&) {
+    return 0;
+  }
+
+  nf7::File::Id                owner;
+  std::weak_ptr<BufferAdaptor> self;
+
+ private:
+  Env* const    env_;
+
+  std::filesystem::path path_;
+  Flags flags_ = 0;
+
+  size_t size_ = 0;
+
+  nf7::NativeFile::Flags impl_flags_;
+  std::shared_ptr<nf7::NativeFile> impl_;
+
+  bool   exclusive_ = false;
+  size_t lock_      = 0;
+  nf7::Queue<LockTask> lockq_;  // pushed from only main thread
+
+  std::mutex mtx_;
+  bool working_ = false;
+  nf7::Queue<Task> taskq_;
+
+  void CreateImplIf() noexcept {
+    if (!impl_) {
+      impl_ = std::make_shared<nf7::NativeFile>(*env_, owner, path_, impl_flags_);
+    }
+  }
+  void CreateLock(const auto& pro) noexcept;
+
+  void HandleNextLock() noexcept {
+    assert(lock_ == 0);
+
+    auto task = [self = self.lock()]() {
+      while (auto task = self->lockq_.Pop()) {
+        if (!(*task)()) {
+          self->lockq_.Interrupt(std::move(*task));
+          break;
+        }
+      }
+    };
+    CreateImplIf();
+    env_->ExecMain(impl_, std::move(task));
+  }
+  void Exec(Task&& task) noexcept {
+    taskq_.Push(std::move(task));
+    std::unique_lock<std::mutex> k(mtx_);
+    if (!std::exchange(working_, true)) {
+      CreateImplIf();
+      env_->ExecAsync(impl_, [self = self.lock()]() { self->HandleTask(); });
+    }
+  }
+  void HandleTask() noexcept {
+    for (;;) {
+      std::unique_lock<std::mutex> k(mtx_);
+      auto task = taskq_.Pop();
+      if (!task) {
+        working_ = false;
+        break;
+      }
+      k.unlock();
+
+      (*task)();
+    }
+  }
+};
+
+class BufferAdaptor::Lock : public nf7::Buffer::ReadLock, public nf7::Buffer::WriteLock {
+ public:
+  Lock(const std::shared_ptr<BufferAdaptor>& buf) noexcept :
+      buf_(buf), impl_(buf_->impl_) {
+    if (buf_->lock_++ == 0) {
+      impl_->Lock();
+    }
+  }
+  ~Lock() noexcept {
+    if (--buf_->lock_ == 0) {
+      buf_->Exec([impl = impl_]() { impl->Unlock(); });
+      buf_->exclusive_ = false;
+      buf_->HandleNextLock();
+    }
+  }
+
+  std::future<size_t> Read(size_t offset, uint8_t* buf, size_t size) noexcept override {
+    auto pro  = std::make_shared<std::promise<size_t>>();
+    auto task = [impl = impl_, pro, offset, buf, size]() {
+      pro->set_value(impl->Read(offset, buf, size));
+    };
+    buf_->Exec(std::move(task));
+    return pro->get_future();
+  }
+  std::future<size_t> Write(size_t offset, const uint8_t* buf, size_t size) noexcept override {
+    auto pro  = std::make_shared<std::promise<size_t>>();
+    auto task = [impl = impl_, pro, offset, buf, size]() {
+      pro->set_value(impl->Write(offset, buf, size));
+    };
+    buf_->Exec(std::move(task));
+    return pro->get_future();
+  }
+  std::future<size_t> Truncate(size_t size) noexcept override {
+    auto pro  = std::make_shared<std::promise<size_t>>();
+    auto task = [impl = impl_, pro, size]() {
+      pro->set_value(impl->Truncate(size));
+    };
+    buf_->Exec(std::move(task));
+    return pro->get_future();
+  }
+
+ private:
+  std::shared_ptr<BufferAdaptor>   buf_;
+  std::shared_ptr<nf7::NativeFile> impl_;
+};
+
+void BufferAdaptor::CreateLock(const auto& pro) noexcept {
+  try {
+    env_->GetFile(owner);
+  } catch (ExpiredException&) {
+    lockq_.Clear();
+    pro->set_exception(std::make_exception_ptr<LockException>({"file expired"}));
+    return;
+  }
+  CreateImplIf();
+
+  auto task = [self = self.lock(), pro]() {
+    try {
+      auto k = std::make_unique<Lock>(self);
+      pro->set_value(std::move(k));
+    } catch (...) {
+      pro->set_exception(std::current_exception());
+    }
+  };
+  Exec(std::move(task));
+}
+
+
+class NativeFile final : public File,
+    public nf7::DirItem {
+ public:
+  static inline const GenericTypeInfo<NativeFile> kType = {
+    "System/NativeFile", {"Buffer", "DirItem"}};
+
+  NativeFile(Env& env, const std::filesystem::path& path = "", std::string_view mode = "") noexcept :
+      File(kType, env), DirItem(DirItem::kMenu | DirItem::kTooltip),
+      logger_(*this), buf_(std::make_shared<BufferAdaptor>(env)),
+      test_win_(*this, "NativeFile Tester"),
+      npath_(path), mode_(mode) {
+    buf_->self = buf_;
+    Reset();
+  }
+
+  NativeFile(Env& env, Deserializer& ar) : NativeFile(env) {
+    ar(npath_, mode_);
+    Reset();
+  }
+  void Serialize(Serializer& ar) const noexcept override {
+    ar(npath_, mode_);
+  }
+  std::unique_ptr<File> Clone(Env& env) const noexcept override {
+    return std::make_unique<NativeFile>(env, npath_, mode_);
+  }
+
+  void Update() noexcept override;
+  void UpdateMenu() noexcept override;
+  void UpdateTooltip() noexcept override;
+
+  void Handle(const Event& ev) noexcept override {
+    switch (ev.type) {
+    case Event::kAdd:
+      buf_->owner = id();
+      return;
+    case Event::kRemove:
+      buf_->owner = 0;
+      return;
+    default:
+      return;
+    }
+  }
+
+  File::Interface* interface(const std::type_info& t) noexcept override {
+    return InterfaceSelector<nf7::Buffer, nf7::DirItem>(t).Select(this, buf_.get());
+  }
+
+ private:
+  nf7::LoggerPool logger_;
+  std::shared_ptr<BufferAdaptor> buf_;
+
+  const char* popup_ = nullptr;
+
+  gui::Window test_win_;
+  uint32_t    test_offset_ = 0;
+  uint32_t    test_size_   = 1;
+  std::string test_msg_;
+  std::vector<uint8_t> test_buf_;
+  nf7::PromiseQueue    testq_;
+
+  // persistent params
+  std::filesystem::path npath_;
+  std::string mode_;
+
+
+  void Reset() noexcept {
+    buf_->Reset(env().npath() / npath_, mode_);
+  }
+
+  void TestLock(std::future<std::unique_ptr<nf7::Buffer::ReadLock>>& fu) noexcept
+  try {
+    auto k = std::move(fu.get());
+    test_msg_ = "OK: lock acquired";
+
+    test_buf_.resize(test_size_);
+    testq_.Push(k->Read(test_offset_, test_buf_.data(), test_size_),
+                [this](auto&& f) { TestRead(f); });
+  } catch (Exception& e) {
+    test_msg_ = "NG:"+e.msg();
+  }
+  void TestRead(std::future<size_t>& fu) noexcept
+  try {
+    const auto n = std::move(fu.get());
+    test_msg_ = "DONE: read "+std::to_string(n)+" bytes";
+  } catch (Exception& e) {
+    test_msg_ = "NG:"+e.msg();
+  }
+};
+
+void NativeFile::Update() noexcept {
+  while (testq_.PopAndExec());
+
+  if (const auto popup = std::exchange(popup_, nullptr)) {
+    ImGui::OpenPopup(popup);
+  }
+
+  if (ImGui::BeginPopup("ConfigPopup")) {
+    static std::string path;
+    static bool flag_exlock;
+    static bool flag_readable;
+    static bool flag_writeable;
+
+    ImGui::TextUnformatted("System/NativeFile: config");
+    if (ImGui::IsWindowAppearing()) {
+      path           = npath_.generic_string();
+      flag_exlock    = mode_.find('x') != std::string::npos;
+      flag_readable  = mode_.find('r') != std::string::npos;
+      flag_writeable = mode_.find('w') != std::string::npos;
+    }
+
+    ImGui::InputText("path", &path);
+    if (ImGui::IsItemHovered()) {
+      ImGui::SetTooltip(
+          "path to the native file system (base: '%s')",
+          env().npath().generic_string().c_str());
+    }
+    ImGui::Checkbox("exclusive lock", &flag_exlock);
+    ImGui::Checkbox("readable",       &flag_readable);
+    ImGui::Checkbox("writeable",      &flag_writeable);
+
+    if (ImGui::Button("ok")) {
+      ImGui::CloseCurrentPopup();
+
+      npath_ = path;
+      mode_  = "";
+      if (flag_exlock)    mode_ += 'x';
+      if (flag_readable)  mode_ += 'r';
+      if (flag_writeable) mode_ += 'w';
+
+      auto ctx = std::make_shared<nf7::GenericContext>(env(), id());
+      ctx->description() = "resetting native file handle";
+      env().ExecMain(ctx, [this]() { Reset(); });
+    }
+
+    if (!std::filesystem::exists(env().npath()/path)) {
+      ImGui::Bullet(); ImGui::TextUnformatted("target file seems to be missing...");
+    }
+    ImGui::EndPopup();
+  }
+
+  if (test_win_.Begin()) {
+    static const uint32_t kMax = 1024*1024, kZero = 0, kOne = 1;
+    ImGui::DragScalar("offset", ImGuiDataType_U32, &test_offset_, 1, &kZero, &kMax);
+    ImGui::DragScalar("size",   ImGuiDataType_U32, &test_size_,   1, &kOne,  &kMax);
+    if (ImGui::Button("exec")) {
+      auto fu = buf_->LockForRead();
+      testq_.Push(std::move(fu), [this](auto&& fu) { TestLock(fu); });
+    }
+    ImGui::TextUnformatted(test_msg_.c_str());
+  }
+  test_win_.End();
+}
+void NativeFile::UpdateMenu() noexcept {
+  if (ImGui::MenuItem("config")) {
+    popup_ = "ConfigPopup";
+  }
+  ImGui::MenuItem("test", nullptr, &test_win_.shown());
+}
+void NativeFile::UpdateTooltip() noexcept {
+  ImGui::Text("basepath: %s", env().npath().generic_string().c_str());
+  ImGui::Text("path    : %s", npath_.generic_string().c_str());
+  ImGui::Text("mode    : %s", mode_.c_str());
+}
+
+}
+}  // namespace nf7