create new project

.gitignore

@@ -0,0 +1 @@
+/build/

CMakeLists.txt

@@ -0,0 +1,27 @@
+cmake_minimum_required(VERSION 3.18)
+
+project(blocky C CXX)
+
+set(CMAKE_CXX_STANDARD          20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+set(BLOCKY_C_FLAGS
+    $<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>:
+      -Wall -Werror -pedantic-errors -Wextra -Wconversion -Wsign-conversion>
+    $<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>>:
+      -Wno-overloaded-virtual>
+    $<$<CXX_COMPILER_ID:MSVC>:
+    /W4 /WX>
+)
+set(BLOCKY_CXX_FLAGS
+  $<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>:
+    -Wall -Werror -pedantic-errors -Wextra -Wconversion -Wsign-conversion>
+  $<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>>:
+    -Wno-overloaded-virtual>
+  $<$<CXX_COMPILER_ID:MSVC>:
+    /W4 /WX>
+)
+
+add_subdirectory(thirdparty)
+add_subdirectory(liblocky)
+add_subdirectory(playground)

README.md

@@ -0,0 +1,12 @@
+blocky
+====
+
+## ffmpeg useful commands
+
+```
+ffmpeg -i in.mp4 -vframes 300 "%d.png"
+```
+
+```
+ffmpeg -r 30 -i "%d.png" -vcodec libx264 -pix_fmt yuv420p out.mp4
+```

liblocky/CMakeLists.txt

@@ -0,0 +1,17 @@
+cmake_minimum_required(VERSION 3.18)
+
+add_library(liblocky)
+set_target_properties(liblocky PROPERTIES PREFIX "")
+target_compile_options(liblocky PRIVATE ${BLOCKY_C_FLAGS})
+
+target_link_libraries(liblocky PUBLIC m)
+target_include_directories(liblocky PUBLIC SYSTEM .)
+target_sources(liblocky
+  PUBLIC
+    liblocky.h
+  PRIVATE
+    block.c
+    extractor.c
+    sensor.c
+    image.c
+)

liblocky/block.c

@@ -0,0 +1,24 @@
+#include "liblocky.h"
+
+
+double blky_block_estimate(
+    const blky_sensor_t* sensors, uint64_t n,
+    double max_var, double min_avg) {
+  double sum = 0;
+  for (uint64_t i = 0; i < n; ++i) {
+    sum += blky_abs(sensors[i].correl);
+  }
+  const double avg = sum / (double) n;
+
+  double var = 0;
+  for (uint64_t i = 0; i < n; ++i) {
+    const double diff = blky_abs(sensors[i].correl) - avg;
+    var += diff*diff;
+  }
+  var /= (double) n;
+
+  // FIXME: calculate probability
+  if (var > max_var) return 0;
+  if (avg < min_avg) return 0;
+  return 1;
+}

liblocky/extractor.c

@@ -0,0 +1,89 @@
+#include "liblocky.h"
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+
+
+void blky_extractor_init(blky_extractor_t* ex) {
+  assert(ex->block_num_x > 0);
+  assert(ex->block_num_y > 0);
+  assert(ex->sensor_num_block_x > 0);
+  assert(ex->sensor_num_block_y > 0);
+  assert(ex->samples_per_pix > 0);
+  assert(ex->utime > 0);
+  assert(ex->utime <= BLKY_SENSOR_MAX_DUR);
+  assert(ex->pix_stride > 0);
+  assert(ex->pix_stride > ex->samples_per_pix);
+  assert(ex->correl_max_var >= 0);
+  assert(ex->correl_min_avg >= 0);
+  assert(ex->correl_min_avg <= 1);
+
+  // set immutable params
+  ex->block_num = ex->block_num_x * ex->block_num_y;
+
+  ex->sensor_num_block =
+    ex->sensor_num_block_x * ex->sensor_num_block_y *
+    ex->samples_per_pix;
+
+  ex->sensor_num_whole = ex->sensor_num_block * ex->block_num;
+
+  ex->block_w = 1./ex->block_num_x;
+  ex->block_h = 1./ex->block_num_y;
+
+  ex->sensor_interval_x = ex->block_w / ex->sensor_num_block_x;
+  ex->sensor_interval_y = ex->block_h / ex->sensor_num_block_y;
+
+  // clear states
+  ex->time = 0;
+
+  const uint64_t sensors_bytes = sizeof(ex->sensors[0]) * ex->sensor_num_whole;
+  const uint64_t probs_bytes   = sizeof(ex->probs[0]) * ex->block_num;
+
+  const uint64_t mem_bytes = sensors_bytes + probs_bytes;
+  ex->mem = calloc(mem_bytes, 1);
+  assert(ex->mem);
+
+  ex->sensors = (blky_sensor_t*) ex->mem;
+  ex->probs   = (double*) (ex->mem + sensors_bytes);
+}
+void blky_extractor_deinit(blky_extractor_t* ex) {
+  free(ex->mem);
+}
+
+bool blky_extractor_feed(
+    blky_extractor_t* ex,
+    const uint8_t* img, uint32_t w, uint32_t h, const double verts[8]) {
+  blky_sensor_t* s = ex->sensors;
+
+  for (uint32_t by = 0; by < ex->block_num_y; ++by) {
+    const double byf = by*ex->block_h + ex->sensor_interval_y/2.;
+    for (uint32_t bx = 0; bx < ex->block_num_x; ++bx) {
+      const double bxf = bx*ex->block_w + ex->sensor_interval_x/2.;
+      for (uint32_t sy = 0; sy < ex->sensor_num_block_y; ++sy) {
+        const double syf = byf + ex->sensor_interval_y*sy;
+        for (uint32_t sx = 0; sx < ex->sensor_num_block_x; ++sx) {
+          const double sxf = bxf + ex->sensor_interval_x*sx;
+
+          const uint8_t* base = img +
+            ex->pix_stride*blky_image_offset(w, h, verts, sxf, syf);
+          for (uint8_t off = 0; off < ex->samples_per_pix; ++off) {
+            const float v = base[off]*1.f / UINT8_MAX;
+            blky_sensor_feed(s++, &v, 1);
+          }
+        }
+      }
+    }
+  }
+
+  ++ex->time;
+  if (ex->time%ex->utime > 0) return false;
+
+  for (uint64_t bi = 0; bi < ex->block_num; ++bi) {
+    blky_sensor_t* s = ex->sensors + (bi*ex->sensor_num_block);
+    ex->probs[bi] = blky_block_estimate(
+        s, ex->sensor_num_block, ex->correl_max_var, ex->correl_min_avg);
+    memset(s, 0, ex->sensor_num_block*sizeof(*s));
+  }
+  return true;
+}

liblocky/image.c

@@ -0,0 +1,32 @@
+#include "liblocky.h"
+
+#include <assert.h>
+
+
+void blky_image_convert_to_normalized_coord(
+    const double verts[8], double* x, double* y) {
+  assert(0 <= *x && *x < 1);
+  assert(0 <= *y && *y < 1);
+
+  const double rx = 1-*x, ry = 1-*y;
+
+  const double top_xf = *x*verts[6] + rx*verts[0];
+  const double bot_xf = *x*verts[4] + rx*verts[2];
+
+  const double lef_yf = *y*verts[3] + ry*verts[1];
+  const double rig_yf = *y*verts[5] + ry*verts[7];
+
+  const double ans_x = *y*bot_xf + ry*top_xf;
+  const double ans_y = *x*rig_yf + rx*lef_yf;
+
+  *x = ans_x;
+  *y = ans_y;
+}
+
+uint64_t blky_image_offset(
+    uint32_t w, uint32_t h, const double verts[8], double x, double y) {
+  blky_image_convert_to_normalized_coord(verts, &x, &y);
+  const uint32_t xi = (uint32_t) (x*w);
+  const uint32_t yi = (uint32_t) (y*h);
+  return yi*w + xi;
+}

liblocky/liblocky.h

@@ -0,0 +1,103 @@
+#pragma once
+
+#include <stdbool.h>
+#include <stdint.h>
+
+
+#define blky_clamp(x, a, b) ((x) < (a)? (a): (x) > (b)? (b): (x))
+#define blky_abs(x) ((x) < 0? -(x): (x))
+
+
+/* ---- Sensor ----
+ * calculates correl from samples sequencially */
+typedef struct blky_sensor_t {
+  double sum, avg, var;
+  uint64_t time;
+
+  double cov;
+  double correl;
+  double prev_correl;
+
+# define BLKY_SENSOR_MAX_DUR 64
+  float values[BLKY_SENSOR_MAX_DUR];
+} blky_sensor_t;
+
+void
+blky_sensor_feed(
+    blky_sensor_t*, const float* v, uint64_t n);
+
+void
+blky_sensor_drop(
+    blky_sensor_t*, uint64_t until);
+
+
+/* ---- Block ----
+ * calculates probability representing how likely the block is a feature,
+ * by dealing with multiple sensors */
+
+double  /* 0~1 probability */
+blky_block_estimate(
+    const blky_sensor_t* sensors, uint64_t n,
+    double max_var, double min_avg);
+
+
+/* ---- Extractor ----
+ * extracts all features from a sequence of multiple frames */
+typedef struct blky_extractor_t {
+  // must be filled before init()
+  uint32_t block_num_x;
+  uint32_t block_num_y;
+  uint32_t sensor_num_block_x;
+  uint32_t sensor_num_block_y;
+  uint32_t samples_per_pix;
+  uint32_t pix_stride;
+  uint32_t utime;
+
+  double correl_max_var;
+  double correl_min_avg;
+
+  // immutable internal params
+  uint32_t block_num;
+  uint32_t sensor_num_block;
+  uint32_t sensor_num_whole;
+
+  double block_w;
+  double block_h;
+  double sensor_interval_x;
+  double sensor_interval_y;
+
+  // mutable internal state
+  uint64_t time;
+
+  // heap objects
+  uint8_t*       mem;
+  blky_sensor_t* sensors;
+  double*        probs;
+} blky_extractor_t;
+
+typedef struct blky_extractor_feat_t {
+  uint32_t block;
+  uint64_t begin;
+  double   prob;
+} blky_extractor_feat_t;
+
+void
+blky_extractor_init(
+    blky_extractor_t* ex);
+
+void
+blky_extractor_deinit(
+    blky_extractor_t* ex);
+
+bool
+blky_extractor_feed(
+    blky_extractor_t* ex,
+    const uint8_t* img, uint32_t w, uint32_t h, const double verts[8]);
+
+
+/* ---- Image utility ---- */
+void blky_image_convert_to_normalized_coord(
+    const double verts[8], double* x, double* y);
+
+uint64_t blky_image_offset(
+    uint32_t w, uint32_t h, const double verts[8], double x, double y);

liblocky/sensor.c

@@ -0,0 +1,53 @@
+#include "liblocky.h"
+
+#include <assert.h>
+#include <math.h>
+#include <string.h>
+
+
+void blky_sensor_feed(blky_sensor_t* b, const float* v, uint64_t n) {
+  assert(b->time+n <= BLKY_SENSOR_MAX_DUR);
+  if (n == 0) return;
+
+  const uint64_t t = b->time;
+  memcpy(b->values+t, v, n*sizeof(float));
+  b->time += n;
+
+  for (uint64_t i = t; i < b->time; ++i) {
+    b->sum += b->values[i];
+  }
+  b->avg = b->sum / (double) b->time;
+
+  b->var = 0;
+  for (uint64_t i = 0; i < b->time; ++i) {
+    const double diff = b->values[i] - b->avg;
+    b->var += diff*diff;
+  }
+  b->var /= (double) b->time;
+
+  b->cov = 0;
+  for (uint64_t i = 0; i < b->time; ++i) {
+    const double diff_v = b->values[i] - b->avg;
+    const double diff_t = (double) i - (double) b->time/2.;
+    b->cov += diff_v * diff_t;
+  }
+  b->cov /= (double) b->time;
+
+  const double tf   = (double) b->time;
+  const double tvar =
+    (tf/6*(tf+1)*(2*tf+2)-tf*tf/2-tf*tf*tf/4)/tf;
+
+  b->prev_correl = b->correl;
+
+  if (b->var == 0) {
+    b->correl = 1;
+  } else {
+    b->correl = b->cov / sqrt(b->var*tvar);
+  }
+}
+
+void blky_sensor_drop(blky_sensor_t* b, uint64_t until) {
+  (void) b;
+  (void) until;
+  // TODO
+}

playground/CMakeLists.txt

@@ -0,0 +1,30 @@
+add_executable(playground)
+target_compile_options(playground PRIVATE ${BLOCKY_CXX_FLAGS})
+
+target_sources(playground
+  PRIVATE
+    app.hh
+    block.cc
+    encoder.cc
+    exporter.cc
+    extractor.cc
+    initiator.hh
+    input.cc
+    input.hh
+    input_gradient.cc
+    input_imgseq.cc
+    input_noise.cc
+    main.cc
+    player.cc
+    sensor.cc
+)
+target_link_libraries(playground
+  PRIVATE
+    liblocky
+    glew
+    glfw
+    imgui
+    implot
+    m
+    stb
+)

playground/app.hh

@@ -0,0 +1,61 @@
+#pragma once
+
+#include <functional>
+#include <map>
+#include <memory>
+#include <string>
+#include <utility>
+
+
+namespace pg {
+
+class App {
+ public:
+  App() = default;
+  virtual ~App() = default;
+  App(const App&) = delete;
+  App(App&&) = delete;
+  App& operator=(const App&) = delete;
+  App& operator=(App&&) = delete;
+
+  virtual void Update() noexcept = 0;
+
+ protected:
+  struct TypeInfo final {
+   public:
+    using Factory = std::function<std::unique_ptr<App>()>;
+
+    template <typename T>
+    static TypeInfo Create(const char* name) noexcept {
+      return {name, []() { return std::make_unique<T>(); }};
+    }
+
+    TypeInfo(const char* name, Factory&& f) noexcept :
+        name_(name), factory_(std::move(f)) {
+      registry_[name_] = this;
+    }
+    ~TypeInfo() noexcept {
+      registry_.erase(name_);
+    }
+
+    std::unique_ptr<App> Create() noexcept {
+      return factory_();
+    }
+
+    const char* name() const noexcept { return name_; }
+
+   private:
+    const char* name_;
+
+    Factory factory_;
+  };
+
+  static const std::map<std::string, TypeInfo*>& registry() noexcept {
+    return registry_;
+  }
+
+ private:
+  static inline std::map<std::string, TypeInfo*> registry_;
+};
+
+}  // namespace pg

playground/block.cc

@@ -0,0 +1,183 @@
+extern "C" {
+# include <liblocky.h>
+}
+
+#include <numeric>
+#include <vector>
+
+#include <imgui.h>
+#include <implot.h>
+
+#include "app.hh"
+#include "input.hh"
+
+
+namespace pg {
+
+class Block final : public App {
+ public:
+  static inline TypeInfo kType = TypeInfo::Create<Block>("Block");
+
+  Block() noexcept {
+  }
+
+  void Update() noexcept {
+    const auto id = "Block | "+
+        std::to_string(reinterpret_cast<uintptr_t>(this));
+
+    const auto em = ImGui::GetFontSize();
+    ImGui::SetNextWindowSize({40*em, 24*em}, ImGuiCond_Once);
+    if (ImGui::Begin(id.c_str())) {
+      UpdateParams();
+      ImGui::SameLine();
+      UpdatePlots();
+      if (msg_.size() > 0) {
+        ImGui::TextUnformatted(msg_.c_str());
+      }
+    }
+    ImGui::End();
+  }
+  void UpdateParams() noexcept {
+    const auto em = ImGui::GetFontSize();
+
+    bool mod = false;
+    ImGui::BeginGroup();
+    ImGui::PushItemWidth(6*em);
+    mod |= ImGui::DragInt("input_slot", &src_, 1, 0, 1024);
+    mod |= ImGui::DragInt("time", &time_, 1, 0, 1024);
+    mod |= ImGui::DragInt("dur", &dur_, 1, 1, BLKY_SENSOR_MAX_DUR);
+    mod |= ImGui::DragInt2("blocks", block_, 1, 1, 16);
+    mod |= ImGui::DragInt2("sensors", sensor_);
+    mod |= ImGui::DragFloat("var thresh", &var_thresh_);
+    ImGui::PopItemWidth();
+    ImGui::EndGroup();
+
+    if (mod) Calc();
+  }
+  void UpdatePlots() noexcept {
+    const auto em = ImGui::GetFontSize();
+
+    ImGui::BeginGroup();
+
+    auto avail = ImGui::GetContentRegionAvail();
+    avail.x -= em;
+    avail.y -= em;
+    if (ImPlot::BeginPlot("correls", avail)) {
+      ImPlot::SetupAxisLimits(ImAxis_X1, 0, 1);
+      ImPlot::SetupAxisLimits(ImAxis_Y1, 0, 1);
+      ImPlot::PlotHeatmap(
+          "correls_avg", correl_avg_.data(), block_[1], block_[0],
+          0, 1, "%.2f");
+      ImPlot::PlotHeatmap(
+          "correls_var", correl_var_.data(), block_[1], block_[0],
+          0, var_thresh_, "%.4f");
+      ImPlot::PlotHeatmap(
+          "correls_avg_filtered", correl_avg_filtered_.data(), block_[1], block_[0],
+          0, 1, "%.2f");
+      ImPlot::EndPlot();
+    }
+    ImGui::EndGroup();
+  }
+
+ private:
+  int src_ = 0;
+  int time_ = 0;
+  int dur_ = 30;
+  int block_[2] = {1, 1};
+  int sensor_[2] = {1, 1};
+
+  float var_thresh_ = .1f;
+
+  std::string msg_;
+
+  std::vector<double> correls_    = {0};
+  std::vector<double> correl_avg_ = {0};
+  std::vector<double> correl_var_ = {0};
+  std::vector<double> correl_avg_filtered_ = {0};
+
+  void Calc() noexcept
+  try {
+    auto data = Input::instance().slots(static_cast<size_t>(src_));
+    if (!data) throw "missing input";
+
+    const auto block_x = static_cast<size_t>(block_[0]);
+    const auto block_y = static_cast<size_t>(block_[1]);
+    const auto block_n = block_x*block_y;
+    const auto block_w = 1.f / static_cast<float>(block_x);
+    const auto block_h = 1.f / static_cast<float>(block_y);
+
+    const auto sensor_x = static_cast<size_t>(sensor_[0]);
+    const auto sensor_y = static_cast<size_t>(sensor_[1]);
+    const auto sensor_n = sensor_x*sensor_y;
+    const auto sensor_interval_x = block_w / static_cast<float>(sensor_x);
+    const auto sensor_interval_y = block_h / static_cast<float>(sensor_y);
+
+    correls_.clear();
+    correls_.reserve(block_n*sensor_n*3);
+    for (size_t by = 0; by < block_y; ++by) {
+      for (size_t bx = 0; bx < block_x; ++bx) {
+        for (size_t sy = 0; sy < sensor_y; ++sy) {
+          for (size_t sx = 0; sx < sensor_x; ++sx) {
+            const auto xf =
+                block_w*static_cast<float>(bx) +
+                sensor_interval_x*(static_cast<float>(sx)+.5f);
+            const auto yf =
+                block_h*static_cast<float>(by) +
+                sensor_interval_y*(static_cast<float>(sy)+.5f);
+
+            for (size_t i = 0; i < 3; ++i) {
+              const auto samp = data->FetchSamples(
+                  static_cast<size_t>(time_),
+                  static_cast<size_t>(dur_),
+                  xf, yf, i);
+
+              blky_sensor_t sensor = {};
+              blky_sensor_feed(&sensor, samp.data(), samp.size());
+              correls_.push_back(sensor.correl);
+            }
+          }
+        }
+      }
+    }
+
+    correl_avg_.clear();
+    correl_avg_.reserve(block_n);
+    correl_var_.clear();
+    correl_var_.reserve(block_n);
+    correl_avg_filtered_.clear();
+    correl_avg_filtered_.reserve(block_n);
+
+    const auto correls_per_block = sensor_n*3;
+    for (size_t bi = 0; bi < block_n; ++bi) {
+      const size_t st = bi*correls_per_block;
+      const size_t ed = st + correls_per_block;
+
+      double sum = 0;
+      for (size_t i = st; i < ed; ++i) {
+        sum += std::abs(correls_[i]);
+      }
+
+      const auto avg = sum / static_cast<double>(correls_per_block);
+      correl_avg_.push_back(avg);
+
+      double var = 0;
+      for (size_t i = st; i < ed; ++i) {
+        const auto diff = std::abs(correls_[i]) - avg;
+        var += diff*diff;
+      }
+      var /= static_cast<double>(correls_per_block);
+      correl_var_.push_back(var);
+
+      if (var < var_thresh_) {
+        correl_avg_filtered_.push_back(avg);
+      } else {
+        correl_avg_filtered_.push_back(-1);
+      }
+    }
+
+  } catch (const char* msg) {
+    msg_ = msg;
+  }
+};
+
+}  // namespace pg

playground/encoder.cc

@@ -0,0 +1,211 @@
+#include "app.hh"
+#include "input.hh"
+
+#include <imgui.h>
+#include <imgui_stdlib.h>
+
+
+namespace pg {
+namespace {
+
+class Encoder final : public App, public Input::Data {
+ public:
+  static inline TypeInfo kType = TypeInfo::Create<Encoder>("Encoder");
+
+  Encoder() noexcept : Data("encoder") {
+  }
+
+  void Update() noexcept override {
+    const auto id = std::to_string(index())+" Encoder | "+
+        std::to_string(reinterpret_cast<uintptr_t>(this));
+
+    if (ImGui::Begin(id.c_str())) {
+      ImGui::DragInt("src", &src_, 1, 0, 1024);
+      ImGui::DragInt("w", &w_, 1, 1, 1024);
+      ImGui::DragInt("h", &h_, 1, 1, 1024);
+      ImGui::DragInt2("block num", block_num_, 1, 1, 1024);
+      ImGui::DragInt("utime", &utime_, 1, 5, 64);
+      ImGui::InputTextMultiline("feats", &feats_);
+      if (ImGui::Button("encode")) {
+        Encode();
+      }
+    }
+    ImGui::End();
+  }
+
+  Frame Fetch(size_t n) noexcept override {
+    if (n >= frames_) return {};
+    const auto w = static_cast<size_t>(w_);
+    const auto h = static_cast<size_t>(h_);
+    return { .w = w, .h = h, .rgba = buf_.data()+4*w*h*n };
+  }
+  size_t frames() noexcept override {
+    return frames_;
+  }
+
+ private:
+  int src_ = 0;
+  int w_ = 128, h_ = 128;
+  int block_num_[2] = {2, 2};
+  int utime_ = 10;
+  std::string feats_;
+
+  std::vector<uint8_t> buf_;
+  size_t frames_ = 0;
+
+
+  struct Feat final {
+   public:
+    size_t block;
+    char type;  // C: const, L: linear
+  };
+
+  void Encode() {
+    const auto src = Input::instance().slots(static_cast<size_t>(src_));
+    if (!src) throw "missing input";
+
+    const auto feats = ParseFeats(feats_);
+
+    const auto w     = static_cast<size_t>(w_);
+    const auto h     = static_cast<size_t>(h_);
+    const auto utime = static_cast<size_t>(utime_);
+    buf_.resize(w*h*4*feats.size()*utime);
+    frames_ = utime * feats.size();
+
+    const auto b_num_x = static_cast<size_t>(block_num_[0]);
+    const auto b_num_y = static_cast<size_t>(block_num_[1]);
+
+    const auto dst_bw   = w/b_num_x;
+    const auto dst_bh   = h/b_num_y;
+    std::vector<std::pair<uint8_t, uint8_t>> block_rgba(dst_bw*dst_bh*4);
+
+    const auto dst_bw_f = static_cast<float>(dst_bw);
+    const auto dst_bh_f = static_cast<float>(dst_bh);
+
+    auto dst_ptr = buf_.data();
+    for (size_t t = 0; t < feats.size(); ++t) {
+      const auto& f = feats[t];
+
+      const auto bx = f.block%b_num_x;
+      const auto by = f.block/b_num_x;
+      if (by < b_num_y) {
+        {
+          // get pixels in start of the feature
+          const auto frame = src->Fetch(t*utime);
+          if (!frame.rgba) throw "got an empty frame";
+
+          const auto bw    = frame.w / b_num_x;
+          const auto bh    = frame.h / b_num_y;
+          const auto bw_f  = static_cast<float>(bw);
+          const auto bh_f  = static_cast<float>(bh);
+          const auto off_x = bw*bx;
+          const auto off_y = bh*by;
+
+          auto itr = block_rgba.begin();
+          for (size_t y = 0; y < dst_bh; ++y) {
+            const auto yf = static_cast<float>(y) / dst_bh_f;
+            for (size_t x = 0; x < dst_bw; ++x) {
+              const auto xf = static_cast<float>(x) / dst_bw_f;
+
+              const auto srcx = static_cast<size_t>(bw_f*xf) + off_x;
+              const auto srcy = static_cast<size_t>(bh_f*yf) + off_y;
+
+              auto ptr = frame.rgba + 4*(frame.w*srcy+srcx);
+              for (size_t i = 0; i < 4; ++i) {
+                itr->first  = *ptr;
+                itr->second = *ptr;
+                ++itr, ++ptr;
+              }
+            }
+          }
+        }
+        if (f.type == 'L') {
+          // get pixels in end frame if it's linear
+          const auto frame = src->Fetch((t+1)*utime);
+          if (!frame.rgba) throw "got an empty frame";
+
+          const auto bw    = frame.w / b_num_x;
+          const auto bh    = frame.h / b_num_y;
+          const auto bw_f  = static_cast<float>(bw);
+          const auto bh_f  = static_cast<float>(bh);
+          const auto off_x = bw*bx;
+          const auto off_y = bh*by;
+
+          auto itr = block_rgba.begin();
+          for (size_t y = 0; y < dst_bh; ++y) {
+            const auto yf = static_cast<float>(y) / dst_bh_f;
+            for (size_t x = 0; x < dst_bw; ++x) {
+              const auto xf = static_cast<float>(x) / dst_bw_f;
+
+              const auto srcx = static_cast<size_t>(bw_f*xf) + off_x;
+              const auto srcy = static_cast<size_t>(bh_f*yf) + off_y;
+
+              auto ptr = frame.rgba + 4*(frame.w*srcy+srcx);
+              for (size_t i = 0; i < 4; ++i) {
+                (itr++)->second = *(ptr++);
+              }
+            }
+          }
+        }
+      }
+
+      const auto bx_off = dst_bw*bx;
+      const auto by_off = dst_bh*by;
+
+      for (size_t bt = 0; bt < utime; ++bt) {
+        const auto bt_f = static_cast<float>(bt) / static_cast<float>(utime);
+
+        const auto frame = src->Fetch(t*utime+bt);
+        if (!frame.rgba) throw "got an empty frame";
+        for (size_t y = 0; y < w; ++y) {
+          for (size_t x = 0; x < h; ++x) {
+            const bool is_target =
+                bx_off <= x && x < bx_off+dst_bw && by_off <= y && y < by_off+dst_bh;
+            if (is_target) {
+              const auto* ptr = &block_rgba[4*((x-bx_off)+(y-by_off)*dst_bw)];
+              for (size_t i = 0; i < 4; ++i) {
+                const auto a = static_cast<float>(ptr->second-ptr->first);
+                const auto b = static_cast<float>(ptr->first);
+                *(dst_ptr++) = static_cast<uint8_t>(bt_f*a+b);
+                ++ptr;
+              }
+            } else {
+              const auto xf = static_cast<float>(x) / static_cast<float>(w);
+              const auto yf = static_cast<float>(y) / static_cast<float>(h);
+
+              const auto src_x = static_cast<size_t>(static_cast<float>(frame.w)*xf);
+              const auto src_y = static_cast<size_t>(static_cast<float>(frame.h)*yf);
+
+              auto ptr = frame.rgba + 4*(src_x+src_y*frame.w);
+              for (size_t i = 0; i < 4; ++i) {
+                *(dst_ptr++) = (*ptr++);
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+  static std::vector<Feat> ParseFeats(const std::string& script) {
+    std::vector<Feat> ret;
+
+    const char* begin = &*script.c_str();
+    const char* end   = begin + script.size();
+
+    size_t time = 0;
+    for (auto itr = begin; itr < end; ++itr, ++time) {
+      char* end;
+      const auto idx = std::strtol(&*itr, &end, 0);
+      if (itr == end) throw "invalid format: expected block index";
+      if (*end != 'C' && *end != 'L') {
+        throw "invalid format: unknown type";
+      }
+      itr = end+1;
+      ret.push_back(Feat { .block = static_cast<size_t>(idx), .type = *end, });
+    }
+    return ret;
+  }
+};
+
+}
+}  // namespace pg

playground/exporter.cc

@@ -0,0 +1,59 @@
+#include <imgui.h>
+#include <imgui_stdlib.h>
+
+#define STB_IMAGE_WRITE_IMPLEMENTATION
+#include <stb_image_write.h>
+
+#include "app.hh"
+#include "input.hh"
+
+
+namespace pg {
+namespace {
+
+class Exporter final : public App {
+ public:
+  static inline TypeInfo kType = TypeInfo::Create<Exporter>("Exporter");
+
+  Exporter() noexcept {
+  }
+
+  void Update() noexcept override {
+    const auto id = " Exporter | "+
+        std::to_string(reinterpret_cast<uintptr_t>(this));
+
+    if (ImGui::Begin(id.c_str())) {
+      ImGui::DragInt("src", &src_);
+      ImGui::InputText("dir", &dir_);
+      if (ImGui::Button("export")) {
+        Export();
+      }
+    }
+    ImGui::End();
+  }
+
+ private:
+  int src_ = 0;
+  std::string dir_;
+
+  void Export() noexcept {
+    const auto src =
+        Input::instance().slots(static_cast<size_t>(src_));
+
+    const auto n = src->frames();
+    for (size_t i = 0; i < n; ++i) {
+      const auto f = src->Fetch(i);
+      if (!f.rgba) break;
+
+      const auto path = dir_+"/"+std::to_string(i)+".png";
+      stbi_write_png(
+          path.c_str(),
+          static_cast<int>(f.w), static_cast<int>(f.h), 4,
+          f.rgba, static_cast<int>(f.w*4));
+    }
+  }
+};
+
+}
+}  // namespace pg
+

playground/extractor.cc

@@ -0,0 +1,130 @@
+extern "C" {
+#include <liblocky.h>
+}
+
+#include <sstream>
+#include <string>
+
+#include <imgui.h>
+#include <imgui_stdlib.h>
+#include <implot.h>
+
+#include "app.hh"
+#include "input.hh"
+
+
+namespace pg {
+
+class Extractor final : public App {
+ public:
+  static inline TypeInfo kType = TypeInfo::Create<Extractor>("Extractor");
+
+  Extractor() noexcept {
+  }
+
+  void Update() noexcept override {
+    const auto em = ImGui::GetFontSize();
+    const auto id = " Extractor | "+
+        std::to_string(reinterpret_cast<uintptr_t>(this));
+
+    ImGui::SetNextWindowSize({32*em, 32*em}, ImGuiCond_Once);
+    if (ImGui::Begin(id.c_str())) {
+      ImGui::DragInt("src", &src_, 1, 0, 1024);
+      ImGui::DragInt2("block num", block_num_, 1, 1, 1024);
+      ImGui::DragInt2("sensor num", sensor_num_, 1, 1, 1024);
+      ImGui::DragInt("utime", &utime_, 1, 5, 64);
+      ImGui::DragInt("len", &len_, 1, 1, 1024);
+      ImGui::DragFloat("correl max var", &correl_max_var_, 1e-3f, 0, 1);
+      ImGui::DragFloat("correl min avg", &correl_min_avg_, 1e-3f, 0, 1);
+      ImGui::DragFloat("feature min probability", &feat_min_prob_, 1e-3f, 0, 1);
+      if (ImGui::Button("extract")) {
+        Extract();
+      }
+
+      time_ = std::clamp(time_, 0, len_-1);
+      ImGui::SliderInt("time", &time_, 0, len_-1);
+
+      const auto offset = static_cast<size_t>(block_num_[0]*block_num_[1]*time_);
+      if (probs_.size() > offset) {
+        auto avail = ImGui::GetContentRegionAvail();
+        avail.x -= 1*em;
+        avail.y -= 1*em;
+        if (ImPlot::BeginPlot("probs", avail)) {
+          ImPlot::SetupAxisLimits(ImAxis_X1, 0, 1);
+          ImPlot::SetupAxisLimits(ImAxis_Y1, 0, 1);
+
+          ImPlot::PlotHeatmap(
+              "probs", probs_.data()+offset, block_num_[1], block_num_[0],
+              0, 1, "%.3f");
+          ImPlot::EndPlot();
+        }
+      } else {
+        ImGui::TextUnformatted("no data");
+      }
+    }
+    ImGui::End();
+  }
+
+ private:
+  int src_ = 0;
+  int block_num_[2] = {2, 2};
+  int sensor_num_[2] = {2, 2};
+  int utime_ = 10;
+  int len_ = 1;
+
+  float correl_max_var_ = 1e-3f;
+  float correl_min_avg_ = 0.8f;
+  float feat_min_prob_  = .5f;
+
+  int time_;
+
+  std::vector<double> probs_;
+
+
+  void Extract() noexcept {
+    const auto src = Input::instance().slots(static_cast<size_t>(src_));
+    if (!src) return;
+
+    const uint64_t dur = static_cast<uint64_t>(utime_ * len_);
+
+    blky_extractor_t ex = {};
+
+    ex.block_num_x = static_cast<uint32_t>(block_num_[0]);
+    ex.block_num_y = static_cast<uint32_t>(block_num_[1]);
+    ex.sensor_num_block_x = static_cast<uint32_t>(sensor_num_[0]);
+    ex.sensor_num_block_y = static_cast<uint32_t>(sensor_num_[1]);
+    ex.samples_per_pix = 3;
+    ex.pix_stride = 4;
+    ex.utime = static_cast<uint32_t>(utime_);
+    ex.correl_max_var = static_cast<double>(correl_max_var_);
+    ex.correl_min_avg = static_cast<double>(correl_min_avg_);
+    blky_extractor_init(&ex);
+
+    static const double verts[] = {
+      0, 0,
+      0, 1,
+      1, 1,
+      1, 0,
+    };
+
+    const uint64_t block_num = ex.block_num_x * ex.block_num_y;
+    probs_.clear();
+    probs_.reserve(block_num*static_cast<uint64_t>(len_));
+    for (uint64_t t = 0; t < dur; ++t) {
+      const auto f = src->Fetch(t);
+      if (!f.rgba) break;
+
+      const bool pop = blky_extractor_feed(
+          &ex, f.rgba,
+          static_cast<uint32_t>(f.w),
+          static_cast<uint32_t>(f.h),
+          verts);
+      if (pop) {
+        probs_.insert(probs_.end(), ex.probs, ex.probs+block_num);
+      }
+    }
+    blky_extractor_deinit(&ex);
+  }
+};
+
+}  // namespace pg

playground/initiator.hh

@@ -0,0 +1,39 @@
+#pragma once
+
+#include <memory>
+#include <vector>
+
+#include <imgui.h>
+
+#include "app.hh"
+
+
+namespace pg {
+
+class Initiator final : App {
+ public:
+  Initiator() noexcept {
+  }
+
+  void Update() noexcept override {
+    if (ImGui::BeginMainMenuBar()) {
+      if (ImGui::BeginMenu("Apps")) {
+        for (const auto& type : registry()) {
+          if (ImGui::MenuItem(type.first.c_str())) {
+            apps_.push_back(type.second->Create());
+          }
+        }
+        ImGui::EndMenu();
+      }
+      ImGui::EndMainMenuBar();
+    }
+    for (auto& app : apps_) {
+      app->Update();
+    }
+  }
+
+ private:
+  std::vector<std::unique_ptr<App>> apps_;
+};
+
+}  // namespace pg

playground/input.cc

@@ -0,0 +1,52 @@
+#include "input.hh"
+
+#include <imgui.h>
+#include <imgui_stdlib.h>
+
+
+namespace pg {
+
+Input Input::instance_;
+
+
+size_t Input::Add(Data* data) noexcept {
+  slots_.push_back(data);
+  return slots_.size()-1;
+}
+
+void Input::Update() noexcept {
+  if (ImGui::BeginMainMenuBar()) {
+    if (ImGui::BeginMenu("Input")) {
+      for (size_t i = 0; i < slots_.size(); ++i) {
+        const auto str =
+            std::to_string(i)+". "+slots_[i]->name();
+        ImGui::MenuItem(str.c_str());
+      }
+      ImGui::EndMenu();
+    }
+    ImGui::EndMainMenuBar();
+  }
+}
+
+
+std::vector<float> Input::Data::FetchSamples(
+    size_t st, size_t dur, float xf, float yf, size_t offset) noexcept {
+  std::vector<float> ret(dur);
+  for (size_t i = 0; i < dur; ++i) {
+    const auto frame = Fetch(i+st);
+    if (!frame.rgba) continue;
+
+    const auto fwf = static_cast<float>(frame.w);
+    const auto fhf = static_cast<float>(frame.h);
+    const auto fhi = static_cast<intmax_t>(frame.h);
+
+    const auto x = static_cast<intmax_t>(xf*fwf);
+    const auto y = static_cast<intmax_t>(yf*fhf);
+
+    const auto v = frame.rgba + 4*(y*fhi+x);
+    ret[i] = static_cast<float>(v[offset]) / UINT8_MAX;
+  }
+  return ret;
+}
+
+}  // namespace pg

playground/input.hh

@@ -0,0 +1,71 @@
+#pragma once
+
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include "app.hh"
+
+
+namespace pg {
+
+class Input final : App {
+ public:
+  class Data;
+
+  static Input& instance() noexcept { return instance_; }
+
+  Input() = default;
+  Input(const Input&) = delete;
+  Input(Input&&) = delete;
+  Input& operator=(const Input&) = delete;
+  Input& operator=(Input&&) = delete;
+
+  size_t Add(Data*) noexcept;
+  void Update() noexcept override;
+
+  Data* slots(size_t idx) noexcept {
+    return idx < slots_.size()? slots_[idx]: nullptr;
+  }
+
+ private:
+  static Input instance_;
+
+  std::vector<Data*> slots_;
+};
+
+class Input::Data {
+ public:
+  struct Frame {
+   public:
+    size_t w, h;
+    const uint8_t* rgba;
+  };
+
+  Data() = delete;
+  Data(std::string_view name) noexcept : name_(name) {
+    idx_ = Input::instance().Add(this);
+  }
+  virtual ~Data() = default;
+  Data(const Data&) = delete;
+  Data(Data&&) = delete;
+  Data& operator=(const Data&) = delete;
+  Data& operator=(Data&&) = delete;
+
+  virtual Frame Fetch(size_t) noexcept = 0;
+  virtual size_t frames() noexcept = 0;
+
+  size_t index() const noexcept { return idx_; }
+
+  std::vector<float> FetchSamples(
+      size_t st, size_t dur, float x, float y, size_t offset) noexcept;
+
+  const std::string& name() const noexcept { return name_; }
+
+ private:
+  std::string name_;
+
+  size_t idx_;
+};
+
+}  // namespace pg

playground/input_gradient.cc

@@ -0,0 +1,52 @@
+#include "app.hh"
+#include "input.hh"
+
+#include <array>
+#include <cstring>
+
+#include <imgui.h>
+
+
+namespace pg {
+namespace {
+
+class Gradient final : public App, public Input::Data {
+ public:
+  static inline TypeInfo kType = TypeInfo::Create<Gradient>("Input_Gradient");
+
+  Gradient() noexcept : Data("gradient") {
+  }
+
+  void Update() noexcept override {
+    const auto id = std::to_string(index())+" Input_Gradient | "+
+        std::to_string(reinterpret_cast<uintptr_t>(this));
+
+    if (ImGui::Begin(id.c_str())) {
+      ImGui::DragInt("w", &w_, 1, 1, 1024);
+      ImGui::DragInt("h", &h_, 1, 1, 1024);
+      ImGui::DragInt("dur", &dur_, 1, 1, 1024);
+    }
+    ImGui::End();
+  }
+
+  Frame Fetch(size_t n) noexcept override {
+    const auto w = static_cast<size_t>(w_);
+    const auto h = static_cast<size_t>(h_);
+    buf_.resize(w*h*4);
+
+    const auto f = static_cast<float>(n)/static_cast<float>(dur_);
+    std::memset(buf_.data(), static_cast<uint8_t>(f*UINT8_MAX), buf_.size());
+    return Frame {.w = w, .h = h, .rgba = buf_.data()};
+  }
+  size_t frames() noexcept override {
+    return static_cast<size_t>(dur_);
+  }
+
+ private:
+  int w_ = 100, h_ = 100;
+  int dur_ = 100;
+  std::vector<uint8_t> buf_;
+};
+
+}
+}  // namespace pg

playground/input_imgseq.cc

@@ -0,0 +1,96 @@
+#include <string>
+#include <numeric>
+#include <vector>
+
+#include <imgui.h>
+#include <imgui_stdlib.h>
+
+#include "app.hh"
+#include "input.hh"
+
+#define STB_IMAGE_IMPLEMENTATION
+#include <stb_image.h>
+
+
+namespace pg {
+namespace {
+
+class ImgSeq final : public App, public Input::Data {
+ public:
+  static inline TypeInfo kType = TypeInfo::Create<ImgSeq>("Input_ImgSeq");
+
+  ImgSeq() noexcept : Data("imgseq") {
+  }
+  ~ImgSeq() noexcept {
+    DropCache();
+  }
+  void DropCache() noexcept {
+    for (auto& frame : frames_) {
+      stbi_image_free(const_cast<uint8_t*>(frame.second.rgba));
+    }
+    frames_.clear();
+  }
+
+  void Update() noexcept override {
+    const auto id = std::to_string(index())+" Input_ImgSeq | "+
+        std::to_string(reinterpret_cast<uintptr_t>(this));
+
+    if (ImGui::Begin(id.c_str())) {
+      bool mod = false;
+      mod |= ImGui::InputText("dir", &path_);
+      mod |= ImGui::DragInt("start", &st_, 1, 0, 1024);
+      mod |= ImGui::DragInt("end",   &ed_, 1, 0, 1024);
+      if (mod) DropCache();
+
+      if (msg_.size() > 0) {
+        ImGui::TextUnformatted(msg_.c_str());
+      }
+    }
+    ImGui::End();
+  }
+
+  Frame Fetch(size_t n) noexcept override
+  try {
+    n += static_cast<size_t>(st_);
+    if (n >= static_cast<size_t>(ed_)) {
+      throw "frame number out of range";
+    }
+
+    auto itr = frames_.find(n);
+    if (itr != frames_.end()) return itr->second;
+
+    const auto fname = path_+"/"+std::to_string(n)+".png";
+
+    int w, h, comp;
+    uint8_t* buf = stbi_load(fname.c_str(), &w, &h, &comp, 4);
+    if (buf == nullptr) {
+      throw stbi_failure_reason();
+    }
+
+    const Frame ret = {
+      .w = static_cast<size_t>(w),
+      .h = static_cast<size_t>(h),
+      .rgba = buf,
+    };
+    frames_[n] = ret;
+    return ret;
+  } catch (const char* msg) {
+    msg_ = msg;
+    return {.w = 0, .h = 0, .rgba = nullptr};
+  }
+  size_t frames() noexcept override {
+    if (ed_ <= st_) return 0;
+    return static_cast<size_t>(ed_-st_);
+  }
+
+ private:
+  std::string path_;
+  int st_ = 1, ed_ = 100;
+
+  std::string msg_;
+
+  std::unordered_map<size_t, Frame> frames_;
+};
+
+}
+}  // namespace pg

playground/input_noise.cc

@@ -0,0 +1,103 @@
+#include "app.hh"
+#include "input.hh"
+
+#include <vector>
+
+#include <imgui.h>
+
+
+namespace pg {
+namespace {
+
+class Noise final : public App, public Input::Data {
+ public:
+  static inline TypeInfo kType = TypeInfo::Create<Noise>("Input_Noise");
+
+  static uint32_t xorshift(uint32_t x) noexcept {
+    x ^= x<<13;
+    x ^= x>>17;
+    x ^= x<<5;
+    return x;
+  }
+
+  Noise() noexcept : Data("random noise") {
+  }
+
+  void Update() noexcept override {
+    const auto id = std::to_string(index())+" Input_Noise | "+
+        std::to_string(reinterpret_cast<uintptr_t>(this));
+
+    if (ImGui::Begin(id.c_str())) {
+      ImGui::DragInt("src", &src_, 1, 0, 1024);
+
+      ImGui::DragInt("w", &w_, 1, 1, 1024);
+      ImGui::DragInt("h", &h_, 1, 1, 1024);
+
+      ImGui::DragFloat("level", &level_, 1e-4f, 0, 1);
+      ImGui::DragInt("seed", &seed_);
+    }
+    ImGui::End();
+  }
+
+  Frame Fetch(size_t n) noexcept override {
+    auto src = Input::instance().slots(static_cast<size_t>(src_));
+    if (!src) return {};
+
+    const auto w = static_cast<size_t>(w_);
+    const auto h = static_cast<size_t>(h_);
+
+    buf_.resize(w*h*4);
+    uint8_t* buf = buf_.data();
+
+    auto srcf = src->Fetch(n);
+    for (size_t y = 0; y < h; ++y) {
+      const auto yf = static_cast<float>(y)/static_cast<float>(h);
+      for (size_t x = 0; x < w; ++x) {
+        const auto xf = static_cast<float>(x)/static_cast<float>(w);
+        const auto srcx  = static_cast<float>(srcf.w) * xf;
+        const auto srcy  = static_cast<float>(srcf.h) * yf;
+        const auto srcxi = static_cast<size_t>(srcx);
+        const auto srcyi = static_cast<size_t>(srcy);
+
+        auto v = srcf.rgba + 4*(srcxi+srcyi*srcf.w);
+        for (size_t i = 0; i < 4; ++i) {
+          *(buf++) = TryAttack(n, x, y, i, *(v++));
+        }
+      }
+    }
+    return Frame { .w = w, .h = h, .rgba = buf_.data(), };
+  }
+  size_t frames() noexcept override {
+    auto src = Input::instance().slots(static_cast<size_t>(src_));
+    if(!src) return 0;
+    return src->frames();
+  }
+
+  uint8_t TryAttack(
+      size_t n, size_t x, size_t y, size_t i, uint8_t v) const noexcept {
+    if (i == 3) return v;
+
+    const auto s = static_cast<uint32_t>(seed_);
+    const auto seed = static_cast<uint32_t>(s*s*s*n + s*s*x + s*y + i);
+
+    const auto rand = xorshift(seed);
+    if (rand%100 < 50) {
+      const auto t = static_cast<float>(rand%100)/100.f*level_;
+      const auto a = static_cast<uint8_t>(t*UINT8_MAX);
+      return v+a;
+    } else {
+      return v;
+    }
+  }
+
+ private:
+  int src_ = 0;
+  int w_ = 100, h_ = 100;
+  float level_ = 0.01f;
+  int seed_ = 1234;
+
+  std::vector<uint8_t> buf_;
+};
+
+}
+}  // namespace pg

playground/main.cc

@@ -0,0 +1,102 @@
+#include <iostream>
+
+#include <GL/glew.h>
+
+#include <imgui.h>
+#include <imgui_impl_glfw.h>
+#include <imgui_impl_opengl3.h>
+#include <implot.h>
+
+// To prevent conflicts caused by fucking windows.h, include GLFW last.
+#include <GLFW/glfw3.h>
+
+#include "initiator.hh"
+#include "input.hh"
+
+
+#if defined(_MSC_VER) && (_MSC_VER >= 1900) && !defined(IMGUI_DISABLE_WIN32_FUNCTIONS)
+# pragma comment(lib, "legacy_stdio_definitions")
+#endif
+
+
+int main(int, char**) {
+  // init display
+  glfwSetErrorCallback(
+      [](int, const char* msg) {
+        std::cout << "GLFW error: " << msg << std::endl;
+      });
+  if (!glfwInit()) return 1;
+
+  GLFWwindow* window;
+  const char* glsl_version;
+  glfwWindowHint(GLFW_VISIBLE, GLFW_FALSE);
+# if defined(__APPLE__)
+    glsl_version = "#version 150";
+    glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 3);
+    glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 2);
+    glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);
+    glfwWindowHint(GLFW_OPENGL_FORWARD_COMPAT, GL_TRUE);
+# else
+    glsl_version = "#version 130";
+    glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 3);
+    glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 3);
+    glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);
+    glfwWindowHint(GLFW_OPENGL_FORWARD_COMPAT, GL_TRUE);
+# endif
+  window = glfwCreateWindow(1280, 720, "playground", NULL, NULL);
+  if (window == NULL) return 1;
+  glfwMakeContextCurrent(window);
+  glfwSwapInterval(1);
+  if (glewInit() != GLEW_OK) return 1;
+
+  // init ImGUI
+  IMGUI_CHECKVERSION();
+  ImGui::CreateContext();
+  ImPlot::CreateContext();
+
+  auto& io = ImGui::GetIO();
+  io.IniFilename  = nullptr;
+  io.ConfigFlags |= ImGuiConfigFlags_DockingEnable;
+
+  ImGui::StyleColorsDark();
+  ImGui_ImplGlfw_InitForOpenGL(window, true);
+  ImGui_ImplOpenGL3_Init(glsl_version);
+
+  pg::Initiator initiator;
+  glfwShowWindow(window);
+
+  // main loop
+  while (!glfwWindowShouldClose(window)) {
+    // new frame
+    glfwPollEvents();
+    ImGui_ImplOpenGL3_NewFrame();
+    ImGui_ImplGlfw_NewFrame();
+    ImGui::NewFrame();
+
+    // update GUI state
+    initiator.Update();
+    pg::Input::instance().Update();
+
+    // render windows
+    ImGui::Render();
+
+    int w, h;
+    glfwGetFramebufferSize(window, &w, &h);
+    glViewport(0, 0, w, h);
+
+    glClear(GL_COLOR_BUFFER_BIT);
+    ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData());
+
+    glfwSwapBuffers(window);
+  }
+  // teardown ImGUI
+  ImGui_ImplOpenGL3_Shutdown();
+  ImGui_ImplGlfw_Shutdown();
+  ImPlot::DestroyContext();
+  ImGui::DestroyContext();
+
+  // teardown display
+  glfwDestroyWindow(window);
+  glfwTerminate();
+  return 0;
+}

playground/player.cc

@@ -0,0 +1,161 @@
+#include <algorithm>
+#include <cassert>
+#include <string>
+#include <vector>
+
+#include <GL/glew.h>
+#include <imgui.h>
+
+#include "app.hh"
+#include "input.hh"
+
+
+namespace pg {
+namespace {
+
+class Player final : public App {
+ public:
+  static inline TypeInfo kType = TypeInfo::Create<Player>("Player");
+
+  static constexpr size_t kTexW = 1024;
+  static constexpr size_t kTexH = 1024;
+
+  Player() noexcept {
+    glGenTextures(1, &tex_);
+
+    glBindTexture(GL_TEXTURE_2D, tex_);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    glBindTexture(GL_TEXTURE_2D, 0);
+  }
+  ~Player() noexcept {
+    glDeleteTextures(1, &tex_);
+  }
+
+  void Update() noexcept override {
+    const auto em = ImGui::GetFontSize();
+    const auto id = "Player | "+
+        std::to_string(reinterpret_cast<uintptr_t>(this));
+
+    ImGui::SetNextWindowSize({32*em, 16*em}, ImGuiCond_Once);
+    if (ImGui::Begin(id.c_str())) {
+      bool mod = false;
+
+      ImGui::BeginGroup();
+      {
+        ImGui::PushItemWidth(4*em);
+        mod |= ImGui::DragInt("input_slot", &slot_, 1, 0, 1024);
+        mod |= ImGui::DragInt("time", &t_, 1, 0, 1024);
+
+        ImGui::Checkbox("auto increment", &auto_inc_);
+        if (auto_inc_) {
+          ++t_; mod = true;
+        }
+
+        ImGui::PopItemWidth();
+      }
+      ImGui::EndGroup();
+      ImGui::SameLine();
+
+      ImVec2 img_min = {0, 0};
+      ImVec2 img_max = {0, 0};
+      if (mod) UpdateTex();
+      const auto wf  = static_cast<float>(w_);
+      const auto hf  = static_cast<float>(h_);
+
+      ImGui::BeginGroup();
+      {
+        if (w_ && h_) {
+          const auto tex = (void*) (uintptr_t) tex_;
+          const auto z   = wf / ImGui::GetContentRegionAvail().x;
+          ImGui::Image(tex, {wf/z, hf/z}, {0, 0}, uv_);
+          img_min = ImGui::GetItemRectMin();
+          img_max = ImGui::GetItemRectMax();
+        }
+      }
+      ImGui::EndGroup();
+
+      if (msg_.size() > 0) {
+        ImGui::TextUnformatted(msg_.c_str());
+      } else if (w_ && h_) {
+        const auto m  = ImGui::GetMousePos();
+        const auto xf = (m.x-img_min.x)/(img_max.x-img_min.x);
+        const auto yf = (m.y-img_min.y)/(img_max.y-img_min.y);
+        ImGui::Text("x=%f, y=%f, xf=%f, yf=%f", xf*wf, yf*hf, xf, yf);
+      } else {
+        ImGui::TextUnformatted("no image shown");
+      }
+    }
+    ImGui::End();
+  }
+
+ private:
+  std::string msg_;
+
+  int slot_      = 0;
+  int t_         = 0;
+  bool auto_inc_ = false;
+
+  GLsizei w_ = 0, h_ = 0;
+  ImVec2 uv_;
+
+  GLuint tex_;
+  GLsizei texw_ = 0, texh_ = 0;
+
+
+  void UpdateTex() noexcept
+  try {
+    msg_ = "";
+
+    auto data = Input::instance().slots(static_cast<size_t>(slot_));
+    if (data == nullptr) {
+      throw "missing slot";
+    }
+    if (static_cast<size_t>(t_) >= data->frames()) {
+      throw "time out of range";
+    }
+    const auto frame = data->Fetch(static_cast<size_t>(t_));
+    if (!frame.rgba) {
+      throw "got an empty frame";
+    }
+    w_ = static_cast<GLsizei>(frame.w);
+    h_ = static_cast<GLsizei>(frame.h);
+
+    glBindTexture(GL_TEXTURE_2D, tex_);
+
+    const auto ptexw = texw_;
+    const auto ptexh = texh_;
+    texw_ = std::max(texw_, NextPowerOf2(w_));
+    texh_ = std::max(texh_, NextPowerOf2(h_));
+    if (texw_ != ptexw || texh_ != ptexh) {
+      glTexImage2D(
+          GL_TEXTURE_2D, 0, GL_RGBA, texw_, texh_,
+          0, GL_RGBA, GL_UNSIGNED_BYTE, nullptr);
+    }
+
+    glTexSubImage2D(
+        GL_TEXTURE_2D, 0, 0, 0, w_, h_,
+        GL_RGBA, GL_UNSIGNED_BYTE, frame.rgba);
+    uv_ = {
+      static_cast<float>(w_)/static_cast<float>(texw_),
+      static_cast<float>(h_)/static_cast<float>(texh_)};
+
+    glBindTexture(GL_TEXTURE_2D, 0);
+    assert(glGetError() == 0);
+  } catch (const char* msg) {
+    msg_ = msg;
+  }
+
+
+  template <typename I>
+  static I NextPowerOf2(I x) {
+    I y = 1;
+    while (y < x) y *= 2;
+    return y;
+  }
+};
+
+}
+}  // namespace pg

playground/sensor.cc

@@ -0,0 +1,192 @@
+extern "C" {
+# include <liblocky.h>
+}
+
+#include <algorithm>
+#include <cinttypes>
+#include <cmath>
+#include <numeric>
+#include <string>
+#include <vector>
+
+#include <imgui.h>
+#include <implot.h>
+
+#include "app.hh"
+#include "input.hh"
+
+
+namespace pg {
+namespace {
+
+class Sensor final : public App {
+ public:
+  static inline TypeInfo kType = TypeInfo::Create<Sensor>("Sensor");
+
+  Sensor() noexcept {
+  }
+
+  void Update() noexcept override {
+    const auto em = ImGui::GetFontSize();
+    const auto id = "Sensor | "+
+        std::to_string(reinterpret_cast<uintptr_t>(this));
+
+    ImGui::SetNextWindowSize({64*em, 24*em}, ImGuiCond_Once);
+    if (ImGui::Begin(id.c_str())) {
+      bool mod = false;
+      ImGui::BeginGroup();
+      {
+        ImGui::PushItemWidth(6*em);
+        mod |= ImGui::DragInt("input slot", &src_, 1, 0, 1024);
+        mod |= ImGui::DragInt("start", &start_, 1, 0, 1024);
+        mod |= ImGui::DragInt("dur", &dur_, 1, 1, BLKY_SENSOR_MAX_DUR);
+
+        ImGui::Spacing();
+        ImGui::DragFloat2("pos", pos_, .001f);
+        ImGui::DragInt("offset", &offset_, 1, 0, 3);
+        if (ImGui::Button("add")) {
+          mod = true;
+          data_.emplace_back(pos_[0], pos_[1], offset_);
+        }
+        if (ImGui::BeginListBox("##points")) {
+          for (size_t i = 0; i < data_.size(); ++i) {
+            const auto& data = data_[i];
+            const auto  name =
+                std::to_string(i)+". "+
+                std::to_string(data.pos[0])+","+
+                std::to_string(data.pos[1]);
+            ImGui::Selectable(name.c_str());
+            if (ImGui::BeginPopupContextItem()) {
+              if (ImGui::MenuItem("clear all")) {
+                mod = true;
+                data_.clear();
+              }
+              ImGui::EndPopup();
+            }
+            if (ImGui::IsItemHovered()) {
+              ImGui::SetTooltip(
+                  "pos   : %f,%f + %" PRIiMAX "\n"
+                  "avg   : %f\n"
+                  "var   : %f\n"
+                  "correl: %f",
+                  data.pos[0], data.pos[1], data.offset,
+                  data.avg, data.var, data.correl);
+            }
+          }
+          ImGui::EndListBox();
+        }
+
+        double correl_abs_avg = 0;
+        for (auto& data : data_) {
+          correl_abs_avg += fabs(data.correl);
+        }
+        correl_abs_avg /= static_cast<float>(data_.size());
+        ImGui::Text("avg(abs(r)) = %f", correl_abs_avg);
+
+        double correl_abs_var = 0;
+        for (auto& data : data_) {
+          const auto diff = fabs(data.correl)-correl_abs_avg;
+          correl_abs_var += diff*diff;
+        }
+        correl_abs_var /= static_cast<float>(data_.size());
+        ImGui::Text("var(abs(r)) = %f", correl_abs_var);
+
+        ImGui::PopItemWidth();
+        if (mod) {
+          try {
+            Calc();
+            msg_ = "";
+          } catch (const char* msg) {
+            msg_ = msg;
+          }
+        }
+      }
+      ImGui::EndGroup();
+      ImGui::SameLine();
+      ImGui::BeginGroup();
+      auto avail = ImGui::GetContentRegionAvail();
+      avail.x -= em;
+      avail.y -= em;
+
+      const ImVec2 size = {avail.x/2, avail.y};
+
+      if (ImPlot::BeginPlot("input value", size)) {
+        ImPlot::SetupAxis(ImAxis_X1, nullptr, ImPlotAxisFlags_AutoFit);
+        ImPlot::SetupAxisLimits(ImAxis_Y1, -0.1, 1.1);
+        for (size_t i = 0; i < data_.size(); ++i) {
+          const auto& d = data_[i];
+          ImPlot::PlotLine(
+              std::to_string(i).c_str(),
+              d.values.data(), static_cast<int>(d.values.size()), 1, start_);
+        }
+        ImPlot::EndPlot();
+      }
+      ImGui::SameLine();
+      if (ImPlot::BeginPlot("input histogram", size)) {
+        ImPlot::SetupAxisLimits(ImAxis_X1, 0, 1);
+        ImPlot::SetupAxis(ImAxis_Y1, nullptr, ImPlotAxisFlags_AutoFit);
+        for (size_t i = 0; i < data_.size(); ++i) {
+          const auto& d = data_[i];
+          ImPlot::PlotHistogram(
+              std::to_string(i).c_str(),
+              d.values.data(), static_cast<int>(d.values.size()));
+        }
+        ImPlot::EndPlot();
+      }
+      ImGui::EndGroup();
+      ImGui::TextUnformatted(msg_.c_str());
+    }
+    ImGui::End();
+  }
+
+ private:
+  int src_ = 0, start_ = 0, dur_ = 30;
+  float pos_[2];
+  int offset_;
+
+  struct Data final {
+   public:
+    float pos[2];
+    size_t offset;
+    std::vector<float> values;
+
+    double avg;
+    double var;
+    double cov;
+    double correl;
+
+    Data(float x, float y, int off) noexcept :
+        pos{x, y}, offset(static_cast<size_t>(off)) {
+    }
+  };
+  std::vector<Data> data_;
+  std::string msg_;
+
+  void Calc();
+};
+
+void Sensor::Calc() {
+  auto in = Input::instance().slots(static_cast<size_t>(src_));
+  if (!in) throw "missing slot";
+
+  if (dur_ == 0) throw "invalid time range";
+
+  for (auto& data : data_) {
+    const auto xf = std::clamp(data.pos[0], 0.f, 1.f);
+    const auto yf = std::clamp(data.pos[1], 0.f, 1.f);
+
+    const auto dur = static_cast<size_t>(dur_);
+    const auto st  = static_cast<size_t>(start_);
+    data.values = in->FetchSamples(st, dur, xf, yf, data.offset);
+
+    blky_sensor_t sensor = {};
+    blky_sensor_feed(&sensor, data.values.data(), data.values.size());
+    data.avg    = sensor.avg;
+    data.var    = sensor.var;
+    data.cov    = sensor.cov;
+    data.correl = sensor.correl;
+  }
+}
+
+}
+}  // namespace pg

thirdparty/CMakeLists.txt

@@ -0,0 +1,120 @@
+include(FetchContent)
+
+
+# ---- GLEW ----
+# repository: https://github.com/Perlmint/glew-cmake
+# license   :  Modified BSD License, the Mesa 3-D License (MIT) and the Khronos License (MIT).
+
+FetchContent_Declare(
+  glew
+  URL "https://github.com/Perlmint/glew-cmake/archive/refs/tags/glew-cmake-2.2.0.zip"
+)
+FetchContent_MakeAvailable(glew)
+
+if (KINGTAKER_STATIC)
+  add_library(glew ALIAS libglew_static)
+else()
+  add_library(glew ALIAS libglew_shared)
+endif()
+
+
+# ---- GLFW ----
+# repository: https://github.com/glfw/glfw
+# license   : zlib
+
+FetchContent_Declare(
+  glfw
+  URL "https://github.com/glfw/glfw/archive/refs/tags/3.3.4.zip"
+)
+
+set(GLFW_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
+set(GLFW_BUILD_TESTS    OFF CACHE BOOL "" FORCE)
+set(GLFW_BUILD_DOCS     OFF CACHE BOOL "" FORCE)
+set(GLFW_INSTALL        OFF CACHE BOOL "" FORCE)
+
+FetchContent_MakeAvailable(glfw)
+
+
+# ---- ImGUI (docking branch) ----
+# repository: https://github.com/ocornut/imgui/
+# license   : MIT
+
+FetchContent_Declare(
+  imgui
+  URL "https://github.com/ocornut/imgui/archive/9b0c26b0b2adae3ccf66dc9552fae4945d735a0c.zip"
+)
+FetchContent_Populate(imgui)
+
+add_library(imgui)
+target_sources(imgui
+  PRIVATE
+    "${imgui_SOURCE_DIR}/imgui.cpp"
+    "${imgui_SOURCE_DIR}/imgui_demo.cpp"
+    "${imgui_SOURCE_DIR}/imgui_draw.cpp"
+    "${imgui_SOURCE_DIR}/imgui_internal.h"
+    "${imgui_SOURCE_DIR}/imgui_tables.cpp"
+    "${imgui_SOURCE_DIR}/imgui_widgets.cpp"
+    "${imgui_SOURCE_DIR}/backends/imgui_impl_glfw.cpp"
+    "${imgui_SOURCE_DIR}/backends/imgui_impl_opengl3.cpp"
+    "${imgui_SOURCE_DIR}/misc/cpp/imgui_stdlib.cpp"
+  PUBLIC
+    "${imgui_SOURCE_DIR}/imgui.h"
+    "${imgui_SOURCE_DIR}/imstb_rectpack.h"
+    "${imgui_SOURCE_DIR}/imstb_textedit.h"
+    "${imgui_SOURCE_DIR}/imstb_truetype.h"
+    "${imgui_SOURCE_DIR}/backends/imgui_impl_glfw.h"
+    "${imgui_SOURCE_DIR}/backends/imgui_impl_opengl3.h"
+    "${imgui_SOURCE_DIR}/misc/cpp/imgui_stdlib.h"
+)
+target_include_directories(imgui SYSTEM
+  PUBLIC
+    "${imgui_SOURCE_DIR}"
+    "${imgui_SOURCE_DIR}/backends"
+    "${imgui_SOURCE_DIR}/misc/cpp"
+)
+target_link_libraries(imgui
+  PRIVATE glfw
+)
+
+
+# ---- ImPlot ----
+# repository: https://github.com/epezent/implot
+# license   : MIT
+
+FetchContent_Declare(
+  implot
+  URL "https://github.com/epezent/implot/archive/refs/heads/master.zip"
+)
+FetchContent_Populate(implot)
+
+add_library(implot)
+target_link_libraries(implot
+  PRIVATE
+    imgui
+)
+target_include_directories(implot SYSTEM
+  PUBLIC
+    "${implot_SOURCE_DIR}"
+)
+target_sources(implot
+  PUBLIC
+    "${implot_SOURCE_DIR}/implot.h"
+    "${implot_SOURCE_DIR}/implot_internal.h"
+  PRIVATE
+    "${implot_SOURCE_DIR}/implot.cpp"
+    "${implot_SOURCE_DIR}/implot_items.cpp"
+)
+
+
+# ---- stb ----
+FetchContent_Declare(
+  stb
+  URL "https://github.com/nothings/stb/archive/af1a5bc352164740c1cc1354942b1c6b72eacb8a.zip"
+)
+FetchContent_Populate(stb)
+
+add_library(stb INTERFACE)
+target_include_directories(stb SYSTEM
+  INTERFACE
+    "${stb_SOURCE_DIR}"
+)