diff --git a/blocky/CMakeLists.txt b/blocky/CMakeLists.txt index e73b709..af0e8b0 100644 --- a/blocky/CMakeLists.txt +++ b/blocky/CMakeLists.txt @@ -7,6 +7,8 @@ target_sources(blocky bytes.hh common.hh + embed.hh + extract.hh video_encoder.hh video_decoder.hh ) diff --git a/blocky/embed.hh b/blocky/embed.hh index af8712f..9bd9047 100644 --- a/blocky/embed.hh +++ b/blocky/embed.hh @@ -8,6 +8,8 @@ #include "video_encoder.hh" +namespace blky { + void Embed(const std::vector& features, const std::string& dst_path, const std::string& video_path, @@ -68,21 +70,24 @@ void Embed(const std::vector& features, feat_pix.resize(feat_size_x*feat_size_y); for (uint32_t y = 0; y < feat_size_y; ++y) { const uint32_t ay = y+feat_offset_y; - std::memcpy(feat_pix.data()+(y/2)*(feat_size_x/2), U.data()+(ay/2)*(w/2)+feat_offset_x/2, feat_size_x/2); - std::memcpy(feat_pix.data()+(y/2)*(feat_size_x/2)+feat_pix.size()/2, V.data()+(ay/2)*(w/2)+feat_offset_x/2, feat_size_x/2); + std::memcpy(feat_pix.data()+y*feat_size_x, Y.data()+ay*w+feat_offset_x, feat_size_x); + //std::memcpy(feat_pix.data()+(y/2)*(feat_size_x/2), U.data()+(ay/2)*(w/2)+feat_offset_x/2, feat_size_x/2); + //std::memcpy(feat_pix.data()+(y/2)*(feat_size_x/2)+feat_pix.size()/2, V.data()+(ay/2)*(w/2)+feat_offset_x/2, feat_size_x/2); } last_feat = feat; + if (enc) enc->ForceIntraFrame(); } for (uint32_t y = 0; y < feat_size_y; ++y) { const uint32_t ay = y+feat_offset_y; - std::memcpy(U.data()+(ay/2)*(w/2)+feat_offset_x/2, feat_pix.data()+(y/2)*(feat_size_x/2), feat_size_x/2); - std::memcpy(V.data()+(ay/2)*(w/2)+feat_offset_x/2, feat_pix.data()+(y/2)*(feat_size_x/2)+feat_pix.size()/2, feat_size_x/2); + std::memcpy(Y.data()+ay*w+feat_offset_x, feat_pix.data()+y*feat_size_x, feat_size_x); + //std::memcpy(U.data()+(ay/2)*(w/2)+feat_offset_x/2, feat_pix.data()+(y/2)*(feat_size_x/2), feat_size_x/2); + //std::memcpy(V.data()+(ay/2)*(w/2)+feat_offset_x/2, feat_pix.data()+(y/2)*(feat_size_x/2)+feat_pix.size()/2, feat_size_x/2); } // create an encoder if not yet if (!enc) { SEncParamBase param = {}; - param.iUsageType = SCREEN_CONTENT_REAL_TIME; + param.iUsageType = CAMERA_VIDEO_REAL_TIME; param.iPicWidth = static_cast(w); param.iPicHeight = static_cast(h); param.fMaxFrameRate = 30; @@ -105,3 +110,5 @@ void Embed(const std::vector& features, enc->Encode(dst); } } + +} // namespace blky diff --git a/blocky/extract.hh b/blocky/extract.hh new file mode 100644 index 0000000..bb3d186 --- /dev/null +++ b/blocky/extract.hh @@ -0,0 +1,65 @@ +#pragma once + +#include +#include +#include +#include + +#include "common.hh" +#include "video_decoder.hh" + + +namespace blky { + +std::vector Extract( + VideoDecoder& dec, + std::tuple block_num, + std::tuple sensor_num, + uint32_t utime) { + std::vector ret; + + blky_extractor_t ex = {}; + ex.block_num_x = std::get<0>(block_num); + ex.block_num_y = std::get<1>(block_num); + ex.sensor_num_block_x = std::get<0>(sensor_num); + ex.sensor_num_block_y = std::get<1>(sensor_num); + ex.samples_per_pix = 1; + ex.pix_stride = 1; + ex.utime = utime; + ex.correl_max_var = 0.3; + ex.correl_min_avg = 0.8; + + blky_extractor_init(&ex); + while (dec.Decode()) { + const auto& frame = dec.frame(); + if (frame.iBufferStatus != 1) { + continue; + } + + const uint32_t w = static_cast(frame.UsrData.sSystemBuffer.iWidth); + const uint32_t h = static_cast(frame.UsrData.sSystemBuffer.iHeight); + + const uint32_t stride_y = static_cast(frame.UsrData.sSystemBuffer.iStride[0]); + //const uint32_t stride_uv = static_cast(frame.UsrData.sSystemBuffer.iStride[1]); + + const uint8_t* const* srcp = frame.pDst; + + const double wf = static_cast(w) / static_cast(stride_y); + const double verts[8] = { + 0, 0, + 0, 1, + wf, 1, + wf, 0, + }; + if (blky_extractor_feed(&ex, srcp[0], stride_y, h, verts)) { + const size_t psize = ret.size(); + ret.resize(ret.size() + ex.block_num); + std::memcpy(ret.data() + psize, ex.probs, ex.block_num * sizeof(ret[0])); + } + } + blky_extractor_deinit(&ex); + + return ret; +} + +} // namespace blky diff --git a/blocky/main.cc b/blocky/main.cc index 37259d9..afd544d 100644 --- a/blocky/main.cc +++ b/blocky/main.cc @@ -8,6 +8,7 @@ #include "common.hh" #include "bytes.hh" #include "embed.hh" +#include "extract.hh" #include "features.hh" #include @@ -49,7 +50,7 @@ args::ValueFlag> param_block_num { "int>0,int>0", "number of features", {"feature-num"}, - {16, 16} + {8, 8} }; args::ValueFlag param_block_first { param_group, @@ -85,6 +86,13 @@ args::ValueFlag param_utime { {"utime"}, 10 }; +args::ValueFlag> param_sensor_num { + param_group, + "int>0,int>0", + "number of sensors (for extraction)", + {"sensor-num"}, + {16, 16} +}; args::Group probgen_group { parser, "params for feature probability generator", args::Group::Validators::DontCare @@ -217,8 +225,15 @@ try { switch (args::get(from)) { case kVideo: if (args::get(to) == kVideo) break; - // TODO extract feature probs // features = XX - assert(false); + if (!decoder) { + throw std::runtime_error {"decoder is empty"}; + } + feature_probs = Extract( + *decoder, + args::get(param_block_num), + args::get(param_sensor_num), + args::get(param_utime)); + /* fallthrough */ case kFeatureProbs: if (args::get(to) == kFeatureProbs) break; diff --git a/blocky/video_encoder.hh b/blocky/video_encoder.hh index f855738..0645a79 100644 --- a/blocky/video_encoder.hh +++ b/blocky/video_encoder.hh @@ -58,6 +58,9 @@ class VideoEncoder final { } } } + void ForceIntraFrame() noexcept { + encoder_->ForceIntraFrame(true); + } private: std::ofstream file_; diff --git a/liblocky/block.c b/liblocky/block.c index ee6b80b..a2def36 100644 --- a/liblocky/block.c +++ b/liblocky/block.c @@ -18,7 +18,13 @@ double blky_block_estimate( var /= (double) n; // FIXME: calculate probability - if (var > max_var) return 0; - if (avg < min_avg) return 0; - return 1; + // if (var > max_var) return 0; + // if (avg < min_avg) return 0; + // return 1; + // + //if (var > 0.001) return 0; + (void) max_var; + (void) min_avg; + if (var > 0.01) return 0; + return blky_clamp(avg, 0, 1); } diff --git a/liblocky/extractor.c b/liblocky/extractor.c index 531e0e6..e8a0738 100644 --- a/liblocky/extractor.c +++ b/liblocky/extractor.c @@ -14,7 +14,7 @@ void blky_extractor_init(blky_extractor_t* ex) { assert(ex->utime > 0); assert(ex->utime <= BLKY_SENSOR_MAX_DUR); assert(ex->pix_stride > 0); - assert(ex->pix_stride > ex->samples_per_pix); + assert(ex->pix_stride >= ex->samples_per_pix); assert(ex->correl_max_var >= 0); assert(ex->correl_min_avg >= 0); assert(ex->correl_min_avg <= 1);