Merges thirdparty modules into this repo.

thirdparty/dast/.gitignore

@@ -0,0 +1,4 @@
+/.bin
+/.dub
+
+*.swp

thirdparty/dast/dub.json

@@ -0,0 +1,6 @@
+{
+  "name": "dast",
+
+  "targetType": "library",
+  "targetPath": ".bin"
+}

thirdparty/dast/src/dast/parse/exception.d

@@ -0,0 +1,16 @@
+/// License: MIT
+module dast.parse.exception;
+
+import dast.tokenize;
+
+///
+class ParseException(Token) : Exception if (IsToken!Token) {
+ public:
+  ///
+  this(string msg, Token token, string file = __FILE__, size_t line = __LINE__) {
+    super(msg, file, line);
+    this.token = token;
+  }
+
+  Token token;
+}

thirdparty/dast/src/dast/parse/itemset.d

@@ -0,0 +1,219 @@
+/// License: MIT
+module dast.parse.itemset;
+
+import std.algorithm,
+       std.array,
+       std.conv,
+       std.range,
+       std.range.primitives,
+       std.traits,
+       std.typecons;
+
+import dast.parse.rule,
+       dast.parse.ruleset,
+       dast.util.range;
+
+///
+unittest {
+  import std;
+
+  enum TokenType {
+    Ident,
+    End,
+  }
+  alias Token = dast.tokenize.Token!(TokenType, string);
+
+  struct RuleSet {
+   public:
+   @ParseRule:
+    int ParseWhole(string, @(TokenType.End) Token) {
+      assert(false);
+    }
+    string ParseString(@(TokenType.Ident) Token) {
+      assert(false);
+    }
+  }
+
+  enum rules   = CreateRulesFromRuleSet!(RuleSet, TokenType)();
+  enum itemset = rules.CreateItemSetFromRules(typeid(int));
+}
+
+///
+struct RuleWithCursor(T) if (IsRule!T) {
+ public:
+  alias entity this;
+
+  ///
+  bool opEquals(in RuleWithCursor other) const {
+    return
+      entity == other.entity &&
+      cursor == other.cursor &&
+      follows.equal(other.follows);
+  }
+
+  ///
+  @property RuleWithCursor advanced() const in (canAdvance) {
+    return RuleWithCursor(entity, cursor+1, follows);
+  }
+
+  ///
+  @property Term!(T.TokenType) prev() const in (cursor > 0) {
+    return cast(Term!(T.TokenType)) entity.rhs[cursor-1];
+  }
+  ///
+  @property Term!(T.TokenType) next() const in (canAdvance) {
+    return cast(Term!(T.TokenType)) entity.rhs[cursor];
+  }
+
+  ///
+  @property bool canAdvance() const {
+    return cursor < entity.rhs.length;
+  }
+
+  ///
+  const T entity;
+
+  ///
+  const size_t cursor;
+
+  ///
+  const T.TokenType[] follows;
+}
+
+///
+struct ItemSet(T) if (IsRule!T) {
+ public:
+  ///
+  TypeInfo whole;
+
+  ///
+  const Term!(T.TokenType)[] terms;
+  ///
+  const size_t[size_t][] automaton;
+  ///
+  const RuleWithCursor!T[][] states;
+}
+
+///
+ItemSet!(ElementType!R) CreateItemSetFromRules(R)(R rules, TypeInfo whole)
+    if (isInputRange!R && IsRule!(ElementType!R))
+in {
+  assert(rules.count!(x => x.lhs is whole) == 1);
+}
+do {
+  alias T = ElementType!R;
+
+  Term!(T.TokenType)[] terms;
+  size_t FindOrRegisterTermIndices(typeof(terms[0]) term) {
+    auto index = terms.countUntil!(x => x == term);
+    if (index < 0) {
+      index  = terms.length.to!(typeof(index));
+      terms ~= term;
+    }
+    return index.to!size_t;
+  }
+
+  RuleWithCursor!T[] Resolve(
+      RuleWithCursor!T[] items, TypeInfo[] prev_resolved_types = [null]) {
+    auto resolved_types = prev_resolved_types.appender;
+
+    T[] new_items;
+    foreach (item; items) {
+      const type =
+        !item.canAdvance || item.next.isTerminal? null: item.next.nonTerminalType;
+      if (resolved_types[].canFind!"a is b"(type)) continue;
+
+      rules.
+        filter!(x => x.lhs is type).
+        each  !(x => new_items ~= x);
+      resolved_types ~= type;
+    }
+
+    auto result = items ~ new_items.
+      map!(x => RuleWithCursor!T(x)).
+      array;
+    return new_items.length > 0?
+      Resolve(result, resolved_types[]): result;
+  }
+  RuleWithCursor!T[][size_t] Advance(in RuleWithCursor!T[] items) {
+    RuleWithCursor!T[][size_t] new_items;
+
+    foreach (item; items.filter!"a.canAdvance".map!"a.advanced") {
+      auto  term  = item.prev;
+      const index = FindOrRegisterTermIndices(term);
+      if (index !in new_items) {
+        new_items[index] = [];
+      }
+      new_items[index] ~= item;
+    }
+    foreach (ref state; new_items) {
+      state = Resolve(state).AttachFollowSet();
+    }
+    return new_items;
+  }
+
+  size_t[size_t][]     automaton;
+  RuleWithCursor!T[][] states;
+
+  const first_rule = rules.find!"a.lhs is b"(whole)[0];
+
+  size_t current_state;
+  states = [Resolve([RuleWithCursor!T(first_rule)]).AttachFollowSet()];
+
+  while (true) {
+    automaton.length = current_state+1;
+
+    const advanced_states = Advance(states[current_state]);
+    foreach (termi, state; advanced_states) {
+      auto  index = states.countUntil!(x => x.equal(state));
+      if (index < 0) {
+        index   = states.length.to!(typeof(index));
+        states ~= state.dup;
+      }
+      automaton[current_state][termi] = index.to!size_t;
+    }
+    if (++current_state >= states.length) break;
+  }
+  return ItemSet!T(whole, terms, automaton, states);
+}
+
+private RuleWithCursor!T[] AttachFollowSet(T)(RuleWithCursor!T[] rules) {
+  alias TokenType = T.TokenType;
+
+  TokenType[] CreateFollowSet(TypeInfo type, TypeInfo[] skip_types = []) {
+    if (skip_types.canFind!"a is b"(type)) return [];
+
+    auto result = appender!(TokenType[]);
+    skip_types ~= type;
+
+    foreach (const ref rule; rules) {
+      if (rule.lhs is type) {
+        result ~= rule.follows;
+      }
+      if (!rule.canAdvance || rule.next != type) continue;
+
+      if (rule.cursor+1 < rule.rhs.length) {
+        const follow_term = rule.rhs[rule.cursor+1];
+        if (follow_term.isTerminal) {
+          result ~= follow_term.terminalType;
+        } else {
+          result ~= rules.CreateFirstSet!TokenType(follow_term.nonTerminalType);
+        }
+      } else {
+        result ~= CreateFollowSet(rule.lhs, skip_types);
+      }
+    }
+    return result[];
+  }
+
+  TokenType[][TypeInfo] caches;
+
+  RuleWithCursor!T[] result;
+  foreach (const ref rule; rules) {
+    if (rule.lhs !in caches) {
+      caches[rule.lhs] = CreateFollowSet(rule.lhs).DropDuplicated;
+    }
+    result ~= RuleWithCursor!T(rule.entity, rule.cursor, caches[rule.lhs]);
+  }
+  return result;
+}

thirdparty/dast/src/dast/parse/package.d

@@ -0,0 +1,201 @@
+/// License: MIT
+module dast.parse;
+
+import std.algorithm,
+       std.conv,
+       std.range.primitives,
+       std.traits,
+       std.variant;
+
+import dast.tokenize;
+
+import dast.parse.itemset,
+       dast.parse.rule,
+       dast.parse.ruleset,
+       dast.parse.table;
+
+public {
+  import dast.parse.exception,
+         dast.parse.ruleset;
+}
+
+///
+unittest {
+  import std;
+
+  enum TokenType {
+    Unknown,
+
+    @TextCompleteMatcher!"," Comma,
+
+    @TextAllMatcher!isDigit Number,
+    @TextAllMatcher!isWhite Whitespace,
+
+    End,
+  }
+  alias Token = dast.tokenize.Token!(TokenType, string);
+
+  struct Whole {
+    int[] numbers;
+  }
+  class RuleSet {
+   public:
+   @ParseRule:
+    Whole ParseWhole(int[] nums, @(TokenType.End) Token) {
+      return Whole(nums);
+    }
+    int[] ParseFirstNumber(@(TokenType.Number) Token num) {
+      return [num.text.to!int];
+    }
+    int[] ParseNextNumber(int[] nums, @(TokenType.Comma) Token, @(TokenType.Number) Token num) {
+      return nums ~ num.text.to!int;
+    }
+  }
+
+  auto ruleset = new RuleSet;
+
+  int[] ParseNumbers(string src) {
+    return src.
+      Tokenize!TokenType.
+      filter!(x => x.type != TokenType.Whitespace).
+      chain([Token("", TokenType.End)]).
+      Parse!Whole(ruleset).numbers;
+  }
+  assert(ParseNumbers("0, 1, 2, 3").equal([0, 1, 2, 3]));
+
+  assertThrown!(ParseException!Token)(ParseNumbers("0 1 2 3"));
+}
+
+///
+Whole Parse(Whole, Tokenizer, RuleSet)(Tokenizer tokenizer, RuleSet ruleset)
+    if (isInputRange!Tokenizer && IsToken!(ElementType!Tokenizer)) {
+  alias Token     = ElementType!Tokenizer;
+  alias Exception = ParseException!Token;
+
+  enum rules = CreateRulesFromRuleSet!(RuleSet, Token.TokenType);
+  static assert(rules.length > 0);
+
+  enum itemset  = rules.CreateItemSetFromRules(typeid(Whole));
+  enum table    = itemset.CreateTableFromItemSet();
+  enum statelen = table.shift.length;
+
+  enum stackitem_size = itemset.terms.
+    filter!"!a.isTerminal".
+    map   !"a.nonTerminalType.tsize".
+    maxElement.
+    max(Token.sizeof);
+
+  alias StackItem = VariantN!(stackitem_size);
+
+  StackItem[] stack;
+  size_t[]    status = [0];
+
+  size_t Reduce(Token, string name)() {
+    alias Func   = mixin("RuleSet."~name);
+    alias Params = Parameters!Func;
+
+    Params params;
+    static foreach (parami; 0..Params.length) {
+      static if (is(Params[$-parami-1] == Token)) {
+        params[$-parami-1] = stack[$-parami-1].get!Token;
+      } else {
+        params[$-parami-1] = stack[$-parami-1].get!(Params[$-parami-1]);
+      }
+    }
+    stack  = stack[0..$-Params.length];
+    stack ~= StackItem(mixin("ruleset."~name~"(params)"));
+    return Params.length;
+  }
+
+  void GoStateAfterReduce() {
+    auto  stack_top_type = stack[$-1].type;
+    const current_status = status[$-1];
+
+    TypeInfo temp;
+    static foreach (statei; 0..statelen) {
+      if (statei == current_status) {
+        static foreach (type, number; table.go[statei]) {
+          if (type is stack_top_type) {
+            status ~= table.go[statei][stack_top_type];
+            return;
+          }
+        }
+      }
+    }
+    assert(false);  // broken go table
+  }
+
+  Token prev_token;
+  while (status[$-1] != table.accept_state) {
+    if (tokenizer.empty) {
+      throw new Exception("all tokens are consumed without acception", prev_token);
+    }
+    const token = tokenizer.front;
+    scope(exit) prev_token = token;
+
+MAIN_SWITCH: switch (status[$-1]) {
+      static foreach (statei; 0..statelen) {
+        case statei:
+          static foreach (token_type, reduce; table.reduce[statei]) {
+            if (token_type == token.type) {
+              const pop = Reduce!(Token, reduce);
+              status    = status[0..$-pop];
+              GoStateAfterReduce();
+              break MAIN_SWITCH;
+            }
+          }
+          static foreach (token_type, shift; table.shift[statei]) {
+            if (token_type == token.type) {
+              stack  ~= StackItem(cast(Unqual!Token) token);
+              status ~= shift;
+              tokenizer.popFront();
+              break MAIN_SWITCH;
+            }
+          }
+          throw new Exception("unexpected token", token);
+      }
+      default: assert(false);
+    }
+  }
+
+  Reduce!(Token, itemset.states[0][0].name);
+  return stack[0].get!Whole;
+}
+
+debug void PrintItemSet(TokenType, RuleSet, Whole)()
+    if (is(TokenType == enum)) {
+  import std;
+
+  enum itemset =
+    CreateRulesFromRuleSet!(RuleSet, TokenType).
+    CreateItemSetFromRules(typeid(Whole));
+
+  void PrintTerm(T2)(T2 term) {
+    if (term.isTerminal) {
+      term.terminalType.write;
+    } else {
+      term.nonTerminalType.write;
+    }
+  }
+
+  "[states]".writeln;
+  foreach (statei, state; itemset.states) {
+    "  (%d) ".writef(statei);
+    foreach (termi, go; itemset.automaton[statei]) {
+      PrintTerm(itemset.terms[termi]);
+      ":%d, ".writef(go);
+    }
+    writeln;
+
+    foreach (rule; state) {
+      "    %s => ".writef(rule.lhs);
+      foreach (termi, term; rule.rhs) {
+        if (termi == rule.cursor) "/ ".write;
+        PrintTerm(term);
+        " ".write;
+      }
+      if (rule.rhs.length == rule.cursor) "/ ".write;
+      rule.follows.writeln;
+    }
+  }
+}

thirdparty/dast/src/dast/parse/rule.d

@@ -0,0 +1,172 @@
+/// License: MIT
+module dast.parse.rule;
+
+import std.algorithm,
+       std.array,
+       std.traits;
+
+import dast.tokenize;
+
+///
+unittest {
+  enum TokenType {
+    Number,
+  }
+  alias Token = dast.tokenize.Token!(TokenType, string);
+
+  struct RuleSet {
+   public:
+    int ParseWhole(@(TokenType.Number) Token, string v) {
+      assert(false);
+    }
+  }
+  enum rule_ctfe = Rule!TokenType.CreateFromNonOverloadedMethod!(RuleSet, "ParseWhole");
+
+  static assert(rule_ctfe.lhs is typeid(int));
+
+  static assert(rule_ctfe.rhs.length == 2);
+  static assert(rule_ctfe.rhs[0] == TokenType.Number);
+  static assert(rule_ctfe.rhs[1] == typeid(string));
+}
+
+///
+template IsRule(T) {
+  static if (__traits(compiles, T.TokenType)) {
+    enum IsRule = is(T : Rule!(T.TokenType));
+  } else {
+    enum IsRule = false;
+  }
+}
+
+///
+struct NamedRule(TokenType) if (is(TokenType == enum)) {
+ public:
+  alias entity this;
+
+  ///
+  const string name;
+  ///
+  const Rule!TokenType entity;
+}
+
+///
+struct Rule(TokenType_) if (is(TokenType_ == enum)) {
+ public:
+  ///
+  alias TokenType = TokenType_;
+
+  ///
+  static Rule CreateFromNonOverloadedMethod(T, string name)()
+      if (__traits(hasMember, T, name)) {
+    alias method = mixin("T." ~ name);
+    alias Params = Parameters!method;
+
+    auto rhs = appender!(Term!TokenType[])();
+    static foreach (parami; 0..Params.length) {
+      static if (IsToken!(Params[parami])) {{
+        static foreach (attr; __traits(getAttributes, Params[parami..parami+1])) {
+          static if (is(typeof(attr) == TokenType)) {
+            static assert(!__traits(compiles, found_attr));
+            enum found_attr = true;
+            rhs ~= Term!TokenType(attr);
+          }
+        }
+        static assert(__traits(compiles, found_attr));
+      }} else {
+        rhs ~= Term!TokenType(TypeInfoWithSize.CreateFromType!(Params[parami]));
+      }
+    }
+    return Rule(typeid(ReturnType!method), rhs.array);
+  }
+
+  ///
+  bool opEquals(Rule other) const {
+    return lhs is other.lhs && rhs_.equal(other.rhs_);
+  }
+
+  ///
+  @property TypeInfo lhs() const {
+    return cast(TypeInfo) lhs_;
+  }
+  ///
+  @property const(Term!TokenType[]) rhs() const {
+    return rhs_;
+  }
+
+ private:
+  const TypeInfo lhs_;
+  invariant(lhs_ !is null);
+
+  const Term!TokenType[] rhs_;
+  invariant(rhs_.length > 0);
+}
+
+///
+struct Term(TokenType) if (is(TokenType == enum)) {
+ public:
+  @disable this();
+
+  ///
+  this(TypeInfoWithSize type) {
+    data_.type_info = type;
+    terminal_       = false;
+  }
+  ///
+  this(TokenType type) {
+    data_.token_type = type;
+    terminal_        = true;
+  }
+
+  ///
+  bool opEquals(in Term rhs) const {
+    return isTerminal?
+      rhs.isTerminal  && terminalType    == rhs.terminalType:
+      !rhs.isTerminal && nonTerminalType is rhs.nonTerminalType;
+  }
+  ///
+  bool opEquals(in TypeInfo type) const {
+    return !isTerminal && nonTerminalType is type;
+  }
+  ///
+  bool opEquals(TokenType type) const {
+    return isTerminal && terminalType == type;
+  }
+
+  ///
+  @property bool isTerminal() const {
+    return terminal_;
+  }
+
+  ///
+  @property TokenType terminalType() const in (isTerminal) {
+    return data_.token_type;
+  }
+  ///
+  @property TypeInfoWithSize nonTerminalType() const in (!isTerminal) {
+    return cast(TypeInfoWithSize) data_.type_info;
+  }
+
+ private:
+  union Data {
+    TypeInfoWithSize type_info;
+    TokenType        token_type;
+  }
+  Data data_;
+  bool terminal_;
+}
+
+///
+struct TypeInfoWithSize {
+ public:
+  alias entity this;
+
+  ///
+  static TypeInfoWithSize CreateFromType(T)() {
+    return TypeInfoWithSize(T.sizeof, typeid(T));
+  }
+
+  ///
+  const size_t tsize;
+  ///
+  TypeInfo entity;
+}

thirdparty/dast/src/dast/parse/ruleset.d

@@ -0,0 +1,96 @@
+/// License: MIT
+module dast.parse.ruleset;
+
+import std.algorithm,
+       std.array,
+       std.meta,
+       std.range.primitives,
+       std.traits;
+
+import dast.tokenize;
+
+import dast.parse.rule;
+
+///
+unittest {
+  import std;
+  import dast.util.range;
+
+  enum TokenType {
+    Number,
+    Ident,
+    End,
+  }
+  alias Token = dast.tokenize.Token!(TokenType, string);
+
+  struct RuleSet {
+   public:
+   @ParseRule:
+    int ParseWhole(string, @(TokenType.End) Token) {
+      assert(false);
+    }
+    string ParseStr(@(TokenType.Ident) Token) {
+      assert(false);
+    }
+    string ParseStrFromFloat(float) {
+      assert(false);
+    }
+    float ParseFloat(@(TokenType.Number) Token) {
+      assert(false);
+    }
+    float ParseFloatFromStr(@(TokenType.Ident) Token, string, @(TokenType.Ident) Token) {
+      assert(false);
+    }
+  }
+
+  enum rules = CreateRulesFromRuleSet!(RuleSet, TokenType)();
+  static assert(rules.length == 5);
+
+  enum firsts = rules.CreateFirstSet!TokenType(typeid(int)).DropDuplicated;
+  static assert(firsts.equal([TokenType.Ident, TokenType.Number]));
+}
+
+/// A struct which is attributed to rule methods.
+struct ParseRule {}
+
+/// Creates rules from RuleSet's methods attributed by ParseRule.
+NamedRule!TokenType[] CreateRulesFromRuleSet(RuleSet, TokenType)() {
+  alias members = __traits(allMembers, RuleSet);
+
+  NamedRule!TokenType[] result;  // Appender causes CTFE errors :(
+
+  static foreach (name; members) {{
+    enum method = "RuleSet." ~ name;
+    static if (__traits(compiles, mixin(method))) {
+      alias attrs = __traits(getAttributes, mixin(method));
+      static if (staticIndexOf!(ParseRule, attrs) >= 0) {
+        static assert(__traits(getOverloads, RuleSet, name).length == 1);
+        auto rule = Rule!TokenType.
+          CreateFromNonOverloadedMethod!(RuleSet, name);
+        result ~= NamedRule!TokenType(name, rule);
+      }
+    }
+  }}
+  return result;
+}
+
+/// Creates a first set of the target.
+///
+/// Items in the result can be duplicated.
+TokenType[] CreateFirstSet(TokenType, R)(R rules, TypeInfo target, TypeInfo[] skip_types = [])
+    if (isInputRange!R && is(Unqual!(ElementType!R) : Rule!TokenType)) {
+  auto result = appender!(TokenType[]);
+  skip_types ~= target;
+
+  foreach (const ref rule; rules.filter!(x => x.lhs is target)) {
+    const first = rule.rhs[0];
+    if (first.isTerminal) {
+      result ~= first.terminalType;
+    } else if (!skip_types.canFind!"a is b"(first.nonTerminalType)) {
+      result ~= rules.
+        CreateFirstSet!(TokenType)(first.nonTerminalType, skip_types);
+      skip_types ~= first.nonTerminalType;
+    }
+  }
+  return result.array;
+}

thirdparty/dast/src/dast/parse/table.d

@@ -0,0 +1,80 @@
+/// License: MIT
+module dast.parse.table;
+
+import std.algorithm;
+
+import dast.parse.itemset,
+       dast.parse.rule,
+       dast.parse.ruleset;
+
+///
+unittest {
+  import std;
+
+  enum TokenType {
+    Ident,
+    End,
+  }
+  alias Token = dast.tokenize.Token!(TokenType, string);
+
+  struct RuleSet {
+   public:
+   @ParseRule:
+    int ParseWhole(string, @(TokenType.End) Token) {
+      assert(false);
+    }
+    string ParseString(@(TokenType.Ident) Token) {
+      assert(false);
+    }
+  }
+
+  enum rules   = CreateRulesFromRuleSet!(RuleSet, TokenType)();
+  enum itemset = rules.CreateItemSetFromRules(typeid(int));
+  enum table   = itemset.CreateTableFromItemSet();
+}
+
+///
+struct Table(TokenType) if (is(TokenType == enum)) {
+ public:
+  ///
+  const size_t[TokenType][] shift;
+  ///
+  const string[TokenType][] reduce;
+  ///
+  const size_t[TypeInfo][] go;
+
+  ///
+  const size_t accept_state;
+}
+
+///
+Table!(T.TokenType) CreateTableFromItemSet(T)(in ItemSet!T itemset) {
+  size_t[T.TokenType][] shift;
+  string[T.TokenType][] reduce;
+  size_t[TypeInfo][] go;
+
+  size_t accept_state;
+
+  shift.length  = itemset.automaton.length;
+  reduce.length = itemset.automaton.length;
+  go.length     = itemset.automaton.length;
+
+  foreach (state, numbers; itemset.automaton) {
+    foreach (k, v; numbers) {
+      const term = itemset.terms[k];
+      if (term.isTerminal) {
+        shift[state][term.terminalType] = v;
+      } else {
+        go[state][term.nonTerminalType] = v;
+      }
+    }
+  }
+
+  foreach (state, rules; itemset.states) {
+    foreach (rule; rules.filter!"!a.canAdvance") {
+      if (rule.lhs is itemset.whole) accept_state = state;
+      rule.follows.each!(x => reduce[state][x] = rule.name);
+    }
+  }
+  return Table!(T.TokenType)(shift, reduce, go, accept_state);
+}

thirdparty/dast/src/dast/tokenize/data.d

@@ -0,0 +1,44 @@
+/// License: MIT
+module dast.tokenize.data;
+
+import std.traits;
+
+///
+template IsToken(T) {
+  private enum hasRequiredMembers =
+    __traits(hasMember, T, "text") &&
+    __traits(hasMember, T, "type") &&
+    __traits(hasMember, T, "pos");
+
+  static if (hasRequiredMembers) {
+    enum IsToken =
+      isSomeString!(typeof(T.text)) &&
+      is(typeof(T.type) == enum) &&
+      is(typeof(T.pos)  == TokenPos);
+  } else {
+    enum IsToken = false;
+  }
+}
+
+///
+enum IsToken(T, S) = IsToken!T && is(typeof(T.text) == S);
+
+///
+struct Token(TokenType_, S)
+    if (is(TokenType_ == enum) && isSomeString!S) {
+  alias TokenType = TokenType_;
+
+  S         text;
+  TokenType type;
+
+  TokenPos pos;
+}
+
+///
+struct TokenPos {
+  size_t stline;
+  size_t edline;
+
+  size_t stchar;
+  size_t edchar;
+}

thirdparty/dast/src/dast/tokenize/exception.d

@@ -0,0 +1,31 @@
+/// License: MIT
+module dast.tokenize.exception;
+
+import std.format;
+
+///
+class TokenizeException : Exception {
+ public:
+  ///
+  this(string msg,
+       size_t srcline,
+       size_t srchar,
+       string src  = "",
+       string file = __FILE__,
+       size_t line = __LINE__) {
+    if (src == "") {
+      msg ~= " at (%d, %d)".format(srcline, srcchar);
+    } else {
+      msg ~= " at token '%s' (%d, %d)".format(src, srcline, srcchar);
+    }
+    super(msg, file, line);
+
+    this.srcline = srcline;
+    this.srcchar = srcchar;
+  }
+
+  ///
+  const size_t srcline;
+  ///
+  const size_t srcchar;
+}

thirdparty/dast/src/dast/tokenize/match.d

@@ -0,0 +1,142 @@
+/// License: MIT
+module dast.tokenize.match;
+
+import std.algorithm,
+       std.ascii,
+       std.conv,
+       std.traits;
+
+///
+unittest {
+  import std;
+
+  enum TokenType {
+    @TextAllMatcher!isDigit  Number,
+    @TextCompleteMatcher!"12" OneTwo,
+
+    @TextFuncMatcher!((string text) {
+      return text.length >= 3? 3LU: 0LU;
+    }) ThreeLetters,
+
+    @TextAllMatcher!isUpper
+    @TextAllMatcher!isLower Ident,
+  }
+
+  with (TokenType) {
+    assert(MatchTextAsTokenType!Number("0123")  == 4);
+    assert(MatchTextAsTokenType!Number("01a23") == 2);
+    assert(MatchTextAsTokenType!Number("a0123") == 0);
+
+    assert(MatchTextAsTokenType!OneTwo("12")  == 2);
+    assert(MatchTextAsTokenType!OneTwo("12a") == 2);
+    assert(MatchTextAsTokenType!OneTwo("1")   == 0);
+
+    assert(MatchTextAsTokenType!ThreeLetters("abc")  == 3);
+    assert(MatchTextAsTokenType!ThreeLetters("abcd") == 3);
+    assert(MatchTextAsTokenType!ThreeLetters("ab")   == 0);
+
+    assert(MatchTextAsTokenType!Ident("abC") == 2);
+    assert(MatchTextAsTokenType!Ident("AB0") == 2);
+    assert(MatchTextAsTokenType!Ident("0Ab") == 0);
+
+    {
+      const result = FindMatchedTokenTypes!TokenType("012");
+      assert(result.types.equal([Number, ThreeLetters]));
+      assert(result.length == 3);
+    }
+    {
+      const result = FindMatchedTokenTypes!TokenType("12");
+      assert(result.types.equal([Number, OneTwo]));
+      assert(result.length == 2);
+    }
+  }
+}
+
+/// Checks if the matcher is a text matcher for S.
+enum IsTextMatcher(alias matcher, S) =
+  __traits(compiles, (S str) => matcher.Match(str)) &&
+  is(ReturnType!((S str) => matcher.Match(str)) == size_t);
+/// ditto
+enum IsTextMatcher(alias matcher) =
+  IsTextMatcher!(matcher,  string) ||
+  IsTextMatcher!(matcher, wstring) ||
+  IsTextMatcher!(matcher, dstring);
+
+///
+struct TextAllMatcher(alias func) {
+ public:
+  ///
+  static size_t Match(S)(S text) if (isSomeString!S) {
+    const index = text.countUntil!(x => !func(x));
+    return index >= 0? index.to!size_t: text.length;
+  }
+  static assert(IsTextMatcher!TextAllMatcher);
+}
+
+///
+struct TextCompleteMatcher(alias str) if (isSomeString!(typeof(str))) {
+ public:
+  ///
+  static size_t Match(typeof(str) text) {
+    if (text.length < str.length || text[0..str.length] != str) {
+      return 0;
+    }
+    return str.length;
+  }
+  static assert(IsTextMatcher!TextCompleteMatcher);
+}
+
+///
+struct TextFuncMatcher(alias func)
+  if ((is(typeof((string  x) => func(x))) ||
+       is(typeof((wstring x) => func(x))) ||
+       is(typeof((dstring x) => func(x)))) &&
+      is(ReturnType!func == size_t)) {
+ public:
+  ///
+  static size_t Match(S)(S text)
+      if (isSomeString!S && __traits(compiles, func(text))) {
+    return func(text);
+  }
+  static assert(IsTextMatcher!(TextFuncMatcher, string));
+}
+
+/// Finds matched token types from TokenType enum.
+auto FindMatchedTokenTypes(TokenType, S)(S src)
+    if (is(TokenType == enum) && isSomeString!S) {
+  struct Result {
+    TokenType[] types;
+    size_t      length;
+  }
+  Result result;
+
+  size_t length = void;
+  static foreach (type; EnumMembers!TokenType) {
+    length = MatchTextAsTokenType!type(src);
+    if (length == 0) {
+
+    } else if (length > result.length) {
+      result.types  = [type];
+      result.length = length;
+
+    } else if (length == result.length) {
+      result.types ~= type;
+    }
+  }
+  return result;
+}
+
+/// Checks if the src can be a token with the type.
+size_t MatchTextAsTokenType(alias type, S)(S src)
+    if (is(typeof(type) == enum) && isSomeString!S) {
+  alias TokenType = typeof(type);
+  enum  typestr   = "TokenType." ~ type.to!string;
+
+  size_t result;
+  static foreach (attr; __traits(getAttributes, mixin(typestr))) {
+    static if (IsTextMatcher!(attr, S)) {
+      result = result.max(attr.Match(src));
+    }
+  }
+  return result;
+}

thirdparty/dast/src/dast/tokenize/package.d

@@ -0,0 +1,122 @@
+/// License: MIT
+module dast.tokenize;
+
+import std.format,
+       std.range,
+       std.traits,
+       std.typecons;
+
+public {
+  import dast.tokenize.data,
+         dast.tokenize.exception,
+         dast.tokenize.match;
+}
+
+///
+unittest {
+  import std;
+
+  enum TokenType {
+    @TextAllMatcher!isDigit Number,
+    @TextAllMatcher!isAlpha Ident,
+    @TextAllMatcher!isWhite Whitespace,
+
+    @TextCompleteMatcher!"0"  Zero,
+    @TextCompleteMatcher!"0a" ZeroAndA,
+  }
+
+  with (TokenType) {
+    auto tokens = "1 abc 2".Tokenize!(TokenType, string)();
+    assert(tokens.
+        map!"a.type".
+        equal([Number, Whitespace, Ident, Whitespace, Number]));
+    assert(tokens.
+        filter!(x => x.type != Whitespace).
+        map!"a.text".
+        equal(["1", "abc", "2"]));
+  }
+  with (TokenType) {
+    auto tokens = "0a 1 2".Tokenize!(TokenType, string)();
+    assert(tokens.
+        map!"a.type".
+        equal([ZeroAndA, Whitespace, Number, Whitespace, Number]));
+    assert(tokens.
+        filter!(x => x.type != Whitespace).
+        map!"a.text".
+        equal(["0a", "1", "2"]));
+  }
+  assertThrown!TokenizeException("0".Tokenize!(TokenType, string));
+}
+
+/// Tokenizes the src into the TokenType.
+/// Returns: an input range of Token!(TokenType, S) as a voldemorte type
+auto Tokenize(TokenType, S)(S src)
+    if (is(TokenType == enum) && isSomeString!S) {
+  return Tokenizer!(TokenType, S)(src).drop(1);
+}
+
+private struct Tokenizer(TokenType_, S)
+  if (is(TokenType_ == enum) && isSomeString!S) {
+ public:
+  alias TokenType = TokenType_;
+  alias Token     = dast.tokenize.data.Token!(TokenType, S);
+
+  void PopFront() in (!empty) {
+    if (cursor_ < src.length) {
+      TokenizeNext();
+    } else {
+      end_ = true;
+    }
+  }
+  alias popFront = PopFront;
+
+  @property Token front() const in (!empty) {
+    return last_tokenized_;
+  }
+  @property bool empty() const {
+    return end_;
+  }
+
+  const S src;
+
+ private:
+  void TokenizeNext() in (!empty) {
+    last_tokenized_.pos.stline = line_;
+    last_tokenized_.pos.stchar = char_;
+
+    const match = FindMatchedTokenTypes!TokenType(src[cursor_..$]);
+    if (match.types.length == 0 || match.length == 0) {
+      throw new TokenizeException(
+          "found the uncategorizable token",
+          line_, char_, src[cursor_..cursor_+1]);
+    }
+    if (match.types.length > 1) {
+      throw new TokenizeException(
+          "found the token which can be categorizable to multiple types %s".
+            format(match.types),
+          line_, char_, src[cursor_..cursor_+1]);
+    }
+
+    last_tokenized_.text = src[cursor_..cursor_+match.length];
+    last_tokenized_.type = match.types[0];
+
+    foreach (c; last_tokenized_.text) {
+      ++char_;
+      if (c == '\n') {
+        ++line_;
+        char_ = 0;
+      }
+    }
+    cursor_ += match.length;
+
+    last_tokenized_.pos.edline = line_;
+    last_tokenized_.pos.edchar = char_;
+  }
+
+  Token last_tokenized_;
+
+  bool end_;
+
+  size_t cursor_;
+  size_t line_, char_;
+}

thirdparty/dast/src/dast/util/range.d

@@ -0,0 +1,19 @@
+/// License: MIT
+module dast.util.range;
+
+import std.algorithm,
+       std.array,
+       std.range.primitives;
+
+/// Returns: an input range which has unique items
+auto DropDuplicated(R)(R src) if (isInputRange!R) {
+  auto dest = appender!(ElementType!R[]);
+  dest.reserve(src.length);
+
+  foreach (item; src) if (!dest[].canFind(item)) dest ~= item;
+  return dest[];
+}
+///
+unittest {
+  static assert([0, 1, 0, 1].DropDuplicated.equal([0, 1]));
+}

thirdparty/dast/test/all.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+if [ ! -d ".git" ]; then
+  echo "plz run at root of this repository"
+  exit 1
+fi
+
+function test() {
+  result=`$1 "$2"`
+  if [ "$result" != "$3" ]; then
+    echo "\`$1 \"$2\"\` != \`$3\`"
+  fi
+}
+
+test ./test/math.d "1+2-3*4/5" "1"

thirdparty/dast/test/math.d

@@ -0,0 +1,91 @@
+#!/usr/bin/env dub
+/+ dub.json:
+{
+  "name": "math",
+  "dependencies": {
+    "dast": {"path": "../"}
+  }
+} +/
+
+import std;
+import dast.parse,
+       dast.tokenize;
+
+enum TokenType {
+  @TextAllMatcher!isDigit Number,
+
+  @TextCompleteMatcher!"+" Add,
+  @TextCompleteMatcher!"-" Sub,
+  @TextCompleteMatcher!"*" Mul,
+  @TextCompleteMatcher!"/" Div,
+
+  @TextCompleteMatcher!"(" OpenParen,
+  @TextCompleteMatcher!")" CloseParen,
+
+  End,
+}
+alias Token = dast.tokenize.Token!(TokenType, string);
+
+struct Whole {
+  int result;
+}
+struct TermList {
+  int value;
+}
+struct Term {
+  int value;
+}
+
+class RuleSet {
+ public:
+ @ParseRule:
+  static Whole ParseWhole(TermList terms, @(TokenType.End) Token) {
+    return Whole(terms.value);
+  }
+
+  static TermList ParseTermListFromAddedNextTerm(
+      TermList lterms, @(TokenType.Add) Token, Term term) {
+    return TermList(lterms.value + term.value);
+  }
+  static TermList ParseTermListFromSubtractedNextTerm(
+      TermList lterms, @(TokenType.Sub) Token, Term term) {
+    return TermList(lterms.value - term.value);
+  }
+  static TermList ParseTermListFirstItem(Term term) {
+    return TermList(term.value);
+  }
+
+  static Term ParseTermFromFirstNumber(@(TokenType.Number) Token num) {
+    return Term(num.text.to!int);
+  }
+
+  static Term ParseTermFromTermList(
+      @(TokenType.OpenParen) Token, TermList terms, @(TokenType.CloseParen) Token) {
+    return Term(terms.value);
+  }
+  static Term ParseMultipledTerm(
+      Term lterm, @(TokenType.Mul) Token, @(TokenType.Number) Token num) {
+    return Term(lterm.value * num.text.to!int);
+  }
+  static Term ParseDividedTerm(
+      Term lterm, @(TokenType.Div) Token, @(TokenType.Number) Token num) {
+    return Term(lterm.value / num.text.to!int);
+  }
+}
+
+void main(string[] args) {
+  assert(args.length == 2);
+
+  // PrintItemSet!(TokenType, RuleSet, Whole);
+
+  try {
+    args[1].
+      Tokenize!TokenType.
+      chain([Token("", TokenType.End)]).
+      Parse!Whole(cast(RuleSet) null).
+      result.writeln;
+  } catch (ParseException!Token e) {
+    "%s at token '%s' [%s] at (%d, %d)".
+      writefln(e.msg, e.token.text, e.token.type, e.token.pos.stline, e.token.pos.stchar);
+  }
+}