falsycat/shapes-juke
Archived
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

[update] Improved the tokenizing algorithm.

Browse Source
This commit is contained in:
falsycat 2019-10-04 00:00:00 +00:00
parent e7ebe7ddef
commit 53295526b3
3 changed files with 41 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
sjscript/src/sjscript/Token.d
View File

@ -7,58 +7,54 @@ import std.algorithm,
import dast.tokenize; import dast.tokenize;
///
unittest {
import std;
with (TokenType) {
assert("0 0.1 _hoge0_ $hoge".
Tokenize!TokenType.
map!"a.type".
filter!(x => x != Whitespace).
equal([Number, Number, Ident, PreprocessCommand]));
}
}
/// ///
alias TokenPos = dast.tokenize.TokenPos; alias TokenPos = dast.tokenize.TokenPos;
/// ///
enum TokenType { enum TokenType {
@TextFuncMatcher!((string text, string next) { @TextFuncMatcher!((string text) {
const point_index = text.countUntil('.'); const integral_len = text.countUntil!(x => !x.isDigit);
if (point_index < 0) { if (integral_len < 0) return text.length;
if (text.all!isDigit) { if (integral_len == 0) return 0;
if (next.length == 1 && (next[0].isDigit || next[0] == '.')) {
return TextMatchResult.Probably; if (text[integral_len] != '.') return integral_len;
} if (text[integral_len+1..$] == "") return integral_len;
return TextMatchResult.Completely;
} const fraction_len =
} else { text[integral_len+1..$].countUntil!(x => !x.isDigit);
if ((text[0..point_index]~text[point_index+1..$]).all!isDigit) { if (fraction_len < 0) return text.length;
if (next.length == 1 && next[0].isDigit) { if (fraction_len == 0) return integral_len;
return TextMatchResult.Probably;
} return integral_len + 1 + fraction_len;
return TextMatchResult.Completely;
}
}
return TextMatchResult.Improbably;
}) Number, }) Number,
@TextFuncMatcher!((string text, string next) { @TextFuncMatcher!((string text) {
const head = text[0].isAlpha || text[0] == '_'; if (text.length == 0) return 0;
const body = text[1..$].all!( if (!text[0].isAlpha && text[0] != '_') return 0;
x => x.isAlpha || x.isDigit || x == '_');
const nexthead = next.length > 0 && (
next[0].isAlpha || next[0].isDigit || next[0] == '_');
if (head && body && !nexthead) return TextMatchResult.Completely; const index = text[1..$].countUntil!(x => !x.isAlpha && !x.isDigit && x != '_');
if (head && body && nexthead) return TextMatchResult.Probably; return index >= 0? index.to!size_t+1: text.length;
return TextMatchResult.Improbably;
}) Ident, }) Ident,
@TextFuncMatcher!((string text, string next) { @TextFuncMatcher!((string text) {
const head = text[0] == '$'; if (text.length < 2 || text[0] != '$') return 0;
if (!head || text.length <= 1) { if (!text[1].isAlpha && text[1] != '_') return 0;
return head? TextMatchResult.Probably: TextMatchResult.Improbably;
} const index = text[2..$].countUntil!(x => !x.isAlpha && !x.isDigit && x != '_');
if (text[1] != '_' && !text[1].isAlpha) { return index >= 0? index.to!size_t+2: text.length;
return TextMatchResult.Improbably;
}
if (text[1..$].all!(x => x.isAlpha || x.isDigit || x == '_')) {
if (next.length > 0 && (next[0].isAlpha || next[0].isDigit || next[0] == '_')) {
return TextMatchResult.Probably;
}
return TextMatchResult.Completely;
}
return TextMatchResult.Improbably;
}) PreprocessCommand, }) PreprocessCommand,
@TextCompleteMatcher!"{" OpenBrace, @TextCompleteMatcher!"{" OpenBrace,
@ -87,13 +83,6 @@ enum TokenType {
End, End,
} }
///
unittest {
with (TokenType) {
"0 0.1 _hoge0_ $hoge".Tokenize!TokenType.map!"a.type".equal(
[Number, Whitespace, Number, Whitespace, Ident, Whitespace, PreprocessCommand]);
}
}
/// ///
alias Token = dast.tokenize.Token!(TokenType, string); alias Token = dast.tokenize.Token!(TokenType, string);

3
sjscript/src/sjscript/preprocess.d
View File

@ -4,6 +4,7 @@ module sjscript.preprocess;
import std.algorithm, import std.algorithm,
std.array, std.array,
std.conv, std.conv,
std.format,
std.range, std.range,
std.range.primitives, std.range.primitives,
std.typecons; std.typecons;
@ -212,7 +213,7 @@ private struct Preprocessor(R)
const counter = GetCounterValue(name); const counter = GetCounterValue(name);
if (counter.isNull) { if (counter.isNull) {
(name in templates_). (name in templates_).
enforce("the template is unknown", front); enforce("the template (%s) is unknown".format(name), front);
body = templates_[name]; body = templates_[name];
} else { } else {
body = [Token(counter.get.to!string, TokenType.Number)]; body = [Token(counter.get.to!string, TokenType.Number)];

2
thirdparty/dast vendored

@ -1 +1 @@
Subproject commit 1865e404a6f318d31b3655317141d998de6654eb Subproject commit d99fcb9bf502683ab4e5f6fb19a64a3bda98c83e