1 module pry.atoms; 2 3 import pry.stream, pry.traits; 4 import std.conv, std.range.primitives; 5 6 template parsers(Stream) 7 if(is(Stream == string)) 8 { 9 import pry.stream; 10 alias parsers = parsers!(SimpleStream!string); 11 } 12 13 template parsers(Stream) 14 if(!is(Stream == string)) 15 { 16 struct Tk(alias c) 17 if(is(typeof(c) : ElementType!Stream)) { 18 static immutable msg = "expected '" ~ to!string(c)~"'"; 19 20 bool parse(ref Stream stream, ref ElementType!Stream value, ref Stream.Error err) const { 21 if(stream.empty) { 22 err.location = stream.location; 23 err.reason = "unexpected end of stream"; 24 return false; 25 } 26 if(stream.front == c){ 27 value = c; 28 stream.popFront(); 29 return true; 30 } 31 else { 32 err.location = stream.location; 33 err.reason = msg; 34 return false; 35 } 36 } 37 } 38 39 /// Single element token. 40 auto tk(alias c)(){ return Tk!c(); } 41 42 /// Single element token, skipping any whitespace at front. 43 auto stk(alias c)() 44 if(is(typeof(c) : ElementType!Stream) && is(typeof(c) : dchar)){ 45 import pry.combinators; 46 return tk!c.skipWs; 47 } 48 49 struct Range(alias low, alias high) 50 if(is(typeof(low): ElementType!Stream) && is(typeof(high) : ElementType!Stream)){ 51 static immutable msg = "expected in a range of " ~ to!string(low) ~ ".." ~ to!string(high); 52 53 bool parse(ref Stream stream, ref ElementType!Stream value, ref Stream.Error err) const { 54 if(stream.empty) { 55 err.location = stream.location; 56 err.reason = "unexpected end of stream"; 57 return false; 58 } 59 auto v = stream.front; 60 if(v >= low && v <= high) { 61 value = v; 62 stream.popFront(); 63 return true; 64 } 65 else { 66 err.location = stream.location; 67 err.reason = msg; 68 return false; 69 } 70 } 71 } 72 73 // In a range of elements. 74 auto range(alias low, alias high)(){ return Range!(low, high)(); } 75 76 interface DynamicParser(V) { 77 bool parse(ref Stream stream, ref V value, ref Stream.Error err) const; 78 } 79 80 // Use LINE & FILE to provide unique types of dynamic. 81 auto dynamic(V, size_t line=__LINE__, string file=__FILE__)(){ 82 static class Dynamic : DynamicParser!V { 83 DynamicParser!V wrapped; 84 final: 85 void opAssign(P)(P parser) 86 if(isParser!P && !is(P : Dynamic)){ 87 wrapped = wrap(parser); 88 } 89 90 bool parse(ref Stream stream, ref V value, ref Stream.Error err) const { 91 assert(wrapped, "Use of empty dynamic parser"); 92 return wrapped.parse(stream, value, err); 93 } 94 } 95 return new Dynamic(); 96 } 97 98 auto wrap(Parser)(Parser parser) 99 if(isParser!Parser){ 100 alias V = ParserValue!Parser; 101 static class Wrapped: DynamicParser!V { 102 Parser p; 103 104 this(Parser p){ 105 this.p = p; 106 } 107 108 bool parse(ref Stream stream, ref V value, ref Stream.Error err) const { 109 return p.parse(stream, value, err); 110 } 111 } 112 return new Wrapped(parser); 113 } 114 115 struct Set(alias set) { 116 import std.uni, std.conv; 117 enum val = set.byInterval.length; 118 static if(val <= 6) { 119 mixin("static " ~ set.toSourceCode("test")); 120 } 121 else { 122 alias Trie = CodepointSetTrie!(13, 8); 123 alias makeTrie = codepointSetTrie!(13, 8); 124 125 static struct BitTable { 126 uint[4] table; 127 128 this(CodepointSet set){ 129 foreach (iv; set.byInterval) 130 { 131 foreach (v; iv.a .. iv.b) 132 add(v); 133 } 134 } 135 136 void add()(dchar ch){ 137 immutable i = ch & 0x7F; 138 table[i >> 5] |= 1<<(i & 31); 139 } 140 141 bool opIndex()(dchar ch) const{ 142 immutable i = ch & 0x7F; 143 return (table[i >> 5]>>(i & 31)) & 1; 144 } 145 } 146 147 static struct CharMatcher { 148 BitTable ascii; // fast path for ASCII 149 Trie trie; // slow path for Unicode 150 151 this(CodepointSet set){ 152 auto asciiSet = set & unicode.ASCII; 153 ascii = BitTable(asciiSet); 154 trie = makeTrie(set); 155 } 156 157 bool opIndex()(dchar ch) const{ 158 if (ch < 0x80) 159 return ascii[ch]; 160 else 161 return trie[ch]; 162 } 163 } 164 165 static immutable matcher = CharMatcher(set); 166 167 static bool test(dchar ch){ 168 return matcher[ch]; 169 } 170 } 171 172 static immutable string msg = (){ 173 import std.format; 174 string message = "expected one of "; 175 set.toString((const(char)[] s){ message ~= s; }, FormatSpec!char("%x")); 176 return message; 177 }(); 178 179 bool parse(ref Stream stream, ref dchar value, ref Stream.Error err) const { 180 if(stream.empty){ 181 err.location = stream.location; 182 err.reason = "unexpected end of stream"; 183 return false; 184 } 185 immutable c = stream.front; 186 if(test(c)){ 187 value = stream.front; 188 stream.popFront(); 189 return true; 190 } 191 err.location = stream.location; 192 err.reason = msg; 193 return false; 194 } 195 } 196 197 auto set(alias s)(){ 198 import std.uni; 199 static assert(isCodepointSet!(typeof(s)), "set only works with std.uni.CodepointSet"); 200 return Set!s(); 201 } 202 203 struct _Literal(alias literal) { 204 static immutable msg = "expected '"~literal~"' literal"; 205 206 bool parse(ref Stream stream, ref Stream.Range value, ref Stream.Error err) const { 207 auto m = stream.mark(); 208 auto t = literal.save(); 209 for(;;) { 210 if (t.empty) { 211 value = stream.slice(m); 212 return true; 213 } 214 if (stream.empty) { 215 stream.restore(m); 216 err.location = stream.location; 217 err.reason = "unexpected end of stream"; 218 return false; 219 } 220 if (t.front != stream.front) { 221 stream.restore(m); 222 err.location = stream.location; 223 err.reason = msg; 224 return false; 225 } 226 t.popFront(); 227 stream.popFront(); 228 } 229 } 230 } 231 232 auto literal(alias lit)() 233 if(isForwardRange!(typeof(lit)) 234 && is(ElementType!(typeof(lit)) : ElementType!(Stream.Range))) { 235 return _Literal!lit(); 236 } 237 238 struct Eof { 239 bool parse(ref Stream stream, ref Nothing _, ref Stream.Error err) const { 240 bool r = stream.empty; 241 if(!r) { 242 err.location = stream.location; 243 err.reason = "input not fully consumed"; 244 } 245 return r; 246 } 247 } 248 249 /// Zero-width assertion that parses successfully on input end. 250 /// Use to make sure the whole stream was consumed. 251 auto eof(){ 252 return Eof(); 253 } 254 } 255 256 unittest { 257 alias S = SimpleStream!string; 258 with(parsers!S) { 259 auto parser = dynamic!dchar; 260 parser = tk!'a'; 261 S s = S("a"); 262 dchar c; 263 S.Error err; 264 assert(parser.parse(s, c, err)); 265 assert(c == 'a'); 266 assert(s.empty); 267 } 268 } 269 270 unittest { 271 alias S = SimpleStream!string; 272 with(parsers!S) { 273 auto s = " a".stream; 274 auto p = stk!'a'; 275 dchar c; 276 S.Error err; 277 assert(p.parse(s, c, err)); 278 assert(c == 'a'); 279 assert(s.empty); 280 281 auto s2 = "a".stream; 282 assert(p.parse(s2, c, err)); 283 assert(c == 'a'); 284 assert(s2.empty); 285 } 286 } 287 288 unittest { 289 import std.uni; 290 alias S = SimpleStream!string; 291 with(parsers!S) { 292 auto p = set!(CodepointSet('A', 'Z'+1, 'a', 'z'+1)); 293 auto s = "aZ0".stream; 294 dchar c; 295 S.Error err; 296 assert(p.parse(s, c, err)); 297 assert(c == 'a'); 298 assert(p.parse(s, c, err)); 299 assert(c == 'Z'); 300 assert(!p.parse(s, c, err)); 301 assert(s.front == '0'); 302 auto p2 = set!(unicode.L); 303 s = "Яz".stream; 304 assert(p2.parse(s, c, err)); 305 assert(c == 'Я'); 306 assert(p2.parse(s, c, err)); 307 assert(c == 'z'); 308 } 309 } 310 311 unittest { 312 alias S = SimpleStream!string; 313 with(parsers!S) { 314 auto p = literal!"abc"; 315 auto s = "abcd".stream; 316 S.Error err; 317 string slice; 318 assert(p.parse(s, slice, err)); 319 assert(s.front == 'd'); 320 assert(slice == "abc"); 321 s = "ab".stream; 322 assert(!p.parse(s, slice, err)); 323 assert(s.front == 'a'); 324 s = "abd".stream; 325 assert(!p.parse(s, slice, err)); 326 assert(s.front == 'a'); 327 } 328 } 329 330 unittest { 331 alias S = SimpleStream!string; 332 with(parsers!S) { 333 auto p = eof; 334 Nothing n; 335 S.Error err; 336 auto s = "".stream; 337 assert(p.parse(s, n, err)); 338 } 339 }