1 module pry.atoms; 2 3 import pry.stream, pry.traits; 4 import std.conv, std.range.primitives, std.uni, std.conv; 5 6 template parsers(Stream) 7 if(is(Stream == string)) 8 { 9 import pry.stream; 10 alias parsers = parsers!(SimpleStream!string); 11 } 12 13 template parsers(Stream) 14 if(!is(Stream == string)) 15 { 16 struct Tk(alias c) 17 if(is(typeof(c) : ElementType!Stream)) { 18 static immutable msg = "expected '" ~ to!string(c)~"'"; 19 20 bool parse(ref Stream stream, ref ElementType!Stream value, ref Stream.Error err) const { 21 if(stream.empty) { 22 err.location = stream.location; 23 err.reason = "unexpected end of stream"; 24 return false; 25 } 26 if(stream.front == c){ 27 value = c; 28 stream.popFront(); 29 return true; 30 } 31 else { 32 err.location = stream.location; 33 err.reason = msg; 34 return false; 35 } 36 } 37 } 38 39 /// Single element token. 40 auto tk(alias c)(){ return Tk!c(); } 41 42 /// Single element token, skipping any whitespace at front. 43 auto stk(alias c)() 44 if(is(typeof(c) : ElementType!Stream) && is(typeof(c) : dchar)){ 45 import pry.combinators; 46 return tk!c.skipWs; 47 } 48 49 struct Range(alias low, alias high) 50 if(is(typeof(low): ElementType!Stream) && is(typeof(high) : ElementType!Stream)){ 51 static immutable msg = "expected in a range of " ~ to!string(low) ~ ".." ~ to!string(high); 52 53 bool parse(ref Stream stream, ref ElementType!Stream value, ref Stream.Error err) const { 54 if(stream.empty) { 55 err.location = stream.location; 56 err.reason = "unexpected end of stream"; 57 return false; 58 } 59 auto v = stream.front; 60 if(v >= low && v <= high) { 61 value = v; 62 stream.popFront(); 63 return true; 64 } 65 else { 66 err.location = stream.location; 67 err.reason = msg; 68 return false; 69 } 70 } 71 } 72 73 // In a range of elements. 74 auto range(alias low, alias high)(){ return Range!(low, high)(); } 75 76 interface DynamicParser(V) { 77 bool parse(ref Stream stream, ref V value, ref Stream.Error err) const; 78 } 79 80 // Use LINE & FILE to provide unique types of dynamic. 81 auto dynamic(V, size_t line=__LINE__, string file=__FILE__)(){ 82 static class Dynamic : DynamicParser!V { 83 DynamicParser!V wrapped; 84 final: 85 void opAssign(P)(P parser) 86 if(isParser!P && !is(P : Dynamic)){ 87 wrapped = wrap(parser); 88 } 89 90 bool parse(ref Stream stream, ref V value, ref Stream.Error err) const { 91 assert(wrapped, "Use of empty dynamic parser"); 92 return wrapped.parse(stream, value, err); 93 } 94 } 95 return new Dynamic(); 96 } 97 98 auto wrap(Parser)(Parser parser) 99 if(isParser!Parser){ 100 alias V = ParserValue!Parser; 101 static class Wrapped: DynamicParser!V { 102 Parser p; 103 104 this(Parser p){ 105 this.p = p; 106 } 107 108 bool parse(ref Stream stream, ref V value, ref Stream.Error err) const { 109 return p.parse(stream, value, err); 110 } 111 } 112 return new Wrapped(parser); 113 } 114 115 struct Set { 116 CodepointSet set; 117 size_t val; 118 CharMatcher matcher; 119 120 alias Trie = CodepointSetTrie!(13, 8); 121 alias makeTrie = codepointSetTrie!(13, 8); 122 123 this(CodepointSet set) { 124 this.set = set; 125 val = set.byInterval.length; 126 matcher = CharMatcher(set); 127 } 128 129 static struct BitTable { 130 uint[4] table; 131 132 this(CodepointSet set){ 133 foreach (iv; set.byInterval) 134 { 135 foreach (v; iv.a .. iv.b) 136 add(v); 137 } 138 } 139 140 void add()(dchar ch){ 141 immutable i = ch & 0x7F; 142 table[i >> 5] |= 1<<(i & 31); 143 } 144 145 bool opIndex()(dchar ch) const{ 146 immutable i = ch & 0x7F; 147 return (table[i >> 5]>>(i & 31)) & 1; 148 } 149 } 150 151 static struct CharMatcher { 152 BitTable ascii; // fast path for ASCII 153 Trie trie; // slow path for Unicode 154 155 this(CodepointSet set){ 156 auto asciiSet = set & unicode.ASCII; 157 ascii = BitTable(asciiSet); 158 trie = makeTrie(set); 159 } 160 161 bool opIndex()(dchar ch) const{ 162 if (ch < 0x80) 163 return ascii[ch]; 164 else 165 return trie[ch]; 166 } 167 } 168 169 bool test(dchar ch) const { 170 return matcher[ch]; 171 } 172 173 immutable string msg = (){ 174 import std.format; 175 string message = "expected one of"; 176 return message; 177 }(); 178 179 bool parse(ref Stream stream, ref dchar value, ref Stream.Error err) const { 180 if(stream.empty){ 181 err.location = stream.location; 182 err.reason = "unexpected end of stream"; 183 return false; 184 } 185 const c = stream.front; 186 if(test(c)){ 187 value = stream.front; 188 stream.popFront(); 189 return true; 190 } 191 err.location = stream.location; 192 err.reason = msg; 193 return false; 194 } 195 } 196 197 auto set(CodepointSet set){ 198 static assert(isCodepointSet!(typeof(set)), "set only works with std.uni.CodepointSet"); 199 return Set(set); 200 } 201 202 struct Literal(alias literal) { 203 static immutable msg = "expected '"~literal~"' literal"; 204 205 bool parse(ref Stream stream, ref Stream.Range value, ref Stream.Error err) const { 206 auto m = stream.mark(); 207 auto t = literal.save(); 208 for(;;) { 209 if (t.empty) { 210 value = stream.slice(m); 211 return true; 212 } 213 if (stream.empty) { 214 stream.restore(m); 215 err.location = stream.location; 216 err.reason = "unexpected end of stream"; 217 return false; 218 } 219 if (t.front != stream.front) { 220 stream.restore(m); 221 err.location = stream.location; 222 err.reason = msg; 223 return false; 224 } 225 t.popFront(); 226 stream.popFront(); 227 } 228 } 229 } 230 231 auto literal(alias lit)() 232 if(isForwardRange!(typeof(lit)) 233 && is(ElementType!(typeof(lit)) : ElementType!(Stream.Range))) { 234 return Literal!lit(); 235 } 236 237 struct Eof { 238 bool parse(ref Stream stream, ref Nothing _, ref Stream.Error err) const { 239 bool r = stream.empty; 240 if(!r) { 241 err.location = stream.location; 242 err.reason = "input not fully consumed"; 243 } 244 return r; 245 } 246 } 247 248 /// Zero-width assertion that parses successfully on input end. 249 /// Use to make sure the whole stream was consumed. 250 auto eof(){ 251 return Eof(); 252 } 253 } 254 255 unittest { 256 alias S = SimpleStream!string; 257 with(parsers!S) { 258 auto parser = dynamic!dchar; 259 parser = tk!'a'; 260 S s = S("a"); 261 dchar c; 262 S.Error err; 263 assert(parser.parse(s, c, err)); 264 assert(c == 'a'); 265 assert(s.empty); 266 } 267 } 268 269 unittest { 270 alias S = SimpleStream!string; 271 with(parsers!S) { 272 auto s = " a".stream; 273 auto p = stk!'a'; 274 dchar c; 275 S.Error err; 276 assert(p.parse(s, c, err)); 277 assert(c == 'a'); 278 assert(s.empty); 279 280 auto s2 = "a".stream; 281 assert(p.parse(s2, c, err)); 282 assert(c == 'a'); 283 assert(s2.empty); 284 } 285 } 286 287 unittest { 288 import std.uni; 289 alias S = SimpleStream!string; 290 with(parsers!S) { 291 auto p = set(CodepointSet('A', 'Z'+1, 'a', 'z'+1)); 292 auto s = "aZ0".stream; 293 dchar c; 294 S.Error err; 295 assert(p.parse(s, c, err)); 296 assert(c == 'a'); 297 assert(p.parse(s, c, err)); 298 assert(c == 'Z'); 299 assert(!p.parse(s, c, err)); 300 assert(s.front == '0'); 301 auto p2 = set(unicode.L); 302 s = "Яz".stream; 303 assert(p2.parse(s, c, err)); 304 assert(c == 'Я'); 305 assert(p2.parse(s, c, err)); 306 assert(c == 'z'); 307 } 308 } 309 310 unittest { 311 alias S = SimpleStream!string; 312 with(parsers!S) { 313 auto p = literal!"abc"; 314 auto s = "abcd".stream; 315 S.Error err; 316 string slice; 317 assert(p.parse(s, slice, err)); 318 assert(s.front == 'd'); 319 assert(slice == "abc"); 320 s = "ab".stream; 321 assert(!p.parse(s, slice, err)); 322 assert(s.front == 'a'); 323 s = "abd".stream; 324 assert(!p.parse(s, slice, err)); 325 assert(s.front == 'a'); 326 } 327 } 328 329 unittest { 330 alias S = SimpleStream!string; 331 with(parsers!S) { 332 auto p = eof; 333 Nothing n; 334 S.Error err; 335 auto s = "".stream; 336 assert(p.parse(s, n, err)); 337 } 338 }