1 // Compiler implementation of the D programming language 2 // Copyright (c) 1999-2015 by Digital Mars 3 // All Rights Reserved 4 // written by Walter Bright 5 // http://www.digitalmars.com 6 // Distributed under the Boost Software License, Version 1.0. 7 // http://www.boost.org/LICENSE_1_0.txt 8 9 module ddmd.tokens; 10 11 import core.stdc.ctype; 12 import core.stdc.stdio; 13 import core.stdc..string; 14 import ddmd.globals; 15 import ddmd.id; 16 import ddmd.identifier; 17 import ddmd.root.port; 18 import ddmd.root.outbuffer; 19 import ddmd.root.rmem; 20 import ddmd.utf; 21 22 enum TOK : int 23 { 24 TOKreserved, 25 26 // Other 27 TOKlparen, 28 TOKrparen, 29 TOKlbracket, 30 TOKrbracket, 31 TOKlcurly, 32 TOKrcurly, 33 TOKcolon, 34 TOKneg, 35 TOKsemicolon, 36 TOKdotdotdot, 37 TOKeof, 38 TOKcast, 39 TOKnull, 40 TOKassert, 41 TOKtrue, 42 TOKfalse, 43 TOKarray, 44 TOKcall, 45 TOKaddress, 46 TOKtype, 47 TOKthrow, 48 TOKnew, 49 TOKdelete, 50 TOKstar, 51 TOKsymoff, 52 TOKvar, 53 TOKdotvar, 54 TOKdotid, 55 TOKdotti, 56 TOKdottype, 57 TOKslice, 58 TOKarraylength, 59 TOKversion, 60 TOKmodule, 61 TOKdollar, 62 TOKtemplate, 63 TOKdottd, 64 TOKdeclaration, 65 TOKtypeof, 66 TOKpragma, 67 TOKdsymbol, 68 TOKtypeid, 69 TOKuadd, 70 TOKremove, 71 TOKnewanonclass, 72 TOKcomment, 73 TOKarrayliteral, 74 TOKassocarrayliteral, 75 TOKstructliteral, 76 TOKclassreference, 77 TOKthrownexception, 78 TOKdelegateptr, 79 TOKdelegatefuncptr, 80 81 // 54 82 // Operators 83 TOKlt, 84 TOKgt, 85 TOKle, 86 TOKge, 87 TOKequal, 88 TOKnotequal, 89 TOKidentity, 90 TOKnotidentity, 91 TOKindex, 92 TOKis, 93 TOKtobool, 94 95 // 65 96 // NCEG floating point compares 97 // !<>= <> <>= !> !>= !< !<= !<> 98 TOKunord, 99 TOKlg, 100 TOKleg, 101 TOKule, 102 TOKul, 103 TOKuge, 104 TOKug, 105 TOKue, 106 107 // 73 108 TOKshl, 109 TOKshr, 110 TOKshlass, 111 TOKshrass, 112 TOKushr, 113 TOKushrass, 114 TOKcat, 115 TOKcatass, // ~ ~= 116 TOKadd, 117 TOKmin, 118 TOKaddass, 119 TOKminass, 120 TOKmul, 121 TOKdiv, 122 TOKmod, 123 TOKmulass, 124 TOKdivass, 125 TOKmodass, 126 TOKand, 127 TOKor, 128 TOKxor, 129 TOKandass, 130 TOKorass, 131 TOKxorass, 132 TOKassign, 133 TOKnot, 134 TOKtilde, 135 TOKplusplus, 136 TOKminusminus, 137 TOKconstruct, 138 TOKblit, 139 TOKdot, 140 TOKarrow, 141 TOKcomma, 142 TOKquestion, 143 TOKandand, 144 TOKoror, 145 TOKpreplusplus, 146 TOKpreminusminus, 147 148 // 112 149 // Numeric literals 150 TOKint32v, 151 TOKuns32v, 152 TOKint64v, 153 TOKuns64v, 154 TOKint128v, 155 TOKuns128v, 156 TOKfloat32v, 157 TOKfloat64v, 158 TOKfloat80v, 159 TOKimaginary32v, 160 TOKimaginary64v, 161 TOKimaginary80v, 162 163 // Char constants 164 TOKcharv, 165 TOKwcharv, 166 TOKdcharv, 167 168 // Leaf operators 169 TOKidentifier, 170 TOKstring, 171 TOKxstring, 172 TOKthis, 173 TOKsuper, 174 TOKhalt, 175 TOKtuple, 176 TOKerror, 177 178 // Basic types 179 TOKvoid, 180 TOKint8, 181 TOKuns8, 182 TOKint16, 183 TOKuns16, 184 TOKint32, 185 TOKuns32, 186 TOKint64, 187 TOKuns64, 188 TOKint128, 189 TOKuns128, 190 TOKfloat32, 191 TOKfloat64, 192 TOKfloat80, 193 TOKimaginary32, 194 TOKimaginary64, 195 TOKimaginary80, 196 TOKcomplex32, 197 TOKcomplex64, 198 TOKcomplex80, 199 TOKchar, 200 TOKwchar, 201 TOKdchar, 202 TOKbool, 203 204 // 159 205 // Aggregates 206 TOKstruct, 207 TOKclass, 208 TOKinterface, 209 TOKunion, 210 TOKenum, 211 TOKimport, 212 TOKtypedef, 213 TOKalias, 214 TOKoverride, 215 TOKdelegate, 216 TOKfunction, 217 TOKmixin, 218 TOKalign, 219 TOKextern, 220 TOKprivate, 221 TOKprotected, 222 TOKpublic, 223 TOKexport, 224 TOKstatic, 225 TOKfinal, 226 TOKconst, 227 TOKabstract, 228 TOKvolatile, 229 TOKdebug, 230 TOKdeprecated, 231 TOKin, 232 TOKout, 233 TOKinout, 234 TOKlazy, 235 TOKauto, 236 TOKpackage, 237 TOKmanifest, 238 TOKimmutable, 239 240 // Statements 241 TOKif, 242 TOKelse, 243 TOKwhile, 244 TOKfor, 245 TOKdo, 246 TOKswitch, 247 TOKcase, 248 TOKdefault, 249 TOKbreak, 250 TOKcontinue, 251 TOKwith, 252 TOKsynchronized, 253 TOKreturn, 254 TOKgoto, 255 TOKtry, 256 TOKcatch, 257 TOKfinally, 258 TOKasm, 259 TOKforeach, 260 TOKforeach_reverse, 261 TOKscope, 262 TOKon_scope_exit, 263 TOKon_scope_failure, 264 TOKon_scope_success, 265 266 // Contracts 267 TOKbody, 268 TOKinvariant, 269 270 // Testing 271 TOKunittest, 272 273 // Added after 1.0 274 TOKargTypes, 275 TOKref, 276 TOKmacro, 277 278 TOKparameters, 279 TOKtraits, 280 TOKoverloadset, 281 TOKpure, 282 TOKnothrow, 283 TOKgshared, 284 TOKline, 285 TOKfile, 286 TOKmodulestring, 287 TOKfuncstring, 288 TOKprettyfunc, 289 TOKshared, 290 TOKat, 291 TOKpow, 292 TOKpowass, 293 TOKgoesto, 294 TOKvector, 295 TOKpound, 296 297 TOKinterval, 298 TOKvoidexp, 299 TOKcantexp, 300 301 TOKMAX, 302 } 303 304 alias TOKreserved = TOK.TOKreserved; 305 alias TOKlparen = TOK.TOKlparen; 306 alias TOKrparen = TOK.TOKrparen; 307 alias TOKlbracket = TOK.TOKlbracket; 308 alias TOKrbracket = TOK.TOKrbracket; 309 alias TOKlcurly = TOK.TOKlcurly; 310 alias TOKrcurly = TOK.TOKrcurly; 311 alias TOKcolon = TOK.TOKcolon; 312 alias TOKneg = TOK.TOKneg; 313 alias TOKsemicolon = TOK.TOKsemicolon; 314 alias TOKdotdotdot = TOK.TOKdotdotdot; 315 alias TOKeof = TOK.TOKeof; 316 alias TOKcast = TOK.TOKcast; 317 alias TOKnull = TOK.TOKnull; 318 alias TOKassert = TOK.TOKassert; 319 alias TOKtrue = TOK.TOKtrue; 320 alias TOKfalse = TOK.TOKfalse; 321 alias TOKarray = TOK.TOKarray; 322 alias TOKcall = TOK.TOKcall; 323 alias TOKaddress = TOK.TOKaddress; 324 alias TOKtype = TOK.TOKtype; 325 alias TOKthrow = TOK.TOKthrow; 326 alias TOKnew = TOK.TOKnew; 327 alias TOKdelete = TOK.TOKdelete; 328 alias TOKstar = TOK.TOKstar; 329 alias TOKsymoff = TOK.TOKsymoff; 330 alias TOKvar = TOK.TOKvar; 331 alias TOKdotvar = TOK.TOKdotvar; 332 alias TOKdotid = TOK.TOKdotid; 333 alias TOKdotti = TOK.TOKdotti; 334 alias TOKdottype = TOK.TOKdottype; 335 alias TOKslice = TOK.TOKslice; 336 alias TOKarraylength = TOK.TOKarraylength; 337 alias TOKversion = TOK.TOKversion; 338 alias TOKmodule = TOK.TOKmodule; 339 alias TOKdollar = TOK.TOKdollar; 340 alias TOKtemplate = TOK.TOKtemplate; 341 alias TOKdottd = TOK.TOKdottd; 342 alias TOKdeclaration = TOK.TOKdeclaration; 343 alias TOKtypeof = TOK.TOKtypeof; 344 alias TOKpragma = TOK.TOKpragma; 345 alias TOKdsymbol = TOK.TOKdsymbol; 346 alias TOKtypeid = TOK.TOKtypeid; 347 alias TOKuadd = TOK.TOKuadd; 348 alias TOKremove = TOK.TOKremove; 349 alias TOKnewanonclass = TOK.TOKnewanonclass; 350 alias TOKcomment = TOK.TOKcomment; 351 alias TOKarrayliteral = TOK.TOKarrayliteral; 352 alias TOKassocarrayliteral = TOK.TOKassocarrayliteral; 353 alias TOKstructliteral = TOK.TOKstructliteral; 354 alias TOKclassreference = TOK.TOKclassreference; 355 alias TOKthrownexception = TOK.TOKthrownexception; 356 alias TOKdelegateptr = TOK.TOKdelegateptr; 357 alias TOKdelegatefuncptr = TOK.TOKdelegatefuncptr; 358 alias TOKlt = TOK.TOKlt; 359 alias TOKgt = TOK.TOKgt; 360 alias TOKle = TOK.TOKle; 361 alias TOKge = TOK.TOKge; 362 alias TOKequal = TOK.TOKequal; 363 alias TOKnotequal = TOK.TOKnotequal; 364 alias TOKidentity = TOK.TOKidentity; 365 alias TOKnotidentity = TOK.TOKnotidentity; 366 alias TOKindex = TOK.TOKindex; 367 alias TOKis = TOK.TOKis; 368 alias TOKtobool = TOK.TOKtobool; 369 alias TOKunord = TOK.TOKunord; 370 alias TOKlg = TOK.TOKlg; 371 alias TOKleg = TOK.TOKleg; 372 alias TOKule = TOK.TOKule; 373 alias TOKul = TOK.TOKul; 374 alias TOKuge = TOK.TOKuge; 375 alias TOKug = TOK.TOKug; 376 alias TOKue = TOK.TOKue; 377 alias TOKshl = TOK.TOKshl; 378 alias TOKshr = TOK.TOKshr; 379 alias TOKshlass = TOK.TOKshlass; 380 alias TOKshrass = TOK.TOKshrass; 381 alias TOKushr = TOK.TOKushr; 382 alias TOKushrass = TOK.TOKushrass; 383 alias TOKcat = TOK.TOKcat; 384 alias TOKcatass = TOK.TOKcatass; 385 alias TOKadd = TOK.TOKadd; 386 alias TOKmin = TOK.TOKmin; 387 alias TOKaddass = TOK.TOKaddass; 388 alias TOKminass = TOK.TOKminass; 389 alias TOKmul = TOK.TOKmul; 390 alias TOKdiv = TOK.TOKdiv; 391 alias TOKmod = TOK.TOKmod; 392 alias TOKmulass = TOK.TOKmulass; 393 alias TOKdivass = TOK.TOKdivass; 394 alias TOKmodass = TOK.TOKmodass; 395 alias TOKand = TOK.TOKand; 396 alias TOKor = TOK.TOKor; 397 alias TOKxor = TOK.TOKxor; 398 alias TOKandass = TOK.TOKandass; 399 alias TOKorass = TOK.TOKorass; 400 alias TOKxorass = TOK.TOKxorass; 401 alias TOKassign = TOK.TOKassign; 402 alias TOKnot = TOK.TOKnot; 403 alias TOKtilde = TOK.TOKtilde; 404 alias TOKplusplus = TOK.TOKplusplus; 405 alias TOKminusminus = TOK.TOKminusminus; 406 alias TOKconstruct = TOK.TOKconstruct; 407 alias TOKblit = TOK.TOKblit; 408 alias TOKdot = TOK.TOKdot; 409 alias TOKarrow = TOK.TOKarrow; 410 alias TOKcomma = TOK.TOKcomma; 411 alias TOKquestion = TOK.TOKquestion; 412 alias TOKandand = TOK.TOKandand; 413 alias TOKoror = TOK.TOKoror; 414 alias TOKpreplusplus = TOK.TOKpreplusplus; 415 alias TOKpreminusminus = TOK.TOKpreminusminus; 416 alias TOKint32v = TOK.TOKint32v; 417 alias TOKuns32v = TOK.TOKuns32v; 418 alias TOKint64v = TOK.TOKint64v; 419 alias TOKuns64v = TOK.TOKuns64v; 420 alias TOKint128v = TOK.TOKint128v; 421 alias TOKuns128v = TOK.TOKuns128v; 422 alias TOKfloat32v = TOK.TOKfloat32v; 423 alias TOKfloat64v = TOK.TOKfloat64v; 424 alias TOKfloat80v = TOK.TOKfloat80v; 425 alias TOKimaginary32v = TOK.TOKimaginary32v; 426 alias TOKimaginary64v = TOK.TOKimaginary64v; 427 alias TOKimaginary80v = TOK.TOKimaginary80v; 428 alias TOKcharv = TOK.TOKcharv; 429 alias TOKwcharv = TOK.TOKwcharv; 430 alias TOKdcharv = TOK.TOKdcharv; 431 alias TOKidentifier = TOK.TOKidentifier; 432 alias TOKstring = TOK.TOKstring; 433 alias TOKxstring = TOK.TOKxstring; 434 alias TOKthis = TOK.TOKthis; 435 alias TOKsuper = TOK.TOKsuper; 436 alias TOKhalt = TOK.TOKhalt; 437 alias TOKtuple = TOK.TOKtuple; 438 alias TOKerror = TOK.TOKerror; 439 alias TOKvoid = TOK.TOKvoid; 440 alias TOKint8 = TOK.TOKint8; 441 alias TOKuns8 = TOK.TOKuns8; 442 alias TOKint16 = TOK.TOKint16; 443 alias TOKuns16 = TOK.TOKuns16; 444 alias TOKint32 = TOK.TOKint32; 445 alias TOKuns32 = TOK.TOKuns32; 446 alias TOKint64 = TOK.TOKint64; 447 alias TOKuns64 = TOK.TOKuns64; 448 alias TOKint128 = TOK.TOKint128; 449 alias TOKuns128 = TOK.TOKuns128; 450 alias TOKfloat32 = TOK.TOKfloat32; 451 alias TOKfloat64 = TOK.TOKfloat64; 452 alias TOKfloat80 = TOK.TOKfloat80; 453 alias TOKimaginary32 = TOK.TOKimaginary32; 454 alias TOKimaginary64 = TOK.TOKimaginary64; 455 alias TOKimaginary80 = TOK.TOKimaginary80; 456 alias TOKcomplex32 = TOK.TOKcomplex32; 457 alias TOKcomplex64 = TOK.TOKcomplex64; 458 alias TOKcomplex80 = TOK.TOKcomplex80; 459 alias TOKchar = TOK.TOKchar; 460 alias TOKwchar = TOK.TOKwchar; 461 alias TOKdchar = TOK.TOKdchar; 462 alias TOKbool = TOK.TOKbool; 463 alias TOKstruct = TOK.TOKstruct; 464 alias TOKclass = TOK.TOKclass; 465 alias TOKinterface = TOK.TOKinterface; 466 alias TOKunion = TOK.TOKunion; 467 alias TOKenum = TOK.TOKenum; 468 alias TOKimport = TOK.TOKimport; 469 alias TOKtypedef = TOK.TOKtypedef; 470 alias TOKalias = TOK.TOKalias; 471 alias TOKoverride = TOK.TOKoverride; 472 alias TOKdelegate = TOK.TOKdelegate; 473 alias TOKfunction = TOK.TOKfunction; 474 alias TOKmixin = TOK.TOKmixin; 475 alias TOKalign = TOK.TOKalign; 476 alias TOKextern = TOK.TOKextern; 477 alias TOKprivate = TOK.TOKprivate; 478 alias TOKprotected = TOK.TOKprotected; 479 alias TOKpublic = TOK.TOKpublic; 480 alias TOKexport = TOK.TOKexport; 481 alias TOKstatic = TOK.TOKstatic; 482 alias TOKfinal = TOK.TOKfinal; 483 alias TOKconst = TOK.TOKconst; 484 alias TOKabstract = TOK.TOKabstract; 485 alias TOKvolatile = TOK.TOKvolatile; 486 alias TOKdebug = TOK.TOKdebug; 487 alias TOKdeprecated = TOK.TOKdeprecated; 488 alias TOKin = TOK.TOKin; 489 alias TOKout = TOK.TOKout; 490 alias TOKinout = TOK.TOKinout; 491 alias TOKlazy = TOK.TOKlazy; 492 alias TOKauto = TOK.TOKauto; 493 alias TOKpackage = TOK.TOKpackage; 494 alias TOKmanifest = TOK.TOKmanifest; 495 alias TOKimmutable = TOK.TOKimmutable; 496 alias TOKif = TOK.TOKif; 497 alias TOKelse = TOK.TOKelse; 498 alias TOKwhile = TOK.TOKwhile; 499 alias TOKfor = TOK.TOKfor; 500 alias TOKdo = TOK.TOKdo; 501 alias TOKswitch = TOK.TOKswitch; 502 alias TOKcase = TOK.TOKcase; 503 alias TOKdefault = TOK.TOKdefault; 504 alias TOKbreak = TOK.TOKbreak; 505 alias TOKcontinue = TOK.TOKcontinue; 506 alias TOKwith = TOK.TOKwith; 507 alias TOKsynchronized = TOK.TOKsynchronized; 508 alias TOKreturn = TOK.TOKreturn; 509 alias TOKgoto = TOK.TOKgoto; 510 alias TOKtry = TOK.TOKtry; 511 alias TOKcatch = TOK.TOKcatch; 512 alias TOKfinally = TOK.TOKfinally; 513 alias TOKasm = TOK.TOKasm; 514 alias TOKforeach = TOK.TOKforeach; 515 alias TOKforeach_reverse = TOK.TOKforeach_reverse; 516 alias TOKscope = TOK.TOKscope; 517 alias TOKon_scope_exit = TOK.TOKon_scope_exit; 518 alias TOKon_scope_failure = TOK.TOKon_scope_failure; 519 alias TOKon_scope_success = TOK.TOKon_scope_success; 520 alias TOKbody = TOK.TOKbody; 521 alias TOKinvariant = TOK.TOKinvariant; 522 alias TOKunittest = TOK.TOKunittest; 523 alias TOKargTypes = TOK.TOKargTypes; 524 alias TOKref = TOK.TOKref; 525 alias TOKmacro = TOK.TOKmacro; 526 alias TOKparameters = TOK.TOKparameters; 527 alias TOKtraits = TOK.TOKtraits; 528 alias TOKoverloadset = TOK.TOKoverloadset; 529 alias TOKpure = TOK.TOKpure; 530 alias TOKnothrow = TOK.TOKnothrow; 531 alias TOKgshared = TOK.TOKgshared; 532 alias TOKline = TOK.TOKline; 533 alias TOKfile = TOK.TOKfile; 534 alias TOKmodulestring = TOK.TOKmodulestring; 535 alias TOKfuncstring = TOK.TOKfuncstring; 536 alias TOKprettyfunc = TOK.TOKprettyfunc; 537 alias TOKshared = TOK.TOKshared; 538 alias TOKat = TOK.TOKat; 539 alias TOKpow = TOK.TOKpow; 540 alias TOKpowass = TOK.TOKpowass; 541 alias TOKgoesto = TOK.TOKgoesto; 542 alias TOKvector = TOK.TOKvector; 543 alias TOKpound = TOK.TOKpound; 544 alias TOKinterval = TOK.TOKinterval; 545 alias TOKvoidexp = TOK.TOKvoidexp; 546 alias TOKcantexp = TOK.TOKcantexp; 547 alias TOKMAX = TOK.TOKMAX; 548 549 enum TOKwild = TOKinout; 550 551 /*********************************************************** 552 */ 553 extern (C++) struct Token 554 { 555 Token* next; 556 Loc loc; 557 const(char)* ptr; // pointer to first character of this token within buffer 558 TOK value; 559 const(char)* blockComment; // doc comment string prior to this token 560 const(char)* lineComment; // doc comment for previous token 561 562 union 563 { 564 // Integers 565 d_int64 int64value; 566 d_uns64 uns64value; 567 // Floats 568 d_float80 float80value; 569 570 struct 571 { 572 const(char)* ustring; // UTF8 string 573 uint len; 574 ubyte postfix; // 'c', 'w', 'd' 575 } 576 577 Identifier ident; 578 } 579 580 static __gshared const(char)*[TOKMAX] tochars; 581 582 static this() 583 { 584 Identifier.initTable(); 585 foreach (kw; keywords) 586 { 587 //printf("keyword[%d] = '%s'\n",u, keywords[u].name); 588 const(char)* s = kw.name; 589 TOK v = kw.value; 590 auto id = Identifier.idPool(s, strlen(s)); 591 id.value = v; 592 //printf("tochars[%d] = '%s'\n",v, s); 593 Token.tochars[v] = s; 594 } 595 Token.tochars[TOKeof] = "EOF"; 596 Token.tochars[TOKlcurly] = "{"; 597 Token.tochars[TOKrcurly] = "}"; 598 Token.tochars[TOKlparen] = "("; 599 Token.tochars[TOKrparen] = ")"; 600 Token.tochars[TOKlbracket] = "["; 601 Token.tochars[TOKrbracket] = "]"; 602 Token.tochars[TOKsemicolon] = ";"; 603 Token.tochars[TOKcolon] = ":"; 604 Token.tochars[TOKcomma] = ","; 605 Token.tochars[TOKdot] = "."; 606 Token.tochars[TOKxor] = "^"; 607 Token.tochars[TOKxorass] = "^="; 608 Token.tochars[TOKassign] = "="; 609 Token.tochars[TOKconstruct] = "="; 610 Token.tochars[TOKblit] = "="; 611 Token.tochars[TOKlt] = "<"; 612 Token.tochars[TOKgt] = ">"; 613 Token.tochars[TOKle] = "<="; 614 Token.tochars[TOKge] = ">="; 615 Token.tochars[TOKequal] = "=="; 616 Token.tochars[TOKnotequal] = "!="; 617 Token.tochars[TOKnotidentity] = "!is"; 618 Token.tochars[TOKtobool] = "!!"; 619 Token.tochars[TOKunord] = "!<>="; 620 Token.tochars[TOKue] = "!<>"; 621 Token.tochars[TOKlg] = "<>"; 622 Token.tochars[TOKleg] = "<>="; 623 Token.tochars[TOKule] = "!>"; 624 Token.tochars[TOKul] = "!>="; 625 Token.tochars[TOKuge] = "!<"; 626 Token.tochars[TOKug] = "!<="; 627 Token.tochars[TOKnot] = "!"; 628 Token.tochars[TOKtobool] = "!!"; 629 Token.tochars[TOKshl] = "<<"; 630 Token.tochars[TOKshr] = ">>"; 631 Token.tochars[TOKushr] = ">>>"; 632 Token.tochars[TOKadd] = "+"; 633 Token.tochars[TOKmin] = "-"; 634 Token.tochars[TOKmul] = "*"; 635 Token.tochars[TOKdiv] = "/"; 636 Token.tochars[TOKmod] = "%"; 637 Token.tochars[TOKslice] = ".."; 638 Token.tochars[TOKdotdotdot] = "..."; 639 Token.tochars[TOKand] = "&"; 640 Token.tochars[TOKandand] = "&&"; 641 Token.tochars[TOKor] = "|"; 642 Token.tochars[TOKoror] = "||"; 643 Token.tochars[TOKarray] = "[]"; 644 Token.tochars[TOKindex] = "[i]"; 645 Token.tochars[TOKaddress] = "&"; 646 Token.tochars[TOKstar] = "*"; 647 Token.tochars[TOKtilde] = "~"; 648 Token.tochars[TOKdollar] = "$"; 649 Token.tochars[TOKcast] = "cast"; 650 Token.tochars[TOKplusplus] = "++"; 651 Token.tochars[TOKminusminus] = "--"; 652 Token.tochars[TOKpreplusplus] = "++"; 653 Token.tochars[TOKpreminusminus] = "--"; 654 Token.tochars[TOKtype] = "type"; 655 Token.tochars[TOKquestion] = "?"; 656 Token.tochars[TOKneg] = "-"; 657 Token.tochars[TOKuadd] = "+"; 658 Token.tochars[TOKvar] = "var"; 659 Token.tochars[TOKaddass] = "+="; 660 Token.tochars[TOKminass] = "-="; 661 Token.tochars[TOKmulass] = "*="; 662 Token.tochars[TOKdivass] = "/="; 663 Token.tochars[TOKmodass] = "%="; 664 Token.tochars[TOKshlass] = "<<="; 665 Token.tochars[TOKshrass] = ">>="; 666 Token.tochars[TOKushrass] = ">>>="; 667 Token.tochars[TOKandass] = "&="; 668 Token.tochars[TOKorass] = "|="; 669 Token.tochars[TOKcatass] = "~="; 670 Token.tochars[TOKcat] = "~"; 671 Token.tochars[TOKcall] = "call"; 672 Token.tochars[TOKidentity] = "is"; 673 Token.tochars[TOKnotidentity] = "!is"; 674 Token.tochars[TOKidentifier] = "identifier"; 675 Token.tochars[TOKat] = "@"; 676 Token.tochars[TOKpow] = "^^"; 677 Token.tochars[TOKpowass] = "^^="; 678 Token.tochars[TOKgoesto] = "=>"; 679 Token.tochars[TOKpound] = "#"; 680 681 // For debugging 682 Token.tochars[TOKerror] = "error"; 683 Token.tochars[TOKdotid] = "dotid"; 684 Token.tochars[TOKdottd] = "dottd"; 685 Token.tochars[TOKdotti] = "dotti"; 686 Token.tochars[TOKdotvar] = "dotvar"; 687 Token.tochars[TOKdottype] = "dottype"; 688 Token.tochars[TOKsymoff] = "symoff"; 689 Token.tochars[TOKarraylength] = "arraylength"; 690 Token.tochars[TOKarrayliteral] = "arrayliteral"; 691 Token.tochars[TOKassocarrayliteral] = "assocarrayliteral"; 692 Token.tochars[TOKstructliteral] = "structliteral"; 693 Token.tochars[TOKstring] = "string"; 694 Token.tochars[TOKdsymbol] = "symbol"; 695 Token.tochars[TOKtuple] = "tuple"; 696 Token.tochars[TOKdeclaration] = "declaration"; 697 Token.tochars[TOKon_scope_exit] = "scope(exit)"; 698 Token.tochars[TOKon_scope_success] = "scope(success)"; 699 Token.tochars[TOKon_scope_failure] = "scope(failure)"; 700 } 701 702 static __gshared Token* freelist = null; 703 704 static Token* alloc() 705 { 706 if (Token.freelist) 707 { 708 Token* t = freelist; 709 freelist = t.next; 710 t.next = null; 711 return t; 712 } 713 return new Token(); 714 } 715 716 void free() 717 { 718 next = freelist; 719 freelist = &this; 720 } 721 722 int isKeyword() const 723 { 724 foreach (kw; keywords) 725 { 726 if (kw.value == value) 727 return 1; 728 } 729 return 0; 730 } 731 732 debug 733 { 734 void print() 735 { 736 fprintf(stderr, "%s\n", toChars()); 737 } 738 } 739 740 /**** 741 * Set to contents of ptr[0..length] 742 * Params: 743 * ptr = pointer to string 744 * length = length of string 745 */ 746 final void setString(const(char)* ptr, size_t length) 747 { 748 auto s = cast(char*)mem.xmalloc(length + 1); 749 memcpy(s, ptr, length); 750 s[length] = 0; 751 ustring = s; 752 len = cast(uint)length; 753 postfix = 0; 754 } 755 756 /**** 757 * Set to contents of buf 758 * Params: 759 * buf = string (not zero terminated) 760 */ 761 final void setString(const ref OutBuffer buf) 762 { 763 setString(cast(const(char)*)buf.data, buf.offset); 764 } 765 766 /**** 767 * Set to empty string 768 */ 769 final void setString() 770 { 771 ustring = ""; 772 len = 0; 773 postfix = 0; 774 } 775 776 extern (C++) const(char)* toChars() const 777 { 778 __gshared char[3 + 3 * float80value.sizeof + 1] buffer; 779 const(char)* p = &buffer[0]; 780 switch (value) 781 { 782 case TOKint32v: 783 sprintf(&buffer[0], "%d", cast(d_int32)int64value); 784 break; 785 case TOKuns32v: 786 case TOKcharv: 787 case TOKwcharv: 788 case TOKdcharv: 789 sprintf(&buffer[0], "%uU", cast(d_uns32)uns64value); 790 break; 791 case TOKint64v: 792 sprintf(&buffer[0], "%lldL", cast(long)int64value); 793 break; 794 case TOKuns64v: 795 sprintf(&buffer[0], "%lluUL", cast(ulong)uns64value); 796 break; 797 case TOKfloat32v: 798 Port.ld_sprint(&buffer[0], 'g', float80value); 799 strcat(&buffer[0], "f"); 800 break; 801 case TOKfloat64v: 802 Port.ld_sprint(&buffer[0], 'g', float80value); 803 break; 804 case TOKfloat80v: 805 Port.ld_sprint(&buffer[0], 'g', float80value); 806 strcat(&buffer[0], "L"); 807 break; 808 case TOKimaginary32v: 809 Port.ld_sprint(&buffer[0], 'g', float80value); 810 strcat(&buffer[0], "fi"); 811 break; 812 case TOKimaginary64v: 813 Port.ld_sprint(&buffer[0], 'g', float80value); 814 strcat(&buffer[0], "i"); 815 break; 816 case TOKimaginary80v: 817 Port.ld_sprint(&buffer[0], 'g', float80value); 818 strcat(&buffer[0], "Li"); 819 break; 820 case TOKstring: 821 { 822 OutBuffer buf; 823 buf.writeByte('"'); 824 for (size_t i = 0; i < len;) 825 { 826 dchar c; 827 utf_decodeChar(ustring, len, i, c); 828 switch (c) 829 { 830 case 0: 831 break; 832 case '"': 833 case '\\': 834 buf.writeByte('\\'); 835 goto default; 836 default: 837 if (c <= 0x7F) 838 { 839 if (isprint(c)) 840 buf.writeByte(c); 841 else 842 buf.printf("\\x%02x", c); 843 } 844 else if (c <= 0xFFFF) 845 buf.printf("\\u%04x", c); 846 else 847 buf.printf("\\U%08x", c); 848 continue; 849 } 850 break; 851 } 852 buf.writeByte('"'); 853 if (postfix) 854 buf.writeByte(postfix); 855 p = buf.extractString(); 856 } 857 break; 858 case TOKxstring: 859 { 860 OutBuffer buf; 861 buf.writeByte('x'); 862 buf.writeByte('"'); 863 foreach (size_t i; 0 .. len) 864 { 865 if (i) 866 buf.writeByte(' '); 867 buf.printf("%02x", ustring[i]); 868 } 869 buf.writeByte('"'); 870 if (postfix) 871 buf.writeByte(postfix); 872 buf.writeByte(0); 873 p = buf.extractData(); 874 break; 875 } 876 case TOKidentifier: 877 case TOKenum: 878 case TOKstruct: 879 case TOKimport: 880 case TOKwchar: 881 case TOKdchar: 882 case TOKbool: 883 case TOKchar: 884 case TOKint8: 885 case TOKuns8: 886 case TOKint16: 887 case TOKuns16: 888 case TOKint32: 889 case TOKuns32: 890 case TOKint64: 891 case TOKuns64: 892 case TOKint128: 893 case TOKuns128: 894 case TOKfloat32: 895 case TOKfloat64: 896 case TOKfloat80: 897 case TOKimaginary32: 898 case TOKimaginary64: 899 case TOKimaginary80: 900 case TOKcomplex32: 901 case TOKcomplex64: 902 case TOKcomplex80: 903 case TOKvoid: 904 p = ident.toChars(); 905 break; 906 default: 907 p = toChars(value); 908 break; 909 } 910 return p; 911 } 912 913 static const(char)* toChars(TOK value) 914 { 915 static __gshared char[3 + 3 * value.sizeof + 1] buffer; 916 const(char)* p = tochars[value]; 917 if (!p) 918 { 919 sprintf(&buffer[0], "TOK%d", value); 920 p = &buffer[0]; 921 } 922 return p; 923 } 924 } 925 926 /**************************************** 927 */ 928 struct Keyword 929 { 930 immutable(char)* name; 931 TOK value; 932 } 933 934 immutable Keyword[] keywords = 935 [ 936 Keyword("this", TOKthis), 937 Keyword("super", TOKsuper), 938 Keyword("assert", TOKassert), 939 Keyword("null", TOKnull), 940 Keyword("true", TOKtrue), 941 Keyword("false", TOKfalse), 942 Keyword("cast", TOKcast), 943 Keyword("new", TOKnew), 944 Keyword("delete", TOKdelete), 945 Keyword("throw", TOKthrow), 946 Keyword("module", TOKmodule), 947 Keyword("pragma", TOKpragma), 948 Keyword("typeof", TOKtypeof), 949 Keyword("typeid", TOKtypeid), 950 Keyword("template", TOKtemplate), 951 Keyword("void", TOKvoid), 952 Keyword("byte", TOKint8), 953 Keyword("ubyte", TOKuns8), 954 Keyword("short", TOKint16), 955 Keyword("ushort", TOKuns16), 956 Keyword("int", TOKint32), 957 Keyword("uint", TOKuns32), 958 Keyword("long", TOKint64), 959 Keyword("ulong", TOKuns64), 960 Keyword("cent", TOKint128), 961 Keyword("ucent", TOKuns128), 962 Keyword("float", TOKfloat32), 963 Keyword("double", TOKfloat64), 964 Keyword("real", TOKfloat80), 965 Keyword("bool", TOKbool), 966 Keyword("char", TOKchar), 967 Keyword("wchar", TOKwchar), 968 Keyword("dchar", TOKdchar), 969 Keyword("ifloat", TOKimaginary32), 970 Keyword("idouble", TOKimaginary64), 971 Keyword("ireal", TOKimaginary80), 972 Keyword("cfloat", TOKcomplex32), 973 Keyword("cdouble", TOKcomplex64), 974 Keyword("creal", TOKcomplex80), 975 Keyword("delegate", TOKdelegate), 976 Keyword("function", TOKfunction), 977 Keyword("is", TOKis), 978 Keyword("if", TOKif), 979 Keyword("else", TOKelse), 980 Keyword("while", TOKwhile), 981 Keyword("for", TOKfor), 982 Keyword("do", TOKdo), 983 Keyword("switch", TOKswitch), 984 Keyword("case", TOKcase), 985 Keyword("default", TOKdefault), 986 Keyword("break", TOKbreak), 987 Keyword("continue", TOKcontinue), 988 Keyword("synchronized", TOKsynchronized), 989 Keyword("return", TOKreturn), 990 Keyword("goto", TOKgoto), 991 Keyword("try", TOKtry), 992 Keyword("catch", TOKcatch), 993 Keyword("finally", TOKfinally), 994 Keyword("with", TOKwith), 995 Keyword("asm", TOKasm), 996 Keyword("foreach", TOKforeach), 997 Keyword("foreach_reverse", TOKforeach_reverse), 998 Keyword("scope", TOKscope), 999 Keyword("struct", TOKstruct), 1000 Keyword("class", TOKclass), 1001 Keyword("interface", TOKinterface), 1002 Keyword("union", TOKunion), 1003 Keyword("enum", TOKenum), 1004 Keyword("import", TOKimport), 1005 Keyword("mixin", TOKmixin), 1006 Keyword("static", TOKstatic), 1007 Keyword("final", TOKfinal), 1008 Keyword("const", TOKconst), 1009 Keyword("typedef", TOKtypedef), 1010 Keyword("alias", TOKalias), 1011 Keyword("override", TOKoverride), 1012 Keyword("abstract", TOKabstract), 1013 Keyword("volatile", TOKvolatile), 1014 Keyword("debug", TOKdebug), 1015 Keyword("deprecated", TOKdeprecated), 1016 Keyword("in", TOKin), 1017 Keyword("out", TOKout), 1018 Keyword("inout", TOKinout), 1019 Keyword("lazy", TOKlazy), 1020 Keyword("auto", TOKauto), 1021 Keyword("align", TOKalign), 1022 Keyword("extern", TOKextern), 1023 Keyword("private", TOKprivate), 1024 Keyword("package", TOKpackage), 1025 Keyword("protected", TOKprotected), 1026 Keyword("public", TOKpublic), 1027 Keyword("export", TOKexport), 1028 Keyword("body", TOKbody), 1029 Keyword("invariant", TOKinvariant), 1030 Keyword("unittest", TOKunittest), 1031 Keyword("version", TOKversion), 1032 Keyword("__argTypes", TOKargTypes), 1033 Keyword("__parameters", TOKparameters), 1034 Keyword("ref", TOKref), 1035 Keyword("macro", TOKmacro), 1036 Keyword("pure", TOKpure), 1037 Keyword("nothrow", TOKnothrow), 1038 Keyword("__gshared", TOKgshared), 1039 Keyword("__traits", TOKtraits), 1040 Keyword("__vector", TOKvector), 1041 Keyword("__overloadset", TOKoverloadset), 1042 Keyword("__FILE__", TOKfile), 1043 Keyword("__LINE__", TOKline), 1044 Keyword("__MODULE__", TOKmodulestring), 1045 Keyword("__FUNCTION__", TOKfuncstring), 1046 Keyword("__PRETTY_FUNCTION__", TOKprettyfunc), 1047 Keyword("shared", TOKshared), 1048 Keyword("immutable", TOKimmutable), 1049 ];