1 // Compiler implementation of the D programming language 2 // Copyright (c) 1999-2015 by Digital Mars 3 // All Rights Reserved 4 // written by Walter Bright 5 // http://www.digitalmars.com 6 // Distributed under the Boost Software License, Version 1.0. 7 // http://www.boost.org/LICENSE_1_0.txt 8 9 module ddmd.tokens; 10 11 import core.stdc.ctype, core.stdc.stdio, core.stdc..string; 12 import ddmd.globals, ddmd.id, ddmd.identifier, ddmd.root.longdouble, ddmd.root.outbuffer, ddmd.root.rmem, ddmd.utf; 13 14 enum TOK : int 15 { 16 TOKreserved, 17 // Other 18 TOKlparen, 19 TOKrparen, 20 TOKlbracket, 21 TOKrbracket, 22 TOKlcurly, 23 TOKrcurly, 24 TOKcolon, 25 TOKneg, 26 TOKsemicolon, 27 TOKdotdotdot, 28 TOKeof, 29 TOKcast, 30 TOKnull, 31 TOKassert, 32 TOKtrue, 33 TOKfalse, 34 TOKarray, 35 TOKcall, 36 TOKaddress, 37 TOKtype, 38 TOKthrow, 39 TOKnew, 40 TOKdelete, 41 TOKstar, 42 TOKsymoff, 43 TOKvar, 44 TOKdotvar, 45 TOKdotti, 46 TOKdotexp, 47 TOKdottype, 48 TOKslice, 49 TOKarraylength, 50 TOKversion, 51 TOKmodule, 52 TOKdollar, 53 TOKtemplate, 54 TOKdottd, 55 TOKdeclaration, 56 TOKtypeof, 57 TOKpragma, 58 TOKdsymbol, 59 TOKtypeid, 60 TOKuadd, 61 TOKremove, 62 TOKnewanonclass, 63 TOKcomment, 64 TOKarrayliteral, 65 TOKassocarrayliteral, 66 TOKstructliteral, 67 TOKclassreference, 68 TOKthrownexception, 69 TOKdelegateptr, 70 TOKdelegatefuncptr, 71 // 54 72 // Operators 73 TOKlt, 74 TOKgt, 75 TOKle, 76 TOKge, 77 TOKequal, 78 TOKnotequal, 79 TOKidentity, 80 TOKnotidentity, 81 TOKindex, 82 TOKis, 83 TOKtobool, 84 // 65 85 // NCEG floating point compares 86 // !<>= <> <>= !> !>= !< !<= !<> 87 TOKunord, 88 TOKlg, 89 TOKleg, 90 TOKule, 91 TOKul, 92 TOKuge, 93 TOKug, 94 TOKue, 95 // 73 96 TOKshl, 97 TOKshr, 98 TOKshlass, 99 TOKshrass, 100 TOKushr, 101 TOKushrass, 102 TOKcat, 103 TOKcatass, // ~ ~= 104 TOKadd, 105 TOKmin, 106 TOKaddass, 107 TOKminass, 108 TOKmul, 109 TOKdiv, 110 TOKmod, 111 TOKmulass, 112 TOKdivass, 113 TOKmodass, 114 TOKand, 115 TOKor, 116 TOKxor, 117 TOKandass, 118 TOKorass, 119 TOKxorass, 120 TOKassign, 121 TOKnot, 122 TOKtilde, 123 TOKplusplus, 124 TOKminusminus, 125 TOKconstruct, 126 TOKblit, 127 TOKdot, 128 TOKarrow, 129 TOKcomma, 130 TOKquestion, 131 TOKandand, 132 TOKoror, 133 TOKpreplusplus, 134 TOKpreminusminus, 135 // 112 136 // Numeric literals 137 TOKint32v, 138 TOKuns32v, 139 TOKint64v, 140 TOKuns64v, 141 TOKint128v, 142 TOKuns128v, 143 TOKfloat32v, 144 TOKfloat64v, 145 TOKfloat80v, 146 TOKimaginary32v, 147 TOKimaginary64v, 148 TOKimaginary80v, 149 // Char constants 150 TOKcharv, 151 TOKwcharv, 152 TOKdcharv, 153 // Leaf operators 154 TOKidentifier, 155 TOKstring, 156 TOKxstring, 157 TOKthis, 158 TOKsuper, 159 TOKhalt, 160 TOKtuple, 161 TOKerror, 162 // Basic types 163 TOKvoid, 164 TOKint8, 165 TOKuns8, 166 TOKint16, 167 TOKuns16, 168 TOKint32, 169 TOKuns32, 170 TOKint64, 171 TOKuns64, 172 TOKint128, 173 TOKuns128, 174 TOKfloat32, 175 TOKfloat64, 176 TOKfloat80, 177 TOKimaginary32, 178 TOKimaginary64, 179 TOKimaginary80, 180 TOKcomplex32, 181 TOKcomplex64, 182 TOKcomplex80, 183 TOKchar, 184 TOKwchar, 185 TOKdchar, 186 TOKbool, 187 // 159 188 // Aggregates 189 TOKstruct, 190 TOKclass, 191 TOKinterface, 192 TOKunion, 193 TOKenum, 194 TOKimport, 195 TOKtypedef, 196 TOKalias, 197 TOKoverride, 198 TOKdelegate, 199 TOKfunction, 200 TOKmixin, 201 TOKalign, 202 TOKextern, 203 TOKprivate, 204 TOKprotected, 205 TOKpublic, 206 TOKexport, 207 TOKstatic, 208 TOKfinal, 209 TOKconst, 210 TOKabstract, 211 TOKvolatile, 212 TOKdebug, 213 TOKdeprecated, 214 TOKin, 215 TOKout, 216 TOKinout, 217 TOKlazy, 218 TOKauto, 219 TOKpackage, 220 TOKmanifest, 221 TOKimmutable, 222 // Statements 223 TOKif, 224 TOKelse, 225 TOKwhile, 226 TOKfor, 227 TOKdo, 228 TOKswitch, 229 TOKcase, 230 TOKdefault, 231 TOKbreak, 232 TOKcontinue, 233 TOKwith, 234 TOKsynchronized, 235 TOKreturn, 236 TOKgoto, 237 TOKtry, 238 TOKcatch, 239 TOKfinally, 240 TOKasm, 241 TOKforeach, 242 TOKforeach_reverse, 243 TOKscope, 244 TOKon_scope_exit, 245 TOKon_scope_failure, 246 TOKon_scope_success, 247 // Contracts 248 TOKbody, 249 TOKinvariant, 250 // Testing 251 TOKunittest, 252 // Added after 1.0 253 TOKargTypes, 254 TOKref, 255 TOKmacro, 256 TOKparameters, 257 TOKtraits, 258 TOKoverloadset, 259 TOKpure, 260 TOKnothrow, 261 TOKgshared, 262 TOKline, 263 TOKfile, 264 TOKmodulestring, 265 TOKfuncstring, 266 TOKprettyfunc, 267 TOKshared, 268 TOKat, 269 TOKpow, 270 TOKpowass, 271 TOKgoesto, 272 TOKvector, 273 TOKpound, 274 TOKinterval, 275 TOKvoidexp, 276 TOKcantexp, 277 TOKMAX, 278 } 279 280 alias TOKreserved = TOK.TOKreserved; 281 alias TOKlparen = TOK.TOKlparen; 282 alias TOKrparen = TOK.TOKrparen; 283 alias TOKlbracket = TOK.TOKlbracket; 284 alias TOKrbracket = TOK.TOKrbracket; 285 alias TOKlcurly = TOK.TOKlcurly; 286 alias TOKrcurly = TOK.TOKrcurly; 287 alias TOKcolon = TOK.TOKcolon; 288 alias TOKneg = TOK.TOKneg; 289 alias TOKsemicolon = TOK.TOKsemicolon; 290 alias TOKdotdotdot = TOK.TOKdotdotdot; 291 alias TOKeof = TOK.TOKeof; 292 alias TOKcast = TOK.TOKcast; 293 alias TOKnull = TOK.TOKnull; 294 alias TOKassert = TOK.TOKassert; 295 alias TOKtrue = TOK.TOKtrue; 296 alias TOKfalse = TOK.TOKfalse; 297 alias TOKarray = TOK.TOKarray; 298 alias TOKcall = TOK.TOKcall; 299 alias TOKaddress = TOK.TOKaddress; 300 alias TOKtype = TOK.TOKtype; 301 alias TOKthrow = TOK.TOKthrow; 302 alias TOKnew = TOK.TOKnew; 303 alias TOKdelete = TOK.TOKdelete; 304 alias TOKstar = TOK.TOKstar; 305 alias TOKsymoff = TOK.TOKsymoff; 306 alias TOKvar = TOK.TOKvar; 307 alias TOKdotvar = TOK.TOKdotvar; 308 alias TOKdotti = TOK.TOKdotti; 309 alias TOKdotexp = TOK.TOKdotexp; 310 alias TOKdottype = TOK.TOKdottype; 311 alias TOKslice = TOK.TOKslice; 312 alias TOKarraylength = TOK.TOKarraylength; 313 alias TOKversion = TOK.TOKversion; 314 alias TOKmodule = TOK.TOKmodule; 315 alias TOKdollar = TOK.TOKdollar; 316 alias TOKtemplate = TOK.TOKtemplate; 317 alias TOKdottd = TOK.TOKdottd; 318 alias TOKdeclaration = TOK.TOKdeclaration; 319 alias TOKtypeof = TOK.TOKtypeof; 320 alias TOKpragma = TOK.TOKpragma; 321 alias TOKdsymbol = TOK.TOKdsymbol; 322 alias TOKtypeid = TOK.TOKtypeid; 323 alias TOKuadd = TOK.TOKuadd; 324 alias TOKremove = TOK.TOKremove; 325 alias TOKnewanonclass = TOK.TOKnewanonclass; 326 alias TOKcomment = TOK.TOKcomment; 327 alias TOKarrayliteral = TOK.TOKarrayliteral; 328 alias TOKassocarrayliteral = TOK.TOKassocarrayliteral; 329 alias TOKstructliteral = TOK.TOKstructliteral; 330 alias TOKclassreference = TOK.TOKclassreference; 331 alias TOKthrownexception = TOK.TOKthrownexception; 332 alias TOKdelegateptr = TOK.TOKdelegateptr; 333 alias TOKdelegatefuncptr = TOK.TOKdelegatefuncptr; 334 alias TOKlt = TOK.TOKlt; 335 alias TOKgt = TOK.TOKgt; 336 alias TOKle = TOK.TOKle; 337 alias TOKge = TOK.TOKge; 338 alias TOKequal = TOK.TOKequal; 339 alias TOKnotequal = TOK.TOKnotequal; 340 alias TOKidentity = TOK.TOKidentity; 341 alias TOKnotidentity = TOK.TOKnotidentity; 342 alias TOKindex = TOK.TOKindex; 343 alias TOKis = TOK.TOKis; 344 alias TOKtobool = TOK.TOKtobool; 345 alias TOKunord = TOK.TOKunord; 346 alias TOKlg = TOK.TOKlg; 347 alias TOKleg = TOK.TOKleg; 348 alias TOKule = TOK.TOKule; 349 alias TOKul = TOK.TOKul; 350 alias TOKuge = TOK.TOKuge; 351 alias TOKug = TOK.TOKug; 352 alias TOKue = TOK.TOKue; 353 alias TOKshl = TOK.TOKshl; 354 alias TOKshr = TOK.TOKshr; 355 alias TOKshlass = TOK.TOKshlass; 356 alias TOKshrass = TOK.TOKshrass; 357 alias TOKushr = TOK.TOKushr; 358 alias TOKushrass = TOK.TOKushrass; 359 alias TOKcat = TOK.TOKcat; 360 alias TOKcatass = TOK.TOKcatass; 361 alias TOKadd = TOK.TOKadd; 362 alias TOKmin = TOK.TOKmin; 363 alias TOKaddass = TOK.TOKaddass; 364 alias TOKminass = TOK.TOKminass; 365 alias TOKmul = TOK.TOKmul; 366 alias TOKdiv = TOK.TOKdiv; 367 alias TOKmod = TOK.TOKmod; 368 alias TOKmulass = TOK.TOKmulass; 369 alias TOKdivass = TOK.TOKdivass; 370 alias TOKmodass = TOK.TOKmodass; 371 alias TOKand = TOK.TOKand; 372 alias TOKor = TOK.TOKor; 373 alias TOKxor = TOK.TOKxor; 374 alias TOKandass = TOK.TOKandass; 375 alias TOKorass = TOK.TOKorass; 376 alias TOKxorass = TOK.TOKxorass; 377 alias TOKassign = TOK.TOKassign; 378 alias TOKnot = TOK.TOKnot; 379 alias TOKtilde = TOK.TOKtilde; 380 alias TOKplusplus = TOK.TOKplusplus; 381 alias TOKminusminus = TOK.TOKminusminus; 382 alias TOKconstruct = TOK.TOKconstruct; 383 alias TOKblit = TOK.TOKblit; 384 alias TOKdot = TOK.TOKdot; 385 alias TOKarrow = TOK.TOKarrow; 386 alias TOKcomma = TOK.TOKcomma; 387 alias TOKquestion = TOK.TOKquestion; 388 alias TOKandand = TOK.TOKandand; 389 alias TOKoror = TOK.TOKoror; 390 alias TOKpreplusplus = TOK.TOKpreplusplus; 391 alias TOKpreminusminus = TOK.TOKpreminusminus; 392 alias TOKint32v = TOK.TOKint32v; 393 alias TOKuns32v = TOK.TOKuns32v; 394 alias TOKint64v = TOK.TOKint64v; 395 alias TOKuns64v = TOK.TOKuns64v; 396 alias TOKint128v = TOK.TOKint128v; 397 alias TOKuns128v = TOK.TOKuns128v; 398 alias TOKfloat32v = TOK.TOKfloat32v; 399 alias TOKfloat64v = TOK.TOKfloat64v; 400 alias TOKfloat80v = TOK.TOKfloat80v; 401 alias TOKimaginary32v = TOK.TOKimaginary32v; 402 alias TOKimaginary64v = TOK.TOKimaginary64v; 403 alias TOKimaginary80v = TOK.TOKimaginary80v; 404 alias TOKcharv = TOK.TOKcharv; 405 alias TOKwcharv = TOK.TOKwcharv; 406 alias TOKdcharv = TOK.TOKdcharv; 407 alias TOKidentifier = TOK.TOKidentifier; 408 alias TOKstring = TOK.TOKstring; 409 alias TOKxstring = TOK.TOKxstring; 410 alias TOKthis = TOK.TOKthis; 411 alias TOKsuper = TOK.TOKsuper; 412 alias TOKhalt = TOK.TOKhalt; 413 alias TOKtuple = TOK.TOKtuple; 414 alias TOKerror = TOK.TOKerror; 415 alias TOKvoid = TOK.TOKvoid; 416 alias TOKint8 = TOK.TOKint8; 417 alias TOKuns8 = TOK.TOKuns8; 418 alias TOKint16 = TOK.TOKint16; 419 alias TOKuns16 = TOK.TOKuns16; 420 alias TOKint32 = TOK.TOKint32; 421 alias TOKuns32 = TOK.TOKuns32; 422 alias TOKint64 = TOK.TOKint64; 423 alias TOKuns64 = TOK.TOKuns64; 424 alias TOKint128 = TOK.TOKint128; 425 alias TOKuns128 = TOK.TOKuns128; 426 alias TOKfloat32 = TOK.TOKfloat32; 427 alias TOKfloat64 = TOK.TOKfloat64; 428 alias TOKfloat80 = TOK.TOKfloat80; 429 alias TOKimaginary32 = TOK.TOKimaginary32; 430 alias TOKimaginary64 = TOK.TOKimaginary64; 431 alias TOKimaginary80 = TOK.TOKimaginary80; 432 alias TOKcomplex32 = TOK.TOKcomplex32; 433 alias TOKcomplex64 = TOK.TOKcomplex64; 434 alias TOKcomplex80 = TOK.TOKcomplex80; 435 alias TOKchar = TOK.TOKchar; 436 alias TOKwchar = TOK.TOKwchar; 437 alias TOKdchar = TOK.TOKdchar; 438 alias TOKbool = TOK.TOKbool; 439 alias TOKstruct = TOK.TOKstruct; 440 alias TOKclass = TOK.TOKclass; 441 alias TOKinterface = TOK.TOKinterface; 442 alias TOKunion = TOK.TOKunion; 443 alias TOKenum = TOK.TOKenum; 444 alias TOKimport = TOK.TOKimport; 445 alias TOKtypedef = TOK.TOKtypedef; 446 alias TOKalias = TOK.TOKalias; 447 alias TOKoverride = TOK.TOKoverride; 448 alias TOKdelegate = TOK.TOKdelegate; 449 alias TOKfunction = TOK.TOKfunction; 450 alias TOKmixin = TOK.TOKmixin; 451 alias TOKalign = TOK.TOKalign; 452 alias TOKextern = TOK.TOKextern; 453 alias TOKprivate = TOK.TOKprivate; 454 alias TOKprotected = TOK.TOKprotected; 455 alias TOKpublic = TOK.TOKpublic; 456 alias TOKexport = TOK.TOKexport; 457 alias TOKstatic = TOK.TOKstatic; 458 alias TOKfinal = TOK.TOKfinal; 459 alias TOKconst = TOK.TOKconst; 460 alias TOKabstract = TOK.TOKabstract; 461 alias TOKvolatile = TOK.TOKvolatile; 462 alias TOKdebug = TOK.TOKdebug; 463 alias TOKdeprecated = TOK.TOKdeprecated; 464 alias TOKin = TOK.TOKin; 465 alias TOKout = TOK.TOKout; 466 alias TOKinout = TOK.TOKinout; 467 alias TOKlazy = TOK.TOKlazy; 468 alias TOKauto = TOK.TOKauto; 469 alias TOKpackage = TOK.TOKpackage; 470 alias TOKmanifest = TOK.TOKmanifest; 471 alias TOKimmutable = TOK.TOKimmutable; 472 alias TOKif = TOK.TOKif; 473 alias TOKelse = TOK.TOKelse; 474 alias TOKwhile = TOK.TOKwhile; 475 alias TOKfor = TOK.TOKfor; 476 alias TOKdo = TOK.TOKdo; 477 alias TOKswitch = TOK.TOKswitch; 478 alias TOKcase = TOK.TOKcase; 479 alias TOKdefault = TOK.TOKdefault; 480 alias TOKbreak = TOK.TOKbreak; 481 alias TOKcontinue = TOK.TOKcontinue; 482 alias TOKwith = TOK.TOKwith; 483 alias TOKsynchronized = TOK.TOKsynchronized; 484 alias TOKreturn = TOK.TOKreturn; 485 alias TOKgoto = TOK.TOKgoto; 486 alias TOKtry = TOK.TOKtry; 487 alias TOKcatch = TOK.TOKcatch; 488 alias TOKfinally = TOK.TOKfinally; 489 alias TOKasm = TOK.TOKasm; 490 alias TOKforeach = TOK.TOKforeach; 491 alias TOKforeach_reverse = TOK.TOKforeach_reverse; 492 alias TOKscope = TOK.TOKscope; 493 alias TOKon_scope_exit = TOK.TOKon_scope_exit; 494 alias TOKon_scope_failure = TOK.TOKon_scope_failure; 495 alias TOKon_scope_success = TOK.TOKon_scope_success; 496 alias TOKbody = TOK.TOKbody; 497 alias TOKinvariant = TOK.TOKinvariant; 498 alias TOKunittest = TOK.TOKunittest; 499 alias TOKargTypes = TOK.TOKargTypes; 500 alias TOKref = TOK.TOKref; 501 alias TOKmacro = TOK.TOKmacro; 502 alias TOKparameters = TOK.TOKparameters; 503 alias TOKtraits = TOK.TOKtraits; 504 alias TOKoverloadset = TOK.TOKoverloadset; 505 alias TOKpure = TOK.TOKpure; 506 alias TOKnothrow = TOK.TOKnothrow; 507 alias TOKgshared = TOK.TOKgshared; 508 alias TOKline = TOK.TOKline; 509 alias TOKfile = TOK.TOKfile; 510 alias TOKmodulestring = TOK.TOKmodulestring; 511 alias TOKfuncstring = TOK.TOKfuncstring; 512 alias TOKprettyfunc = TOK.TOKprettyfunc; 513 alias TOKshared = TOK.TOKshared; 514 alias TOKat = TOK.TOKat; 515 alias TOKpow = TOK.TOKpow; 516 alias TOKpowass = TOK.TOKpowass; 517 alias TOKgoesto = TOK.TOKgoesto; 518 alias TOKvector = TOK.TOKvector; 519 alias TOKpound = TOK.TOKpound; 520 alias TOKinterval = TOK.TOKinterval; 521 alias TOKvoidexp = TOK.TOKvoidexp; 522 alias TOKcantexp = TOK.TOKcantexp; 523 alias TOKMAX = TOK.TOKMAX; 524 525 enum TOKwild = TOKinout; 526 527 struct Token 528 { 529 Token* next; 530 Loc loc; 531 const(char)* ptr; // pointer to first character of this token within buffer 532 TOK value; 533 const(char)* blockComment; // doc comment string prior to this token 534 const(char)* lineComment; // doc comment for previous token 535 536 union 537 { 538 // Integers 539 d_int64 int64value; 540 d_uns64 uns64value; 541 // Floats 542 d_float80 float80value; 543 544 struct 545 { 546 char* ustring; // UTF8 string 547 uint len; 548 ubyte postfix; // 'c', 'w', 'd' 549 } 550 551 Identifier ident; 552 } 553 554 extern (C++) static __gshared const(char)*[TOKMAX] tochars; 555 556 extern (C++) static void initTokens() 557 { 558 for (nkeywords = 0; keywords[nkeywords].name; nkeywords++) 559 { 560 //printf("keyword[%d] = '%s'\n",u, keywords[u].name); 561 const(char)* s = keywords[nkeywords].name; 562 TOK v = keywords[nkeywords].value; 563 Identifier id = Identifier.idPool(s); 564 id.value = v; 565 //printf("tochars[%d] = '%s'\n",v, s); 566 Token.tochars[v] = s; 567 } 568 Token.tochars[TOKeof] = "EOF"; 569 Token.tochars[TOKlcurly] = "{"; 570 Token.tochars[TOKrcurly] = "}"; 571 Token.tochars[TOKlparen] = "("; 572 Token.tochars[TOKrparen] = ")"; 573 Token.tochars[TOKlbracket] = "["; 574 Token.tochars[TOKrbracket] = "]"; 575 Token.tochars[TOKsemicolon] = ";"; 576 Token.tochars[TOKcolon] = ":"; 577 Token.tochars[TOKcomma] = ","; 578 Token.tochars[TOKdot] = "."; 579 Token.tochars[TOKxor] = "^"; 580 Token.tochars[TOKxorass] = "^="; 581 Token.tochars[TOKassign] = "="; 582 Token.tochars[TOKconstruct] = "="; 583 Token.tochars[TOKblit] = "="; 584 Token.tochars[TOKlt] = "<"; 585 Token.tochars[TOKgt] = ">"; 586 Token.tochars[TOKle] = "<="; 587 Token.tochars[TOKge] = ">="; 588 Token.tochars[TOKequal] = "=="; 589 Token.tochars[TOKnotequal] = "!="; 590 Token.tochars[TOKnotidentity] = "!is"; 591 Token.tochars[TOKtobool] = "!!"; 592 Token.tochars[TOKunord] = "!<>="; 593 Token.tochars[TOKue] = "!<>"; 594 Token.tochars[TOKlg] = "<>"; 595 Token.tochars[TOKleg] = "<>="; 596 Token.tochars[TOKule] = "!>"; 597 Token.tochars[TOKul] = "!>="; 598 Token.tochars[TOKuge] = "!<"; 599 Token.tochars[TOKug] = "!<="; 600 Token.tochars[TOKnot] = "!"; 601 Token.tochars[TOKtobool] = "!!"; 602 Token.tochars[TOKshl] = "<<"; 603 Token.tochars[TOKshr] = ">>"; 604 Token.tochars[TOKushr] = ">>>"; 605 Token.tochars[TOKadd] = "+"; 606 Token.tochars[TOKmin] = "-"; 607 Token.tochars[TOKmul] = "*"; 608 Token.tochars[TOKdiv] = "/"; 609 Token.tochars[TOKmod] = "%"; 610 Token.tochars[TOKslice] = ".."; 611 Token.tochars[TOKdotdotdot] = "..."; 612 Token.tochars[TOKand] = "&"; 613 Token.tochars[TOKandand] = "&&"; 614 Token.tochars[TOKor] = "|"; 615 Token.tochars[TOKoror] = "||"; 616 Token.tochars[TOKarray] = "[]"; 617 Token.tochars[TOKindex] = "[i]"; 618 Token.tochars[TOKaddress] = "&"; 619 Token.tochars[TOKstar] = "*"; 620 Token.tochars[TOKtilde] = "~"; 621 Token.tochars[TOKdollar] = "$"; 622 Token.tochars[TOKcast] = "cast"; 623 Token.tochars[TOKplusplus] = "++"; 624 Token.tochars[TOKminusminus] = "--"; 625 Token.tochars[TOKpreplusplus] = "++"; 626 Token.tochars[TOKpreminusminus] = "--"; 627 Token.tochars[TOKtype] = "type"; 628 Token.tochars[TOKquestion] = "?"; 629 Token.tochars[TOKneg] = "-"; 630 Token.tochars[TOKuadd] = "+"; 631 Token.tochars[TOKvar] = "var"; 632 Token.tochars[TOKaddass] = "+="; 633 Token.tochars[TOKminass] = "-="; 634 Token.tochars[TOKmulass] = "*="; 635 Token.tochars[TOKdivass] = "/="; 636 Token.tochars[TOKmodass] = "%="; 637 Token.tochars[TOKshlass] = "<<="; 638 Token.tochars[TOKshrass] = ">>="; 639 Token.tochars[TOKushrass] = ">>>="; 640 Token.tochars[TOKandass] = "&="; 641 Token.tochars[TOKorass] = "|="; 642 Token.tochars[TOKcatass] = "~="; 643 Token.tochars[TOKcat] = "~"; 644 Token.tochars[TOKcall] = "call"; 645 Token.tochars[TOKidentity] = "is"; 646 Token.tochars[TOKnotidentity] = "!is"; 647 Token.tochars[TOKorass] = "|="; 648 Token.tochars[TOKidentifier] = "identifier"; 649 Token.tochars[TOKat] = "@"; 650 Token.tochars[TOKpow] = "^^"; 651 Token.tochars[TOKpowass] = "^^="; 652 Token.tochars[TOKgoesto] = "=>"; 653 Token.tochars[TOKpound] = "#"; 654 // For debugging 655 Token.tochars[TOKerror] = "error"; 656 Token.tochars[TOKdotexp] = "dotexp"; 657 Token.tochars[TOKdotti] = "dotti"; 658 Token.tochars[TOKdotvar] = "dotvar"; 659 Token.tochars[TOKdottype] = "dottype"; 660 Token.tochars[TOKsymoff] = "symoff"; 661 Token.tochars[TOKarraylength] = "arraylength"; 662 Token.tochars[TOKarrayliteral] = "arrayliteral"; 663 Token.tochars[TOKassocarrayliteral] = "assocarrayliteral"; 664 Token.tochars[TOKstructliteral] = "structliteral"; 665 Token.tochars[TOKstring] = "string"; 666 Token.tochars[TOKdsymbol] = "symbol"; 667 Token.tochars[TOKtuple] = "tuple"; 668 Token.tochars[TOKdeclaration] = "declaration"; 669 Token.tochars[TOKdottd] = "dottd"; 670 Token.tochars[TOKon_scope_exit] = "scope(exit)"; 671 Token.tochars[TOKon_scope_success] = "scope(success)"; 672 Token.tochars[TOKon_scope_failure] = "scope(failure)"; 673 } 674 675 /************************* Token **********************************************/ 676 extern (C++) static __gshared Token* freelist = null; 677 678 extern (C++) static Token* alloc() 679 { 680 if (Token.freelist) 681 { 682 Token* t = freelist; 683 freelist = t.next; 684 t.next = null; 685 return t; 686 } 687 return new Token(); 688 } 689 690 extern (C++) void free() 691 { 692 next = freelist; 693 freelist = &this; 694 } 695 696 extern (C++) int isKeyword() 697 { 698 for (size_t u = 0; u < nkeywords; u++) 699 { 700 if (keywords[u].value == value) 701 return 1; 702 } 703 return 0; 704 } 705 706 debug 707 { 708 extern (C++) void print() 709 { 710 fprintf(stderr, "%s\n", toChars()); 711 } 712 } 713 714 extern (C++) const(char)* toChars() 715 { 716 static __gshared char[3 + 3 * float80value.sizeof + 1] buffer; 717 const(char)* p = &buffer[0]; 718 switch (value) 719 { 720 case TOKint32v: 721 sprintf(&buffer[0], "%d", cast(d_int32)int64value); 722 break; 723 case TOKuns32v: 724 case TOKcharv: 725 case TOKwcharv: 726 case TOKdcharv: 727 sprintf(&buffer[0], "%uU", cast(d_uns32)uns64value); 728 break; 729 case TOKint64v: 730 sprintf(&buffer[0], "%lldL", cast(long)int64value); 731 break; 732 case TOKuns64v: 733 sprintf(&buffer[0], "%lluUL", cast(ulong)uns64value); 734 break; 735 case TOKfloat32v: 736 ld_sprint(&buffer[0], 'g', float80value); 737 strcat(&buffer[0], "f"); 738 break; 739 case TOKfloat64v: 740 ld_sprint(&buffer[0], 'g', float80value); 741 break; 742 case TOKfloat80v: 743 ld_sprint(&buffer[0], 'g', float80value); 744 strcat(&buffer[0], "L"); 745 break; 746 case TOKimaginary32v: 747 ld_sprint(&buffer[0], 'g', float80value); 748 strcat(&buffer[0], "fi"); 749 break; 750 case TOKimaginary64v: 751 ld_sprint(&buffer[0], 'g', float80value); 752 strcat(&buffer[0], "i"); 753 break; 754 case TOKimaginary80v: 755 ld_sprint(&buffer[0], 'g', float80value); 756 strcat(&buffer[0], "Li"); 757 break; 758 case TOKstring: 759 { 760 OutBuffer buf; 761 buf.writeByte('"'); 762 for (size_t i = 0; i < len;) 763 { 764 uint c; 765 utf_decodeChar(cast(char*)ustring, len, &i, &c); 766 switch (c) 767 { 768 case 0: 769 break; 770 case '"': 771 case '\\': 772 buf.writeByte('\\'); 773 default: 774 if (c <= 0x7F) 775 { 776 if (isprint(c)) 777 buf.writeByte(c); 778 else 779 buf.printf("\\x%02x", c); 780 } 781 else if (c <= 0xFFFF) 782 buf.printf("\\u%04x", c); 783 else 784 buf.printf("\\U%08x", c); 785 continue; 786 } 787 break; 788 } 789 buf.writeByte('"'); 790 if (postfix) 791 buf.writeByte(postfix); 792 p = buf.extractString(); 793 } 794 break; 795 case TOKxstring: 796 { 797 OutBuffer buf; 798 buf.writeByte('x'); 799 buf.writeByte('"'); 800 for (size_t i = 0; i < len; i++) 801 { 802 if (i) 803 buf.writeByte(' '); 804 buf.printf("%02x", ustring[i]); 805 } 806 buf.writeByte('"'); 807 if (postfix) 808 buf.writeByte(postfix); 809 buf.writeByte(0); 810 p = cast(char*)buf.extractData(); 811 break; 812 } 813 case TOKidentifier: 814 case TOKenum: 815 case TOKstruct: 816 case TOKimport: 817 case TOKwchar: 818 case TOKdchar: 819 case TOKbool: 820 case TOKchar: 821 case TOKint8: 822 case TOKuns8: 823 case TOKint16: 824 case TOKuns16: 825 case TOKint32: 826 case TOKuns32: 827 case TOKint64: 828 case TOKuns64: 829 case TOKint128: 830 case TOKuns128: 831 case TOKfloat32: 832 case TOKfloat64: 833 case TOKfloat80: 834 case TOKimaginary32: 835 case TOKimaginary64: 836 case TOKimaginary80: 837 case TOKcomplex32: 838 case TOKcomplex64: 839 case TOKcomplex80: 840 case TOKvoid: 841 p = ident.toChars(); 842 break; 843 default: 844 p = toChars(value); 845 break; 846 } 847 return p; 848 } 849 850 extern (C++) static const(char)* toChars(TOK value) 851 { 852 static __gshared char[3 + 3 * value.sizeof + 1] buffer; 853 const(char)* p = tochars[value]; 854 if (!p) 855 { 856 sprintf(&buffer[0], "TOK%d", value); 857 p = &buffer[0]; 858 } 859 return p; 860 } 861 } 862 863 /**************************************** 864 */ 865 struct Keyword 866 { 867 const(char)* name; 868 TOK value; 869 } 870 871 extern (C++) __gshared size_t nkeywords; 872 extern (C++) __gshared Keyword* keywords = 873 [ 874 Keyword("this", TOKthis), 875 Keyword("super", TOKsuper), 876 Keyword("assert", TOKassert), 877 Keyword("null", TOKnull), 878 Keyword("true", TOKtrue), 879 Keyword("false", TOKfalse), 880 Keyword("cast", TOKcast), 881 Keyword("new", TOKnew), 882 Keyword("delete", TOKdelete), 883 Keyword("throw", TOKthrow), 884 Keyword("module", TOKmodule), 885 Keyword("pragma", TOKpragma), 886 Keyword("typeof", TOKtypeof), 887 Keyword("typeid", TOKtypeid), 888 Keyword("template", TOKtemplate), 889 Keyword("void", TOKvoid), 890 Keyword("byte", TOKint8), 891 Keyword("ubyte", TOKuns8), 892 Keyword("short", TOKint16), 893 Keyword("ushort", TOKuns16), 894 Keyword("int", TOKint32), 895 Keyword("uint", TOKuns32), 896 Keyword("long", TOKint64), 897 Keyword("ulong", TOKuns64), 898 Keyword("cent", TOKint128), 899 Keyword("ucent", TOKuns128), 900 Keyword("float", TOKfloat32), 901 Keyword("double", TOKfloat64), 902 Keyword("real", TOKfloat80), 903 Keyword("bool", TOKbool), 904 Keyword("char", TOKchar), 905 Keyword("wchar", TOKwchar), 906 Keyword("dchar", TOKdchar), 907 Keyword("ifloat", TOKimaginary32), 908 Keyword("idouble", TOKimaginary64), 909 Keyword("ireal", TOKimaginary80), 910 Keyword("cfloat", TOKcomplex32), 911 Keyword("cdouble", TOKcomplex64), 912 Keyword("creal", TOKcomplex80), 913 Keyword("delegate", TOKdelegate), 914 Keyword("function", TOKfunction), 915 Keyword("is", TOKis), 916 Keyword("if", TOKif), 917 Keyword("else", TOKelse), 918 Keyword("while", TOKwhile), 919 Keyword("for", TOKfor), 920 Keyword("do", TOKdo), 921 Keyword("switch", TOKswitch), 922 Keyword("case", TOKcase), 923 Keyword("default", TOKdefault), 924 Keyword("break", TOKbreak), 925 Keyword("continue", TOKcontinue), 926 Keyword("synchronized", TOKsynchronized), 927 Keyword("return", TOKreturn), 928 Keyword("goto", TOKgoto), 929 Keyword("try", TOKtry), 930 Keyword("catch", TOKcatch), 931 Keyword("finally", TOKfinally), 932 Keyword("with", TOKwith), 933 Keyword("asm", TOKasm), 934 Keyword("foreach", TOKforeach), 935 Keyword("foreach_reverse", TOKforeach_reverse), 936 Keyword("scope", TOKscope), 937 Keyword("struct", TOKstruct), 938 Keyword("class", TOKclass), 939 Keyword("interface", TOKinterface), 940 Keyword("union", TOKunion), 941 Keyword("enum", TOKenum), 942 Keyword("import", TOKimport), 943 Keyword("mixin", TOKmixin), 944 Keyword("static", TOKstatic), 945 Keyword("final", TOKfinal), 946 Keyword("const", TOKconst), 947 Keyword("typedef", TOKtypedef), 948 Keyword("alias", TOKalias), 949 Keyword("override", TOKoverride), 950 Keyword("abstract", TOKabstract), 951 Keyword("volatile", TOKvolatile), 952 Keyword("debug", TOKdebug), 953 Keyword("deprecated", TOKdeprecated), 954 Keyword("in", TOKin), 955 Keyword("out", TOKout), 956 Keyword("inout", TOKinout), 957 Keyword("lazy", TOKlazy), 958 Keyword("auto", TOKauto), 959 Keyword("align", TOKalign), 960 Keyword("extern", TOKextern), 961 Keyword("private", TOKprivate), 962 Keyword("package", TOKpackage), 963 Keyword("protected", TOKprotected), 964 Keyword("public", TOKpublic), 965 Keyword("export", TOKexport), 966 Keyword("body", TOKbody), 967 Keyword("invariant", TOKinvariant), 968 Keyword("unittest", TOKunittest), 969 Keyword("version", TOKversion), 970 Keyword("__argTypes", TOKargTypes), 971 Keyword("__parameters", TOKparameters), 972 Keyword("ref", TOKref), 973 Keyword("macro", TOKmacro), 974 Keyword("pure", TOKpure), 975 Keyword("nothrow", TOKnothrow), 976 Keyword("__gshared", TOKgshared), 977 Keyword("__traits", TOKtraits), 978 Keyword("__vector", TOKvector), 979 Keyword("__overloadset", TOKoverloadset), 980 Keyword("__FILE__", TOKfile), 981 Keyword("__LINE__", TOKline), 982 Keyword("__MODULE__", TOKmodulestring), 983 Keyword("__FUNCTION__", TOKfuncstring), 984 Keyword("__PRETTY_FUNCTION__", TOKprettyfunc), 985 Keyword("shared", TOKshared), 986 Keyword("immutable", TOKimmutable), 987 Keyword(null, TOKreserved) 988 ];