1 /** 2 * This file is part of DCD, a development tool for the D programming language. 3 * Copyright (C) 2014 Brian Schott 4 * 5 * This program is free software: you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation, either version 3 of the License, or 8 * (at your option) any later version. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 */ 18 19 module dcd.server.autocomplete.util; 20 21 import std.algorithm; 22 import stdx.allocator; 23 import std.experimental.logger; 24 import std.range; 25 import std..string; 26 import std.typecons; 27 28 import dcd.common.messages; 29 30 import dparse.lexer; 31 import dparse.rollback_allocator; 32 33 import dsymbol.builtin.names; 34 import dsymbol.builtin.symbols; 35 import dsymbol.conversion; 36 import dsymbol.modulecache; 37 import dsymbol.scope_; 38 import dsymbol.string_interning; 39 import dsymbol.symbol; 40 41 enum ImportKind : ubyte 42 { 43 selective, 44 normal, 45 neither 46 } 47 48 struct SymbolStuff 49 { 50 void destroy() 51 { 52 typeid(DSymbol).destroy(symbol); 53 typeid(Scope).destroy(scope_); 54 } 55 56 DSymbol*[] symbols; 57 DSymbol* symbol; 58 Scope* scope_; 59 } 60 61 /** 62 * Params: 63 * completionType = the completion type being requested 64 * kind = the kind of the current item in the completion chain 65 * current = the index of the current item in the symbol chain 66 * max = the number of items in the symbol chain 67 * Returns: 68 * true if the symbol should be swapped with its type field 69 */ 70 bool shouldSwapWithType(CompletionType completionType, CompletionKind kind, 71 size_t current, size_t max) pure nothrow @safe 72 { 73 // packages never have types, so always return false 74 if (kind == CompletionKind.packageName 75 || kind == CompletionKind.className 76 || kind == CompletionKind.structName 77 || kind == CompletionKind.interfaceName 78 || kind == CompletionKind.enumName 79 || kind == CompletionKind.unionName 80 || kind == CompletionKind.templateName 81 || kind == CompletionKind.keyword) 82 { 83 return false; 84 } 85 // Swap out every part of a chain with its type except the last part 86 if (current < max) 87 return true; 88 // Only swap out types for these kinds 89 immutable bool isInteresting = 90 kind == CompletionKind.variableName 91 || kind == CompletionKind.memberVariableName 92 || kind == CompletionKind.importSymbol 93 || kind == CompletionKind.aliasName 94 || kind == CompletionKind.enumMember 95 || kind == CompletionKind.functionName; 96 return isInteresting && (completionType == CompletionType.identifiers 97 || (completionType == completionType.calltips && kind == CompletionKind.variableName)) ; 98 } 99 100 istring stringToken()(auto ref const Token a) 101 { 102 return internString(a.text is null ? str(a.type) : a.text); 103 } 104 105 //void dumpTokens(const Token[] tokens) 106 //{ 107 //foreach (t; tokens) 108 //writeln(t.line, ":", t.column, " ", stringToken(t)); 109 //} 110 111 /** 112 * Params: 113 * sourceCode = the source code of the file being edited 114 * cursorPosition = the cursor position in bytes 115 * Returns: 116 * a sorted range of tokens before the cursor position 117 */ 118 auto getTokensBeforeCursor(const(ubyte[]) sourceCode, size_t cursorPosition, 119 ref StringCache cache, out const(Token)[] tokenArray) 120 { 121 LexerConfig config; 122 config.fileName = ""; 123 tokenArray = getTokensForParser(cast(ubyte[]) sourceCode, config, &cache); 124 auto sortedTokens = assumeSorted(tokenArray); 125 return sortedTokens.lowerBound(cast(size_t) cursorPosition); 126 } 127 128 /** 129 * Params: 130 * request = the autocompletion request 131 * type = type the autocompletion type 132 * Returns: 133 * all symbols that should be considered for the autocomplete list based on 134 * the request's source code, cursor position, and completion type. 135 */ 136 SymbolStuff getSymbolsForCompletion(const AutocompleteRequest request, 137 const CompletionType type, IAllocator allocator, RollbackAllocator* rba, 138 ref StringCache cache, ref ModuleCache moduleCache) 139 { 140 const(Token)[] tokenArray; 141 auto beforeTokens = getTokensBeforeCursor(request.sourceCode, 142 request.cursorPosition, cache, tokenArray); 143 ScopeSymbolPair pair = generateAutocompleteTrees(tokenArray, allocator, 144 rba, request.cursorPosition, moduleCache); 145 auto expression = getExpression(beforeTokens); 146 return SymbolStuff(getSymbolsByTokenChain(pair.scope_, expression, 147 request.cursorPosition, type), pair.symbol, pair.scope_); 148 } 149 150 bool isSliceExpression(T)(T tokens, size_t index) 151 { 152 while (index < tokens.length) switch (tokens[index].type) 153 { 154 case tok!"[": 155 tokens.skipParen(index, tok!"[", tok!"]"); 156 break; 157 case tok!"(": 158 tokens.skipParen(index, tok!"(", tok!")"); 159 break; 160 case tok!"]": 161 case tok!"}": 162 return false; 163 case tok!"..": 164 return true; 165 default: 166 index++; 167 break; 168 } 169 return false; 170 } 171 172 /** 173 * 174 */ 175 DSymbol*[] getSymbolsByTokenChain(T)(Scope* completionScope, 176 T tokens, size_t cursorPosition, CompletionType completionType) 177 { 178 //writeln(">>>"); 179 //dumpTokens(tokens.release); 180 //writeln(">>>"); 181 182 // Find the symbol corresponding to the beginning of the chain 183 DSymbol*[] symbols; 184 if (tokens.length == 0) 185 return []; 186 // Recurse in case the symbol chain starts with an expression in parens 187 // e.g. (a.b!c).d 188 if (tokens[0] == tok!"(") 189 { 190 size_t j; 191 tokens.skipParen(j, tok!"(", tok!")"); 192 symbols = getSymbolsByTokenChain(completionScope, tokens[1 .. j], 193 cursorPosition, completionType); 194 tokens = tokens[j + 1 .. $]; 195 //writeln("<<<"); 196 //dumpTokens(tokens.release); 197 //writeln("<<<"); 198 if (tokens.length == 0) // workaround (#371) 199 return []; 200 } 201 else if (tokens[0] == tok!"." && tokens.length > 1) 202 { 203 tokens = tokens[1 .. $]; 204 if (tokens.length == 0) // workaround (#371) 205 return []; 206 symbols = completionScope.getSymbolsAtGlobalScope(stringToken(tokens[0])); 207 } 208 else 209 symbols = completionScope.getSymbolsByNameAndCursor(stringToken(tokens[0]), cursorPosition); 210 211 if (symbols.length == 0) 212 { 213 //TODO: better bugfix for issue #368, see test case 52 or pull #371 214 if (tokens.length) 215 warning("Could not find declaration of ", stringToken(tokens[0]), 216 " from position ", cursorPosition); 217 else assert(0, "internal error"); 218 return []; 219 } 220 221 // If the `symbols` array contains functions, and one of them returns 222 // void and the others do not, this is a property function. For the 223 // purposes of chaining auto-complete we want to ignore the one that 224 // returns void. This is a no-op if we are getting doc comments. 225 void filterProperties() @nogc @safe 226 { 227 if (symbols.length == 0 || completionType == CompletionType.ddoc) 228 return; 229 if (symbols[0].kind == CompletionKind.functionName 230 || symbols[0].qualifier == SymbolQualifier.func) 231 { 232 int voidRets = 0; 233 int nonVoidRets = 0; 234 size_t firstNonVoidIndex = size_t.max; 235 foreach (i, sym; symbols) 236 { 237 if (sym.type is null) 238 return; 239 if (&sym.type.name[0] == &getBuiltinTypeName(tok!"void")[0]) 240 voidRets++; 241 else 242 { 243 nonVoidRets++; 244 firstNonVoidIndex = min(firstNonVoidIndex, i); 245 } 246 } 247 if (voidRets > 0 && nonVoidRets > 0) 248 symbols = symbols[firstNonVoidIndex .. $]; 249 } 250 } 251 252 filterProperties(); 253 254 if (shouldSwapWithType(completionType, symbols[0].kind, 0, tokens.length - 1)) 255 { 256 //trace("Swapping types"); 257 if (symbols.length == 0 || symbols[0].type is null || symbols[0].type is symbols[0]) 258 return []; 259 else if (symbols[0].type.kind == CompletionKind.functionName) 260 { 261 if (symbols[0].type.type is null) 262 symbols = []; 263 else 264 symbols = [symbols[0].type.type]; 265 } 266 else 267 symbols = [symbols[0].type]; 268 } 269 270 loop: for (size_t i = 1; i < tokens.length; i++) 271 { 272 void skip(IdType open, IdType close) 273 { 274 tokens.skipParen(i, open, close); 275 } 276 277 switch (tokens[i].type) 278 { 279 case tok!"int": 280 case tok!"uint": 281 case tok!"long": 282 case tok!"ulong": 283 case tok!"char": 284 case tok!"wchar": 285 case tok!"dchar": 286 case tok!"bool": 287 case tok!"byte": 288 case tok!"ubyte": 289 case tok!"short": 290 case tok!"ushort": 291 case tok!"cent": 292 case tok!"ucent": 293 case tok!"float": 294 case tok!"ifloat": 295 case tok!"cfloat": 296 case tok!"idouble": 297 case tok!"cdouble": 298 case tok!"double": 299 case tok!"real": 300 case tok!"ireal": 301 case tok!"creal": 302 case tok!"this": 303 case tok!"super": 304 symbols = symbols[0].getPartsByName(internString(str(tokens[i].type))); 305 if (symbols.length == 0) 306 break loop; 307 break; 308 case tok!"identifier": 309 //trace(symbols[0].qualifier, " ", symbols[0].kind); 310 filterProperties(); 311 312 if (symbols.length == 0) 313 break loop; 314 315 // Use type instead of the symbol itself for certain symbol kinds 316 while (symbols[0].qualifier == SymbolQualifier.func 317 || symbols[0].kind == CompletionKind.functionName 318 || (symbols[0].kind == CompletionKind.moduleName 319 && symbols[0].type !is null && symbols[0].type.kind == CompletionKind.importSymbol) 320 || symbols[0].kind == CompletionKind.importSymbol 321 || symbols[0].kind == CompletionKind.aliasName) 322 { 323 symbols = symbols[0].type is null || symbols[0].type is symbols[0] ? [] : [symbols[0].type]; 324 if (symbols.length == 0) 325 break loop; 326 } 327 328 //trace("looking for ", tokens[i].text, " in ", symbols[0].name); 329 symbols = symbols[0].getPartsByName(internString(tokens[i].text)); 330 //trace("symbols: ", symbols.map!(a => a.name)); 331 filterProperties(); 332 if (symbols.length == 0) 333 { 334 //trace("Couldn't find it."); 335 break loop; 336 } 337 if (shouldSwapWithType(completionType, symbols[0].kind, i, tokens.length - 1)) 338 { 339 symbols = symbols[0].type is null || symbols[0].type is symbols[0] ? [] : [symbols[0].type]; 340 if (symbols.length == 0) 341 break loop; 342 } 343 if ((symbols[0].kind == CompletionKind.aliasName 344 || symbols[0].kind == CompletionKind.moduleName) 345 && (completionType == CompletionType.identifiers 346 || i + 1 < tokens.length)) 347 { 348 symbols = symbols[0].type is null || symbols[0].type is symbols[0] ? [] : [symbols[0].type]; 349 } 350 if (symbols.length == 0) 351 break loop; 352 if (tokens[i].type == tok!"!") 353 { 354 i++; 355 if (tokens[i].type == tok!"(") 356 goto case; 357 else 358 i++; 359 } 360 break; 361 case tok!"(": 362 skip(tok!"(", tok!")"); 363 break; 364 case tok!"[": 365 if (symbols[0].qualifier == SymbolQualifier.array) 366 { 367 skip(tok!"[", tok!"]"); 368 if (!isSliceExpression(tokens, i)) 369 { 370 symbols = symbols[0].type is null || symbols[0].type is symbols[0] ? [] : [symbols[0].type]; 371 if (symbols.length == 0) 372 break loop; 373 } 374 } 375 else if (symbols[0].qualifier == SymbolQualifier.assocArray) 376 { 377 symbols = symbols[0].type is null || symbols[0].type is symbols[0] ? [] : [symbols[0].type]; 378 skip(tok!"[", tok!"]"); 379 } 380 else 381 { 382 skip(tok!"[", tok!"]"); 383 DSymbol*[] overloads; 384 if (isSliceExpression(tokens, i)) 385 overloads = symbols[0].getPartsByName(internString("opSlice")); 386 else 387 overloads = symbols[0].getPartsByName(internString("opIndex")); 388 if (overloads.length > 0) 389 { 390 symbols = overloads[0].type is null ? [] : [overloads[0].type]; 391 } 392 else 393 return []; 394 } 395 break; 396 case tok!".": 397 break; 398 default: 399 break loop; 400 } 401 } 402 return symbols; 403 } 404 405 enum TYPE_IDENT_CASES = q{ 406 case tok!"int": 407 case tok!"uint": 408 case tok!"long": 409 case tok!"ulong": 410 case tok!"char": 411 case tok!"wchar": 412 case tok!"dchar": 413 case tok!"bool": 414 case tok!"byte": 415 case tok!"ubyte": 416 case tok!"short": 417 case tok!"ushort": 418 case tok!"cent": 419 case tok!"ucent": 420 case tok!"float": 421 case tok!"ifloat": 422 case tok!"cfloat": 423 case tok!"idouble": 424 case tok!"cdouble": 425 case tok!"double": 426 case tok!"real": 427 case tok!"ireal": 428 case tok!"creal": 429 case tok!"this": 430 case tok!"super": 431 case tok!"identifier": 432 }; 433 434 enum STRING_LITERAL_CASES = q{ 435 case tok!"stringLiteral": 436 case tok!"wstringLiteral": 437 case tok!"dstringLiteral": 438 }; 439 440 enum TYPE_IDENT_AND_LITERAL_CASES = TYPE_IDENT_CASES ~ STRING_LITERAL_CASES; 441 442 /** 443 * 444 */ 445 T getExpression(T)(T beforeTokens) 446 { 447 enum EXPRESSION_LOOP_BREAK = q{ 448 if (i + 1 < beforeTokens.length) switch (beforeTokens[i + 1].type) 449 { 450 mixin (TYPE_IDENT_AND_LITERAL_CASES); 451 i++; 452 break expressionLoop; 453 default: 454 break; 455 } 456 }; 457 458 if (beforeTokens.length == 0) 459 return beforeTokens[0 .. 0]; 460 size_t i = beforeTokens.length - 1; 461 size_t sliceEnd = beforeTokens.length; 462 IdType open; 463 IdType close; 464 uint skipCount = 0; 465 466 expressionLoop: while (true) 467 { 468 switch (beforeTokens[i].type) 469 { 470 case tok!"import": 471 i++; 472 break expressionLoop; 473 mixin (TYPE_IDENT_AND_LITERAL_CASES); 474 mixin (EXPRESSION_LOOP_BREAK); 475 break; 476 case tok!".": 477 break; 478 case tok!")": 479 open = tok!")"; 480 close = tok!"("; 481 goto skip; 482 case tok!"]": 483 open = tok!"]"; 484 close = tok!"["; 485 skip: 486 mixin (EXPRESSION_LOOP_BREAK); 487 immutable bookmark = i; 488 i = beforeTokens.skipParenReverse(i, open, close); 489 490 skipCount++; 491 492 // check the current token after skipping parens to the left. 493 // if it's a loop keyword, pretend we never skipped the parens. 494 if (i > 0) switch (beforeTokens[i - 1].type) 495 { 496 case tok!"scope": 497 case tok!"if": 498 case tok!"while": 499 case tok!"for": 500 case tok!"foreach": 501 case tok!"foreach_reverse": 502 case tok!"do": 503 case tok!"cast": 504 case tok!"catch": 505 i = bookmark + 1; 506 break expressionLoop; 507 case tok!"!": 508 if (skipCount == 1) 509 { 510 sliceEnd = i - 1; 511 i -= 2; 512 } 513 break expressionLoop; 514 default: 515 break; 516 } 517 break; 518 default: 519 i++; 520 break expressionLoop; 521 } 522 if (i == 0) 523 break; 524 else 525 i--; 526 } 527 return beforeTokens[i .. sliceEnd]; 528 } 529 530 /** 531 * Determines if an import is selective, whole-module, or neither. 532 */ 533 ImportKind determineImportKind(T)(T tokens) 534 { 535 assert (tokens.length > 1); 536 size_t i = tokens.length - 1; 537 if (!(tokens[i] == tok!":" || tokens[i] == tok!"," || tokens[i] == tok!"." 538 || tokens[i] == tok!"identifier")) 539 return ImportKind.neither; 540 bool foundColon = false; 541 while (true) switch (tokens[i].type) 542 { 543 case tok!":": 544 foundColon = true; 545 goto case; 546 case tok!"identifier": 547 case tok!"=": 548 case tok!".": 549 case tok!",": 550 if (i == 0) 551 return ImportKind.neither; 552 else 553 i--; 554 break; 555 case tok!"import": 556 return foundColon ? ImportKind.selective : ImportKind.normal; 557 default: 558 return ImportKind.neither; 559 } 560 return ImportKind.neither; 561 } 562 563 unittest 564 { 565 import std.stdio : writeln; 566 567 Token[] t = [ 568 Token(tok!"import"), Token(tok!"identifier"), Token(tok!"."), 569 Token(tok!"identifier"), Token(tok!":"), Token(tok!"identifier"), Token(tok!",") 570 ]; 571 assert(determineImportKind(t) == ImportKind.selective); 572 Token[] t2; 573 t2 ~= Token(tok!"else"); 574 t2 ~= Token(tok!":"); 575 assert(determineImportKind(t2) == ImportKind.neither); 576 writeln("Unittest for determineImportKind() passed"); 577 } 578 579 bool isUdaExpression(T)(ref T tokens) 580 { 581 bool result; 582 ptrdiff_t skip; 583 auto i = cast(ptrdiff_t) tokens.length - 2; 584 585 if (i < 1) 586 return result; 587 588 // skips the UDA ctor 589 if (tokens[i].type == tok!")") 590 { 591 ++skip; 592 --i; 593 while (i >= 2) 594 { 595 skip += tokens[i].type == tok!")"; 596 skip -= tokens[i].type == tok!"("; 597 --i; 598 if (skip == 0) 599 { 600 // @UDA!(TemplateParameters)(FunctionParameters) 601 if (i > 3 && tokens[i].type == tok!"!" && tokens[i-1].type == tok!")") 602 { 603 skip = 1; 604 i -= 2; 605 continue; 606 } 607 else break; 608 } 609 } 610 } 611 612 if (skip == 0) 613 { 614 // @UDA!SingleTemplateParameter 615 if (i > 2 && tokens[i].type == tok!"identifier" && tokens[i-1].type == tok!"!") 616 { 617 i -= 2; 618 } 619 620 // @UDA 621 if (i > 0 && tokens[i].type == tok!"identifier" && tokens[i-1].type == tok!"@") 622 { 623 result = true; 624 } 625 } 626 627 return result; 628 } 629 630 /** 631 * Traverses a token slice in reverse to find the opening parentheses or square bracket 632 * that begins the block the last token is in. 633 */ 634 size_t goBackToOpenParen(T)(T beforeTokens) 635 in 636 { 637 assert (beforeTokens.length > 0); 638 } 639 body 640 { 641 size_t i = beforeTokens.length - 1; 642 while (true) switch (beforeTokens[i].type) 643 { 644 case tok!",": 645 case tok!".": 646 case tok!"*": 647 case tok!"&": 648 case tok!"doubleLiteral": 649 case tok!"floatLiteral": 650 case tok!"idoubleLiteral": 651 case tok!"ifloatLiteral": 652 case tok!"intLiteral": 653 case tok!"longLiteral": 654 case tok!"realLiteral": 655 case tok!"irealLiteral": 656 case tok!"uintLiteral": 657 case tok!"ulongLiteral": 658 case tok!"characterLiteral": 659 mixin(TYPE_IDENT_AND_LITERAL_CASES); 660 if (i == 0) 661 return size_t.max; 662 else 663 i--; 664 break; 665 case tok!"(": 666 case tok!"[": 667 return i + 1; 668 case tok!")": 669 i = beforeTokens.skipParenReverseBefore(i, tok!")", tok!"("); 670 break; 671 case tok!"}": 672 i = beforeTokens.skipParenReverseBefore(i, tok!"}", tok!"{"); 673 break; 674 case tok!"]": 675 i = beforeTokens.skipParenReverseBefore(i, tok!"]", tok!"["); 676 break; 677 default: 678 return size_t.max; 679 } 680 } 681 682 /** 683 * Skips blocks of parentheses until the starting block has been closed 684 */ 685 void skipParen(T)(T tokenSlice, ref size_t i, IdType open, IdType close) 686 { 687 if (i >= tokenSlice.length || tokenSlice.length <= 0) 688 return; 689 int depth = 1; 690 while (depth != 0 && i + 1 != tokenSlice.length) 691 { 692 i++; 693 if (tokenSlice[i].type == open) 694 depth++; 695 else if (tokenSlice[i].type == close) 696 depth--; 697 } 698 } 699 700 /** 701 * Skips blocks of parentheses in reverse until the starting block has been opened 702 */ 703 size_t skipParenReverse(T)(T beforeTokens, size_t i, IdType open, IdType close) 704 { 705 if (i == 0) 706 return 0; 707 int depth = 1; 708 while (depth != 0 && i != 0) 709 { 710 i--; 711 if (beforeTokens[i].type == open) 712 depth++; 713 else if (beforeTokens[i].type == close) 714 depth--; 715 } 716 return i; 717 } 718 719 size_t skipParenReverseBefore(T)(T beforeTokens, size_t i, IdType open, IdType close) 720 { 721 i = skipParenReverse(beforeTokens, i, open, close); 722 if (i != 0) 723 i--; 724 return i; 725 } 726 727 /// 728 unittest 729 { 730 Token[] t = [ 731 Token(tok!"identifier"), Token(tok!"identifier"), Token(tok!"("), 732 Token(tok!"identifier"), Token(tok!"("), Token(tok!")"), Token(tok!",") 733 ]; 734 size_t i = t.length - 1; 735 i = skipParenReverse(t, i, tok!")", tok!"("); 736 assert(i == 2); 737 i = t.length - 1; 738 i = skipParenReverseBefore(t, i, tok!")", tok!"("); 739 assert(i == 1); 740 } 741 742 AutocompleteResponse.Completion makeSymbolCompletionInfo(const DSymbol* symbol, char kind) 743 { 744 string definition; 745 if ((kind == CompletionKind.variableName || kind == CompletionKind.memberVariableName) && symbol.type) 746 definition = symbol.type.name ~ ' ' ~ symbol.name; 747 else if (kind == CompletionKind.enumMember) 748 definition = symbol.name; // TODO: add enum value to definition string 749 else 750 definition = symbol.callTip; 751 // TODO: definition strings could include more information, like on classes inheritance 752 return AutocompleteResponse.Completion(symbol.name, kind, definition, 753 symbol.symbolFile, symbol.location, symbol.doc); 754 }