Merge #5930

5930: Migrate to the latest Semantic Tokens Proposal for LSP 3.16 r=matklad a=kjeremy

This stabilizes call hierarchy and semantic tokens features on the client side and changes the server-side semantic tokens protocol to match the latest proposal for 3.16.

The server-side change will break clients depending on the earlier semantic tokens draft.

Fixes #4942 

Co-authored-by: kjeremy <[email protected]>

Author: bors[bot]

Cargo.lock

@@ -21,7 +21,7 @@ env_logger = { version = "0.7.1", default-features = false }
 itertools = "0.9.0"
 jod-thread = "0.1.0"
 log = "0.4.8"
-lsp-types = { version = "0.79.0", features = ["proposed"] }
+lsp-types = { version = "0.80.0", features = ["proposed"] }
 parking_lot = "0.11.0"
 pico-args = "0.3.1"
 oorandom = "11.1.2"

crates/rust-analyzer/Cargo.toml

@@ -6,7 +6,7 @@ use lsp_types::{
     CodeActionProviderCapability, CodeLensOptions, CompletionOptions,
     DocumentOnTypeFormattingOptions, FoldingRangeProviderCapability, HoverProviderCapability,
     ImplementationProviderCapability, RenameOptions, RenameProviderCapability, SaveOptions,
-    SelectionRangeProviderCapability, SemanticTokensDocumentProvider, SemanticTokensLegend,
+    SelectionRangeProviderCapability, SemanticTokensFullOptions, SemanticTokensLegend,
     SemanticTokensOptions, ServerCapabilities, SignatureHelpOptions, TextDocumentSyncCapability,
     TextDocumentSyncKind, TextDocumentSyncOptions, TypeDefinitionProviderCapability,
     WorkDoneProgressOptions,
@@ -76,10 +76,8 @@ pub fn server_capabilities(client_caps: &ClientCapabilities) -> ServerCapabiliti
                     token_modifiers: semantic_tokens::SUPPORTED_MODIFIERS.to_vec(),
                 },
 
-                document_provider: Some(SemanticTokensDocumentProvider::Edits {
-                    edits: Some(true),
-                }),
-                range_provider: Some(true),
+                full: Some(SemanticTokensFullOptions::Delta { delta: Some(true) }),
+                range: Some(true),
                 work_done_progress_options: Default::default(),
             }
             .into(),

crates/rust-analyzer/src/caps.rs

@@ -17,8 +17,8 @@ use lsp_types::{
     CallHierarchyOutgoingCall, CallHierarchyOutgoingCallsParams, CallHierarchyPrepareParams,
     CodeActionKind, CodeLens, Command, CompletionItem, Diagnostic, DocumentFormattingParams,
     DocumentHighlight, DocumentSymbol, FoldingRange, FoldingRangeParams, HoverContents, Location,
-    Position, PrepareRenameResponse, Range, RenameParams, SemanticTokensEditResult,
-    SemanticTokensEditsParams, SemanticTokensParams, SemanticTokensRangeParams,
+    Position, PrepareRenameResponse, Range, RenameParams, SemanticTokensDeltaParams,
+    SemanticTokensFullDeltaResult, SemanticTokensParams, SemanticTokensRangeParams,
     SemanticTokensRangeResult, SemanticTokensResult, SymbolInformation, SymbolTag,
     TextDocumentIdentifier, Url, WorkspaceEdit,
 };
@@ -1171,11 +1171,11 @@ pub(crate) fn handle_call_hierarchy_outgoing(
     Ok(Some(res))
 }
 
-pub(crate) fn handle_semantic_tokens(
+pub(crate) fn handle_semantic_tokens_full(
     snap: GlobalStateSnapshot,
     params: SemanticTokensParams,
 ) -> Result<Option<SemanticTokensResult>> {
-    let _p = profile::span("handle_semantic_tokens");
+    let _p = profile::span("handle_semantic_tokens_full");
 
     let file_id = from_proto::file_id(&snap, &params.text_document.uri)?;
     let text = snap.analysis.file_text(file_id)?;
@@ -1190,11 +1190,11 @@ pub(crate) fn handle_semantic_tokens(
     Ok(Some(semantic_tokens.into()))
 }
 
-pub(crate) fn handle_semantic_tokens_edits(
+pub(crate) fn handle_semantic_tokens_full_delta(
     snap: GlobalStateSnapshot,
-    params: SemanticTokensEditsParams,
-) -> Result<Option<SemanticTokensEditResult>> {
-    let _p = profile::span("handle_semantic_tokens_edits");
+    params: SemanticTokensDeltaParams,
+) -> Result<Option<SemanticTokensFullDeltaResult>> {
+    let _p = profile::span("handle_semantic_tokens_full_delta");
 
     let file_id = from_proto::file_id(&snap, &params.text_document.uri)?;
     let text = snap.analysis.file_text(file_id)?;
@@ -1209,9 +1209,9 @@ pub(crate) fn handle_semantic_tokens_edits(
 
     if let Some(prev_id) = &cached_tokens.result_id {
         if *prev_id == params.previous_result_id {
-            let edits = to_proto::semantic_token_edits(&cached_tokens, &semantic_tokens);
+            let delta = to_proto::semantic_token_delta(&cached_tokens, &semantic_tokens);
             *cached_tokens = semantic_tokens;
-            return Ok(Some(edits.into()));
+            return Ok(Some(delta.into()));
         }
     }
 

crates/rust-analyzer/src/handlers.rs

@@ -407,9 +407,11 @@ impl GlobalState {
             .on::<lsp_types::request::CallHierarchyOutgoingCalls>(
                 handlers::handle_call_hierarchy_outgoing,
             )?
-            .on::<lsp_types::request::SemanticTokensRequest>(handlers::handle_semantic_tokens)?
-            .on::<lsp_types::request::SemanticTokensEditsRequest>(
-                handlers::handle_semantic_tokens_edits,
+            .on::<lsp_types::request::SemanticTokensFullRequest>(
+                handlers::handle_semantic_tokens_full,
+            )?
+            .on::<lsp_types::request::SemanticTokensFullDeltaRequest>(
+                handlers::handle_semantic_tokens_full_delta,
             )?
             .on::<lsp_types::request::SemanticTokensRangeRequest>(
                 handlers::handle_semantic_tokens_range,

crates/rust-analyzer/src/main_loop.rs

@@ -24,14 +24,14 @@ macro_rules! define_semantic_token_types {
             SemanticTokenType::CLASS,
             SemanticTokenType::INTERFACE,
             SemanticTokenType::ENUM,
+            SemanticTokenType::ENUM_MEMBER,
             SemanticTokenType::TYPE_PARAMETER,
             SemanticTokenType::FUNCTION,
             SemanticTokenType::MEMBER,
             SemanticTokenType::PROPERTY,
             SemanticTokenType::MACRO,
             SemanticTokenType::VARIABLE,
             SemanticTokenType::PARAMETER,
-            SemanticTokenType::LABEL,
             $($ident),*
         ];
     };
@@ -41,7 +41,6 @@ define_semantic_token_types![
     (ATTRIBUTE, "attribute"),
     (BOOLEAN, "boolean"),
     (BUILTIN_TYPE, "builtinType"),
-    (ENUM_MEMBER, "enumMember"),
     (ESCAPE_SEQUENCE, "escapeSequence"),
     (FORMAT_SPECIFIER, "formatSpecifier"),
     (GENERIC, "generic"),

crates/rust-analyzer/src/semantic_tokens.rs

@@ -334,13 +334,13 @@ pub(crate) fn semantic_tokens(
     builder.build()
 }
 
-pub(crate) fn semantic_token_edits(
+pub(crate) fn semantic_token_delta(
     previous: &lsp_types::SemanticTokens,
     current: &lsp_types::SemanticTokens,
-) -> lsp_types::SemanticTokensEdits {
+) -> lsp_types::SemanticTokensDelta {
     let result_id = current.result_id.clone();
     let edits = semantic_tokens::diff_tokens(&previous.data, &current.data);
-    lsp_types::SemanticTokensEdits { result_id, edits }
+    lsp_types::SemanticTokensDelta { result_id, edits }
 }
 
 fn semantic_token_type_and_modifiers(
@@ -369,7 +369,7 @@ fn semantic_token_type_and_modifiers(
             mods |= lsp_types::SemanticTokenModifier::STATIC;
             lsp_types::SemanticTokenType::VARIABLE
         }
-        HighlightTag::EnumVariant => semantic_tokens::ENUM_MEMBER,
+        HighlightTag::EnumVariant => lsp_types::SemanticTokenType::ENUM_MEMBER,
         HighlightTag::Macro => lsp_types::SemanticTokenType::MACRO,
         HighlightTag::ValueParam => lsp_types::SemanticTokenType::PARAMETER,
         HighlightTag::Local => lsp_types::SemanticTokenType::VARIABLE,

crates/rust-analyzer/src/to_proto.rs

@@ -36,7 +36,7 @@
     },
     "dependencies": {
         "node-fetch": "^2.6.0",
-        "vscode-languageclient": "7.0.0-next.1"
+        "vscode-languageclient": "7.0.0-next.9"
     },
     "devDependencies": {
         "@rollup/plugin-commonjs": "^13.0.2",

editors/code/package-lock.json

@@ -1,10 +1,7 @@
-import * as lc from 'vscode-languageclient';
+import * as lc from 'vscode-languageclient/node';
 import * as vscode from 'vscode';
 import * as ra from '../src/lsp_ext';
-import * as Is from 'vscode-languageclient/lib/utils/is';
-
-import { CallHierarchyFeature } from 'vscode-languageclient/lib/callHierarchy.proposed';
-import { SemanticTokensFeature } from 'vscode-languageclient/lib/semanticTokens.proposed';
+import * as Is from 'vscode-languageclient/lib/common/utils/is';
 import { assert } from './util';
 
 function renderCommand(cmd: ra.CommandLink) {
@@ -57,7 +54,7 @@ export function createClient(serverPath: string, cwd: string): lc.LanguageClient
                         return hover;
                     },
                     (error) => {
-                        client.logFailedRequest(lc.HoverRequest.type, error);
+                        client.handleFailedRequest(lc.HoverRequest.type, error, null);
                         return Promise.resolve(null);
                     });
             },
@@ -140,12 +137,6 @@ export function createClient(serverPath: string, cwd: string): lc.LanguageClient
     );
 
     // To turn on all proposed features use: client.registerProposedFeatures();
-    // Here we want to enable CallHierarchyFeature and SemanticTokensFeature
-    // since they are available on stable.
-    // Note that while these features are stable in vscode their LSP protocol
-    // implementations are still in the "proposed" category for 3.16.
-    client.registerFeature(new CallHierarchyFeature(client));
-    client.registerFeature(new SemanticTokensFeature(client));
     client.registerFeature(new ExperimentalFeatures());
 
     return client;

editors/code/package.json

@@ -63,7 +63,7 @@ export function memoryUsage(ctx: Ctx): Cmd {
         provideTextDocumentContent(_uri: vscode.Uri): vscode.ProviderResult<string> {
             if (!vscode.window.activeTextEditor) return '';
 
-            return ctx.client.sendRequest(ra.memoryUsage, null).then((mem) => {
+            return ctx.client.sendRequest(ra.memoryUsage, null).then((mem: any) => {
                 return 'Per-query memory usage:\n' + mem + '\n(note: database has been cleared)';
             });
         }
@@ -121,7 +121,7 @@ export function joinLines(ctx: Ctx): Cmd {
             textDocument: { uri: editor.document.uri.toString() },
         });
         editor.edit((builder) => {
-            client.protocol2CodeConverter.asTextEdits(items).forEach((edit) => {
+            client.protocol2CodeConverter.asTextEdits(items).forEach((edit: any) => {
                 builder.replace(edit.range, edit.newText);
             });
         });
@@ -140,8 +140,8 @@ export function onEnter(ctx: Ctx): Cmd {
             position: client.code2ProtocolConverter.asPosition(
                 editor.selection.active,
             ),
-        }).catch(_error => {
-            // client.logFailedRequest(OnEnterRequest.type, error);
+        }).catch((_error: any) => {
+            // client.handleFailedRequest(OnEnterRequest.type, error, null);
             return null;
         });
         if (!lcEdits) return false;

editors/code/src/client.ts

@@ -1,5 +1,5 @@
 import * as vscode from 'vscode';
-import * as lc from 'vscode-languageclient';
+import * as lc from 'vscode-languageclient/node';
 import * as ra from './lsp_ext';
 
 import { Config } from './config';

editors/code/src/commands.ts

@@ -1,4 +1,4 @@
-import * as lc from "vscode-languageclient";
+import * as lc from "vscode-languageclient/node";
 import * as vscode from "vscode";
 import { strict as nativeAssert } from "assert";
 import { spawnSync } from "child_process";