From 4860acd97dd81652b8f3ba0bdadda2f0e05e25a3 Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Mon, 26 Oct 2020 16:34:37 -0700 Subject: [PATCH] Language server middleware stop gap for notebooks (#14522) * Rebase changes onto main * Change how jupyter extension is downloaded * Try a different way * Fix build failure? * Azure Devops is a pain * Missing paren * Just download the file * Wrong way to build * Missed file from old branch * Fix unit tests * Workaround powershell issue * Put in correct folder * Specify full path to vsix * Directory still not found * Building wrong bundle * See why not found * Invalid syntax in yml * Fail on error * VSIX was being cleaned * copy is not a bash command * Add jupyter into requirements --- build/ci/templates/globals.yml | 2 + build/ci/templates/test_phases.yml | 53 ++ build/ci/vscode-python-pr-validation.yaml | 3 + build/smoke-test-requirements.txt | 6 + package.json | 4 +- src/client/activation/jedi/manager.ts | 8 +- .../activation/languageClientMiddleware.ts | 329 ++++----- .../activation/languageServer/manager.ts | 8 +- src/client/activation/node/manager.ts | 8 +- src/client/common/application/notebook.ts | 145 ++++ src/client/common/application/types.ts | 45 +- src/client/common/configSettings.ts | 6 +- src/client/common/serviceRegistry.ts | 3 + src/client/common/startPage/webviewHost.ts | 12 +- .../languageserver/notebookConcatDocument.ts | 140 ++++ .../languageserver/notebookConverter.ts | 625 ++++++++++++++++++ .../languageserver/notebookMiddlewareAddon.ts | 459 +++++++++++++ .../languageServer/manager.unit.test.ts | 9 - .../insiders/languageServer.insiders.test.ts | 110 +++ src/test/smoke/common.ts | 27 + src/test/smoke/datascience.smoke.test.ts | 89 +++ src/test/standardTest.ts | 87 ++- .../smokeTests/definitions.ipynb | 88 +++ 23 files changed, 2043 insertions(+), 223 deletions(-) create mode 100644 build/smoke-test-requirements.txt create mode 100644 src/client/common/application/notebook.ts create mode 100644 src/client/datascience/languageserver/notebookConcatDocument.ts create mode 100644 src/client/datascience/languageserver/notebookConverter.ts create mode 100644 src/client/datascience/languageserver/notebookMiddlewareAddon.ts create mode 100644 src/test/insiders/languageServer.insiders.test.ts create mode 100644 src/test/smoke/datascience.smoke.test.ts create mode 100644 src/testMultiRootWkspc/smokeTests/definitions.ipynb diff --git a/build/ci/templates/globals.yml b/build/ci/templates/globals.yml index b77fa3605082..af885a1bb049 100644 --- a/build/ci/templates/globals.yml +++ b/build/ci/templates/globals.yml @@ -11,3 +11,5 @@ variables: npm_config_cache: $(Pipeline.Workspace)/.npm vmImageMacOS: 'macOS-latest' TS_NODE_FILES: true # Temporarily enabled to allow using types from vscode.proposed.d.ts from ts-node (for tests). + VSIX_NAME_JUPYTER: ms-toolsai-jupyter-insiders.vsix + VSIX_NAME_JUPYTER_ZIP: ms-toolsai-jupyter-insiders.zip diff --git a/build/ci/templates/test_phases.yml b/build/ci/templates/test_phases.yml index 1e366cbb8978..7672a9bc2168 100644 --- a/build/ci/templates/test_phases.yml +++ b/build/ci/templates/test_phases.yml @@ -118,6 +118,12 @@ steps: displayName: 'pip install functional requirements' condition: and(succeeded(), eq(variables['NeedsPythonFunctionalReqs'], 'true')) + # Install smoke test specific requirements + - bash: | + python -m pip install --upgrade -r ./build/smoke-test-requirements.txt + displayName: 'pip install smoke test requirements' + condition: and(succeeded(), contains(variables['TestsToRun'], 'testSmoke')) + # Add CONDA to the path so anaconda works # # This task will only run if variable `NeedsPythonFunctionalReqs` is true. @@ -445,6 +451,33 @@ steps: env: DISPLAY: :10 + # Note this section below should download an extension when this code is moved to github actions + # name: Azure Login + # uses: azure/login@v1 + # with: + # creds: ${{ secrets.AZURE_CREDENTIALS }} + # name: Download JUPYTER VSIX + # run: az storage blob download --file $(VSIX_NAME_JUPYTER) --account-name pvsc --container-name extension-builds-jupyter --name $(VSIX_NAME_JUPYTER) --auth-mode login + # condition: and(succeeded(), or(contains(variables['TestsToRun'], 'testSmoke'), contains(variables['TestsToRun'], 'testInsiders')) + # Otherwise have to run a powershell task + + # Work around https://github.com/actions/virtual-environments/issues/1378 + - bash: | + sudo chmod -R 777 /run/user + displayName: 'Fix powershell problem on Ubuntu 20' + condition: ne(variables['Agent.Os'], 'Windows_NT') + + # Now actually install + - powershell: | + Invoke-WebRequest -Uri https://pvsc.blob.core.windows.net/extension-builds-jupyter/$(VSIX_NAME_JUPYTER) -OutFile "$(Build.SourcesDirectory)/$(VSIX_NAME_JUPYTER_ZIP)" + if (![System.IO.File]::Exists("$(Build.SourcesDirectory)/$(VSIX_NAME_JUPYTER_ZIP)")) { + Write-Warning "Jupyter extension not downloaded" + throw new Exception("Jupyter extension not downloaded") + } + condition: and(succeeded(), or(contains(variables['TestsToRun'], 'testSmoke'), contains(variables['TestsToRun'], 'testInsiders'))) + failOnStderr: true + displayName: 'Download Jupyter VSIX' + # Run the smoke tests. # # This task only runs if the string 'testSmoke' exists in variable `TestsToRun`. @@ -461,11 +494,31 @@ steps: npm run updateBuildNumber -- --buildNumber $BUILD_BUILDID npm run package npx tsc -p ./ + cp $(VSIX_NAME_JUPYTER_ZIP) $(VSIX_NAME_JUPYTER) node --no-force-async-hooks-checks ./out/test/smokeTest.js displayName: 'Run Smoke Tests' condition: and(succeeded(), contains(variables['TestsToRun'], 'testSmoke')) env: DISPLAY: :10 + INSTALL_JUPYTER_EXTENSION: true + + # Run the insiders tests. + # + # This task only runs if the string 'testInsiders' exists in variable `TestsToRun`. + # + - bash: | + npm run prePublish + cp $(VSIX_NAME_JUPYTER_ZIP) $(VSIX_NAME_JUPYTER) + node ./out/test/standardTest.js + displayName: 'Run Tests with Insiders' + condition: and(succeeded(), contains(variables['TestsToRun'], 'testInsiders')) + env: + DISPLAY: :10 + INSTALL_JUPYTER_EXTENSION: true + INSTALL_PYLANCE_EXTENSION: true + VSC_PYTHON_CI_TEST_VSC_CHANNEL: insiders + TEST_FILES_SUFFIX: insiders.test + CODE_TESTS_WORKSPACE: ./src/testMultiRootWkspc/smokeTests - task: PublishBuildArtifacts@1 inputs: diff --git a/build/ci/vscode-python-pr-validation.yaml b/build/ci/vscode-python-pr-validation.yaml index f5787d3fa0cd..ac2fb0d23865 100644 --- a/build/ci/vscode-python-pr-validation.yaml +++ b/build/ci/vscode-python-pr-validation.yaml @@ -52,6 +52,9 @@ stages: 'Smoke': TestsToRun: 'testSmoke' NeedsPythonTestReqs: true + 'Insiders': + TestsToRun: 'testInsiders' + NeedsPythonTestReqs: true pool: vmImage: 'ubuntu-20.04' steps: diff --git a/build/smoke-test-requirements.txt b/build/smoke-test-requirements.txt new file mode 100644 index 000000000000..7d5ac3da00d9 --- /dev/null +++ b/build/smoke-test-requirements.txt @@ -0,0 +1,6 @@ +# List of requirements for smoke tests (they will attempt to run a kernel) +jupyter +numpy +matplotlib +pandas +livelossplot \ No newline at end of file diff --git a/package.json b/package.json index bf9ae138606f..e244f66f9be2 100644 --- a/package.json +++ b/package.json @@ -1935,6 +1935,7 @@ }, "scripts": { "package": "gulp clean && gulp prePublishBundle && vsce package -o ms-python-insiders.vsix", + "prePublish": "gulp clean && gulp prePublishNonBundle", "compile": "tsc -watch -p ./", "compiled": "deemon npm run compile", "kill-compiled": "deemon --kill npm run compile", @@ -1956,7 +1957,8 @@ "testJediLSP": "node ./out/test/languageServers/jedi/lspSetup.js && cross-env CODE_TESTS_WORKSPACE=src/test VSC_PYTHON_CI_TEST_GREP='Language Server:' node ./out/test/testBootstrap.js ./out/test/standardTest.js && node ./out/test/languageServers/jedi/lspTeardown.js", "testMultiWorkspace": "node ./out/test/testBootstrap.js ./out/test/multiRootTest.js", "testPerformance": "node ./out/test/testBootstrap.js ./out/test/performanceTest.js", - "testSmoke": "node ./out/test/smokeTest.js", + "testSmoke": "cross-env INSTALL_JUPYTER_EXTENSION=true \"node ./out/test/smokeTest.js\"", + "testInsiders": "cross-env VSC_PYTHON_CI_TEST_VSC_CHANNEL=insiders INSTALL_PYLANCE_EXTENSION=true TEST_FILES_SUFFIX=insiders.test CODE_TESTS_WORKSPACE=src/testMultiRootWkspc/smokeTests \"node ./out/test/standardTest.js\"", "lint-staged": "node gulpfile.js", "lint": "tslint src/**/*.ts -t verbose", "prettier-fix": "prettier 'src/**/*.ts*' --write && prettier 'build/**/*.js' --write", diff --git a/src/client/activation/jedi/manager.ts b/src/client/activation/jedi/manager.ts index 9ae8ac8692dc..ee19d1f428f9 100644 --- a/src/client/activation/jedi/manager.ts +++ b/src/client/activation/jedi/manager.ts @@ -8,7 +8,7 @@ import { inject, injectable, named } from 'inversify'; import { ICommandManager } from '../../common/application/types'; import { traceDecorators } from '../../common/logger'; -import { IConfigurationService, IDisposable, IExperimentsManager, Resource } from '../../common/types'; +import { IDisposable, Resource } from '../../common/types'; import { debounceSync } from '../../common/utils/decorators'; import { EXTENSION_ROOT_DIR } from '../../constants'; import { IServiceContainer } from '../../ioc/types'; @@ -39,8 +39,6 @@ export class JediLanguageServerManager implements ILanguageServerManager { @inject(ILanguageServerAnalysisOptions) @named(LanguageServerType.Jedi) private readonly analysisOptions: ILanguageServerAnalysisOptions, - @inject(IExperimentsManager) private readonly experimentsManager: IExperimentsManager, - @inject(IConfigurationService) private readonly configService: IConfigurationService, @inject(ICommandManager) commandManager: ICommandManager ) { this.disposables.push( @@ -128,9 +126,9 @@ export class JediLanguageServerManager implements ILanguageServerManager { const options = await this.analysisOptions.getAnalysisOptions(); options.middleware = this.middleware = new LanguageClientMiddleware( - this.experimentsManager, - this.configService, + this.serviceContainer, LanguageServerType.Jedi, + () => this.languageServerProxy?.languageClient, this.lsVersion ); diff --git a/src/client/activation/languageClientMiddleware.ts b/src/client/activation/languageClientMiddleware.ts index 2232af85e039..961e5191372f 100644 --- a/src/client/activation/languageClientMiddleware.ts +++ b/src/client/activation/languageClientMiddleware.ts @@ -4,10 +4,8 @@ import * as path from 'path'; import { CancellationToken, CodeAction, - CodeActionContext, CodeLens, Command, - CompletionContext, CompletionItem, Declaration as VDeclaration, Definition, @@ -16,16 +14,10 @@ import { DocumentHighlight, DocumentLink, DocumentSymbol, - FormattingOptions, Location, - Position, - Position as VPosition, ProviderResult, Range, - SignatureHelp, - SignatureHelpContext, SymbolInformation, - TextDocument, TextEdit, Uri, WorkspaceEdit @@ -35,34 +27,20 @@ import { ConfigurationRequest, HandleDiagnosticsSignature, HandlerResult, + LanguageClient, Middleware, - PrepareRenameSignature, - ProvideCodeActionsSignature, - ProvideCodeLensesSignature, - ProvideCompletionItemsSignature, - ProvideDefinitionSignature, - ProvideDocumentFormattingEditsSignature, - ProvideDocumentHighlightsSignature, - ProvideDocumentLinksSignature, - ProvideDocumentRangeFormattingEditsSignature, - ProvideDocumentSymbolsSignature, - ProvideHoverSignature, - ProvideOnTypeFormattingEditsSignature, - ProvideReferencesSignature, - ProvideRenameEditsSignature, - ProvideSignatureHelpSignature, - ProvideWorkspaceSymbolsSignature, - ResolveCodeLensSignature, - ResolveCompletionItemSignature, - ResolveDocumentLinkSignature, ResponseError } from 'vscode-languageclient/node'; +import { IJupyterExtensionDependencyManager, IVSCodeNotebook } from '../common/application/types'; -import { ProvideDeclarationSignature } from 'vscode-languageclient/lib/common/declaration'; -import { HiddenFilePrefix } from '../common/constants'; +import { HiddenFilePrefix, PYTHON_LANGUAGE } from '../common/constants'; import { CollectLSRequestTiming, CollectNodeLSRequestTiming } from '../common/experiments/groups'; -import { IConfigurationService, IExperimentsManager } from '../common/types'; +import { IFileSystem } from '../common/platform/types'; +import { IConfigurationService, IDisposableRegistry, IExperimentsManager, IExtensions } from '../common/types'; +import { isThenable } from '../common/utils/async'; import { StopWatch } from '../common/utils/stopWatch'; +import { NotebookMiddlewareAddon } from '../datascience/languageserver/notebookMiddlewareAddon'; +import { IServiceContainer } from '../ioc/types'; import { sendTelemetryEvent } from '../telemetry'; import { EventName } from '../telemetry/constants'; import { LanguageServerType } from './types'; @@ -91,6 +69,8 @@ export class LanguageClientMiddleware implements Middleware { token: CancellationToken, next: ConfigurationRequest.HandlerSignature ): HandlerResult => { + const configService = this.serviceContainer.get(IConfigurationService); + // Hand-collapse "Thenable | Thenable | Thenable" into just "Thenable" to make TS happy. const result: any[] | ResponseError | Thenable> = next(params, token); @@ -104,7 +84,7 @@ export class LanguageClientMiddleware implements Middleware { params.items.forEach((item, i) => { if (item.section === 'python') { const uri = item.scopeUri ? Uri.parse(item.scopeUri) : undefined; - settings[i].pythonPath = this.configService.getSettings(uri).pythonPath; + settings[i].pythonPath = configService.getSettings(uri).pythonPath; } }); @@ -119,16 +99,23 @@ export class LanguageClientMiddleware implements Middleware { } // tslint:enable:no-any }; + private notebookAddon: NotebookMiddlewareAddon | undefined; private connected = false; // Default to not forwarding to VS code. public constructor( - experimentsManager: IExperimentsManager, - private readonly configService: IConfigurationService, + readonly serviceContainer: IServiceContainer, serverType: LanguageServerType, + getClient: () => LanguageClient | undefined, public readonly serverVersion?: string ) { this.handleDiagnostics = this.handleDiagnostics.bind(this); // VS Code calls function without context. + this.didOpen = this.didOpen.bind(this); + this.didSave = this.didSave.bind(this); + this.didChange = this.didChange.bind(this); + this.didClose = this.didClose.bind(this); + this.willSave = this.willSave.bind(this); + this.willSaveWaitUntil = this.willSaveWaitUntil.bind(this); let group: { experiment: string; control: string } | undefined; @@ -142,10 +129,46 @@ export class LanguageClientMiddleware implements Middleware { return; } - if (!experimentsManager.inExperiment(group.experiment)) { + const experimentsManager = this.serviceContainer.get(IExperimentsManager); + const jupyterDependencyManager = this.serviceContainer.get( + IJupyterExtensionDependencyManager + ); + const notebookApi = this.serviceContainer.get(IVSCodeNotebook); + const disposables = this.serviceContainer.get(IDisposableRegistry) || []; + const extensions = this.serviceContainer.get(IExtensions); + const fileSystem = this.serviceContainer.get(IFileSystem); + + if (experimentsManager && !experimentsManager.inExperiment(group.experiment)) { this.eventName = undefined; experimentsManager.sendTelemetryIfInExperiment(group.control); } + // Enable notebook support if jupyter support is installed + if (jupyterDependencyManager && jupyterDependencyManager.isJupyterExtensionInstalled) { + this.notebookAddon = new NotebookMiddlewareAddon( + notebookApi, + getClient, + fileSystem, + PYTHON_LANGUAGE, + /.*\.ipynb/m + ); + } + disposables.push( + extensions?.onDidChange(() => { + if (jupyterDependencyManager) { + if (this.notebookAddon && !jupyterDependencyManager.isJupyterExtensionInstalled) { + this.notebookAddon = undefined; + } else if (!this.notebookAddon && jupyterDependencyManager.isJupyterExtensionInstalled) { + this.notebookAddon = new NotebookMiddlewareAddon( + notebookApi, + getClient, + fileSystem, + PYTHON_LANGUAGE, + /.*\.ipynb/m + ); + } + } + }) + ); } public connect() { @@ -156,217 +179,161 @@ export class LanguageClientMiddleware implements Middleware { this.connected = false; } + public didChange() { + if (this.connected) { + return this.callNext('didChange', arguments); + } + } + + public didOpen() { + // Special case, open and close happen before we connect. + return this.callNext('didOpen', arguments); + } + + public didClose() { + // Special case, open and close happen before we connect. + return this.callNext('didClose', arguments); + } + + public didSave() { + if (this.connected) { + return this.callNext('didSave', arguments); + } + } + + public willSave() { + if (this.connected) { + return this.callNext('willSave', arguments); + } + } + + public willSaveWaitUntil() { + if (this.connected) { + return this.callNext('willSaveWaitUntil', arguments); + } + } + @captureTelemetryForLSPMethod('textDocument/completion', debounceFrequentCall) - public provideCompletionItem( - document: TextDocument, - position: Position, - context: CompletionContext, - token: CancellationToken, - next: ProvideCompletionItemsSignature - ) { + public provideCompletionItem() { if (this.connected) { - return next(document, position, context, token); + return this.callNext('provideCompletionItem', arguments); } } @captureTelemetryForLSPMethod('textDocument/hover', debounceFrequentCall) - public provideHover( - document: TextDocument, - position: Position, - token: CancellationToken, - next: ProvideHoverSignature - ) { + public provideHover() { if (this.connected) { - return next(document, position, token); + return this.callNext('provideHover', arguments); } } - public handleDiagnostics(uri: Uri, diagnostics: Diagnostic[], next: HandleDiagnosticsSignature) { + public handleDiagnostics(uri: Uri, _diagnostics: Diagnostic[], _next: HandleDiagnosticsSignature) { if (this.connected) { // Skip sending if this is a special file. const filePath = uri.fsPath; const baseName = filePath ? path.basename(filePath) : undefined; if (!baseName || !baseName.startsWith(HiddenFilePrefix)) { - next(uri, diagnostics); + return this.callNext('handleDiagnostics', arguments); } } } @captureTelemetryForLSPMethod('completionItem/resolve', debounceFrequentCall) - public resolveCompletionItem( - item: CompletionItem, - token: CancellationToken, - next: ResolveCompletionItemSignature - ): ProviderResult { + public resolveCompletionItem(): ProviderResult { if (this.connected) { - return next(item, token); + return this.callNext('resolveCompletionItem', arguments); } } @captureTelemetryForLSPMethod('textDocument/signatureHelp', debounceFrequentCall) - public provideSignatureHelp( - document: TextDocument, - position: Position, - context: SignatureHelpContext, - token: CancellationToken, - next: ProvideSignatureHelpSignature - ): ProviderResult { + public provideSignatureHelp() { if (this.connected) { - return next(document, position, context, token); + return this.callNext('provideSignatureHelp', arguments); } } @captureTelemetryForLSPMethod('textDocument/definition', debounceRareCall) - public provideDefinition( - document: TextDocument, - position: Position, - token: CancellationToken, - next: ProvideDefinitionSignature - ): ProviderResult { + public provideDefinition(): ProviderResult { if (this.connected) { - return next(document, position, token); + return this.callNext('provideDefinition', arguments); } } @captureTelemetryForLSPMethod('textDocument/references', debounceRareCall) - public provideReferences( - document: TextDocument, - position: Position, - options: { - includeDeclaration: boolean; - }, - token: CancellationToken, - next: ProvideReferencesSignature - ): ProviderResult { + public provideReferences(): ProviderResult { if (this.connected) { - return next(document, position, options, token); + return this.callNext('provideReferences', arguments); } } - public provideDocumentHighlights( - document: TextDocument, - position: Position, - token: CancellationToken, - next: ProvideDocumentHighlightsSignature - ): ProviderResult { + public provideDocumentHighlights(): ProviderResult { if (this.connected) { - return next(document, position, token); + return this.callNext('provideDocumentHighlights', arguments); } } @captureTelemetryForLSPMethod('textDocument/documentSymbol', debounceFrequentCall) - public provideDocumentSymbols( - document: TextDocument, - token: CancellationToken, - next: ProvideDocumentSymbolsSignature - ): ProviderResult { + public provideDocumentSymbols(): ProviderResult { if (this.connected) { - return next(document, token); + return this.callNext('provideDocumentSymbols', arguments); } } @captureTelemetryForLSPMethod('workspace/symbol', debounceRareCall) - public provideWorkspaceSymbols( - query: string, - token: CancellationToken, - next: ProvideWorkspaceSymbolsSignature - ): ProviderResult { + public provideWorkspaceSymbols(): ProviderResult { if (this.connected) { - return next(query, token); + return this.callNext('provideWorkspaceSymbols', arguments); } } @captureTelemetryForLSPMethod('textDocument/codeAction', debounceFrequentCall) - public provideCodeActions( - document: TextDocument, - range: Range, - context: CodeActionContext, - token: CancellationToken, - next: ProvideCodeActionsSignature - ): ProviderResult<(Command | CodeAction)[]> { + public provideCodeActions(): ProviderResult<(Command | CodeAction)[]> { if (this.connected) { - return next(document, range, context, token); + return this.callNext('provideCodeActions', arguments); } } @captureTelemetryForLSPMethod('textDocument/codeLens', debounceFrequentCall) - public provideCodeLenses( - document: TextDocument, - token: CancellationToken, - next: ProvideCodeLensesSignature - ): ProviderResult { + public provideCodeLenses(): ProviderResult { if (this.connected) { - return next(document, token); + return this.callNext('provideCodeLenses', arguments); } } @captureTelemetryForLSPMethod('codeLens/resolve', debounceFrequentCall) - public resolveCodeLens( - codeLens: CodeLens, - token: CancellationToken, - next: ResolveCodeLensSignature - ): ProviderResult { + public resolveCodeLens(): ProviderResult { if (this.connected) { - return next(codeLens, token); + return this.callNext('resolveCodeLens', arguments); } } - public provideDocumentFormattingEdits( - document: TextDocument, - options: FormattingOptions, - token: CancellationToken, - next: ProvideDocumentFormattingEditsSignature - ): ProviderResult { + public provideDocumentFormattingEdits(): ProviderResult { if (this.connected) { - return next(document, options, token); + return this.callNext('provideDocumentFormattingEdits', arguments); } } - public provideDocumentRangeFormattingEdits( - document: TextDocument, - range: Range, - options: FormattingOptions, - token: CancellationToken, - next: ProvideDocumentRangeFormattingEditsSignature - ): ProviderResult { + public provideDocumentRangeFormattingEdits(): ProviderResult { if (this.connected) { - return next(document, range, options, token); + return this.callNext('provideDocumentRangeFormattingEdits', arguments); } } - public provideOnTypeFormattingEdits( - document: TextDocument, - position: Position, - ch: string, - options: FormattingOptions, - token: CancellationToken, - next: ProvideOnTypeFormattingEditsSignature - ): ProviderResult { + public provideOnTypeFormattingEdits(): ProviderResult { if (this.connected) { - return next(document, position, ch, options, token); + return this.callNext('provideOnTypeFormattingEdits', arguments); } } @captureTelemetryForLSPMethod('textDocument/rename', debounceRareCall) - public provideRenameEdits( - document: TextDocument, - position: Position, - newName: string, - token: CancellationToken, - next: ProvideRenameEditsSignature - ): ProviderResult { + public provideRenameEdits(): ProviderResult { if (this.connected) { - return next(document, position, newName, token); + return this.callNext('provideRenameEdits', arguments); } } @captureTelemetryForLSPMethod('textDocument/prepareRename', debounceRareCall) - public prepareRename( - document: TextDocument, - position: Position, - token: CancellationToken, - next: PrepareRenameSignature - ): ProviderResult< + public prepareRename(): ProviderResult< | Range | { range: Range; @@ -374,39 +341,38 @@ export class LanguageClientMiddleware implements Middleware { } > { if (this.connected) { - return next(document, position, token); + return this.callNext('prepareRename', arguments); } } - public provideDocumentLinks( - document: TextDocument, - token: CancellationToken, - next: ProvideDocumentLinksSignature - ): ProviderResult { + public provideDocumentLinks(): ProviderResult { if (this.connected) { - return next(document, token); + return this.callNext('provideDocumentLinks', arguments); } } - public resolveDocumentLink( - link: DocumentLink, - token: CancellationToken, - next: ResolveDocumentLinkSignature - ): ProviderResult { + public resolveDocumentLink(): ProviderResult { if (this.connected) { - return next(link, token); + return this.callNext('resolveDocumentLink', arguments); } } @captureTelemetryForLSPMethod('textDocument/declaration', debounceRareCall) - public provideDeclaration( - document: TextDocument, - position: VPosition, - token: CancellationToken, - next: ProvideDeclarationSignature - ): ProviderResult { + public provideDeclaration(): ProviderResult { if (this.connected) { - return next(document, position, token); + return this.callNext('provideDeclaration', arguments); + } + } + + private callNext(funcName: keyof NotebookMiddlewareAddon, args: IArguments) { + // This function uses the last argument to call the 'next' item. If we're allowing notebook + // middleware, it calls into the notebook middleware first. + if (this.notebookAddon) { + // It would be nice to use args.callee, but not supported in strict mode + // tslint:disable-next-line: no-any + return (this.notebookAddon as any)[funcName](...args); + } else { + return args[args.length - 1](...args); } } } @@ -469,8 +435,3 @@ function captureTelemetryForLSPMethod(method: string, debounceMilliseconds: numb return descriptor; }; } - -// tslint:disable-next-line: no-any -function isThenable(v: any): v is Thenable { - return typeof v?.then === 'function'; -} diff --git a/src/client/activation/languageServer/manager.ts b/src/client/activation/languageServer/manager.ts index 91c01769fc53..1c81e5f90405 100644 --- a/src/client/activation/languageServer/manager.ts +++ b/src/client/activation/languageServer/manager.ts @@ -6,7 +6,7 @@ import { inject, injectable, named } from 'inversify'; import { ICommandManager } from '../../common/application/types'; import { traceDecorators } from '../../common/logger'; -import { IConfigurationService, IDisposable, IExperimentsManager, Resource } from '../../common/types'; +import { IDisposable, Resource } from '../../common/types'; import { debounceSync } from '../../common/utils/decorators'; import { IServiceContainer } from '../../ioc/types'; import { PythonEnvironment } from '../../pythonEnvironments/info'; @@ -40,8 +40,6 @@ export class DotNetLanguageServerManager implements ILanguageServerManager { private readonly analysisOptions: ILanguageServerAnalysisOptions, @inject(ILanguageServerExtension) private readonly lsExtension: ILanguageServerExtension, @inject(ILanguageServerFolderService) private readonly folderService: ILanguageServerFolderService, - @inject(IExperimentsManager) private readonly experimentsManager: IExperimentsManager, - @inject(IConfigurationService) private readonly configService: IConfigurationService, @inject(ICommandManager) commandManager: ICommandManager ) { this.disposables.push( @@ -124,9 +122,9 @@ export class DotNetLanguageServerManager implements ILanguageServerManager { const options = await this.analysisOptions!.getAnalysisOptions(); options.middleware = this.middleware = new LanguageClientMiddleware( - this.experimentsManager, - this.configService, + this.serviceContainer, LanguageServerType.Microsoft, + () => this.languageServerProxy?.languageClient, this.lsVersion ); diff --git a/src/client/activation/node/manager.ts b/src/client/activation/node/manager.ts index 26b2a33a1c5c..8e4c717d3ec3 100644 --- a/src/client/activation/node/manager.ts +++ b/src/client/activation/node/manager.ts @@ -6,7 +6,7 @@ import { inject, injectable, named } from 'inversify'; import { ICommandManager } from '../../common/application/types'; import { traceDecorators } from '../../common/logger'; -import { IConfigurationService, IDisposable, IExperimentsManager, Resource } from '../../common/types'; +import { IDisposable, Resource } from '../../common/types'; import { debounceSync } from '../../common/utils/decorators'; import { IServiceContainer } from '../../ioc/types'; import { PythonEnvironment } from '../../pythonEnvironments/info'; @@ -39,8 +39,6 @@ export class NodeLanguageServerManager implements ILanguageServerManager { private readonly analysisOptions: ILanguageServerAnalysisOptions, @inject(ILanguageServerFolderService) private readonly folderService: ILanguageServerFolderService, - @inject(IExperimentsManager) private readonly experimentsManager: IExperimentsManager, - @inject(IConfigurationService) private readonly configService: IConfigurationService, @inject(ICommandManager) commandManager: ICommandManager ) { this.disposables.push( @@ -120,9 +118,9 @@ export class NodeLanguageServerManager implements ILanguageServerManager { const options = await this.analysisOptions!.getAnalysisOptions(); options.middleware = this.middleware = new LanguageClientMiddleware( - this.experimentsManager, - this.configService, + this.serviceContainer, LanguageServerType.Node, + () => this.languageServerProxy?.languageClient, this.lsVersion ); diff --git a/src/client/common/application/notebook.ts b/src/client/common/application/notebook.ts new file mode 100644 index 000000000000..85014f3b36c1 --- /dev/null +++ b/src/client/common/application/notebook.ts @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { inject, injectable } from 'inversify'; +import { Disposable, DocumentSelector, Event, EventEmitter, NotebookConcatTextDocument } from 'vscode'; +import type { + notebook, + NotebookCellMetadata, + NotebookCellsChangeEvent as VSCNotebookCellsChangeEvent, + NotebookContentProvider, + NotebookDocument, + NotebookDocumentFilter, + NotebookEditor, + NotebookKernel, + NotebookKernelProvider +} from 'vscode-proposed'; +import { UseProposedApi } from '../constants'; +import { IDisposableRegistry } from '../types'; +import { IApplicationEnvironment, IVSCodeNotebook, NotebookCellChangedEvent } from './types'; + +@injectable() +export class VSCodeNotebook implements IVSCodeNotebook { + public get onDidChangeActiveNotebookKernel(): Event<{ + document: NotebookDocument; + kernel: NotebookKernel | undefined; + }> { + return this.canUseNotebookApi + ? this.notebook.onDidChangeActiveNotebookKernel + : new EventEmitter<{ + document: NotebookDocument; + kernel: NotebookKernel | undefined; + }>().event; + } + public get onDidChangeActiveNotebookEditor(): Event { + return this.canUseNotebookApi + ? this.notebook.onDidChangeActiveNotebookEditor + : new EventEmitter().event; + } + public get onDidOpenNotebookDocument(): Event { + return this.canUseNotebookApi + ? this.notebook.onDidOpenNotebookDocument + : new EventEmitter().event; + } + public get onDidCloseNotebookDocument(): Event { + return this.canUseNotebookApi + ? this.notebook.onDidCloseNotebookDocument + : new EventEmitter().event; + } + public get onDidSaveNotebookDocument(): Event { + return this.canUseNotebookApi + ? this.notebook.onDidSaveNotebookDocument + : new EventEmitter().event; + } + public get notebookDocuments(): ReadonlyArray { + return this.canUseNotebookApi ? this.notebook.notebookDocuments : []; + } + public get notebookEditors() { + return this.canUseNotebookApi ? this.notebook.visibleNotebookEditors : []; + } + public get onDidChangeNotebookDocument(): Event { + return this.canUseNotebookApi + ? this._onDidChangeNotebookDocument.event + : new EventEmitter().event; + } + public get activeNotebookEditor(): NotebookEditor | undefined { + if (!this.useProposedApi) { + return; + } + return this.notebook.activeNotebookEditor; + } + private get notebook() { + if (!this._notebook) { + // tslint:disable-next-line: no-require-imports + this._notebook = require('vscode').notebook; + } + return this._notebook!; + } + private readonly _onDidChangeNotebookDocument = new EventEmitter(); + private addedEventHandlers?: boolean; + private _notebook?: typeof notebook; + private readonly canUseNotebookApi?: boolean; + private readonly handledCellChanges = new WeakSet(); + constructor( + @inject(UseProposedApi) private readonly useProposedApi: boolean, + @inject(IDisposableRegistry) private readonly disposables: IDisposableRegistry, + @inject(IApplicationEnvironment) readonly env: IApplicationEnvironment + ) { + if (this.useProposedApi && this.env.channel === 'insiders') { + this.addEventHandlers(); + this.canUseNotebookApi = true; + } + } + public createConcatTextDocument(doc: NotebookDocument, selector?: DocumentSelector): NotebookConcatTextDocument { + if (this.useProposedApi) { + // tslint:disable-next-line: no-any + return this.notebook.createConcatTextDocument(doc, selector) as any; // Types of Position are different for some reason. Fix this later. + } + throw new Error('createConcatDocument not supported'); + } + public registerNotebookContentProvider( + notebookType: string, + provider: NotebookContentProvider, + options?: { + transientOutputs: boolean; + transientMetadata: { [K in keyof NotebookCellMetadata]?: boolean }; + } + ): Disposable { + return this.notebook.registerNotebookContentProvider(notebookType, provider, options); + } + public registerNotebookKernelProvider( + selector: NotebookDocumentFilter, + provider: NotebookKernelProvider + ): Disposable { + return this.notebook.registerNotebookKernelProvider(selector, provider); + } + private addEventHandlers() { + if (this.addedEventHandlers) { + return; + } + this.addedEventHandlers = true; + this.disposables.push( + ...[ + this.notebook.onDidChangeCellLanguage((e) => + this._onDidChangeNotebookDocument.fire({ ...e, type: 'changeCellLanguage' }) + ), + this.notebook.onDidChangeCellMetadata((e) => + this._onDidChangeNotebookDocument.fire({ ...e, type: 'changeCellMetadata' }) + ), + this.notebook.onDidChangeNotebookDocumentMetadata((e) => + this._onDidChangeNotebookDocument.fire({ ...e, type: 'changeNotebookMetadata' }) + ), + this.notebook.onDidChangeCellOutputs((e) => + this._onDidChangeNotebookDocument.fire({ ...e, type: 'changeCellOutputs' }) + ), + this.notebook.onDidChangeNotebookCells((e) => { + if (this.handledCellChanges.has(e)) { + return; + } + this.handledCellChanges.add(e); + this._onDidChangeNotebookDocument.fire({ ...e, type: 'changeCells' }); + }) + ] + ); + } +} diff --git a/src/client/common/application/types.ts b/src/client/common/application/types.ts index 7469b54deb60..6f16f5a0de2a 100644 --- a/src/client/common/application/types.ts +++ b/src/client/common/application/types.ts @@ -24,6 +24,7 @@ import { InputBoxOptions, MessageItem, MessageOptions, + NotebookConcatTextDocument, OpenDialogOptions, OutputChannel, Progress, @@ -60,10 +61,17 @@ import { } from 'vscode'; import type { NotebookCellLanguageChangeEvent as VSCNotebookCellLanguageChangeEvent, + NotebookCellMetadata, NotebookCellMetadataChangeEvent as VSCNotebookCellMetadataChangeEvent, NotebookCellOutputsChangeEvent as VSCNotebookCellOutputsChangeEvent, NotebookCellsChangeEvent as VSCNotebookCellsChangeEvent, - NotebookDocumentMetadataChangeEvent as VSCNotebookDocumentMetadataChangeEvent + NotebookContentProvider, + NotebookDocument, + NotebookDocumentFilter, + NotebookDocumentMetadataChangeEvent as VSCNotebookDocumentMetadataChangeEvent, + NotebookEditor, + NotebookKernel, + NotebookKernelProvider } from 'vscode-proposed'; import { IAsyncDisposable, Resource } from '../types'; @@ -1521,3 +1529,38 @@ export type NotebookCellChangedEvent = | NotebookCellMetadataChangeEvent | NotebookDocumentMetadataChangeEvent | NotebookCellLanguageChangeEvent; +export const IVSCodeNotebook = Symbol('IVSCodeNotebook'); +export interface IVSCodeNotebook { + readonly onDidChangeActiveNotebookKernel: Event<{ + document: NotebookDocument; + kernel: NotebookKernel | undefined; + }>; + readonly notebookDocuments: ReadonlyArray; + readonly onDidOpenNotebookDocument: Event; + readonly onDidCloseNotebookDocument: Event; + readonly onDidSaveNotebookDocument: Event; + readonly onDidChangeActiveNotebookEditor: Event; + readonly onDidChangeNotebookDocument: Event; + readonly notebookEditors: Readonly; + readonly activeNotebookEditor: NotebookEditor | undefined; + registerNotebookContentProvider( + notebookType: string, + provider: NotebookContentProvider, + options?: { + /** + * Controls if outputs change will trigger notebook document content change and if it will be used in the diff editor + * Default to false. If the content provider doesn't persisit the outputs in the file document, this should be set to true. + */ + transientOutputs: boolean; + /** + * Controls if a meetadata property change will trigger notebook document content change and if it will be used in the diff editor + * Default to false. If the content provider doesn't persisit a metadata property in the file document, it should be set to true. + */ + transientMetadata: { [K in keyof NotebookCellMetadata]?: boolean }; + } + ): Disposable; + + registerNotebookKernelProvider(selector: NotebookDocumentFilter, provider: NotebookKernelProvider): Disposable; + + createConcatTextDocument(notebook: NotebookDocument, selector?: DocumentSelector): NotebookConcatTextDocument; +} diff --git a/src/client/common/configSettings.ts b/src/client/common/configSettings.ts index 9ebf3a4fe08e..f53eaf1f6870 100644 --- a/src/client/common/configSettings.ts +++ b/src/client/common/configSettings.ts @@ -611,7 +611,11 @@ export class PythonSettings implements IPythonSettings { ? this.interpreterPathService.get(this.workspaceRoot) : pythonSettings.get('pythonPath') )!; - if (!process.env.CI_DISABLE_AUTO_SELECTION && (this.pythonPath.length === 0 || this.pythonPath === 'python')) { + if ( + !process.env.CI_DISABLE_AUTO_SELECTION && + (this.pythonPath.length === 0 || this.pythonPath === 'python') && + this.interpreterAutoSelectionService + ) { const autoSelectedPythonInterpreter = this.interpreterAutoSelectionService.getAutoSelectedInterpreter( this.workspaceRoot ); diff --git a/src/client/common/serviceRegistry.ts b/src/client/common/serviceRegistry.ts index 39158f035ea1..5fdd57165c66 100644 --- a/src/client/common/serviceRegistry.ts +++ b/src/client/common/serviceRegistry.ts @@ -18,6 +18,7 @@ import { DebugSessionTelemetry } from './application/debugSessionTelemetry'; import { DocumentManager } from './application/documentManager'; import { Extensions } from './application/extensions'; import { LanguageService } from './application/languageService'; +import { VSCodeNotebook } from './application/notebook'; import { TerminalManager } from './application/terminalManager'; import { IActiveResourceService, @@ -31,6 +32,7 @@ import { IJupyterExtensionDependencyManager, ILanguageService, ITerminalManager, + IVSCodeNotebook, IWorkspaceService } from './application/types'; import { WorkspaceService } from './application/workspace'; @@ -126,6 +128,7 @@ export function registerTypes(serviceManager: IServiceManager) { serviceManager.addSingleton(ITerminalServiceFactory, TerminalServiceFactory); serviceManager.addSingleton(IPathUtils, PathUtils); serviceManager.addSingleton(IApplicationShell, ApplicationShell); + serviceManager.addSingleton(IVSCodeNotebook, VSCodeNotebook); serviceManager.addSingleton(IClipboard, ClipboardService); serviceManager.addSingleton(ICurrentProcess, CurrentProcess); serviceManager.addSingleton(IInstaller, ProductInstaller); diff --git a/src/client/common/startPage/webviewHost.ts b/src/client/common/startPage/webviewHost.ts index 168f6a531079..63ce630cf6bf 100644 --- a/src/client/common/startPage/webviewHost.ts +++ b/src/client/common/startPage/webviewHost.ts @@ -105,7 +105,17 @@ export abstract class WebviewHost implements IDisposable { protected async generateExtraSettings(): Promise { const resource = this.owningResource; - return this.configService.getSettings(resource); + // tslint:disable-next-line: no-any + const prunedSettings = this.configService.getSettings(resource) as any; + + // Remove keys that aren't serializable + const keys = Object.keys(prunedSettings); + keys.forEach((k) => { + if (k.includes('Manager') || k.includes('Service') || k.includes('onDid')) { + delete prunedSettings[k]; + } + }); + return prunedSettings; } protected async sendLocStrings() { diff --git a/src/client/datascience/languageserver/notebookConcatDocument.ts b/src/client/datascience/languageserver/notebookConcatDocument.ts new file mode 100644 index 000000000000..e52748bcc37a --- /dev/null +++ b/src/client/datascience/languageserver/notebookConcatDocument.ts @@ -0,0 +1,140 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import * as path from 'path'; +import * as uuid from 'uuid/v4'; +import { + Disposable, + DocumentSelector, + EndOfLine, + NotebookConcatTextDocument, + NotebookDocument, + Position, + Range, + TextDocument, + Uri +} from 'vscode'; +import { IVSCodeNotebook } from '../../common/application/types'; +import { IDisposable } from '../../common/types'; + +export const NotebookConcatPrefix = '_NotebookConcat_'; + +/** + * This helper class is used to present a converted document to an LS + */ +export class NotebookConcatDocument implements TextDocument, IDisposable { + public get notebookUri() { + return this.notebook.uri; + } + + public get uri() { + return this.dummyUri; + } + + public get fileName() { + return this.dummyFilePath; + } + + public get isUntitled() { + return this.notebook.isUntitled; + } + + public get languageId() { + return this.notebook.languages[0]; + } + + public get version() { + return this._version; + } + + public get isDirty() { + return this.notebook.isDirty; + } + + public get isClosed() { + return this.concatDocument.isClosed; + } + public get eol() { + return EndOfLine.LF; + } + public get lineCount() { + return this.notebook.cells.map((c) => c.document.lineCount).reduce((p, c) => p + c); + } + public firedOpen = false; + public firedClose = false; + public concatDocument: NotebookConcatTextDocument; + private dummyFilePath: string; + private dummyUri: Uri; + private _version = 1; + private onDidChangeSubscription: Disposable; + constructor(public notebook: NotebookDocument, notebookApi: IVSCodeNotebook, selector: DocumentSelector) { + const dir = path.dirname(notebook.uri.fsPath); + // Note: Has to be different than the prefix for old notebook editor (HiddenFileFormat) so + // that the caller doesn't remove diagnostics for this document. + this.dummyFilePath = path.join(dir, `${NotebookConcatPrefix}${uuid().replace(/-/g, '')}.py`); + this.dummyUri = Uri.file(this.dummyFilePath); + this.concatDocument = notebookApi.createConcatTextDocument(notebook, selector); + this.onDidChangeSubscription = this.concatDocument.onDidChange(this.onDidChange, this); + } + + public dispose() { + this.onDidChangeSubscription.dispose(); + } + + public isCellOfDocument(uri: Uri) { + return this.concatDocument.contains(uri); + } + public save(): Thenable { + // Not used + throw new Error('Not implemented'); + } + public lineAt(posOrNumber: Position | number) { + const position = typeof posOrNumber === 'number' ? new Position(posOrNumber, 0) : posOrNumber; + + // convert this position into a cell location + // (we need the translated location, that's why we can't use getCellAtPosition) + const location = this.concatDocument.locationAt(position); + + // Get the cell at this location + const cell = this.notebook.cells.find((c) => c.uri.toString() === location.uri.toString()); + return cell!.document.lineAt(location.range.start); + } + + public offsetAt(position: Position) { + return this.concatDocument.offsetAt(position); + } + + public positionAt(offset: number) { + return this.concatDocument.positionAt(offset); + } + + public getText(range?: Range | undefined) { + return range ? this.concatDocument.getText(range) : this.concatDocument.getText(); + } + + public getWordRangeAtPosition(position: Position, regexp?: RegExp | undefined) { + // convert this position into a cell location + // (we need the translated location, that's why we can't use getCellAtPosition) + const location = this.concatDocument.locationAt(position); + + // Get the cell at this location + const cell = this.notebook.cells.find((c) => c.uri.toString() === location.uri.toString()); + return cell!.document.getWordRangeAtPosition(location.range.start, regexp); + } + + public validateRange(range: Range) { + return this.concatDocument.validateRange(range); + } + + public validatePosition(pos: Position) { + return this.concatDocument.validatePosition(pos); + } + + public getCellAtPosition(position: Position) { + const location = this.concatDocument.locationAt(position); + return this.notebook.cells.find((c) => c.uri === location.uri); + } + + private onDidChange() { + this._version += 1; + } +} diff --git a/src/client/datascience/languageserver/notebookConverter.ts b/src/client/datascience/languageserver/notebookConverter.ts new file mode 100644 index 000000000000..5ded3226fd63 --- /dev/null +++ b/src/client/datascience/languageserver/notebookConverter.ts @@ -0,0 +1,625 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import * as os from 'os'; +import { + CodeAction, + CodeActionContext, + CodeLens, + Command, + CompletionItem, + CompletionList, + Diagnostic, + DiagnosticRelatedInformation, + Disposable, + DocumentHighlight, + DocumentLink, + DocumentSelector, + DocumentSymbol, + Hover, + Location, + LocationLink, + NotebookConcatTextDocument, + NotebookDocument, + Position, + Range, + SymbolInformation, + TextDocument, + TextDocumentChangeEvent, + TextDocumentContentChangeEvent, + TextEdit, + Uri, + WorkspaceEdit +} from 'vscode'; +import { IVSCodeNotebook } from '../../common/application/types'; +import { IFileSystem } from '../../common/platform/types'; +import { NotebookConcatDocument } from './notebookConcatDocument'; + +/* Used by code actions. Disabled for now. +function toRange(rangeLike: Range): Range { + return new Range(toPosition(rangeLike.start), toPosition(rangeLike.end)); +} + +function toPosition(positionLike: Position): Position { + return new Position(positionLike.line, positionLike.character); +} +*/ + +export class NotebookConverter implements Disposable { + private activeDocuments: Map = new Map(); + private activeDocumentsOutgoingMap: Map = new Map(); + private disposables: Disposable[] = []; + + constructor( + private api: IVSCodeNotebook, + private fs: IFileSystem, + private cellSelector: DocumentSelector, + private notebookFilter: RegExp + ) { + this.disposables.push(api.onDidOpenNotebookDocument(this.onDidOpenNotebook.bind(this))); + this.disposables.push(api.onDidCloseNotebookDocument(this.onDidCloseNotebook.bind(this))); + + // Call open on all of the active notebooks too + api.notebookDocuments.forEach(this.onDidOpenNotebook.bind(this)); + } + + public dispose() { + this.disposables.forEach((d) => d.dispose()); + } + + public hasCell(cell: TextDocument) { + const concat = this.getConcatDocument(cell); + return concat && concat.contains(cell.uri); + } + + public hasFiredOpen(cell: TextDocument) { + const wrapper = this.getTextDocumentWrapper(cell); + if (wrapper) { + return wrapper.firedOpen; + } + } + + public firedOpen(cell: TextDocument) { + const wrapper = this.getTextDocumentWrapper(cell); + if (wrapper) { + wrapper.firedOpen = true; + } + } + + public hasFiredClose(cell: TextDocument) { + const wrapper = this.getTextDocumentWrapper(cell); + if (wrapper) { + return wrapper.firedClose; + } + } + + public firedClose(cell: TextDocument) { + const wrapper = this.getTextDocumentWrapper(cell); + if (wrapper) { + wrapper.firedClose = true; + wrapper.firedOpen = false; + } + } + + public toIncomingDiagnosticsMap(uri: Uri, diagnostics: Diagnostic[]): Map { + const wrapper = this.getWrapperFromOutgoingUri(uri); + const result = new Map(); + + if (wrapper) { + // Diagnostics are supposed to be per file and are updated each time + // Make sure to clear out old ones first + wrapper.notebook.cells.forEach((c) => { + result.set(c.uri, []); + }); + + // Then for all the new ones, set their values. + diagnostics.forEach((d) => { + const location = wrapper.concatDocument.locationAt(d.range); + let list = result.get(location.uri); + if (!list) { + list = []; + result.set(location.uri, list); + } + list.push(this.toIncomingDiagnostic(location.uri, d)); + }); + } else { + result.set(uri, diagnostics); + } + + return result; + } + + public toIncomingWorkspaceSymbols(symbols: SymbolInformation[] | null | undefined) { + if (Array.isArray(symbols)) { + return symbols.map(this.toIncomingWorkspaceSymbol.bind(this)); + } + return symbols; + } + + public toIncomingWorkspaceEdit(workspaceEdit: WorkspaceEdit | null | undefined) { + if (workspaceEdit) { + // Translate all of the text edits into a URI map + const translated = new Map(); + workspaceEdit.entries().forEach(([key, values]) => { + values.forEach((e) => { + // Location may move this edit to a different cell. + const location = this.toIncomingLocation(key, e.range); + + // Save this in the entry + let list = translated.get(location.uri); + if (!list) { + list = []; + translated.set(location.uri, list); + } + list.push({ + ...e, + range: location.range + }); + }); + }); + + // Add translated entries to the new edit + const newWorkspaceEdit = new WorkspaceEdit(); + translated.forEach((v, k) => newWorkspaceEdit.set(k, v)); + return newWorkspaceEdit; + } + return workspaceEdit; + } + + public toOutgoingDocument(cell: TextDocument): TextDocument { + const result = this.getTextDocumentWrapper(cell); + return result ? result : cell; + } + + public toOutgoingUri(cell: TextDocument | Uri): Uri { + const uri = cell instanceof Uri ? cell : cell.uri; + const result = this.getTextDocumentWrapper(cell); + return result ? result.uri : uri; + } + + public toOutgoingChangeEvent(cellEvent: TextDocumentChangeEvent) { + return { + document: this.toOutgoingDocument(cellEvent.document), + contentChanges: cellEvent.contentChanges.map( + this.toOutgoingContentChangeEvent.bind(this, cellEvent.document) + ) + }; + } + + public toOutgoingPosition(cell: TextDocument, position: Position) { + const concat = this.getConcatDocument(cell); + return concat ? concat.positionAt(new Location(cell.uri, position)) : position; + } + + public toOutgoingRange(cell: TextDocument, cellRange: Range): Range { + const concat = this.getConcatDocument(cell); + if (concat) { + const startPos = concat.positionAt(new Location(cell.uri, cellRange.start)); + const endPos = concat.positionAt(new Location(cell.uri, cellRange.end)); + return new Range(startPos, endPos); + } + return cellRange; + } + + public toOutgoingOffset(cell: TextDocument, offset: number) { + const concat = this.getConcatDocument(cell); + if (concat) { + const position = cell.positionAt(offset); + const overallPosition = concat.positionAt(new Location(cell.uri, position)); + return concat.offsetAt(overallPosition); + } + return offset; + } + + public toOutgoingContext(cell: TextDocument, context: CodeActionContext): CodeActionContext { + return { + ...context, + diagnostics: context.diagnostics.map(this.toOutgoingDiagnostic.bind(this, cell)) + }; + } + + public toIncomingHover(cell: TextDocument, hover: Hover | null | undefined) { + if (hover && hover.range) { + return { + ...hover, + range: this.toIncomingRange(cell, hover.range) + }; + } + return hover; + } + public toIncomingCompletions( + cell: TextDocument, + completions: CompletionItem[] | CompletionList | null | undefined + ) { + if (completions) { + if (Array.isArray(completions)) { + return completions.map(this.toIncomingCompletion.bind(this, cell)); + } else { + return { + ...completions, + items: completions.items.map(this.toIncomingCompletion.bind(this, cell)) + }; + } + } + return completions; + } + + public toIncomingLocations( + cell: TextDocument, + location: Location | Location[] | LocationLink[] | null | undefined + ) { + if (Array.isArray(location)) { + // tslint:disable-next-line: no-any + return (location).map(this.toIncomingLocationFromLink.bind(this, cell)); + } else if (location?.range) { + return this.toIncomingLocation(location.uri, location.range); + } + return location; + } + + public toIncomingHighlight(cell: TextDocument, highlight: DocumentHighlight[] | null | undefined) { + if (highlight) { + return highlight.map((h) => { + return { + ...h, + range: this.toIncomingRange(cell, h.range) + }; + }); + } + return highlight; + } + + public toIncomingSymbols(cell: TextDocument, symbols: SymbolInformation[] | DocumentSymbol[] | null | undefined) { + if (symbols && Array.isArray(symbols) && symbols.length) { + if (symbols[0] instanceof DocumentSymbol) { + return (symbols).map(this.toIncomingSymbolFromDocumentSymbol.bind(this, cell)); + } else { + return (symbols).map(this.toIncomingSymbolFromSymbolInformation.bind(this, cell)); + } + } + return symbols; + } + + public toIncomingSymbolFromSymbolInformation(cell: TextDocument, symbol: SymbolInformation): SymbolInformation { + return { + ...symbol, + location: this.toIncomingLocation(cell, symbol.location.range) + }; + } + + public toIncomingDiagnostic(cell: TextDocument | Uri, diagnostic: Diagnostic): Diagnostic { + return { + ...diagnostic, + range: this.toIncomingRange(cell, diagnostic.range), + relatedInformation: diagnostic.relatedInformation + ? diagnostic.relatedInformation.map(this.toIncomingRelatedInformation.bind(this, cell)) + : undefined + }; + } + + public toIncomingActions(_cell: TextDocument, actions: (Command | CodeAction)[] | null | undefined) { + if (Array.isArray(actions)) { + // Disable for now because actions are handled directly by the LS sometimes (at least in pylance) + // If we translate or use them they will either + // 1) Do nothing because the LS doesn't know about the ipynb + // 2) Crash (pylance is doing this now) + return undefined; + } + return actions; + } + + public toIncomingCodeLenses(cell: TextDocument, lenses: CodeLens[] | null | undefined) { + if (Array.isArray(lenses)) { + return lenses.map((c) => { + return { + ...c, + range: this.toIncomingRange(cell, c.range) + }; + }); + } + return lenses; + } + + public toIncomingEdits(cell: TextDocument, edits: TextEdit[] | null | undefined) { + if (Array.isArray(edits)) { + return edits.map((e) => { + return { + ...e, + range: this.toIncomingRange(cell, e.range) + }; + }); + } + return edits; + } + + public toIncomingRename( + cell: TextDocument, + rangeOrRename: + | Range + | { + range: Range; + placeholder: string; + } + | null + | undefined + ) { + if (rangeOrRename) { + if (rangeOrRename instanceof Range) { + return this.toIncomingLocation(cell, rangeOrRename).range; + } else { + return { + ...rangeOrRename, + range: this.toIncomingLocation(cell, rangeOrRename.range).range + }; + } + } + return rangeOrRename; + } + + public toIncomingDocumentLinks(cell: TextDocument, links: DocumentLink[] | null | undefined) { + if (links && Array.isArray(links)) { + return links.map((l) => { + const uri = l.target ? l.target : cell.uri; + const location = this.toIncomingLocation(uri, l.range); + return { + ...l, + range: location.range, + target: l.target ? location.uri : undefined + }; + }); + } + return links; + } + + public toIncomingRange(cell: TextDocument | Uri, range: Range) { + // This is dangerous as the URI is not remapped (location uri may be different) + return this.toIncomingLocation(cell, range).range; + } + + public toIncomingPosition(cell: TextDocument | Uri, position: Position) { + // This is dangerous as the URI is not remapped (location uri may be different) + return this.toIncomingLocation(cell, new Range(position, position)).range.start; + } + + private getCellAtLocation(location: Location) { + const key = this.getDocumentKey(location.uri); + const wrapper = this.activeDocuments.get(key); + if (wrapper) { + return wrapper.getCellAtPosition(location.range.start); + } + } + + private toIncomingWorkspaceSymbol(symbol: SymbolInformation): SymbolInformation { + // Figure out what cell if any the symbol is for + const cell = this.getCellAtLocation(symbol.location); + if (cell) { + return this.toIncomingSymbolFromSymbolInformation(cell.document, symbol); + } + return symbol; + } + + /* Renable this if actions can be translated + private toIncomingAction(cell: TextDocument, action: Command | CodeAction): Command | CodeAction { + if (action instanceof CodeAction) { + return { + ...action, + command: action.command ? this.toIncomingCommand(cell, action.command) : undefined, + diagnostics: action.diagnostics + ? action.diagnostics.map(this.toIncomingDiagnostic.bind(this, cell)) + : undefined + }; + } + return this.toIncomingCommand(cell, action); + } + + private toIncomingCommand(cell: TextDocument, command: Command): Command { + return { + ...command, + arguments: command.arguments ? command.arguments.map(this.toIncomingArgument.bind(this, cell)) : undefined + }; + } + + // tslint:disable-next-line: no-any + private toIncomingArgument(cell: TextDocument, argument: any): any { + // URIs in a command should be remapped to the cell document if part + // of one of our open notebooks + if (isUri(argument)) { + const wrapper = this.getWrapperFromOutgoingUri(argument); + if (wrapper) { + return cell.uri; + } + } + if (typeof argument === 'string' && argument.includes(NotebookConcatPrefix)) { + const wrapper = this.getWrapperFromOutgoingUri(Uri.file(argument)); + if (wrapper) { + return cell.uri; + } + } + if (typeof argument === 'object' && argument.hasOwnProperty('start') && argument.hasOwnProperty('end')) { + // This is a range like object. Convert it too. + return this.toIncomingRange(cell, this.toRange(argument)); + } + if (typeof argument === 'object' && argument.hasOwnProperty('line') && argument.hasOwnProperty('character')) { + // This is a position like object. Convert it too. + return this.toIncomingPosition(cell, this.toPosition(argument)); + } + return argument; + } + */ + + private toOutgoingDiagnostic(cell: TextDocument, diagnostic: Diagnostic): Diagnostic { + return { + ...diagnostic, + range: this.toOutgoingRange(cell, diagnostic.range), + relatedInformation: diagnostic.relatedInformation + ? diagnostic.relatedInformation.map(this.toOutgoingRelatedInformation.bind(this, cell)) + : undefined + }; + } + + private toOutgoingRelatedInformation( + cell: TextDocument, + relatedInformation: DiagnosticRelatedInformation + ): DiagnosticRelatedInformation { + const outgoingDoc = this.toOutgoingDocument(cell); + return { + ...relatedInformation, + location: + relatedInformation.location.uri === outgoingDoc.uri + ? this.toOutgoingLocation(cell, relatedInformation.location) + : relatedInformation.location + }; + } + + private toOutgoingLocation(cell: TextDocument, location: Location): Location { + return { + uri: this.toOutgoingDocument(cell).uri, + range: this.toOutgoingRange(cell, location.range) + }; + } + + private toIncomingRelatedInformation( + cell: TextDocument | Uri, + relatedInformation: DiagnosticRelatedInformation + ): DiagnosticRelatedInformation { + const outgoingUri = this.toOutgoingUri(cell); + return { + ...relatedInformation, + location: + relatedInformation.location.uri === outgoingUri + ? this.toIncomingLocationFromLink(cell, relatedInformation.location) + : relatedInformation.location + }; + } + + private toIncomingSymbolFromDocumentSymbol(cell: TextDocument, docSymbol: DocumentSymbol): DocumentSymbol { + return { + ...docSymbol, + range: this.toIncomingRange(cell, docSymbol.range), + selectionRange: this.toIncomingRange(cell, docSymbol.selectionRange), + children: docSymbol.children.map(this.toIncomingSymbolFromDocumentSymbol.bind(this, cell)) + }; + } + + private toIncomingLocationFromLink(_cell: TextDocument | Uri, location: Location | LocationLink) { + const locationLink = location; + const locationNorm = location; + const uri = this.toIncomingUri( + locationLink.targetUri || locationNorm.uri, + locationLink.targetRange ? locationLink.targetRange : locationNorm.range + ); + return { + originSelectionRange: locationLink.originSelectionRange + ? this.toIncomingRange(uri, locationLink.originSelectionRange) + : undefined, + uri, + range: locationLink.targetRange + ? this.toIncomingRange(uri, locationLink.targetRange) + : this.toIncomingRange(uri, locationNorm.range), + targetSelectionRange: locationLink.targetSelectionRange + ? this.toIncomingRange(uri, locationLink.targetSelectionRange) + : undefined + }; + } + + private toIncomingUri(outgoingUri: Uri, range: Range) { + const wrapper = this.getWrapperFromOutgoingUri(outgoingUri); + if (wrapper) { + const location = wrapper.concatDocument.locationAt(range); + return location.uri; + } + return outgoingUri; + } + + private toIncomingCompletion(cell: TextDocument, item: CompletionItem) { + if (item.range) { + if (item.range instanceof Range) { + return { + ...item, + range: this.toIncomingRange(cell, item.range) + }; + } else { + return { + ...item, + range: { + inserting: this.toIncomingRange(cell, item.range.inserting), + replacing: this.toIncomingRange(cell, item.range.replacing) + } + }; + } + } + return item; + } + + private toIncomingLocation(cell: TextDocument | Uri, range: Range): Location { + const uri = cell instanceof Uri ? cell : cell.uri; + const concatDocument = this.getConcatDocument(cell); + if (concatDocument) { + const startLoc = concatDocument.locationAt(range.start); + const endLoc = concatDocument.locationAt(range.end); + return { + uri: startLoc.uri, + range: new Range(startLoc.range.start, endLoc.range.end) + }; + } + return { + uri, + range + }; + } + + private toOutgoingContentChangeEvent(cell: TextDocument, ev: TextDocumentContentChangeEvent) { + return { + range: this.toOutgoingRange(cell, ev.range), + rangeLength: ev.rangeLength, + rangeOffset: this.toOutgoingOffset(cell, ev.rangeOffset), + text: ev.text + }; + } + + private getDocumentKey(uri: Uri): string { + // Use the path of the doc uri. It should be the same for all cells + if (os.platform() === 'win32') { + return uri.fsPath.toLowerCase(); + } + return uri.fsPath; + } + + private onDidOpenNotebook(doc: NotebookDocument) { + if (this.notebookFilter.test(doc.fileName)) { + this.getTextDocumentWrapper(doc.uri); + } + } + + private onDidCloseNotebook(doc: NotebookDocument) { + if (this.notebookFilter.test(doc.fileName)) { + const key = this.getDocumentKey(doc.uri); + const wrapper = this.getTextDocumentWrapper(doc.uri); + this.activeDocuments.delete(key); + this.activeDocumentsOutgoingMap.delete(this.getDocumentKey(wrapper.uri)); + } + } + + private getWrapperFromOutgoingUri(outgoingUri: Uri): NotebookConcatDocument | undefined { + return this.activeDocumentsOutgoingMap.get(this.getDocumentKey(outgoingUri)); + } + + private getTextDocumentWrapper(cell: TextDocument | Uri): NotebookConcatDocument { + const uri = cell instanceof Uri ? cell : cell.uri; + const key = this.getDocumentKey(uri); + let result = this.activeDocuments.get(key); + if (!result) { + const doc = this.api.notebookDocuments.find((n) => this.fs.arePathsSame(uri.fsPath, n.uri.fsPath)); + if (!doc) { + throw new Error(`Invalid uri, not a notebook: ${uri.fsPath}`); + } + result = new NotebookConcatDocument(doc, this.api, this.cellSelector); + this.activeDocuments.set(key, result); + this.activeDocumentsOutgoingMap.set(this.getDocumentKey(result.uri), result); + } + return result; + } + + private getConcatDocument(cell: TextDocument | Uri): NotebookConcatTextDocument | undefined { + return this.getTextDocumentWrapper(cell)?.concatDocument; + } +} diff --git a/src/client/datascience/languageserver/notebookMiddlewareAddon.ts b/src/client/datascience/languageserver/notebookMiddlewareAddon.ts new file mode 100644 index 000000000000..9f116d0b4d83 --- /dev/null +++ b/src/client/datascience/languageserver/notebookMiddlewareAddon.ts @@ -0,0 +1,459 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import { + CancellationToken, + CodeAction, + CodeActionContext, + CodeLens, + Command, + CompletionContext, + CompletionItem, + Declaration as VDeclaration, + Definition, + DefinitionLink, + Diagnostic, + Disposable, + DocumentHighlight, + DocumentLink, + DocumentSelector, + DocumentSymbol, + FormattingOptions, + Location, + Position, + Position as VPosition, + ProviderResult, + Range, + SignatureHelp, + SignatureHelpContext, + SymbolInformation, + TextDocument, + TextDocumentChangeEvent, + TextDocumentWillSaveEvent, + TextEdit, + Uri, + WorkspaceEdit +} from 'vscode'; +import { + DidChangeTextDocumentNotification, + HandleDiagnosticsSignature, + LanguageClient, + Middleware, + PrepareRenameSignature, + ProvideCodeActionsSignature, + ProvideCodeLensesSignature, + ProvideCompletionItemsSignature, + ProvideDefinitionSignature, + ProvideDocumentFormattingEditsSignature, + ProvideDocumentHighlightsSignature, + ProvideDocumentLinksSignature, + ProvideDocumentRangeFormattingEditsSignature, + ProvideDocumentSymbolsSignature, + ProvideHoverSignature, + ProvideOnTypeFormattingEditsSignature, + ProvideReferencesSignature, + ProvideRenameEditsSignature, + ProvideSignatureHelpSignature, + ProvideWorkspaceSymbolsSignature, + ResolveCodeLensSignature, + ResolveCompletionItemSignature, + ResolveDocumentLinkSignature +} from 'vscode-languageclient/node'; + +import { ProvideDeclarationSignature } from 'vscode-languageclient/lib/common/declaration'; +import { IVSCodeNotebook } from '../../common/application/types'; +import { traceInfo } from '../../common/logger'; +import { IFileSystem } from '../../common/platform/types'; +import { isThenable } from '../../common/utils/async'; +import { isNotebookCell } from '../../common/utils/misc'; +import { NotebookConverter } from './notebookConverter'; + +/** + * This class is a temporary solution to handling intellisense and diagnostics in python based notebooks. + * + * It is responsible for generating a concatenated document of all of the cells in a notebook and using that as the + * document for LSP requests. + */ +export class NotebookMiddlewareAddon implements Middleware, Disposable { + private converter: NotebookConverter; + + constructor( + notebookApi: IVSCodeNotebook, + private readonly getClient: () => LanguageClient | undefined, + fs: IFileSystem, + cellSelector: DocumentSelector, + notebookFileRegex: RegExp + ) { + this.converter = new NotebookConverter(notebookApi, fs, cellSelector, notebookFileRegex); + } + + public dispose() { + this.converter.dispose(); + } + + public didChange(event: TextDocumentChangeEvent, next: (ev: TextDocumentChangeEvent) => void) { + // We need to talk directly to the language client here. + const client = this.getClient(); + + // If this is a notebook cell, change this into a concat document event + if (isNotebookCell(event.document.uri) && client) { + const newEvent = this.converter.toOutgoingChangeEvent(event); + + // Next will not use our params here. We need to send directly as next with the event + // doesn't let the event change the value + const params = client.code2ProtocolConverter.asChangeTextDocumentParams(newEvent); + client.sendNotification(DidChangeTextDocumentNotification.type, params); + } else { + next(event); + } + } + + public didOpen(document: TextDocument, next: (ev: TextDocument) => void) { + // If this is a notebook cell, change this into a concat document if this is the first time. + if (isNotebookCell(document.uri)) { + if (!this.converter.hasFiredOpen(document)) { + this.converter.firedOpen(document); + const newDoc = this.converter.toOutgoingDocument(document); + return next(newDoc); + } + } else { + next(document); + } + } + + public didClose(document: TextDocument, next: (ev: TextDocument) => void) { + // If this is a notebook cell, change this into a concat document if this is the first time. + if (isNotebookCell(document.uri)) { + // Cell delete causes this callback, but won't fire the close event because it's not + // in the document anymore. + if (this.converter.hasCell(document) && !this.converter.hasFiredClose(document)) { + this.converter.firedClose(document); + const newDoc = this.converter.toOutgoingDocument(document); + return next(newDoc); + } + } else { + next(document); + } + } + + public didSave(event: TextDocument, next: (ev: TextDocument) => void) { + return next(event); + } + + public willSave(event: TextDocumentWillSaveEvent, next: (ev: TextDocumentWillSaveEvent) => void) { + return next(event); + } + + public willSaveWaitUntil( + event: TextDocumentWillSaveEvent, + next: (ev: TextDocumentWillSaveEvent) => Thenable + ) { + return next(event); + } + + public provideCompletionItem( + document: TextDocument, + position: Position, + context: CompletionContext, + token: CancellationToken, + next: ProvideCompletionItemsSignature + ) { + if (isNotebookCell(document.uri)) { + const newDoc = this.converter.toOutgoingDocument(document); + const newPos = this.converter.toOutgoingPosition(document, position); + const result = next(newDoc, newPos, context, token); + if (isThenable(result)) { + return result.then(this.converter.toIncomingCompletions.bind(this.converter, document)); + } + return this.converter.toIncomingCompletions(document, result); + } + return next(document, position, context, token); + } + + public provideHover( + document: TextDocument, + position: Position, + token: CancellationToken, + next: ProvideHoverSignature + ) { + if (isNotebookCell(document.uri)) { + const newDoc = this.converter.toOutgoingDocument(document); + const newPos = this.converter.toOutgoingPosition(document, position); + const result = next(newDoc, newPos, token); + if (isThenable(result)) { + return result.then(this.converter.toIncomingHover.bind(this.converter, document)); + } + return this.converter.toIncomingHover(document, result); + } + return next(document, position, token); + } + + public resolveCompletionItem( + item: CompletionItem, + token: CancellationToken, + next: ResolveCompletionItemSignature + ): ProviderResult { + // Range should have already been remapped. + // tslint:disable-next-line: no-suspicious-comment + // TODO: What if the LS needs to read the range? It won't make sense. This might mean + // doing this at the extension level is not possible. + return next(item, token); + } + + public provideSignatureHelp( + document: TextDocument, + position: Position, + context: SignatureHelpContext, + token: CancellationToken, + next: ProvideSignatureHelpSignature + ): ProviderResult { + if (isNotebookCell(document.uri)) { + const newDoc = this.converter.toOutgoingDocument(document); + const newPos = this.converter.toOutgoingPosition(document, position); + return next(newDoc, newPos, context, token); + } + return next(document, position, context, token); + } + + public provideDefinition( + document: TextDocument, + position: Position, + token: CancellationToken, + next: ProvideDefinitionSignature + ): ProviderResult { + if (isNotebookCell(document.uri)) { + const newDoc = this.converter.toOutgoingDocument(document); + const newPos = this.converter.toOutgoingPosition(document, position); + const result = next(newDoc, newPos, token); + if (isThenable(result)) { + return result.then(this.converter.toIncomingLocations.bind(this.converter, document)); + } + return this.converter.toIncomingLocations(document, result); + } + return next(document, position, token); + } + + public provideReferences( + document: TextDocument, + position: Position, + options: { + includeDeclaration: boolean; + }, + token: CancellationToken, + next: ProvideReferencesSignature + ): ProviderResult { + if (isNotebookCell(document.uri)) { + const newDoc = this.converter.toOutgoingDocument(document); + const newPos = this.converter.toOutgoingPosition(document, position); + const result = next(newDoc, newPos, options, token); + if (isThenable(result)) { + return result.then(this.converter.toIncomingLocations.bind(this.converter, document)); + } + return this.converter.toIncomingLocations(document, result); + } + return next(document, position, options, token); + } + + public provideDocumentHighlights( + document: TextDocument, + position: Position, + token: CancellationToken, + next: ProvideDocumentHighlightsSignature + ): ProviderResult { + if (isNotebookCell(document.uri)) { + const newDoc = this.converter.toOutgoingDocument(document); + const newPos = this.converter.toOutgoingPosition(document, position); + const result = next(newDoc, newPos, token); + if (isThenable(result)) { + return result.then(this.converter.toIncomingHighlight.bind(this.converter, document)); + } + return this.converter.toIncomingHighlight(document, result); + } + return next(document, position, token); + } + + public provideDocumentSymbols( + document: TextDocument, + token: CancellationToken, + next: ProvideDocumentSymbolsSignature + ): ProviderResult { + if (isNotebookCell(document.uri)) { + const newDoc = this.converter.toOutgoingDocument(document); + const result = next(newDoc, token); + if (isThenable(result)) { + return result.then(this.converter.toIncomingSymbols.bind(this.converter, document)); + } + return this.converter.toIncomingSymbols(document, result); + } + return next(document, token); + } + + public provideWorkspaceSymbols( + query: string, + token: CancellationToken, + next: ProvideWorkspaceSymbolsSignature + ): ProviderResult { + const result = next(query, token); + if (isThenable(result)) { + return result.then(this.converter.toIncomingWorkspaceSymbols.bind(this)); + } + return this.converter.toIncomingWorkspaceSymbols(result); + } + + public provideCodeActions( + document: TextDocument, + range: Range, + context: CodeActionContext, + token: CancellationToken, + next: ProvideCodeActionsSignature + ): ProviderResult<(Command | CodeAction)[]> { + if (isNotebookCell(document.uri)) { + traceInfo(`provideCodeActions not currently supported for notebooks`); + return undefined; + } + return next(document, range, context, token); + } + + public provideCodeLenses( + document: TextDocument, + token: CancellationToken, + next: ProvideCodeLensesSignature + ): ProviderResult { + if (isNotebookCell(document.uri)) { + traceInfo(`provideCodeLenses not currently supported for notebooks`); + return undefined; + } + return next(document, token); + } + + public resolveCodeLens( + codeLens: CodeLens, + token: CancellationToken, + next: ResolveCodeLensSignature + ): ProviderResult { + // Range should have already been remapped. + // tslint:disable-next-line: no-suspicious-comment + // TODO: What if the LS needs to read the range? It won't make sense. This might mean + // doing this at the extension level is not possible. + return next(codeLens, token); + } + + public provideDocumentFormattingEdits( + document: TextDocument, + options: FormattingOptions, + token: CancellationToken, + next: ProvideDocumentFormattingEditsSignature + ): ProviderResult { + if (isNotebookCell(document.uri)) { + traceInfo(`provideDocumentFormattingEdits not currently supported for notebooks`); + return undefined; + } + return next(document, options, token); + } + + public provideDocumentRangeFormattingEdits( + document: TextDocument, + range: Range, + options: FormattingOptions, + token: CancellationToken, + next: ProvideDocumentRangeFormattingEditsSignature + ): ProviderResult { + if (isNotebookCell(document.uri)) { + traceInfo(`provideDocumentRangeFormattingEdits not currently supported for notebooks`); + return undefined; + } + return next(document, range, options, token); + } + + public provideOnTypeFormattingEdits( + document: TextDocument, + position: Position, + ch: string, + options: FormattingOptions, + token: CancellationToken, + next: ProvideOnTypeFormattingEditsSignature + ): ProviderResult { + if (isNotebookCell(document.uri)) { + traceInfo(`provideOnTypeFormattingEdits not currently supported for notebooks`); + return undefined; + } + return next(document, position, ch, options, token); + } + + public provideRenameEdits( + document: TextDocument, + position: Position, + newName: string, + token: CancellationToken, + next: ProvideRenameEditsSignature + ): ProviderResult { + if (isNotebookCell(document.uri)) { + traceInfo(`provideRenameEdits not currently supported for notebooks`); + return undefined; + } + return next(document, position, newName, token); + } + + public prepareRename( + document: TextDocument, + position: Position, + token: CancellationToken, + next: PrepareRenameSignature + ): ProviderResult< + | Range + | { + range: Range; + placeholder: string; + } + > { + if (isNotebookCell(document.uri)) { + traceInfo(`prepareRename not currently supported for notebooks`); + return undefined; + } + return next(document, position, token); + } + + public provideDocumentLinks( + document: TextDocument, + token: CancellationToken, + next: ProvideDocumentLinksSignature + ): ProviderResult { + if (isNotebookCell(document.uri)) { + traceInfo(`provideDocumentLinks not currently supported for notebooks`); + return undefined; + } + return next(document, token); + } + + public resolveDocumentLink( + link: DocumentLink, + token: CancellationToken, + next: ResolveDocumentLinkSignature + ): ProviderResult { + // Range should have already been remapped. + // tslint:disable-next-line: no-suspicious-comment + // TODO: What if the LS needs to read the range? It won't make sense. This might mean + // doing this at the extension level is not possible. + return next(link, token); + } + + public provideDeclaration( + document: TextDocument, + position: VPosition, + token: CancellationToken, + next: ProvideDeclarationSignature + ): ProviderResult { + if (isNotebookCell(document.uri)) { + traceInfo(`provideDeclaration not currently supported for notebooks`); + return undefined; + } + return next(document, position, token); + } + + public handleDiagnostics(uri: Uri, diagnostics: Diagnostic[], next: HandleDiagnosticsSignature) { + // Remap any wrapped documents so that diagnostics appear in the cells. Note that if we + // get a diagnostics list for our concated document, we have to tell VS code about EVERY cell. + // Otherwise old messages for cells that didn't change this time won't go away. + const newDiagMapping = this.converter.toIncomingDiagnosticsMap(uri, diagnostics); + [...newDiagMapping.keys()].forEach((k) => next(k, newDiagMapping.get(k)!)); + } +} diff --git a/src/test/activation/languageServer/manager.unit.test.ts b/src/test/activation/languageServer/manager.unit.test.ts index 09a24a0df6d0..d8130c399977 100644 --- a/src/test/activation/languageServer/manager.unit.test.ts +++ b/src/test/activation/languageServer/manager.unit.test.ts @@ -19,9 +19,6 @@ import { } from '../../../client/activation/types'; import { CommandManager } from '../../../client/common/application/commandManager'; import { ICommandManager } from '../../../client/common/application/types'; -import { ConfigurationService } from '../../../client/common/configuration/service'; -import { ExperimentsManager } from '../../../client/common/experiments/manager'; -import { IConfigurationService, IExperimentsManager } from '../../../client/common/types'; import { ServiceContainer } from '../../../client/ioc/container'; import { IServiceContainer } from '../../../client/ioc/types'; import { sleep } from '../../core'; @@ -38,8 +35,6 @@ suite('Language Server - Manager', () => { let lsExtension: ILanguageServerExtension; let onChangeAnalysisHandler: Function; let folderService: ILanguageServerFolderService; - let experimentsManager: IExperimentsManager; - let configService: IConfigurationService; let commandManager: ICommandManager; const languageClientOptions = ({ x: 1 } as any) as LanguageClientOptions; setup(() => { @@ -48,8 +43,6 @@ suite('Language Server - Manager', () => { languageServer = mock(DotNetLanguageServerProxy); lsExtension = mock(LanguageServerExtension); folderService = mock(DotNetLanguageServerFolderService); - experimentsManager = mock(ExperimentsManager); - configService = mock(ConfigurationService); commandManager = mock(CommandManager); const disposable = mock(Disposable); @@ -60,8 +53,6 @@ suite('Language Server - Manager', () => { instance(analysisOptions), instance(lsExtension), instance(folderService), - instance(experimentsManager), - instance(configService), instance(commandManager) ); }); diff --git a/src/test/insiders/languageServer.insiders.test.ts b/src/test/insiders/languageServer.insiders.test.ts new file mode 100644 index 000000000000..9fefe15c90dd --- /dev/null +++ b/src/test/insiders/languageServer.insiders.test.ts @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +// tslint:disable:max-func-body-length no-invalid-this no-any + +import * as assert from 'assert'; +import { expect } from 'chai'; +import * as path from 'path'; +import * as vscode from 'vscode'; +import { updateSetting } from '../common'; +import { EXTENSION_ROOT_DIR_FOR_TESTS } from '../constants'; +import { sleep } from '../core'; +import { closeActiveWindows, initialize, initializeTest } from '../initialize'; +import { openFileAndWaitForLS, openNotebookAndWaitForLS } from '../smoke/common'; + +const fileDefinitions = path.join( + EXTENSION_ROOT_DIR_FOR_TESTS, + 'src', + 'testMultiRootWkspc', + 'smokeTests', + 'definitions.py' +); + +const notebookDefinitions = path.join( + EXTENSION_ROOT_DIR_FOR_TESTS, + 'src', + 'testMultiRootWkspc', + 'smokeTests', + 'definitions.ipynb' +); + +suite('Insiders Test: Language Server', () => { + suiteSetup(async function () { + // This test should only run in the insiders build + if (vscode.env.appName.includes('Insider')) { + await updateSetting( + 'linting.ignorePatterns', + ['**/dir1/**'], + vscode.workspace.workspaceFolders![0].uri, + vscode.ConfigurationTarget.WorkspaceFolder + ); + await initialize(); + } else { + this.skip(); + } + }); + setup(async () => { + await initializeTest(); + await closeActiveWindows(); + }); + suiteTeardown(async () => { + await closeActiveWindows(); + await updateSetting( + 'linting.ignorePatterns', + undefined, + vscode.workspace.workspaceFolders![0].uri, + vscode.ConfigurationTarget.WorkspaceFolder + ); + }); + teardown(closeActiveWindows); + + test('Definitions', async () => { + const startPosition = new vscode.Position(13, 6); + const textDocument = await openFileAndWaitForLS(fileDefinitions); + let tested = false; + for (let i = 0; i < 5; i += 1) { + const locations = await vscode.commands.executeCommand( + 'vscode.executeDefinitionProvider', + textDocument.uri, + startPosition + ); + if (locations && locations.length > 0) { + expect(locations![0].uri.fsPath).to.contain(path.basename(fileDefinitions)); + tested = true; + break; + } else { + // Wait for LS to start. + await sleep(5_000); + } + } + if (!tested) { + assert.fail('Failled to test definitions'); + } + }); + test('Notebooks', async () => { + const startPosition = new vscode.Position(0, 6); + const notebookDocument = await openNotebookAndWaitForLS(notebookDefinitions); + let tested = false; + for (let i = 0; i < 5; i += 1) { + const locations = await vscode.commands.executeCommand( + 'vscode.executeDefinitionProvider', + notebookDocument.cells[2].uri, // Second cell should have a function with the decorator on it + startPosition + ); + if (locations && locations.length > 0) { + expect(locations![0].uri.fsPath).to.contain(path.basename(notebookDefinitions)); + tested = true; + break; + } else { + // Wait for LS to start. + await sleep(5_000); + } + } + if (!tested) { + assert.fail('Failled to test definitions'); + } + }); +}); diff --git a/src/test/smoke/common.ts b/src/test/smoke/common.ts index 4c8fa12b97cd..9484748bdae2 100644 --- a/src/test/smoke/common.ts +++ b/src/test/smoke/common.ts @@ -10,6 +10,7 @@ import * as fs from 'fs-extra'; import * as glob from 'glob'; import * as path from 'path'; import * as vscode from 'vscode'; +import { JUPYTER_EXTENSION_ID } from '../../client/common/constants'; import { SMOKE_TEST_EXTENSIONS_DIR } from '../constants'; import { noop, sleep } from '../core'; @@ -41,6 +42,26 @@ export async function enableJedi(enable: boolean | undefined) { } await updateSetting('languageServer', 'Jedi'); } + +export async function openNotebookAndWaitForLS(file: string): Promise { + await verifyExtensionIsAvailable(JUPYTER_EXTENSION_ID); + await vscode.commands.executeCommand('vscode.openWith', vscode.Uri.file(file), 'jupyter-notebook'); + const notebook = vscode.notebook.activeNotebookEditor; + assert(notebook, 'Notebook did not open'); + + // Make sure LS completes file loading and analysis. + // In test mode it awaits for the completion before trying + // to fetch data for completion, hover.etc. + await vscode.commands.executeCommand( + 'vscode.executeCompletionItemProvider', + notebook.document.cells[0].uri, + new vscode.Position(0, 0) + ); + // For for LS to get extracted. + await sleep(10_000); + return notebook.document; +} + export async function openFileAndWaitForLS(file: string): Promise { const textDocument = await vscode.workspace.openTextDocument(file); await vscode.window.showTextDocument(textDocument); @@ -57,3 +78,9 @@ export async function openFileAndWaitForLS(file: string): Promise { + const extension = vscode.extensions.all.find((e) => e.id === extensionId); + assert.ok(extension, `Extension ${extensionId} not installed.`); + await extension.activate(); +} diff --git a/src/test/smoke/datascience.smoke.test.ts b/src/test/smoke/datascience.smoke.test.ts new file mode 100644 index 000000000000..138bce12d19b --- /dev/null +++ b/src/test/smoke/datascience.smoke.test.ts @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +// tslint:disable:max-func-body-length no-invalid-this no-any + +import * as fs from 'fs-extra'; +import * as path from 'path'; +import * as vscode from 'vscode'; +import { JUPYTER_EXTENSION_ID } from '../../client/common/constants'; +import { openFile, setAutoSaveDelayInWorkspaceRoot, waitForCondition } from '../common'; +import { EXTENSION_ROOT_DIR_FOR_TESTS, IS_SMOKE_TEST } from '../constants'; +import { noop, sleep } from '../core'; +import { closeActiveWindows, initialize, initializeTest } from '../initialize'; +import { verifyExtensionIsAvailable } from './common'; + +const timeoutForCellToRun = 3 * 60 * 1_000; + +suite('Smoke Test: Interactive Window', () => { + suiteSetup(async function () { + if (!IS_SMOKE_TEST) { + return this.skip(); + } + await verifyExtensionIsAvailable(JUPYTER_EXTENSION_ID); + await initialize(); + await setAutoSaveDelayInWorkspaceRoot(1); + const jupyterConfig = vscode.workspace.getConfiguration('jupyter', (null as any) as vscode.Uri); + await jupyterConfig.update('alwaysTrustNotebooks', true, true); + }); + setup(initializeTest); + suiteTeardown(closeActiveWindows); + teardown(closeActiveWindows); + + test('Run Cell in interactive window', async () => { + const file = path.join( + EXTENSION_ROOT_DIR_FOR_TESTS, + 'src', + 'test', + 'pythonFiles', + 'datascience', + 'simple_note_book.py' + ); + const outputFile = path.join(path.dirname(file), 'ds.log'); + if (await fs.pathExists(outputFile)) { + await fs.unlink(outputFile); + } + const textDocument = await openFile(file); + + // Wait for code lenses to get detected. + await sleep(1_000); + + await vscode.commands.executeCommand('jupyter.runallcells', textDocument.uri); + const checkIfFileHasBeenCreated = () => fs.pathExists(outputFile); + await waitForCondition(checkIfFileHasBeenCreated, timeoutForCellToRun, `"${outputFile}" file not created`); + }).timeout(timeoutForCellToRun); + + test('Run Cell in native editor', async () => { + const file = path.join( + EXTENSION_ROOT_DIR_FOR_TESTS, + 'src', + 'test', + 'pythonFiles', + 'datascience', + 'simple_nb.ipynb' + ); + const fileContents = await fs.readFile(file, { encoding: 'utf-8' }); + const outputFile = path.join(path.dirname(file), 'ds_n.log'); + await fs.writeFile(file, fileContents.replace("'ds_n.log'", `'${outputFile.replace(/\\/g, '/')}'`), { + encoding: 'utf-8' + }); + if (await fs.pathExists(outputFile)) { + await fs.unlink(outputFile); + } + // Ignore exceptions (as native editor closes the document as soon as its opened); + await openFile(file).catch(noop); + + // Wait for 15 seconds for notebook to launch. + // Unfortunately there's no way to know for sure it has completely loaded. + await sleep(15_000); + + await vscode.commands.executeCommand('jupyter.notebookeditor.runallcells'); + const checkIfFileHasBeenCreated = () => fs.pathExists(outputFile); + await waitForCondition(checkIfFileHasBeenCreated, timeoutForCellToRun, `"${outputFile}" file not created`); + + // Give time for the file to be saved before we shutdown + await sleep(300); + }).timeout(timeoutForCellToRun); +}); diff --git a/src/test/standardTest.ts b/src/test/standardTest.ts index 870bce78b598..268a9630f0e5 100644 --- a/src/test/standardTest.ts +++ b/src/test/standardTest.ts @@ -1,7 +1,10 @@ // tslint:disable:no-console +import { spawnSync } from 'child_process'; +import * as fs from 'fs-extra'; import * as path from 'path'; -import { runTests } from 'vscode-test'; +import { downloadAndUnzipVSCode, resolveCliPathFromVSCodeExecutablePath, runTests } from 'vscode-test'; +import { EXTENSION_ROOT_DIR, PYLANCE_EXTENSION_ID } from '../client/common/constants'; import { EXTENSION_ROOT_DIR_FOR_TESTS } from './constants'; // If running smoke tests, we don't have access to this. @@ -10,6 +13,12 @@ if (process.env.TEST_FILES_SUFFIX !== 'smoke.test') { const logger = require('./testLogger'); logger.initializeLogger(); } +function requiresJupyterExtensionToBeInstalled() { + return process.env.INSTALL_JUPYTER_EXTENSION === 'true'; +} +function requiresPylanceExtensionToBeInstalled() { + return process.env.INSTALL_PYLANCE_EXTENSION === 'true'; +} process.env.IS_CI_SERVER_TEST_DEBUGGER = ''; process.env.VSC_PYTHON_CI_TEST = '1'; @@ -22,20 +31,76 @@ const extensionDevelopmentPath = process.env.CODE_EXTENSIONS_PATH const channel = process.env.VSC_PYTHON_CI_TEST_VSC_CHANNEL || 'stable'; -function start() { +/** + * Smoke tests & tests running in VSCode require Jupyter extension to be installed. + */ +async function installJupyterExtension(vscodeExecutablePath: string) { + const jupyterVSIX = process.env.VSIX_NAME_JUPYTER + ? path.join(EXTENSION_ROOT_DIR, process.env.VSIX_NAME_JUPYTER) + : undefined; + if (!requiresJupyterExtensionToBeInstalled() || !jupyterVSIX) { + console.info('Jupyter Extension not required'); + return; + } + console.info('Installing Jupyter Extension'); + const cliPath = resolveCliPathFromVSCodeExecutablePath(vscodeExecutablePath); + spawnSync(cliPath, ['--install-extension', jupyterVSIX], { + encoding: 'utf-8', + stdio: 'inherit' + }); +} + +async function installPylanceExtension(vscodeExecutablePath: string) { + if (!requiresPylanceExtensionToBeInstalled()) { + console.info('Pylance Extension not required'); + return; + } + console.info('Installing Pylance Extension'); + const cliPath = resolveCliPathFromVSCodeExecutablePath(vscodeExecutablePath); + + // For now install pylance from the marketplace + spawnSync(cliPath, ['--install-extension', PYLANCE_EXTENSION_ID], { + encoding: 'utf-8', + stdio: 'inherit' + }); + + // Make sure to enable it by writing to our workspace path settings + await fs.ensureDir(path.join(workspacePath, '.vscode')); + const settingsPath = path.join(workspacePath, '.vscode', 'settings.json'); + if (await fs.pathExists(settingsPath)) { + let settings = JSON.parse(await fs.readFile(settingsPath, 'utf-8')); + settings = { ...settings, 'python.languageServer': 'Pylance' }; + await fs.writeFile(settingsPath, JSON.stringify(settings)); + } else { + const settings = `{ "python.languageServer": "Pylance" }`; + await fs.writeFile(settingsPath, settings); + } +} + +async function start() { console.log('*'.repeat(100)); console.log('Start Standard tests'); - runTests({ + const vscodeExecutablePath = await downloadAndUnzipVSCode(channel); + const baseLaunchArgs = + requiresJupyterExtensionToBeInstalled() || requiresPylanceExtensionToBeInstalled() + ? [] + : ['--disable-extensions']; + await installJupyterExtension(vscodeExecutablePath); + await installPylanceExtension(vscodeExecutablePath); + const launchArgs = baseLaunchArgs + .concat([workspacePath]) + .concat(channel === 'insiders' ? ['--enable-proposed-api'] : []) + .concat(['--timeout', '5000']); + console.log(`Starting vscode ${channel} with args ${launchArgs.join(' ')}`); + await runTests({ extensionDevelopmentPath: extensionDevelopmentPath, - extensionTestsPath: path.join(EXTENSION_ROOT_DIR_FOR_TESTS, 'out', 'test', 'index'), - launchArgs: ['--disable-extensions', workspacePath] - .concat(channel === 'insiders' ? ['--enable-proposed-api'] : []) - .concat(['--timeout', '5000']), + extensionTestsPath: path.join(EXTENSION_ROOT_DIR_FOR_TESTS, 'out', 'test'), + launchArgs, version: channel, extensionTestsEnv: { ...process.env, UITEST_DISABLE_INSIDERS: '1' } - }).catch((ex) => { - console.error('End Standard tests (with errors)', ex); - process.exit(1); }); } -start(); +start().catch((ex) => { + console.error('End Standard tests (with errors)', ex); + process.exit(1); +}); diff --git a/src/testMultiRootWkspc/smokeTests/definitions.ipynb b/src/testMultiRootWkspc/smokeTests/definitions.ipynb new file mode 100644 index 000000000000..ee5427bbff0f --- /dev/null +++ b/src/testMultiRootWkspc/smokeTests/definitions.ipynb @@ -0,0 +1,88 @@ +{ + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": 3 + }, + "orig_nbformat": 2 + }, + "nbformat": 4, + "nbformat_minor": 2, + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from contextlib import contextmanager" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def my_decorator(fn):\n", + " \"\"\"\n", + " This is my decorator.\n", + " \"\"\"\n", + " def wrapper(*args, **kwargs):\n", + " \"\"\"\n", + " This is the wrapper.\n", + " \"\"\"\n", + " return 42\n", + " return wrapper\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@my_decorator\n", + "def thing(arg):\n", + " \"\"\"\n", + " Thing which is decorated.\n", + " \"\"\"\n", + " pass\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@contextmanager\n", + "def my_context_manager():\n", + " \"\"\"\n", + " This is my context manager.\n", + " \"\"\"\n", + " print(\"before\")\n", + " yield\n", + " print(\"after\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "with my_context_manager():\n", + " thing(19)" + ] + } + ] +} \ No newline at end of file