Skip to content

Commit 063c33d

Browse files
rchiodorchiodo
authored
Fix data frame info while debugging (microsoft#11713)
* Potential idea * Make debug variables work after restart and get shape/count * Add restart testing * Force refresh during test * Make stepping with variables work * All tests working
1 parent ae74c2a commit 063c33d

File tree

17 files changed

+453
-237
lines changed

17 files changed

+453
-237
lines changed

‎news/3 Code Health/11657.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Get shape and count when showing debugger variables.

‎pythonFiles/vscode_datascience_helpers/dataframes/vscodeGetVariableInfo.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Query Jupyter server for the info about a dataframe
2+
import json as _VSCODE_json
3+
import pandas as _VSCODE_pd
4+
import pandas.io.json as _VSCODE_pd_json
5+
import builtins as _VSCODE_builtins
6+
import vscodeDataFrameHelpers as _VSCODE_dataFrameHelpers
7+
8+
# Function to do our work. It will return the object
9+
def _VSCODE_getVariableInfo(var):
10+
# Start out without the information
11+
result = {}
12+
result["shape"] = ""
13+
result["count"] = 0
14+
15+
# Find shape and count if available
16+
if hasattr(var, "shape"):
17+
try:
18+
# Get a bit more restrictive with exactly what we want to count as a shape, since anything can define it
19+
if isinstance(var.shape, tuple):
20+
_VSCODE_shapeStr = str(var.shape)
21+
if (
22+
len(_VSCODE_shapeStr) >= 3
23+
and _VSCODE_shapeStr[0] == "("
24+
and _VSCODE_shapeStr[-1] == ")"
25+
and "," in _VSCODE_shapeStr
26+
):
27+
result["shape"] = _VSCODE_shapeStr
28+
del _VSCODE_shapeStr
29+
except TypeError:
30+
pass
31+
32+
if hasattr(var, "__len__"):
33+
try:
34+
result["count"] = len(var)
35+
except TypeError:
36+
pass
37+
38+
# return our json object as a string
39+
return _VSCODE_json.dumps(result)

‎src/client/datascience/constants.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -385,6 +385,9 @@ export namespace DataFrameLoading {
385385
export const DataFrameRowImportName = '_VSCODE_RowImport';
386386
export const DataFrameRowImport = `import vscodeGetDataFrameRows as ${DataFrameRowImportName}`;
387387
export const DataFrameRowFunc = `${DataFrameRowImportName}._VSCODE_getDataFrameRows`;
388+
export const VariableInfoImportName = '_VSCODE_VariableImport';
389+
export const VariableInfoImport = `import vscodeGetVariableInfo as ${VariableInfoImportName}`;
390+
export const VariableInfoFunc = `${VariableInfoImportName}._VSCODE_getVariableInfo`;
388391
}
389392

390393
export namespace Identifiers {

‎src/client/datascience/interactive-common/intellisense/intellisenseProvider.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -752,10 +752,10 @@ export class IntellisenseProvider implements IInteractiveWindowListener {
752752
if (wordAtPosition) {
753753
const notebook = await this.getNotebook();
754754
if (notebook) {
755-
const variable = await this.variableProvider.getMatchingVariable(notebook, wordAtPosition);
756-
if (variable && variable.value && variable.name) {
755+
const value = await this.variableProvider.getMatchingVariableValue(notebook, wordAtPosition);
756+
if (value) {
757757
return {
758-
contents: [`${variable.name} : ${variable.value}`]
758+
contents: [`${wordAtPosition} : ${value}`]
759759
};
760760
}
761761
}

‎src/client/datascience/jupyter/debuggerVariables.ts‎

Lines changed: 146 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,16 @@
33
'use strict';
44
import { inject, injectable, named } from 'inversify';
55

6-
import { DebugAdapterTracker, Event, EventEmitter } from 'vscode';
6+
import { DebugAdapterTracker, Disposable, Event, EventEmitter } from 'vscode';
77
import { DebugProtocol } from 'vscode-debugprotocol';
88
import { IDebugService } from '../../common/application/types';
99
import { traceError } from '../../common/logger';
1010
import { IConfigurationService, Resource } from '../../common/types';
1111
import { DataFrameLoading, Identifiers } from '../constants';
1212
import {
13+
IConditionalJupyterVariables,
1314
IJupyterDebugService,
1415
IJupyterVariable,
15-
IJupyterVariables,
1616
IJupyterVariablesRequest,
1717
IJupyterVariablesResponse,
1818
INotebook
@@ -22,11 +22,13 @@ const DataViewableTypes: Set<string> = new Set<string>(['DataFrame', 'list', 'di
2222
const KnownExcludedVariables = new Set<string>(['In', 'Out', 'exit', 'quit']);
2323

2424
@injectable()
25-
export class DebuggerVariables implements IJupyterVariables, DebugAdapterTracker {
26-
private imported = false;
25+
export class DebuggerVariables implements IConditionalJupyterVariables, DebugAdapterTracker {
2726
private refreshEventEmitter = new EventEmitter<void>();
2827
private lastKnownVariables: IJupyterVariable[] = [];
2928
private topMostFrameId = 0;
29+
private importedIntoKernel = new Set<string>();
30+
private watchedNotebooks = new Map<string, Disposable[]>();
31+
private debuggingStarted = false;
3032
constructor(
3133
@inject(IJupyterDebugService) @named(Identifiers.MULTIPLEXING_DEBUGSERVICE) private debugService: IDebugService,
3234
@inject(IConfigurationService) private configService: IConfigurationService
@@ -36,50 +38,68 @@ export class DebuggerVariables implements IJupyterVariables, DebugAdapterTracker
3638
return this.refreshEventEmitter.event;
3739
}
3840

41+
public get active(): boolean {
42+
return this.debugService.activeDebugSession !== undefined && this.debuggingStarted;
43+
}
44+
3945
// IJupyterVariables implementation
4046
public async getVariables(
41-
_notebook: INotebook,
47+
notebook: INotebook,
4248
request: IJupyterVariablesRequest
4349
): Promise<IJupyterVariablesResponse> {
50+
// Listen to notebook events if we haven't already
51+
this.watchNotebook(notebook);
52+
4453
const result: IJupyterVariablesResponse = {
4554
executionCount: request.executionCount,
4655
pageStartIndex: 0,
4756
pageResponse: [],
4857
totalCount: 0
4958
};
5059

51-
if (this.debugService.activeDebugSession) {
52-
result.pageResponse = this.lastKnownVariables;
60+
if (this.active) {
61+
const startPos = request.startIndex ? request.startIndex : 0;
62+
const chunkSize = request.pageSize ? request.pageSize : 100;
63+
result.pageStartIndex = startPos;
64+
65+
// Do one at a time. All at once doesn't work as they all have to wait for each other anyway
66+
for (let i = startPos; i < startPos + chunkSize && i < this.lastKnownVariables.length; i += 1) {
67+
const fullVariable = !this.lastKnownVariables[i].truncated
68+
? this.lastKnownVariables[i]
69+
: await this.getFullVariable(this.lastKnownVariables[i], notebook);
70+
this.lastKnownVariables[i] = fullVariable;
71+
result.pageResponse.push(fullVariable);
72+
}
5373
result.totalCount = this.lastKnownVariables.length;
5474
}
5575

5676
return result;
5777
}
5878

59-
public async getMatchingVariable(_notebook: INotebook, name: string): Promise<IJupyterVariable | undefined> {
60-
if (this.debugService.activeDebugSession) {
61-
return this.lastKnownVariables.find((v) => v.name === name);
79+
public async getMatchingVariableValue(_notebook: INotebook, name: string): Promise<string | undefined> {
80+
if (this.active) {
81+
// Note, full variable results isn't necessary for this call. It only really needs the variable value.
82+
return this.lastKnownVariables.find((v) => v.name === name)?.value;
6283
}
6384
}
6485

65-
public async getDataFrameInfo(targetVariable: IJupyterVariable, _notebook: INotebook): Promise<IJupyterVariable> {
66-
if (!this.debugService.activeDebugSession) {
86+
public async getDataFrameInfo(targetVariable: IJupyterVariable, notebook: INotebook): Promise<IJupyterVariable> {
87+
if (!this.active) {
6788
// No active server just return the unchanged target variable
6889
return targetVariable;
6990
}
91+
// Listen to notebook events if we haven't already
92+
this.watchNotebook(notebook);
7093

7194
// See if we imported or not into the kernel our special function
72-
if (!this.imported) {
73-
this.imported = await this.importDataFrameScripts();
74-
}
95+
await this.importDataFrameScripts(notebook);
7596

7697
// Then eval calling the main function with our target variable
77-
const results = await this.debugService.activeDebugSession.customRequest('evaluate', {
78-
expression: `${DataFrameLoading.DataFrameInfoFunc}(${targetVariable.name})`,
98+
const results = await this.evaluate(
99+
`${DataFrameLoading.DataFrameInfoFunc}(${targetVariable.name})`,
79100
// tslint:disable-next-line: no-any
80-
frameId: (targetVariable as any).frameId || this.topMostFrameId,
81-
context: 'repl'
82-
});
101+
(targetVariable as any).frameId
102+
);
83103

84104
// Results should be the updated variable.
85105
return {
@@ -90,7 +110,7 @@ export class DebuggerVariables implements IJupyterVariables, DebugAdapterTracker
90110

91111
public async getDataFrameRows(
92112
targetVariable: IJupyterVariable,
93-
_notebook: INotebook,
113+
notebook: INotebook,
94114
start: number,
95115
end: number
96116
): Promise<{}> {
@@ -99,58 +119,115 @@ export class DebuggerVariables implements IJupyterVariables, DebugAdapterTracker
99119
// No active server just return no rows
100120
return {};
101121
}
122+
// Listen to notebook events if we haven't already
123+
this.watchNotebook(notebook);
102124

103125
// See if we imported or not into the kernel our special function
104-
if (!this.imported) {
105-
this.imported = await this.importDataFrameScripts();
106-
}
126+
await this.importDataFrameScripts(notebook);
107127

108-
// Then eval calling the main function with our target variable
109-
const minnedEnd = Math.min(end, targetVariable.rowCount || 0);
110-
const results = await this.debugService.activeDebugSession.customRequest('evaluate', {
111-
expression: `${DataFrameLoading.DataFrameRowFunc}(${targetVariable.name}, ${start}, ${minnedEnd})`,
112-
// tslint:disable-next-line: no-any
113-
frameId: (targetVariable as any).frameId || this.topMostFrameId,
114-
context: 'repl'
115-
});
128+
// Since the debugger splits up long requests, split this based on the number of items.
116129

117-
// Results should be the row.
118-
return JSON.parse(results.result.slice(1, -1));
130+
// Maximum 100 cells at a time or one row
131+
// tslint:disable-next-line: no-any
132+
let output: any;
133+
const minnedEnd = Math.min(targetVariable.rowCount || 0, end);
134+
const totalRowCount = end - start;
135+
const cellsPerRow = targetVariable.columns!.length;
136+
const chunkSize = Math.floor(Math.max(1, Math.min(100 / cellsPerRow, totalRowCount / cellsPerRow)));
137+
for (let pos = start; pos < end; pos += chunkSize) {
138+
const chunkEnd = Math.min(pos + chunkSize, minnedEnd);
139+
const results = await this.evaluate(
140+
`${DataFrameLoading.DataFrameRowFunc}(${targetVariable.name}, ${pos}, ${chunkEnd})`,
141+
// tslint:disable-next-line: no-any
142+
(targetVariable as any).frameId
143+
);
144+
const chunkResults = JSON.parse(results.result.slice(1, -1));
145+
if (output && output.data) {
146+
output = {
147+
...output,
148+
data: output.data.concat(chunkResults.data)
149+
};
150+
} else {
151+
output = chunkResults;
152+
}
153+
}
154+
155+
// Results should be the rows.
156+
return output;
119157
}
120158

121-
public onDidSendMessage(message: DebugProtocol.Response) {
159+
// tslint:disable-next-line: no-any
160+
public onDidSendMessage(message: any) {
122161
// If using the interactive debugger, update our variables.
123-
if (message.type === 'response' && message.command === 'variables') {
162+
if (message.type === 'response' && message.command === 'initialize') {
163+
this.debuggingStarted = true;
164+
} else if (message.type === 'response' && message.command === 'variables') {
124165
// tslint:disable-next-line: no-suspicious-comment
125166
// TODO: Figure out what resource to use
126167
this.updateVariables(undefined, message as DebugProtocol.VariablesResponse);
127168
} else if (message.type === 'response' && message.command === 'stackTrace') {
128169
// This should be the top frame. We need to use this to compute the value of a variable
129170
this.updateStackFrame(message as DebugProtocol.StackTraceResponse);
171+
} else if (message.type === 'event' && message.event === 'terminated') {
172+
// When the debugger exits, make sure the variables are cleared
173+
this.lastKnownVariables = [];
174+
this.topMostFrameId = 0;
175+
this.debuggingStarted = false;
176+
this.refreshEventEmitter.fire();
130177
}
131178
}
132179

180+
private watchNotebook(notebook: INotebook) {
181+
const key = notebook.identity.toString();
182+
if (!this.watchedNotebooks.has(key)) {
183+
const disposables: Disposable[] = [];
184+
disposables.push(notebook.onKernelChanged(this.resetImport.bind(this, key)));
185+
disposables.push(notebook.onKernelRestarted(this.resetImport.bind(this, key)));
186+
disposables.push(
187+
notebook.onDisposed(() => {
188+
this.resetImport(key);
189+
disposables.forEach((d) => d.dispose());
190+
this.watchedNotebooks.delete(key);
191+
})
192+
);
193+
this.watchedNotebooks.set(key, disposables);
194+
}
195+
}
196+
197+
private resetImport(key: string) {
198+
this.importedIntoKernel.delete(key);
199+
}
200+
133201
// tslint:disable-next-line: no-any
134-
private async evalute(code: string): Promise<any> {
202+
private async evaluate(code: string, frameId?: number): Promise<any> {
135203
if (this.debugService.activeDebugSession) {
136-
return this.debugService.activeDebugSession.customRequest('evaluate', {
204+
const results = await this.debugService.activeDebugSession.customRequest('evaluate', {
137205
expression: code,
138-
frameId: this.topMostFrameId,
206+
frameId: this.topMostFrameId || frameId,
139207
context: 'repl'
140208
});
209+
if (results && results.result !== 'None') {
210+
return results;
211+
} else {
212+
traceError(`Cannot evaluate ${code}`);
213+
return undefined;
214+
}
141215
}
142216
throw Error('Debugger is not active, cannot evaluate.');
143217
}
144218

145-
private async importDataFrameScripts(): Promise<boolean> {
219+
private async importDataFrameScripts(notebook: INotebook): Promise<void> {
146220
try {
147-
await this.evalute(DataFrameLoading.DataFrameSysImport);
148-
await this.evalute(DataFrameLoading.DataFrameInfoImport);
149-
await this.evalute(DataFrameLoading.DataFrameRowImport);
150-
return true;
221+
const key = notebook.identity.toString();
222+
if (!this.importedIntoKernel.has(key)) {
223+
await this.evaluate(DataFrameLoading.DataFrameSysImport);
224+
await this.evaluate(DataFrameLoading.DataFrameInfoImport);
225+
await this.evaluate(DataFrameLoading.DataFrameRowImport);
226+
await this.evaluate(DataFrameLoading.VariableInfoImport);
227+
this.importedIntoKernel.add(key);
228+
}
151229
} catch (exc) {
152230
traceError('Error attempting to import in debugger', exc);
153-
return false;
154231
}
155232
}
156233

@@ -160,6 +237,29 @@ export class DebuggerVariables implements IJupyterVariables, DebugAdapterTracker
160237
}
161238
}
162239

240+
private async getFullVariable(variable: IJupyterVariable, notebook: INotebook): Promise<IJupyterVariable> {
241+
// See if we imported or not into the kernel our special function
242+
await this.importDataFrameScripts(notebook);
243+
244+
// Then eval calling the variable info function with our target variable
245+
const results = await this.evaluate(
246+
`${DataFrameLoading.VariableInfoFunc}(${variable.name})`,
247+
// tslint:disable-next-line: no-any
248+
(variable as any).frameId
249+
);
250+
if (results) {
251+
// Results should be the updated variable.
252+
return {
253+
...variable,
254+
truncated: false,
255+
...JSON.parse(results.result.slice(1, -1))
256+
};
257+
} else {
258+
// If no results, just return current value. Better than nothing.
259+
return variable;
260+
}
261+
}
262+
163263
private updateVariables(resource: Resource, variablesResponse: DebugProtocol.VariablesResponse) {
164264
const exclusionList = this.configService.getSettings(resource).datascience.variableExplorerExclude
165265
? this.configService.getSettings().datascience.variableExplorerExclude?.split(';')
@@ -193,7 +293,7 @@ export class DebuggerVariables implements IJupyterVariables, DebugAdapterTracker
193293
size: 0,
194294
supportsDataExplorer: DataViewableTypes.has(v.type || ''),
195295
value: v.value,
196-
truncated: false,
296+
truncated: true,
197297
frameId: v.variablesReference
198298
};
199299
});

‎src/client/datascience/jupyter/jupyterDebugger.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import * as uuid from 'uuid/v4';
88
import { DebugConfiguration } from 'vscode';
99
import * as vsls from 'vsls/vscode';
1010
import { concatMultilineStringOutput } from '../../../datascience-ui/common';
11+
import { ServerStatus } from '../../../datascience-ui/interactive-common/mainState';
1112
import { IApplicationShell } from '../../common/application/types';
1213
import { DebugAdapterNewPtvsd } from '../../common/experimentGroups';
1314
import { traceError, traceInfo, traceWarning } from '../../common/logger';
@@ -101,7 +102,9 @@ export class JupyterDebugger implements IJupyterDebugger, ICellHashListener {
101102

102103
// Disable tracing after we disconnect because we don't want to step through this
103104
// code if the user was in step mode.
104-
await this.executeSilently(notebook, this.tracingDisableCode);
105+
if (notebook.status !== ServerStatus.Dead && notebook.status !== ServerStatus.NotStarted) {
106+
await this.executeSilently(notebook, this.tracingDisableCode);
107+
}
105108
}
106109
}
107110

0 commit comments

Comments
 (0)