Skip to content

Commit 417c063

Browse files
liuxy0551liuxy0551
committed
feat: add splitInputBySeparator
1 parent 18e55e4 commit 417c063

File tree

8 files changed

+111
-67
lines changed

8 files changed

+111
-67
lines changed

‎src/parser/common/basicSQL.ts

+90-18
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ import type { SplitListener } from './splitListener';
2121
import type { EntityCollector } from './entityCollector';
2222
import { EntityContext } from './entityCollector';
2323

24+
const SEPARATOR: string = ';';
25+
2426
/**
2527
* Basic SQL class, every sql needs extends it.
2628
*/
@@ -65,13 +67,11 @@ export abstract class BasicSQL<
6567
* @param candidates candidate list
6668
* @param allTokens all tokens from input
6769
* @param caretTokenIndex tokenIndex of caretPosition
68-
* @param tokenIndexOffset offset of the tokenIndex in the candidates compared to the tokenIndex in allTokens
6970
*/
7071
protected abstract processCandidates(
7172
candidates: CandidatesCollection,
7273
allTokens: Token[],
73-
caretTokenIndex: number,
74-
tokenIndexOffset: number
74+
caretTokenIndex: number
7575
): Suggestions<Token>;
7676

7777
/**
@@ -251,6 +251,67 @@ export abstract class BasicSQL<
251251
return res;
252252
}
253253

254+
/**
255+
* Get the smaller range of input
256+
* @param input string
257+
* @param caretTokenIndex tokenIndex of caretPosition
258+
* @returns inputSlice: string, caretTokenIndex: number, allTokens: Token[]
259+
*/
260+
private splitInputBySeparator(
261+
input: string,
262+
caretTokenIndex: number
263+
): { inputSlice: string; caretTokenIndex: number; allTokens: Token[] } {
264+
const allTokens = this.getAllTokens(input);
265+
266+
/**
267+
* set startToken
268+
*/
269+
let startToken: Token | null = null;
270+
for (let tokenIndex = caretTokenIndex; tokenIndex >= 0; tokenIndex--) {
271+
const token = allTokens[tokenIndex];
272+
if (token?.text === SEPARATOR) {
273+
startToken = allTokens[tokenIndex + 1];
274+
break;
275+
}
276+
}
277+
if (startToken === null) {
278+
startToken = allTokens[0];
279+
}
280+
281+
/**
282+
* set stopToken
283+
*/
284+
let stopToken: Token | null = null;
285+
for (let tokenIndex = caretTokenIndex; tokenIndex < allTokens.length; tokenIndex++) {
286+
const token = allTokens[tokenIndex];
287+
if (token?.text === SEPARATOR) {
288+
stopToken = token;
289+
break;
290+
}
291+
}
292+
if (stopToken === null) {
293+
stopToken = allTokens[allTokens.length - 1];
294+
}
295+
296+
let startIndex = startToken?.start ?? 0;
297+
let stopIndex = stopToken?.stop + 1 ?? input.length;
298+
299+
/**
300+
* Save offset of the tokenIndex in the range of input
301+
* compared to the tokenIndex in the whole input
302+
*/
303+
const tokenIndexOffset = startToken?.tokenIndex ?? 0;
304+
const _caretTokenIndex = caretTokenIndex - tokenIndexOffset;
305+
306+
/**
307+
* Get the smaller range of _input
308+
*/
309+
const _input = input.slice(startIndex, stopIndex);
310+
const _allTokens = this.getAllTokens(_input);
311+
312+
return { inputSlice: _input, caretTokenIndex: _caretTokenIndex, allTokens: _allTokens };
313+
}
314+
254315
/**
255316
* Get suggestions of syntax and token at caretPosition
256317
* @param input source string
@@ -262,12 +323,13 @@ export abstract class BasicSQL<
262323
caretPosition: CaretPosition
263324
): Suggestions | null {
264325
const splitListener = this.splitListener;
326+
let inputSlice = input;
265327

266-
this.parseWithCache(input);
328+
this.parseWithCache(inputSlice);
267329
if (!this._parseTree) return null;
268330

269331
let sqlParserIns = this._parser;
270-
const allTokens = this.getAllTokens(input);
332+
let allTokens = this.getAllTokens(inputSlice);
271333
let caretTokenIndex = findCaretTokenIndex(caretPosition, allTokens);
272334
let c3Context: ParserRuleContext = this._parseTree;
273335
let tokenIndexOffset: number = 0;
@@ -321,22 +383,37 @@ export abstract class BasicSQL<
321383
}
322384

323385
// A boundary consisting of the index of the input.
324-
const startIndex = startStatement?.start?.start ?? 0;
325-
const stopIndex = stopStatement?.stop?.stop ?? input.length - 1;
386+
let startIndex = startStatement?.start?.start ?? 0;
387+
let stopIndex = stopStatement?.stop?.stop ?? inputSlice.length - 1;
326388

327389
/**
328390
* Save offset of the tokenIndex in the range of input
329391
* compared to the tokenIndex in the whole input
330392
*/
331393
tokenIndexOffset = startStatement?.start?.tokenIndex ?? 0;
332394
caretTokenIndex = caretTokenIndex - tokenIndexOffset;
395+
inputSlice = inputSlice.slice(startIndex, stopIndex);
396+
}
333397

334-
/**
335-
* Reparse the input fragment，
336-
* and c3 will collect candidates in the newly generated parseTree.
337-
*/
338-
const inputSlice = input.slice(startIndex, stopIndex);
398+
/**
399+
* Split the inputSlice by separator to get the smaller range of inputSlice.
400+
*/
401+
if (inputSlice.includes(SEPARATOR)) {
402+
const {
403+
inputSlice: _input,
404+
caretTokenIndex: _caretTokenIndex,
405+
allTokens: _allTokens,
406+
} = this.splitInputBySeparator(inputSlice, caretTokenIndex);
407+
408+
caretTokenIndex = _caretTokenIndex;
409+
inputSlice = _input;
410+
allTokens = _allTokens;
411+
}
339412

413+
/**
414+
* Reparse the input fragment, and c3 will collect candidates in the newly generated parseTree when input changed.
415+
*/
416+
if (inputSlice !== input) {
340417
const lexer = this.createLexer(inputSlice);
341418
lexer.removeErrorListeners();
342419
const tokenStream = new CommonTokenStream(lexer);
@@ -356,12 +433,7 @@ export abstract class BasicSQL<
356433
core.preferredRules = this.preferredRules;
357434

358435
const candidates = core.collectCandidates(caretTokenIndex, c3Context);
359-
const originalSuggestions = this.processCandidates(
360-
candidates,
361-
allTokens,
362-
caretTokenIndex,
363-
tokenIndexOffset
364-
);
436+
const originalSuggestions = this.processCandidates(candidates, allTokens, caretTokenIndex);
365437

366438
const syntaxSuggestions: SyntaxSuggestion<WordRange>[] = originalSuggestions.syntax.map(
367439
(syntaxCtx) => {

‎src/parser/flink/index.ts

+3-7
Original file line numberDiff line numberDiff line change
@@ -50,19 +50,15 @@ export class FlinkSQL extends BasicSQL<FlinkSqlLexer, ProgramContext, FlinkSqlPa
5050
protected processCandidates(
5151
candidates: CandidatesCollection,
5252
allTokens: Token[],
53-
caretTokenIndex: number,
54-
tokenIndexOffset: number
53+
caretTokenIndex: number
5554
): Suggestions<Token> {
5655
const originalSyntaxSuggestions: SyntaxSuggestion<Token>[] = [];
5756
const keywords: string[] = [];
5857

5958
for (let candidate of candidates.rules) {
6059
const [ruleType, candidateRule] = candidate;
61-
const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
62-
const tokenRanges = allTokens.slice(
63-
startTokenIndex,
64-
caretTokenIndex + tokenIndexOffset + 1
65-
);
60+
const startTokenIndex = candidateRule.startTokenIndex;
61+
const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + 1);
6662

6763
let syntaxContextType: EntityContextType | StmtContextType | undefined = void 0;
6864
switch (ruleType) {

‎src/parser/hive/index.ts

+3-7
Original file line numberDiff line numberDiff line change
@@ -51,18 +51,14 @@ export class HiveSQL extends BasicSQL<HiveSqlLexer, ProgramContext, HiveSqlParse
5151
protected processCandidates(
5252
candidates: CandidatesCollection,
5353
allTokens: Token[],
54-
caretTokenIndex: number,
55-
tokenIndexOffset: number
54+
caretTokenIndex: number
5655
): Suggestions<Token> {
5756
const originalSyntaxSuggestions: SyntaxSuggestion<Token>[] = [];
5857
const keywords: string[] = [];
5958
for (let candidate of candidates.rules) {
6059
const [ruleType, candidateRule] = candidate;
61-
const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
62-
const tokenRanges = allTokens.slice(
63-
startTokenIndex,
64-
caretTokenIndex + tokenIndexOffset + 1
65-
);
60+
const startTokenIndex = candidateRule.startTokenIndex;
61+
const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + 1);
6662

6763
let syntaxContextType: EntityContextType | StmtContextType | undefined = void 0;
6864
switch (ruleType) {

‎src/parser/impala/index.ts

+3-7
Original file line numberDiff line numberDiff line change
@@ -49,18 +49,14 @@ export class ImpalaSQL extends BasicSQL<ImpalaSqlLexer, ProgramContext, ImpalaSq
4949
protected processCandidates(
5050
candidates: CandidatesCollection,
5151
allTokens: Token[],
52-
caretTokenIndex: number,
53-
tokenIndexOffset: number
52+
caretTokenIndex: number
5453
): Suggestions<Token> {
5554
const originalSyntaxSuggestions: SyntaxSuggestion<Token>[] = [];
5655
const keywords: string[] = [];
5756
for (let candidate of candidates.rules) {
5857
const [ruleType, candidateRule] = candidate;
59-
const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
60-
const tokenRanges = allTokens.slice(
61-
startTokenIndex,
62-
caretTokenIndex + tokenIndexOffset + 1
63-
);
58+
const startTokenIndex = candidateRule.startTokenIndex;
59+
const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + 1);
6460

6561
let syntaxContextType: EntityContextType | StmtContextType | undefined = void 0;
6662
switch (ruleType) {

‎src/parser/mysql/index.ts

+3-7
Original file line numberDiff line numberDiff line change
@@ -49,19 +49,15 @@ export class MySQL extends BasicSQL<MySqlLexer, ProgramContext, MySqlParser> {
4949
protected processCandidates(
5050
candidates: CandidatesCollection,
5151
allTokens: Token[],
52-
caretTokenIndex: number,
53-
tokenIndexOffset: number
52+
caretTokenIndex: number
5453
): Suggestions<Token> {
5554
const originalSyntaxSuggestions: SyntaxSuggestion<Token>[] = [];
5655
const keywords: string[] = [];
5756

5857
for (const candidate of candidates.rules) {
5958
const [ruleType, candidateRule] = candidate;
60-
const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
61-
const tokenRanges = allTokens.slice(
62-
startTokenIndex,
63-
caretTokenIndex + tokenIndexOffset + 1
64-
);
59+
const startTokenIndex = candidateRule.startTokenIndex;
60+
const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + 1);
6561

6662
let syntaxContextType: EntityContextType | StmtContextType | undefined = void 0;
6763
switch (ruleType) {

‎src/parser/postgresql/index.ts

+3-7
Original file line numberDiff line numberDiff line change
@@ -54,18 +54,14 @@ export class PostgreSQL extends BasicSQL<PostgreSqlLexer, ProgramContext, Postgr
5454
protected processCandidates(
5555
candidates: CandidatesCollection,
5656
allTokens: Token[],
57-
caretTokenIndex: number,
58-
tokenIndexOffset: number
57+
caretTokenIndex: number
5958
): Suggestions<Token> {
6059
const originalSyntaxSuggestions: SyntaxSuggestion<Token>[] = [];
6160
const keywords: string[] = [];
6261
for (let candidate of candidates.rules) {
6362
const [ruleType, candidateRule] = candidate;
64-
const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
65-
const tokenRanges = allTokens.slice(
66-
startTokenIndex,
67-
caretTokenIndex + tokenIndexOffset + 1
68-
);
63+
const startTokenIndex = candidateRule.startTokenIndex;
64+
const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + 1);
6965

7066
let syntaxContextType: EntityContextType | StmtContextType | undefined = void 0;
7167
switch (ruleType) {

‎src/parser/spark/index.ts

+3-7
Original file line numberDiff line numberDiff line change
@@ -49,19 +49,15 @@ export class SparkSQL extends BasicSQL<SparkSqlLexer, ProgramContext, SparkSqlPa
4949
protected processCandidates(
5050
candidates: CandidatesCollection,
5151
allTokens: Token[],
52-
caretTokenIndex: number,
53-
tokenIndexOffset: number
52+
caretTokenIndex: number
5453
): Suggestions<Token> {
5554
const originalSyntaxSuggestions: SyntaxSuggestion<Token>[] = [];
5655
const keywords: string[] = [];
5756

5857
for (const candidate of candidates.rules) {
5958
const [ruleType, candidateRule] = candidate;
60-
const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
61-
const tokenRanges = allTokens.slice(
62-
startTokenIndex,
63-
caretTokenIndex + tokenIndexOffset + 1
64-
);
59+
const startTokenIndex = candidateRule.startTokenIndex;
60+
const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + 1);
6561

6662
let syntaxContextType: EntityContextType | StmtContextType | undefined = void 0;
6763
switch (ruleType) {

‎src/parser/trino/index.ts

+3-7
Original file line numberDiff line numberDiff line change
@@ -51,19 +51,15 @@ export class TrinoSQL extends BasicSQL<TrinoSqlLexer, ProgramContext, TrinoSqlPa
5151
protected processCandidates(
5252
candidates: CandidatesCollection,
5353
allTokens: Token[],
54-
caretTokenIndex: number,
55-
tokenIndexOffset: number
54+
caretTokenIndex: number
5655
): Suggestions<Token> {
5756
const originalSyntaxSuggestions: SyntaxSuggestion<Token>[] = [];
5857
const keywords: string[] = [];
5958

6059
for (let candidate of candidates.rules) {
6160
const [ruleType, candidateRule] = candidate;
62-
const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
63-
const tokenRanges = allTokens.slice(
64-
startTokenIndex,
65-
caretTokenIndex + tokenIndexOffset + 1
66-
);
61+
const startTokenIndex = candidateRule.startTokenIndex;
62+
const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + 1);
6763

6864
let syntaxContextType: EntityContextType | StmtContextType | undefined = void 0;
6965
switch (ruleType) {

0 commit comments

Comments
 (0)