Skip to content

Commit de3b760

Browse files
liuxy0551liuxy0551
committed
feat: add splitInputBySeparator
1 parent 68c00c2 commit de3b760

File tree

8 files changed

+129
-68
lines changed

8 files changed

+129
-68
lines changed

‎src/parser/common/basicSQL.ts

+108-19
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ import type { SplitListener } from './splitListener';
2121
import type { EntityCollector } from './entityCollector';
2222
import { EntityContext } from './entityCollector';
2323

24+
const SEPARATOR: string = ';';
25+
2426
/**
2527
* Basic SQL class, every sql needs extends it.
2628
*/
@@ -65,13 +67,11 @@ export abstract class BasicSQL<
6567
* @param candidates candidate list
6668
* @param allTokens all tokens from input
6769
* @param caretTokenIndex tokenIndex of caretPosition
68-
* @param tokenIndexOffset offset of the tokenIndex in the candidates compared to the tokenIndex in allTokens
6970
*/
7071
protected abstract processCandidates(
7172
candidates: CandidatesCollection,
7273
allTokens: Token[],
73-
caretTokenIndex: number,
74-
tokenIndexOffset: number
74+
caretTokenIndex: number
7575
): Suggestions<Token>;
7676

7777
/**
@@ -251,6 +251,78 @@ export abstract class BasicSQL<
251251
return res;
252252
}
253253

254+
/**
255+
* Get the smaller range of input
256+
* @param input string
257+
* @param allTokens all tokens from input
258+
* @param tokenIndexOffset offset of the tokenIndex in the range of input
259+
* @param caretTokenIndex tokenIndex of caretPosition
260+
* @returns inputSlice: string, caretTokenIndex: number
261+
*/
262+
private splitInputBySeparator(
263+
input: string,
264+
allTokens: Token[],
265+
tokenIndexOffset: number,
266+
caretTokenIndex: number
267+
): { inputSlice: string; allTokens: Token[]; caretTokenIndex: number } {
268+
const tokens = allTokens.slice(tokenIndexOffset);
269+
/**
270+
* Set startToken
271+
*/
272+
let startToken: Token | null = null;
273+
for (let tokenIndex = caretTokenIndex - tokenIndexOffset; tokenIndex >= 0; tokenIndex--) {
274+
const token = tokens[tokenIndex];
275+
if (token?.text === SEPARATOR) {
276+
startToken = tokens[tokenIndex + 1];
277+
break;
278+
}
279+
}
280+
if (startToken === null) {
281+
startToken = tokens[0];
282+
}
283+
284+
/**
285+
* Set stopToken
286+
*/
287+
let stopToken: Token | null = null;
288+
for (
289+
let tokenIndex = caretTokenIndex - tokenIndexOffset;
290+
tokenIndex < tokens.length;
291+
tokenIndex++
292+
) {
293+
const token = tokens[tokenIndex];
294+
if (token?.text === SEPARATOR) {
295+
stopToken = token;
296+
break;
297+
}
298+
}
299+
if (stopToken === null) {
300+
stopToken = tokens[tokens.length - 1];
301+
}
302+
303+
const indexOffset = tokens[0].start;
304+
let startIndex = startToken.start - indexOffset;
305+
let stopIndex = stopToken.stop + 1 - indexOffset;
306+
307+
/**
308+
* Save offset of the tokenIndex in the range of input
309+
* compared to the tokenIndex in the whole input
310+
*/
311+
const _tokenIndexOffset = startToken.tokenIndex;
312+
const _caretTokenIndex = caretTokenIndex - _tokenIndexOffset;
313+
314+
/**
315+
* Get the smaller range of _input
316+
*/
317+
const _input = input.slice(startIndex, stopIndex);
318+
319+
return {
320+
inputSlice: _input,
321+
allTokens: allTokens.slice(_tokenIndexOffset),
322+
caretTokenIndex: _caretTokenIndex,
323+
};
324+
}
325+
254326
/**
255327
* Get suggestions of syntax and token at caretPosition
256328
* @param input source string
@@ -262,12 +334,13 @@ export abstract class BasicSQL<
262334
caretPosition: CaretPosition
263335
): Suggestions | null {
264336
const splitListener = this.splitListener;
337+
let inputSlice = input;
265338

266-
this.parseWithCache(input);
339+
this.parseWithCache(inputSlice);
267340
if (!this._parseTree) return null;
268341

269342
let sqlParserIns = this._parser;
270-
const allTokens = this.getAllTokens(input);
343+
let allTokens = this.getAllTokens(inputSlice);
271344
let caretTokenIndex = findCaretTokenIndex(caretPosition, allTokens);
272345
let c3Context: ParserRuleContext = this._parseTree;
273346
let tokenIndexOffset: number = 0;
@@ -321,22 +394,43 @@ export abstract class BasicSQL<
321394
}
322395

323396
// A boundary consisting of the index of the input.
324-
const startIndex = startStatement?.start?.start ?? 0;
325-
const stopIndex = stopStatement?.stop?.stop ?? input.length - 1;
397+
let startIndex = startStatement?.start?.start ?? 0;
398+
let stopIndex = stopStatement?.stop?.stop ?? inputSlice.length - 1;
326399

327400
/**
328401
* Save offset of the tokenIndex in the range of input
329402
* compared to the tokenIndex in the whole input
330403
*/
331404
tokenIndexOffset = startStatement?.start?.tokenIndex ?? 0;
332-
caretTokenIndex = caretTokenIndex - tokenIndexOffset;
405+
inputSlice = inputSlice.slice(startIndex, stopIndex);
406+
}
333407

334-
/**
335-
* Reparse the input fragment，
336-
* and c3 will collect candidates in the newly generated parseTree.
337-
*/
338-
const inputSlice = input.slice(startIndex, stopIndex);
408+
/**
409+
* Split the inputSlice by separator to get the smaller range of inputSlice.
410+
*/
411+
if (inputSlice.includes(SEPARATOR)) {
412+
const {
413+
inputSlice: _input,
414+
allTokens: _allTokens,
415+
caretTokenIndex: _caretTokenIndex,
416+
} = this.splitInputBySeparator(
417+
inputSlice,
418+
allTokens,
419+
tokenIndexOffset,
420+
caretTokenIndex
421+
);
422+
423+
allTokens = _allTokens;
424+
caretTokenIndex = _caretTokenIndex;
425+
inputSlice = _input;
426+
} else {
427+
caretTokenIndex = caretTokenIndex - tokenIndexOffset;
428+
}
339429

430+
/**
431+
* Reparse the input fragment, and c3 will collect candidates in the newly generated parseTree when input changed.
432+
*/
433+
if (inputSlice !== input) {
340434
const lexer = this.createLexer(inputSlice);
341435
lexer.removeErrorListeners();
342436
const tokenStream = new CommonTokenStream(lexer);
@@ -356,12 +450,7 @@ export abstract class BasicSQL<
356450
core.preferredRules = this.preferredRules;
357451

358452
const candidates = core.collectCandidates(caretTokenIndex, c3Context);
359-
const originalSuggestions = this.processCandidates(
360-
candidates,
361-
allTokens,
362-
caretTokenIndex,
363-
tokenIndexOffset
364-
);
453+
const originalSuggestions = this.processCandidates(candidates, allTokens, caretTokenIndex);
365454

366455
const syntaxSuggestions: SyntaxSuggestion<WordRange>[] = originalSuggestions.syntax.map(
367456
(syntaxCtx) => {

‎src/parser/flink/index.ts

+3-7
Original file line numberDiff line numberDiff line change
@@ -50,19 +50,15 @@ export class FlinkSQL extends BasicSQL<FlinkSqlLexer, ProgramContext, FlinkSqlPa
5050
protected processCandidates(
5151
candidates: CandidatesCollection,
5252
allTokens: Token[],
53-
caretTokenIndex: number,
54-
tokenIndexOffset: number
53+
caretTokenIndex: number
5554
): Suggestions<Token> {
5655
const originalSyntaxSuggestions: SyntaxSuggestion<Token>[] = [];
5756
const keywords: string[] = [];
5857

5958
for (let candidate of candidates.rules) {
6059
const [ruleType, candidateRule] = candidate;
61-
const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
62-
const tokenRanges = allTokens.slice(
63-
startTokenIndex,
64-
caretTokenIndex + tokenIndexOffset + 1
65-
);
60+
const startTokenIndex = candidateRule.startTokenIndex;
61+
const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + 1);
6662

6763
let syntaxContextType: EntityContextType | StmtContextType | undefined = void 0;
6864
switch (ruleType) {

‎src/parser/hive/index.ts

+3-7
Original file line numberDiff line numberDiff line change
@@ -51,18 +51,14 @@ export class HiveSQL extends BasicSQL<HiveSqlLexer, ProgramContext, HiveSqlParse
5151
protected processCandidates(
5252
candidates: CandidatesCollection,
5353
allTokens: Token[],
54-
caretTokenIndex: number,
55-
tokenIndexOffset: number
54+
caretTokenIndex: number
5655
): Suggestions<Token> {
5756
const originalSyntaxSuggestions: SyntaxSuggestion<Token>[] = [];
5857
const keywords: string[] = [];
5958
for (let candidate of candidates.rules) {
6059
const [ruleType, candidateRule] = candidate;
61-
const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
62-
const tokenRanges = allTokens.slice(
63-
startTokenIndex,
64-
caretTokenIndex + tokenIndexOffset + 1
65-
);
60+
const startTokenIndex = candidateRule.startTokenIndex;
61+
const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + 1);
6662

6763
let syntaxContextType: EntityContextType | StmtContextType | undefined = void 0;
6864
switch (ruleType) {

‎src/parser/impala/index.ts

+3-7
Original file line numberDiff line numberDiff line change
@@ -49,18 +49,14 @@ export class ImpalaSQL extends BasicSQL<ImpalaSqlLexer, ProgramContext, ImpalaSq
4949
protected processCandidates(
5050
candidates: CandidatesCollection,
5151
allTokens: Token[],
52-
caretTokenIndex: number,
53-
tokenIndexOffset: number
52+
caretTokenIndex: number
5453
): Suggestions<Token> {
5554
const originalSyntaxSuggestions: SyntaxSuggestion<Token>[] = [];
5655
const keywords: string[] = [];
5756
for (let candidate of candidates.rules) {
5857
const [ruleType, candidateRule] = candidate;
59-
const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
60-
const tokenRanges = allTokens.slice(
61-
startTokenIndex,
62-
caretTokenIndex + tokenIndexOffset + 1
63-
);
58+
const startTokenIndex = candidateRule.startTokenIndex;
59+
const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + 1);
6460

6561
let syntaxContextType: EntityContextType | StmtContextType | undefined = void 0;
6662
switch (ruleType) {

‎src/parser/mysql/index.ts

+3-7
Original file line numberDiff line numberDiff line change
@@ -49,19 +49,15 @@ export class MySQL extends BasicSQL<MySqlLexer, ProgramContext, MySqlParser> {
4949
protected processCandidates(
5050
candidates: CandidatesCollection,
5151
allTokens: Token[],
52-
caretTokenIndex: number,
53-
tokenIndexOffset: number
52+
caretTokenIndex: number
5453
): Suggestions<Token> {
5554
const originalSyntaxSuggestions: SyntaxSuggestion<Token>[] = [];
5655
const keywords: string[] = [];
5756

5857
for (const candidate of candidates.rules) {
5958
const [ruleType, candidateRule] = candidate;
60-
const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
61-
const tokenRanges = allTokens.slice(
62-
startTokenIndex,
63-
caretTokenIndex + tokenIndexOffset + 1
64-
);
59+
const startTokenIndex = candidateRule.startTokenIndex;
60+
const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + 1);
6561

6662
let syntaxContextType: EntityContextType | StmtContextType | undefined = void 0;
6763
switch (ruleType) {

‎src/parser/postgresql/index.ts

+3-7
Original file line numberDiff line numberDiff line change
@@ -54,18 +54,14 @@ export class PostgreSQL extends BasicSQL<PostgreSqlLexer, ProgramContext, Postgr
5454
protected processCandidates(
5555
candidates: CandidatesCollection,
5656
allTokens: Token[],
57-
caretTokenIndex: number,
58-
tokenIndexOffset: number
57+
caretTokenIndex: number
5958
): Suggestions<Token> {
6059
const originalSyntaxSuggestions: SyntaxSuggestion<Token>[] = [];
6160
const keywords: string[] = [];
6261
for (let candidate of candidates.rules) {
6362
const [ruleType, candidateRule] = candidate;
64-
const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
65-
const tokenRanges = allTokens.slice(
66-
startTokenIndex,
67-
caretTokenIndex + tokenIndexOffset + 1
68-
);
63+
const startTokenIndex = candidateRule.startTokenIndex;
64+
const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + 1);
6965

7066
let syntaxContextType: EntityContextType | StmtContextType | undefined = void 0;
7167
switch (ruleType) {

‎src/parser/spark/index.ts

+3-7
Original file line numberDiff line numberDiff line change
@@ -49,19 +49,15 @@ export class SparkSQL extends BasicSQL<SparkSqlLexer, ProgramContext, SparkSqlPa
4949
protected processCandidates(
5050
candidates: CandidatesCollection,
5151
allTokens: Token[],
52-
caretTokenIndex: number,
53-
tokenIndexOffset: number
52+
caretTokenIndex: number
5453
): Suggestions<Token> {
5554
const originalSyntaxSuggestions: SyntaxSuggestion<Token>[] = [];
5655
const keywords: string[] = [];
5756

5857
for (const candidate of candidates.rules) {
5958
const [ruleType, candidateRule] = candidate;
60-
const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
61-
const tokenRanges = allTokens.slice(
62-
startTokenIndex,
63-
caretTokenIndex + tokenIndexOffset + 1
64-
);
59+
const startTokenIndex = candidateRule.startTokenIndex;
60+
const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + 1);
6561

6662
let syntaxContextType: EntityContextType | StmtContextType | undefined = void 0;
6763
switch (ruleType) {

‎src/parser/trino/index.ts

+3-7
Original file line numberDiff line numberDiff line change
@@ -51,19 +51,15 @@ export class TrinoSQL extends BasicSQL<TrinoSqlLexer, ProgramContext, TrinoSqlPa
5151
protected processCandidates(
5252
candidates: CandidatesCollection,
5353
allTokens: Token[],
54-
caretTokenIndex: number,
55-
tokenIndexOffset: number
54+
caretTokenIndex: number
5655
): Suggestions<Token> {
5756
const originalSyntaxSuggestions: SyntaxSuggestion<Token>[] = [];
5857
const keywords: string[] = [];
5958

6059
for (let candidate of candidates.rules) {
6160
const [ruleType, candidateRule] = candidate;
62-
const startTokenIndex = candidateRule.startTokenIndex + tokenIndexOffset;
63-
const tokenRanges = allTokens.slice(
64-
startTokenIndex,
65-
caretTokenIndex + tokenIndexOffset + 1
66-
);
61+
const startTokenIndex = candidateRule.startTokenIndex;
62+
const tokenRanges = allTokens.slice(startTokenIndex, caretTokenIndex + 1);
6763

6864
let syntaxContextType: EntityContextType | StmtContextType | undefined = void 0;
6965
switch (ruleType) {

0 commit comments

Comments
 (0)