From b13a8ea61b4af654c3d158e137e9ac7d95319ed0 Mon Sep 17 00:00:00 2001 From: Frank Stenzhorn Date: Fri, 5 Dec 2025 11:57:03 +0100 Subject: [PATCH 1/4] feature: add support to split sql strings --- src/DatabaseLibrary/query.py | 175 ++++++++++++++++++++--------------- 1 file changed, 99 insertions(+), 76 deletions(-) diff --git a/src/DatabaseLibrary/query.py b/src/DatabaseLibrary/query.py index 2bbb709..455dddf 100644 --- a/src/DatabaseLibrary/query.py +++ b/src/DatabaseLibrary/query.py @@ -16,7 +16,7 @@ import inspect import re import sys -from typing import List, Optional, Tuple +from typing import List, Optional, Tuple, Union import sqlparse from robot.api import logger @@ -328,9 +328,7 @@ def execute_sql_script( else: statements_to_execute = self.split_sql_script(script_path, external_parser=external_parser) for statement in statements_to_execute: - proc_end_pattern = re.compile("end(?!( if;| loop;| case;| while;| repeat;)).*;()?") - line_ends_with_proc_end = re.compile(r"(\s|;)" + proc_end_pattern.pattern + "$") - omit_semicolon = not line_ends_with_proc_end.search(statement.lower()) + omit_semicolon = self._omit_semicolon_needed(statement) self._execute_sql(cur, statement, omit_semicolon, replace_robot_variables=replace_robot_variables) self._commit_if_needed(db_connection, no_transaction) except Exception as e: @@ -350,72 +348,82 @@ def split_sql_script( """ with open(script_path, encoding="UTF-8") as sql_file: logger.info("Splitting script file into statements...") - statements_to_execute = [] - if external_parser: - split_statements = sqlparse.split(sql_file.read()) - for statement in split_statements: - statement_without_comments = sqlparse.format(statement, strip_comments=True) - if statement_without_comments: - statements_to_execute.append(statement_without_comments) - else: - current_statement = "" - inside_statements_group = False - proc_start_pattern = re.compile("create( or replace)? (procedure|function){1}( )?") - proc_end_pattern = re.compile("end(?!( if;| loop;| case;| while;| repeat;)).*;()?") - for line in sql_file: - line = line.strip() - if line.startswith("#") or line.startswith("--") or line == "/": - continue - - # check if the line matches the creating procedure regexp pattern - if proc_start_pattern.match(line.lower()): - inside_statements_group = True - elif line.lower().startswith("begin"): - inside_statements_group = True - - # semicolons inside the line? use them to separate statements - # ... but not if they are inside a begin/end block (aka. statements group) - sqlFragments = line.split(";") - # no semicolons - if len(sqlFragments) == 1: - current_statement += line + " " - continue + return self.split_sql_string(sql_file.read(), external_parser=external_parser) + + def split_sql_string(self, sql_string: str, external_parser: bool = False): + if external_parser: + return self._split_statements_using_external_parser(sql_string) + else: + return self._parse_sql_internally(sql_string.splitlines()) + + def _parse_sql_internally(self, sql_file: List[str]) -> list[str]: + statements_to_execute = [] + current_statement = "" + inside_statements_group = False + proc_start_pattern = re.compile("create( or replace)? (procedure|function){1}( )?") + proc_end_pattern = re.compile("end(?!( if;| loop;| case;| while;| repeat;)).*;()?") + for line in sql_file: + line = line.strip() + if line.startswith("#") or line.startswith("--") or line == "/": + continue + + # check if the line matches the creating procedure regexp pattern + if proc_start_pattern.match(line.lower()): + inside_statements_group = True + elif line.lower().startswith("begin"): + inside_statements_group = True + + # semicolons inside the line? use them to separate statements + # ... but not if they are inside a begin/end block (aka. statements group) + sqlFragments = line.split(";") + # no semicolons + if len(sqlFragments) == 1: + current_statement += line + " " + continue + quotes = 0 + # "select * from person;" -> ["select..", ""] + for sqlFragment in sqlFragments: + if len(sqlFragment.strip()) == 0: + continue + + if inside_statements_group: + # if statements inside a begin/end block have semicolns, + # they must persist - even with oracle + sqlFragment += "; " + + if proc_end_pattern.match(sqlFragment.lower()): + inside_statements_group = False + elif proc_start_pattern.match(sqlFragment.lower()): + inside_statements_group = True + elif sqlFragment.lower().startswith("begin"): + inside_statements_group = True + + # check if the semicolon is a part of the value (quoted string) + quotes += sqlFragment.count("'") + quotes -= sqlFragment.count("\\'") + inside_quoted_string = quotes % 2 != 0 + if inside_quoted_string: + sqlFragment += ";" # restore the semicolon + + current_statement += sqlFragment + if not inside_statements_group and not inside_quoted_string: + statements_to_execute.append(current_statement.strip()) + current_statement = "" quotes = 0 - # "select * from person;" -> ["select..", ""] - for sqlFragment in sqlFragments: - if len(sqlFragment.strip()) == 0: - continue - - if inside_statements_group: - # if statements inside a begin/end block have semicolns, - # they must persist - even with oracle - sqlFragment += "; " - - if proc_end_pattern.match(sqlFragment.lower()): - inside_statements_group = False - elif proc_start_pattern.match(sqlFragment.lower()): - inside_statements_group = True - elif sqlFragment.lower().startswith("begin"): - inside_statements_group = True - - # check if the semicolon is a part of the value (quoted string) - quotes += sqlFragment.count("'") - quotes -= sqlFragment.count("\\'") - inside_quoted_string = quotes % 2 != 0 - if inside_quoted_string: - sqlFragment += ";" # restore the semicolon - - current_statement += sqlFragment - if not inside_statements_group and not inside_quoted_string: - statements_to_execute.append(current_statement.strip()) - current_statement = "" - quotes = 0 - - current_statement = current_statement.strip() - if len(current_statement) != 0: - statements_to_execute.append(current_statement) - - return statements_to_execute + + current_statement = current_statement.strip() + if len(current_statement) != 0: + statements_to_execute.append(current_statement) + return statements_to_execute + + def _split_statements_using_external_parser(self, sql_file_content: str): + statements_to_execute = [] + split_statements = sqlparse.split(sql_file_content) + for statement in split_statements: + statement_without_comments = sqlparse.format(statement, strip_comments=True) + if statement_without_comments: + statements_to_execute.append(statement_without_comments) + return statements_to_execute @renamed_args( mapping={ @@ -436,6 +444,8 @@ def execute_sql_string( sqlString: Optional[str] = None, sansTran: Optional[bool] = None, omitTrailingSemicolon: Optional[bool] = None, + split: bool = False, + external_parser: bool = False, ): """ Executes the ``sql_string`` as a single SQL command. @@ -473,17 +483,30 @@ def execute_sql_string( cur = db_connection.client.cursor() if omit_trailing_semicolon is None: omit_trailing_semicolon = db_connection.omit_trailing_semicolon - self._execute_sql( - cur, - sql_string, - omit_trailing_semicolon=omit_trailing_semicolon, - parameters=parameters, - replace_robot_variables=replace_robot_variables, - ) + if not split: + self._execute_sql( + cur, + sql_string, + omit_trailing_semicolon=omit_trailing_semicolon, + parameters=parameters, + replace_robot_variables=replace_robot_variables, + ) + else: + statements_to_execute = self.split_sql_string(sql_string, external_parser=external_parser) + for statement in statements_to_execute: + omit_semicolon = self._omit_semicolon_needed(statement) + self._execute_sql(cur, statement, omit_semicolon, replace_robot_variables=replace_robot_variables) + self._commit_if_needed(db_connection, no_transaction) except Exception as e: self._rollback_and_raise(db_connection, no_transaction, e) + def _omit_semicolon_needed(self, statement: str) -> bool: + proc_end_pattern = re.compile("end(?!( if;| loop;| case;| while;| repeat;)).*;()?") + line_ends_with_proc_end = re.compile(r"(\s|;)" + proc_end_pattern.pattern + "$") + omit_semicolon = not line_ends_with_proc_end.search(statement.lower()) + return omit_semicolon + @renamed_args(mapping={"spName": "procedure_name", "spParams": "procedure_params", "sansTran": "no_transaction"}) def call_stored_procedure( self, From 486cc5104911b1926dc157aaca04ce64aa9ee80c Mon Sep 17 00:00:00 2001 From: amochin Date: Mon, 8 Dec 2025 14:05:16 +0100 Subject: [PATCH 2/4] Tests for SQL splitting --- test/tests/common_tests/basic_tests.robot | 2 +- test/tests/common_tests/script_files.robot | 35 ----- test/tests/common_tests/split_sql.robot | 123 ++++++++++++++++++ .../sql_script_split_commands.robot | 22 ---- 4 files changed, 124 insertions(+), 58 deletions(-) create mode 100644 test/tests/common_tests/split_sql.robot delete mode 100644 test/tests/custom_db_tests/sql_script_split_commands.robot diff --git a/test/tests/common_tests/basic_tests.robot b/test/tests/common_tests/basic_tests.robot index d850bec..f79288c 100644 --- a/test/tests/common_tests/basic_tests.robot +++ b/test/tests/common_tests/basic_tests.robot @@ -161,4 +161,4 @@ Verify Query - Row Count foobar table 0 row Query Returns Zero Results [Documentation] Tests that nothing crashes when there are zero results ${results}= Query SELECT * FROM person WHERE id < 0 - Should Be Empty ${results} \ No newline at end of file + Should Be Empty ${results} diff --git a/test/tests/common_tests/script_files.robot b/test/tests/common_tests/script_files.robot index afe11a7..9b4f0a7 100644 --- a/test/tests/common_tests/script_files.robot +++ b/test/tests/common_tests/script_files.robot @@ -6,7 +6,6 @@ Suite Teardown Disconnect From Database Test Setup Create Person Table Test Teardown Drop Tables Person And Foobar - *** Test Cases *** Semicolons As Statement Separators In One Line Run SQL Script File statements_in_one_line @@ -35,40 +34,6 @@ Semicolons And Quotes In Values Should Be Equal As Strings ${results}[0] (5, 'Miles', "O'Brian") Should Be Equal As Strings ${results}[1] (6, 'Keiko', "O'Brian") -Split Script Into Statements - Internal Parser - Insert Data In Person Table Using SQL Script - @{Expected commands}= Create List - ... SELECT * FROM person - ... SELECT * FROM person WHERE id=1 - ${extracted commands}= Split Sql Script ${Script files dir}/split_commands.sql - Lists Should Be Equal ${Expected commands} ${extracted commands} - FOR ${command} IN @{extracted commands} - ${results}= Query ${command} - END - -Split Script Into Statements - External Parser - Insert Data In Person Table Using SQL Script - @{Expected commands}= Create List - ... SELECT * FROM person; - ... SELECT * FROM person WHERE id=1; - ${extracted commands}= Split Sql Script ${Script files dir}/split_commands.sql external_parser=True - Lists Should Be Equal ${Expected commands} ${extracted commands} - FOR ${command} IN @{extracted commands} - ${results}= Query ${command} - END - -Split Script Into Statements - External Parser - Comments Are Removed - Insert Data In Person Table Using SQL Script - @{Expected commands}= Create List - ... SELECT * FROM person; - ... SELECT * FROM person WHERE id=1; - ${extracted commands}= Split Sql Script ${Script files dir}/split_commands_comments.sql external_parser=True - Lists Should Be Equal ${Expected commands} ${extracted commands} - FOR ${command} IN @{extracted commands} - ${results}= Query ${command} - END - - *** Keywords *** Run SQL Script File [Arguments] ${File Name} diff --git a/test/tests/common_tests/split_sql.robot b/test/tests/common_tests/split_sql.robot new file mode 100644 index 0000000..69afa90 --- /dev/null +++ b/test/tests/common_tests/split_sql.robot @@ -0,0 +1,123 @@ +*** Settings *** +Documentation Tests for the splitting SQL scripts and string into separate statements. +... +... First time implementation of the _split_ parameter see in: +... https://github.com/MarketSquare/Robotframework-Database-Library/issues/184 + +Resource ../../resources/common.resource +Suite Setup Connect To DB +Suite Teardown Disconnect From Database +Test Setup Create Person Table +Test Teardown Drop Tables Person And Foobar + +*** Variables *** +@{SQL Commands No Semicolons} +... SELECT * FROM person +... SELECT * FROM person WHERE id=1 + +@{SQL Commands With Semicolons} +... SELECT * FROM person; +... SELECT * FROM person WHERE id=1; + +*** Test Cases *** +Run Script With Commands Splitting + [Documentation] Such a simple script works always, + ... just check in the logs if the parameter value was processed properly + Run SQL Script File insert_data_in_person_table split=True + +Run Script Without Commands Splitting + [Documentation] Running such a script as a single statement works for PostgreSQL, + ... but fails in Oracle. Check in the logs if the splitting was disabled. + Skip If $DB_MODULE != "psycopg2" + Run SQL Script File insert_data_in_person_table split=False + +Run Script Split With External Parser + [Documentation] We don't want to test the external parser itself, but just assure + ... the parameter works properly + Run SQL Script File insert_data_in_person_table split=True external_parser=True + +Run Script With Semicolons As Statement Separators In One Line + Run SQL Script File statements_in_one_line split=True + ${sql}= Catenate select * from person + ... where id=6 or id=7 + ${results}= Query ${sql} + Length Should Be ${results} 2 + Should Be Equal As Strings ${results}[0] (6, 'Julian', 'Bashir') + Should Be Equal As Strings ${results}[1] (7, 'Jadzia', 'Dax') + +Run Script With Semicolons In Values + Run SQL Script File semicolons_in_values split=True + ${sql}= Catenate select * from person + ... where id=3 or id=4 + ${results}= Query ${sql} + Length Should Be ${results} 2 + Should Be Equal As Strings ${results}[0] (3, 'Hello; world', 'Another; value') + Should Be Equal As Strings ${results}[1] (4, 'May the Force; ', 'be with you;') + +Run Script With Semicolons And Quotes In Values + Run SQL Script File semicolons_and_quotes_in_values split=True + ${sql}= Catenate select * from person + ... where LAST_NAME='O''Brian' + ${results}= Query ${sql} + Length Should Be ${results} 2 + Should Be Equal As Strings ${results}[0] (5, 'Miles', "O'Brian") + Should Be Equal As Strings ${results}[1] (6, 'Keiko', "O'Brian") + +Split Script Into Statements - Internal Parser + Insert Data In Person Table Using SQL Script + ${extracted commands}= Split Sql Script ${Script files dir}/split_commands.sql + Lists Should Be Equal ${SQL Commands No Semicolons} ${extracted commands} + FOR ${command} IN @{extracted commands} + ${results}= Query ${command} + END + +Split Script Into Statements - External Parser + Insert Data In Person Table Using SQL Script + ${extracted commands}= Split Sql Script ${Script files dir}/split_commands.sql external_parser=True + Lists Should Be Equal ${SQL Commands With Semicolons} ${extracted commands} + FOR ${command} IN @{extracted commands} + ${results}= Query ${command} + END + +Split Script Into Statements - External Parser - Comments Are Removed + Insert Data In Person Table Using SQL Script + ${extracted commands}= Split Sql Script ${Script files dir}/split_commands_comments.sql external_parser=True + Lists Should Be Equal ${SQL Commands With Semicolons} ${extracted commands} + FOR ${command} IN @{extracted commands} + ${results}= Query ${command} + END + +Split SQL String Into Statements - Internal Parser + Insert Data In Person Table Using SQL Script + ${merged command}= Catenate @{SQL Commands With Semicolons} + ${extracted commands}= Split Sql String ${merged command} + Lists Should Be Equal ${extracted commands} ${SQL Commands No Semicolons} + +Split SQL String Into Statements - External Parser + Insert Data In Person Table Using SQL Script + ${merged command}= Catenate @{SQL Commands With Semicolons} + ${extracted commands}= Split Sql String ${merged command} external_parser=True + Lists Should Be Equal ${extracted commands} ${SQL Commands With Semicolons} + +Execute SQL String Without Splitting + [Documentation] Running such a command as a single statement works for PostgreSQL, + ... but fails in Oracle. Check in the logs if the splitting was disabled. + Skip If $DB_MODULE != "psycopg2" + Insert Data In Person Table Using SQL Script + ${merged command}= Catenate @{SQL Commands With Semicolons} + Execute Sql String ${merged command} split=False + +Execute SQL String With Splitting - Internal Parser + Insert Data In Person Table Using SQL Script + ${merged command}= Catenate @{SQL Commands With Semicolons} + Execute Sql String ${merged command} split=True + +Execute SQL String With Splitting - External Parser + Insert Data In Person Table Using SQL Script + ${merged command}= Catenate @{SQL Commands With Semicolons} + Execute Sql String ${merged command} split=True external_parser=True + +*** Keywords *** +Run SQL Script File + [Arguments] ${File Name} ${split} ${external_parser}=False + Execute Sql Script ${Script files dir}/${File Name}.sql split=${Split} external_parser=${external_parser} diff --git a/test/tests/custom_db_tests/sql_script_split_commands.robot b/test/tests/custom_db_tests/sql_script_split_commands.robot deleted file mode 100644 index e6223cc..0000000 --- a/test/tests/custom_db_tests/sql_script_split_commands.robot +++ /dev/null @@ -1,22 +0,0 @@ -*** Settings *** -Documentation Tests for the parameter _split_ in the keyword -... _Execute SQL Script_ - special for the issue #184: -... https://github.com/MarketSquare/Robotframework-Database-Library/issues/184 - -Resource ../../resources/common.resource -Suite Setup Connect To DB -Suite Teardown Disconnect From Database -Test Setup Create Person Table -Test Teardown Drop Tables Person And Foobar - - -*** Test Cases *** -Split Commands - [Documentation] Such a simple script works always, - ... just check if the logs if the parameter value was processed properly - Execute Sql Script ${CURDIR}/../../resources/insert_data_in_person_table.sql split=True - -Don't Split Commands - [Documentation] Running such a script as a single statement works for PostgreSQL, - ... but fails in Oracle. Check in the logs if the splitting was disabled. - Execute Sql Script ${CURDIR}/../../resources/insert_data_in_person_table.sql split=False From e2033721a2f2c078d3156be85a1d2b07b76d1fea Mon Sep 17 00:00:00 2001 From: amochin Date: Mon, 8 Dec 2025 14:09:17 +0100 Subject: [PATCH 3/4] Refactor tests - script files structure --- test/resources/common.resource | 6 +++--- .../{ => script_files}/create_stored_procedures_mssql.sql | 0 .../{ => script_files}/create_stored_procedures_mysql.sql | 0 .../create_stored_procedures_oracle.sql | 0 .../create_stored_procedures_postgres.sql | 0 .../{ => script_files}/excel_db_test_insertData.sql | 0 .../{ => script_files}/insert_data_in_person_table.sql | 0 .../insert_data_in_person_table_utf8.sql | 0 .../select_with_robot_variables.sql | 0 .../semicolons_and_quotes_in_values.sql | 0 .../semicolons_in_values.sql | 0 .../split_commands.sql | 0 .../split_commands_comments.sql | 0 .../statements_in_one_line.sql | 0 test/tests/common_tests/encoding.robot | 2 +- test/tests/common_tests/stored_procedures.robot | 8 ++++---- test/tests/custom_db_tests/excel.robot | 2 +- 17 files changed, 9 insertions(+), 9 deletions(-) rename test/resources/{ => script_files}/create_stored_procedures_mssql.sql (100%) rename test/resources/{ => script_files}/create_stored_procedures_mysql.sql (100%) rename test/resources/{ => script_files}/create_stored_procedures_oracle.sql (100%) rename test/resources/{ => script_files}/create_stored_procedures_postgres.sql (100%) rename test/resources/{ => script_files}/excel_db_test_insertData.sql (100%) rename test/resources/{ => script_files}/insert_data_in_person_table.sql (100%) rename test/resources/{ => script_files}/insert_data_in_person_table_utf8.sql (100%) rename test/resources/{script_file_tests => script_files}/select_with_robot_variables.sql (100%) rename test/resources/{script_file_tests => script_files}/semicolons_and_quotes_in_values.sql (100%) rename test/resources/{script_file_tests => script_files}/semicolons_in_values.sql (100%) rename test/resources/{script_file_tests => script_files}/split_commands.sql (100%) rename test/resources/{script_file_tests => script_files}/split_commands_comments.sql (100%) rename test/resources/{script_file_tests => script_files}/statements_in_one_line.sql (100%) diff --git a/test/resources/common.resource b/test/resources/common.resource index 7487618..a49973e 100644 --- a/test/resources/common.resource +++ b/test/resources/common.resource @@ -17,7 +17,7 @@ ${DB_NAME} db ${DB_PASS} pass ${DB_PORT} 5432 ${DB_USER} db_user -${Script files dir} ${CURDIR}/script_file_tests +${Script files dir} ${CURDIR}/script_files # used for MySQL via PyODBC only ${DB_DRIVER} ODBC Driver 18 for SQL Server @@ -96,9 +96,9 @@ Create Person Table And Insert Data Insert Data In Person Table Using SQL Script [Arguments] ${alias}=${None} IF $alias is None - ${output}= Execute SQL Script ${CURDIR}/insert_data_in_person_table.sql + ${output}= Execute SQL Script ${Script files dir}/insert_data_in_person_table.sql ELSE - ${output}= Execute SQL Script ${CURDIR}/insert_data_in_person_table.sql alias=${alias} + ${output}= Execute SQL Script ${Script files dir}/insert_data_in_person_table.sql alias=${alias} END RETURN ${output} diff --git a/test/resources/create_stored_procedures_mssql.sql b/test/resources/script_files/create_stored_procedures_mssql.sql similarity index 100% rename from test/resources/create_stored_procedures_mssql.sql rename to test/resources/script_files/create_stored_procedures_mssql.sql diff --git a/test/resources/create_stored_procedures_mysql.sql b/test/resources/script_files/create_stored_procedures_mysql.sql similarity index 100% rename from test/resources/create_stored_procedures_mysql.sql rename to test/resources/script_files/create_stored_procedures_mysql.sql diff --git a/test/resources/create_stored_procedures_oracle.sql b/test/resources/script_files/create_stored_procedures_oracle.sql similarity index 100% rename from test/resources/create_stored_procedures_oracle.sql rename to test/resources/script_files/create_stored_procedures_oracle.sql diff --git a/test/resources/create_stored_procedures_postgres.sql b/test/resources/script_files/create_stored_procedures_postgres.sql similarity index 100% rename from test/resources/create_stored_procedures_postgres.sql rename to test/resources/script_files/create_stored_procedures_postgres.sql diff --git a/test/resources/excel_db_test_insertData.sql b/test/resources/script_files/excel_db_test_insertData.sql similarity index 100% rename from test/resources/excel_db_test_insertData.sql rename to test/resources/script_files/excel_db_test_insertData.sql diff --git a/test/resources/insert_data_in_person_table.sql b/test/resources/script_files/insert_data_in_person_table.sql similarity index 100% rename from test/resources/insert_data_in_person_table.sql rename to test/resources/script_files/insert_data_in_person_table.sql diff --git a/test/resources/insert_data_in_person_table_utf8.sql b/test/resources/script_files/insert_data_in_person_table_utf8.sql similarity index 100% rename from test/resources/insert_data_in_person_table_utf8.sql rename to test/resources/script_files/insert_data_in_person_table_utf8.sql diff --git a/test/resources/script_file_tests/select_with_robot_variables.sql b/test/resources/script_files/select_with_robot_variables.sql similarity index 100% rename from test/resources/script_file_tests/select_with_robot_variables.sql rename to test/resources/script_files/select_with_robot_variables.sql diff --git a/test/resources/script_file_tests/semicolons_and_quotes_in_values.sql b/test/resources/script_files/semicolons_and_quotes_in_values.sql similarity index 100% rename from test/resources/script_file_tests/semicolons_and_quotes_in_values.sql rename to test/resources/script_files/semicolons_and_quotes_in_values.sql diff --git a/test/resources/script_file_tests/semicolons_in_values.sql b/test/resources/script_files/semicolons_in_values.sql similarity index 100% rename from test/resources/script_file_tests/semicolons_in_values.sql rename to test/resources/script_files/semicolons_in_values.sql diff --git a/test/resources/script_file_tests/split_commands.sql b/test/resources/script_files/split_commands.sql similarity index 100% rename from test/resources/script_file_tests/split_commands.sql rename to test/resources/script_files/split_commands.sql diff --git a/test/resources/script_file_tests/split_commands_comments.sql b/test/resources/script_files/split_commands_comments.sql similarity index 100% rename from test/resources/script_file_tests/split_commands_comments.sql rename to test/resources/script_files/split_commands_comments.sql diff --git a/test/resources/script_file_tests/statements_in_one_line.sql b/test/resources/script_files/statements_in_one_line.sql similarity index 100% rename from test/resources/script_file_tests/statements_in_one_line.sql rename to test/resources/script_files/statements_in_one_line.sql diff --git a/test/tests/common_tests/encoding.robot b/test/tests/common_tests/encoding.robot index b7dd2d9..fa9d321 100644 --- a/test/tests/common_tests/encoding.robot +++ b/test/tests/common_tests/encoding.robot @@ -21,7 +21,7 @@ Read SQL Script Files As UTF8 ... Pytho might have an issue opening this file on Windows, as it doesn't use UTF8 by default. ... In this case you the library should excplicitely set the UTF8 encoding when opening the script file. ... https://dev.to/methane/python-use-utf-8-mode-on-windows-212i - Execute Sql Script ${CURDIR}/../../resources/insert_data_in_person_table_utf8.sql + Execute Sql Script ${Script files dir}/insert_data_in_person_table_utf8.sql ${results}= Query ... SELECT LAST_NAME FROM person WHERE FIRST_NAME='Jürgen' Should Be Equal ${results}[0][0] Gernegroß \ No newline at end of file diff --git a/test/tests/common_tests/stored_procedures.robot b/test/tests/common_tests/stored_procedures.robot index 7cc136a..db79d8d 100644 --- a/test/tests/common_tests/stored_procedures.robot +++ b/test/tests/common_tests/stored_procedures.robot @@ -112,13 +112,13 @@ MSSQL Procedure Returns OUT Param Without Result Sets Create And Fill Tables And Stored Procedures Create Person Table And Insert Data IF "${DB_MODULE}" in ["oracledb", "cx_Oracle"] - Execute SQL Script ${CURDIR}/../../resources/create_stored_procedures_oracle.sql + Execute SQL Script ${Script files dir}/create_stored_procedures_oracle.sql ELSE IF "${DB_MODULE}" in ["pymysql"] - Execute SQL Script ${CURDIR}/../../resources/create_stored_procedures_mysql.sql + Execute SQL Script ${Script files dir}/create_stored_procedures_mysql.sql ELSE IF "${DB_MODULE}" in ["psycopg2", "psycopg3"] - Execute SQL Script ${CURDIR}/../../resources/create_stored_procedures_postgres.sql + Execute SQL Script ${Script files dir}/create_stored_procedures_postgres.sql ELSE IF "${DB_MODULE}" in ["pymssql"] - Execute SQL Script ${CURDIR}/../../resources/create_stored_procedures_mssql.sql + Execute SQL Script ${Script files dir}/create_stored_procedures_mssql.sql ELSE Skip Don't know how to create stored procedures for '${DB_MODULE}' END diff --git a/test/tests/custom_db_tests/excel.robot b/test/tests/custom_db_tests/excel.robot index f61b793..332d5c3 100644 --- a/test/tests/custom_db_tests/excel.robot +++ b/test/tests/custom_db_tests/excel.robot @@ -24,7 +24,7 @@ Create person table Execute SQL Script - Insert Data person table log to console ${DBName} - ${output} = Execute SQL Script ${CURDIR}/../../resources/excel_db_test_insertData.sql + ${output} = Execute SQL Script ${Script files dir}/excel_db_test_insertData.sql Log ${output} Should Be Equal As Strings ${output} None From 64de753c19ec81dddc9a34029ef5a44d662aa4e4 Mon Sep 17 00:00:00 2001 From: amochin Date: Mon, 8 Dec 2025 14:10:48 +0100 Subject: [PATCH 4/4] Finalize the structure, make minor fixes, add docs --- src/DatabaseLibrary/query.py | 163 +++++++++++++++++++---------------- 1 file changed, 87 insertions(+), 76 deletions(-) diff --git a/src/DatabaseLibrary/query.py b/src/DatabaseLibrary/query.py index 455dddf..d816889 100644 --- a/src/DatabaseLibrary/query.py +++ b/src/DatabaseLibrary/query.py @@ -16,7 +16,7 @@ import inspect import re import sys -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Tuple import sqlparse from robot.api import logger @@ -347,82 +347,82 @@ def split_sql_script( Set ``external_parser`` to _True_ to use the external library [https://pypi.org/project/sqlparse/|sqlparse]. """ with open(script_path, encoding="UTF-8") as sql_file: - logger.info("Splitting script file into statements...") return self.split_sql_string(sql_file.read(), external_parser=external_parser) - def split_sql_string(self, sql_string: str, external_parser: bool = False): - if external_parser: - return self._split_statements_using_external_parser(sql_string) - else: - return self._parse_sql_internally(sql_string.splitlines()) + def split_sql_string(self, sql_string: str, external_parser=False): + """ + Splits the content of the ``sql_string`` into individual SQL commands + and returns them as a list of strings. + SQL commands are expected to be delimited by a semicolon (';'). - def _parse_sql_internally(self, sql_file: List[str]) -> list[str]: + Set ``external_parser`` to _True_ to use the external library [https://pypi.org/project/sqlparse/|sqlparse]. + """ + logger.info(f"Splitting SQL into statements. Using external parser: {external_parser}") statements_to_execute = [] - current_statement = "" - inside_statements_group = False - proc_start_pattern = re.compile("create( or replace)? (procedure|function){1}( )?") - proc_end_pattern = re.compile("end(?!( if;| loop;| case;| while;| repeat;)).*;()?") - for line in sql_file: - line = line.strip() - if line.startswith("#") or line.startswith("--") or line == "/": - continue - - # check if the line matches the creating procedure regexp pattern - if proc_start_pattern.match(line.lower()): - inside_statements_group = True - elif line.lower().startswith("begin"): - inside_statements_group = True - - # semicolons inside the line? use them to separate statements - # ... but not if they are inside a begin/end block (aka. statements group) - sqlFragments = line.split(";") - # no semicolons - if len(sqlFragments) == 1: - current_statement += line + " " - continue - quotes = 0 - # "select * from person;" -> ["select..", ""] - for sqlFragment in sqlFragments: - if len(sqlFragment.strip()) == 0: + if external_parser: + split_statements = sqlparse.split(sql_string) + for statement in split_statements: + statement_without_comments = sqlparse.format(statement, strip_comments=True) + if statement_without_comments: + statements_to_execute.append(statement_without_comments) + else: + current_statement = "" + inside_statements_group = False + proc_start_pattern = re.compile("create( or replace)? (procedure|function){1}( )?") + proc_end_pattern = re.compile("end(?!( if;| loop;| case;| while;| repeat;)).*;()?") + for line in sql_string.splitlines(): + line = line.strip() + if line.startswith("#") or line.startswith("--") or line == "/": continue - if inside_statements_group: - # if statements inside a begin/end block have semicolns, - # they must persist - even with oracle - sqlFragment += "; " - - if proc_end_pattern.match(sqlFragment.lower()): - inside_statements_group = False - elif proc_start_pattern.match(sqlFragment.lower()): + # check if the line matches the creating procedure regexp pattern + if proc_start_pattern.match(line.lower()): inside_statements_group = True - elif sqlFragment.lower().startswith("begin"): + elif line.lower().startswith("begin"): inside_statements_group = True - # check if the semicolon is a part of the value (quoted string) - quotes += sqlFragment.count("'") - quotes -= sqlFragment.count("\\'") - inside_quoted_string = quotes % 2 != 0 - if inside_quoted_string: - sqlFragment += ";" # restore the semicolon - - current_statement += sqlFragment - if not inside_statements_group and not inside_quoted_string: - statements_to_execute.append(current_statement.strip()) - current_statement = "" - quotes = 0 - - current_statement = current_statement.strip() - if len(current_statement) != 0: - statements_to_execute.append(current_statement) - return statements_to_execute + # semicolons inside the line? use them to separate statements + # ... but not if they are inside a begin/end block (aka. statements group) + sqlFragments = line.split(";") + # no semicolons + if len(sqlFragments) == 1: + current_statement += line + " " + continue + quotes = 0 + # "select * from person;" -> ["select..", ""] + for sqlFragment in sqlFragments: + if len(sqlFragment.strip()) == 0: + continue + + if inside_statements_group: + # if statements inside a begin/end block have semicolns, + # they must persist - even with oracle + sqlFragment += "; " + + if proc_end_pattern.match(sqlFragment.lower()): + inside_statements_group = False + elif proc_start_pattern.match(sqlFragment.lower()): + inside_statements_group = True + elif sqlFragment.lower().startswith("begin"): + inside_statements_group = True + + # check if the semicolon is a part of the value (quoted string) + quotes += sqlFragment.count("'") + quotes -= sqlFragment.count("\\'") + inside_quoted_string = quotes % 2 != 0 + if inside_quoted_string: + sqlFragment += ";" # restore the semicolon + + current_statement += sqlFragment + if not inside_statements_group and not inside_quoted_string: + statements_to_execute.append(current_statement.strip()) + current_statement = "" + quotes = 0 + + current_statement = current_statement.strip() + if len(current_statement) != 0: + statements_to_execute.append(current_statement) - def _split_statements_using_external_parser(self, sql_file_content: str): - statements_to_execute = [] - split_statements = sqlparse.split(sql_file_content) - for statement in split_statements: - statement_without_comments = sqlparse.format(statement, strip_comments=True) - if statement_without_comments: - statements_to_execute.append(statement_without_comments) return statements_to_execute @renamed_args( @@ -441,14 +441,20 @@ def execute_sql_string( omit_trailing_semicolon: Optional[bool] = None, *, replace_robot_variables=False, + split: bool = False, + external_parser: bool = False, sqlString: Optional[str] = None, sansTran: Optional[bool] = None, omitTrailingSemicolon: Optional[bool] = None, - split: bool = False, - external_parser: bool = False, ): """ - Executes the ``sql_string`` as a single SQL command. + Executes the ``sql_string`` - as a single SQL command (default) or as separate statements. + + Set ``split`` to _True_ to enable dividing the string into SQL commands similar to the `Execute SQL Script` + keyword. The commands are expected to be delimited by a semicolon (';') in this case - + they will be split and executed separately. + + Set ``external_parser`` to _True_ to use the external library [https://pypi.org/project/sqlparse/|sqlparse] for splitting the script. Set ``no_transaction`` to _True_ to run command without explicit transaction commit or rollback in case of error. @@ -501,12 +507,6 @@ def execute_sql_string( except Exception as e: self._rollback_and_raise(db_connection, no_transaction, e) - def _omit_semicolon_needed(self, statement: str) -> bool: - proc_end_pattern = re.compile("end(?!( if;| loop;| case;| while;| repeat;)).*;()?") - line_ends_with_proc_end = re.compile(r"(\s|;)" + proc_end_pattern.pattern + "$") - omit_semicolon = not line_ends_with_proc_end.search(statement.lower()) - return omit_semicolon - @renamed_args(mapping={"spName": "procedure_name", "spParams": "procedure_params", "sansTran": "no_transaction"}) def call_stored_procedure( self, @@ -831,6 +831,17 @@ def set_logging_query_results(self, enabled: Optional[bool] = None, log_head: Op raise ValueError(f"Wrong log head value provided: {log_head}. The value can't be negative!") self.LOG_QUERY_RESULTS_HEAD = log_head + def _omit_semicolon_needed(self, statement: str) -> bool: + """ + Checks if the `statement` ends with a procedure ending keyword - so that semicolon should be omitted - + and returns the result. + The function is used when running multiple SQL statements from a script or an SQL string. + """ + proc_end_pattern = re.compile("end(?!( if;| loop;| case;| while;| repeat;)).*;()?") + line_ends_with_proc_end = re.compile(r"(\s|;)" + proc_end_pattern.pattern + "$") + omit_semicolon = not line_ends_with_proc_end.search(statement.lower()) + return omit_semicolon + def _execute_sql( self, cur,