Skip to content

SNOW-1936603: fix limit 0 bug with show #3090

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Apr 13, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
- Fixed a bug in `DataFrame.group_by().pivot().agg` when the pivot column and aggregate column are the same.
- Fixed a bug in local testing where transient `__pycache__` directory was unintentionally copied during stored procedure execution via import.
- Fixed a bug in `DataFrameReader.dbapi` (PrPr) where a `TypeError` was raised when `create_connection` returned a connection object of an unsupported driver type.
- Fixed a bug where `df.limit(0)` call would not properly apply.

#### Deprecations

Expand All @@ -38,6 +39,7 @@

- Fixed a bug in local testing that created incorrect result for `Column.like` calls.
- Fixed a bug in local testing that caused `Column.getItem` and `snowpark.snowflake.functions.get` to raise `IndexError` rather than return null.
- Fixed a bug in local testing where `df.limit(0)` call would not properly apply.

### Snowpark pandas API Updates

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1375,7 +1375,7 @@ def limit(self, n: int, *, offset: int = 0) -> "SelectStatement":
else:
new = copy(self)
new.from_ = self.from_.to_subqueryable()
new.limit_ = min(self.limit_, n) if self.limit_ else n
new.limit_ = min(self.limit_, n) if self.limit_ is not None else n
new.offset = offset or self.offset
new.column_states = self.column_states
new.pre_actions = new.from_.pre_actions
Expand Down
3 changes: 1 addition & 2 deletions src/snowflake/snowpark/dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -4738,13 +4738,12 @@ def _show_string_spark(
truncate: Union[bool, int] = True,
vertical: bool = False,
_spark_column_names: List[str] = None,
_emit_ast: bool = True,
**kwargs,
) -> str:
"""Spark's show() logic - translated from scala to python."""
# Fetch one more rows to check whether the result is truncated.
result, meta = self._get_result_and_meta_for_show(
num_rows + 1, _emit_ast, **kwargs
num_rows + 1, _emit_ast=False, **kwargs
)

# handle empty dataframe
Expand Down
2 changes: 1 addition & 1 deletion src/snowflake/snowpark/mock/_select_statement.py
Original file line number Diff line number Diff line change
Expand Up @@ -521,7 +521,7 @@ def set_operator(
def limit(self, n: int, *, offset: int = 0) -> "SelectStatement":
new = copy(self)
new.from_ = self.from_.to_subqueryable()
new.limit_ = min(self.limit_, n) if self.limit_ else n
new.limit_ = min(self.limit_, n) if self.limit_ is not None else n
new.offset = (self.offset + offset) if self.offset else offset
new._column_states = self._column_states
return new
Expand Down
9 changes: 3 additions & 6 deletions src/snowflake/snowpark/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -476,12 +476,9 @@ def valueType(self):

def _fill_ast(self, ast: proto.DataType) -> None:
ast.map_type.structured = self.structured
if self.key_type is None or self.value_type is None:
raise NotImplementedError(
"SNOW-1862700: AST does not support empty key or value type."
)
self.key_type._fill_ast(ast.map_type.key_ty)
self.value_type._fill_ast(ast.map_type.value_ty)
if self.key_type is not None and self.value_type is not None:
self.key_type._fill_ast(ast.map_type.key_ty)
self.value_type._fill_ast(ast.map_type.value_ty)


class VectorType(DataType):
Expand Down
44 changes: 43 additions & 1 deletion tests/ast/data/RelationalGroupedDataFrame.test
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ res14 = df.group_by("a").count()

df = session.create_dataframe([("SF", 21.0), ("SF", 17.5), ("SF", 24.0), ("NY", 30.9), ("NY", 33.6)], schema=["location", "temp_c"])

res16 = udtf("_ApplyInPandas", output_schema=PandasDataFrameType(StringType(), FloatType(), FloatType(), "LOCATION", "TEMP_C", "TEMP_F"), input_types=[StringType(), FloatType()], copy_grants=False, _registered_object_name="\"MOCK_DATABASE\".\"MOCK_SCHEMA\".SNOWPARK_TEMP_TABLE_FUNCTION_xxx")
res16 = udtf("_ApplyInPandas", output_schema=PandasDataFrameType(StringType(), FloatType(), FloatType(), "LOCATION", "TEMP_C", "TEMP_F"), input_types=[StringType(), FloatType()], artifact_repository=None, artifact_repository_packages=None, copy_grants=False, resource_constraint=None, _registered_object_name="\"MOCK_DATABASE\".\"MOCK_SCHEMA\".SNOWPARK_TEMP_TABLE_FUNCTION_xxx")

df.group_by("location").apply_in_pandas(convert, StructType(fields=[StructField("location", StringType(), nullable=True), StructField("temp_c", FloatType(), nullable=True), StructField("temp_f", FloatType(), nullable=True)], structured=False), input_types=[StringType(), FloatType()], input_names=["LOCATION", "TEMP_C"]).sort("temp_c").collect()

Expand Down Expand Up @@ -1086,6 +1086,34 @@ body {
input_types {
float_type: true
}
kwargs {
_1: "artifact_repository"
_2 {
null_val {
src {
end_column: 75
end_line: 50
file: 2
start_column: 8
start_line: 47
}
}
}
}
kwargs {
_1: "artifact_repository_packages"
_2 {
null_val {
src {
end_column: 75
end_line: 50
file: 2
start_column: 8
start_line: 47
}
}
}
}
kwargs {
_1: "copy_grants"
_2 {
Expand All @@ -1100,6 +1128,20 @@ body {
}
}
}
kwargs {
_1: "resource_constraint"
_2 {
null_val {
src {
end_column: 75
end_line: 50
file: 2
start_column: 8
start_line: 47
}
}
}
}
output_schema {
udtf_schema__type {
return_type {
Expand Down
44 changes: 43 additions & 1 deletion tests/ast/data/Session.call.test
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ df2 = session.call("my_sproc", 2, "one", dict(), False)

## EXPECTED UNPARSER OUTPUT

my_sproc_sp = sproc("my_sproc", return_type=StringType(), input_types=[LongType(), StringType(), MapType(StringType(), StringType(), structured=False), BooleanType()], name="my_sproc", replace=True, comment="The parameters are useless.", _registered_object_name="\"MOCK_DATABASE\".\"MOCK_SCHEMA\".\"MY_SPROC\"")
my_sproc_sp = sproc("my_sproc", return_type=StringType(), input_types=[LongType(), StringType(), MapType(StringType(), StringType(), structured=False), BooleanType()], name="my_sproc", replace=True, comment="The parameters are useless.", artifact_repository=None, artifact_repository_packages=None, resource_constraint=None, _registered_object_name="\"MOCK_DATABASE\".\"MOCK_SCHEMA\".\"MY_SPROC\"")

df = session.call("my_sproc", 1, "two", {"param1": 10, "param2": "twenty"}, True)

Expand Down Expand Up @@ -80,6 +80,48 @@ body {
input_types {
boolean_type: true
}
kwargs {
_1: "artifact_repository"
_2 {
null_val {
src {
end_column: 124
end_line: 30
file: 2
start_column: 22
start_line: 30
}
}
}
}
kwargs {
_1: "artifact_repository_packages"
_2 {
null_val {
src {
end_column: 124
end_line: 30
file: 2
start_column: 22
start_line: 30
}
}
}
}
kwargs {
_1: "resource_constraint"
_2 {
null_val {
src {
end_column: 124
end_line: 30
file: 2
start_column: 22
start_line: 30
}
}
}
}
name {
name {
name_flat {
Expand Down
88 changes: 86 additions & 2 deletions tests/ast/data/Session.table_function.test
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,9 @@ df7 = session.table_function(fn4)

## EXPECTED UNPARSER OUTPUT

my_sproc_sp = sproc("my_sproc", return_type=StringType(), input_types=[LongType(), StringType(), BooleanType()], name="my_fn1", replace=True, comment="The parameters are useless.", _registered_object_name="\"MOCK_DATABASE\".\"MOCK_SCHEMA\".\"MY_FN1\"")
my_sproc_sp = sproc("my_sproc", return_type=StringType(), input_types=[LongType(), StringType(), BooleanType()], name="my_fn1", replace=True, comment="The parameters are useless.", artifact_repository=None, artifact_repository_packages=None, resource_constraint=None, _registered_object_name="\"MOCK_DATABASE\".\"MOCK_SCHEMA\".\"MY_FN1\"")

my_sproc_sp2 = sproc("my_sproc", return_type=StringType(), input_types=[LongType(), StringType(), BooleanType()], name="my_fn2", replace=True, comment="Hello!", _registered_object_name="\"MOCK_DATABASE\".\"MOCK_SCHEMA\".\"MY_FN2\"")
my_sproc_sp2 = sproc("my_sproc", return_type=StringType(), input_types=[LongType(), StringType(), BooleanType()], name="my_fn2", replace=True, comment="Hello!", artifact_repository=None, artifact_repository_packages=None, resource_constraint=None, _registered_object_name="\"MOCK_DATABASE\".\"MOCK_SCHEMA\".\"MY_FN2\"")

df1 = session.table_function("my_fn1", lit(1), lit("two"), lit(True))

Expand Down Expand Up @@ -91,6 +91,48 @@ body {
input_types {
boolean_type: true
}
kwargs {
_1: "artifact_repository"
_2 {
null_val {
src {
end_column: 122
end_line: 33
file: 2
start_column: 22
start_line: 33
}
}
}
}
kwargs {
_1: "artifact_repository_packages"
_2 {
null_val {
src {
end_column: 122
end_line: 33
file: 2
start_column: 22
start_line: 33
}
}
}
}
kwargs {
_1: "resource_constraint"
_2 {
null_val {
src {
end_column: 122
end_line: 33
file: 2
start_column: 22
start_line: 33
}
}
}
}
name {
name {
name_flat {
Expand Down Expand Up @@ -153,6 +195,48 @@ body {
input_types {
boolean_type: true
}
kwargs {
_1: "artifact_repository"
_2 {
null_val {
src {
end_column: 102
end_line: 35
file: 2
start_column: 23
start_line: 35
}
}
}
}
kwargs {
_1: "artifact_repository_packages"
_2 {
null_val {
src {
end_column: 102
end_line: 35
file: 2
start_column: 23
start_line: 35
}
}
}
}
kwargs {
_1: "resource_constraint"
_2 {
null_val {
src {
end_column: 102
end_line: 35
file: 2
start_column: 23
start_line: 35
}
}
}
}
name {
name {
name_flat {
Expand Down
Loading