Skip to content

Commit 8d488a4

Browse files
cpcloudcpcloud
committed
Remove long-deprecated code and clean up warnings created by ibis
Author: Phillip Cloud <[email protected]> Author: Krisztián Szűcs <[email protected]> Closes ibis-project#1714 from cpcloud/address-warnings and squashes the following commits: f68ac03 [Phillip Cloud] Remove last remnants of named interval APIs 3b8771f [Phillip Cloud] Fix interval usage d7803ac [Phillip Cloud] Fixup docstring 7a2d918 [Phillip Cloud] Remove services 56a3c81 [Krisztián Szűcs] fix deprecations in notebook f5d8d2e [Phillip Cloud] Allow restart to use ${SERVICES} 557f010 [Phillip Cloud] Remove gz usage in ci/setup_docker_volume.sh 6e624bf [Phillip Cloud] Fix interval usage in docs e0af86c [Phillip Cloud] Formatting 737dfbb [Phillip Cloud] Clean up manual tempfile creation 78c2577 [Phillip Cloud] Refer to named argument rather than position 5436d22 [Phillip Cloud] Remove add_column from api.rst db6e0f3 [Phillip Cloud] Make it easy to start up a specific service, e.g., if it goes down f165e29 [Phillip Cloud] Fix importing and skipping b3c0a8f [Phillip Cloud] Add session kwarg docstring 3fb9d42 [Phillip Cloud] Docstring cleanup 533100b [Phillip Cloud] Pass session explicitly cc50c96 [Phillip Cloud] Simplify make test 51e5b74 [Phillip Cloud] Use the right variable 200c495 [Phillip Cloud] Map plural to singular 1a0a5ab [Phillip Cloud] Remove usage of named intervals d073e6f [Phillip Cloud] Address race condition in tear down 6fb7808 [Phillip Cloud] Fix docstring 9f683ea [Phillip Cloud] Address bigquery date offset arithmetic warning in test 00c4a6f [Phillip Cloud] Fix bigquery warning with infer dtype 958f900 [Krisztián Szűcs] rename remaining ibis.month definitions 8e05336 [Phillip Cloud] Fix warning cee5496 [Phillip Cloud] Constrain pandas to make sure we have the skipna argument to infer_dtype 9b0cb94 [Phillip Cloud] Pass skipna=True to infer_dtype from pandas 6360bd3 [Phillip Cloud] Fix numpy warning d363749 [Phillip Cloud] More raw fixes f42b713 [Phillip Cloud] SQLAlchemy warning 330c60b [Phillip Cloud] Fix usage of intervals and tz 8e43237 [Phillip Cloud] Fix raw passing 8787f2b [Phillip Cloud] ix is inclusive apparently 2d38bac [Phillip Cloud] Ignore bytecode related files and directories 635a989 [Phillip Cloud] Remove internal usage of timedelta api c0256fe [Phillip Cloud] Remove timedelta api f9ed0b3 [Phillip Cloud] Put raw in the correct place a89970b [Phillip Cloud] Remove usage of ix ae700bd [Phillip Cloud] Fix pytest test 50febc1 [Phillip Cloud] Add PYTEST_OPTIONS to makefile ffd3371 [Phillip Cloud] Move all unit usages to interval 888dfb5 [Phillip Cloud] Use raw to preserve behavior in apply 9c49630 [Phillip Cloud] Binary has been renamed to LargeBinary in SQLAlchemy 068cff5 [Phillip Cloud] Use tz_localize instead of tz argument 7da184e [Phillip Cloud] Address pytest raises message parameter warning 96581ff [Phillip Cloud] Use collections.abc for ABCs 274d8aa [Phillip Cloud] Remove add_column
1 parent 86253b0 commit 8d488a4

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+324
-377
lines changed

‎.dockerignore

+2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
11
ci/udf/CMakeCache.txt
22
ci/udf/CMakeFiles/
33
ci/udf/Makefile
4+
**/*.pyc
5+
**/__pycache__

‎Makefile

+12-6
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ MAKEFILE_DIR = $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
66
COMPOSE_FILE := "$(MAKEFILE_DIR)/ci/docker-compose.yml"
77
DOCKER := ENVKIND=$(ENVKIND) docker-compose -f $(COMPOSE_FILE)
88
DOCKER_RUN := $(DOCKER) run --rm
9+
PYTEST_OPTIONS :=
10+
SERVICES := mapd postgres mysql clickhouse impala kudu-master kudu-tserver
911

1012
clean:
1113
@python setup.py clean
@@ -25,15 +27,15 @@ lint:
2527

2628
stop:
2729
# stop all running docker compose services
28-
@$(DOCKER) rm --force --stop
30+
@$(DOCKER) rm --force --stop ${SERVICES}
2931

3032
build:
3133
# build the ibis image
3234
@$(DOCKER) build --pull ibis
3335

3436
start:
3537
# start all docker compose services
36-
@$(DOCKER) up -d --no-build mapd postgres mysql clickhouse impala kudu-master kudu-tserver
38+
@$(DOCKER) up -d --no-build ${SERVICES}
3739
# wait for services to start
3840
@$(DOCKER_RUN) waiter
3941

@@ -47,21 +49,25 @@ init: restart
4749
@$(MAKE) build
4850
@$(MAKE) load
4951

50-
test:
52+
testparallel:
5153
@ENVKIND=$(ENVKIND) $(MAKEFILE_DIR)/ci/test.sh -n auto -m 'not udf' \
54+
--doctest-modules --doctest-ignore-import-errors ${PYTEST_OPTIONS}
55+
56+
test:
57+
@ENVKIND=$(ENVKIND) $(MAKEFILE_DIR)/ci/test.sh ${PYTEST_OPTIONS} \
5258
--doctest-modules --doctest-ignore-import-errors
5359

5460
testmost:
5561
@ENVKIND=$(ENVKIND) $(MAKEFILE_DIR)/ci/test.sh -n auto -m 'not (udf or impala or hdfs)' \
56-
--doctest-modules --doctest-ignore-import-errors
62+
--doctest-modules --doctest-ignore-import-errors ${PYTEST_OPTIONS}
5763

5864
testfast:
5965
@ENVKIND=$(ENVKIND) $(MAKEFILE_DIR)/ci/test.sh -n auto -m 'not (udf or impala or hdfs or bigquery)' \
60-
--doctest-modules --doctest-ignore-import-errors
66+
--doctest-modules --doctest-ignore-import-errors ${PYTEST_OPTIONS}
6167

6268
testparams:
6369
@echo 'not (udf or impala or hdfs or postgresql or mysql or mapd or clickhouse)' \
64-
--doctest-modules --doctest-ignore-import-errors
70+
--doctest-modules --doctest-ignore-import-errors ${PYTEST_OPTIONS}
6571

6672
docclean:
6773
@$(DOCKER_RUN) ibis rm -rf /tmp/docs.ibis-project.org

‎benchmarks/benchmarks.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -153,13 +153,13 @@ def setup(self):
153153
]].sort_by(['low_card_key', 'key'])
154154

155155
low_card_window = ibis.trailing_range_window(
156-
2 * ibis.day(),
156+
ibis.interval(days=2),
157157
order_by=t.repeated_timestamps,
158158
group_by=t.low_card_key)
159159
self.low_card_grouped_rolling = t.value.mean().over(low_card_window)
160160

161161
high_card_window = ibis.trailing_range_window(
162-
2 * ibis.day(),
162+
ibis.interval(days=2),
163163
order_by=t.repeated_timestamps,
164164
group_by=t.key)
165165
self.high_card_grouped_rolling = t.value.mean().over(high_card_window)

‎ci/impalamgr.py

+1-2
Original file line numberDiff line numberDiff line change
@@ -211,8 +211,7 @@ def upload_udfs(con):
211211

212212
@click.group(context_settings=dict(help_option_names=['-h', '--help']))
213213
def main():
214-
"""Manage test data for Ibis"""
215-
pass
214+
"""Manage impala test data for Ibis."""
216215

217216

218217
@main.command()

‎ci/requirements-dev.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ lz4
1212
multipledispatch>=0.6.0
1313
mypy
1414
numpy
15-
pandas
15+
pandas>=0.21
1616
plumbum
1717
pre_commit
1818
psycopg2

‎ci/setup_docker_volume.sh

-4
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,3 @@ fi
3232

3333
mkdir -p /tmp/ibis
3434
cp "${GOOGLE_APPLICATION_CREDENTIALS}" /tmp/ibis/gcloud-service-key.json
35-
cp -rf "${IBIS_TEST_DATA_DIRECTORY}" /tmp/ibis
36-
37-
gzipprog="$([ "$(which pigz)" ] && echo pigz || echo gzip)"
38-
tar -I "${gzipprog}" -cf /tmp/ibis/ibis-testing-data.tar.gz "${IBIS_TEST_DATA_DIRECTORY}" 2> /dev/null

‎docs/source/api.rst

-1
Original file line numberDiff line numberDiff line change
@@ -256,7 +256,6 @@ Table methods
256256
.. autosummary::
257257
:toctree: generated/
258258

259-
TableExpr.add_column
260259
TableExpr.aggregate
261260
TableExpr.count
262261
TableExpr.distinct

‎docs/source/notebooks/tutorial/3-Projection-Join-Sort.ipynb

+1-18
Original file line numberDiff line numberDiff line change
@@ -119,23 +119,6 @@
119119
"proj2.limit(5)"
120120
]
121121
},
122-
{
123-
"cell_type": "markdown",
124-
"metadata": {},
125-
"source": [
126-
"Adding columns is a shortcut for projection. In Ibis, adding columns always produces a new table reference"
127-
]
128-
},
129-
{
130-
"cell_type": "code",
131-
"execution_count": null,
132-
"metadata": {},
133-
"outputs": [],
134-
"source": [
135-
"table2 = table.add_column(bigger_expr)\n",
136-
"table2.limit(5)"
137-
]
138-
},
139122
{
140123
"cell_type": "markdown",
141124
"metadata": {},
@@ -506,7 +489,7 @@
506489
"name": "python",
507490
"nbconvert_exporter": "python",
508491
"pygments_lexer": "ipython3",
509-
"version": "3.6.3"
492+
"version": "3.6.7"
510493
}
511494
},
512495
"nbformat": 4,

‎docs/source/notebooks/tutorial/4-More-Value-Expressions.ipynb

+3-3
Original file line numberDiff line numberDiff line change
@@ -488,7 +488,7 @@
488488
"outputs": [],
489489
"source": [
490490
"table[table.timestamp_col < \n",
491-
" (ibis.timestamp('2010-01-01') + ibis.month(3))].count()"
491+
" (ibis.timestamp('2010-01-01') + ibis.interval(months=3))].count()"
492492
]
493493
},
494494
{
@@ -497,7 +497,7 @@
497497
"metadata": {},
498498
"outputs": [],
499499
"source": [
500-
"expr = (table.timestamp_col + ibis.day(1) + ibis.hour(4)).name('offset')\n",
500+
"expr = (table.timestamp_col + ibis.interval(days=1) + ibis.interval(hours=4)).name('offset')\n",
501501
"table[table.timestamp_col, expr, ibis.now().name('current_time')].limit(10)"
502502
]
503503
}
@@ -518,7 +518,7 @@
518518
"name": "python",
519519
"nbconvert_exporter": "python",
520520
"pygments_lexer": "ipython3",
521-
"version": "3.6.3"
521+
"version": "3.6.7"
522522
}
523523
},
524524
"nbformat": 4,

‎docs/source/sql.rst

+1-1
Original file line numberDiff line numberDiff line change
@@ -1187,7 +1187,7 @@ arithmetic. For example:
11871187

11881188
.. ipython:: python
11891189
1190-
expr = events[events.ts > (ibis.now() - ibis.year())]
1190+
expr = events[events.ts > (ibis.now() - ibis.interval(years=1))]
11911191
print(ibis.impala.compile(expr))
11921192
11931193
The implementation of each timedelta offset will depend on the query engine.

‎ibis/__init__.py

+26-16
Original file line numberDiff line numberDiff line change
@@ -58,31 +58,41 @@
5858

5959
def hdfs_connect(host='localhost', port=50070, protocol='webhdfs',
6060
use_https='default', auth_mechanism='NOSASL',
61-
verify=True, **kwds):
62-
"""
63-
Connect to HDFS
61+
verify=True, session=None, **kwds):
62+
"""Connect to HDFS.
6463
6564
Parameters
6665
----------
67-
host : string, Host name of the HDFS NameNode
68-
port : int, NameNode's WebHDFS port (default 50070)
69-
protocol : {'webhdfs'}
70-
use_https : boolean, default 'default'
66+
host : str
67+
Host name of the HDFS NameNode
68+
port : int
69+
NameNode's WebHDFS port
70+
protocol : str,
71+
The protocol used to communicate with HDFS. The only valid value is
72+
``'webhdfs'``.
73+
use_https : bool
7174
Connect to WebHDFS with HTTPS, otherwise plain HTTP. For secure
72-
authentication, the default for this is True, otherwise False
73-
auth_mechanism : string, Set to NOSASL or PLAIN for non-secure clusters.
75+
authentication, the default for this is True, otherwise False.
76+
auth_mechanism : str
77+
Set to NOSASL or PLAIN for non-secure clusters.
7478
Set to GSSAPI or LDAP for Kerberos-secured clusters.
75-
verify : boolean, Set to False to turn off verifying SSL certificates.
76-
(default True)
79+
verify : bool
80+
Set to :data:`False` to turn off verifying SSL certificates.
81+
session : Optional[requests.Session]
82+
A custom :class:`requests.Session` object.
7783
78-
Other keywords are forwarded to hdfs library classes
84+
Notes
85+
-----
86+
Other keywords are forwarded to HDFS library classes.
7987
8088
Returns
8189
-------
82-
client : WebHDFS
90+
WebHDFS
91+
8392
"""
8493
import requests
85-
session = kwds.setdefault('session', requests.Session())
94+
if session is None:
95+
session = requests.Session()
8696
session.verify = verify
8797
if auth_mechanism in ('GSSAPI', 'LDAP'):
8898
if use_https == 'default':
@@ -100,15 +110,15 @@ def hdfs_connect(host='localhost', port=50070, protocol='webhdfs',
100110
# note SSL
101111
url = '{0}://{1}:{2}'.format(prefix, host, port)
102112
kwds.setdefault('mutual_auth', 'OPTIONAL')
103-
hdfs_client = KerberosClient(url, **kwds)
113+
hdfs_client = KerberosClient(url, session=session, **kwds)
104114
else:
105115
if use_https == 'default':
106116
prefix = 'http'
107117
else:
108118
prefix = 'https' if use_https else 'http'
109119
from hdfs.client import InsecureClient
110120
url = '{}://{}:{}'.format(prefix, host, port)
111-
hdfs_client = InsecureClient(url, **kwds)
121+
hdfs_client = InsecureClient(url, session=session, **kwds)
112122
return WebHDFS(hdfs_client)
113123

114124

‎ibis/bigquery/tests/test_compiler.py

+9-9
Original file line numberDiff line numberDiff line change
@@ -271,14 +271,14 @@ def test_range_window_function(alltypes, project_id):
271271
('preceding', 'value'),
272272
[
273273
(5, 5),
274-
(ibis.nanosecond(), 0.001),
275-
(ibis.microsecond(), 1),
276-
(ibis.second(), 1000000),
277-
(ibis.minute(), 1000000 * 60),
278-
(ibis.hour(), 1000000 * 60 * 60),
279-
(ibis.day(), 1000000 * 60 * 60 * 24),
280-
(2 * ibis.day(), 1000000 * 60 * 60 * 24 * 2),
281-
(ibis.week(), 1000000 * 60 * 60 * 24 * 7),
274+
(ibis.interval(nanoseconds=1), 0.001),
275+
(ibis.interval(microseconds=1), 1),
276+
(ibis.interval(seconds=1), 1000000),
277+
(ibis.interval(minutes=1), 1000000 * 60),
278+
(ibis.interval(hours=1), 1000000 * 60 * 60),
279+
(ibis.interval(days=1), 1000000 * 60 * 60 * 24),
280+
(2 * ibis.interval(days=1), 1000000 * 60 * 60 * 24 * 2),
281+
(ibis.interval(weeks=1), 1000000 * 60 * 60 * 24 * 7),
282282
]
283283
)
284284
def test_trailing_range_window(alltypes, preceding, value, project_id):
@@ -298,7 +298,7 @@ def test_trailing_range_window(alltypes, preceding, value, project_id):
298298
@pytest.mark.parametrize(
299299
('preceding', 'value'),
300300
[
301-
(ibis.year(), None),
301+
(ibis.interval(years=1), None),
302302
]
303303
)
304304
def test_trailing_range_window_unsupported(alltypes, preceding, value):

‎ibis/clickhouse/tests/test_select.py

+4-3
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ def test_limit_offset(alltypes):
7676
tm.assert_frame_equal(alltypes.limit(4).execute(), expected.head(4))
7777
tm.assert_frame_equal(alltypes.limit(8).execute(), expected.head(8))
7878
tm.assert_frame_equal(alltypes.limit(4, offset=4).execute(),
79-
expected.ix[4:7].reset_index(drop=True))
79+
expected.iloc[4:8].reset_index(drop=True))
8080

8181

8282
def test_subquery(alltypes, df):
@@ -423,9 +423,10 @@ def test_timestamp_scalar_in_filter(alltypes, translate):
423423
table = alltypes
424424

425425
expr = (table.filter([table.timestamp_col <
426-
(ibis.timestamp('2010-01-01') + ibis.week(3)),
426+
(ibis.timestamp('2010-01-01') +
427+
ibis.interval(weeks=3)),
427428
table.timestamp_col < (ibis.now() +
428-
ibis.day(10))
429+
ibis.interval(days=10))
429430
])
430431
.count())
431432
expr.execute()

0 commit comments

Comments
 (0)