From fc1c4da77018e9bc9b13f48364b197f31961bec0 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Sun, 8 Jul 2018 11:43:25 -0500 Subject: [PATCH 01/25] CI: Migrate to circleci 2.0 --- ci/install_circle.sh | 15 +++---- circle.yml | 105 +++++++++++++++++++++++++++++-------------- 2 files changed, 79 insertions(+), 41 deletions(-) diff --git a/ci/install_circle.sh b/ci/install_circle.sh index 5ffff84c88488..2e67b23ade38f 100755 --- a/ci/install_circle.sh +++ b/ci/install_circle.sh @@ -6,14 +6,7 @@ echo "[home_dir: $home_dir]" echo "[ls -ltr]" ls -ltr -echo "[Using clean Miniconda install]" -rm -rf "$MINICONDA_DIR" - -# install miniconda -wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -q -O miniconda.sh || exit 1 -bash miniconda.sh -b -p "$MINICONDA_DIR" || exit 1 - -export PATH="$MINICONDA_DIR/bin:$PATH" +apt-get update -y && apt-get install build-essential -y echo "[update conda]" conda config --set ssl_verify false || exit 1 @@ -48,6 +41,12 @@ source $ENVS_FILE # edit the locale override if needed if [ -n "$LOCALE_OVERRIDE" ]; then + + apt-get update && apt-get install locales -y + + sed -i -e 's/# $LOCALE_OVERRIDE UTF-8/$LOCAL_OVERRIDE UTF-8/' /etc/locale.gen && \ + locale-gen + echo "[Adding locale to the first line of pandas/__init__.py]" rm -f pandas/__init__.pyc sedc="3iimport locale\nlocale.setlocale(locale.LC_ALL, '$LOCALE_OVERRIDE')\n" diff --git a/circle.yml b/circle.yml index 66415defba6fe..fe8f1467a4b56 100644 --- a/circle.yml +++ b/circle.yml @@ -1,38 +1,77 @@ -machine: - environment: - # these are globally set - MINICONDA_DIR: /home/ubuntu/miniconda3 +version: 2 +jobs: + py27_compat: + docker: + - image: continuumio/miniconda:latest + environment: + JOB: "2.7_COMPAT" + ENV_FILE: "ci/circle-27-compat.yaml" + LOCALE_OVERRIDE: "it_IT.UTF-8" + MINICONDA_DIR: /home/ubuntu/miniconda3 + steps: + - checkout + - run: + name: build + command: | + ./ci/install_circle.sh + ./ci/show_circle.sh + - run: + name: install database + command: ./ci/install_db_circle.sh + - run: + name: test + command: ./ci/run_circle.sh --skip-slow --skip-network + py36_locale: + docker: + - image: continuumio/miniconda:latest + environment: + JOB: "3.6_LOCALE" + ENV_FILE: "ci/circle-36-locale.yaml" + LOCALE_OVERRIDE: "zh_CN.UTF-8" + MINICONDA_DIR: /home/ubuntu/miniconda3 + steps: + - checkout + - run: + name: build + command: | + ./ci/install_circle.sh + ./ci/show_circle.sh + - run: + name: install database + command: ./ci/install_db_circle.sh + - run: + name: test + command: ./ci/run_circle.sh --skip-slow --skip-network -database: - override: - - ./ci/install_db_circle.sh + py35_ascii: + docker: + - image: continuumio/miniconda:latest + environment: + JOB: "3.5_ASCII" + ENV_FILE: "ci/circle-35-ascii.yaml" + LOCALE_OVERRIDE: "C" + MINICONDA_DIR: /home/ubuntu/miniconda3 + steps: + - checkout + - run: + name: build + command: | + ./ci/install_circle.sh + ./ci/show_circle.sh + - run: + name: install database + command: ./ci/install_db_circle.sh + - run: + name: test + command: ./ci/run_circle.sh --skip-slow --skip-network -checkout: - post: - # since circleci does a shallow fetch - # we need to populate our tags - - git fetch --depth=1000 +workflows: + version: 2 + build_and_test: + jobs: + - py27_compat + - py36_locale + - py35_ascii - -dependencies: - override: - - > - case $CIRCLE_NODE_INDEX in - 0) - sudo apt-get install language-pack-it && ./ci/install_circle.sh JOB="2.7_COMPAT" ENV_FILE="ci/circle-27-compat.yaml" LOCALE_OVERRIDE="it_IT.UTF-8" ;; - 1) - sudo apt-get install language-pack-zh-hans && ./ci/install_circle.sh JOB="3.6_LOCALE" ENV_FILE="ci/circle-36-locale.yaml" LOCALE_OVERRIDE="zh_CN.UTF-8" ;; - 2) - sudo apt-get install language-pack-zh-hans && ./ci/install_circle.sh JOB="3.6_LOCALE_SLOW" ENV_FILE="ci/circle-36-locale_slow.yaml" LOCALE_OVERRIDE="zh_CN.UTF-8" ;; - 3) - ./ci/install_circle.sh JOB="3.5_ASCII" ENV_FILE="ci/circle-35-ascii.yaml" LOCALE_OVERRIDE="C" ;; - esac - - ./ci/show_circle.sh - - -test: - override: - - case $CIRCLE_NODE_INDEX in 0) ./ci/run_circle.sh --skip-slow --skip-network ;; 1) ./ci/run_circle.sh --only-slow --skip-network ;; 2) ./ci/run_circle.sh --skip-slow --skip-network ;; 3) ./ci/run_circle.sh --skip-slow --skip-network ;; esac: - parallel: true From f3953ff1606e903926c4f92126c006c04ac1dbee Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Sun, 8 Jul 2018 13:49:52 -0500 Subject: [PATCH 02/25] fail fast --- ci/install_circle.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/install_circle.sh b/ci/install_circle.sh index 2e67b23ade38f..5f00ba531a53e 100755 --- a/ci/install_circle.sh +++ b/ci/install_circle.sh @@ -47,6 +47,8 @@ if [ -n "$LOCALE_OVERRIDE" ]; then sed -i -e 's/# $LOCALE_OVERRIDE UTF-8/$LOCAL_OVERRIDE UTF-8/' /etc/locale.gen && \ locale-gen + python -c 'import locale; locale.setlocale(locale.LC_ALL, "$LOCALE_OVERRIDE")' + echo "[Adding locale to the first line of pandas/__init__.py]" rm -f pandas/__init__.pyc sedc="3iimport locale\nlocale.setlocale(locale.LC_ALL, '$LOCALE_OVERRIDE')\n" From cd588aa78aee026d69beb2ca32bab780f229604b Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Sun, 8 Jul 2018 17:06:01 -0500 Subject: [PATCH 03/25] Fixup escaping --- ci/install_circle.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ci/install_circle.sh b/ci/install_circle.sh index 5f00ba531a53e..6b952b37636fa 100755 --- a/ci/install_circle.sh +++ b/ci/install_circle.sh @@ -44,14 +44,17 @@ if [ -n "$LOCALE_OVERRIDE" ]; then apt-get update && apt-get install locales -y - sed -i -e 's/# $LOCALE_OVERRIDE UTF-8/$LOCAL_OVERRIDE UTF-8/' /etc/locale.gen && \ + sed -i -e "s/# $LOCALE_OVERRIDE UTF-8/$LOCALE_OVERRIDE UTF-8/" /etc/locale.gen && \ locale-gen - python -c 'import locale; locale.setlocale(locale.LC_ALL, "$LOCALE_OVERRIDE")' + export LANG=$LOCALE_OVERRIDE + export LC_ALL=$LOCALE_OVERRIDE + + python -c "import locale; locale.setlocale(locale.LC_ALL, \"$LOCALE_OVERRIDE\")" || exit 1; echo "[Adding locale to the first line of pandas/__init__.py]" rm -f pandas/__init__.pyc - sedc="3iimport locale\nlocale.setlocale(locale.LC_ALL, '$LOCALE_OVERRIDE')\n" + sedc="3iimport locale\nlocale.setlocale(locale.LC_ALL, \"$LOCALE_OVERRIDE\")\n" sed -i "$sedc" pandas/__init__.py echo "[head -4 pandas/__init__.py]" head -4 pandas/__init__.py From 8f9b00ca97f70e88e505574ba215c809b9b8bfc7 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 12 Jul 2018 14:59:19 -0500 Subject: [PATCH 04/25] Update locale setting --- ci/install_circle.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ci/install_circle.sh b/ci/install_circle.sh index 6b952b37636fa..c2ebc255b8c1c 100755 --- a/ci/install_circle.sh +++ b/ci/install_circle.sh @@ -44,8 +44,11 @@ if [ -n "$LOCALE_OVERRIDE" ]; then apt-get update && apt-get install locales -y - sed -i -e "s/# $LOCALE_OVERRIDE UTF-8/$LOCALE_OVERRIDE UTF-8/" /etc/locale.gen && \ - locale-gen + echo "LC_ALL=$LOCALE_OVERRIDE" >> /etc/environment + echo "$LOCALE_OVERRIDE UTF-8" >> /etc/locale.gen + echo "LANG=$LOCALE_OVERRIDE" > /etc/locale.conf + + locale-gen $LOCALE_OVERRIDE export LANG=$LOCALE_OVERRIDE export LC_ALL=$LOCALE_OVERRIDE From 9f1dfdade470486a9ae7d8353d6f44d746009910 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 24 Jul 2018 06:20:36 -0500 Subject: [PATCH 05/25] Update locale setting --- ci/install_circle.sh | 8 +------- circle.yml | 1 - 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/ci/install_circle.sh b/ci/install_circle.sh index c2ebc255b8c1c..cbf1e2cf538ab 100755 --- a/ci/install_circle.sh +++ b/ci/install_circle.sh @@ -42,13 +42,7 @@ source $ENVS_FILE # edit the locale override if needed if [ -n "$LOCALE_OVERRIDE" ]; then - apt-get update && apt-get install locales -y - - echo "LC_ALL=$LOCALE_OVERRIDE" >> /etc/environment - echo "$LOCALE_OVERRIDE UTF-8" >> /etc/locale.gen - echo "LANG=$LOCALE_OVERRIDE" > /etc/locale.conf - - locale-gen $LOCALE_OVERRIDE + apt-get update && apt-get -y install locales locales-all export LANG=$LOCALE_OVERRIDE export LC_ALL=$LOCALE_OVERRIDE diff --git a/circle.yml b/circle.yml index fe8f1467a4b56..06e96add46d49 100644 --- a/circle.yml +++ b/circle.yml @@ -74,4 +74,3 @@ workflows: - py27_compat - py36_locale - py35_ascii - From 5b1940005e34c1e6f9d62db3d06c70d726b3627e Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 24 Jul 2018 06:25:41 -0500 Subject: [PATCH 06/25] Misc * Fixed locale setting * Added 36_slow * fixed command for slow / not slow --- circle.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/circle.yml b/circle.yml index 06e96add46d49..df49f839331eb 100644 --- a/circle.yml +++ b/circle.yml @@ -1,5 +1,9 @@ version: 2 jobs: + + # -------------------------------------------------------------------------- + # 0. py27_compat + # -------------------------------------------------------------------------- py27_compat: docker: - image: continuumio/miniconda:latest @@ -22,6 +26,9 @@ jobs: name: test command: ./ci/run_circle.sh --skip-slow --skip-network + # -------------------------------------------------------------------------- + # 1. py36_locale + # -------------------------------------------------------------------------- py36_locale: docker: - image: continuumio/miniconda:latest @@ -44,6 +51,34 @@ jobs: name: test command: ./ci/run_circle.sh --skip-slow --skip-network + # -------------------------------------------------------------------------- + # 2. py36_locale_slow + # -------------------------------------------------------------------------- + py36_locale_slow: + docker: + - image: continuumio/miniconda:latest + environment: + JOB: "3.6_LOCALE_SLOW" + ENV_FILE: "ci/circle-36-locale_slow.yaml" + LOCALE_OVERRIDE: "zh_CN.UTF-8" + MINICONDA_DIR: /home/ubuntu/miniconda3 + steps: + - checkout + - run: + name: build + command: | + ./ci/install_circle.sh + ./ci/show_circle.sh + - run: + name: install database + command: ./ci/install_db_circle.sh + - run: + name: test + command: ./ci/run_circle.sh --only-slow --skip-network + + # -------------------------------------------------------------------------- + # 3. py35_ascii + # -------------------------------------------------------------------------- py35_ascii: docker: - image: continuumio/miniconda:latest @@ -73,4 +108,5 @@ workflows: jobs: - py27_compat - py36_locale + - py36_locale_slow - py35_ascii From 82ce6e4107e841072a1d32fb4d17125f97df8a4e Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 24 Jul 2018 08:07:17 -0500 Subject: [PATCH 07/25] WIP db --- ci/install_circle.sh | 2 +- ci/install_db_circle.sh | 4 ++-- circle.yml | 5 +++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ci/install_circle.sh b/ci/install_circle.sh index cbf1e2cf538ab..f8bcf6bcffc99 100755 --- a/ci/install_circle.sh +++ b/ci/install_circle.sh @@ -6,7 +6,7 @@ echo "[home_dir: $home_dir]" echo "[ls -ltr]" ls -ltr -apt-get update -y && apt-get install build-essential -y +apt-get update -y && apt-get install -y build-essential postgresql-client-9.6 echo "[update conda]" conda config --set ssl_verify false || exit 1 diff --git a/ci/install_db_circle.sh b/ci/install_db_circle.sh index a00f74f009f54..fbe4d6e57d878 100755 --- a/ci/install_db_circle.sh +++ b/ci/install_db_circle.sh @@ -1,8 +1,8 @@ #!/bin/bash echo "installing dbs" -mysql -e 'create database pandas_nosetest;' -psql -c 'create database pandas_nosetest;' -U postgres +# mysql -e 'create database pandas_nosetest;' +psql -d $POSTGRESQL_URL -c 'create database pandas_nosetest;' -U postgres || exit 1 echo "done" exit 0 diff --git a/circle.yml b/circle.yml index df49f839331eb..34e1cff44ef02 100644 --- a/circle.yml +++ b/circle.yml @@ -7,11 +7,16 @@ jobs: py27_compat: docker: - image: continuumio/miniconda:latest + - image: circleci/postgres:9.6.5-alpine-ram + environment: + POSTGRES_USER: postgres + POSTGRES_DB: pandas_nosetest environment: JOB: "2.7_COMPAT" ENV_FILE: "ci/circle-27-compat.yaml" LOCALE_OVERRIDE: "it_IT.UTF-8" MINICONDA_DIR: /home/ubuntu/miniconda3 + POSTGRESQL_URL: "postgresql://postgres@localhost/pandas_nosetest" steps: - checkout - run: From ab4fd0d387a13bb23d5aed671ef8672e4cad19aa Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 24 Jul 2018 08:20:11 -0500 Subject: [PATCH 08/25] More database --- ci/install_db_circle.sh | 8 -------- circle.yml | 45 +++++++++++++++++++++++++++++------------ 2 files changed, 32 insertions(+), 21 deletions(-) delete mode 100755 ci/install_db_circle.sh diff --git a/ci/install_db_circle.sh b/ci/install_db_circle.sh deleted file mode 100755 index fbe4d6e57d878..0000000000000 --- a/ci/install_db_circle.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -echo "installing dbs" -# mysql -e 'create database pandas_nosetest;' -psql -d $POSTGRESQL_URL -c 'create database pandas_nosetest;' -U postgres || exit 1 - -echo "done" -exit 0 diff --git a/circle.yml b/circle.yml index 34e1cff44ef02..8a1eb2981dfde 100644 --- a/circle.yml +++ b/circle.yml @@ -7,16 +7,20 @@ jobs: py27_compat: docker: - image: continuumio/miniconda:latest + # databases configuration - image: circleci/postgres:9.6.5-alpine-ram environment: POSTGRES_USER: postgres POSTGRES_DB: pandas_nosetest + - image: circleci/mysql:8-ram + MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_ROOT_PASSWORD: '' + MYSQL_DATABASE: pandas_nosetest environment: JOB: "2.7_COMPAT" ENV_FILE: "ci/circle-27-compat.yaml" LOCALE_OVERRIDE: "it_IT.UTF-8" MINICONDA_DIR: /home/ubuntu/miniconda3 - POSTGRESQL_URL: "postgresql://postgres@localhost/pandas_nosetest" steps: - checkout - run: @@ -24,9 +28,6 @@ jobs: command: | ./ci/install_circle.sh ./ci/show_circle.sh - - run: - name: install database - command: ./ci/install_db_circle.sh - run: name: test command: ./ci/run_circle.sh --skip-slow --skip-network @@ -37,6 +38,15 @@ jobs: py36_locale: docker: - image: continuumio/miniconda:latest + # databases configuration + - image: circleci/postgres:9.6.5-alpine-ram + environment: + POSTGRES_USER: postgres + POSTGRES_DB: pandas_nosetest + - image: circleci/mysql:8-ram + MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_ROOT_PASSWORD: '' + MYSQL_DATABASE: pandas_nosetest environment: JOB: "3.6_LOCALE" ENV_FILE: "ci/circle-36-locale.yaml" @@ -49,9 +59,6 @@ jobs: command: | ./ci/install_circle.sh ./ci/show_circle.sh - - run: - name: install database - command: ./ci/install_db_circle.sh - run: name: test command: ./ci/run_circle.sh --skip-slow --skip-network @@ -62,6 +69,15 @@ jobs: py36_locale_slow: docker: - image: continuumio/miniconda:latest + # databases configuration + - image: circleci/postgres:9.6.5-alpine-ram + environment: + POSTGRES_USER: postgres + POSTGRES_DB: pandas_nosetest + - image: circleci/mysql:8-ram + MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_ROOT_PASSWORD: '' + MYSQL_DATABASE: pandas_nosetest environment: JOB: "3.6_LOCALE_SLOW" ENV_FILE: "ci/circle-36-locale_slow.yaml" @@ -74,9 +90,6 @@ jobs: command: | ./ci/install_circle.sh ./ci/show_circle.sh - - run: - name: install database - command: ./ci/install_db_circle.sh - run: name: test command: ./ci/run_circle.sh --only-slow --skip-network @@ -87,6 +100,15 @@ jobs: py35_ascii: docker: - image: continuumio/miniconda:latest + # databases configuration + - image: circleci/postgres:9.6.5-alpine-ram + environment: + POSTGRES_USER: postgres + POSTGRES_DB: pandas_nosetest + - image: circleci/mysql:8-ram + MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_ROOT_PASSWORD: '' + MYSQL_DATABASE: pandas_nosetest environment: JOB: "3.5_ASCII" ENV_FILE: "ci/circle-35-ascii.yaml" @@ -99,9 +121,6 @@ jobs: command: | ./ci/install_circle.sh ./ci/show_circle.sh - - run: - name: install database - command: ./ci/install_db_circle.sh - run: name: test command: ./ci/run_circle.sh --skip-slow --skip-network From 596ad25d3b9cd7a3f8b8364414c63be4deda1bee Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 8 Aug 2018 09:37:42 -0500 Subject: [PATCH 09/25] move --- circle.yml => .circleci/config.yml | 35 ++++++++++++++++++++---------- ci/run_circle.sh | 2 +- 2 files changed, 24 insertions(+), 13 deletions(-) rename circle.yml => .circleci/config.yml (84%) diff --git a/circle.yml b/.circleci/config.yml similarity index 84% rename from circle.yml rename to .circleci/config.yml index 8a1eb2981dfde..e947f30d285cd 100644 --- a/circle.yml +++ b/.circleci/config.yml @@ -13,9 +13,11 @@ jobs: POSTGRES_USER: postgres POSTGRES_DB: pandas_nosetest - image: circleci/mysql:8-ram - MYSQL_ALLOW_EMPTY_PASSWORD: yes - MYSQL_ROOT_PASSWORD: '' - MYSQL_DATABASE: pandas_nosetest + environment: + MYSQL_USER: "root" + MYSQL_HOST: "localhost" + MYSQL_ALLOW_EMPTY_PASSWORD: "true" + MYSQL_DATABASE: "pandas_nosetest" environment: JOB: "2.7_COMPAT" ENV_FILE: "ci/circle-27-compat.yaml" @@ -44,9 +46,12 @@ jobs: POSTGRES_USER: postgres POSTGRES_DB: pandas_nosetest - image: circleci/mysql:8-ram - MYSQL_ALLOW_EMPTY_PASSWORD: yes - MYSQL_ROOT_PASSWORD: '' - MYSQL_DATABASE: pandas_nosetest + environment: + MYSQL_USER: "root" + MYSQL_HOST: "localhost" + MYSQL_ALLOW_EMPTY_PASSWORD: "true" + MYSQL_DATABASE: "pandas_nosetest" + environment: JOB: "3.6_LOCALE" ENV_FILE: "ci/circle-36-locale.yaml" @@ -75,9 +80,12 @@ jobs: POSTGRES_USER: postgres POSTGRES_DB: pandas_nosetest - image: circleci/mysql:8-ram - MYSQL_ALLOW_EMPTY_PASSWORD: yes - MYSQL_ROOT_PASSWORD: '' - MYSQL_DATABASE: pandas_nosetest + environment: + MYSQL_USER: "root" + MYSQL_HOST: "localhost" + MYSQL_ALLOW_EMPTY_PASSWORD: "true" + MYSQL_DATABASE: "pandas_nosetest" + environment: JOB: "3.6_LOCALE_SLOW" ENV_FILE: "ci/circle-36-locale_slow.yaml" @@ -106,9 +114,12 @@ jobs: POSTGRES_USER: postgres POSTGRES_DB: pandas_nosetest - image: circleci/mysql:8-ram - MYSQL_ALLOW_EMPTY_PASSWORD: yes - MYSQL_ROOT_PASSWORD: '' - MYSQL_DATABASE: pandas_nosetest + environment: + MYSQL_USER: "root" + MYSQL_HOST: "localhost" + MYSQL_ALLOW_EMPTY_PASSWORD: "true" + MYSQL_DATABASE: "pandas_nosetest" + environment: JOB: "3.5_ASCII" ENV_FILE: "ci/circle-35-ascii.yaml" diff --git a/ci/run_circle.sh b/ci/run_circle.sh index 435985bd42148..fc2a8b849a354 100755 --- a/ci/run_circle.sh +++ b/ci/run_circle.sh @@ -6,4 +6,4 @@ export PATH="$MINICONDA_DIR/bin:$PATH" source activate pandas echo "pytest --strict --junitxml=$CIRCLE_TEST_REPORTS/reports/junit.xml $@ pandas" -pytest --strict --junitxml=$CIRCLE_TEST_REPORTS/reports/junit.xml $@ pandas +pytest --strict --color=no --junitxml=$CIRCLE_TEST_REPORTS/reports/junit.xml $@ pandas From 92c1c939a9043f1aa4b11e1d4d1c2c7732bccc74 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 14 Aug 2018 07:43:43 -0500 Subject: [PATCH 10/25] Added skips for s3 non-us locale --- pandas/tests/io/json/test_compression.py | 2 ++ pandas/tests/io/json/test_pandas.py | 2 ++ pandas/tests/io/parser/test_network.py | 4 ++++ pandas/tests/io/test_excel.py | 1 + 4 files changed, 9 insertions(+) diff --git a/pandas/tests/io/json/test_compression.py b/pandas/tests/io/json/test_compression.py index f2e72e5fe00e1..b411744f7bac2 100644 --- a/pandas/tests/io/json/test_compression.py +++ b/pandas/tests/io/json/test_compression.py @@ -2,6 +2,7 @@ import pandas as pd import pandas.util.testing as tm +import pandas.util._test_decorators as td from pandas.util.testing import assert_frame_equal, assert_raises_regex @@ -31,6 +32,7 @@ def test_read_zipped_json(datapath): assert_frame_equal(uncompressed_df, compressed_df) +@td.skip_if_not_us_locale def test_with_s3_url(compression): boto3 = pytest.importorskip('boto3') pytest.importorskip('s3fs') diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index 0715521a74819..04f0220839523 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -15,6 +15,7 @@ assert_series_equal, network, ensure_clean, assert_index_equal) import pandas.util.testing as tm +import pandas.util._test_decorators as td _seriesd = tm.getSeriesData() _tsd = tm.getTimeSeriesData() @@ -1047,6 +1048,7 @@ def test_read_inline_jsonl(self): expected = DataFrame([[1, 2], [1, 2]], columns=['a', 'b']) assert_frame_equal(result, expected) + @td.skip_if_not_us_locale def test_read_s3_jsonl(self, s3_resource): # GH17200 diff --git a/pandas/tests/io/parser/test_network.py b/pandas/tests/io/parser/test_network.py index a7cc3ad989ea1..31579aa4ef0ef 100644 --- a/pandas/tests/io/parser/test_network.py +++ b/pandas/tests/io/parser/test_network.py @@ -14,7 +14,9 @@ from pandas import DataFrame from pandas.io.parsers import read_csv from pandas.compat import BytesIO, StringIO +import locale +locale.setlocale(locale.LC_ALL, "zh_CN.UTF-8") @pytest.mark.network @pytest.mark.parametrize( @@ -55,10 +57,12 @@ def tips_df(datapath): @pytest.mark.usefixtures("s3_resource") +@td.skip_if_not_us_locale() class TestS3(object): def test_parse_public_s3_bucket(self, tips_df): pytest.importorskip('s3fs') + # more of an integration test due to the not-public contents portion # can probably mock this though. for ext, comp in [('', None), ('.gz', 'gzip'), ('.bz2', 'bz2')]: diff --git a/pandas/tests/io/test_excel.py b/pandas/tests/io/test_excel.py index fa5a8f6a1900c..5f27ff719fda1 100644 --- a/pandas/tests/io/test_excel.py +++ b/pandas/tests/io/test_excel.py @@ -586,6 +586,7 @@ def test_read_from_http_url(self, ext): tm.assert_frame_equal(url_table, local_table) @td.skip_if_no('s3fs') + @td.skip_if_not_us_locale def test_read_from_s3_url(self, ext): boto3 = pytest.importorskip('boto3') moto = pytest.importorskip('moto') From 776066aa42202a6eb3c8a1cdb6054d58ae7dcaf8 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 14 Aug 2018 07:51:44 -0500 Subject: [PATCH 11/25] remove debug code --- pandas/tests/io/parser/test_network.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pandas/tests/io/parser/test_network.py b/pandas/tests/io/parser/test_network.py index 31579aa4ef0ef..bfe33980ac617 100644 --- a/pandas/tests/io/parser/test_network.py +++ b/pandas/tests/io/parser/test_network.py @@ -14,9 +14,7 @@ from pandas import DataFrame from pandas.io.parsers import read_csv from pandas.compat import BytesIO, StringIO -import locale -locale.setlocale(locale.LC_ALL, "zh_CN.UTF-8") @pytest.mark.network @pytest.mark.parametrize( From 8220a281bc45f93961d578bc9c94c37ef98c4535 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 14 Aug 2018 08:39:15 -0500 Subject: [PATCH 12/25] Normalize before comparison in tests --- pandas/tests/indexes/datetimes/test_misc.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_misc.py b/pandas/tests/indexes/datetimes/test_misc.py index 056924f2c6663..2f4e9fa22fbae 100644 --- a/pandas/tests/indexes/datetimes/test_misc.py +++ b/pandas/tests/indexes/datetimes/test_misc.py @@ -1,5 +1,6 @@ import locale import calendar +import unicodedata import pytest @@ -284,10 +285,22 @@ def test_datetime_name_accessors(self, time_locale): dti = DatetimeIndex(freq='M', start='2012', end='2013') result = dti.month_name(locale=time_locale) expected = Index([month.capitalize() for month in expected_months]) + + # work around different normalization schemes + # https://github.com/pandas-dev/pandas/issues/22342 + assert isinstance(result, pd.Index) + + result = pd.Index([unicodedata.normalize("NFD", x) for x in result]) + expected = Index([unicodedata.normalize("NFD", month.capitalize()) + for month in expected_months]) + tm.assert_index_equal(result, expected) for date, expected in zip(dti, expected_months): - result = date.month_name(locale=time_locale) - assert result == expected.capitalize() + result = unicodedata.normalize( + "NFD", date.month_name(locale=time_locale) + ) + expected = unicodedata.normalize("NFD", expected.capitalize()) + assert result == expected dti = dti.append(DatetimeIndex([pd.NaT])) assert np.isnan(dti.month_name(locale=time_locale)[-1]) From 5218a1126df95bd3bcc65aa73e9528659c4278ce Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 14 Aug 2018 10:40:07 -0500 Subject: [PATCH 13/25] py2 compat --- pandas/tests/indexes/datetimes/test_misc.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_misc.py b/pandas/tests/indexes/datetimes/test_misc.py index 2f4e9fa22fbae..67436fa9efd99 100644 --- a/pandas/tests/indexes/datetimes/test_misc.py +++ b/pandas/tests/indexes/datetimes/test_misc.py @@ -290,9 +290,14 @@ def test_datetime_name_accessors(self, time_locale): # https://github.com/pandas-dev/pandas/issues/22342 assert isinstance(result, pd.Index) - result = pd.Index([unicodedata.normalize("NFD", x) for x in result]) - expected = Index([unicodedata.normalize("NFD", month.capitalize()) - for month in expected_months]) + result = pd.Index([ + unicodedata.normalize("NFD", x.encode("utf-8")) + for x in result + ]) + expected = Index([ + unicodedata.normalize("NFD", month.capitalize().encode("utf-8")) + for month in expected_months + ]) tm.assert_index_equal(result, expected) for date, expected in zip(dti, expected_months): From e66e6e4bb74ef2a4e9f3d1ffc95e8ee68ade2edf Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 14 Aug 2018 15:00:55 -0500 Subject: [PATCH 14/25] py2 compat --- pandas/tests/indexes/datetimes/test_misc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_misc.py b/pandas/tests/indexes/datetimes/test_misc.py index 67436fa9efd99..fe73444423229 100644 --- a/pandas/tests/indexes/datetimes/test_misc.py +++ b/pandas/tests/indexes/datetimes/test_misc.py @@ -8,7 +8,7 @@ import pandas as pd import pandas.util.testing as tm from pandas import (Index, DatetimeIndex, datetime, offsets, - date_range, Timestamp) + date_range, Timestamp, compat) class TestTimeSeries(object): @@ -291,11 +291,11 @@ def test_datetime_name_accessors(self, time_locale): assert isinstance(result, pd.Index) result = pd.Index([ - unicodedata.normalize("NFD", x.encode("utf-8")) + unicodedata.normalize("NFD", compat.text_type(x)) for x in result ]) expected = Index([ - unicodedata.normalize("NFD", month.capitalize().encode("utf-8")) + unicodedata.normalize("NFD", compat.text_type(month).capitalize()) for month in expected_months ]) From 53724a8d5d0e21b31e71dc3f4915642d290adc25 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 14 Aug 2018 16:05:16 -0500 Subject: [PATCH 15/25] py2 compat --- pandas/tests/indexes/datetimes/test_misc.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_misc.py b/pandas/tests/indexes/datetimes/test_misc.py index fe73444423229..9ff22ad9271b9 100644 --- a/pandas/tests/indexes/datetimes/test_misc.py +++ b/pandas/tests/indexes/datetimes/test_misc.py @@ -302,9 +302,11 @@ def test_datetime_name_accessors(self, time_locale): tm.assert_index_equal(result, expected) for date, expected in zip(dti, expected_months): result = unicodedata.normalize( - "NFD", date.month_name(locale=time_locale) + "NFD", compat.text_type(date.month_name(locale=time_locale)) + ) + expected = compat.text_type( + unicodedata.normalize("NFD", expected.capitalize()) ) - expected = unicodedata.normalize("NFD", expected.capitalize()) assert result == expected dti = dti.append(DatetimeIndex([pd.NaT])) assert np.isnan(dti.month_name(locale=time_locale)[-1]) From 3256cf9c3e61e164b9dc31d8012586818a47773e Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 14 Aug 2018 21:15:04 -0500 Subject: [PATCH 16/25] Use .str.normalize --- pandas/tests/indexes/datetimes/test_misc.py | 14 +++----------- pandas/tests/series/test_datetime_values.py | 21 ++++++++++++++++++--- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_misc.py b/pandas/tests/indexes/datetimes/test_misc.py index 9ff22ad9271b9..33f10dbb0fb43 100644 --- a/pandas/tests/indexes/datetimes/test_misc.py +++ b/pandas/tests/indexes/datetimes/test_misc.py @@ -288,18 +288,10 @@ def test_datetime_name_accessors(self, time_locale): # work around different normalization schemes # https://github.com/pandas-dev/pandas/issues/22342 - assert isinstance(result, pd.Index) - - result = pd.Index([ - unicodedata.normalize("NFD", compat.text_type(x)) - for x in result - ]) - expected = Index([ - unicodedata.normalize("NFD", compat.text_type(month).capitalize()) - for month in expected_months - ]) - + result = result.str.normalize("NFD") + expected = expected.str.normalize("NDF") tm.assert_index_equal(result, expected) + for date, expected in zip(dti, expected_months): result = unicodedata.normalize( "NFD", compat.text_type(date.month_name(locale=time_locale)) diff --git a/pandas/tests/series/test_datetime_values.py b/pandas/tests/series/test_datetime_values.py index 06eb525bbac56..6518a050a6d86 100644 --- a/pandas/tests/series/test_datetime_values.py +++ b/pandas/tests/series/test_datetime_values.py @@ -3,6 +3,7 @@ import locale import calendar +import unicodedata import pytest from datetime import datetime, time, date @@ -13,7 +14,8 @@ from pandas.core.dtypes.common import is_integer_dtype, is_list_like from pandas import (Index, Series, DataFrame, bdate_range, date_range, period_range, timedelta_range, - PeriodIndex, DatetimeIndex, TimedeltaIndex) + PeriodIndex, DatetimeIndex, TimedeltaIndex, + compat) import pandas.core.common as com import dateutil @@ -308,10 +310,23 @@ def test_dt_accessor_datetime_name_accessors(self, time_locale): s = Series(DatetimeIndex(freq='M', start='2012', end='2013')) result = s.dt.month_name(locale=time_locale) expected = Series([month.capitalize() for month in expected_months]) + + # work around https://github.com/pandas-dev/pandas/issues/22342 + result = result.str.normalize("NFD") + expected = expected.str.normalize("NFD") + tm.assert_series_equal(result, expected) + for s_date, expected in zip(s, expected_months): - result = s_date.month_name(locale=time_locale) - assert result == expected.capitalize() + result = unicodedata.normalize( + "NFD", compat.text_type(s_date.month_name(locale=time_locale)) + ) + expected = compat.text_type( + unicodedata.normalize("NFD", expected.capitalize()) + ) + + assert result == expected + s = s.append(Series([pd.NaT])) assert np.isnan(s.dt.month_name(locale=time_locale).iloc[-1]) From 12ce1b8cf28fbebf1be7b5259d293a0bc2debfff Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 15 Aug 2018 09:46:05 -0500 Subject: [PATCH 17/25] fixups --- pandas/tests/indexes/datetimes/test_misc.py | 2 +- .../tests/scalar/timestamp/test_timestamp.py | 27 ++++++++++++++----- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_misc.py b/pandas/tests/indexes/datetimes/test_misc.py index 33f10dbb0fb43..6b53e4962b56e 100644 --- a/pandas/tests/indexes/datetimes/test_misc.py +++ b/pandas/tests/indexes/datetimes/test_misc.py @@ -289,7 +289,7 @@ def test_datetime_name_accessors(self, time_locale): # work around different normalization schemes # https://github.com/pandas-dev/pandas/issues/22342 result = result.str.normalize("NFD") - expected = expected.str.normalize("NDF") + expected = expected.str.normalize("NFD") tm.assert_index_equal(result, expected) for date, expected in zip(dti, expected_months): diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index 4172bfd41b9db..7bc6bcf9fe2c0 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -5,6 +5,7 @@ import dateutil import calendar import locale +import unicodedata import numpy as np from dateutil.tz import tzutc @@ -20,7 +21,7 @@ from pandas._libs.tslibs.timezones import get_timezone, dateutil_gettz as gettz from pandas.errors import OutOfBoundsDatetime -from pandas.compat import long, PY3 +from pandas.compat import long, PY3, text_type from pandas.compat.numpy import np_datetime64_compat from pandas import Timestamp, Period, Timedelta, NaT @@ -113,11 +114,25 @@ def test_names(self, data, time_locale): expected_month = 'August' else: with tm.set_locale(time_locale, locale.LC_TIME): - expected_day = calendar.day_name[0].capitalize() - expected_month = calendar.month_name[8].capitalize() - - assert data.day_name(time_locale) == expected_day - assert data.month_name(time_locale) == expected_month + expected_day = unicodedata.normalize( + "NFD", text_type(calendar.day_name[0].capitalize()) + ) + expected_month = unicodedata.normalize( + "NFD", calendar.month_name[8].capitalize() + ) + + # Work around https://github.com/pandas-dev/pandas/issues/22342 + # different normalizations + + result_day = unicodedata.normalize( + "NFD", data.day_name(time_locale) + ) + result_month = unicodedata.normalize( + "NFD", data.month_name_name(time_locale) + ) + + assert result_day == expected_day + assert result_month == expected_month # Test NaT nan_ts = Timestamp(NaT) From 92188c0c74bcc371d8e3ef4dab6e135ad7160d82 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 15 Aug 2018 09:54:40 -0500 Subject: [PATCH 18/25] fixup --- pandas/tests/scalar/timestamp/test_timestamp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index 7bc6bcf9fe2c0..13fb1f273f7ba 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -128,7 +128,7 @@ def test_names(self, data, time_locale): "NFD", data.day_name(time_locale) ) result_month = unicodedata.normalize( - "NFD", data.month_name_name(time_locale) + "NFD", data.month_name(time_locale) ) assert result_day == expected_day From 132f76ac149a665a0733087b82136999ed63e864 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 15 Aug 2018 13:21:07 -0500 Subject: [PATCH 19/25] py2 compat --- pandas/tests/series/test_datetime_values.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/tests/series/test_datetime_values.py b/pandas/tests/series/test_datetime_values.py index 6518a050a6d86..556a43e211a2b 100644 --- a/pandas/tests/series/test_datetime_values.py +++ b/pandas/tests/series/test_datetime_values.py @@ -321,8 +321,9 @@ def test_dt_accessor_datetime_name_accessors(self, time_locale): result = unicodedata.normalize( "NFD", compat.text_type(s_date.month_name(locale=time_locale)) ) - expected = compat.text_type( - unicodedata.normalize("NFD", expected.capitalize()) + expected = unicodedata.normalize( + "NFD", + compat.text_type(expected.capitalize()) ) assert result == expected From 7c231d32eca50ff84387d0dcc7c970bed62f5012 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 15 Aug 2018 13:28:02 -0500 Subject: [PATCH 20/25] py2 compat --- pandas/tests/indexes/datetimes/test_misc.py | 4 ++++ .../tests/scalar/timestamp/test_timestamp.py | 22 +++++++++++-------- pandas/tests/series/test_datetime_values.py | 4 ++++ 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_misc.py b/pandas/tests/indexes/datetimes/test_misc.py index 6b53e4962b56e..e67ac9a7c1636 100644 --- a/pandas/tests/indexes/datetimes/test_misc.py +++ b/pandas/tests/indexes/datetimes/test_misc.py @@ -293,6 +293,10 @@ def test_datetime_name_accessors(self, time_locale): tm.assert_index_equal(result, expected) for date, expected in zip(dti, expected_months): + if compat.PY2 and time_locale: + # some locales may have non-ascii characters + # not worth testing at this point + break result = unicodedata.normalize( "NFD", compat.text_type(date.month_name(locale=time_locale)) ) diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index 13fb1f273f7ba..6f123210e0433 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -21,7 +21,7 @@ from pandas._libs.tslibs.timezones import get_timezone, dateutil_gettz as gettz from pandas.errors import OutOfBoundsDatetime -from pandas.compat import long, PY3, text_type +from pandas.compat import long, PY3, text_type, PY2 from pandas.compat.numpy import np_datetime64_compat from pandas import Timestamp, Period, Timedelta, NaT @@ -124,15 +124,19 @@ def test_names(self, data, time_locale): # Work around https://github.com/pandas-dev/pandas/issues/22342 # different normalizations - result_day = unicodedata.normalize( - "NFD", data.day_name(time_locale) - ) - result_month = unicodedata.normalize( - "NFD", data.month_name(time_locale) - ) + if not (PY2 and time_locale): + # some locales may have non-ascii characters + # not worth testing at this point - assert result_day == expected_day - assert result_month == expected_month + result_day = unicodedata.normalize( + "NFD", data.day_name(time_locale) + ) + result_month = unicodedata.normalize( + "NFD", data.month_name(time_locale) + ) + + assert result_day == expected_day + assert result_month == expected_month # Test NaT nan_ts = Timestamp(NaT) diff --git a/pandas/tests/series/test_datetime_values.py b/pandas/tests/series/test_datetime_values.py index 556a43e211a2b..d58c5be6b0098 100644 --- a/pandas/tests/series/test_datetime_values.py +++ b/pandas/tests/series/test_datetime_values.py @@ -318,6 +318,10 @@ def test_dt_accessor_datetime_name_accessors(self, time_locale): tm.assert_series_equal(result, expected) for s_date, expected in zip(s, expected_months): + if compat.PY2 and time_locale: + # some locales may have non-ascii characters + # not worth testing at this point + break result = unicodedata.normalize( "NFD", compat.text_type(s_date.month_name(locale=time_locale)) ) From 96f1faa671299b5696f7e9729a0911e53ff616f8 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 15 Aug 2018 14:12:57 -0500 Subject: [PATCH 21/25] py2 compat --- pandas/tests/indexes/datetimes/test_misc.py | 4 ++-- pandas/tests/scalar/timestamp/test_timestamp.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_misc.py b/pandas/tests/indexes/datetimes/test_misc.py index e67ac9a7c1636..547534eee3149 100644 --- a/pandas/tests/indexes/datetimes/test_misc.py +++ b/pandas/tests/indexes/datetimes/test_misc.py @@ -300,8 +300,8 @@ def test_datetime_name_accessors(self, time_locale): result = unicodedata.normalize( "NFD", compat.text_type(date.month_name(locale=time_locale)) ) - expected = compat.text_type( - unicodedata.normalize("NFD", expected.capitalize()) + expected = unicodedata.normalize( + compat.text_type("NFD", expected.capitalize()) ) assert result == expected dti = dti.append(DatetimeIndex([pd.NaT])) diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index 6f123210e0433..f392a6e0f8de2 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -129,10 +129,10 @@ def test_names(self, data, time_locale): # not worth testing at this point result_day = unicodedata.normalize( - "NFD", data.day_name(time_locale) + "NFD", text_type(data.day_name(time_locale)) ) result_month = unicodedata.normalize( - "NFD", data.month_name(time_locale) + "NFD", text_type(data.month_name(time_locale)) ) assert result_day == expected_day From 8d7647c140eeebdc694b69a92b500522ad4b4016 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 15 Aug 2018 15:14:55 -0500 Subject: [PATCH 22/25] Fixup --- pandas/tests/indexes/datetimes/test_misc.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_misc.py b/pandas/tests/indexes/datetimes/test_misc.py index 547534eee3149..b01477fbac7d0 100644 --- a/pandas/tests/indexes/datetimes/test_misc.py +++ b/pandas/tests/indexes/datetimes/test_misc.py @@ -298,10 +298,12 @@ def test_datetime_name_accessors(self, time_locale): # not worth testing at this point break result = unicodedata.normalize( - "NFD", compat.text_type(date.month_name(locale=time_locale)) + "NFD", + compat.text_type(date.month_name(locale=time_locale)) ) expected = unicodedata.normalize( - compat.text_type("NFD", expected.capitalize()) + "NFD", + compat.text_type(expected.capitalize()) ) assert result == expected dti = dti.append(DatetimeIndex([pd.NaT])) From 11d80dc2125668fed04c36ae6a0c3010abe5ba28 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 16 Aug 2018 08:12:43 -0500 Subject: [PATCH 23/25] py2 compat --- pandas/tests/indexes/datetimes/test_misc.py | 25 +++++++--------- .../tests/scalar/timestamp/test_timestamp.py | 29 ++++++++++--------- pandas/tests/series/test_datetime_values.py | 21 +++++--------- 3 files changed, 35 insertions(+), 40 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_misc.py b/pandas/tests/indexes/datetimes/test_misc.py index b01477fbac7d0..743cbc107cce5 100644 --- a/pandas/tests/indexes/datetimes/test_misc.py +++ b/pandas/tests/indexes/datetimes/test_misc.py @@ -288,23 +288,20 @@ def test_datetime_name_accessors(self, time_locale): # work around different normalization schemes # https://github.com/pandas-dev/pandas/issues/22342 - result = result.str.normalize("NFD") - expected = expected.str.normalize("NFD") + if not compat.PY2: + result = result.str.normalize("NFD") + expected = expected.str.normalize("NFD") + tm.assert_index_equal(result, expected) for date, expected in zip(dti, expected_months): - if compat.PY2 and time_locale: - # some locales may have non-ascii characters - # not worth testing at this point - break - result = unicodedata.normalize( - "NFD", - compat.text_type(date.month_name(locale=time_locale)) - ) - expected = unicodedata.normalize( - "NFD", - compat.text_type(expected.capitalize()) - ) + result = date.month_name(locale=time_locale) + expected = expected.capitalize() + + if not compat.PY2: + result = unicodedata.normalize("NFD", result) + expected = unicodedata.normalize("NFD", result) + assert result == expected dti = dti.append(DatetimeIndex([pd.NaT])) assert np.isnan(dti.month_name(locale=time_locale)[-1]) diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index f392a6e0f8de2..474e7e967d3a8 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -21,7 +21,7 @@ from pandas._libs.tslibs.timezones import get_timezone, dateutil_gettz as gettz from pandas.errors import OutOfBoundsDatetime -from pandas.compat import long, PY3, text_type, PY2 +from pandas.compat import long, PY3, PY2 from pandas.compat.numpy import np_datetime64_compat from pandas import Timestamp, Period, Timedelta, NaT @@ -114,29 +114,32 @@ def test_names(self, data, time_locale): expected_month = 'August' else: with tm.set_locale(time_locale, locale.LC_TIME): - expected_day = unicodedata.normalize( - "NFD", text_type(calendar.day_name[0].capitalize()) - ) - expected_month = unicodedata.normalize( - "NFD", calendar.month_name[8].capitalize() - ) + expected_day = calendar.day_name[0].capitalize() + expected_month = calendar.month_name[8].capitalize() + + result_day = data.day_name(time_locale) + result_month = data.month_name(time_locale) # Work around https://github.com/pandas-dev/pandas/issues/22342 # different normalizations if not (PY2 and time_locale): - # some locales may have non-ascii characters - # not worth testing at this point + expected_day = unicodedata.normalize( + "NFD", expected_day + ) + expected_month = unicodedata.normalize( + "NFD", expected_month + ) result_day = unicodedata.normalize( - "NFD", text_type(data.day_name(time_locale)) + "NFD", result_day, ) result_month = unicodedata.normalize( - "NFD", text_type(data.month_name(time_locale)) + "NFD", result_month ) - assert result_day == expected_day - assert result_month == expected_month + assert result_day == expected_day + assert result_month == expected_month # Test NaT nan_ts = Timestamp(NaT) diff --git a/pandas/tests/series/test_datetime_values.py b/pandas/tests/series/test_datetime_values.py index d58c5be6b0098..9b5d3b4f11406 100644 --- a/pandas/tests/series/test_datetime_values.py +++ b/pandas/tests/series/test_datetime_values.py @@ -312,23 +312,18 @@ def test_dt_accessor_datetime_name_accessors(self, time_locale): expected = Series([month.capitalize() for month in expected_months]) # work around https://github.com/pandas-dev/pandas/issues/22342 - result = result.str.normalize("NFD") - expected = expected.str.normalize("NFD") + if not compat.PY2: + result = result.str.normalize("NFD") + expected = expected.str.normalize("NFD") tm.assert_series_equal(result, expected) for s_date, expected in zip(s, expected_months): - if compat.PY2 and time_locale: - # some locales may have non-ascii characters - # not worth testing at this point - break - result = unicodedata.normalize( - "NFD", compat.text_type(s_date.month_name(locale=time_locale)) - ) - expected = unicodedata.normalize( - "NFD", - compat.text_type(expected.capitalize()) - ) + expected = expected.capitalize() + + if not compat.PY2: + result = unicodedata.normalize("NFD", result) + expected = unicodedata.normalize("NFD", expected) assert result == expected From 57489588e95e30408a0bafc023b37853c4bb2aaf Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 16 Aug 2018 08:26:02 -0500 Subject: [PATCH 24/25] fixup --- pandas/tests/series/test_datetime_values.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/series/test_datetime_values.py b/pandas/tests/series/test_datetime_values.py index 9b5d3b4f11406..e2dcbfae78b16 100644 --- a/pandas/tests/series/test_datetime_values.py +++ b/pandas/tests/series/test_datetime_values.py @@ -319,6 +319,7 @@ def test_dt_accessor_datetime_name_accessors(self, time_locale): tm.assert_series_equal(result, expected) for s_date, expected in zip(s, expected_months): + result = s_date.month_name(locale=time_locale) expected = expected.capitalize() if not compat.PY2: From 03e732f00bbb259892b1728954566cb4eb3fb762 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 16 Aug 2018 08:27:56 -0500 Subject: [PATCH 25/25] fixup --- .../tests/scalar/timestamp/test_timestamp.py | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index 474e7e967d3a8..58146cae587fe 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -123,20 +123,12 @@ def test_names(self, data, time_locale): # Work around https://github.com/pandas-dev/pandas/issues/22342 # different normalizations - if not (PY2 and time_locale): - expected_day = unicodedata.normalize( - "NFD", expected_day - ) - expected_month = unicodedata.normalize( - "NFD", expected_month - ) - - result_day = unicodedata.normalize( - "NFD", result_day, - ) - result_month = unicodedata.normalize( - "NFD", result_month - ) + if not PY2: + expected_day = unicodedata.normalize("NFD", expected_day) + expected_month = unicodedata.normalize("NFD", expected_month) + + result_day = unicodedata.normalize("NFD", result_day,) + result_month = unicodedata.normalize("NFD", result_month) assert result_day == expected_day assert result_month == expected_month