From 0a1ea75a384d87f6df123e29ebe74d696275cfae Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 10 Dec 2018 17:53:33 -0500 Subject: [PATCH 01/24] FIX: Immutable buffers should not be set writeable --- nibabel/streamlines/tck.py | 4 +--- nibabel/streamlines/tests/test_trk.py | 3 +-- nibabel/volumeutils.py | 8 ++++++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/nibabel/streamlines/tck.py b/nibabel/streamlines/tck.py index 1ba2d76625..bf732e10d8 100644 --- a/nibabel/streamlines/tck.py +++ b/nibabel/streamlines/tck.py @@ -432,10 +432,8 @@ def _read(cls, fileobj, header, buffer_size=4): for point_part in point_parts: # Read floats. pts = np.frombuffer(point_part, dtype=dtype) - # Enforce ability to write to underlying bytes object - pts.flags.writeable = True # Convert data to little-endian if needed. - yield pts.astype(' Date: Tue, 11 Dec 2018 19:02:56 -0500 Subject: [PATCH 02/24] Revert "FIX: Immutable buffers should not be set writeable" This reverts commit 0a1ea75a384d87f6df123e29ebe74d696275cfae. --- nibabel/streamlines/tck.py | 4 +++- nibabel/streamlines/tests/test_trk.py | 3 ++- nibabel/volumeutils.py | 8 ++------ 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/nibabel/streamlines/tck.py b/nibabel/streamlines/tck.py index bf732e10d8..1ba2d76625 100644 --- a/nibabel/streamlines/tck.py +++ b/nibabel/streamlines/tck.py @@ -432,8 +432,10 @@ def _read(cls, fileobj, header, buffer_size=4): for point_part in point_parts: # Read floats. pts = np.frombuffer(point_part, dtype=dtype) + # Enforce ability to write to underlying bytes object + pts.flags.writeable = True # Convert data to little-endian if needed. - yield pts.astype(' Date: Tue, 11 Dec 2018 19:05:38 -0500 Subject: [PATCH 03/24] MAINT: Add bz2file dependency for Python 2 --- nibabel/info.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nibabel/info.py b/nibabel/info.py index 204412c5d2..2788a0875a 100644 --- a/nibabel/info.py +++ b/nibabel/info.py @@ -209,4 +209,5 @@ def cmp_pkg_version(version_str, pkg_version_str=__version__): ISRELEASE = _version_extra == '' VERSION = __version__ PROVIDES = ["nibabel", 'nisext'] -REQUIRES = ["numpy (>=%s)" % NUMPY_MIN_VERSION] +REQUIRES = ["numpy (>=%s)" % NUMPY_MIN_VERSION, + 'bz2file; python_version < "3.0"'] From 2452a0f107e57e4cda3f925b7765270fe7ca01ad Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 11 Dec 2018 19:10:56 -0500 Subject: [PATCH 04/24] RF: Drop SAFE_STRINGERS, depend on bz2file to provide .readinto() --- nibabel/volumeutils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nibabel/volumeutils.py b/nibabel/volumeutils.py index 1880aa63bf..44a4618f2e 100644 --- a/nibabel/volumeutils.py +++ b/nibabel/volumeutils.py @@ -42,9 +42,6 @@ #: file-like classes known to hold compressed data COMPRESSED_FILE_LIKES = (gzip.GzipFile, bz2.BZ2File) -#: file-like classes known to return string values that are safe to modify -SAFE_STRINGERS = (gzip.GzipFile, bz2.BZ2File) - class Recoder(object): ''' class to return canonical code(s) from code or aliases @@ -530,7 +527,7 @@ def array_from_file(shape, in_dtype, infile, offset=0, order='F', mmap=True): else: data_bytes = infile.read(n_bytes) n_read = len(data_bytes) - needs_copy = not isinstance(infile, SAFE_STRINGERS) + needs_copy = True if n_bytes != n_read: raise IOError('Expected {0} bytes, got {1} bytes from {2}\n' ' - could the file be damaged?'.format( From 81e775f52091b7191b24ff306e8d74c421b29f11 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 12 Dec 2018 09:14:19 -0500 Subject: [PATCH 05/24] CI: Upgrade setuptools --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b6e69d09ba..f6f07e513c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -96,7 +96,7 @@ before_install: - virtualenv --python=python venv - source venv/bin/activate - python --version # just to check - - pip install -U pip wheel # needed at one point + - pip install -U pip setuptools>=27.0 wheel - retry pip install nose flake8 mock # always - pip install $EXTRA_PIP_FLAGS $DEPENDS $OPTIONAL_DEPENDS - if [ "${COVERAGE}" == "1" ]; then From 4977b45aa7049ba37e9c449526cf58f24366f51c Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Mon, 17 Dec 2018 13:20:10 -0800 Subject: [PATCH 06/24] RF: Circumvents a deprecation warning from `np.fromstring` ``` /Users/arokem/.virtualenvs/afq/lib/python3.7/site-packages/nibabel/streamlines/trk.py:562: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead header_rec = np.fromstring(string=header_str, dtype=header_2_dtype) ``` --- nibabel/streamlines/trk.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nibabel/streamlines/trk.py b/nibabel/streamlines/trk.py index 36a605d53f..ba9df7c14f 100644 --- a/nibabel/streamlines/trk.py +++ b/nibabel/streamlines/trk.py @@ -559,8 +559,8 @@ def _read_header(fileobj): # Read the header in one block. header_str = f.read(header_2_dtype.itemsize) - header_rec = np.fromstring(string=header_str, dtype=header_2_dtype) - + header_rec = np.frombuffer(buffer=header_str, dtype=header_2_dtype) + header_rec.setflags(write=1) # Check endianness endianness = native_code if header_rec['hdr_size'] != TrkFile.HEADER_SIZE: From 53d941351d967423ae7b0cd5556744af0d192da3 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Mon, 17 Dec 2018 13:48:38 -0800 Subject: [PATCH 07/24] RF: fromstring => frombuffer in externals.netcdf. --- nibabel/externals/netcdf.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nibabel/externals/netcdf.py b/nibabel/externals/netcdf.py index 24b17706b8..e485533cd7 100644 --- a/nibabel/externals/netcdf.py +++ b/nibabel/externals/netcdf.py @@ -37,7 +37,7 @@ import numpy as np # noqa from ..py3k import asbytes, asstr -from numpy import fromstring, ndarray, dtype, empty, array, asarray +from numpy import frombuffer, ndarray, dtype, empty, array, asarray from numpy import little_endian as LITTLE_ENDIAN from functools import reduce @@ -519,7 +519,7 @@ def _read(self): if not magic == b'CDF': raise TypeError("Error: %s is not a valid NetCDF 3 file" % self.filename) - self.__dict__['version_byte'] = fromstring(self.fp.read(1), '>b')[0] + self.__dict__['version_byte'] = frombuffer(self.fp.read(1), '>b')[0] # Read file headers and set data. self._read_numrecs() @@ -608,7 +608,7 @@ def _read_var_array(self): # Calculate size to avoid problems with vsize (above) a_size = reduce(mul, shape, 1) * size if self.file_bytes >= 0 and begin_ + a_size > self.file_bytes: - data = fromstring(b'\x00'*a_size, dtype=dtype_) + data = frombuffer(b'\x00'*a_size, dtype=dtype_) elif self.use_mmap: mm = mmap(self.fp.fileno(), begin_+a_size, access=ACCESS_READ) data = ndarray.__new__(ndarray, shape, dtype=dtype_, @@ -622,7 +622,7 @@ def _read_var_array(self): buf = self.fp.read(a_size) if len(buf) < a_size: buf = b'\x00'*a_size - data = fromstring(buf, dtype=dtype_) + data = frombuffer(buf, dtype=dtype_) data.shape = shape self.fp.seek(pos) @@ -644,7 +644,7 @@ def _read_var_array(self): else: pos = self.fp.tell() self.fp.seek(begin) - rec_array = fromstring(self.fp.read(self._recs*self._recsize), dtype=dtypes) + rec_array = frombuffer(self.fp.read(self._recs*self._recsize), dtype=dtypes) rec_array.shape = (self._recs,) self.fp.seek(pos) @@ -687,7 +687,7 @@ def _read_values(self): self.fp.read(-count % 4) # read padding if typecode is not 'c': - values = fromstring(values, dtype='>%s' % typecode) + values = frombuffer(values, dtype='>%s' % typecode) if values.shape == (1,): values = values[0] else: @@ -705,14 +705,14 @@ def _pack_int(self, value): _pack_int32 = _pack_int def _unpack_int(self): - return int(fromstring(self.fp.read(4), '>i')[0]) + return int(frombuffer(self.fp.read(4), '>i')[0]) _unpack_int32 = _unpack_int def _pack_int64(self, value): self.fp.write(array(value, '>q').tostring()) def _unpack_int64(self): - return fromstring(self.fp.read(8), '>q')[0] + return frombuffer(self.fp.read(8), '>q')[0] def _pack_string(self, s): count = len(s) From 5d11c9e9b5b459b522fa04544e4a919ac1d6dd95 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Mon, 17 Dec 2018 13:50:37 -0800 Subject: [PATCH 08/24] RF: fromstring => frombuffer in gifti/cifti2 --- nibabel/cifti2/tests/test_cifti2.py | 4 ++-- nibabel/gifti/parse_gifti_fast.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nibabel/cifti2/tests/test_cifti2.py b/nibabel/cifti2/tests/test_cifti2.py index 9a2dcec728..ce71b92bcc 100644 --- a/nibabel/cifti2/tests/test_cifti2.py +++ b/nibabel/cifti2/tests/test_cifti2.py @@ -58,7 +58,7 @@ def test_cifti2_metadata(): assert_equal(md.data, dict(metadata_test)) assert_equal(list(iter(md)), list(iter(collections.OrderedDict(metadata_test)))) - + md.update({'a': 'aval', 'b': 'bval'}) assert_equal(md.data, dict(metadata_test)) @@ -310,7 +310,7 @@ def test_matrix(): assert_raises(ci.Cifti2HeaderError, m.insert, 0, mim_none) assert_equal(m.mapped_indices, []) - + h = ci.Cifti2Header(matrix=m) assert_equal(m.mapped_indices, []) m.insert(0, mim_0) diff --git a/nibabel/gifti/parse_gifti_fast.py b/nibabel/gifti/parse_gifti_fast.py index 4cdbd3d768..de02f4c76b 100644 --- a/nibabel/gifti/parse_gifti_fast.py +++ b/nibabel/gifti/parse_gifti_fast.py @@ -47,7 +47,7 @@ def read_data_block(encoding, endian, ordering, datatype, shape, data): dec = base64.b64decode(data.encode('ascii')) dt = data_type_codes.type[datatype] sh = tuple(shape) - newarr = np.fromstring(dec, dtype=dt) + newarr = np.frombuffer(dec, dtype=dt) if len(newarr.shape) != len(sh): newarr = newarr.reshape(sh, order=ord) @@ -59,7 +59,7 @@ def read_data_block(encoding, endian, ordering, datatype, shape, data): zdec = zlib.decompress(dec) dt = data_type_codes.type[datatype] sh = tuple(shape) - newarr = np.fromstring(zdec, dtype=dt) + newarr = np.frombuffer(zdec, dtype=dt) if len(newarr.shape) != len(sh): newarr = newarr.reshape(sh, order=ord) From b7e4850bb82cf24b186cb958ed008b1bfced9aa7 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Mon, 17 Dec 2018 13:58:36 -0800 Subject: [PATCH 09/24] RF: fromstring => frombuffer. --- nibabel/nifti1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nibabel/nifti1.py b/nibabel/nifti1.py index 60ff818e57..84cfed956a 100644 --- a/nibabel/nifti1.py +++ b/nibabel/nifti1.py @@ -579,7 +579,7 @@ def from_fileobj(klass, fileobj, size, byteswap): # otherwise there should be a full extension header if not len(ext_def) == 8: raise HeaderDataError('failed to read extension header') - ext_def = np.fromstring(ext_def, dtype=np.int32) + ext_def = np.frombuffer(ext_def, dtype=np.int32) if byteswap: ext_def = ext_def.byteswap() # be extra verbose From 8c0b21da4580fa9fdbb61aa4142ff27faff085d3 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Mon, 17 Dec 2018 14:10:53 -0800 Subject: [PATCH 10/24] BF: Use a writeable bytearray for the header data. This is because newer numpy doesn't allow to change the writeable flag. --- nibabel/streamlines/trk.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nibabel/streamlines/trk.py b/nibabel/streamlines/trk.py index ba9df7c14f..88858165c0 100644 --- a/nibabel/streamlines/trk.py +++ b/nibabel/streamlines/trk.py @@ -559,8 +559,8 @@ def _read_header(fileobj): # Read the header in one block. header_str = f.read(header_2_dtype.itemsize) - header_rec = np.frombuffer(buffer=header_str, dtype=header_2_dtype) - header_rec.setflags(write=1) + header_rec = np.frombuffer(buffer=bytearray(header_str), + dtype=header_2_dtype) # Check endianness endianness = native_code if header_rec['hdr_size'] != TrkFile.HEADER_SIZE: From a696d0d117691563b2df9387edf74c9a72f0b680 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Mon, 17 Dec 2018 15:37:43 -0800 Subject: [PATCH 11/24] NF: Implement a `readinto` method for Opener. --- nibabel/openers.py | 3 +++ nibabel/streamlines/trk.py | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/nibabel/openers.py b/nibabel/openers.py index f64ab23b37..4f5a340747 100644 --- a/nibabel/openers.py +++ b/nibabel/openers.py @@ -209,6 +209,9 @@ def fileno(self): def read(self, *args, **kwargs): return self.fobj.read(*args, **kwargs) + def readinto(self, *args, **kwargs): + return self.fobj.readinto(*args, **kwargs) + def write(self, *args, **kwargs): return self.fobj.write(*args, **kwargs) diff --git a/nibabel/streamlines/trk.py b/nibabel/streamlines/trk.py index 88858165c0..5e942b48d3 100644 --- a/nibabel/streamlines/trk.py +++ b/nibabel/streamlines/trk.py @@ -557,10 +557,10 @@ def _read_header(fileobj): with Opener(fileobj) as f: - # Read the header in one block. - header_str = f.read(header_2_dtype.itemsize) - header_rec = np.frombuffer(buffer=bytearray(header_str), - dtype=header_2_dtype) + # Read the header into a bytearray. + header_buf = bytearray(header_2_dtype.itemsize) + n_read = f.readinto(header_buf) + header_rec = np.frombuffer(buffer=header_buf, dtype=header_2_dtype) # Check endianness endianness = native_code if header_rec['hdr_size'] != TrkFile.HEADER_SIZE: From a97b067897889ecfd032b940e14af94fc41e611c Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 21 Dec 2018 11:32:50 -0500 Subject: [PATCH 12/24] MAINT: Move to setuptools exclusively --- nibabel/info.py | 2 +- nisext/sexts.py | 1 - setup.py | 11 ++--------- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/nibabel/info.py b/nibabel/info.py index 2788a0875a..56cdcb2c80 100644 --- a/nibabel/info.py +++ b/nibabel/info.py @@ -209,5 +209,5 @@ def cmp_pkg_version(version_str, pkg_version_str=__version__): ISRELEASE = _version_extra == '' VERSION = __version__ PROVIDES = ["nibabel", 'nisext'] -REQUIRES = ["numpy (>=%s)" % NUMPY_MIN_VERSION, +REQUIRES = ["numpy>=%s" % NUMPY_MIN_VERSION, 'bz2file; python_version < "3.0"'] diff --git a/nisext/sexts.py b/nisext/sexts.py index b0d34348e5..8fe5c3f338 100644 --- a/nisext/sexts.py +++ b/nisext/sexts.py @@ -191,7 +191,6 @@ def version_getter(pkg_name): optional, dependency) return - _add_append_key(setuptools_args, 'install_requires', dependency) return diff --git a/setup.py b/setup.py index 27f85d3e99..6607f867aa 100755 --- a/setup.py +++ b/setup.py @@ -19,14 +19,7 @@ if os.path.exists('MANIFEST'): os.remove('MANIFEST') -# For some commands, use setuptools. -if len(set(('develop', 'bdist_egg', 'bdist_rpm', 'bdist', 'bdist_dumb', - 'install_egg_info', 'egg_info', 'easy_install', 'bdist_wheel', - 'bdist_mpkg')).intersection(sys.argv)) > 0: - # setup_egg imports setuptools setup, thus monkeypatching distutils. - import setup_egg # noqa - -from distutils.core import setup +from setuptools import setup # Commit hash writing, and dependency checking from nisext.sexts import (get_comrec_build, package_check, install_scripts_bat, @@ -77,8 +70,8 @@ def main(**extra_args): author_email=INFO.AUTHOR_EMAIL, platforms=INFO.PLATFORMS, version=INFO.VERSION, - requires=INFO.REQUIRES, provides=INFO.PROVIDES, + install_requires=INFO.REQUIRES, packages = ['nibabel', 'nibabel.externals', 'nibabel.externals.tests', From c79a29112dc42815946b142e34d4beabf8804d98 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 21 Dec 2018 11:41:28 -0500 Subject: [PATCH 13/24] FIX: Use bz2file for Python 2 --- nibabel/openers.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nibabel/openers.py b/nibabel/openers.py index f64ab23b37..ececb1591e 100644 --- a/nibabel/openers.py +++ b/nibabel/openers.py @@ -9,7 +9,11 @@ """ Context manager openers for various fileobject types """ -import bz2 +import sys +if sys.version_info[0] < 3: + from bz2file import BZ2File +else: + from bz2 import BZ2File import gzip import sys import warnings @@ -127,7 +131,7 @@ class Opener(object): for \*args """ gz_def = (_gzip_open, ('mode', 'compresslevel', 'keep_open')) - bz2_def = (bz2.BZ2File, ('mode', 'buffering', 'compresslevel')) + bz2_def = (BZ2File, ('mode', 'buffering', 'compresslevel')) compress_ext_map = { '.gz': gz_def, '.bz2': bz2_def, From 62b8cc9456d391a32814c9bd11fe22bb43f6b8e9 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 21 Dec 2018 11:47:04 -0500 Subject: [PATCH 14/24] FIX: Rely on openers.BZ2File to provide correct class --- nibabel/tests/test_openers.py | 3 +-- nibabel/tests/test_volumeutils.py | 6 +++--- nibabel/volumeutils.py | 5 ++--- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/nibabel/tests/test_openers.py b/nibabel/tests/test_openers.py index 6b5f231fc3..6aeb66aaf7 100644 --- a/nibabel/tests/test_openers.py +++ b/nibabel/tests/test_openers.py @@ -10,12 +10,11 @@ import os import contextlib from gzip import GzipFile -from bz2 import BZ2File from io import BytesIO, UnsupportedOperation from distutils.version import StrictVersion from ..py3k import asstr, asbytes -from ..openers import Opener, ImageOpener, HAVE_INDEXED_GZIP +from ..openers import Opener, ImageOpener, HAVE_INDEXED_GZIP, BZ2File from ..tmpdirs import InTemporaryDirectory from ..volumeutils import BinOpener diff --git a/nibabel/tests/test_volumeutils.py b/nibabel/tests/test_volumeutils.py index aa145af2e9..758ef07758 100644 --- a/nibabel/tests/test_volumeutils.py +++ b/nibabel/tests/test_volumeutils.py @@ -46,7 +46,7 @@ _dt_min_max, _write_data, ) -from ..openers import Opener +from ..openers import Opener, BZ2File from ..casting import (floor_log2, type_info, OK_FLOATS, shared_range) from numpy.testing import (assert_array_almost_equal, @@ -71,7 +71,7 @@ def test__is_compressed_fobj(): with InTemporaryDirectory(): for ext, opener, compressed in (('', open, False), ('.gz', gzip.open, True), - ('.bz2', bz2.BZ2File, True)): + ('.bz2', BZ2File, True)): fname = 'test.bin' + ext for mode in ('wb', 'rb'): fobj = opener(fname, mode) @@ -94,7 +94,7 @@ def make_array(n, bytes): with InTemporaryDirectory(): for n, opener in itertools.product( (256, 1024, 2560, 25600), - (open, gzip.open, bz2.BZ2File)): + (open, gzip.open, BZ2File)): in_arr = np.arange(n, dtype=dtype) # Write array to file fobj_w = opener(fname, 'wb') diff --git a/nibabel/volumeutils.py b/nibabel/volumeutils.py index 44a4618f2e..2b8349d369 100644 --- a/nibabel/volumeutils.py +++ b/nibabel/volumeutils.py @@ -12,7 +12,6 @@ import sys import warnings import gzip -import bz2 from collections import OrderedDict from os.path import exists, splitext from operator import mul @@ -21,7 +20,7 @@ import numpy as np from .casting import (shared_range, type_info, OK_FLOATS) -from .openers import Opener +from .openers import Opener, BZ2File from .deprecated import deprecate_with_version from .externals.oset import OrderedSet @@ -40,7 +39,7 @@ default_compresslevel = 1 #: file-like classes known to hold compressed data -COMPRESSED_FILE_LIKES = (gzip.GzipFile, bz2.BZ2File) +COMPRESSED_FILE_LIKES = (gzip.GzipFile, BZ2File) class Recoder(object): From 28412d9348f5682b47650544176a2d21161ca02e Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 21 Dec 2018 12:22:36 -0500 Subject: [PATCH 15/24] CI: Upgrade pip, setuptools, wheel on AppVeyor --- appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor.yml b/appveyor.yml index 772bfa142d..d0a8be788b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -22,6 +22,7 @@ install: - SET PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% # Install the dependencies of the project. + - pip install --upgrade pip setuptools wheel - pip install numpy scipy matplotlib nose h5py mock pydicom - pip install . - SET NIBABEL_DATA_DIR=%CD%\nibabel-data From d7a619fc7d9f578ea8483308123aadd191357f34 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 21 Dec 2018 12:40:03 -0500 Subject: [PATCH 16/24] TEST: fobj string assumptions need only apply to bytearrays --- nibabel/tests/test_volumeutils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nibabel/tests/test_volumeutils.py b/nibabel/tests/test_volumeutils.py index 758ef07758..29c0edaf07 100644 --- a/nibabel/tests/test_volumeutils.py +++ b/nibabel/tests/test_volumeutils.py @@ -103,7 +103,8 @@ def make_array(n, bytes): # Read back from file fobj_r = opener(fname, 'rb') try: - contents1 = fobj_r.read() + contents1 = bytearray(4 * n) + fobj_r.readinto(contents1) # Second element is 1 assert_false(contents1[0:8] == b'\x00' * 8) out_arr = make_array(n, contents1) @@ -114,7 +115,8 @@ def make_array(n, bytes): assert_equal(contents1[:8], b'\x00' * 8) # Reread, to get unmodified contents fobj_r.seek(0) - contents2 = fobj_r.read() + contents2 = bytearray(4 * n) + fobj_r.readinto(contents2) out_arr2 = make_array(n, contents2) assert_array_equal(in_arr, out_arr2) assert_equal(out_arr[1], 0) From f5eafcf8d5645ceed9dbf194620dfb6eac58f426 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 21 Dec 2018 14:14:39 -0500 Subject: [PATCH 17/24] FIX: Load TckFile streamlines using bytearray --- nibabel/streamlines/tck.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/nibabel/streamlines/tck.py b/nibabel/streamlines/tck.py index 1ba2d76625..48f7d4f8aa 100644 --- a/nibabel/streamlines/tck.py +++ b/nibabel/streamlines/tck.py @@ -405,18 +405,19 @@ def _read(cls, fileobj, header, buffer_size=4): n_streams = 0 while not eof: + buff = bytearray(buffer_size) + n_read = f.readinto(buff) + eof = n_read != buffer_size - bytes_read = f.read(buffer_size) - buffs.append(bytes_read) - eof = len(bytes_read) != buffer_size + buffs.append(buff) # Make sure we've read enough to find a streamline delimiter. - if fiber_marker not in bytes_read: + if fiber_marker not in buff: # If we've read the whole file, then fail. if eof: # Could have minimal buffering, and have read only the # EOF delimiter - buffs = [b''.join(buffs)] + buffs = [bytearray().join(buffs)] if not buffs[0] == eof_marker: raise DataError( "Cannot find a streamline delimiter. This file" @@ -425,15 +426,13 @@ def _read(cls, fileobj, header, buffer_size=4): # Otherwise read a bit more. continue - all_parts = b''.join(buffs).split(fiber_marker) + all_parts = bytearray().join(buffs).split(fiber_marker) point_parts, buffs = all_parts[:-1], all_parts[-1:] point_parts = [p for p in point_parts if p != b''] for point_part in point_parts: # Read floats. pts = np.frombuffer(point_part, dtype=dtype) - # Enforce ability to write to underlying bytes object - pts.flags.writeable = True # Convert data to little-endian if needed. yield pts.astype(' Date: Fri, 21 Dec 2018 14:19:49 -0500 Subject: [PATCH 18/24] TEST: Return mutable bytearray from trk_with_bytes --- nibabel/streamlines/tests/test_trk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nibabel/streamlines/tests/test_trk.py b/nibabel/streamlines/tests/test_trk.py index c4c10bbcbd..a0a3d8a1f3 100644 --- a/nibabel/streamlines/tests/test_trk.py +++ b/nibabel/streamlines/tests/test_trk.py @@ -103,7 +103,7 @@ def test_load_complex_file(self): def trk_with_bytes(self, trk_key='simple_trk_fname', endian='<'): """ Return example trk file bytes and struct view onto bytes """ with open(DATA[trk_key], 'rb') as fobj: - trk_bytes = fobj.read() + trk_bytes = bytearray(fobj.read()) dt = trk_module.header_2_dtype.newbyteorder(endian) trk_struct = np.ndarray((1,), dt, buffer=trk_bytes) trk_struct.flags.writeable = True From ed42db395e433f97c67cd1a2040cdd3a1187f108 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 21 Dec 2018 14:36:46 -0500 Subject: [PATCH 19/24] FIX: Truncate buffer on EOF --- nibabel/streamlines/tck.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nibabel/streamlines/tck.py b/nibabel/streamlines/tck.py index 48f7d4f8aa..31f0be0ab5 100644 --- a/nibabel/streamlines/tck.py +++ b/nibabel/streamlines/tck.py @@ -408,6 +408,8 @@ def _read(cls, fileobj, header, buffer_size=4): buff = bytearray(buffer_size) n_read = f.readinto(buff) eof = n_read != buffer_size + if eof: + buff = buff[:n_read] buffs.append(buff) From 24cba1becc50dd46c4ccd5a0e10e73cebd705176 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 21 Dec 2018 14:37:27 -0500 Subject: [PATCH 20/24] CI: Try alternative pip install --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index d0a8be788b..93438cfc0f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -22,7 +22,7 @@ install: - SET PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% # Install the dependencies of the project. - - pip install --upgrade pip setuptools wheel + - python -m pip install --upgrade pip setuptools wheel - pip install numpy scipy matplotlib nose h5py mock pydicom - pip install . - SET NIBABEL_DATA_DIR=%CD%\nibabel-data From 03086a425b07bcb94c516cdc9ff7126d1761ed06 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 21 Dec 2018 14:50:17 -0500 Subject: [PATCH 21/24] FIX: Use io.BytesIO to test TckFile --- nibabel/streamlines/tests/test_tck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nibabel/streamlines/tests/test_tck.py b/nibabel/streamlines/tests/test_tck.py index d6d0edef6c..f5752bbe8b 100644 --- a/nibabel/streamlines/tests/test_tck.py +++ b/nibabel/streamlines/tests/test_tck.py @@ -3,7 +3,7 @@ import numpy as np from os.path import join as pjoin -from six import BytesIO +from io import BytesIO from nibabel.py3k import asbytes from ..array_sequence import ArraySequence From d4f447786cd1ad4ebd672e52b785e2cd5e5e76dd Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 21 Dec 2018 15:03:59 -0500 Subject: [PATCH 22/24] STY: Drop unused variable --- nibabel/streamlines/trk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nibabel/streamlines/trk.py b/nibabel/streamlines/trk.py index 5e942b48d3..eda7c8abeb 100644 --- a/nibabel/streamlines/trk.py +++ b/nibabel/streamlines/trk.py @@ -559,7 +559,7 @@ def _read_header(fileobj): # Read the header into a bytearray. header_buf = bytearray(header_2_dtype.itemsize) - n_read = f.readinto(header_buf) + f.readinto(header_buf) header_rec = np.frombuffer(buffer=header_buf, dtype=header_2_dtype) # Check endianness endianness = native_code From 7cf14a3f0867e8695118d936a8c2e9fe96beaf94 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Sat, 22 Dec 2018 09:13:47 -0500 Subject: [PATCH 23/24] RF: Leave nisext alone, pop install_requires --- nisext/sexts.py | 1 + setup.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/nisext/sexts.py b/nisext/sexts.py index 8fe5c3f338..b0d34348e5 100644 --- a/nisext/sexts.py +++ b/nisext/sexts.py @@ -191,6 +191,7 @@ def version_getter(pkg_name): optional, dependency) return + _add_append_key(setuptools_args, 'install_requires', dependency) return diff --git a/setup.py b/setup.py index 6607f867aa..222ad8562a 100755 --- a/setup.py +++ b/setup.py @@ -120,4 +120,6 @@ def main(**extra_args): if __name__ == "__main__": + # Do not use nisext's dynamically updated install_requires + extra_setuptools_args.pop('install_requires', None) main(**extra_setuptools_args) From ea1b0cc1a6bb6ff005fbea82045121c2b3f22937 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Sat, 22 Dec 2018 09:18:52 -0500 Subject: [PATCH 24/24] DOC: Add a comment explaining readinto/frombuffer idiom --- nibabel/streamlines/trk.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nibabel/streamlines/trk.py b/nibabel/streamlines/trk.py index eda7c8abeb..629673e943 100644 --- a/nibabel/streamlines/trk.py +++ b/nibabel/streamlines/trk.py @@ -556,8 +556,8 @@ def _read_header(fileobj): start_position = fileobj.tell() if hasattr(fileobj, 'tell') else None with Opener(fileobj) as f: - - # Read the header into a bytearray. + # Reading directly from a file into a (mutable) bytearray enables a zero-copy + # cast to a mutable numpy object with frombuffer header_buf = bytearray(header_2_dtype.itemsize) f.readinto(header_buf) header_rec = np.frombuffer(buffer=header_buf, dtype=header_2_dtype)