svmloader.pyx

cimport cython
from libc.stdlib cimport strtol, strtod
from libc.limits cimport INT_MAX
import array
from cpython cimport array
import numpy as np
cimport numpy as cnp
import scipy.sparse as sp


# SciPy uses int32 indices by default and switches to int64 for large
# matrices, so save_svmfile must accept both.
ctypedef fused csr_index_t:
    cnp.int32_t
    cnp.int64_t

_CSR_CLASSES = (sp.csr_matrix,
                sp.csr_array) if hasattr(sp, 'csr_array') else (sp.csr_matrix,)


@cython.boundscheck(False) # turn off bounds-checking for entire function
@cython.wraparound(False)  # turn off negative index wrapping for entire function
#@cython.initializedcheck(False)
cdef _load_svmfile(fp, dtype, ltype, bint zero_based, bint multilabels):
    cdef char * s
    cdef char * end
    cdef Py_ssize_t idx
    cdef Py_ssize_t last_idx
    cdef double value

    cdef char dt = 'f' if dtype == 'f' else 'd'  # defaut is double
    cdef char lt = 'd' if ltype == 'd' else 'i'  # default is int
    cdef array.array data = array.array(dtype)
    cdef array.array indices = array.array('i')
    cdef array.array indptr = array.array('i', [0])

    if not multilabels:
        labels = array.array(ltype)
    else:
        labels = []
        cls = float if lt == 'd' else int

    cdef Py_ssize_t sz = 0
    cdef Py_ssize_t nrows = 0

    for line in fp:
        s = line

        if s[0] == '#':
            continue

        nrows += 1

        # get the label
        if not multilabels:
            array.resize_smart(labels, nrows)
            if lt == 'i':
                labels[nrows-1] = strtol(s, &end, 10)
            else:
                labels[nrows-1] = strtod(s, &end)
            if s==end:
                raise ValueError('invalid label')
            s = end
        else:
            labl, line = line.split(None, 1)
            s = line
            label = sorted([cls(val) for val in labl.split(b',')])
            labels.append(tuple(label))

        # process the line
        last_idx = -1
        # A '\r' ends the line only when followed by '\n' (CRLF) or EOF.
        # A stray '\r' is left to strtol(), which raises "invalid index".
        while (s[0] != '#' and s[0] != '\n' and s[0] != 0
               and not (s[0] == '\r' and (s[1] == '\n' or s[1] == 0))):
            # get index
            idx = strtol(s, &end, 10)
            if s == end:
                raise ValueError('invalid index')
            if idx < 0 or idx > INT_MAX:
                raise ValueError('invalid index (out of range)')
            s = end

            if not zero_based:
                if idx == 0:
                    raise ValueError('invalid index 0 with one-based indexes')
                idx -= 1

            if idx <= last_idx:
                raise ValueError('indices should be sorted and uniques')
            last_idx = idx

            # ensure we have correct separator
            while s[0] == ' ' or s[0] == '\t':
                s += 1
            if s[0] != ':':
                raise ValueError('invalid separator')
            s += 1

            # get value
            value = strtod(s, &end)
            if s == end:
                raise ValueError('invalid value')
            s = end
            while s[0] == ' ' or s[0] == '\t':
                s += 1

            sz += 1
            if sz > INT_MAX:
                # indptr holds C ints, which would silently wrap around
                raise ValueError('too many non-zero values (more than %d)' % INT_MAX)
            array.resize_smart(indices, sz)
            array.resize_smart(data, sz)
            indices.data.as_ints[sz-1] = idx
            if dt == 'd':
                data.data.as_doubles[sz-1] = value
            else:
                data.data.as_floats[sz-1] = value

        array.resize_smart(indptr, nrows+1)
        indptr.data.as_ints[nrows] = sz

    if not multilabels:
        labels = np.frombuffer(labels, dtype=ltype)

    return (np.frombuffer(data, dtype=dtype),
            np.frombuffer(indices, dtype='i'),
            np.frombuffer(indptr, dtype='i'),
            labels)



def _openfile(filename):
    import os.path
    _, ext = os.path.splitext(filename)
    if ext == ".gz":
        import gzip
        fp = gzip.open(filename, "rb")
    elif ext == ".bz2":
        from bz2 import BZ2File
        fp = BZ2File(filename, "rb")
    else:
        fp = open(filename, "rb")
    return fp


def _nfeatures(indices):
    """Number of columns implied by the indices: 0 for a file without values."""
    return int(indices.max()) + 1 if len(indices) else 0


def load_svmfile(filename, dtype='d', ltype='i', nfeatures=None, zero_based=True, multilabels=False):
    """\
    Load a sparse CSR matrix from filename at svmlib format.

    Files in .gz or .bz2 format will be uncompressed on the fly.

    :param filename: the file name
    :type filename: str
    :param dtype: type of data, must be either 'd' (double) or 'f' (float)
    :type dtype: str
    :param ltype: type of labels, must be either 'i' (int) or 'd' (double)
    :type ltype: str
    :param nfeatures: the number of columns (infered from file if is None)
    :type nfeatures: int
    :param zero_based: indicates if columns indexes are zero-based or one-based
    :type zero_based: bool
    :param multilabels: indicates if file uses multiple labels per row
    :type multilabels: bool
    :returns: (X, y) tuple, where X is the sparse matrix and y its labels
    :rtype: (:class:`scipy.sparse.csr_matrix`, :class:`numpy.ndarray`)
    """
    assert(dtype=='f' or dtype=='d'), 'dtype must be "d" or "f"'
    assert(ltype=='i' or ltype=='d'), 'ltype must be "i" or "d"'

    with _openfile(filename) as fp:
        data, indices, indptr, y = _load_svmfile(fp, dtype, ltype, zero_based, multilabels)

    if nfeatures is None:
        nfeatures = _nfeatures(indices)
    X = sp.csr_matrix((data, indices, indptr), shape=(len(indptr)-1, nfeatures), copy=False)

    return X, y

def load_svmfiles(filenames, dtype='d', ltype='i', zero_based=True, multilabels=False):
    """\
    Load a sparse CSR matrix list from list of filenames at svmlib format.

    Files in .gz or .bz2 format will be uncompressed on the fly.

    The number of features will be infered from the maximum indice found
    on all files.

    :param filenames: the list of files names
    :type filenames: list
    :param dtype: type of data, must be either 'd' (double) or 'f' (float)
    :type dtype: str
    :param ltype: type of labels, must be either 'i' (int) or 'd' (double)
    :type ltype: str
    :param zero_based: indicates if columns indexes are zero-based or one-based
    :type zero_based: bool
    :param multilabels: indicates if file uses multiple labels per row
    :type multilabels: bool
    :returns: a list [matrix_0, labels_0, .., matrix_n, labels_n]
    """
    assert(dtype=='f' or dtype=='d'), 'dtype must be "d" or "f"'
    assert(ltype=='i' or ltype=='d'), 'ltype must be "i" or "d"'

    Xlst = []
    ylst = []
    for filename in filenames:
        with _openfile(filename) as fp:
            try:
                data, indices, indptr, y = _load_svmfile(fp, dtype, ltype, zero_based, multilabels)
            except ValueError as err:
                err.args = 'in %s, %s' % (filename, err.args[0]),
                raise
            Xlst.append((data, indices, indptr))
            ylst.append(y)
    # get nfeatures as the maximum indice
    nfeatures = max([_nfeatures(indices) for _, indices, _ in Xlst], default=0)
    lst = []
    for (data, indices, indptr), y in zip(Xlst, ylst):
        X = sp.csr_matrix((data, indices, indptr), shape=(len(indptr)-1, nfeatures), copy=False)    
        lst.append(X)
        lst.append(y)
    return lst

@cython.boundscheck(False) # turn off bounds-checking for entire function
@cython.wraparound(False)  # turn off negative index wrapping for entire function
cdef _save_svmfile(filename, mat, labels, bint zero_based,
                   const csr_index_t[:] indices, const csr_index_t[:] indptr):
    cdef cnp.ndarray data = mat.data
    cdef Py_ssize_t i
    cdef Py_ssize_t j
    cdef Py_ssize_t offset = 0 if zero_based else 1
    # '\n' line endings on every platform, as libsvm files have
    with open(filename, 'w', newline='\n') as fp:
        for i in range(indptr.shape[0] - 1):
            if labels is not None:
                fp.write('%s ' % labels[i])
            row = ' '.join(['%d:%g' % (indices[j] + offset, data[j])
                            for j in range(indptr[i], indptr[i+1])])
            fp.write(row)
            fp.write('\n')


def save_svmfile(filename, mat, labels=None, zero_based=False):
    """\
    Save a sparse CSR matrix to filename at svmlib format (convenient function).

    :param filename: the output filesname
    :type filename: str
    :param mat: the sparse csr matrix
    :type mat: :class:`scipy.sparse.csr_matrix` or :class:`scipy.sparse.csr_array`
    :param labels: the rows labels
    :type labels: indexable
    :param zero_based: indicates if columns indexes are zero-based or one-based
    :type zero_based: bool
    """
    assert isinstance(mat, _CSR_CLASSES), 'not a CSR matrix'
    if mat.indices.dtype != mat.indptr.dtype:
        raise TypeError('mat.indices and mat.indptr must have the same dtype, '
                        'got %s and %s' % (mat.indices.dtype, mat.indptr.dtype))
    cdef const cnp.int32_t[:] indices32
    cdef const cnp.int32_t[:] indptr32
    cdef const cnp.int64_t[:] indices64
    cdef const cnp.int64_t[:] indptr64
    if mat.indices.dtype == np.int32:
        indices32 = mat.indices
        indptr32 = mat.indptr
        _save_svmfile(filename, mat, labels, <bint>zero_based,
                      indices32, indptr32)
    elif mat.indices.dtype == np.int64:
        indices64 = mat.indices
        indptr64 = mat.indptr
        _save_svmfile(filename, mat, labels, <bint>zero_based,
                      indices64, indptr64)
    else:
        raise TypeError('unsupported index dtype: %s' % mat.indices.dtype)