"""A FactorInfo object is a simple class that provides some metadata about

def __init__(self, factor, type, state,

num_columns=None, categories=None):

self.factor = factor

self.type = type

if self.type not in ["numerical", "categorical"]:

raise ValueError("FactorInfo.type must be "

"'numerical' or 'categorical', not %r"

% (self.type,))

self.state = state

if self.type == "numerical":

if not isinstance(num_columns, six.integer_types):

raise ValueError("For numerical factors, num_columns "

"must be an integer")

if categories is not None:

raise ValueError("For numerical factors, categories "

"must be None")

else:

assert self.type == "categorical"

if num_columns is not None:

raise ValueError("For categorical factors, num_columns "

"must be None")

categories = tuple(categories)

self.num_columns = num_columns

self.categories = categories

__repr__ = repr_pretty_delegate

def _repr_pretty_(self, p, cycle):

assert not cycle

class FactorState(object):

def __repr__(self):

return "<factor state>"

kwlist = [("factor", self.factor),

("type", self.type),

# Don't put the state in people's faces, it will

# just encourage them to pay attention to the

# contents :-). Plus it's a bunch of gobbledygook

# they don't care about. They can always look at

# self.state if they want to know...

("state", FactorState()),

]

if self.type == "numerical":

kwlist.append(("num_columns", self.num_columns))

else:

kwlist.append(("categories", self.categories))

repr_pretty_impl(p, self, [], kwlist)

fi1 = FactorInfo("asdf", "numerical", {"a": 1}, num_columns=10)

assert fi1.factor == "asdf"

assert fi1.state == {"a": 1}

assert fi1.type == "numerical"

assert fi1.num_columns == 10

assert fi1.categories is None

# smoke test

repr(fi1)

fi2 = FactorInfo("asdf", "categorical", {"a": 2}, categories=["z", "j"])

assert fi2.factor == "asdf"

assert fi2.state == {"a": 2}

assert fi2.type == "categorical"

assert fi2.num_columns is None

assert fi2.categories == ("z", "j")

# smoke test

repr(fi2)

from nose.tools import assert_raises

assert_raises(ValueError, FactorInfo, "asdf", "non-numerical", {})

assert_raises(ValueError, FactorInfo, "asdf", "numerical", {})

assert_raises(ValueError, FactorInfo, "asdf", "numerical", {},

num_columns="asdf")

assert_raises(ValueError, FactorInfo, "asdf", "numerical", {},

num_columns=1, categories=1)

assert_raises(TypeError, FactorInfo, "asdf", "categorical", {})

assert_raises(ValueError, FactorInfo, "asdf", "categorical", {},

num_columns=1)

assert_raises(TypeError, FactorInfo, "asdf", "categorical", {},

categories=1)

# Make sure longs are legal for num_columns

# (Important on python2+win64, where array shapes are tuples-of-longs)

if not six.PY3:

fi_long = FactorInfo("asdf", "numerical", {"a": 1},

num_columns=long(10))

"""A SubtermInfo object is a simple metadata container describing a single

def __init__(self, factors, contrast_matrices, num_columns):

self.factors = tuple(factors)

factor_set = frozenset(factors)

if not isinstance(contrast_matrices, dict):

raise ValueError("contrast_matrices must be dict")

for factor, contrast_matrix in six.iteritems(contrast_matrices):

if factor not in factor_set:

raise ValueError("Unexpected factor in contrast_matrices dict")

if not isinstance(contrast_matrix, ContrastMatrix):

raise ValueError("Expected a ContrastMatrix, not %r"

% (contrast_matrix,))

self.contrast_matrices = contrast_matrices

if not isinstance(num_columns, six.integer_types):

raise ValueError("num_columns must be an integer")

self.num_columns = num_columns

__repr__ = repr_pretty_delegate

def _repr_pretty_(self, p, cycle):

assert not cycle

repr_pretty_impl(p, self, [],

[("factors", self.factors),

("contrast_matrices", self.contrast_matrices),

("num_columns", self.num_columns)])

cm = ContrastMatrix(np.ones((2, 2)), ["[1]", "[2]"])

s = SubtermInfo(["a", "x"], {"a": cm}, 4)

assert s.factors == ("a", "x")

assert s.contrast_matrices == {"a": cm}

assert s.num_columns == 4

# Make sure longs are accepted for num_columns

if not six.PY3:

s = SubtermInfo(["a", "x"], {"a": cm}, long(4))

assert s.num_columns == 4

# smoke test

repr(s)

from nose.tools import assert_raises

assert_raises(TypeError, SubtermInfo, 1, {}, 1)

assert_raises(ValueError, SubtermInfo, ["a", "x"], 1, 1)

assert_raises(ValueError, SubtermInfo, ["a", "x"], {"z": cm}, 1)

assert_raises(ValueError, SubtermInfo, ["a", "x"], {"a": 1}, 1)

"""A DesignInfo object holds metadata about a design matrix.

def __init__(self, column_names,

factor_infos=None, term_codings=None):

self.column_name_indexes = OrderedDict(zip(column_names,

range(len(column_names))))

if (factor_infos is None) != (term_codings is None):

raise ValueError("Must specify either both or neither of "

"factor_infos= and term_codings=")

self.factor_infos = factor_infos

self.term_codings = term_codings

# factor_infos is a dict containing one entry for every factor

# mentioned in our terms

# and mapping each to FactorInfo object

if self.factor_infos is not None:

if not isinstance(self.factor_infos, dict):

raise ValueError("factor_infos should be a dict")

if not isinstance(self.term_codings, OrderedDict):

raise ValueError("term_codings must be an OrderedDict")

for term, subterms in six.iteritems(self.term_codings):

if not isinstance(term, Term):

raise ValueError("expected a Term, not %r" % (term,))

if not isinstance(subterms, list):

raise ValueError("term_codings must contain lists")

term_factors = set(term.factors)

for subterm in subterms:

if not isinstance(subterm, SubtermInfo):

raise ValueError("expected SubtermInfo, "

"not %r" % (subterm,))

if not term_factors.issuperset(subterm.factors):

raise ValueError("unexpected factors in subterm")

all_factors = set()

for term in self.term_codings:

all_factors.update(term.factors)

if all_factors != set(self.factor_infos):

raise ValueError("Provided Term objects and factor_infos "

"do not match")

for factor, factor_info in six.iteritems(self.factor_infos):

if not isinstance(factor_info, FactorInfo):

raise ValueError("expected FactorInfo object, not %r"

% (factor_info,))

if factor != factor_info.factor:

raise ValueError("mismatched factor_info.factor")

for term, subterms in six.iteritems(self.term_codings):

for subterm in subterms:

exp_cols = 1

cat_factors = set()

for factor in subterm.factors:

fi = self.factor_infos[factor]

if fi.type == "numerical":

exp_cols *= fi.num_columns

else:

assert fi.type == "categorical"

cm = subterm.contrast_matrices[factor].matrix

if cm.shape[0] != len(fi.categories):

raise ValueError("Mismatched contrast matrix "

"for factor %r" % (factor,))

cat_factors.add(factor)

exp_cols *= cm.shape[1]

if cat_factors != set(subterm.contrast_matrices):

raise ValueError("Mismatch between contrast_matrices "

"and categorical factors")

if exp_cols != subterm.num_columns:

raise ValueError("Unexpected num_columns")

if term_codings is None:

# Need to invent term information

self.term_slices = None

# We invent one term per column, with the same name as the column

term_names = column_names

slices = [slice(i, i + 1) for i in range(len(column_names))]

self.term_name_slices = OrderedDict(zip(term_names, slices))

else:

# Need to derive term information from term_codings

self.term_slices = OrderedDict()

idx = 0

for term, subterm_infos in six.iteritems(self.term_codings):

term_columns = 0

for subterm_info in subterm_infos:

term_columns += subterm_info.num_columns

self.term_slices[term] = slice(idx, idx + term_columns)

idx += term_columns

if idx != len(self.column_names):

raise ValueError("mismatch between column_names and columns "

"coded by given terms")

self.term_name_slices = OrderedDict(

[(term.name(), slice_)

for (term, slice_) in six.iteritems(self.term_slices)])

# Guarantees:

# term_name_slices is never None

# The slices in term_name_slices are in order and exactly cover the

# whole range of columns.

# term_slices may be None

# If term_slices is not None, then its slices match the ones in

# term_name_slices.

assert self.term_name_slices is not None

if self.term_slices is not None:

assert (list(self.term_slices.values())

== list(self.term_name_slices.values()))

# These checks probably aren't necessary anymore now that we always

# generate the slices ourselves, but we'll leave them in just to be

# safe.

covered = 0

for slice_ in six.itervalues(self.term_name_slices):

start, stop, step = slice_.indices(len(column_names))

assert start == covered

assert step == 1

covered = stop

assert covered == len(column_names)

# If there is any name overlap between terms and columns, they refer

# to the same columns.

for column_name, index in six.iteritems(self.column_name_indexes):

if column_name in self.term_name_slices:

slice_ = self.term_name_slices[column_name]

if slice_ != slice(index, index + 1):

raise ValueError("term/column name collision")

__repr__ = repr_pretty_delegate

def _repr_pretty_(self, p, cycle):

assert not cycle

repr_pretty_impl(p, self,

[self.column_names],

[("factor_infos", self.factor_infos),

("term_codings", self.term_codings)])

@property

def column_names(self):

"A list of the column names, in order."

return list(self.column_name_indexes)

@property

def terms(self):

"A list of :class:`Terms`, in order, or else None."

if self.term_slices is None:

return None

return list(self.term_slices)

@property

def term_names(self):

"A list of terms, in order."

return list(self.term_name_slices)

@property

def builder(self):

".. deprecated:: 0.4.0"

warnings.warn(DeprecationWarning(

"The DesignInfo.builder attribute is deprecated starting in "

"patsy v0.4.0; distinct builder objects have been eliminated "

"and design_info.builder is now just a long-winded way of "

"writing 'design_info' (i.e. the .builder attribute just "

"returns self)"), stacklevel=2)

return self

@property

def design_info(self):

".. deprecated:: 0.4.0"

warnings.warn(DeprecationWarning(

"Starting in patsy v0.4.0, the DesignMatrixBuilder class has "

"been merged into the DesignInfo class. So there's no need to "

"use builder.design_info to access the DesignInfo; 'builder' "

"already *is* a DesignInfo."), stacklevel=2)

return self

def slice(self, columns_specifier):

"""Locate a subset of design matrix columns, specified symbolically.

if isinstance(columns_specifier, slice):

return columns_specifier

if np.issubsctype(type(columns_specifier), np.integer):

return slice(columns_specifier, columns_specifier + 1)

if (self.term_slices is not None

and columns_specifier in self.term_slices):

return self.term_slices[columns_specifier]

if columns_specifier in self.term_name_slices:

return self.term_name_slices[columns_specifier]

if columns_specifier in self.column_name_indexes:

idx = self.column_name_indexes[columns_specifier]

return slice(idx, idx + 1)

raise PatsyError("unknown column specified '%s'"

% (columns_specifier,))

def linear_constraint(self, constraint_likes):

"""Construct a linear constraint in matrix form from a (possibly

return linear_constraint(constraint_likes, self.column_names)

def describe(self):

"""Returns a human-readable string describing this design info.

names = []

for name in self.term_names:

if name == "Intercept":

names.append("1")

else:

names.append(name)

return " + ".join(names)

def subset(self, which_terms):

"""Create a new :class:`DesignInfo` for design matrices that contain a

if isinstance(which_terms, str):

desc = ModelDesc.from_formula(which_terms)

if desc.lhs_termlist:

raise PatsyError("right-hand-side-only formula required")

which_terms = [term.name() for term in desc.rhs_termlist]

if self.term_codings is None:

# This is a minimal DesignInfo

# If the name is unknown we just let the KeyError escape

new_names = []

for t in which_terms:

new_names += self.column_names[self.term_name_slices[t]]

return DesignInfo(new_names)

else:

term_name_to_term = {}

for term in self.term_codings:

term_name_to_term[term.name()] = term

new_column_names = []

new_factor_infos = {}

new_term_codings = OrderedDict()

for name_or_term in which_terms:

term = term_name_to_term.get(name_or_term, name_or_term)

# If the name is unknown we just let the KeyError escape

s = self.term_slices[term]

new_column_names += self.column_names[s]

for f in term.factors:

new_factor_infos[f] = self.factor_infos[f]

new_term_codings[term] = self.term_codings[term]

return DesignInfo(new_column_names,

factor_infos=new_factor_infos,

term_codings=new_term_codings)

@classmethod

def from_array(cls, array_like, default_column_prefix="column"):

"""Find or construct a DesignInfo appropriate for a given array_like.

if hasattr(array_like, "design_info") and isinstance(array_like.design_info, cls):

return array_like.design_info

arr = atleast_2d_column_default(array_like, preserve_pandas=True)

if arr.ndim > 2:

raise ValueError("design matrix can't have >2 dimensions")

columns = getattr(arr, "columns", range(arr.shape[1]))

if (hasattr(columns, "dtype")

and not safe_issubdtype(columns.dtype, np.integer)):

column_names = [str(obj) for obj in columns]

else:

column_names = ["%s%s" % (default_column_prefix, i)

for i in columns]

return DesignInfo(column_names)

from nose.tools import assert_raises

class _MockFactor(object):

def __init__(self, name):

self._name = name

def name(self):

return self._name

f_x = _MockFactor("x")

f_y = _MockFactor("y")

t_x = Term([f_x])

t_y = Term([f_y])

factor_infos = {f_x:

FactorInfo(f_x, "numerical", {}, num_columns=3),

f_y:

FactorInfo(f_y, "numerical", {}, num_columns=1),

}

term_codings = OrderedDict([(t_x, [SubtermInfo([f_x], {}, 3)]),

(t_y, [SubtermInfo([f_y], {}, 1)])])

di = DesignInfo(["x1", "x2", "x3", "y"], factor_infos, term_codings)

assert di.column_names == ["x1", "x2", "x3", "y"]

assert di.term_names == ["x", "y"]

assert di.terms == [t_x, t_y]

assert di.column_name_indexes == {"x1": 0, "x2": 1, "x3": 2, "y": 3}

assert di.term_name_slices == {"x": slice(0, 3), "y": slice(3, 4)}

assert di.term_slices == {t_x: slice(0, 3), t_y: slice(3, 4)}

assert di.describe() == "x + y"

assert di.slice(1) == slice(1, 2)

assert di.slice("x1") == slice(0, 1)

assert di.slice("x2") == slice(1, 2)

assert di.slice("x3") == slice(2, 3)

assert di.slice("x") == slice(0, 3)

assert di.slice(t_x) == slice(0, 3)

assert di.slice("y") == slice(3, 4)

assert di.slice(t_y) == slice(3, 4)

assert di.slice(slice(2, 4)) == slice(2, 4)

assert_raises(PatsyError, di.slice, "asdf")

# smoke test

repr(di)

assert_no_pickling(di)

# One without term objects

di = DesignInfo(["a1", "a2", "a3", "b"])

assert di.column_names == ["a1", "a2", "a3", "b"]

assert di.term_names == ["a1", "a2", "a3", "b"]

assert di.terms is None

assert di.column_name_indexes == {"a1": 0, "a2": 1, "a3": 2, "b": 3}

assert di.term_name_slices == {"a1": slice(0, 1),

"a2": slice(1, 2),

"a3": slice(2, 3),

"b": slice(3, 4)}

assert di.term_slices is None

assert di.describe() == "a1 + a2 + a3 + b"

assert di.slice(1) == slice(1, 2)

assert di.slice("a1") == slice(0, 1)

assert di.slice("a2") == slice(1, 2)

assert di.slice("a3") == slice(2, 3)

assert di.slice("b") == slice(3, 4)

# Check intercept handling in describe()

assert DesignInfo(["Intercept", "a", "b"]).describe() == "1 + a + b"

# Failure modes

# must specify either both or neither of factor_infos and term_codings:

assert_raises(ValueError, DesignInfo,

["x1", "x2", "x3", "y"], factor_infos=factor_infos)

assert_raises(ValueError, DesignInfo,

["x1", "x2", "x3", "y"], term_codings=term_codings)

# factor_infos must be a dict

assert_raises(ValueError, DesignInfo,

["x1", "x2", "x3", "y"], list(factor_infos), term_codings)

# wrong number of column names:

assert_raises(ValueError, DesignInfo,

["x1", "x2", "x3", "y1", "y2"], factor_infos, term_codings)

assert_raises(ValueError, DesignInfo,

["x1", "x2", "x3"], factor_infos, term_codings)

# name overlap problems

assert_raises(ValueError, DesignInfo,

["x1", "x2", "y", "y2"], factor_infos, term_codings)

# duplicate name

assert_raises(ValueError, DesignInfo,

["x1", "x1", "x1", "y"], factor_infos, term_codings)

# f_y is in factor_infos, but not mentioned in any term

term_codings_x_only = OrderedDict(term_codings)

del term_codings_x_only[t_y]

assert_raises(ValueError, DesignInfo,

["x1", "x2", "x3"], factor_infos, term_codings_x_only)

# f_a is in a term, but not in factor_infos

f_a = _MockFactor("a")

t_a = Term([f_a])

term_codings_with_a = OrderedDict(term_codings)

term_codings_with_a[t_a] = [SubtermInfo([f_a], {}, 1)]

assert_raises(ValueError, DesignInfo,

["x1", "x2", "x3", "y", "a"],

factor_infos, term_codings_with_a)

# bad factor_infos

not_factor_infos = dict(factor_infos)

not_factor_infos[f_x] = "what is this I don't even"

assert_raises(ValueError, DesignInfo,

["x1", "x2", "x3", "y"], not_factor_infos, term_codings)

mismatch_factor_infos = dict(factor_infos)

mismatch_factor_infos[f_x] = FactorInfo(f_a, "numerical", {}, num_columns=3)

assert_raises(ValueError, DesignInfo,

["x1", "x2", "x3", "y"], mismatch_factor_infos, term_codings)

# bad term_codings

assert_raises(ValueError, DesignInfo,

["x1", "x2", "x3", "y"], factor_infos, dict(term_codings))

not_term_codings = OrderedDict(term_codings)

not_term_codings["this is a string"] = term_codings[t_x]

assert_raises(ValueError, DesignInfo,

["x1", "x2", "x3", "y"], factor_infos, not_term_codings)

non_list_term_codings = OrderedDict(term_codings)

non_list_term_codings[t_y] = tuple(term_codings[t_y])

assert_raises(ValueError, DesignInfo,

["x1", "x2", "x3", "y"], factor_infos, non_list_term_codings)

non_subterm_term_codings = OrderedDict(term_codings)

non_subterm_term_codings[t_y][0] = "not a SubtermInfo"

assert_raises(ValueError, DesignInfo,

["x1", "x2", "x3", "y"], factor_infos, non_subterm_term_codings)

bad_subterm = OrderedDict(term_codings)

# f_x is a factor in this model, but it is not a factor in t_y

term_codings[t_y][0] = SubtermInfo([f_x], {}, 1)

assert_raises(ValueError, DesignInfo,

["x1", "x2", "x3", "y"], factor_infos, bad_subterm)

# contrast matrix has wrong number of rows

factor_codings_a = {f_a:

FactorInfo(f_a, "categorical", {},

categories=["a1", "a2"])}

term_codings_a_bad_rows = OrderedDict([

(t_a,

[SubtermInfo([f_a],

{f_a: ContrastMatrix(np.ones((3, 2)),

["[1]", "[2]"])},

2)])])

assert_raises(ValueError, DesignInfo,

["a[1]", "a[2]"],

factor_codings_a,

term_codings_a_bad_rows)

# have a contrast matrix for a non-categorical factor

t_ax = Term([f_a, f_x])

factor_codings_ax = {f_a:

FactorInfo(f_a, "categorical", {},

categories=["a1", "a2"]),

f_x:

FactorInfo(f_x, "numerical", {},

num_columns=2)}

term_codings_ax_extra_cm = OrderedDict([

(t_ax,

[SubtermInfo([f_a, f_x],

{f_a: ContrastMatrix(np.ones((2, 2)), ["[1]", "[2]"]),

f_x: ContrastMatrix(np.ones((2, 2)), ["[1]", "[2]"])},

4)])])

assert_raises(ValueError, DesignInfo,

["a[1]:x[1]", "a[2]:x[1]", "a[1]:x[2]", "a[2]:x[2]"],

factor_codings_ax,

term_codings_ax_extra_cm)

# no contrast matrix for a categorical factor

term_codings_ax_missing_cm = OrderedDict([

(t_ax,

[SubtermInfo([f_a, f_x],

{},

4)])])

# This actually fails before it hits the relevant check with a KeyError,

# but that's okay... the previous test still exercises the check.

assert_raises((ValueError, KeyError), DesignInfo,

["a[1]:x[1]", "a[2]:x[1]", "a[1]:x[2]", "a[2]:x[2]"],

factor_codings_ax,

term_codings_ax_missing_cm)

# subterm num_columns doesn't match the value computed from the individual

# factors

term_codings_ax_wrong_subterm_columns = OrderedDict([

(t_ax,

[SubtermInfo([f_a, f_x],

{f_a: ContrastMatrix(np.ones((2, 3)),

["[1]", "[2]", "[3]"])},

# should be 2 * 3 = 6

5)])])

assert_raises(ValueError, DesignInfo,

["a[1]:x[1]", "a[2]:x[1]", "a[3]:x[1]",

"a[1]:x[2]", "a[2]:x[2]", "a[3]:x[2]"],

factor_codings_ax,

di = DesignInfo.from_array([1, 2, 3])

assert di.column_names == ["column0"]

di2 = DesignInfo.from_array([[1, 2], [2, 3], [3, 4]])

assert di2.column_names == ["column0", "column1"]

di3 = DesignInfo.from_array([1, 2, 3], default_column_prefix="x")

assert di3.column_names == ["x0"]

di4 = DesignInfo.from_array([[1, 2], [2, 3], [3, 4]],

default_column_prefix="x")

assert di4.column_names == ["x0", "x1"]

m = DesignMatrix([1, 2, 3], di3)

assert DesignInfo.from_array(m) is di3

# But weird objects are ignored

m.design_info = "asdf"

di_weird = DesignInfo.from_array(m)

assert di_weird.column_names == ["column0"]

from nose.tools import assert_raises

assert_raises(ValueError, DesignInfo.from_array, np.ones((2, 2, 2)))

from patsy.util import have_pandas

if have_pandas:

import pandas

# with named columns

di5 = DesignInfo.from_array(pandas.DataFrame([[1, 2]],

columns=["a", "b"]))

assert di5.column_names == ["a", "b"]

# with irregularly numbered columns

di6 = DesignInfo.from_array(pandas.DataFrame([[1, 2]],

columns=[0, 10]))

assert di6.column_names == ["column0", "column10"]

# with .design_info attr

df = pandas.DataFrame([[1, 2]])

df.design_info = di6

di = DesignInfo(["a1", "a2", "a3", "b"])

con = di.linear_constraint(["2 * a1 = b + 1", "a3"])

assert con.variable_names == ["a1", "a2", "a3", "b"]

assert np.all(con.coefs == [[2, 0, 0, -1], [0, 0, 1, 0]])

d = DesignInfo(["a1", "a2"])

def check(attr):

with warnings.catch_warnings(record=True) as w:

warnings.simplefilter("always")

assert getattr(d, attr) is d

assert len(w) == 1

assert w[0].category is DeprecationWarning

check("builder")

format_str = "%." + str(precision) + "f"

assert col.ndim == 1

# We don't want to look at numbers like "1e-5" or "nan" when stripping.

simple_float_chars = set("+-0123456789.")

col_strs = np.array([format_str % (x,) for x in col], dtype=object)

# Really every item should have a decimal, but just in case, we don't want

# to strip zeros off the end of "10" or something like that.

mask = np.array([simple_float_chars.issuperset(col_str) and "." in col_str

for col_str in col_strs])

mask_idxes = np.nonzero(mask)[0]

strip_char = "0"

if np.any(mask):

while True:

if np.all([s.endswith(strip_char) for s in col_strs[mask]]):

for idx in mask_idxes:

col_strs[idx] = col_strs[idx][:-1]

else:

if strip_char == "0":

strip_char = "."

else:

break

def t(precision, numbers, expected):

got = _format_float_column(precision, np.asarray(numbers))

print(got, expected)

assert np.array_equal(got, expected)

# This acts weird on old python versions (e.g. it can be "-nan"), so don't

# hardcode it:

nan_string = "%.3f" % (np.nan,)

t(3, [1, 2.1234, 2.1239, np.nan], ["1.000", "2.123", "2.124", nan_string])

t(3, [1, 2, 3, np.nan], ["1", "2", "3", nan_string])

t(3, [1.0001, 2, 3, np.nan], ["1", "2", "3", nan_string])

"""A simple numpy array subclass that carries design matrix metadata.