unexpected data passed to log_prob method of model_wrapper class

[mdzeig]

I recently discovered Edward and I’ve been working on getting some psychometric applications up and running, starting with a simple example, the Rasch model. The Rasch model is a simple model for binary matrices X which decomposes the logit of the probability that each X_{pi} = 1 into the difference between two components, a row component \theta_p and a column component \beta_i.

I’m attempting to implement the Rasch model using a class model wrapper. The model has two latent variables — a 200 x 1 tensor trait containing the theta values and a 1 x 25 tensor thresh containing the beta values. To track down why variational inference is diverging, I inserted the print statement inside the log_prob method. Here's the code

#!/usr/bin/env python

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import tensorflow as tf
import edward as ed
import matplotlib.pyplot as plt
from edward.models import Normal, Bernoulli
from edward.stats import norm
from scipy.special import expit

class RaschModel:
    """Full Rasch model with N(0, 1) priors on person (\theta_p)
    and item parameters (\beta_i).

    .. math::

    log p(X_{pi} = 1) - log P(X_{pi} = 0) = \theta_p - \beta_i

    """

    def log_prob(self, xs, zs):
        X = xs['X']
        trait, thresh = zs['trait'], zs['thresh']
        print(trait, thresh, flush=True)
        log_prior = tf.reduce_sum(norm.log_pdf(trait, 0.0, 1.0))
        log_prior += tf.reduce_sum(norm.log_pdf(thresh, 0.0, 1.0))
        logit = tf.sub(trait, thresh)
        log_lik = tf.reduce_sum(tf.mul(X, logit))
        log_lik -= tf.reduce_sum(tf.log(1 + tf.exp(logit)))
        # log_like -= tf.reduce_sum(tf.nn.softplus(logit))
        return log_prior + log_lik

def create_data(nsubj, nitem):
    trait = np.random.normal(size=[nsubj, 1])
    thresh = np.random.normal(size=[1, nitem])
    X = np.random.binomial(1, expit(trait - thresh))
    return locals()

nsubj = 200
nitem = 25
dataset = create_data(nsubj, nitem)
model = RaschModel()

q_trait = Normal(mu=tf.Variable(tf.random_normal([nsubj, 1])),
                 sigma=tf.nn.softplus(tf.Variable(tf.random_normal([nsubj, 1]))))

q_thresh = Normal(mu=tf.Variable(tf.random_normal([1, nitem])),
                  sigma=tf.nn.softplus(tf.Variable(tf.random_normal([1, nitem]))))

inference = ed.MFVI({'trait': q_trait, 'thresh': q_thresh},
                    {'X': dataset['X']}, model)
#inference.initialize()
inference.run(n_iter=2500, n_samples=10, n_minibatch=20)

plt.scatter(dataset['trait'], q_trait.mu.eval())
plt.show()

The output from my system is below. I would expect the first Tensor to have shape (200, 1) rather than (1, 25).

Tensor("inference_0/Normal_1/sample/sample:0", shape=(1, 25), dtype=float32) Tensor("inference_0/Normal_1/sample/sample:0", shape=(1, 25), dtype=float32)
Tensor("inference_1/Normal_1/sample/sample:0", shape=(1, 25), dtype=float32) Tensor("inference_1/Normal_1/sample/sample:0", shape=(1, 25), dtype=float32)
Tensor("inference_2/Normal_1/sample/sample:0", shape=(1, 25), dtype=float32) Tensor("inference_2/Normal_1/sample/sample:0", shape=(1, 25), dtype=float32)
Tensor("inference_3/Normal_1/sample/sample:0", shape=(1, 25), dtype=float32) Tensor("inference_3/Normal_1/sample/sample:0", shape=(1, 25), dtype=float32)
Tensor("inference_4/Normal_1/sample/sample:0", shape=(1, 25), dtype=float32) Tensor("inference_4/Normal_1/sample/sample:0", shape=(1, 25), dtype=float32)
Tensor("inference_5/Normal_1/sample/sample:0", shape=(1, 25), dtype=float32) Tensor("inference_5/Normal_1/sample/sample:0", shape=(1, 25), dtype=float32)
Tensor("inference_6/Normal_1/sample/sample:0", shape=(1, 25), dtype=float32) Tensor("inference_6/Normal_1/sample/sample:0", shape=(1, 25), dtype=float32)
Tensor("inference_7/Normal_1/sample/sample:0", shape=(1, 25), dtype=float32) Tensor("inference_7/Normal_1/sample/sample:0", shape=(1, 25), dtype=float32)
Tensor("inference_8/Normal_1/sample/sample:0", shape=(1, 25), dtype=float32) Tensor("inference_8/Normal_1/sample/sample:0", shape=(1, 25), dtype=float32)
Tensor("inference_9/Normal_1/sample/sample:0", shape=(1, 25), dtype=float32) Tensor("inference_9/Normal_1/sample/sample:0", shape=(1, 25), dtype=float32)
iter 1 loss 59674.46
iter 100 loss 7335925087847055360.00
W tensorflow/core/framework/op_kernel.cc:940] Invalid argument: assertion failed: [] [Condition x > 0 did not hold element-wise: x = ] [Softplus_37:0] [1.9073468e-06 1.7881378e-06 3.5762781e-07...]
	 [[Node: Normal_660/assert_positive/assert_less/Assert = Assert[T=[DT_STRING, DT_STRING, DT_STRING, DT_FLOAT], summarize=3, _device="/job:localhost/replica:0/task:0/cpu:0"](Normal_660/assert_positive/assert_less/All, Normal_660/assert_positive/assert_less/Assert/data_0, Normal_660/assert_positive/assert_less/Assert/data_1, Normal_660/assert_positive/assert_less/Assert/data_2, Softplus_37)]]
---------------------------------------------------------------------------
InvalidArgumentError                      Traceback (most recent call last)
/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
    964     try:
--> 965       return fn(*args)
    966     except errors.OpError as e:

/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/tensorflow/python/client/session.py in _run_fn(session, feed_dict, fetch_list, target_list, options, run_metadata)
    946                                  feed_dict, fetch_list, target_list,
--> 947                                  status, run_metadata)
    948 

/~/Documents/mzeigenfuse/anaconda/lib/python3.5/contextlib.py in __exit__(self, type, value, traceback)
     65             try:
---> 66                 next(self.gen)
     67             except StopIteration:

/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/tensorflow/python/framework/errors.py in raise_exception_on_not_ok_status()
    449           compat.as_text(pywrap_tensorflow.TF_Message(status)),
--> 450           pywrap_tensorflow.TF_GetCode(status))
    451   finally:

InvalidArgumentError: assertion failed: [] [Condition x > 0 did not hold element-wise: x = ] [Softplus_37:0] [1.9073468e-06 1.7881378e-06 3.5762781e-07...]
	 [[Node: Normal_660/assert_positive/assert_less/Assert = Assert[T=[DT_STRING, DT_STRING, DT_STRING, DT_FLOAT], summarize=3, _device="/job:localhost/replica:0/task:0/cpu:0"](Normal_660/assert_positive/assert_less/All, Normal_660/assert_positive/assert_less/Assert/data_0, Normal_660/assert_positive/assert_less/Assert/data_1, Normal_660/assert_positive/assert_less/Assert/data_2, Softplus_37)]]

During handling of the above exception, another exception occurred:

InvalidArgumentError                      Traceback (most recent call last)
/~/Documents/mzeigenfuse/Projects/irtvb/raschmodel.py in <module>()
     55                     {'X': dataset['X']}, model)
     56 #inference.initialize()
---> 57 inference.run(n_iter=2500, n_samples=10, n_minibatch=20)
     58 
     59 plt.scatter(dataset['trait'], q_trait.mu.eval())

/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/edward/inferences/inference.py in run(self, logdir, variables, use_coordinator, *args, **kwargs)
    173 
    174     for _ in range(self.n_iter):
--> 175       info_dict = self.update()
    176       self.print_progress(info_dict)
    177 

/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/edward/inferences/variational_inference.py in update(self, feed_dict)
    139 
    140     sess = get_session()
--> 141     _, t, loss = sess.run([self.train, self.increment_t, self.loss], feed_dict)
    142     return {'t': t, 'loss': loss}
    143 

/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/tensorflow/python/client/session.py in run(self, fetches, feed_dict, options, run_metadata)
    708     try:
    709       result = self._run(None, fetches, feed_dict, options_ptr,
--> 710                          run_metadata_ptr)
    711       if run_metadata:
    712         proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)

/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/tensorflow/python/client/session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
    906     if final_fetches or final_targets:
    907       results = self._do_run(handle, final_targets, final_fetches,
--> 908                              feed_dict_string, options, run_metadata)
    909     else:
    910       results = []

/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/tensorflow/python/client/session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
    956     if handle is None:
    957       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 958                            target_list, options, run_metadata)
    959     else:
    960       return self._do_call(_prun_fn, self._session, handle, feed_dict,

/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
    976         except KeyError:
    977           pass
--> 978       raise type(e)(node_def, op, message)
    979 
    980   def _extend_graph(self):

InvalidArgumentError: assertion failed: [] [Condition x > 0 did not hold element-wise: x = ] [Softplus_37:0] [1.9073468e-06 1.7881378e-06 3.5762781e-07...]
	 [[Node: Normal_660/assert_positive/assert_less/Assert = Assert[T=[DT_STRING, DT_STRING, DT_STRING, DT_FLOAT], summarize=3, _device="/job:localhost/replica:0/task:0/cpu:0"](Normal_660/assert_positive/assert_less/All, Normal_660/assert_positive/assert_less/Assert/data_0, Normal_660/assert_positive/assert_less/Assert/data_1, Normal_660/assert_positive/assert_less/Assert/data_2, Softplus_37)]]
Caused by op 'Normal_660/assert_positive/assert_less/Assert', defined at:
  File "/~/Documents/mzeigenfuse/anaconda/bin/ipython", line 6, in <module>
    sys.exit(IPython.start_ipython())
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/IPython/__init__.py", line 119, in start_ipython
    return launch_new_instance(argv=argv, **kwargs)
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/traitlets/config/application.py", line 653, in launch_instance
    app.start()
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/IPython/terminal/ipapp.py", line 348, in start
    self.shell.mainloop()
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/IPython/terminal/interactiveshell.py", line 440, in mainloop
    self.interact()
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/IPython/terminal/interactiveshell.py", line 431, in interact
    self.run_cell(code, store_history=True)
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 2717, in run_cell
    interactivity=interactivity, compiler=compiler, result=result)
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 2827, in run_ast_nodes
    if self.run_code(code, result):
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 2881, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-49-57e6ba9c8ee1>", line 1, in <module>
    get_ipython().magic('run raschmodel.py')
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 2158, in magic
    return self.run_line_magic(magic_name, magic_arg_s)
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 2079, in run_line_magic
    result = fn(*args,**kwargs)
  File "<decorator-gen-57>", line 2, in run
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/IPython/core/magic.py", line 188, in <lambda>
    call = lambda f, *a, **k: f(*a, **k)
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/IPython/core/magics/execution.py", line 742, in run
    run()
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/IPython/core/magics/execution.py", line 728, in run
    exit_ignore=exit_ignore)
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 2481, in safe_execfile
    self.compile if kw['shell_futures'] else None)
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/IPython/utils/py3compat.py", line 186, in execfile
    exec(compiler(f.read(), fname, 'exec'), glob, loc)
  File "/~/Documents/mzeigenfuse/Projects/irtvb/raschmodel.py", line 52, in <module>
    sigma=tf.nn.softplus(tf.Variable(tf.random_normal([1, nitem]))))
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/edward/models/random_variables.py", line 36, in __init__
    RandomVariable.__init__(self, *args, **kwargs)
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/edward/models/random_variable.py", line 58, in __init__
    super(RandomVariable, self).__init__(*args, **kwargs)
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/tensorflow/contrib/distributions/python/ops/normal.py", line 115, in __init__
    validate_args else []):
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/tensorflow/python/ops/check_ops.py", line 178, in assert_positive
    return assert_less(zero, x, data=data, summarize=summarize)
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/tensorflow/python/ops/check_ops.py", line 354, in assert_less
    return logging_ops.Assert(condition, data, summarize=summarize)
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/tensorflow/python/ops/logging_ops.py", line 58, in Assert
    return gen_logging_ops._assert(condition, data, summarize, name)
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/tensorflow/python/ops/gen_logging_ops.py", line 37, in _assert
    summarize=summarize, name=name)
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py", line 710, in apply_op
    op_def=op_def)
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 2317, in create_op
    original_op=self._default_original_op, op_def=op_def)
  File "/~/Documents/mzeigenfuse/anaconda/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 1239, in __init__
    self._traceback = _extract_stack()

[dustinvtran]

on running your code (without the flush print argument, which didn't work for me), the tensors above are indeed the right shape. in your output, it also looks like it's printing the same tensor both times(?).

i get a different error. it is a result of using n_minibatch=20, so that the input data xs['X'] has shape (20, 25) but the logits argument has shape (200, 25).

Tensor("inference_0/Normal/sample/Reshape:0", shape=(200, 1), dtype=float32) Tensor("inference_0/Normal_1/sample/Reshape:0", shape=(1, 25), dtype=float32)
Traceback (most recent call last):
  File "raschmodel.py", line 57, in <module>
    inference.run(n_iter=2500, n_samples=10, n_minibatch=20)
  File "/Users/dvt/Envs/venv/lib/python2.7/site-packages/edward/inferences/inference.py", line 157, in run
    self.initialize(*args, **kwargs)
  File "/Users/dvt/Envs/venv/lib/python2.7/site-packages/edward/inferences/klqp.py", line 41, in initialize
    return super(KLqp, self).initialize(*args, **kwargs)
  File "/Users/dvt/Envs/venv/lib/python2.7/site-packages/edward/inferences/variational_inference.py", line 106, in initialize
    self.loss, grads_and_vars = self.build_loss_and_gradients(scope=scope)
  File "/Users/dvt/Envs/venv/lib/python2.7/site-packages/edward/inferences/klqp.py", line 81, in build_loss_and_gradients
    loss = build_reparam_loss(self)
  File "/Users/dvt/Envs/venv/lib/python2.7/site-packages/edward/inferences/klqp.py", line 319, in build_reparam_loss
    p_log_prob[s] = inference.model_wrapper.log_prob(x, z_sample)
  File "raschmodel.py", line 32, in log_prob
    log_lik = tf.reduce_sum(tf.mul(X, logit))
  File "/Users/dvt/Envs/venv/lib/python2.7/site-packages/tensorflow/python/ops/gen_math_ops.py", line 1519, in mul
    result = _op_def_lib.apply_op("Mul", x=x, y=y, name=name)
  File "/Users/dvt/Envs/venv/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 749, in apply_op
    op_def=op_def)
  File "/Users/dvt/Envs/venv/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2382, in create_op
    set_shapes_for_outputs(ret)
  File "/Users/dvt/Envs/venv/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1783, in set_shapes_for_outputs
    shapes = shape_func(op)
  File "/Users/dvt/Envs/venv/lib/python2.7/site-packages/tensorflow/python/framework/common_shapes.py", line 596, in call_cpp_shape_fn
    raise ValueError(err.message)
ValueError: Dimensions must be equal, but are 20 and 200```

[dustinvtran]

upon removing the n_minbatch argument, the code runs for me.

Iteration    1 [  0%]: Loss = 5773.036
Iteration  250 [ 10%]: Loss = 2890.587
Iteration  500 [ 20%]: Loss = 2892.991
Iteration  750 [ 30%]: Loss = 2890.511
Iteration 1000 [ 40%]: Loss = 2889.903
Iteration 1250 [ 50%]: Loss = 2889.706
Iteration 1500 [ 60%]: Loss = 2888.860
Iteration 1750 [ 70%]: Loss = 2888.186
Iteration 2000 [ 80%]: Loss = 2888.164
Iteration 2250 [ 90%]: Loss = 2889.640
Iteration 2500 [100%]: Loss = 2888.406

if you're still having shape troubles, could you try reporting your edward version and tensorflow version? not sure how it's even running with the incompatible X and logit multiplication.

[dustinvtran]

here's a simplified version of the script, leveraging Edward's native language. let me know if this one doesn't run.

#!/usr/bin/env python
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import edward as ed
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt

from edward.models import Normal, Bernoulli
from scipy.special import expit


def create_data(nsubj, nitem):
    trait = np.random.normal(size=[nsubj, 1])
    thresh = np.random.normal(size=[1, nitem])
    X = np.random.binomial(1, expit(trait - thresh))
    return locals()


# DATA
nsubj = 200
nitem = 25
dataset = create_data(nsubj, nitem)

# MODEL
trait = Normal(mu=tf.zeros([nsubj, 1]), sigma=tf.ones([nsubj, 1]))
thresh = Normal(mu=tf.zeros([1, nitem]), sigma=tf.ones([1, nitem]))
X = Bernoulli(logits=tf.sub(trait, thresh))

# INFERENCE
q_trait = Normal(mu=tf.Variable(tf.random_normal([nsubj, 1])),
                 sigma=tf.nn.softplus(tf.Variable(tf.random_normal([nsubj, 1]))))
q_thresh = Normal(mu=tf.Variable(tf.random_normal([1, nitem])),
                  sigma=tf.nn.softplus(tf.Variable(tf.random_normal([1, nitem]))))

inference = ed.KLqp({trait: q_trait, thresh: q_thresh},
                    {X: dataset['X']})
inference.run(n_iter=2500, n_samples=10)

# CRITICISM
plt.scatter(dataset['trait'], q_trait.mu.eval())
plt.show()

[mdzeig]

It actually was supposed to fail, because of the error. I became confused when it didn't. As far as I can tell, the issue has something to do with the ipython environment. When I run code from the command line, it works perfectly. When I call it from ipython (5.1.0, with python 3.5.2) using the %run magic, it fails miserably with the above errors.

[dustinvtran]

oh i see. interesting. what ipython does separately from the command line is essentially black magic to me. do similar inconsistencies occur when using the version written in edward's native language?

[mdzeig]

yeah, i don't really know what it's doing either. i appreciate the shorthand for some things, but it does add an extra layer of complexity. i get the same result from both versions (native and model_wrapper).