Learning(Distribu.ons(over(Logical(Forms(for( - - PowerPoint PPT Presentation

Learning(Distribu.ons(over(Logical(Forms(for( Referring(Expression(Genera.on(

Nicholas(FitzGerald(((((Yoav(Artzi(((((Luke(ZeClemoyer(

Referring(Expressions(

Photo:(pwenzel(on(Flikr(

Referring(Expressions(

Photo:(pwenzel(on(Flikr(

Referring(Expressions(

“the(red(beans”(

Photo:(pwenzel(on(Flikr(

Referring(Expressions(

“the(red(beans”( “the(second(jar(from(the( leK(on(the(middle(shelf”(

Photo:(pwenzel(on(Flikr(

Referring(Expressions(

“the(red(beans”( “the(second(jar(from(the( leK(on(the(middle(shelf”( “the(jar(underneath(the( space(between(the(two( biggest(white(jars”(

Photo:(pwenzel(on(Flikr(

``The(green( and(red( balls.’’(

Grounded(Language(Problems(

Physical(Scene( Language(

``The(green( and(red( balls.’’(

Grounded(Language(Problems(

Physical(Scene( Logical(Form((LF)( Language(

ιx.(green(x) ∨ red(x)) ∧sphere(x)

``The(green( and(red( balls.’’(

Grounded(Language(Problems(

Physical(Scene( Logical(Form((LF)( Language(

ιx.(green(x) ∨ red(x)) ∧sphere(x)

``The(green( and(red( spheres.’’(

Grounded(Language(Problems(

Physical(Scene( Logical(Form((LF)( Language(

ιx.(green(x) ∨ red(x)) ∧sphere(x)

``The(green( and(red( spheres.’’(

Grounded(Language(Problems(

Grounded(Language(Understanding((

[Matuszek(et.(al(2012],([Krishnamurthy(et.(al(2013](

( Physical(Scene( Logical(Form((LF)( Language(

ιx.(green(x) ∨ red(x)) ∧sphere(x)

``The(green( and(red( spheres.’’(

Grounded(Language(Problems(

Realiza.on(from(LF(

[White(and(Rajkumar(2009],([Lu(and(Ng(2011](

Physical(Scene( Logical(Form((LF)( Language(

ιx.(green(x) ∨ red(x)) ∧sphere(x)

Grounded(Language(Understanding((

[Matuszek(et.(al(2012],([Krishnamurthy(et.(al(2013](

(

``The(green( and(red( spheres.’’(

Grounded(Language(Problems(

Realiza.on(from(LF(

[White(and(Rajkumar(2009],([Lu(and(Ng(2011](

Physical(Scene( Logical(Form((LF)( Language( LF(Genera.on( Focus(of(this(talk:( Generate(Distribu.on(

ιx.(green(x) ∨ red(x)) ∧sphere(x)

Grounded(Language(Understanding((

[Matuszek(et.(al(2012],([Krishnamurthy(et.(al(2013](

(

Goal:(Generate(Distribu.ons(over(LFs(

ιx.green(x) ∧ sphere(x) ∪ιx.red(x) ∧ sphere(x)

ιx.apple(x) ∪ ιx.pear(x)

ιx.(green(x) ∨ red(x)) ∧sphere(x)

. ..

“The(green(ball( and(the(red(ball.”( “The(green(and(red(spheres.”( ( “The(green(and(red(balls.”( “The(apple(and(the(pear.”(

Model(how(people(refer(to(objects(

• Many(different(expressions(for(each(referent(
• Some(are(more(likely(to(be(used(in(prac.ce(
• We(need(to(learn(a(probability(distribu.on(

Goal:(Generate(Distribu.ons(over(LFs(

ιx.green(x) ∧ sphere(x) ∪ιx.red(x) ∧ sphere(x)

ιx.apple(x) ∪ ιx.pear(x)

ιx.(green(x) ∨ red(x)) ∧sphere(x)

. ..

“The(green(ball( and(the(red(ball.”( “The(green(and(red(spheres.”( ( “The(green(and(red(balls.”( “The(apple(and(the(pear.”(

0.2( 0.3( 0.1(

P(exp)(

Model(how(people(refer(to(objects(

• Many(different(expressions(for(each(referent(
• Some(are(more(likely(to(be(used(in(prac.ce(
• We(need(to(learn(a(probability(distribu.on(

0.4(

Goal:(Generate(Distribu.ons(over(LFs(

ιx.green(x) ∧ sphere(x) ∪ιx.red(x) ∧ sphere(x)

ιx.apple(x) ∪ ιx.pear(x)

ιx.(green(x) ∨ red(x)) ∧sphere(x)

. . .

0.3( 0.2( 0.1(

. . .

Several(advantages(

• Natural(varia.on(for(genera.on(
• Useful(prior(for(understanding(systems(

Input:( Output:(

Learn(from(Labeled(Examples(

“The(green,(red,(orange(and(yellow(toys”( “The(green,(red,(yellow,(and(orange(objects”( “All(the(pieces(that(are(not(blue(or(brown”( “All(items(that(are(not(brown(or(blue”( “Everything(that(is(not(brown(or(blue”( …..(

{( ,( }( {( ,(

“The(red(and(green(balls.”( “The(red(and(greed(spheres.”( “The(pear(and(the(apple”( “The(red(ball(and(the(green(pear”( “All(the(balls(except(the(yellow(one”( ……(

(

}(

Lots(of(varia.on(in(prac.ce(

• Collected(20(sentences(per(scene(
• Mean(of(6(unique(logical(forms(per(scene(
• Max(of(13((

Learn(from(Labeled(Examples(

{(

Lots(of(varia.on(in(prac.ce(

• Collected(20(sentences(per(scene(
• Mean(of(6(unique(logical(forms(per(scene(
• Max(of(13((

,( }( {( ,( }(

ιx.green(x) ∧ sphere(x) ∪ιx.red(x) ∧ sphere(x)

ιx.apple(x) ∪ ιx.pear(x) ιx.(green(x) ∨ red(x)) ∧sphere(x)

0.3( 0.2( 0.1(

ιx.(red(x) ∨ green(x) ∨ orange(x) ∨yellow(x)) ∧ obj(x)

Ex.obj(x) \ (ιx.brown(x) ∧ triangle(x)

∪(ιx.blue(x) ∧ lego(x)

. . . . . .

0.4( 0.3(

Overview(

Space(of(Referring(Expressions( Probabilis.c(Model( Learning( Experiments( Results( Conclusion( ( (

Overview(

Space(of(Referring(Expressions(

(Seman.c(Modeling( (Enumera.ng(Referring(Expressions(

Probabilis.c(Model( Learning( Experiments( Results( Conclusion( ( (

Seman.c(Modeling(

• Simplyityped(lambda(calculus(

– [Steedman(1996],([Carpenter(1997],([Steedman(2011](

• Extended(to(model(set(reference(

– Capture(dis.nc.ons(present(in(data( – As(simple(as(possible(

Seman.c(Modeling(

e

:(((Sets(of(Objects(

t

:((([True,(False]( Two(Simple(Types:(

λx.blue(x)

Seman.c(Modeling(

< e, t >

ACributes(

Seman.c(Modeling(

< e, t >

ACributes(

λx.triangle(x)

Seman.c(Modeling(

< e, t >

ACributes(

λx.¬blue(x)

Logical(Operators(

λx.blue(x) ∧ triangle(x)

Seman.c(Modeling(

ACributes(

< e, t >

Logical(Operators(

λx.blue(x) ∨ triangle(x)

Seman.c(Modeling(

ACributes(

< e, t >

Logical(Operators(

Seman.c(Modeling(

ιx.triangle(x) << e, t >, e >

Determiners( ACribute(Coordina.on( Logical(Operators(

Seman.c(Modeling(

[ι, E, A] << e, t >, e >

Determiners( ACribute(Coordina.on( Logical(Operators(

E ≈ ∀ A ≈ ∃

<< e, e >, e >

Seman.c(Modeling(

Set(Coordina.on(

ιx.triangle(x) ∪ιx.apple(x)

Determiners( ACribute(Coordina.on( Logical(Operators(

<< e, e >, e >

Seman.c(Modeling(

Set(Coordina.on(

ιx.triangle(x) \ιx.blue(x) ∧triangle(x)

Determiners( ACribute(Coordina.on( Logical(Operators(

Seman.c(Modeling(

Set(Coordina.on( Determiners( ACribute(Coordina.on( Logical(Operators( Plurality( Cardinality(

Seman.c(Modeling(

• Simplyityped(lambda(calculus(

– [Steedman(1996],([Carpenter(1997],([Steedman(2011](

• Extended(to(model(set(reference(

– Capture(dis.nc.ons(present(in(data( – As(simple(as(possible(

• Two(new(contribu.ons:(

– Sets(as(a(primi.ve(type((plurals)( – Coordina.on(

Enumera.ng(Logical(Forms(

• Enumerate(candidate(Logical(Forms(
• Problem:(

– Infinite(in(general(

• Goal:(

– finite(set(with(good(empirical(coverage( – strategy(for(enumera.ng(

Enumera.ng(Logical(Forms(

λx.red(x) λx.blue(x) λx.cube(x) λx.object(x)

. . .

λx.rect(x)

1 2 3 4 5

Enumera.ng(Logical(Forms(

λx.red(x) λx.blue(x) λx.cube(x) λx.object(x)

. . .

λx.rect(x)

λx.¬red(x)

1 2 3 4 5

Enumera.ng(Logical(Forms(

λx.red(x) λx.blue(x) λx.cube(x) λx.object(x)

. . .

λx.rect(x)

λx.¬red(x) λx.¬blue(x)

1 2 3 4 5

Enumera.ng(Logical(Forms(

λx.red(x) λx.blue(x) λx.cube(x) λx.object(x)

. . .

λx.rect(x)

λx.¬red(x) λx.¬blue(x) ιx.red(x) ιx.cube(x) Ex.object(x)

. . . 1 2 3 4 5

Enumera.ng(Logical(Forms(

λx.red(x) λx.blue(x) λx.cube(x) λx.object(x)

. . .

λx.rect(x)

λx.¬red(x) λx.¬blue(x) ιx.red(x) ιx.cube(x) Ex.object(x)

. . .

Ex.¬red(x)

ιx.¬blue(x)

1 2 3 4 5

Enumera.ng(Logical(Forms(

λx.red(x) λx.blue(x) λx.cube(x) λx.object(x)

. . .

λx.rect(x)

λx.¬red(x) λx.¬blue(x) ιx.red(x) ιx.cube(x) Ex.object(x)

. . .

Ex.¬red(x)

ιx.¬blue(x)

λx.red(x) ∧ cube(x)

1 2 3 4 5

Enumera.ng(Logical(Forms(

λx.red(x) λx.blue(x) λx.cube(x) λx.object(x)

. . .

λx.rect(x)

λx.¬red(x) λx.¬blue(x) ιx.red(x) ιx.cube(x) Ex.object(x)

. . .

Ex.¬red(x)

ιx.¬blue(x)

λx.red(x) ∧ cube(x) λx.red(x) ∧ object(x)

1 2 3 4 5

Enumera.ng(Logical(Forms(

λx.red(x) λx.blue(x) λx.cube(x) λx.object(x)

. . .

λx.rect(x)

λx.¬red(x) λx.¬blue(x) ιx.red(x) ιx.cube(x) Ex.object(x)

. . .

Ex.¬red(x)

ιx.¬blue(x)

λx.red(x) ∧ cube(x) λx.red(x) ∧ object(x) λx.red(x) ∨ cube(x)

. . . 1 2 3 4 5

Enumera.ng(Logical(Forms(

λx.red(x) λx.blue(x) λx.cube(x) λx.object(x) λx.¬red(x) λx.¬blue(x) ιx.red(x) ιx.cube(x) λx.red(x) ∧ object(x) λx.red(x) ∧ cube(x) λx.red(x) ∨ cube(x) ιx.red(x) ∧ object(x) Ax.red(x) ∨ object(x) Ex.object(x) Ex.¬red(x) λx.¬cube(x) ∧ object(x) λx.¬(cube(x) ∨ object(x))

. . . . . .

ιx.¬blue(x)

. . . . . .

λx.rect(x)

1 2 3 4 5

Enumera.ng(Logical(Forms(

λx.red(x) λx.blue(x) λx.cube(x) λx.object(x) λx.¬red(x) λx.¬blue(x) ιx.red(x) ιx.cube(x) λx.red(x) ∧ object(x) λx.red(x) ∧ cube(x) λx.red(x) ∨ cube(x) ιx.red(x) ∧ object(x) Ax.red(x) ∨ object(x) Ex.object(x) Ex.¬red(x) λx.¬cube(x) ∧ object(x) λx.¬(cube(x) ∨ object(x)) ιx.red(x) ∪ ιx.cube(x) Ex.object(x) \ ιx.cube(x)

. . . . . .

ιx.¬blue(x)

. . . . . .

λx.rect(x)

. . .

λx.object(x) ∧ equal(x, Ay.cube(y))

1 2 3 4 5

Enumera.ng(Logical(Forms(

λx.red(x) λx.blue(x) λx.cube(x) λx.object(x) λx.¬red(x) λx.¬blue(x) ιx.red(x) ιx.cube(x) λx.red(x) ∧ object(x) λx.red(x) ∧ cube(x) λx.red(x) ∨ cube(x) ιx.red(x) ∧ object(x) Ax.red(x) ∨ object(x) Ex.object(x) Ex.¬red(x) λx.¬cube(x) ∧ object(x) λx.¬(cube(x) ∨ object(x)) ιx.red(x) ∪ ιx.cube(x) Ex.object(x) \ ιx.cube(x)

. . . . . .

ιx.¬blue(x)

. . . . . .

λx.rect(x)

. . .

λx.object(x) ∧ equal(x, Ay.cube(y))

M 1 2 3 4 5

Overview(

Space(of(Referring(Expressions( Probabilis.c(Model(

(Global(Model( (Explicit(Pruning(Model( (Features(

Learning( Experiments( Results( Conclusion( ( (

Global(Model(

P(z | S, G), z ∈ Z

Global(Model(

P(z | S, G), z ∈ Z

• bj1: red, sphere, apple
• bj2: brown, triangle
• bj3: yellow, fries

… …

World(State(

Target(Set(

Global(Model(

P(z | S, G), z ∈ Z

Global(Model(

• Global(DensityiEs.ma.on(Model(

– Mul.nomial(Logilinear(over(expressions(z(that(name(the( set(G(in(state(S(

PG(z | S, G; θ) = 1 C eθ·φ(z,S,G)

θ ∈ Rn φ(z, S, G) ∈ Rn

C = X

z0∈Z

eθ·φ(z0,S,G)

Parameters( Features( Normaliza.on( constant(

Global(Model(

• Global(DensityiEs.ma.on(Model(

– Mul.nomial(Logilinear(

PG(z | S, G; θ) = 1 C eθ·φ(z,S,G)

θ ∈ Rn φ(z, S, G) ∈ Rn

C = X

z0∈Z

eθ·φ(z0,S,G)

Parameters( Features( Normaliza.on( constant(

Too(Big!(

(Exponen.al(in(max(number(

• f(constants(M)(

Pruning(

λx.red(x) λx.blue(x) λx.cube(x) λx.object(x) λx.¬red(x) λx.¬blue(x) ιx.red(x) ιx.cube(x) λx.red(x) ∧ object(x) λx.red(x) ∧ cube(x) λx.red(x) ∨ cube(x) ιx.red(x) ∧ object(x) Ax.red(x) ∨ object(x) Ex.object(x) Ex.¬red(x) λx.¬cube(x) ∧ object(x) λx.¬(cube(x) ∨ object(x)) ιx.red(x) ∪ ιx.cube(x) Ex.object(x) \ ιx.cube(x)

. . . . . .

ιx.¬blue(x)

. . . . . .

λx.rect(x)

. . .

λx.object(x) ∧ equal(x, Ay.cube(y))

M 1 2 3 4 5

Pruning(

λx.red(x) λx.blue(x) λx.cube(x) λx.object(x) λx.¬red(x) λx.¬blue(x) ιx.red(x) ιx.cube(x) λx.red(x) ∧ object(x) λx.red(x) ∧ cube(x) λx.red(x) ∨ cube(x) ιx.red(x) ∧ object(x) Ax.red(x) ∨ object(x) Ex.object(x) Ex.¬red(x) λx.¬cube(x) ∧ object(x) λx.¬(cube(x) ∨ object(x)) ιx.red(x) ∪ ιx.cube(x) Ex.object(x) \ ιx.cube(x)

ιx.¬blue(x) λx.rect(x) λx.object(x) ∧ equal(x, Ay.cube(y))

. . . . . . . . . . . . . . . M 1 2 3 4 5

Pruning(

λx.red(x) λx.blue(x) λx.cube(x) λx.object(x) λx.¬red(x) λx.¬blue(x) ιx.red(x) ιx.cube(x) λx.red(x) ∧ object(x) λx.red(x) ∧ cube(x) λx.red(x) ∨ cube(x) ιx.red(x) ∧ object(x) Ax.red(x) ∨ object(x) Ex.object(x) Ex.¬red(x) λx.¬cube(x) ∧ object(x) λx.¬(cube(x) ∨ object(x)) ιx.red(x) ∪ ιx.cube(x) Ex.object(x) \ ιx.cube(x)

ιx.¬blue(x) λx.rect(x) λx.object(x) ∧ equal(x, Ay.cube(y))

Topik( (Beam(Search)( k

M 1 2 3 4 5

Pruning(Model(

Good(Referring( Expression( Good(Subi Expression(

6=

Pj(a | S, G) =

Pruning(Model(

Good(Referring( Expression( Good(Subi Expression(

6=

Binary(probability(distribu.on(indica.ng( whether(an(expression(should(be(pruned( at(complexityilevel(( j

Pruning(Model(

• Binary(LogiLinear(Model(for(each(complexityi

level(

Pj(a | S, G; πj) = eπj·φ(a,S,G) 1 + eπj·φ(a,S,G)

πj

Parameters(

j

Features(

• Structural(Features(

– Logical(form((((only( – Capture(common(combina.ons(of(predicates(

• Situated(Features(

– LF(((,(worldistate(S(and(targetiset( – Capture(how(subiexpressions(of(z(group(sets(of(

• bjects(in(the(scene(
• Complexity(Feature(

z S G z

Structural(Features(

ιx.red(x) ∧ object(x)

Structural(Features(

∧ ιx red(x)

• bject(x)

[∧, color]

Structural(Features(

∧ ιx red(x)

• bject(x)

[ι, ∧] [∧, object]

Head(Bigram(

[∧; color, object]

Structural(Features(

∧ ιx red(x)

• bject(x)

Coordina.on(Children( Head(Bigram(

Structural(Features(

∧ ιx red(x)

• bject(x)

Coordina.on(Duplicate( Coordina.on(Children( Head(Bigram( Head(Predicate( Head(Trigram(

Situated(Features(

Coverage(

G :

ιx.red(x) ∧ object(x)

Situated(Features(

SUB

Coverage(

G :

ιx.red(x) ∧ object(x)

Situated(Features(

SUB SPR ALL DISJ EMPTY OTHER

Coverage(

G :

ιx.red(x) ∧ object(x)

Structural(Features(

[ι, SUB] [∧, SUB] [color, SUB] [object, ALL]

ιx.red(x) ∧ object(x)

Head(Predicate(and((Coverage(

Situated(Features(

∧ ιx red(x)

• bject(x)

Head(Predicate(and((Coverage(

[∧; SUB, ALL]

Coordina.on(Child(Coverage(

Situated(Features(

∧ ιx red(x)

• bject(x)

Head(Predicate(and((Coverage( Coordina.on(Child(Coverage( Coordina.on(Rela.ve(Cov.(

Overview(

Space(of(Referring(Expressions( Probabilis.c(Model( Learning(

(Data( (Algorithm ((

Experiments( Results( Conclusion( ( (

Learning(–(Data(

{(Si, Gi, Zi) : i = 1 . . . n}

Learning(–(Data(

{(Si, Gi, Zi) : i = 1 . . . n}

• bj1: red, sphere, apple
• bj2: brown, triangle
• bj3: yellow, fries

… …

World(State(

Learning(–(Data(

{(Si, Gi, Zi) : i = 1 . . . n} Target(Set(

Learning(–(Data(

{(Si, Gi, Zi) : i = 1 . . . n}

Ex.¬(brown(x) ∨ blue(x)) ∧ object(x) ∧ sg(x)) ιx.¬(brown(x) ∨ blue(x)) ∧ object(x) ∧ plu(x) ιx.¬(brown(x) ∨ blue(x)) ∧ object(x) ∧ plu(x) ιx.(yellow(x) ∨ orange(x) ∨ red(x) ∨ green(x)) ∧ object(x) ∧ plu(x) ιx.(yellow(x) ∨ orange(x) ∨ red(x) ∨ green(x)) ∧ object(x) ∧ plu(x) ιx.(yellow(x) ∨ orange(x) ∨ red(x) ∨ green(x)) ∧ object(x) ∧ plu(x) . . . . . .

Labeled(Logical(Forms(

Learning(–(Data(

{(Si, Gi, Zi) : i = 1 . . . n}

ˆ Q(z | Si, Gi)

Qi

Empirical(Distribu.on:(

Ex.¬(brown(x) ∨ blue(x)) ∧ object(x) ∧ sg(x)) ιx.¬(brown(x) ∨ blue(x)) ∧ object(x) ∧ plu(x) ιx.(yellow(x) ∨ orange(x) ∨ red(x) ∨ green(x)) ∧ object(x) ∧ plu(x) 0.1( 0.3( 0.2(

. . . . . .

Learning(Algorithm(

• Online(
• Stochas.c(Gradient(Descent(

Learning(Algorithm(

For t = 1 . . . T, i = 1 . . . n: Step 1: (Update Global Model)

• a. Compute the stochastic gradient
• b. Update the parameters

Step 2: (Update Pruning Model) For j = 1 . . . M

• a. Construct a set of positive and negative examples
• b. Compute mini-batch stochastic gradient
• c. Update complexity-j pruning parameters

For t = 1 . . . T, i = 1 . . . n: Step 1: (Update Global Model)

• a. Compute the stochastic gradient

∆θ ← EQi(z|Si,Gi)[φi(z)] − E ˆ

P (z|Gi,Si;θ,Π)[φi(z)]

• b. Update the parameters

γ ←

α0 1+c×τ where τ = i + t × n

θ ← θ + γ∆θ Step 2: (Update Pruning Model)

Learning(Algorithm(

For t = 1 . . . T, i = 1 . . . n: Step 1: (Update Global Model) Step 2: (Update Pruning Model) For j = 1 . . . M

• a. Construct a set of positive and negative examples

D+ ← S

z∈Zi SUB(j, z).

D− ← Aj \ D+

• b. Compute mini-batch stochastic gradient
• c. Update complexity-j pruning parameters

Learning(Algorithm(

For t = 1 . . . T, i = 1 . . . n: Step 1: (Update Global Model) Step 2: (Update Pruning Model) For j = 1 . . . M

• a. Construct a set of positive and negative examples
• b. Compute mini-batch stochastic gradient

∆Πj ←

1 |D+|

P

z∈D+(1 − Pj(z | Si, Gi; Πj))φi(z)

−

1 |D−|

P

z∈D− Pj(z | Si, G; Πj)φi(z)

• c. Update complexity-j pruning parameters

Πj ← Πj + γ∆Πj

Learning(Algorithm(

Overview(

Space(of(Referring(Expressions( Probabilis.c(Model( Learning( Experiments( Results( Conclusion( ( (

Data(Collec.on(

• 269(scenes(

20(expressions(/(scene( 5380(expressions(

• Data(Split(

– Training:(

• 196(scenes((3920(exps)(
• Labeled(semiiautoma.cally(

– Dev:(

• 20(scenes((400(exps)(
• HandiLabeled(

– Test:(

• 43(scenes((860(exps)(
• HandiLabeled(

“Please(pick(up( _______________”(

Training(Data(

• SemiiAutoma.c(Labeling(

– Trained(seman.c(parser(on(ini.aliza.on(set((

• 10(scenes,(100(sentenceiexpression(pairs(

– Hand(engineered(lexicon( – 95%(precision,(70%(recall( – Labeled(196(scenes((3920(exps)( – Use(scenes(with(at(least(15(successful(labels(

• Total(training(set:(

– 141(scenes,(2587(expressions(

Related(Work(

• Most(previous(systems(are(determinis.c((

[Dale(and(Reiter(1995],([van(Deempter(2002],([Gardent(2002],([Horacek(2004],( [GaC(and(van(Deempter(2007],([Areces(et(al.(2008],([Ren(et(al.(2010],([Krahmer( and(van(Deempter(2012],([van(Deempter(et(al.(2012],(…(

• Learning(to(refer(to(a(single(objects(

– Compare(state(of(the(art(approach(( – Visual(Objects(Algorithm([Mitchell(et(al(2013](

• Learning(to(refer(to(sets(of(objects(

– Requires(more(complex(logical(expressions( – Present(the(first(learning(results(+(abla.ons(

Evalua.on(

• Mean(absolute(error((100(i(MAE)(

( (

• Coverage(

– All(logical(forms:( – Unique(logical(forms:(

• Topi1(Accuracy(

MAE = 1 2n

n

X

i=1

X

z∈Z

|P(z | Si, Gi) − Q(z | Si, Gi)|

%dup %uniq

Results(–(Single(Objects(

72.7( 92.5( 98.2( 60.3( 72.7( 100( 100( 74.2( 0( 20( 40( 60( 80( 100( Top1( %dup( %uniq( (100(i( MAE)( GenX( VOA(

0( 20( 40( 60( 80( 100( Top1( %uniq( %dup( (100iMAE)( GenX( NoPrune( NoCov( NoStruc( HeadExp(

Results(–(Object(Sets(

0( 20( 40( 60( 80( 100( Top1( %uniq( %dup( (100iMAE)( GenX( NoPrune( NoStruc( NoCov( HeadExp(

Results(–(Object(Sets(

Qualita.ve(Results(

Q ˆ P z .750 .320 ι(λx.object(x) ∧ (yellow(x) ∨ red(x))) .114 ι(λx.lego(x)) ∪ ι(λx.red(x) ∧ apple(x)) .114 ι(λx.yellow(x) ∧ lego(x))) ∪ ι(λx.apple(x)) .044 ι(λx.lego(x) ∨ (red(x) ∧ apple(x))) .044 ι(λx.(yellow(x) ∧ lego(x)) ∨ apple(x)) .036 ι(λx.lego(x)) ∪ ι(λx.red(x) ∧ sphere(x)) .026 ι(λx.red(x) ∧ lego(x)) ∪ ι(λx.red(x) ∧ sphere(x)) .050 .021 ι(λx.(lego(x) ∧ yellow(x)) ∨ (red(x) ∧ apple(x))) .017 ι(λx.(lego(x) ∧ yellow(x)) ∨ (red(x) ∧ sphere(x))) .014 ι(λx.yellow(x) ∧ lego(x)) ∪ ι(λx.red(x) ∧ sphere(x)) .100 .010 ι(λx.yellow(x) ∧ object(x)) ∪ ι(λx.apple(x)) .050 .007 ι(λx.yellow(x) ∧ object(x)) ∪ ι(λx.red(x) ∧ sphere(x)) .050 .005 ι(λx.yellow(x) ∧ object(x)) ∪ ι(λx.red(x) ∧ object(x))

Conclusion(

• Referring(Expression(Genera.on(as(Density(

Es.ma.on(

– Global(Model( – Learned(Pruning(Model(

• First(results(on(density(es.ma.on(for(set(

reference(

• Stateiofitheiart(results(on(single(objects(

Future(Work(

• Full(joint(approach(to(REG(

Future(Work(

• Full(joint(approach(to(REG(

``The(green( and(red( spheres.’’(

Physical(Scene( Logical(Form((LF)( Sentence(

ιx.(green(x) ∨ red(x)) ∧sphere(x)

Future(Work(

• Joint(approach(to(REG(
• Extend(to(more(general(referring(expressions(

“the(second(jar(to(the( leK(of(the(middle(shelf”(

Future(Work(

• Joint(approach(to(REG(
• Extend(to(more(general(referring(expressions(
• Extend(to(other(grounded(language(problems(

States

Abbr. Capital Pop.

AL Montgomery 3.9 AK Juneau 0.4 AZ Phoenix 2.7 WA Olympia 4.1 NY Albany 17.5 IL Springfield 11.4

``What(is(the( largest(state?’’( ``Move(to(the(chair( and(turn(right.’’(

Ques.ons?(

hCp://nfitz.net( ( Data:( (Available(Now( Code:( (Available(Soon(