0123,%%4 · drug: {low blood pressure, xanax, lamictal, generic birth control, lipitor, side...

!"#$"%&'()*•  +&,-*'%*,*."/,(01*23,%%4*

5*

!"#$"%&'()*•  +&,-*'%*,*."/,(01*23,%%4*•  6'7"*,(*"8,/93"*:#*%"/,(01*13,%%*

;*

!"#$"%&'()*•  +&,-*'%*,*."/,(01*23,%%4*•  6'7"*,(*"8,/93"*:#*%"/,(01*13,%%*

•  +&,-*'%*,(*'(%-,(1"4*

<*

!"#$"%&'()*•  +&,-*'%*,*."/,(01*23,%%4*•  6'7"*,(*"8,/93"*:#*%"/,(01*13,%%*

•  +&,-*'%*,(*'(%-,(1"4*

•  6'7"*,(*"8,/93"*:#*'(%-,(1"*

>*

!"#$"%&'()*•  +&,-*'%*,*."/,(01*23,%%4*•  6'7"*,(*"8,/93"*:#*%"/,(01*13,%%*

•  +&,-*'%*,(*'(%-,(1"4*

•  6'7"*,(*"8,/93"*:#*'(%-,(1"*

•  +&,-*'%*,(*:(-:3:)?4*

@*

!"#$"%&'()*•  +&,-*'%*,*."/,(01*23,%%4*•  6'7"*,(*"8,/93"*:#*%"/,(01*13,%%*

•  +&,-*'%*,(*'(%-,(1"4*

•  6'7"*,(*"8,/93"*:#*'(%-,(1"*

•  +&,-*'%*,(*:(-:3:)?4*

•  6'7"*,(*"8,/93"*:#*:(-:3:)?*

A*

B*

."/,(01*23,%%**

C  D"E('0:(F*G*%"/,(01*13,%%*1:(-,'(%*H:$I%*-&,-*%&,$"*,*%"/,(01*9$:9"$-?J*

C  K8,/93"F*C  !"#$%&'()*$&+),,$(-F*L!"#$%&'()*+&$)*M*,-.'/"-*0'NO*

C  ./--)0,&1$2$(03",&4505$-F*L1$2%3)-M*40%-)*M*56++)-'NO*

C  '$"6%$F*L&$-78$%M*+&69$*&M*3)%"M*#2:M*;)""'<=)&8'NO*

!"#$"%&'()*•  +&,-*'%*,*."/,(01*23,%%4*•  6'7"*,(*"8,/93"*:#*%"/,(01*13,%%*

•  +&,-*'%*,(*'(%-,(1"4*

•  6'7"*,(*"8,/93"*:#*'(%-,(1"*

•  +&,-*'%*,(*:(-:3:)?4*

•  6'7"*,(*"8,/93"*:#*:(-:3:)?*

•  +&?*I:*H"*(""I*-:*3",$(*%"/,(01*13,%%"%4*

P*

Q*

K/:0:(,3.-,-"*

R3?S:*T'7"U(*

SV%&'$-*

!"#$"%&'()*•  +&,-*'%*,*."/,(01*23,%%4*•  6'7"*,(*"8,/93"*:#*%"/,(01*13,%%*

•  +&,-*'%*,(*'(%-,(1"4*

•  6'7"*,(*"8,/93"*:#*'(%-,(1"*

•  +&,-*'%*,(*:(-:3:)?4*

•  6'7"*,(*"8,/93"*:#*:(-:3:)?*

•  +&?*I:*H"*(""I*-:*3",$(*%"/,(01*13,%%"%4*

•  +&,-*,993'1,0:(%*(""I*:(-:3:)'"%4*

;5*

S,%W*•  6'7"(*,*I:/,'(*:#*'(-"$"%-*,(I*%:/"*$"3,0:(%*•  T",$(*#$:/*-&"*+"X*H'-&*/'('/,3*%Y9"$7'%':(*

;;*

9":93"*

Z$J*.9'("%*S$,1?*

["-"$*

1&'3I$"(*&"$:*)$:H(Y9%* E10:(,3*1&,$,1-"$*

U(%-,(1"%*

."/,(01*23,%%"%*

S,8:(:/?*

!"1W3"%%\*]::-%-$,99'()*

;<*

•  U(%-,(0,-"*DG[*H'-&*>"-++?-=$*,(I*:("*@+$$9A''(%-,(1"*

'+&-&$+''%Y1&*,%''

* * * **

!"-#-=-' ,(I*\*B$.-+'

,)++)++)CC)'

!%0-*+-+'

•  R""I*-&"*("H3?*3",$("I*-"$/%*:(*@+$$9A*9:%'0:(*

•  2:(IY1-*,*X$",I-&VE$%-*%",$1&*

\-&"$"*'%*(:*1&"1W*,-*,(?*9:'(-*

T",$('()*2Y$7"%*

7,)80%-&

90-):&;$<$%&=",:$65-&

>,5$(8$2)05$&=",:$65-&

>5$(03",-&

?&"@&)5$8

-&%$0(,$

2& &),-50,:$-&

-),A$(-&-/:B&0-&!&0,2&"#

;=*

5JBP*

5JP*

5JP<*

5JP>*

5JPA*

5JPP*

5JQ*

5JQ<*

;* <* =* >*

&-),A$(-&-/:B&0-&!&0,2&"#

>5$(03",-&&6($:)-)",

&

JJQ5*

JJP5*

*[$:X3"/F*%",$1&*(""I%*)Y'I,(1"***.:3Y0:(F*$,(W*-&"*3",$("I*'(%-,(1"%*

^?9:(?/*[,_"$(*T'(W,)"*6$,9&*

•  ^[T6`abMKc*H&"$"*7"$-"8***********'%*,(*'(%-,(1"M*,(I********************'%*,(*"I)"*X"-H""(*-H:*'(%-,(1"%*

*7$%&-)*'$%&%'$D'$()*#&$#+,&-&.&#&/0'B$.-+D'+826"9'E2%#)9'C%-:$%+'&8-&'-%$'"$9''

•  S&"*H"')&-*F*:#*,(*"I)"*'%*-&"*#$"dY"(1?*H'-&*H&'1&*6*)"("$,-"I*G'

G3,X,/,* S"8,%*F`;@*

6' G'

;>*

D?(,/'1*6$,9&%*•  e%"$*&,%*3'/'-"I*,11"%%*-:*+"X*dY"$'"%*

•  f(* ",1&* '-"$,0:(* :(3?* -&"* &')&"%-* %1:$'()*Y("893:$"I*(:I"* '%* ,II"I* -:* -&"* -$Y%-"I* 13,%%*'(%-,(1"%*,(I*%""I"I*#:$*-&"*("8-*'-"$,0:(*

•  Z",%Y$"%*$"3?*:(*C2C6"-%)&:M*-&"*1,9,X'3'-?*:#*,*13,%%* '(%-,(1"* -:* X"* I'%1:7"$"I* X?* :-&"$* 13,%%*'(%-,(1"%*

;@*

.1:$'()*#Y(10:(%*

•  '(D")$""* a'(Dc*:#* ,*(:I"* aGc* '%* -&"* %Y/*:#* -&"*H"')&-%*:#*,33*'(1:/'()*"I)"%*a6MGcM*H&"$"*a6c*'%*,* -$Y%-"I* '(%-,(1"M* (:$/,3'g"I* X?* (Y/X"$* :#*(:I"%*'(*-&"*)$,9&*/'(Y%*:("**

•  S&"*/",%Y$"*$"3'"%*"(0$"3?*:(*9:9Y3,$'-?*

;A*

Q*

;@*

.1:$'()*#Y(10:(%*

•  ]"%-*"I)"*a]Kc*:#*,*(:I"*aGc*'%*-&"*/,8'/Y/*"I)"*H"')&-*,/:()*-&"*'(1:/'()*"I)"%*a6MGcM*H&"$"*6*'%*,*-$Y%-"I*'(%-,(1"*

;B*

Q*

;@*

P*B*

<*

;*>*

<*

.1:$'()*#Y(10:(%*

•  ]"%-*"I)"*a]Kc*:#*,*(:I"*aGc*'%*-&"*/,8'/Y/*"I)"*H"')&-*,/:()*-&"*'(1:/'()*"I)"%*a6MGcM*H&"$"*6*'%*,*-$Y%-"I*'(%-,(1"*

;P*

Q*

;@*

P*B*

<*

;*>*

<*

.1:$'()*#Y(10:(%*

•  h"?*[3,?"$*[$:X3"/*a*h[[c*:#*(:I"*aGc*'%*

* * * * * * * * * * * * * * * * * * * * M* &')&*h[[* '(I'1,-"%* %-$:()*1:(("107'-?* ,(I* 9$:8'/'-?* -:* -&"* $"%-* :#* -&"*(:I"%*'(*-&"*)$,9&*

;Q*

Q*

;@*P*

B*<*

[$"1:/9'3"I*6$,9&%*•  e%"$*&,%*Y(3'/'-"I*,11"%%*-:*+"X*dY"$'"%*

•  !"V$,(W'()* $Y(%* ,i"$* -&"* "8&,Y%07"*X::-%-$,99'()* %",$1&* :#* ,33* '(%-,(1"%* ,(I*1:(%'I"$%*,33*(:I"%*,(I*"I)"%*

•  Z",%Y$"%* $"3?* :(* C%2967HG)&:M* H&'1&* '%* -&"*,X'3'-?*:#*,(*'(%-,(1"*-:*I'%1:7"$*("H*'(%-,(1"%j*,%*H"33*,%*X:-&*-&"*C%2967HG)&:*,(I*C2C6"-%)&:'

<5*

!"V$,(W'()*#Y(10:(%*

•  :Y-D")$""*a:Y-Dc*:#*,*(:I"*aGc*'%*-&"*%Y/*:#*,33*:Y-):'()*"I)"%* #$:/*G*(:$/,3'g"I*X?*-&"*-:-,3*(Y/X"$*:#*(:I"%*'(*-&"*)$,9&*/'(Y%*:("*

•  S&"*/",%Y$"*$"3'"%*"(0$"3?*:(*9$:IY107'-?*

<;*

@*<*

B*;5*

;*<*

=*

Q*

P*

=*

!"V$,(W'()*#Y(10:(%*

•  -:-,3D")$""*a-:-c*:#*,*(:I"*aGc*'%*-&"*%Y/*:#*,33*'(D")$""*,(I*:Y-D")$""*"I)"%*:#*G*(:$/,3'g"I*X?*-:-,3*(Y/X"$*:#*(:I"%*/'(Y%*:("*

•  2,9-Y$"%*9$:IY107'-?*,(I*9:9Y3,$'-?*

<<*

@*<*

B*;5*

;*<*

=*

Q*

P*

=*

!"V$,(W'()*#Y(10:(%*

•  ]"-H""(("%%* a]Kc* 1:(%'I"$%* -&"* (Y/X"$* :#*%&:$-"%-* 9,-&%* -&,-* 9,%%* -&$:Y)&* ,* (:I"* GJ* G*(:I"* '%* '/9:$-,(-* H&"(* '-* '%* 13:%"* -:* /:%-*(:I"%*'(*-&"*("-H:$W*

•  [,)"!,(W*a[!c*:#*,*(:I"*aGc*'%**

***********************^",7?*(:I"%*,$"*/:$"*'/9:$-,(-*

<=*

T",$('()*2Y$7"*2Y-Vfk*

<>*

•  D"-"$/'("* ,* 9:'(-* '(*H&'1&F*-  ,33* '(%-,(1"%* :#* -&"*%"/,(01* 13,%%* ,$"*3",$("I*

-  -&"* ,3):$'-&/* '%* &')&3?*1:(EI"(-* ,X:Y-* -&"*dY,3'-?* :#* -&"* 3",$("I*'(#:$/,0:(*

•  2Y-V:k* '%* X,%"I* :(* g"$:*:Y-D")$""*

U(%-,(1"%*,-*$,(W*!*

[$"1'%':(*

D,-,*2:33"10:(*•  [$:1"IY$"F*

– Y%"*6::)3"*:$*l,&::m*]:%%*G[U*– 1:33"1-*,33*%('99"-%*/,-1&'()*-&"*dY"$?*– 13",(*%('99"-%*#$:/*8&="',(I*9,$-V:#V%9""1&*-,)*–  $Y(*9$:1"IY$"*#:$*-"(*'-"$,0:(%*

•  !"%Y3-%F*– 1:33"1-"I*n=6]*:#*I,-,*–  3",$("I*:7"$*n@M555*-"$/%*

<@*

."/,(01*23,%%"%*•  K893:$"*-&"*&,$7"%0()*9:H"$*:#*^[T6*H'-&*I'k"$"(-*%'g"*13,%%"%*

– 13:%"IM*%/,33*,(I*X')*•  e.G*%-,-"%*a@5*"3"/"(-%c ! •  1:Y(-$?*a*;Q>*"3"/"(-%c**

– :9"(*,(I*X')**•  E%&*a):3I*%-,(I,$I*+'W'9"I',c*

•  %'()"$%*a/,(Y,33?*$"7'"H"Ic*

•  ,('/,3%*a/,(Y,33?*$"7'"H"Ic*•  9":93"*a/,(Y,33?*$"7'"H"Ic*

<A*

K7,3Y,0:(*/",%Y$"%\*

\*,33*'(%-,(1"%*'(*-&"*)"("$,-"I*3'%-%*,$"*Y('dY"*<B*

["$#:$/,(1"*

<P*

4505$-&

[:9Y3,$'-?*

op* ]K* h[[* '(D*

<@* ;J5* ;J5* ;J5&

@5* JQA* JQP* JQP*

A>* JBB* JBP* JBB*

(Y/X"$*:#*3",$("I*'(%-,(1"%*

]K*q*X"%-*"I)"*h[[*q*W"?*93,?"$*9$:X3"/*'(D*q*'(VD")$""**

["$#:$/,(1"*

<Q*

4505$-&

[:9Y3,$'-?* [:9r[$I*

op* ]K* h[[* '(D* -:-D* ]S* [!*

<@* ;J5* ;J5* ;J5* CDE& JPP* JPP*

@5* JQA* JQP* JQP* CDE& JPA* JP<*

A>* JBB* JBP* JBB* JBP* JBB* JAB*

]K*q*X"%-*"I)"*h[[*q*W"?*93,?"$*9$:X3"/*'(D*q*'(VD")$""**-:-D*q*-:-,3*I")$""*]S*q*X"-H""(("%%*[!*q*[,)"*!,(W*

["$#:$/,(1"*

=5*

4505$-&

[:9Y3,$'-?* [:9r[$I* [$I*

op* ]K* h[[* '(D* -:-D* ]S* [!* :Y-D*

<@* ;J5* ;J5* ;J5* CDE& JPP* JPP* CDE&

@5* JQA* JQP* JQP* CDE& JPA* JP<* CDE&

A>* JBB* JBP* JBB* JBP* JBB* JAB* JBP*

•  ^[T6%*9"$#:$/*X"_"$*-&,(*X::-%-$,99'()*

•  :Y-D*,(I*-:-D*I'%1:7"$*,33*%-,-"*/"/X"$%*

["$#:$/,(1"*

=;*

4505$-&

[:9Y3,$'-?* [:9r[$I* [$I*

op* ]K* h[[* '(D* -:-D* ]S* [!* :Y-D*

<@* ;J5* ;J5* ;J5* CDE& JPP* JPP* CDE&

@5* JQA* JQP* JQP* CDE& JPA* JP<* CDE&

A>* JBB* JBP* JBB* JBP* JBB* JAB* JBP*

•  ^[T6%*9"$#:$/*X"_"$*-&,(*X::-%-$,99'()*

•  :Y-D*,(I*-:-D*I'%1:7"$*,33*%-,-"*/"/X"$%*

•  '#*-&"$"*,$"*:(3?*@5*e.*%-,-"%M*H&?*I:"%*-&"*,3):$'-&/**

*****W""9*:(*3",$('()*

S&"*S$:YX3"%:/"*R:Y$-""(*

•  U(%-,(1"%*,i"$*-&"*3",$("I*@5*e.*%-,-"%F** !Y%%',M* eW$,'("M* egX"W'%-,(M* Gg"$X,'s,(M* Z:3I,7,M*S,s'W'%-,(M* G$/"(',M* 2&'1,):M* ]:%-:(M* G-3,(-,M* D"-$:'-M*[&'3,I"39&',M*S,/9,M*Z:3I,7',*

=<*

S&"*S$:YX3"%:/"*R:Y$-""(*

•  U(%-,(1"%*,i"$*-&"*3",$("I*@5*e.*%-,-"%F** !Y%%',M* eW$,'("M* egX"W'%-,(M* Gg"$X,'s,(M* Z:3I,7,M*S,s'W'%-,(M* G$/"(',M* 2&'1,):M* ]:%-:(M* G-3,(-,M* D"-$:'-M*[&'3,I"39&',M*S,/9,M*Z:3I,7',*

==*

t,Y-&:$'-,$',(*@"(8$(&4"<)$5&-505$-*%Y1&*,%*F$"(A)0&,(I*GH(0),$u*

tR'(I3,?*&,%*<5*$"%-,Y$,(-%*'(*%-,-"%*%Y1&*,%*1%"()20&,(I*=B):0A"u&

["$#:$/,(1"*

=>*

1)-B&

[:9* [$I*

p* h[[* :Y-D*

;5* JQ5* CDE&

<@* JPP* CDE&

@5* JP5* CDE&

B@* JAQ* JQ=*

;55* JAP* JP>*

;;A* JA@* JP5*

["$#:$/,(1"*

=@*

1)-B&

[:9* [$I*

p* h[[* :Y-D*

;5* JQ5* CDE&

<@* JPP* CDE&

@5* JP5* CDE&

B@* JAQ* JQ=*

;55* JAP* JP>*

;;A* JA@* JP5*

4),A$(-&

[:9* [$I*

p* '(D* :Y-D*

;5* JQ<* CDE&

<@* JQ;* CDE&

@5* JQ<* DIJ&

B@* JQ;* JQA*

;55* JPQ* JQA*

;@5* JPP* JQ@*

;P5* JPB* JQ;*

["$#:$/,(1"*

=A*

1)-B&

[:9* [$I*

p* h[[* :Y-D*

;5* JQ5* CDE&

<@* JPP* CDE&

@5* JP5* CDE&

B@* JAQ* JQ=*

;55* JAP* JP>*

;;A* JA@* JP5*

4),A$(-&

[:9* [$I*

p* '(D* :Y-D*

;5* JQ<* CDE&

<@* JQ;* CDE&

@5* JQ<* DIJ&

B@* JQ;* JQA*

;55* JPQ* JQA*

;@5* JPP* JQ@*

;P5* JPB* JQ;*

="/,5()$-&

[:9* [$I*

p* '(D* :Y-D*

@5* JQP* CDE&

;55* JQ>* CDE&

;@5* JQ;* CDE&

<55* JP=* JQ5*

=55* JA;* JA;*

=<=* J@B* J@B*

K$$:$*G(,3?%'%*•  S?9"*;F*'(1:$$"1-*9$:9"$*(,/"*"8-$,10:(*

'I+&-&$+'+678'-+'1$2%3)-'-*9'(*3")+8'+C$-0)*3'726*&%)$+J'

•  S?9"*<F*'(%-,(1"%*-&,-*#:$/"$3?*X"3:()"I*-:*-&"*%"/,(01*13,%%*' ''I<$%#)-K,2*&$*$3%2JM'I>L$782+"2G-0)-J'

•  S?9"*=F*%9"33'()*7,$',(-%** *IM:%3:+&-*J'7%'IM:%3:L8+&-*J'

•  S?9"*>F*%"(-"(1"%*H'-&*H$:()*#,1-Y,3*,%%"$0:(%** *I)*96+&%:')*'726*&%)$+'+678'-+'N%-*7$'-*9'?2%&8'!=$%)7-J'

•  S?9"*@F*X$:W"(*"89$"%%':(%**********I)++6$'8-+'#$$*'&2638'E2%'+&-&$+'+678'-+'B$.-+'-*9#?$FJ**

=B*

2:/9,$'%:(*DG[*,(I*h(:HU-G33*

•  ]:-&*%?%-"/*$",1&*1:/9,$,X3"*9"$#:$/,(1"*•  p:-"*h(:HU-G33*aK-g':('*"-*,3JM5@c**

–  Y%"I*%'()3?V,(1&:$"I*9,_"$(%**–  $"3'"I*:(*/Y3093"*9,_"$(%*#:$*-&"*%,/"*%"/,(01*13,%%*–  $"dY'$"I*,%*'(9Y-*%?(:(?/%*:#*-&"*%"/,(01*13,%%*–  "8-$,1-"I*'(%-,(1"%*#$:/*;555*%('99"-%*

=P*

:"/,5(K& L,"M>57%%&C& L,"M>57%%&N& "/5O$A($$&

[$"1'%':(* BQv* QBv* ;55v*

!"1,33* PQv* @Pv* BBv*

U(%-,(1"*K8-$,10:(*#$:/*wY"$?*T:)%*•  6'7"(F*

*****V*G*%"-*:#*dY"$?*3:)%*****************V*G*%"-*:#*'(%-,(1"%*:#*13,%%*>'

•  .-"9;F*K8-$,1-*dY"$?*-"/93,-"%*/,-1&'()*%""I*'(%-,(1"%*

•  .-"9<F*6"("$,-"*,*9::3*:#*1,(I'I,-"*'(%-,(1"%*

=Q*

[buy] [in uk]postfixprefix

[can] [make you tired]postfixprefix

postfix[long term] [use]

prefix


prefix

[how does] [work]postfixprefix

[does] [cause weight gain]postfixprefix





prefix



Query logs

Target classes (seed instances)

side effects of generic birth control pills can low blood pressure make you tired

prescription vicodin online long term xanax use propanolol and vicodin interaction

buy xanax in uk taking beta blockers during pregnancy how oxycontin works

can lamictal be crushed how does lipitor work effect of beta blockers on exercise

Pool of candidate instances

Search!signature vectors (one per candidate instance)

long term lamictal use propanolol and lamictal interaction is xanax habit forming

vicodin, beta blockers, ...}

causes of low blood pressure during pregnancy taking lipitor during pregnancy

Drug: xanax

[causes of] [during pregnancy]prefix postfixDrug: low blood pressure

Drug: vioxx

Drug: lamictal postfixprefix[can] [be crushed]

does vioxx cause weight gain can phentermine make you tired buy vioxx in uk

Ranked list of instancesDrug: {viagra, phentermine, ambien, adderall, vicodin, hydrocodone, xanax, vioxx, oxycontin, cialis, valium, lexapro, ritalin, zoloft, percocet, ...}

(2)

(3)

(4)

(5)

Drug: {low blood pressure, xanax, lamictal, generic birth control, lipitor,

side effects of viagra pills long term vioxx use prescription lamictal onlinelong term xanax use what effects does low blood pressure have buy lipitor online

Reference search!signature vectors (one per class)

Drug

Query templates matching seed instances

[side effects of] [pills]postfixprefix

(1)

Drug: {phentermine, viagra, vicodin, vioxx, xanax}

[side effects of] [pills]

[does] [cause weight gain]postfix

[prescription] [online]postfix

prefix

prefix

postfixprefix

Figure 1: Overview of data flow during weakly supervised extraction of named entities from query logs

itly derive signals improving the quality of various tasks suchas information retrieval, whether through re-ranking of theretrieved documents [21], query expansion [5], or the devel-opment of spelling correction models [8]. Conversely, previ-ous studies in large-scale information extraction, includingnamed entity discovery, uniformly choose to capitalize ondocument collections [10] rather than queries as preferreddata source, thus failing to take advantage of the collec-tive knowledge embedded haphazardly within noisy searchqueries.

The proposed extraction method is guided by a small setof seed instances, without any need for handcrafted extrac-tion patterns or further domain-specific knowledge. To en-sure varied experimentation on several dimensions despitethe burden of manually assessing the correctness of the out-put, the evaluation computes the precision of instances ex-tracted for ten di!erent target classes (City, Drug etc.). Theclasses constitute a realistic sample of the wide range ofdomains of interest to Web search users. One of the pre-requisites of the evaluation, illustrating its scope and time-intensive nature, is the manual assessment of the correctnessof more than 4,000 candidate instances. The resulting pre-cision scores are significant both in absolute value (0.96 forprec@50, 0.90 for prec@150, and 0.80 for prec@250), andrelative to the quality of instances extracted from Web doc-uments based on handcrafted patterns (with precision im-proving by 29% for prec@50, 26% for prec@150, and 15%for prec@250).

2. EXTRACTION FROM QUERY LOGS

2.1 Selection of Candidate InstancesGiven a set of target classes and a set of seed instances

for each class, the goal is to extract relevant class instancesfrom query logs, without any further domain knowledge.

As shown in Figure 1, the extraction method consists offive stages: identification of query templates that match theseed instances (Step 1); identification of candidate instances(Step 2); internal representation of candidate instances (Step3) and seed instances (Step 4); and instance ranking (Step5).

A target class (e.g., Drug), for which instances need tobe extracted, is available in the form of a set of represen-tative instances (e.g., phentermine and vioxx). In Step 1,each query that contains a seed instance generates a querytemplate composed of the remainder of the query, that is,the prefix and the postfix around the matched instance. Forexample, the occurrence of the instance vioxx in the query“long term vioxx use” corresponds to the query template[long term]prefix [use]postfix. During matching, all stringcomparisons are case-insensitive.

In another pass over the query logs, Step 2 collects a verylarge (and noisy) pool of candidate instances, by identify-ing the queries (e.g., “long term xanax use”) which matchany of the query templates generated in the previous step,and collecting the query infixes (e.g., xanax) as candidateinstances.

2.2 Representation through Search SignaturesStep 3 (see Figure 1) builds an internal representation

of each candidate instance. Whenever a query contains acandidate instance collected in Step 2, the remainder of thequery, that is, the concatenation of the remaining prefix andpostfix, becomes an entry in a query template vector thatacts as a search signature of the candidate instance withrespect to the class. The weight of each entry in a vectoris the frequency of occurrence of the contributing query inthe query logs. For instance, the query “can lamictal becrushed” results in a new entry being added to the search-signature vector of lamictal with respect to the class Drug.The new entry corresponds to the query template [can]prefix

684




prefix


prefix







prefix



Query logs











Drug: xanax


Drug: vioxx




(2)

(3)

(4)

(5)


side effects of viagra pills long term vioxx use prescription lamictal online

long term xanax use what effects does low blood pressure have buy lipitor online


Drug



(1)





prefix

prefix

postfixprefix



The proposed extraction method is guided by a small setof seed instances, without any need for handcrafted extrac-tion patterns or further domain-specific knowledge. To en-sure varied experimentation on several dimensions despitethe burden of manually assessing the correctness of the out-put, the evaluation computes the precision of instances ex-tracted for ten di!erent target classes (City, Drug etc.). Theclasses constitute a realistic sample of the wide range ofdomains of interest to Web search users. One of the pre-requisites of the evaluation, illustrating its scope and time-intensive nature, is the manual assessment of the correctnessof more than 4,000 candidate instances. The resulting pre-cision scores are significant both in absolute value (0.96 forprec@50, 0.90 for prec@150, and 0.80 for prec@250), andrelative to the quality of instances extracted from Web doc-uments based on handcrafted patterns (with precision im-proving by 29% for prec@50, 26% for prec@150, and 15%for prec@250).









684




prefix


prefix







prefix



Query logs











Drug: xanax


Drug: vioxx




(2)

(3)

(4)

(5)





Drug



(1)





prefix

prefix

postfixprefix



The proposed extraction method is guided by a small setof seed instances, without any need for handcrafted extrac-tion patterns or further domain-specific knowledge. To en-sure varied experimentation on several dimensions despitethe burden of manually assessing the correctness of the out-put, the evaluation computes the precision of instances ex-tracted for ten di erent target classes (City, Drug etc.). Theclasses constitute a realistic sample of the wide range ofdomains of interest to Web search users. One of the pre-requisites of the evaluation, illustrating its scope and time-intensive nature, is the manual assessment of the correctnessof more than 4,000 candidate instances. The resulting pre-cision scores are significant both in absolute value (0.96 forprec@50, 0.90 for prec@150, and 0.80 for prec@250), andrelative to the quality of instances extracted from Web doc-uments based on handcrafted patterns (with precision im-proving by 29% for prec@50, 26% for prec@150, and 15%for prec@250).









684




prefix


prefix







prefix



Query logs











Drug: xanax


Drug: vioxx




(2)

(3)

(4)

(5)





Drug



(1)





prefix

prefix

postfixprefix












684

[,%1,M5B*

U(%-,(1"*K8-$,10:(*#$:/*wY"$?*T:)%*•  .-"9=F*]Y'3I*,*dY"$?*-"/93,-"*7"1-:$*-&,-*,1-%*,%*,*%",$1&*%')(,-Y$"*:#*-&"*1,(I'I,-"*'(%-,(1"*#:$*13,%%*>*

•  .-"9>F*6"("$,-"*:("*$"#"$"(1"*%",$1&*%')(,-Y$"*7"1-:$*#:$*",1&*13,%%*>*

>5*




prefix


prefix







prefix



Query logs











Drug: xanax


Drug: vioxx




(2)

(3)

(4)

(5)





Drug



(1)





prefix

prefix

postfixprefix












684




prefix


prefix







prefix



Query logs


side effects of generic birth control pills can low blood pressure make you tiredprescription vicodin online long term xanax use propanolol and vicodin interaction








Drug: xanax


Drug: vioxx




(2)

(3)

(4)

(5)


side effects of viagra pills long term vioxx use prescription lamictal onlinelong term xanax use what effects does low blood pressure have buy lipitor online


Drug



(1)





prefix

prefix

postfixprefix












684

[,%1,M5B*

U(%-,(1"*K8-$,10:(*#$:/*wY"$?*T:)%*•  .-"9@F*Z",%Y$"*%'/'3,$'-?*X"-H""(*%",$1&*%')(,-Y$"*7"1-:$*:#*",1&*1,(I'I,-"*'(%-,(1"*,(I*-&"*$"#"$"(1"*%",$1&*%')(,-Y$"*7"1-:$*:#*-&"*13,%%*>'

•  !,(W*9$:IY1"I*:Y-9Y-*:#*'(%-,(1"%*#:$*",1&*13,%%*>'

>;*

0

0.2

0.4

0.6

0.8

1

25 50 100 150 250

Prec

isio

n

Rank

Class: City

D-pattQ-seed

0

0.2

0.4

0.6

0.8

1

25 50 100 150 250

Prec

isio

n

Rank

Class: Country

D-pattQ-seed

0

0.2

0.4

0.6

0.8

1

25 50 100 150 250

Prec

isio

n

Rank

Class: Drug

D-pattQ-seed

0

0.2

0.4

0.6

0.8

1

25 50 100 150 250

Prec

isio

n

Rank

Class: Food

D-pattQ-seed

0

0.2

0.4

0.6

0.8

1

25 50 100 150 250

Prec

isio

n

Rank

Class: Location

D-pattQ-seed

0

0.2

0.4

0.6

0.8

1

25 50 100 150 250

Prec

isio

n

Rank

Class: Movie

D-pattQ-seed

0

0.2

0.4

0.6

0.8

1

25 50 100 150 250

Prec

isio

n

Rank

Class: Newspaper

D-pattQ-seed

0

0.2

0.4

0.6

0.8

1

25 50 100 150 250

Prec

isio

n

Rank

Class: Person

D-pattQ-seed

0

0.2

0.4

0.6

0.8

1

25 50 100 150 250

Prec

isio

n

Rank

Class: University

D-pattQ-seed

0

0.2

0.4

0.6

0.8

1

25 50 100 150 250

Prec

isio

n

Rank

Class: VideoGame

D-pattQ-seed

0.6

0.7

0.8

0.9

1

25 50 100 150 250

Prec

isio

n

Rank

Class: Average-Class

D-pattQ-seed

Figure 3: Extraction from Web documents with handcrafted patterns (Dpatt), vs. seed-based extraction fromqueries (Qseed) proposed in this paper, for individual target classes (first ten graphs) and as an average overall target classes (last graph)

respectively). In contrast, the scores at the same rank 50vary between 0.86 and 1.00 in the case of Qseed, as shownearlier in Table 3.

Figure 3 plots precision values for ranks 1 through 250for pattern-based extraction from Web documents (Dpatt)against seed-based extraction from search queries (Qseed).The first ten graphs correspond to each of the ten target

classes. The output of Qseed is better for most target classes,with the exception of the classes City and Newspaper forwhich Dpatt gives better precision at higher ranks, and Coun-try for which Dpatt is better across all ranks. The last graphin Figure 3 shows the comparative precision as an averageover all target classes. Overall, the Qseed method proposedin this paper outperforms Dpatt, with relative precision score

688

["$#:$/,(1"*:#*wY"$?*T:)%**7%J*[,_"$(VX,%"I*,99$:,1&*:#*a[,%1,M5>c*

[,%1,M5B*

2:/9,$'%:(*DG[*,(I*wY"$?*T:)%*

><*

A$,$(05$2&),-50,:$-&P:"/,5(KQ&

'0-:0&EJ&P6($:)-)",Q&

"/5O$A($$&P6($:)-)",Q&

;55* Q@v* ;55v*

;@5* P<v* ;55v*

•  D:YX3?V,(1&:$"I*9,_"$(* aDG[c*1:/X'("I*H'-&* -&"*)$,9&*'(%-,(1"*$,(W'()*9$:IY1"I*/:$"*,11Y$,-"*'(%-,(1"*3'%-*

•  a[,%1,M5Bc* ,3):$'-&/* -,W"%* ,* %-"9* #Y$-&"$* X"'()* ,X3"* -:*)"("$,-"*("H*9,_"$(%*#$:/*-&"*3",$("I*dY"$?*-"/93,-"%*

Country: Drug: VideoGame:what type of government does I have how long does I stay in your system how many copies of I have been soldwhat do people in I eat can I be crushed how much does I costwhat is the weather like in I in march what does the I pill look like where can i play I online for freehow to apply for I visa how much does I cost how to install I modshow did I gain independence how does I a!ect the heart when is I coming out on gamecubewhere is I on the map how is I manufactured how many I levelswhat continent is I on does I cause weight gain what copy protection does I usewhat is I ’s currency when was I fda approved why does I crashwhy did I join the eu can I make you tired how to add bots to I serverwhy is I poor what I is made out of who made I 2

Table 1: Examples of query templates from the reference search-signature vectors generated during namedentity discovery (I stands for a class instance)

[be crushed]postfix. The weight of the new entry is set to thefrequency of the query.

Step 4 of Figure 1 injects weak supervision in the extrac-tion process, through the small input set of per-class in-stances. For each class (e.g., Drug), the vectors generatedin Step 3 are inspected to identify those associated to a seedinstance (e.g., xanax and vioxx), rather than any other candi-date instance. The vectors thus selected are added together,forming a reference search-signature vector. Whereas Step 3generates a vector for each candidate instance of each class,Step 4 produces one reference search-signature vector foreach class. A reference vector can be thought of as a loosesearch fingerprint of the desired output type with respect tothe class [12].

In Step 5, the similarity scores among the search-signaturevector of each candidate instance, on one hand, and thereference search-signature vector of the class, on the otherhand, determine the ordered list of candidate instances alongwith their ranking. The similarity scores are computed basedon a similarity function, namely Jensen-Shannon, that isused frequently in text processing [7].

One of the possible interpretations of the reference search-signature vectors of a class is to view them as a series ofqueries (or questions, when formulated in natural language)that can be asked about the instances in the class. Table 1shows some of the query templates in the reference search-signature vectors collected during named entity discovery fora sample of the target classes. Note how it is incrementallyeasier to guess to which class an instance belongs, solely byinspecting more questions that various people (users) askabout the instance. For example, if valid questions aboutan instance I include“what type of government does I have”and “what do people in I eat”, then it is relatively easy forpeople to guess that I must be a geopolitical entity of somesort, and later refine that guess as they are presented withmore questions about the instance. The intuition behindnamed entity discovery from query logs is similar: given aset of candidate phrases, the system must guess which of thecandidate phrases are more likely to belong to the targetclass, by looking at queries that can be asked about thecandidate phrase, and more generally about all instances inthe target class.

3. EXPERIMENTAL SETTING

3.1 DataThe input to the experiments is a random sample of around

50 million unique, fully-anonymized queries in English sub-

mitted by Web users to the Google search engine in 2006.All queries are considered independently of one another,whether they were submitted by the same user or di!erentusers, within the same or di!erent search sessions.

3.2 Target ClassesTo better test the ability of the extraction method to dis-

cover useful named entities, each target class is specifiedthrough only five seed instances, which are case-insensitive:

• City: {london, paris, san francisco, tokyo, toronto};• Country: {argentina, china, costa rica, france, germany};• Drug: {phentermine, viagra, vicodin, vioxx, xanax};• Food: {chicken, fish, milk, tomatoes, wheat};• Location: {argentina, china, france, san francisco, tokyo};• Movie: {die hard, sin city, star trek, star wars, the ma-

trix};• Newspaper: {le monde, new york times, usa today, wall

street journal, washington post};• Person: {mel gibson, leonardo da vinci, jennifer lopez,

pablo picasso, vincent van gogh};• University: {columbia university, stanford university,

university of chicago, university of pennsylvania, universityof texas at austin};

• VideoGame: {grand theft auto, need for speed, quake,super mario bros., warcraft}.

The resulting set of ten target classes is a mix of coarse-grained classes (Person and Location) used heavily in liter-ature on named entity discovery and extraction, and finer-grained classes (e.g., Drug and VideoGame) that more real-istically model the inherent diversity of Web documents andqueries. Furthermore, the classes di!er from one anotherwith respect to domains of interest (e.g., Health for Drugvs. Entertainment for Movie). Overall, the target classesprovide varied experimentation on several dimensions, whiletaking into account the time intensive nature of manual ac-curacy judgments often required in the evaluation of infor-mation extraction systems [2].

3.3 Evaluation ProcedureMultiple lists of candidate instances (named entities) are

evaluated for each class. The top 250 elements of eachoutput list to be evaluated are sorted alphabetically intoa merged list. Each instance of the merged list is manuallyassigned a correctness label with two possible values (corrector incorrect) within its respective class. Thus, a correctnesslabel is manually assigned to a total of 4,468 instances ex-tracted for the ten target classes, in a process that once againconfirms that evaluation of information extraction methodscan be quite time consuming.

685

(.O'

K7,3Y,0:(F*DG[*,),'(%-*+:$Ip"-*

>=*

?&B0(<$-5$2& '(+!& '(RGS& !"5>,+!&

G('/,3%& Q;=& JBQ& JB;& >P&

[":93"& ;=>>& J<=& JQ@& QPA&

7,)80%-& '$"6%$&

.0,H& .0,H&'($:)-)",&

'($:)-)",&

D'%1Y%%':(*:(*D:YX3?V,(1&:$"I*[,_"$(%*

>>*

•  b"$?*%'/93"*-"1&('dY"M*?"-*7"$?*"x1'"(-*•  DG[* $"dY'$"%* /'('/Y/* '(9Y-F* ,* %"/,(01*13,%%M*,*9,_"$(*,(I*:(3?*:("*%""I*"8,/93"*

•  ^[T6* )$,9&* $,(W'()* 9"$#:$/* X"_"$* -&,(*$"1W3"%%*X::-%-$,99'()*

•  :Y-D")$""* :Y-9"$#:$/%* ,33* /",%Y$"%* ,1$:%%*-&"*I'k"$"(-*1,-"):$'"%*

•  DG[* "8-$,1-%* /:$"* %"/,(01* 13,%%* '(%-,(1"%*H'-&* X"_"$* dY,3'-?* -&,(* .G[* ,(I* "8'%0()*9,_"$(*X,%"I*,99$:,1&"%*:(*-&"*+"X*

4TS7!U>=&=;744&R7.VT4U>!F&.-"9*<*

>@*

^:H*-:*3",$(*."/,(01*23,%%"%\4*

>A*

•  a[,(-"3*"-*,3JM5Ac*13Y%-"$*%'/'3,$*'(%-,(1"%*,(I*/,9*-:*&Y/,(V/,I"*$"9:%'-:$'"%*3'W"*+:$Ip"-*

•  a!'_"$* "-* ,3JM5Qc* Y%"* ^",$%-y%* 3"8'1:V%?(-,101*9,_"$(%*,(I*^ZZ*/:I"3*-:*'(1$",%"*$"1,33*

:$,()"*

9",$*

,993"*

/"3:(*(Y-%*

:$,()"*\*%Y1&*,%*

/:X'3"*:9"$,-:$*

1:3:$*

#$Y'-*

1:Y(-?*

\,*-"$/*H'-&*,*X$:,I"$*/",('()*-&,-*/:$"*%9"1'E1*-"$/%*#,33*Y(I"$*

[:H"$*:#*DG[V;*

>B*

•  e%"* -&"* (,-Y$,3* 9,'$'()* :#* -&"* 3",$("I*>"-++P*+&-*7$* 9,'$%* H'-&* -&"* DG[* 9,_"$(* -:*,Y-:/,01,33?* 3",$(* %"/,(01* 13,%%"%*a&?9"$(?/%c*

' '''''''''Q'+678'-+''

* * * **

:$,()"*,(I*)$,9"#$Y'-*#$Y'-%*

DG[V;*."/,(01*23,%%*^,$7"%0()*

•  R""I*-&"*'(%-,(1"*9,'$%*'(-:*,(*'(7"$%"*DG[V;** * *zz*\*%Y1&*,%*{,$)Y/"(-;|*,(I*{,$)Y/"(-<|u*

•  .YX/'-*,%*,*dY"$?*-:*l,&::m*

>P*

%Y1&*,%*

),2)<)2/0%-&6$(@"(8$(-&

W&

60($,5-&80%$-&

S0(K&S02",,0&X#080&

T%5",&Y"B,&,(I*Y"B,&,(I*

Y"B,&,(I*W&

."/,(01*23,%%*!,(W'()*

•  ]Y'3I*,*I'$"1-"I*)$,9&*Y%'()*),&$$12/$%&/)'312/$%&/)'4#-$'93"%*

>Q*

•  !,(W*"3"/"(-%*•  '(D")$""* :#* ,* &?9"$(?/* (:I"* aGRRc* '%* -&"* %Y/* :#* ,33*

'(1:/'()* "I)"%* #$:/* -&"* '(%-,(1"* 9,'$%* -:H,$I%* GRR*(:$/,3'g"I*X?*byyV;*

D"3-,*

9":93"*/,3"%* 9,$"(-%*

,'$*1,$$'"$%*E)Y$"%* :-&"$%*

["-"$*

D"3-,*

K3-:(*}:&(*

K//,*

!:%"*[,Y3*

Z,I:((,*

}:&(*Z,$?*

."/,(01*23,%%*D,-,*2:33"10:(*•  ."/,(01*23,%%"%F**

– G('/,3%*–  [":93"*

•  [$:1"IY$"F**–  ."(-*DG[V;*dY"$'"%*-:*6::)3"*al,&::m*]:%%c*–  2:33"1-"I*;555*%('99"-%*9"$*dY"$?M*W"9-*:(3?*Y('dY"*,(%H"$%*

•  !"%Y3-%F*–  T",$("I*;J;*6]*:#*%('99"-%*#:$*G('/,3%*,(I*;J@*6]*#:$*[":93"**

@5*

^:H*-:*K7,3Y,-"4*

@;*

•  R:Y(I*%-,))"$'()*7,$'"-?*:#*-"$/%*

•  ZY1&*/:$"*I'7"$%"*-&,(*"89"1-"IF**– [$:X,X3?*Y%"#Y3F*"-#2%-&2%:'-*)=-"+D'E2%$+&'

9F$""$%+D'$*9-*3$%$9'+C$7)$+'S'*

– G$"*-&"?*$",33?*,('/,34F'#-)&D'-""$%3$*+D'+$-E229D'G$7&2%+D'C%2&$)*D'C$+&+'S'''

– +&,-*-:*I:4F**-HG$'-*)=-"+D'"-%3$'=-==-"+&

TZ086%$-&"@&%$0(,$2&4/6$(5K6$&=",:$65-&@"(&7,)80%-[''-=C8)#)-*+D'

-%-78*)9+D'-%$-D'E$")*$+D'T+8$+D'E229D'E2F"D'3%-L$%+D'8$%#)G2%$+D'8$%9'

-*)=-"+D'86*&$%+D')*+$7HG2%$+D')*G$%&$#%-&$+D'"-#2%-&2%:'-*)=-"+D'F-&$%'

-*)=-"+D'F$&"-*9+D''L22'-*)=-"+M*N*

K7,3Y,0:(*+:"%*•  [:'(-*:#*:Y$*H:$W*'%*-:*3",$(*/:$"~("H*1:(1"9-%*-&,(*1Y$$"(-3?*,7,'3,X3"*

•  Z,(?*:#*-&"*3",$("I*-"$/%*,$"*/'%%'()*#$:/*+:$Ip"-*"7"(*-&:Y)&*-&"?*,$"*%"(%'X3"*,(I*9:-"(0,33?*7,3Y,X3"*

•  fY$*%-$,-")?F**–  1:Y(-*:(3?*1:$$"1-*13,%%"%*$"3,-"I*-:*-&"*G('/,3*%"(%"*L\0A/0(*U.GF*:0(,)<"($M*80880%M*-:?M*%9:$-%V-",/M*1,$V/,W"M*:9"$,0()V%?%-"/O*

–  1:/9,$"*,),'(%-*+:$Ip"-*,(I*I:*/,(Y,3*"7,3Y,0:(**

@<*

@=*

•  [,'I*#:Y$*,((:-,-:$%*a%-YI"(-%*#$:/*3'()Y'%01%c*

•  2$",-"I*I"-,'3"I*)Y'I"3'("%*#:$*G('/,3*,(I*[":93"*%"/,(01*13,%%*,((:-,0:(*

•  e%"I*-&"*2:I'()*G(,3?%'%*S::3W'-*a&_9F~~1,-JY1%Y$J9'_J"IY~c*

^Y/,(V],%"I*G((:-,0:(*

7,)80%-& '$"6%$&

2:$$"1-* $"903"M*/,//,3M*13:7"(V&::#"I*,('/,3M*9"-M*9,1WM*9,$,%'-"M*&"$I*

2,Y1,%',(M*%-Y_"$"$M*%1&'g:9&$"('1M*,Y(-M*&"$:M*,-&"'%-*

]:$I"$3'("* I$,):(M*1$'_"$** X'X3'1,3*E)Y$"%M*1:Y93"%*

],%'1T"7"3~U(%-,(1"%* I:)M*&Y//'()X'$I* 1&'3IM*H:/,(M**],$,1W*fX,/,*

6"("$,32:(1"9-~+$:()** /:I"3M*1,-,3?%-M*'(I'7'IY,3%* X::WM*%YXs"1-*

?&B0(<$-5$2& '(+!& '(RGS& !"5>,+!&

7,)80%-& ]^J& DNE& D_J& NE]&

'$"6%$& NI`& D_C& Da_& CEa&

@>*

•  ^Y/,(VX,%"I*H'-&*#:Y$*,((:-,-:$%*ah,99,*`*JPAc**

•  2:/9,$'%:(*-:*+:$Ip"-*

."/,(01*23,%%*K7,3Y,0:(*

7,)80%& '$"6%$&

7,DC& 7,DN& 7,D^& 7,D]& 7,DC& 7,DN& 7,D^& 7,D]&

="(($:5& N]`& N]^& N_C& NÊ& NÎ& N^C& NN_& NNC&

9"(2$(%),$& ]N& N`& NN& NI& CN& CE& `& ]&

>,-50,:$-& N& a& I& N& `& N& I& CE&

F$,$(0%=",:$65& C]J& CÈ& C__& CJ`& Î& _^& _`& `C&

Kk"1-*:#*'(D")$""*2:(1"9-*!,(W'()*

@@*

G('/,3%* [":93"*

'($:)-)",&

'($:)-)",&

*,"&(0,H),A&

M)5B&(0,H),A&

&,"&(0,H),A&

M)5B&(0,H),A&

.0,H& .0,H&

•  '(D")$""*$,(W'()*'%*"k"107"*•  K7,3Y,0:(*1:(E$/%*-&,-*$,(W'()*'/9$:7"%*9$"1'%':(*

,.:&,.:b#&M.:&M.:b#&

,.:&,.:b#&M.:&M.:b#&

'%V,*K7,3Y,0:(*

•  wY"%0:(F*+&"(*,3):$'-&/*13,'/%*-&,-*)+-UVDWXD'&:H*:i"(*'%*V*-$Y3?*,*%YX1:(1"9-*:#*W4***

–  )+-U32-&D'8$%#)G2%$X''–  )+-U32-&D'#)%9X'*

?&;),H-& '(+!& '(RGS& !"5>,+!&

7,)80%-& CI]E& D]J& Daa& aE]&

'$"6%$& IEa& DN^& DI]& _^I&

+"(2!$5&%0:H-&,$0(%K&B0%@&"@&5B$&>47&%),H-c&

@A*

D'%1Y%%':(*:(*."/,(01*23,%%*T",$('()*

@B*

•  [,_"$(VX,%"I*-"1&(:3:)?*:(*-&"*+"X*1,(**•  )"("$,-"*&')&3?*,11Y$,-"*1,-"):$?*3'%-%*•  3",$(* $'1&* ,(I* I'7"$%"* %"-%* :#* 1,-"):$'"%*/'%%'()*#$:/*"8'%0()*W(:H3"I)"*X,%"%*

•  ."/,(01* 13,%%* a&?9"$(?/c* "7,3Y,0:(* '%* 7"$?*I'x1Y3-F*

•  (:*%-,(I,$I'g"IM*1:/93"-"*,(I*1:$$"1-*$"%:Y$1"%*

•  -"$/*%9,1"*W""9%*)$:H'()*,(I*1&,()'()*

UT.S&U7dX!XS>e7U>X!&.-"9*=*

@P*

S,8:(:/?*

@Q*

•  G*-,8:(:/?*'%*,(*:$),('g,0:(*:#*1:(1"9-%*a-"$/%c*'(*,*I:/,'(*•  ,3%:*W(:H(*,%*:(-:3:)?M*1:(1"9-*&'"$,$1&?*

•  U-*/:I"3%F*•  -&"*%"-*:#*1:(1"9-%*'(*,*I:/,'(M*,(I*•  -&"*$"3,0:(%*X"-H""(*-&"%"*1:(1"9-%*

S,8:(:/'g,0:(*

A5*

,('/,3*

3':(* 0)"$*

9Y/,*

"3"9&,(-*

1&""-,&*I""$*

#"3'("*

7"$-"X$,-"*

1,$('7:$"*

1&:$I,-"*

/,//,3*X'$I%*

,('/,3*

1&:$I,-"*

7"$-"X$,-"*

/,//,3*1,$('7:$"*#"3'("*

3':(* 0)"$*1&""-,&*

9Y/,*

),2/:$&

S,8:(:/'g,0:(*

A;*

•  R:$*",1&*%'5.#,(I*'-%*&,$7"%-"I*$(6'5%76'$#Y%"*9,_"$(%*atV*,$"*W'-&,-uM* zzV* %Y1&*,%*WuM*tV' '(13YI'()*WuM*tV' 3'W"*WuM*t%Y1&*V*,%*Wuc*-:*9:%'0:(*-&"*1:(1"9-%*

(a) vertebrates are chordates that (b) chordates are vertebrates that '#*&'-%a,c|&'-%aXc******782%9-&$+''''''G$%&$#%-&$+'''''''''''''''''''''''''"3%"M*G$%&$#%-&$+'''''782%9-&$+'

•  R'(I*-&"*3:()"%-*9,-&*'(*-&"*)$,9&*

!

"

!

"

!

"

K7,3Y,0:(*D'3"//,%*

A<*

•  f(*&,(I*Y%'()*&Y/,(%*•  G),'(%-* ,(*"8'%0()* -,8:(:/?M*H&'1&*H,%*,3$",I?*

XY'3-*X?*"89"$-%*•  S&$:Y)&*,(*"8-"$(,3*,993'1,0:(*

•  U(-"$9$"-*(:Y(*1:/9:Y(I%*•  U/9$:7"*%",$1&*$"%Y3-%*•  K89,(I*W(:H3"I)"* '(* -"8-Y,3*"(-,'3/"(-M*&"39*

'(#"$"(1"M*Y(I"$%-,(I*/"-,9&:$%**•  N**

2,(*?:Y*)"("$,-"*,*S,8:(:/?4*

A=*

")2*'

-*)=-"'

C%$:'C%$9-&2%'

7-%*)G2%$'782%9-&$'

=-==-"'

2,(*?:Y*)"("$,-"*,*S,8:(:/?4*

A>*

")2*'

-*)=-"'

C%$:'C%$9-&2%'

7-%*)G2%$'782%9-&$'

=-==-"'

")2*'

-*)=-"'

C%$:' C%$9-&2%'

7-%*)G2%$'

782%9-&$'

=-==-"'

2,(*?:Y*)"("$,-"*,*S,8:(:/?4*

A@*

")2*'

-*)=-"'

C%$:'C%$9-&2%'

7-%*)G2%$'782%9-&$'

=-==-"'

")2*'

-*)=-"'

C%$:' C%$9-&2%'

7-%*)G2%$'

782%9-&$'

=-==-"'

")2*'

-*)=-"'

C%$:' C%$9-&2%'

7-%*)G2%$'

782%9-&$'

=-==-"'

2,(*?:Y*)"("$,-"*,*S,8:(:/?4*

AA*

")2*'

-*)=-"'

C%$:'C%$9-&2%'

7-%*)G2%$'782%9-&$'

=-==-"'

")2*'

-*)=-"'

C%$:' C%$9-&2%'

7-%*)G2%$'

782%9-&$'

=-==-"'

-*)=-"'

C%$:'

C%$9-&2%'

7-%*)G2%$'

782%9-&$'

=-==-"'

")2*'")2*'

-*)=-"'

C%$:' C%$9-&2%'

7-%*)G2%$'

782%9-&$'

=-==-"'

S,8:(:/?*K7,3Y,0:(*

AB*

•  T"-y%*$"1:(%-$Y1-*+:$Ip"-*7!>S7;4& ';7!U4& VTR>=;T4&

o*-"$/%* AP>* @@>* ;>5*

o*'%V,* >=<B* <<Q>* >;<*

,7"$,)"*I"9-&* AJ<=* >J;<* =JQ;*

/,8*I"9-&* ;<* P* B*

S,8:(:/?*K7,3Y,0:(*

AP*

•  T"-y%*$"1:(%-$Y1-*+:$Ip"-*

•  K7,3Y,0:(*Z",%Y$"%*

7!>S7;4& ';7!U4& VTR>=;T4&

o*-"$/%* AP>* @@>* ;>5*

o*'%V,* >=<B* <<Q>* >;<*

,7"$,)"*I"9-&* AJ<=* >J;<* =JQ;*

/,8*I"9-&* ;<* P* B*

!

Pr ecision =# is " a found in WN ! system# is " a found by system

!

Recall =# is " a found in WN! system

# is " a found in WN

K89"$'/"(-,3*!"%Y3-%*

AQ*

•  p:-*,33*-"$/%*1,(*X"*#:Y(IM*"89$"%%"I*,(I*9:%'0:("I*-&$:Y)&*9,_"$(%*a&"(1"*-&"*3:H*$"1,33cJ*

•  RY-Y$"*):,3%F*•  ^:H*-:*:$),('g"$*-"$/%*3'W"*t+26&8'F$+&$%*'C%$9-&2%+u*,(I*t+=-""'*27&6%*-"'C%$9-&2%+u4*

•  2,(*H"*3",$(*,(I*"7,3Y,-"*9,$,33"3*&'"$,$1&'"%4*•  Z:7"*-:H,$I%*9$:X,X'3'%01*&'"$,$1&?*'(IY10:(*$,-&"$*-&,(*-&"*I"-"$/'('%01J*

7!>S7;4& ';7!U4& VTR>=;T4&

[$"1'%':(* JQP*a;A>=~;APPc* JQB*aQ5@~Q=;c* JQQ*a<>A~<>Qc*

!"1,33* J=P*a;A>=~>=<Bc* J=Q*aQ5@~<<Q>c* JA5*a<>A~>;5c*

K8,/93"*:#*U(IY1"I*S,8:(:/?*

B5*

01*23,%%4* · drug: {low blood pressure, xanax, lamictal, generic birth control, lipitor, side...

Documents

0123,%%4 · drug: {low blood pressure, xanax, lamictal, generic birth control, lipitor, side...