From 8c5d62877ec2005430b740297c93fa50b9421809 Mon Sep 17 00:00:00 2001 From: Joanna Wang Date: Wed, 25 Mar 2020 11:59:17 -0400 Subject: [PATCH] Jennifer's fix --- .DS_Store | Bin 0 -> 6148 bytes code/__pycache__/model.cpython-36.pyc | Bin 0 -> 2141 bytes code/__pycache__/my_layers.cpython-36.pyc | Bin 0 -> 6970 bytes code/__pycache__/optimizers.cpython-36.pyc | Bin 0 -> 913 bytes code/__pycache__/reader.cpython-36.pyc | Bin 0 -> 2936 bytes code/__pycache__/utils.cpython-36.pyc | Bin 0 -> 4794 bytes code/__pycache__/w2vEmbReader.cpython-36.pyc | Bin 0 -> 2305 bytes code/evaluation.py | 292 +++++++------- code/model.py | 134 +++---- code/my_layers.py | 394 ++++++++++--------- code/optimizers.py | 42 +- code/preprocess.py | 102 ++--- code/reader.py | 226 +++++------ code/run_script.sh | 14 +- code/train.py | 353 +++++++++-------- code/utils.py | 330 ++++++++-------- code/w2vEmbReader.py | 142 +++---- code/word2vec.py | 52 +-- 18 files changed, 1044 insertions(+), 1037 deletions(-) create mode 100644 .DS_Store create mode 100644 code/__pycache__/model.cpython-36.pyc create mode 100644 code/__pycache__/my_layers.cpython-36.pyc create mode 100644 code/__pycache__/optimizers.cpython-36.pyc create mode 100644 code/__pycache__/reader.cpython-36.pyc create mode 100644 code/__pycache__/utils.cpython-36.pyc create mode 100644 code/__pycache__/w2vEmbReader.cpython-36.pyc mode change 100755 => 100644 code/optimizers.py mode change 100755 => 100644 code/run_script.sh mode change 100755 => 100644 code/utils.py mode change 100755 => 100644 code/word2vec.py diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..feded73ff076c815b89629dce804489102ff1664 GIT binary patch literal 6148 zcmeHKOHKko5PgNjK-i2cUCz=Ki6?NxA(2E~$N^9o7AE{eNQ|r5co+}h$vl9*>TU@% zxHW33lIqvhUDY-HCYhQ7AgyI_0we%xRK-f2!w%7X)h(%nm3^Y<8J*E|&>v0vwDmxf z-_L-)yDA2lqQEeEe>wNwKAgCV)V}=C#53+Oi~KL>b-L!?z$H33M;q7Za%^IOCoH0< zS4JJLMIB&$7PmLA=wgM(`U zh*d5-;oSZD6H{FvW&ul&T%m{$CHhbaErx{7L3}v(v*2HP^x+VfndgMe5;_zKjfbSp z9g=yp(HJlWHW}FSW}@}K{eAzx8Dy3*U<~{#2BeyI^A@iZYU|eJwAQB7TdIosl^$0i joVZd literal 0 HcmV?d00001 diff --git a/code/__pycache__/model.cpython-36.pyc b/code/__pycache__/model.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..99e74124dc903f03f3ab36c7ed1ff3255f3e7dec GIT binary patch literal 2141 zcmZV<+in|0bZ76oUOTqqHf<@wfKqCqM$q;Fp{gRiAXU&H(UwT7h}Fb1*{rh{cV_G) z*80hn;KdTZ!Kd&G^U4G2FYp3qW9L^X#jKFMf0$i9 zrDFkS$>2FYdyZL}m5c4|+nB>u1s(C>_U)X`X4an{z-5$I=QVuH*BnX6^4uxcBm zwn_^L*7nj9j&N598MRkCaEdRyl~MaZ1;QsGX{}N1*Fmao2t}ZDb*Q-(?0}uiNbiV% zYKsugZmm!aucWR4ltZgmd5FOIh_q6JEy^&gA+Z7wNQ@g2I zcS&z;E_<-6`wL>nq)#^UUeRCn!Q;M?>hlI*vne(w19C;j(wkf%5W~csT&*`JTeVsL zE4j8ZRuBy`wEfjfd+_nX{yLlq5i}ZTo2|Vuq+Bmm0O)OF6M*M{YC{+k1YoWd1;v=y z08VYJ5OgD?ehmR_DV?h>e$QAcYi$ExuU4F2!(}TIsGvrZkl5E@_cZ`Wg{hpW&4B@q zxO;DvBQK#*Y7P(KC0W6llE3{6)+mzb8)=U*7t*6sp5@TnHh;~8v^X@X%U{nh zqtIe`g~ge)NF`(-7%us=f|0Q3Ih77S##1VTiizVYVKg0Car;!adjfsQva{GaO-gAg z6#{2i%6~QfeEj&!gIAm~elUUdJwA9s`H85e2d_##n^HE5` z(#-P2gvNg*w%iyFq1Y1a$dIQvOYp0}J%>+Czx@3^NX#egH z@LYUPF`*2258O$hri+iBm6^zJo-N=_8Y)kRplY6|VN-?0qyN}6{`}8si1}~~iD6Yn zhSkyVq@cJQj`JB8P^-}mY2~zJjtbgW$>GX(^dqe=!T6-kcIK@e~t=Qy9y z_=b|ILz@7USx=D{&NP;z0sBB-0#$$rrK9)g29u$tn@~SizTQ&k z6F{*=lhfEbPdFQCc9HPdp$TLrwiZdrKGjl|aB1fi=hD$#8LR(CQ^Yt_-a^MhLD>+j zp?#Di>_b2@dbgJ&knsC}3^Q_ltBVF!ANi(_Lexhc3%*tdby0*OYk<1IN5F;X7QhXY zJ%W9sE$OKws{dB|1pZ8X=MaxiXi21ZL9jm9#ZjZw>E*h1OqWX+S{MJqLWje3@#3I3 zZwkVjy6X9Ty=RMt-tHFinDz5p+@%Q`3TAe|Qf Z0o*&w=|lCB@h%{hgliD+O$Zxk^KVFDUlsrW literal 0 HcmV?d00001 diff --git a/code/__pycache__/my_layers.cpython-36.pyc b/code/__pycache__/my_layers.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..563ed3ca223ab3c56f2d7619f4a5cf6dcce66425 GIT binary patch literal 6970 zcmb_hOOG5^6|Q$xS3jmbp4f3-3U2C>#q-K+P}2QWkG%kCHVuYNQ?BLHqg5UBx7iftgfZA*Bsg-r|XDp zL3X>Y$TnoJ>xrxnYh7R7P#-k9%~!R^iQJo7oc1TF#?*IpuJ-CE?rn$Telof+f5#iU)9FOvfvaJ% z(;sgw?yTpnt>HvX(qMPo-<+t?TBnt_lX848D#j~*d~R;VX>M_=6fG`qUJEMwls7l3 zO~tLr8`pxx*5!?jYRlT_he>W@u2?CqbnXQK-yQ@VH@A{_xT(0{s==zss>NyvRqpJ( z8>%hZN-YR_!!SvL;LqBBUcLOS>u)A7qU!@S3b(Jn949;JWP1J0u}t$$KZ&FB7n5n+ zOV3}q0gLK!r?2-WQG9*07Yu2S$%W~jI)zDml%$PHa~-{5G<5kh9DRRzzS zwrSvw6IlZnzaDL9P($$}`xY%p>}XIwgax6usoqCz!@S|Xl_y&0T`HewM;@L#6=|@y z#@cPFt#nkkS?yatJqWIKY;4bVIF0jq#X)mx9FEXc-Fv^X%NCq?A}`ojzx_X4>N9v_ zM^_K>XF0dhJ-pEx_LDSkT}-fB+=DAhO;m0U<1yw;<7txnVH5@L#{I4BRBR~hr%b-!@`nBAiDnNr+Mqw>9lw9A)?Ty|3Fzpy>nH~6w>24axC#eVc7OBqgo{Wqh zftakK()708Hd@9*x~2Ol4c*7aJGxP}_LmkWz&WBGU4$2ZxrDO+^b=WOeNE=-n1v<) zJZb{ZP4Lwg-R!7BjoxI0F|qjI*YKW1EKJ`&vzWO$WUhdRv2z3bCmJv!{hW3Hv!kIM zSeYShBgH9!Lc=mCXMmL#9|M6+}vUB)_qIp;a;z{Wj4v+-_9b&*`?q3zVv-!d0{jg0y<4*`bX)DGOD4?U1D zFEIQTb!;7KXW>xAxJP9i%(u2CmMaFP&f<#!3s!iepR8l&zk+t|?Zojk>W`A!|D7A; zQm((2*MI@zWI9RW+>9ovdV-_t@r`LXjsiZAFR6GMhiMR(psPO5A!f2WD(tPprj>@{ z?cBt3xz!6{*v{r~5~eqD7na-~PR1QiHeQ{_Xmx>)=r?kGuWQErfs$9~;ZX~~n})+@ zFt)TvPsblDvY{{Q%do~3w0zj5r>_`a)c0EzyTF5Y01}N-43a)r*aK~#lwcK`2G-H6 zt|MKebR6|1$VatX!D$docGI~>1#r*TF>q$5k1g~$X=uk@!?oVg!35SiHG)5JB?xjq z2u71=cSyMr1aI$#!{UyhvU-Zje3{inR+mua&GHcF#lv9`EK~*RtZ)!rT+WypgoNk- zzJtG3+dJc%{#m)#L-+jRit$RmfojGqgjc|eZhGJqeq=>9ScFw@B=Cy5up{sa;S(-_ zHLqV21dF3Lc1Jzp0xxf*ZxXHm-)f9HU|!#!5k}w!Y7ZNTv`Trkdth7a@_$esY%!% zfE4fmXEy-}HXPWzMFX=c1_p)cnjJ^2;zxa!)#q3}&FWcJTzcFmb}t-9y};&6sJgA< zbciF+2AJVu+l&3Cd90(zeiwjF2l7$&+tsC@pM4B>!HF#HQvDh^<3xekrkvn2Vyi!) zMkSQ5R<$_EN#=V)YoYbX-h9NVQmhQY<>*S#TY4z7c^|CKTRlWdZ}b%BQ7^N)!iv_m zaJRzyUSXe@n?Oa?(`cbTKuKt{p5CzhhG&Z1Qj7V;Q;OG}?7bykM&BZPm-qudwp99- zT|^w`LYykX1`POf7;x6MprTx*@ZpQ=0?jjG6OmbUPUp3P zUEui2{Su)e3yD_*z=BSd90ZJOE8WiBNKK{~EWEv7+cY!bS@{Zi@;m)>ugdWA)4NHW z1j9H~W5g7}bUVZ;x>Pd7lDQ>?J3r4?wOM_am4H{haBLnK>MAR4i&|sFjZs7(Q6zWY z6%0)jZAyJK+HqMD!l`DHt2I!Zg0|z-EP5LH{zJ!XWR8V*fPjnl&VF!hZJCMLrgov z&%HY${eqL1Nch8iMN$Hwpx*%+SV1%<&P7np1mCkzf3n$+Ds{}S&15@n{rU$VP{lOz3>+7s)9BehFC=9sYHNBmS~HDt%Rf@J{nr0hJy7lYx;0VR`6=>Qp_ z%vjM57-m_pR1*-vy9~iB$`6y@*=uFcB7{gW)|0AZ7xwdQb_jwlSWyv2-rbVk$8(9C zbqE~V2wXjVzcKr~W1NR8AYV2M5hCm72n?XL9ch%#Qdg&BaE!8wcuUk}gnwV`=l9g9 z^!Z0-y4=&xJ{j}*pfcymY|bjWd`hf`HuoFOHD^68JchkJl=($7;&neBBMBVISxVmZ zj>w{RHN0^ZNwEI-WS?QolOjhcRKmC;q7ZD72!01W>Lb6&&8%Wxr@EiIJJp%>($dA<+UfqHu0l!1fdYDNJj*zs)Ark?MXApN3 zA=$BKtc*#Alb*rhjiBwMd64qpfsS;EgJ&7f*hqIYim?YwSa_M6IZ=~|327sv8@1s5 zoXca{15pp}u}uQW=a8)0uJ9!I%b3SunX2a21B9{S_>0+Q^L;bd!1aq)6n?iB%tL=p1O0DHVUn(pPs zc-n2I+i^IaTqqNuPWh{^OCqMoaFA*UdWt%g-FkpGh;F&}e32-6fxDit3*5E*^N<;9 MztTS2KHon5e|J2Bc>n+a literal 0 HcmV?d00001 diff --git a/code/__pycache__/optimizers.cpython-36.pyc b/code/__pycache__/optimizers.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c6d47ba166c23719c509331bf1825f5446086952 GIT binary patch literal 913 zcmZ8gOKTKC5bo)jotYh7jk>P6_&7R9f{$B}j4_E3AwdkegvHp|Zf9pRFS=)A5;x!> zpa<9AAl~9Zya?U}5287Ev%!OU3ivz)#DmqHhlw>*O)=kBUsn~=7b+F+<;OcWUzrGf zLAwk?ehFOEAW%e&7`2RPOffZA(6mX7N#tAWfT4+op+_G$s@6ZM^-pO=4H$2L;m~8y;T}fEgn4@> zxEgicgCHHAh@5A6_-FOk`|;W9LHO>ivadnd{r2MV#_Q*Ueo5k(OOrP<>9AZxah6KA z5l4BN@kEwYHc#RX1(`V~H{a*~SGH9(S=d^{t-yC>nZ#knqoSEeGs_F<+!&u!%aZ2g*obt7 zHanFrB)TVQL6%5cMY7;q&=C)Xo=cA)%b*e^(GugUh#!Z^eqS9Ps80#TMZLun(yWg# zF;`@HeJU00obmZcFxodHa@HvNMi&ZB8by?)^+rZneMe06=ZmruvSMcE6I=u&PC!7| zGA!&FHN!J{!K>igc@6WE!0@fc)BaFEUvLH2?qr literal 0 HcmV?d00001 diff --git a/code/__pycache__/reader.cpython-36.pyc b/code/__pycache__/reader.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..828f53789d7df2a144d64be25f81089e978e00a3 GIT binary patch literal 2936 zcmZ`*OOG4J5$>KBhr`#tyt}etU=V2IP||W8+ejQ?BU_MsNrHu;#Fj%-A}DrK%jI%N z_Vh{=(UXJds{#FhoP#9DF{d1I&Tp7okVCM4LGU48^-!yolMK++)z#JA)zwvB&4;yG z^pAf&efYN;WB+28J`en-Xz6nh!2~~L?V{&xZpTjBk^ICJ&XTu1;R^2?*7ikB_#*g* zw*yfUAtWVH7iAGa63Viuj2!hl8H(zXcf5AguC%M|nykx)tStGGb7pzWuJ{`Dg8!N| z8|Sct9WAyWetmua!6#q8zyGrb*E_9(xpuYBX&N8(F4`Tmv<{+~<|mF}v?uoAM@={P z9!(~vc{w>7ca?lB=ea-b>Hcw(=f0kdWb!Ab0xW%b_s-4E7pYWf=R}Qr$DO+}9qGxm z^F@-LO{ID=Or_Yrola$6?|(Mes@K=UNz&<0gzPBU6H>LN&+_sx?c%6MQkBRk2ZV7I zg7(Q$`Zm$<0)^E%?qHbM83k!ov*%p6hV3xn?Xb)hyB#O1xeSza57}>7;(W!7XQkdu+-I}H`%ITa*?7huINR_! zXByfkaP~TEs=Z;; z=C@9^G5^YJ;PlV!>2;jCHDWKg`m_mbl+ws-8^UhvuC@+|z)xPqBC zHDKBLNqtEktRVug9OaRcsqUSrUZN3;QmVyUt=7R*$*G$3Ws0ckik|KrETRJh)i7yk z1Q$hi&eg&{(}Vqw&uO^$K^(_>!&pyr?=*foQ6jxTHr<7PIPHmB=+2T6bqLX0i=8+gNjZ)6vBa)Z+RSddc=vxAxR)M0ne=;)a{e^; zVAsbu%o`*8^H3&8gJJTx$rVNRIfeR5Ue0-Riwsa@A`wW_u~)OC*~sg+aeqa5^qEo< zl?Rkf{WSL`Q<>y`Iz1ig+&z_v&7eH|rF>Q>`cpN8KZA6lv=q5N)N-8WA(pA0#zqkd z9!ssI%KfJbqufn3`2Oi6mCemO5R);Sod@II9Q)^yy;?UNE+p-Gruwp7v99h~pR}um zpI4)Zgx!IXvov?%yWAaMGw*bmNSsG+qtvj{ zscctQtE{F)r-Bfq= z!45N&6eU2ICpJ57=4U|$pvc0%i2PT>SybT_S*>RJ5>FBiKq>1RJA3{k@ns^|_n3C;Fv*$?QGIq4Z$b_%}+=!`r1($`hdw{)AG{l^f?Xn4K#-tX4X5gD z61+o%>RcZ5rpR8Q-o>EVP&O< z@oNEW?9SCST6CSr4UoKq>=^<*)d7j9eqYWID#sR7Q>kxaE~QHq8M1{qf0ZLunn{lh0L`wlylqT z$3*zXHO|s23Oh7{i<#2J46;SL#y6# zjp706Df5!u49`?8m=%}?v?HVy+ZV#`KT-&Ewr}(h8>L$y%svFcQ^+@Li@3T?$AsN3 zfpxcQv+0c*a=D(Y+0e`#B~?tzO8v=r3OrR*j%;kczSYIXNpCE%v5m}6Nbgf3HbyCc qi@1IrV3xBu93Gv3Nk!&A}4gC_rt4EJ>+W?NBlp&bHvgB literal 0 HcmV?d00001 diff --git a/code/__pycache__/utils.cpython-36.pyc b/code/__pycache__/utils.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..16e9afaf367e5c36883ec9678118ad5891be64f5 GIT binary patch literal 4794 zcmbtY&2JmW6`$E}t|-cu72C2LM{(lR1ZvfdQoAt>C89)s)C%4Dqq5g>wiIV2QzE(4 z?8=siWeZqN4+VNi4@FQE=&eBigQCZt`X>|v^wd-Sg`E0(vy{vTabY;3XWrX+Z|2R+ z`y9`Xj{1N9&$G{$jxzRNHuPr#{}e6xD>}ghZ?PbUJmC7<2n-F)z!YbMAdU{47C|qZvC`Ml}F}i02z9@+?P)EcOaTKs9#>FwfQ86Lj z0xXH+;soHBI4LFpkBE1~DKQ0kN5yIJHb%w;zst(gUt_PV*5Cy2=b%|=3uwuo(4}mH z3x0y2ae@i7K2OaJYmfDf%-FDjJA0fnKo?_XX1*K)?qR%;@jd)F1HYe{8e{U0!t7h1 zj$}+&SQ+R4z0tQbduYBrm~Rcuw`sm$*t=WnjXgSdz7;)oe(2s`#@qS5`X~QMstx%0k?VWulzhw<=5btIDd@ z78jHY=JjT0T@}`&bP0GQm6rgeJt=#!TDgB+jowXVRBzv?cSH;HBIws**{-K4z6vi< zzb?c+)5x`ED;j*19p%Q`X(*a9C^h0v61SpoO|PIF+CU@&cQ77|tYPAS8hD_FH1Ibj z$Dzz`78b6o+)pBztZZNdPgWMAWHXJsEB8Ceb~lnwn@J=t%qQKbkzT0oq_W;fn{j8Q zflFA~PMfV{rn{?r93af)x@0D&&@nz?c)ZA|9pOH=@Eg~-oB;K}z6^XELDTm2IS^={ zZyK^l8r?0CwPlPnyv#at7E6syUq0ZlInH`#GrnQMVl41q1E;;=5U;$}F`k3}A^93O zf6X7Tj`1M35u>JHD9VTv<+baZ5$-Ti?(Mr(DPyVZNXkyEoND#K&Aa!M)vc#bl-*3g zRW{@lwx`T!r=jmI?~c)!+uj66*qy{jLZ@Lkx5W)!!r4y($}<>i5Mlq1VuL<2sew+p zVbDngpTr4Z%SKI-3zltp93U6%LI|0%GLzjzS#`PobZ95Ekks2L!uCQL$J#>P!*D{k z%(y&@{@~_E@Il_Kckyo^8QZ|!PvXXaabav)a$1iW7&AH(!hnbCJN*JtYmG+PIy@ZB z+WA8Y@89E#?D22=c9Zp;ojHua%Q5m{-_2O&X11Q=WnSiNI`VnuWS-3?5koFBlU-$7 z4}?Kuge_wy{ld=snS=SK`PhaJDzmR!-f9tDdhNxm@I_6{+FrTbt>l%$` z)|!F4H1oUsT3@K8?Bq$k8!4+Jnr&svdS^XSPQBZWI-)$H?7Eco-N5T?w{eHhk{oNq zEntm!y8|Bu)?SOkRx?SJ35tTVX@W8_L%C5aYDb+kD75Q4p(X_;CJgCHtEsTs>}YE8 z0d7C}Avy+6@p;j(U^nD3CEdu2#*HFy(13e@B`z<3zd@7ek76{n>u5=e4qD+mYZ=4+ zecm@x#M(Wcl4k`1wx)9p#uj01d%O=XBD&C?1h2C4sJ!)raA9XA;<0l=`{9;PW2sI2 z`lkJ5mO7+q9f7~$(+3ttR!4ifOM4=(Ux$r7F7}c8GJc$WX>3`T!+X}bnb|dcJ7q%| zGZz&h{Z(v6fOA|x`4B8p(=*q)^=UjA7L0cH>&fC)jQJ0DuH|Z z*7ZBpYE8Mz^LJ`DYuA-^ZT{wxq$nI%mD@{;LE(N4>`OOm)xci9ar0g^a2{5dmToUA zJD+M%u@P7c59e#jyi;8a+|TB(S8Mm?1G};`zi=yXE4fG@d1VrKK=fo_1HoV>$$?*a zDbJ~Bfv9W-5}isu%dTjdc0~)dDkRmb?2C0kg7P5jP4c+TOZqnv* z2gHONgp%a{|531e#m~@deajli$Fwz2upw(e3WBZbVikTt!KPNW^gQ`9u*jcMN7oNS za0j*F(6)7$Hem`+!Lpx3e-8YOrG7>Nhos8eG@!5N;Et5N8bapokj!6vADK5_N2ayf z+K#dzk))(IKIX005$UalyHTqZKl{y)SjrV|EcV9hh_%;c6m^~t$-MV{WG=mqOm{WZ zQok6IdY`1~@B)PD=7VDFpZLhQMDd#;p6#LRN_BZii5kU@C6IdYE&%qT_-Z`);;zM!cqoQswl`>F5_{qbrv>2rFNmF zQ|y6!Qtzu(EPn~|cPm&fgY*})T(_s_N%Jbr>MN8F!Mv|n8PLL(d<-Dz*fOM!0xw=v zS}Ci*gVBLfPy^!8p0Cp&&O;H|+ZX?55QkoP``7yH+mB0wnaHf zzXSC%Ojqx^N|c`vfeIN(sYOz(mlS>FUFs+}Ns1L(v0BW~?({y7fvUW#c?mK{*PWb2 b=iw3IS^l`^d!{$#jd@1tePH`P+aLcQ%)#kp literal 0 HcmV?d00001 diff --git a/code/__pycache__/w2vEmbReader.cpython-36.pyc b/code/__pycache__/w2vEmbReader.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..34f287749fa9dc6b4e704702f8e2a81b8b626871 GIT binary patch literal 2305 zcmZuy-E$O05TBX-yvtnzDL?`Wr*LJ~k|4qIgQ6&qL`%U~r9yD)blvRC9eW?U$IK=q zS9?{I;lUUF5MO=rZ<#8ee9FJzi+0am_zH71J$=2?Jw4sO?z!4**8lqZ@h=0PkblTa z!v_8)O!X@ePB;jn~ErGmz^^HaROntHzrGW^O&bu zuO&uXf+MF|U73%rwL&{f6EQj8YF*e7Ns&u+t<_FHY+c}xluV_0W(2`;?GE!^Pe^U2 zSvS}97nZc!!<{4Tj&d%BD)cvV$ye`-MB76+S9`EG0(;uo$&+{sq^T;j8;{2#p|@Wdl+q zPXN)5aYoyZ!IwXXTmsp@-`u{|xd-Q{P9I?Ou(KxApvcFadzsoE3%Qdj!7trXV+h2h z_1!|o0L?t>Byg?H@reX>sg@}G6s)SfzfXW$~Vw3h(b73ewp6v}0jJer4#F}T=Tb+Mm7@&M*0 z!BXN@X%!9@6Sqt21gZG5q-#~cE?MFBy}l3S;$fe8W}PKdw{!vS{zebnG2?*3GP%-D zFSpYcUbr>GcnedP#zR0CZ!JGeinGNCpAkU>TpFzf6hJ4Gk&zcIH*tvrV% zHa@(S$?*+f3VWMJBou;Vav?)s0oLHx}CK zLc=Mpt_DsO>Ht5o!ASduXNHR=ys;_rSs!^l!pAYfuk837?5H{r0%g*mt_9&dcO;yz zs3eFz9Jt~%QN(a!X+t!ie5vVIu$ZEKtzPF2^9;!Q0@M@+h0w|ABj?WAD_ZoiFX0Os zMY|FJS^JDwnyW=Dnm8U3m4R2C}Nqd%yLj=*Bz$`$o1S-tkUFmo^?~G uldm_v{7%>4=6B6wBd?$brn1nDH~FXi>(wo9V2~A-olmP@gZdV7b@o3pT|8v~ literal 0 HcmV?d00001 diff --git a/code/evaluation.py b/code/evaluation.py index 6b0f334..7d92c35 100644 --- a/code/evaluation.py +++ b/code/evaluation.py @@ -1,146 +1,146 @@ -import argparse -import logging -import numpy as np -from time import time -import utils as U -from sklearn.metrics import classification_report -import codecs - -######### Get hyper-params in order to rebuild the model architecture ########### -# The hyper parameters should be exactly the same as those used for training -parser = argparse.ArgumentParser() -parser.add_argument("-o", "--out-dir", dest="out_dir_path", type=str, metavar='', required=True, help="The path to the output directory") -parser.add_argument("-e", "--embdim", dest="emb_dim", type=int, metavar='', default=200, help="Embeddings dimension (default=200)") -parser.add_argument("-b", "--batch-size", dest="batch_size", type=int, metavar='', default=50, help="Batch size (default=50)") -parser.add_argument("-v", "--vocab-size", dest="vocab_size", type=int, metavar='', default=9000, help="Vocab size. '0' means no limit (default=9000)") -parser.add_argument("-as", "--aspect-size", dest="aspect_size", type=int, metavar='', default=14, help="The number of aspects specified by users (default=14)") -parser.add_argument("--emb", dest="emb_path", type=str, metavar='', help="The path to the word embeddings file") -parser.add_argument("--epochs", dest="epochs", type=int, metavar='', default=15, help="Number of epochs (default=15)") -parser.add_argument("-n", "--neg-size", dest="neg_size", type=int, metavar='', default=20, help="Number of negative instances (default=20)") -parser.add_argument("--maxlen", dest="maxlen", type=int, metavar='', default=0, help="Maximum allowed number of words during training. '0' means no limit (default=0)") -parser.add_argument("--seed", dest="seed", type=int, metavar='', default=1234, help="Random seed (default=1234)") -parser.add_argument("-a", "--algorithm", dest="algorithm", type=str, metavar='', default='adam', help="Optimization algorithm (rmsprop|sgd|adagrad|adadelta|adam|adamax) (default=adam)") -parser.add_argument("--domain", dest="domain", type=str, metavar='', default='restaurant', help="domain of the corpus {restaurant, beer}") -parser.add_argument("--ortho-reg", dest="ortho_reg", type=float, metavar='', default=0.1, help="The weight of orthogonol regularizaiton (default=0.1)") - -args = parser.parse_args() -out_dir = args.out_dir_path + '/' + args.domain -# out_dir = '../pre_trained_model/' + args.domain -U.print_args(args) - -assert args.algorithm in {'rmsprop', 'sgd', 'adagrad', 'adadelta', 'adam', 'adamax'} -assert args.domain in {'restaurant', 'beer'} - -from keras.preprocessing import sequence -import reader as dataset - -###### Get test data ############# -vocab, train_x, test_x, overall_maxlen = dataset.get_data(args.domain, vocab_size=args.vocab_size, maxlen=args.maxlen) -test_x = sequence.pad_sequences(test_x, maxlen=overall_maxlen) - - -############# Build model architecture, same as the model used for training ######### -from model import create_model -import keras.backend as K -from optimizers import get_optimizer - -optimizer = get_optimizer(args) - -def max_margin_loss(y_true, y_pred): - return K.mean(y_pred) -model = create_model(args, overall_maxlen, vocab) - -## Load the save model parameters -model.load_weights(out_dir+'/model_param') -model.compile(optimizer=optimizer, loss=max_margin_loss, metrics=[max_margin_loss]) - - - -################ Evaluation #################################### - -def evaluation(true, predict, domain): - true_label = [] - predict_label = [] - - if domain == 'restaurant': - - for line in predict: - predict_label.append(line.strip()) - - for line in true: - true_label.append(line.strip()) - - print(classification_report(true_label, predict_label, - ['Food', 'Staff', 'Ambience', 'Anecdotes', 'Price', 'Miscellaneous'], digits=3)) - - else: - for line in predict: - label = line.strip() - if label == 'smell' or label == 'taste': - label = 'taste+smell' - predict_label.append(label) - - for line in true: - label = line.strip() - if label == 'smell' or label == 'taste': - label = 'taste+smell' - true_label.append(label) - - print(classification_report(true_label, predict_label, - ['feel', 'taste+smell', 'look', 'overall', 'None'], digits=3)) - - -def prediction(test_labels, aspect_probs, cluster_map, domain): - label_ids = np.argsort(aspect_probs, axis=1)[:,-1] - predict_labels = [cluster_map[label_id] for label_id in label_ids] - evaluation(open(test_labels), predict_labels, domain) - - -## Create a dictionary that map word index to word -vocab_inv = {} -for w, ind in vocab.items(): - vocab_inv[ind] = w - - -test_fn = K.function([model.get_layer('sentence_input').input, K.learning_phase()], - [model.get_layer('att_weights').output, model.get_layer('p_t').output]) -att_weights, aspect_probs = test_fn([test_x, 0]) - - -## Save attention weights on test sentences into a file -att_out = codecs.open(out_dir + '/att_weights', 'w', 'utf-8') -print 'Saving attention weights on test sentences...' -for c in xrange(len(test_x)): - att_out.write('----------------------------------------\n') - att_out.write(str(c) + '\n') - - word_inds = [i for i in test_x[c] if i!=0] - line_len = len(word_inds) - weights = att_weights[c] - weights = weights[(overall_maxlen-line_len):] - - words = [vocab_inv[i] for i in word_inds] - att_out.write(' '.join(words) + '\n') - for j in range(len(words)): - att_out.write(words[j] + ' '+str(round(weights[j], 3)) + '\n') - - - -###################################################### -# Uncomment the below part for F scores -###################################################### - -## cluster_map need to be specified manually according to the top words in each inferred aspect (save in aspect.log) - -# map for the pre-trained restaurant model (under pre_trained_model/restaurant) -# cluster_map = {0: 'Food', 1: 'Miscellaneous', 2: 'Miscellaneous', 3: 'Food', -# 4: 'Miscellaneous', 5: 'Food', 6:'Price', 7: 'Miscellaneous', 8: 'Staff', -# 9: 'Food', 10: 'Food', 11: 'Anecdotes', -# 12: 'Ambience', 13: 'Staff'} - - -# print '--- Results on %s domain ---' % (args.domain) -# test_labels = '../preprocessed_data/%s/test_label.txt' % (args.domain) -# prediction(test_labels, aspect_probs, cluster_map, domain=args.domain) - - +import argparse +import logging +import numpy as np +from time import time +import utils as U +from sklearn.metrics import classification_report +import codecs + +######### Get hyper-params in order to rebuild the model architecture ########### +# The hyper parameters should be exactly the same as those used for training +parser = argparse.ArgumentParser() +parser.add_argument("-o", "--out-dir", dest="out_dir_path", type=str, metavar='', required=True, help="The path to the output directory") +parser.add_argument("-e", "--embdim", dest="emb_dim", type=int, metavar='', default=200, help="Embeddings dimension (default=200)") +parser.add_argument("-b", "--batch-size", dest="batch_size", type=int, metavar='', default=50, help="Batch size (default=50)") +parser.add_argument("-v", "--vocab-size", dest="vocab_size", type=int, metavar='', default=9000, help="Vocab size. '0' means no limit (default=9000)") +parser.add_argument("-as", "--aspect-size", dest="aspect_size", type=int, metavar='', default=14, help="The number of aspects specified by users (default=14)") +parser.add_argument("--emb", dest="emb_path", type=str, metavar='', help="The path to the word embeddings file") +parser.add_argument("--epochs", dest="epochs", type=int, metavar='', default=15, help="Number of epochs (default=15)") +parser.add_argument("-n", "--neg-size", dest="neg_size", type=int, metavar='', default=20, help="Number of negative instances (default=20)") +parser.add_argument("--maxlen", dest="maxlen", type=int, metavar='', default=0, help="Maximum allowed number of words during training. '0' means no limit (default=0)") +parser.add_argument("--seed", dest="seed", type=int, metavar='', default=1234, help="Random seed (default=1234)") +parser.add_argument("-a", "--algorithm", dest="algorithm", type=str, metavar='', default='adam', help="Optimization algorithm (rmsprop|sgd|adagrad|adadelta|adam|adamax) (default=adam)") +parser.add_argument("--domain", dest="domain", type=str, metavar='', default='restaurant', help="domain of the corpus {restaurant, beer}") +parser.add_argument("--ortho-reg", dest="ortho_reg", type=float, metavar='', default=0.1, help="The weight of orthogonol regularizaiton (default=0.1)") + +args = parser.parse_args() +out_dir = args.out_dir_path + '/' + args.domain +# out_dir = '../pre_trained_model/' + args.domain +U.print_args(args) + +assert args.algorithm in {'rmsprop', 'sgd', 'adagrad', 'adadelta', 'adam', 'adamax'} +assert args.domain in {'restaurant', 'beer'} + +from keras.preprocessing import sequence +import reader as dataset + +###### Get test data ############# +vocab, train_x, test_x, overall_maxlen = dataset.get_data(args.domain, vocab_size=args.vocab_size, maxlen=args.maxlen) +test_x = sequence.pad_sequences(test_x, maxlen=overall_maxlen) + + +############# Build model architecture, same as the model used for training ######### +from model import create_model +import keras.backend as K +from optimizers import get_optimizer + +optimizer = get_optimizer(args) + +def max_margin_loss(y_true, y_pred): + return K.mean(y_pred) +model = create_model(args, overall_maxlen, vocab) + +## Load the save model parameters +model.load_weights(out_dir+'/model_param') +model.compile(optimizer=optimizer, loss=max_margin_loss, metrics=[max_margin_loss]) + + + +################ Evaluation #################################### + +def evaluation(true, predict, domain): + true_label = [] + predict_label = [] + + if domain == 'restaurant': + + for line in predict: + predict_label.append(line.strip()) + + for line in true: + true_label.append(line.strip()) + + print(classification_report(true_label, predict_label, + ['Food', 'Staff', 'Ambience', 'Anecdotes', 'Price', 'Miscellaneous'], digits=3)) + + else: + for line in predict: + label = line.strip() + if label == 'smell' or label == 'taste': + label = 'taste+smell' + predict_label.append(label) + + for line in true: + label = line.strip() + if label == 'smell' or label == 'taste': + label = 'taste+smell' + true_label.append(label) + + print(classification_report(true_label, predict_label, + ['feel', 'taste+smell', 'look', 'overall', 'None'], digits=3)) + + +def prediction(test_labels, aspect_probs, cluster_map, domain): + label_ids = np.argsort(aspect_probs, axis=1)[:,-1] + predict_labels = [cluster_map[label_id] for label_id in label_ids] + evaluation(open(test_labels), predict_labels, domain) + + +## Create a dictionary that map word index to word +vocab_inv = {} +for w, ind in vocab.items(): + vocab_inv[ind] = w + + +test_fn = K.function([model.get_layer('sentence_input').input, K.learning_phase()], + [model.get_layer('att_weights').output, model.get_layer('p_t').output]) +att_weights, aspect_probs = test_fn([test_x, 0]) + + +## Save attention weights on test sentences into a file +att_out = codecs.open(out_dir + '/att_weights', 'w', 'utf-8') +print('Saving attention weights on test sentences...') +for c in xrange(len(test_x)): + att_out.write('----------------------------------------\n') + att_out.write(str(c) + '\n') + + word_inds = [i for i in test_x[c] if i!=0] + line_len = len(word_inds) + weights = att_weights[c] + weights = weights[(overall_maxlen-line_len):] + + words = [vocab_inv[i] for i in word_inds] + att_out.write(' '.join(words) + '\n') + for j in range(len(words)): + att_out.write(words[j] + ' '+str(round(weights[j], 3)) + '\n') + + + +###################################################### +# Uncomment the below part for F scores +###################################################### + +## cluster_map need to be specified manually according to the top words in each inferred aspect (save in aspect.log) + +# map for the pre-trained restaurant model (under pre_trained_model/restaurant) +# cluster_map = {0: 'Food', 1: 'Miscellaneous', 2: 'Miscellaneous', 3: 'Food', +# 4: 'Miscellaneous', 5: 'Food', 6:'Price', 7: 'Miscellaneous', 8: 'Staff', +# 9: 'Food', 10: 'Food', 11: 'Anecdotes', +# 12: 'Ambience', 13: 'Staff'} + + +# print('--- Results on %s domain ---' % (args.domain)) +# test_labels = '../preprocessed_data/%s/test_label.txt' % (args.domain) +# prediction(test_labels, aspect_probs, cluster_map, domain=args.domain) + + diff --git a/code/model.py b/code/model.py index 572864e..60920c6 100644 --- a/code/model.py +++ b/code/model.py @@ -1,67 +1,67 @@ -import logging -import keras.backend as K -from keras.layers import Dense, Activation, Embedding, Input -from keras.models import Model -from my_layers import Attention, Average, WeightedSum, WeightedAspectEmb, MaxMargin - - -logging.basicConfig(level=logging.INFO, - format='%(asctime)s %(levelname)s %(message)s') -logger = logging.getLogger(__name__) - -def create_model(args, maxlen, vocab): - - def ortho_reg(weight_matrix): - ### orthogonal regularization for aspect embedding matrix ### - w_n = weight_matrix / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(weight_matrix), axis=-1, keepdims=True)), K.floatx()) - reg = K.sum(K.square(K.dot(w_n, K.transpose(w_n)) - K.eye(w_n.shape[0].eval()))) - return args.ortho_reg*reg - - vocab_size = len(vocab) - - ##### Inputs ##### - sentence_input = Input(shape=(maxlen,), dtype='int32', name='sentence_input') - neg_input = Input(shape=(args.neg_size, maxlen), dtype='int32', name='neg_input') - - ##### Construct word embedding layer ##### - word_emb = Embedding(vocab_size, args.emb_dim, mask_zero=True, name='word_emb') - - ##### Compute sentence representation ##### - e_w = word_emb(sentence_input) - y_s = Average()(e_w) - att_weights = Attention(name='att_weights')([e_w, y_s]) - z_s = WeightedSum()([e_w, att_weights]) - - ##### Compute representations of negative instances ##### - e_neg = word_emb(neg_input) - z_n = Average()(e_neg) - - ##### Reconstruction ##### - p_t = Dense(args.aspect_size)(z_s) - p_t = Activation('softmax', name='p_t')(p_t) - r_s = WeightedAspectEmb(args.aspect_size, args.emb_dim, name='aspect_emb', - W_regularizer=ortho_reg)(p_t) - - ##### Loss ##### - loss = MaxMargin(name='max_margin')([z_s, z_n, r_s]) - model = Model(input=[sentence_input, neg_input], output=loss) - - ### Word embedding and aspect embedding initialization ###### - if args.emb_path: - from w2vEmbReader import W2VEmbReader as EmbReader - emb_reader = EmbReader(args.emb_path, emb_dim=args.emb_dim) - logger.info('Initializing word embedding matrix') - model.get_layer('word_emb').W.set_value(emb_reader.get_emb_matrix_given_vocab(vocab, model.get_layer('word_emb').W.get_value())) - logger.info('Initializing aspect embedding matrix as centroid of kmean clusters') - model.get_layer('aspect_emb').W.set_value(emb_reader.get_aspect_matrix(args.aspect_size)) - - return model - - - - - - - - - +import logging +import keras.backend as K +from keras.layers import Dense, Activation, Embedding, Input +from keras.models import Model +from my_layers import Attention, Average, WeightedSum, WeightedAspectEmb, MaxMargin + + +logging.basicConfig(level=logging.INFO, + format='%(asctime)s %(levelname)s %(message)s') +logger = logging.getLogger(__name__) + +def create_model(args, maxlen, vocab): + + def ortho_reg(weight_matrix): + ### orthogonal regularization for aspect embedding matrix ### + w_n = weight_matrix / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(weight_matrix), axis=-1, keepdims=True)), K.floatx()) + reg = K.sum(K.square(K.dot(w_n, K.transpose(w_n)) - K.eye(w_n.shape[0]))) + return args.ortho_reg*reg + + vocab_size = len(vocab) + + ##### Inputs ##### + sentence_input = Input(shape=(maxlen,), dtype='int32', name='sentence_input') + neg_input = Input(shape=(args.neg_size, maxlen), dtype='int32', name='neg_input') + + ##### Construct word embedding layer ##### + word_emb = Embedding(vocab_size, args.emb_dim, mask_zero=True, name='word_emb') + print(W) + ##### Compute sentence representation ##### + e_w = word_emb(sentence_input) + y_s = Average()(e_w) + att_weights = Attention(name='att_weights')([e_w, y_s]) + z_s = WeightedSum()([e_w, att_weights]) + + ##### Compute representations of negative instances ##### + e_neg = word_emb(neg_input) + z_n = Average()(e_neg) + + ##### Reconstruction ##### + p_t = Dense(args.aspect_size)(z_s) + p_t = Activation('softmax', name='p_t')(p_t) + r_s = WeightedAspectEmb(args.aspect_size, args.emb_dim, name='aspect_emb', + W_regularizer=ortho_reg)(p_t) + + ##### Loss ##### + loss = MaxMargin(name='max_margin')([z_s, z_n, r_s]) + model = Model(input=[sentence_input, neg_input], output=loss) + + ### Word embedding and aspect embedding initialization ###### + if args.emb_path: + from w2vEmbReader import W2VEmbReader as EmbReader + emb_reader = EmbReader(args.emb_path, emb_dim=args.emb_dim) + logger.info('Initializing word embedding matrix') + model.get_layer('word_emb').W.set_value(emb_reader.get_emb_matrix_given_vocab(vocab, model.get_layer('word_emb').W.get_value())) + logger.info('Initializing aspect embedding matrix as centroid of kmean clusters') + model.get_layer('aspect_emb').W.set_value(emb_reader.get_aspect_matrix(args.aspect_size)) + + return model + + + + + + + + + diff --git a/code/my_layers.py b/code/my_layers.py index f76a124..f529a90 100644 --- a/code/my_layers.py +++ b/code/my_layers.py @@ -1,195 +1,199 @@ -import keras.backend as K -from keras.engine.topology import Layer -from keras import initializations -from keras import regularizers -from keras import constraints -import numpy as np -import theano.tensor as T - -class Attention(Layer): - def __init__(self, W_regularizer=None, b_regularizer=None, - W_constraint=None, b_constraint=None, - bias=True, **kwargs): - """ - Keras Layer that implements an Content Attention mechanism. - Supports Masking. - """ - self.supports_masking = True - self.init = initializations.get('glorot_uniform') - - self.W_regularizer = regularizers.get(W_regularizer) - self.b_regularizer = regularizers.get(b_regularizer) - self.W_constraint = constraints.get(W_constraint) - self.b_constraint = constraints.get(b_constraint) - - self.bias = bias - super(Attention, self).__init__(**kwargs) - - def build(self, input_shape): - assert type(input_shape) == list - assert len(input_shape) == 2 - - self.steps = input_shape[0][1] - - self.W = self.add_weight((input_shape[0][-1], input_shape[1][-1]), - initializer=self.init, - name='{}_W'.format(self.name), - regularizer=self.W_regularizer, - constraint=self.W_constraint) - if self.bias: - self.b = self.add_weight((1,), - initializer='zero', - name='{}_b'.format(self.name), - regularizer=self.b_regularizer, - constraint=self.b_constraint) - self.built = True - - def compute_mask(self, input_tensor, mask=None): - return None - - def call(self, input_tensor, mask=None): - x = input_tensor[0] - y = input_tensor[1] - mask = mask[0] - - y = K.transpose(K.dot(self.W, K.transpose(y))) - y = K.expand_dims(y, dim=-2) - y = K.repeat_elements(y, self.steps, axis=1) - eij = K.sum(x*y, axis=-1) - - if self.bias: - b = K.repeat_elements(self.b, self.steps, axis=0) - eij += b - - eij = K.tanh(eij) - a = K.exp(eij) - - if mask is not None: - a *= K.cast(mask, K.floatx()) - - a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx()) - return a - - def get_output_shape_for(self, input_shape): - return (input_shape[0][0], input_shape[0][1]) - -class WeightedSum(Layer): - def __init__(self, **kwargs): - self.supports_masking = True - super(WeightedSum, self).__init__(**kwargs) - - def call(self, input_tensor, mask=None): - assert type(input_tensor) == list - assert type(mask) == list - - x = input_tensor[0] - a = input_tensor[1] - - a = K.expand_dims(a) - weighted_input = x * a - - return K.sum(weighted_input, axis=1) - - def get_output_shape_for(self, input_shape): - return (input_shape[0][0], input_shape[0][-1]) - - def compute_mask(self, x, mask=None): - return None - -class WeightedAspectEmb(Layer): - def __init__(self, input_dim, output_dim, - init='uniform', input_length=None, - W_regularizer=None, activity_regularizer=None, - W_constraint=None, - weights=None, dropout=0., **kwargs): - self.input_dim = input_dim - self.output_dim = output_dim - self.init = initializations.get(init) - self.input_length = input_length - self.dropout = dropout - - self.W_constraint = constraints.get(W_constraint) - self.W_regularizer = regularizers.get(W_regularizer) - self.activity_regularizer = regularizers.get(activity_regularizer) - - if 0. < self.dropout < 1.: - self.uses_learning_phase = True - self.initial_weights = weights - kwargs['input_shape'] = (self.input_length,) - kwargs['input_dtype'] = K.floatx() - super(WeightedAspectEmb, self).__init__(**kwargs) - - def build(self, input_shape): - self.W = self.add_weight((self.input_dim, self.output_dim), - initializer=self.init, - name='{}_W'.format(self.name), - regularizer=self.W_regularizer, - constraint=self.W_constraint) - - if self.initial_weights is not None: - self.set_weights(self.initial_weights) - self.built = True - - def compute_mask(self, x, mask=None): - return None - - def get_output_shape_for(self, input_shape): - return (input_shape[0], self.output_dim) - - def call(self, x, mask=None): - return K.dot(x, self.W) - - -class Average(Layer): - def __init__(self, **kwargs): - self.supports_masking = True - super(Average, self).__init__(**kwargs) - - def call(self, x, mask=None): - if mask is not None: - mask = K.cast(mask, K.floatx()) - mask = K.expand_dims(mask) - x = x * mask - return K.sum(x, axis=-2) / K.sum(mask, axis=-2) - - def get_output_shape_for(self, input_shape): - return input_shape[0:-2]+input_shape[-1:] - - def compute_mask(self, x, mask=None): - return None - - -class MaxMargin(Layer): - def __init__(self, **kwargs): - super(MaxMargin, self).__init__(**kwargs) - - def call(self, input_tensor, mask=None): - z_s = input_tensor[0] - z_n = input_tensor[1] - r_s = input_tensor[2] - - z_s = z_s / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(z_s), axis=-1, keepdims=True)), K.floatx()) - z_n = z_n / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(z_n), axis=-1, keepdims=True)), K.floatx()) - r_s = r_s / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(r_s), axis=-1, keepdims=True)), K.floatx()) - - steps = z_n.shape[1] - - pos = K.sum(z_s*r_s, axis=-1, keepdims=True) - pos = K.repeat_elements(pos, steps, axis=-1) - r_s = K.expand_dims(r_s, dim=-2) - r_s = K.repeat_elements(r_s, steps, axis=1) - neg = K.sum(z_n*r_s, axis=-1) - - loss = K.cast(K.sum(T.maximum(0., (1. - pos + neg)), axis=-1, keepdims=True), K.floatx()) - return loss - - def compute_mask(self, input_tensor, mask=None): - return None - - def get_output_shape_for(self, input_shape): - return (input_shape[0][0], 1) - - - - - +import keras.backend as K +from keras.engine.topology import Layer +from keras import initializers +from keras import regularizers +from keras import constraints +import numpy as np +import theano.tensor as T + +class Attention(Layer): + def __init__(self, W_regularizer=None, b_regularizer=None, + W_constraint=None, b_constraint=None, + bias=True, **kwargs): + """ + Keras Layer that implements an Content Attention mechanism. + Supports Masking. + """ + self.supports_masking = True + self.init = initializers.get('glorot_uniform') + + self.W_regularizer = regularizers.get(W_regularizer) + self.b_regularizer = regularizers.get(b_regularizer) + self.W_constraint = constraints.get(W_constraint) + self.b_constraint = constraints.get(b_constraint) + + self.bias = bias + super(Attention, self).__init__(**kwargs) + + def build(self, input_shape): + assert type(input_shape) == list + assert len(input_shape) == 2 + + self.steps = input_shape[0][1] + + self.W = self.add_weight(shape=(input_shape[0][-1], input_shape[1][-1]), + initializer=self.init, + name='{}_W'.format(self.name), + regularizer=self.W_regularizer, + constraint=self.W_constraint) + print(self.W) + + if self.bias: + self.b = self.add_weight(shape=(1,), + initializer='zero', + name='{}_b'.format(self.name), + regularizer=self.b_regularizer, + constraint=self.b_constraint) + self.built = True + #super(Attention, self).build(input_shape) + + def compute_mask(self, input_tensor, mask=None): + return None + + def call(self, input_tensor, mask=None): + x = input_tensor[0] + y = input_tensor[1] + mask = mask[0] + + y = K.transpose(K.dot(self.W, K.transpose(y))) + y = K.expand_dims(y, -2) + y = K.repeat_elements(y, self.steps, axis=1) + eij = K.sum(x*y, axis=-1) + + if self.bias: + b = K.repeat_elements(self.b, self.steps, axis=0) + eij += b + + eij = K.tanh(eij) + a = K.exp(eij) + + if mask is not None: + a *= K.cast(mask, K.floatx()) + + a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx()) + return a + + def get_output_shape_for(self, input_shape): + return (input_shape[0][0], input_shape[0][1]) + +class WeightedSum(Layer): + def __init__(self, **kwargs): + self.supports_masking = True + super(WeightedSum, self).__init__(**kwargs) + + def call(self, input_tensor, mask=None): + assert type(input_tensor) == list + assert type(mask) == list + + x = input_tensor[0] + a = input_tensor[1] + + a = K.expand_dims(a) + weighted_input = x * a + + return K.sum(weighted_input, axis=1) + + def get_output_shape_for(self, input_shape): + return (input_shape[0][0], input_shape[0][-1]) + + def compute_mask(self, x, mask=None): + return None + +class WeightedAspectEmb(Layer): + def __init__(self, input_dim, output_dim, + init='uniform', input_length=None, + W_regularizer=None, activity_regularizer=None, + W_constraint=None, + weights=None, dropout=0., **kwargs): + self.input_dim = input_dim + self.output_dim = output_dim + self.init = initializers.get(init) + self.input_length = input_length + self.dropout = dropout + + self.W_constraint = constraints.get(W_constraint) + self.W_regularizer = regularizers.get(W_regularizer) + self.activity_regularizer = regularizers.get(activity_regularizer) + + if 0. < self.dropout < 1.: + self.uses_learning_phase = True + self.initial_weights = weights + kwargs['input_shape'] = (self.input_length,) + kwargs['input_dtype'] = K.floatx() + super(WeightedAspectEmb, self).__init__(**kwargs) + + def build(self, input_shape): + self.W = self.add_weight(shape=(self.input_dim, self.output_dim), + initializer=self.init, + name='{}_W'.format(self.name), + regularizer=self.W_regularizer, + constraint=self.W_constraint) + + if self.initial_weights is not None: + self.set_weights(self.initial_weights) + #super(WeightedAspectEmb, self).build(input_shape) + self.built = True + + def compute_mask(self, x, mask=None): + return None + + def get_output_shape_for(self, input_shape): + return (input_shape[0], self.output_dim) + + def call(self, x, mask=None): + return K.dot(x, self.W) + + +class Average(Layer): + def __init__(self, **kwargs): + self.supports_masking = True + super(Average, self).__init__(**kwargs) + + def call(self, x, mask=None): + if mask is not None: + mask = K.cast(mask, K.floatx()) + mask = K.expand_dims(mask) + x = x * mask + return K.sum(x, axis=-2) / K.sum(mask, axis=-2) + + def get_output_shape_for(self, input_shape): + return input_shape[0:-2]+input_shape[-1:] + + def compute_mask(self, x, mask=None): + return None + + +class MaxMargin(Layer): + def __init__(self, **kwargs): + super(MaxMargin, self).__init__(**kwargs) + + def call(self, input_tensor, mask=None): + z_s = input_tensor[0] + z_n = input_tensor[1] + r_s = input_tensor[2] + + z_s = z_s / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(z_s), axis=-1, keepdims=True)), K.floatx()) + z_n = z_n / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(z_n), axis=-1, keepdims=True)), K.floatx()) + r_s = r_s / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(r_s), axis=-1, keepdims=True)), K.floatx()) + + steps = z_n.shape[1] + + pos = K.sum(z_s*r_s, axis=-1, keepdims=True) + pos = K.repeat_elements(pos, steps, axis=-1) + r_s = K.expand_dims(r_s, -2) + r_s = K.repeat_elements(r_s, steps, axis=1) + neg = K.sum(z_n*r_s, axis=-1) + + loss = K.cast(K.sum(K.maximum(0., (1. - pos + neg)), axis=-1, keepdims=True), K.floatx()) + return loss + + def compute_mask(self, input_tensor, mask=None): + return None + + def get_output_shape_for(self, input_shape): + return (input_shape[0][0], 1) + + + + + diff --git a/code/optimizers.py b/code/optimizers.py old mode 100755 new mode 100644 index 556eb53..6c04d4d --- a/code/optimizers.py +++ b/code/optimizers.py @@ -1,21 +1,21 @@ -import keras.optimizers as opt - -def get_optimizer(args): - - clipvalue = 0 - clipnorm = 10 - - if args.algorithm == 'rmsprop': - optimizer = opt.RMSprop(lr=0.001, rho=0.9, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue) - elif args.algorithm == 'sgd': - optimizer = opt.SGD(lr=0.01, momentum=0.0, decay=0.0, nesterov=False, clipnorm=clipnorm, clipvalue=clipvalue) - elif args.algorithm == 'adagrad': - optimizer = opt.Adagrad(lr=0.01, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue) - elif args.algorithm == 'adadelta': - optimizer = opt.Adadelta(lr=1.0, rho=0.95, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue) - elif args.algorithm == 'adam': - optimizer = opt.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=clipnorm, clipvalue=clipvalue) - elif args.algorithm == 'adamax': - optimizer = opt.Adamax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=clipnorm, clipvalue=clipvalue) - - return optimizer +import keras.optimizers as opt + +def get_optimizer(args): + + clipvalue = 0 + clipnorm = 10 + + if args.algorithm == 'rmsprop': + optimizer = opt.RMSprop(lr=0.001, rho=0.9, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue) + elif args.algorithm == 'sgd': + optimizer = opt.SGD(lr=0.01, momentum=0.0, decay=0.0, nesterov=False, clipnorm=clipnorm, clipvalue=clipvalue) + elif args.algorithm == 'adagrad': + optimizer = opt.Adagrad(lr=0.01, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue) + elif args.algorithm == 'adadelta': + optimizer = opt.Adadelta(lr=1.0, rho=0.95, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue) + elif args.algorithm == 'adam': + optimizer = opt.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=clipnorm, clipvalue=clipvalue) + elif args.algorithm == 'adamax': + optimizer = opt.Adamax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=clipnorm, clipvalue=clipvalue) + + return optimizer diff --git a/code/preprocess.py b/code/preprocess.py index 6932eb7..aaddc07 100644 --- a/code/preprocess.py +++ b/code/preprocess.py @@ -1,51 +1,51 @@ -from sklearn.feature_extraction.text import CountVectorizer -from nltk.corpus import stopwords -from nltk.stem.wordnet import WordNetLemmatizer -import codecs - -def parseSentence(line): - lmtzr = WordNetLemmatizer() - stop = stopwords.words('english') - text_token = CountVectorizer().build_tokenizer()(line.lower()) - text_rmstop = [i for i in text_token if i not in stop] - text_stem = [lmtzr.lemmatize(w) for w in text_rmstop] - return text_stem - -def preprocess_train(domain): - f = codecs.open('../datasets/'+domain+'/train.txt', 'r', 'utf-8') - out = codecs.open('../preprocessed_data/'+domain+'/train.txt', 'w', 'utf-8') - - for line in f: - tokens = parseSentence(line) - if len(tokens) > 0: - out.write(' '.join(tokens)+'\n') - -def preprocess_test(domain): - # For restaurant domain, only keep sentences with single - # aspect label that in {Food, Staff, Ambience} - - f1 = codecs.open('../datasets/'+domain+'/test.txt', 'r', 'utf-8') - f2 = codecs.open('../datasets/'+domain+'/test_label.txt', 'r', 'utf-8') - out1 = codecs.open('../preprocessed_data/'+domain+'/test.txt', 'w', 'utf-8') - out2 = codecs.open('../preprocessed_data/'+domain+'/test_label.txt', 'w', 'utf-8') - - for text, label in zip(f1, f2): - label = label.strip() - if domain == 'restaurant' and label not in ['Food', 'Staff', 'Ambience']: - continue - tokens = parseSentence(text) - if len(tokens) > 0: - out1.write(' '.join(tokens) + '\n') - out2.write(label+'\n') - -def preprocess(domain): - print '\t'+domain+' train set ...' - preprocess_train(domain) - print '\t'+domain+' test set ...' - preprocess_test(domain) - -print 'Preprocessing raw review sentences ...' -preprocess('restaurant') -preprocess('beer') - - +from sklearn.feature_extraction.text import CountVectorizer +from nltk.corpus import stopwords +from nltk.stem.wordnet import WordNetLemmatizer +import codecs + +def parseSentence(line): + lmtzr = WordNetLemmatizer() + stop = stopwords.words('english') + text_token = CountVectorizer().build_tokenizer()(line.lower()) + text_rmstop = [i for i in text_token if i not in stop] + text_stem = [lmtzr.lemmatize(w) for w in text_rmstop] + return text_stem + +def preprocess_train(domain): + f = codecs.open('../datasets/'+domain+'/train.txt', 'r', 'utf-8') + out = codecs.open('../preprocessed_data/'+domain+'/train.txt', 'w', 'utf-8') + + for line in f: + tokens = parseSentence(line) + if len(tokens) > 0: + out.write(' '.join(tokens)+'\n') + +def preprocess_test(domain): + # For restaurant domain, only keep sentences with single + # aspect label that in {Food, Staff, Ambience} + + f1 = codecs.open('../datasets/'+domain+'/test.txt', 'r', 'utf-8') + f2 = codecs.open('../datasets/'+domain+'/test_label.txt', 'r', 'utf-8') + out1 = codecs.open('../preprocessed_data/'+domain+'/test.txt', 'w', 'utf-8') + out2 = codecs.open('../preprocessed_data/'+domain+'/test_label.txt', 'w', 'utf-8') + + for text, label in zip(f1, f2): + label = label.strip() + if domain == 'restaurant' and label not in ['Food', 'Staff', 'Ambience']: + continue + tokens = parseSentence(text) + if len(tokens) > 0: + out1.write(' '.join(tokens) + '\n') + out2.write(label+'\n') + +def preprocess(domain): + print('\t'+domain+' train set ...') + preprocess_train(domain) + print('\t'+domain+' test set ...') + preprocess_test(domain) + +print('Preprocessing raw review sentences ...') +preprocess('restaurant') +preprocess('beer') + + diff --git a/code/reader.py b/code/reader.py index 9737938..c6f7bdb 100644 --- a/code/reader.py +++ b/code/reader.py @@ -1,113 +1,113 @@ -import codecs -import re -import operator - -num_regex = re.compile('^[+-]?[0-9]+\.?[0-9]*$') - -def is_number(token): - return bool(num_regex.match(token)) - -def create_vocab(domain, maxlen=0, vocab_size=0): - assert domain in {'restaurant', 'beer'} - source = '../preprocessed_data/'+domain+'/train.txt' - - total_words, unique_words = 0, 0 - word_freqs = {} - top = 0 - - fin = codecs.open(source, 'r', 'utf-8') - for line in fin: - words = line.split() - if maxlen > 0 and len(words) > maxlen: - continue - - for w in words: - if not is_number(w): - try: - word_freqs[w] += 1 - except KeyError: - unique_words += 1 - word_freqs[w] = 1 - total_words += 1 - - print (' %i total words, %i unique words' % (total_words, unique_words)) - sorted_word_freqs = sorted(word_freqs.items(), key=operator.itemgetter(1), reverse=True) - - vocab = {'':0, '':1, '':2} - index = len(vocab) - for word, _ in sorted_word_freqs: - vocab[word] = index - index += 1 - if vocab_size > 0 and index > vocab_size + 2: - break - if vocab_size > 0: - print (' keep the top %i words' % vocab_size) - - #Write (vocab, frequence) to a txt file - vocab_file = codecs.open('../preprocessed_data/%s/vocab' % domain, mode='w', encoding='utf8') - sorted_vocab = sorted(vocab.items(), key=operator.itemgetter(1)) - for word, index in sorted_vocab: - if index < 3: - vocab_file.write(word+'\t'+str(0)+'\n') - continue - vocab_file.write(word+'\t'+str(word_freqs[word])+'\n') - vocab_file.close() - - return vocab - -def read_dataset(domain, phase, vocab, maxlen): - assert domain in {'restaurant', 'beer'} - assert phase in {'train', 'test'} - - source = '../preprocessed_data/'+domain+'/'+phase+'.txt' - num_hit, unk_hit, total = 0., 0., 0. - maxlen_x = 0 - data_x = [] - - fin = codecs.open(source, 'r', 'utf-8') - for line in fin: - words = line.strip().split() - if maxlen > 0 and len(words) > maxlen: - continue - - indices = [] - for word in words: - if is_number(word): - indices.append(vocab['']) - num_hit += 1 - elif word in vocab: - indices.append(vocab[word]) - else: - indices.append(vocab['']) - unk_hit += 1 - total += 1 - - data_x.append(indices) - if maxlen_x < len(indices): - maxlen_x = len(indices) - - print ' hit rate: %.2f%%, hit rate: %.2f%%' % (100*num_hit/total, 100*unk_hit/total) - return data_x, maxlen_x - - - -def get_data(domain, vocab_size=0, maxlen=0): - print 'Reading data from', domain - print ' Creating vocab ...' - vocab = create_vocab(domain, maxlen, vocab_size) - print ' Reading dataset ...' - print ' train set' - train_x, train_maxlen = read_dataset(domain, 'train', vocab, maxlen) - print ' test set' - test_x, test_maxlen = read_dataset(domain, 'test', vocab, maxlen) - maxlen = max(train_maxlen, test_maxlen) - return vocab, train_x, test_x, maxlen - - - -if __name__ == "__main__": - vocab, train_x, test_x, maxlen = get_data('restaurant') - print len(train_x) - print len(test_x) - print maxlen - +import codecs +import re +import operator + +num_regex = re.compile('^[+-]?[0-9]+\.?[0-9]*$') + +def is_number(token): + return bool(num_regex.match(token)) + +def create_vocab(domain, maxlen=0, vocab_size=0): + assert domain in {'restaurant', 'beer'} + source = '../preprocessed_data/'+domain+'/train.txt' + + total_words, unique_words = 0, 0 + word_freqs = {} + top = 0 + + fin = codecs.open(source, 'r', 'utf-8') + for line in fin: + words = line.split() + if maxlen > 0 and len(words) > maxlen: + continue + + for w in words: + if not is_number(w): + try: + word_freqs[w] += 1 + except KeyError: + unique_words += 1 + word_freqs[w] = 1 + total_words += 1 + + print(' %i total words, %i unique words' % (total_words, unique_words)) + sorted_word_freqs = sorted(word_freqs.items(), key=operator.itemgetter(1), reverse=True) + + vocab = {'':0, '':1, '':2} + index = len(vocab) + for word, _ in sorted_word_freqs: + vocab[word] = index + index += 1 + if vocab_size > 0 and index > vocab_size + 2: + break + if vocab_size > 0: + print(' keep the top %i words' % vocab_size) + + #Write (vocab, frequence) to a txt file + vocab_file = codecs.open('../preprocessed_data/%s/vocab' % domain, mode='w', encoding='utf8') + sorted_vocab = sorted(vocab.items(), key=operator.itemgetter(1)) + for word, index in sorted_vocab: + if index < 3: + vocab_file.write(word+'\t'+str(0)+'\n') + continue + vocab_file.write(word+'\t'+str(word_freqs[word])+'\n') + vocab_file.close() + + return vocab + +def read_dataset(domain, phase, vocab, maxlen): + assert domain in {'restaurant', 'beer'} + assert phase in {'train', 'test'} + + source = '../preprocessed_data/'+domain+'/'+phase+'.txt' + num_hit, unk_hit, total = 0., 0., 0. + maxlen_x = 0 + data_x = [] + + fin = codecs.open(source, 'r', 'utf-8') + for line in fin: + words = line.strip().split() + if maxlen > 0 and len(words) > maxlen: + continue + + indices = [] + for word in words: + if is_number(word): + indices.append(vocab['']) + num_hit += 1 + elif word in vocab: + indices.append(vocab[word]) + else: + indices.append(vocab['']) + unk_hit += 1 + total += 1 + + data_x.append(indices) + if maxlen_x < len(indices): + maxlen_x = len(indices) + + print(' hit rate: %.2f%%, hit rate: %.2f%%' % (100*num_hit/total, 100*unk_hit/total)) + return data_x, maxlen_x + + + +def get_data(domain, vocab_size=0, maxlen=0): + print('Reading data from', domain) + print(' Creating vocab ...') + vocab = create_vocab(domain, maxlen, vocab_size) + print(' Reading dataset ...') + print(' train set') + train_x, train_maxlen = read_dataset(domain, 'train', vocab, maxlen) + print(' test set') + test_x, test_maxlen = read_dataset(domain, 'test', vocab, maxlen) + maxlen = max(train_maxlen, test_maxlen) + return vocab, train_x, test_x, maxlen + + + +if __name__ == "__main__": + vocab, train_x, test_x, maxlen = get_data('restaurant') + print(len(train_x)) + print(len(test_x)) + print(maxlen) + diff --git a/code/run_script.sh b/code/run_script.sh old mode 100755 new mode 100644 index a614ab2..e17c105 --- a/code/run_script.sh +++ b/code/run_script.sh @@ -1,7 +1,7 @@ - - -THEANO_FLAGS="device=gpu0,floatX=float32" python train.py \ ---emb ../preprocessed_data/restaurant/w2v_embedding \ ---domain restaurant \ --o output_dir \ - + + +THEANO_FLAGS="device=gpu0,floatX=float32" python train.py \ +--emb ../preprocessed_data/restaurant/w2v_embedding \ +--domain restaurant \ +-o output_dir \ + diff --git a/code/train.py b/code/train.py index 6b1fdf0..12b0ede 100644 --- a/code/train.py +++ b/code/train.py @@ -1,175 +1,178 @@ -import argparse -import logging -import numpy as np -from time import time -import utils as U -import codecs - -logging.basicConfig( - #filename='out.log', - level=logging.INFO, - format='%(asctime)s %(levelname)s %(message)s') -logger = logging.getLogger(__name__) - - -############################################################################################################################### -## Parse arguments -# - -parser = argparse.ArgumentParser() -parser.add_argument("-o", "--out-dir", dest="out_dir_path", type=str, metavar='', required=True, help="The path to the output directory") -parser.add_argument("-e", "--embdim", dest="emb_dim", type=int, metavar='', default=200, help="Embeddings dimension (default=200)") -parser.add_argument("-b", "--batch-size", dest="batch_size", type=int, metavar='', default=50, help="Batch size (default=50)") -parser.add_argument("-v", "--vocab-size", dest="vocab_size", type=int, metavar='', default=9000, help="Vocab size. '0' means no limit (default=9000)") -parser.add_argument("-as", "--aspect-size", dest="aspect_size", type=int, metavar='', default=14, help="The number of aspects specified by users (default=14)") -parser.add_argument("--emb", dest="emb_path", type=str, metavar='', help="The path to the word embeddings file") -parser.add_argument("--epochs", dest="epochs", type=int, metavar='', default=15, help="Number of epochs (default=15)") -parser.add_argument("-n", "--neg-size", dest="neg_size", type=int, metavar='', default=20, help="Number of negative instances (default=20)") -parser.add_argument("--maxlen", dest="maxlen", type=int, metavar='', default=0, help="Maximum allowed number of words during training. '0' means no limit (default=0)") -parser.add_argument("--seed", dest="seed", type=int, metavar='', default=1234, help="Random seed (default=1234)") -parser.add_argument("-a", "--algorithm", dest="algorithm", type=str, metavar='', default='adam', help="Optimization algorithm (rmsprop|sgd|adagrad|adadelta|adam|adamax) (default=adam)") -parser.add_argument("--domain", dest="domain", type=str, metavar='', default='restaurant', help="domain of the corpus {restaurant, beer}") -parser.add_argument("--ortho-reg", dest="ortho_reg", type=float, metavar='', default=0.1, help="The weight of orthogonol regularizaiton (default=0.1)") - -args = parser.parse_args() -out_dir = args.out_dir_path + '/' + args.domain -U.mkdir_p(out_dir) -U.print_args(args) - -assert args.algorithm in {'rmsprop', 'sgd', 'adagrad', 'adadelta', 'adam', 'adamax'} -assert args.domain in {'restaurant', 'beer'} - -if args.seed > 0: - np.random.seed(args.seed) - - -# ############################################################################################################################### -# ## Prepare data -# # - -from keras.preprocessing import sequence -import reader as dataset - -vocab, train_x, test_x, overall_maxlen = dataset.get_data(args.domain, vocab_size=args.vocab_size, maxlen=args.maxlen) -train_x = sequence.pad_sequences(train_x, maxlen=overall_maxlen) -test_x = sequence.pad_sequences(test_x, maxlen=overall_maxlen) - -print 'Number of training examples: ', len(train_x) -print 'Length of vocab: ', len(vocab) - -def sentence_batch_generator(data, batch_size): - n_batch = len(data) / batch_size - batch_count = 0 - np.random.shuffle(data) - - while True: - if batch_count == n_batch: - np.random.shuffle(data) - batch_count = 0 - - batch = data[batch_count*batch_size: (batch_count+1)*batch_size] - batch_count += 1 - yield batch - -def negative_batch_generator(data, batch_size, neg_size): - data_len = data.shape[0] - dim = data.shape[1] - - while True: - indices = np.random.choice(data_len, batch_size * neg_size) - samples = data[indices].reshape(batch_size, neg_size, dim) - yield samples - - - -############################################################################################################################### -## Optimizaer algorithm -# - -from optimizers import get_optimizer - -optimizer = get_optimizer(args) - - - -############################################################################################################################### -## Building model - -from model import create_model -import keras.backend as K - -logger.info(' Building model') - - -def max_margin_loss(y_true, y_pred): - return K.mean(y_pred) - -model = create_model(args, overall_maxlen, vocab) -# freeze the word embedding layer -model.get_layer('word_emb').trainable=False -model.compile(optimizer=optimizer, loss=max_margin_loss, metrics=[max_margin_loss]) - - -############################################################################################################################### -## Training -# -from keras.models import load_model -from tqdm import tqdm - -logger.info('--------------------------------------------------------------------------------------------------------------------------') - -vocab_inv = {} -for w, ind in vocab.items(): - vocab_inv[ind] = w - - -sen_gen = sentence_batch_generator(train_x, args.batch_size) -neg_gen = negative_batch_generator(train_x, args.batch_size, args.neg_size) -# batches_per_epoch = len(train_x) / args.batch_size -batches_per_epoch = 1000 - -min_loss = float('inf') -for ii in xrange(args.epochs): - t0 = time() - loss, max_margin_loss = 0., 0. - - for b in tqdm(xrange(batches_per_epoch)): - sen_input = sen_gen.next() - neg_input = neg_gen.next() - - batch_loss, batch_max_margin_loss = model.train_on_batch([sen_input, neg_input], np.ones((args.batch_size, 1))) - loss += batch_loss / batches_per_epoch - max_margin_loss += batch_max_margin_loss / batches_per_epoch - - tr_time = time() - t0 - - if loss < min_loss: - min_loss = loss - word_emb = model.get_layer('word_emb').W.get_value() - aspect_emb = model.get_layer('aspect_emb').W.get_value() - word_emb = word_emb / np.linalg.norm(word_emb, axis=-1, keepdims=True) - aspect_emb = aspect_emb / np.linalg.norm(aspect_emb, axis=-1, keepdims=True) - aspect_file = codecs.open(out_dir+'/aspect.log', 'w', 'utf-8') - model.save_weights(out_dir+'/model_param') - - for ind in range(len(aspect_emb)): - desc = aspect_emb[ind] - sims = word_emb.dot(desc.T) - ordered_words = np.argsort(sims)[::-1] - desc_list = [vocab_inv[w] for w in ordered_words[:100]] - print 'Aspect %d:' % ind - print desc_list - aspect_file.write('Aspect %d:\n' % ind) - aspect_file.write(' '.join(desc_list) + '\n\n') - - logger.info('Epoch %d, train: %is' % (ii, tr_time)) - logger.info('Total loss: %.4f, max_margin_loss: %.4f, ortho_reg: %.4f' % (loss, max_margin_loss, loss-max_margin_loss)) - - - - - - - - - +import argparse +import logging +import numpy as np +from time import time +import utils as U +import codecs +import os +os.environ["MKL_THREADING_LAYER"] = "GNU" + +logging.basicConfig( + #filename='out.log', + level=logging.INFO, + format='%(asctime)s %(levelname)s %(message)s') +logger = logging.getLogger(__name__) + + +############################################################################################################################### +## Parse arguments +# + +parser = argparse.ArgumentParser() +parser.add_argument("-o", "--out-dir", dest="out_dir_path", type=str, metavar='', required=True, help="The path to the output directory") +parser.add_argument("-e", "--embdim", dest="emb_dim", type=int, metavar='', default=200, help="Embeddings dimension (default=200)") +parser.add_argument("-b", "--batch-size", dest="batch_size", type=int, metavar='', default=50, help="Batch size (default=50)") +parser.add_argument("-v", "--vocab-size", dest="vocab_size", type=int, metavar='', default=9000, help="Vocab size. '0' means no limit (default=9000)") +parser.add_argument("-as", "--aspect-size", dest="aspect_size", type=int, metavar='', default=14, help="The number of aspects specified by users (default=14)") +parser.add_argument("--emb", dest="emb_path", type=str, metavar='', help="The path to the word embeddings file") +parser.add_argument("--epochs", dest="epochs", type=int, metavar='', default=15, help="Number of epochs (default=15)") +parser.add_argument("-n", "--neg-size", dest="neg_size", type=int, metavar='', default=20, help="Number of negative instances (default=20)") +parser.add_argument("--maxlen", dest="maxlen", type=int, metavar='', default=0, help="Maximum allowed number of words during training. '0' means no limit (default=0)") +parser.add_argument("--seed", dest="seed", type=int, metavar='', default=1234, help="Random seed (default=1234)") +parser.add_argument("-a", "--algorithm", dest="algorithm", type=str, metavar='', default='adam', help="Optimization algorithm (rmsprop|sgd|adagrad|adadelta|adam|adamax) (default=adam)") +parser.add_argument("--domain", dest="domain", type=str, metavar='', default='restaurant', help="domain of the corpus {restaurant, beer}") +parser.add_argument("--ortho-reg", dest="ortho_reg", type=float, metavar='', default=0.1, help="The weight of orthogonol regularizaiton (default=0.1)") + +args = parser.parse_args() +out_dir = args.out_dir_path + '/' + args.domain +U.mkdir_p(out_dir) +U.print_args(args) + +assert args.algorithm in {'rmsprop', 'sgd', 'adagrad', 'adadelta', 'adam', 'adamax'} +assert args.domain in {'restaurant', 'beer'} + +if args.seed > 0: + np.random.seed(args.seed) + + +# ############################################################################################################################### +# ## Prepare data +# # + +from keras.preprocessing import sequence +import reader as dataset + +vocab, train_x, test_x, overall_maxlen = dataset.get_data(args.domain, vocab_size=args.vocab_size, maxlen=args.maxlen) +train_x = sequence.pad_sequences(train_x, maxlen=overall_maxlen) +test_x = sequence.pad_sequences(test_x, maxlen=overall_maxlen) + +print('Number of training examples: ', len(train_x)) +print('Length of vocab: ', len(vocab)) + +def sentence_batch_generator(data, batch_size): + n_batch = len(data) / batch_size + batch_count = 0 + np.random.shuffle(data) + + while True: + if batch_count == n_batch: + np.random.shuffle(data) + batch_count = 0 + + batch = data[batch_count*batch_size: (batch_count+1)*batch_size] + batch_count += 1 + yield batch + +def negative_batch_generator(data, batch_size, neg_size): + data_len = data.shape[0] + dim = data.shape[1] + + while True: + indices = np.random.choice(data_len, batch_size * neg_size) + samples = data[indices].reshape(batch_size, neg_size, dim) + yield samples + + + +############################################################################################################################### +## Optimizaer algorithm +# + +from optimizers import get_optimizer + +optimizer = get_optimizer(args) + + + +############################################################################################################################### +## Building model + +from model import create_model +import keras.backend as K + +logger.info(' Building model') + +print(W) +print("***") +def max_margin_loss(y_true, y_pred): + return K.mean(y_pred) + +model = create_model(args, overall_maxlen, vocab) +# freeze the word embedding layer +model.get_layer('word_emb').trainable=False +model.compile(optimizer=optimizer, loss=max_margin_loss, metrics=[max_margin_loss]) + + +############################################################################################################################### +## Training +# +from keras.models import load_model +from tqdm import tqdm + +logger.info('--------------------------------------------------------------------------------------------------------------------------') + +vocab_inv = {} +for w, ind in vocab.items(): + vocab_inv[ind] = w + + +sen_gen = sentence_batch_generator(train_x, args.batch_size) +neg_gen = negative_batch_generator(train_x, args.batch_size, args.neg_size) +# batches_per_epoch = len(train_x) / args.batch_size +batches_per_epoch = 1000 + +min_loss = float('inf') +for ii in range(args.epochs): + t0 = time() + loss, max_margin_loss = 0., 0. + + for b in tqdm(range(batches_per_epoch)): + sen_input = next(sen_gen) + neg_input = next(neg_gen) + + batch_loss, batch_max_margin_loss = model.train_on_batch([sen_input, neg_input], np.ones((args.batch_size, 1))) + loss += batch_loss / batches_per_epoch + max_margin_loss += batch_max_margin_loss / batches_per_epoch + + tr_time = time() - t0 + + if loss < min_loss: + min_loss = loss + word_emb = model.get_layer('word_emb').W.get_value() + aspect_emb = model.get_layer('aspect_emb').W.get_value() + word_emb = word_emb / np.linalg.norm(word_emb, axis=-1, keepdims=True) + aspect_emb = aspect_emb / np.linalg.norm(aspect_emb, axis=-1, keepdims=True) + aspect_file = codecs.open(out_dir+'/aspect.log', 'w', 'utf-8') + model.save_weights(out_dir+'/model_param') + + for ind in range(len(aspect_emb)): + desc = aspect_emb[ind] + sims = word_emb.dot(desc.T) + ordered_words = np.argsort(sims)[::-1] + desc_list = [vocab_inv[w] for w in ordered_words[:100]] + print('Aspect %d:' % ind) + print(desc_list) + aspect_file.write('Aspect %d:\n' % ind) + aspect_file.write(' '.join(desc_list) + '\n\n') + + logger.info('Epoch %d, train: %is' % (ii, tr_time)) + logger.info('Total loss: %.4f, max_margin_loss: %.4f, ortho_reg: %.4f' % (loss, max_margin_loss, loss-max_margin_loss)) + + + + + + + + + diff --git a/code/utils.py b/code/utils.py old mode 100755 new mode 100644 index a134e62..33509ac --- a/code/utils.py +++ b/code/utils.py @@ -1,165 +1,165 @@ -import sys -import os, errno -import logging - -#-----------------------------------------------------------------------------------------------------------# - -def set_logger(out_dir=None): - console_format = BColors.OKBLUE + '[%(levelname)s]' + BColors.ENDC + ' (%(name)s) %(message)s' - #datefmt='%Y-%m-%d %Hh-%Mm-%Ss' - logger = logging.getLogger() - logger.setLevel(logging.DEBUG) - console = logging.StreamHandler() - console.setLevel(logging.DEBUG) - console.setFormatter(logging.Formatter(console_format)) - logger.addHandler(console) - if out_dir: - file_format = '[%(levelname)s] (%(name)s) %(message)s' - log_file = logging.FileHandler(out_dir + '/log.txt', mode='w') - log_file.setLevel(logging.DEBUG) - log_file.setFormatter(logging.Formatter(file_format)) - logger.addHandler(log_file) - -#-----------------------------------------------------------------------------------------------------------# - -def mkdir_p(path): - if path == '': - return - try: - os.makedirs(path) - except OSError as exc: # Python >2.5 - if exc.errno == errno.EEXIST and os.path.isdir(path): - pass - else: raise - -def get_root_dir(): - return os.path.dirname(sys.argv[0]) - -def bincounts(array): - num_rows = array.shape[0] - if array.ndim > 1: - num_cols = array.shape[1] - else: - num_cols = 1 - array = array[:, None] - counters = [] - mfe_list = [] - for col in range(num_cols): - counter = {} - for row in range(num_rows): - element = array[row,col] - if element in counter: - counter[element] += 1 - else: - counter[element] = 1 - max_count = 0 - for element in counter: - if counter[element] > max_count: - max_count = counter[element] - mfe = element - counters.append(counter) - mfe_list.append(mfe) - return counters, mfe_list - -# Convert all arguments to strings -def ltos(*args): - outputs = [] - for arg in args: - if type(arg) == list: - out = ' '.join(['%.3f' % e for e in arg]) - if len(arg) == 1: - outputs.append(out) - else: - outputs.append('[' + out + ']') - else: - outputs.append(str(arg)) - return tuple(outputs) - -#-----------------------------------------------------------------------------------------------------------# - -import re - -class BColors: - HEADER = '\033[95m' - OKBLUE = '\033[94m' - OKGREEN = '\033[92m' - WARNING = '\033[93m' - FAIL = '\033[91m' - ENDC = '\033[0m' - BOLD = '\033[1m' - UNDERLINE = '\033[4m' - WHITE = '\033[37m' - YELLOW = '\033[33m' - GREEN = '\033[32m' - BLUE = '\033[34m' - CYAN = '\033[36m' - RED = '\033[31m' - MAGENTA = '\033[35m' - BLACK = '\033[30m' - BHEADER = BOLD + '\033[95m' - BOKBLUE = BOLD + '\033[94m' - BOKGREEN = BOLD + '\033[92m' - BWARNING = BOLD + '\033[93m' - BFAIL = BOLD + '\033[91m' - BUNDERLINE = BOLD + '\033[4m' - BWHITE = BOLD + '\033[37m' - BYELLOW = BOLD + '\033[33m' - BGREEN = BOLD + '\033[32m' - BBLUE = BOLD + '\033[34m' - BCYAN = BOLD + '\033[36m' - BRED = BOLD + '\033[31m' - BMAGENTA = BOLD + '\033[35m' - BBLACK = BOLD + '\033[30m' - - @staticmethod - def cleared(s): - return re.sub("\033\[[0-9][0-9]?m", "", s) - -def red(message): - return BColors.RED + str(message) + BColors.ENDC - -def b_red(message): - return BColors.BRED + str(message) + BColors.ENDC - -def blue(message): - return BColors.BLUE + str(message) + BColors.ENDC - -def b_yellow(message): - return BColors.BYELLOW + str(message) + BColors.ENDC - -def green(message): - return BColors.GREEN + str(message) + BColors.ENDC - -def b_green(message): - return BColors.BGREEN + str(message) + BColors.ENDC - -#-----------------------------------------------------------------------------------------------------------# - -def print_args(args, path=None): - if path: - output_file = open(path, 'w') - logger = logging.getLogger(__name__) - logger.info("Arguments:") - args.command = ' '.join(sys.argv) - items = vars(args) - for key in sorted(items.keys(), key=lambda s: s.lower()): - value = items[key] - if not value: - value = "None" - logger.info(" " + key + ": " + str(items[key])) - if path is not None: - output_file.write(" " + key + ": " + str(items[key]) + "\n") - if path: - output_file.close() - del args.command - -def get_args(args): - items = vars(args) - output_string = '' - for key in sorted(items.keys(), key=lambda s: s.lower()): - value = items[key] - if not value: - value = "None" - output_string += " " + key + ": " + str(items[key] + "\n") - return output_string - +import sys +import os, errno +import logging + +#-----------------------------------------------------------------------------------------------------------# + +def set_logger(out_dir=None): + console_format = BColors.OKBLUE + '[%(levelname)s]' + BColors.ENDC + ' (%(name)s) %(message)s' + #datefmt='%Y-%m-%d %Hh-%Mm-%Ss' + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + console = logging.StreamHandler() + console.setLevel(logging.DEBUG) + console.setFormatter(logging.Formatter(console_format)) + logger.addHandler(console) + if out_dir: + file_format = '[%(levelname)s] (%(name)s) %(message)s' + log_file = logging.FileHandler(out_dir + '/log.txt', mode='w') + log_file.setLevel(logging.DEBUG) + log_file.setFormatter(logging.Formatter(file_format)) + logger.addHandler(log_file) + +#-----------------------------------------------------------------------------------------------------------# + +def mkdir_p(path): + if path == '': + return + try: + os.makedirs(path) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: raise + +def get_root_dir(): + return os.path.dirname(sys.argv[0]) + +def bincounts(array): + num_rows = array.shape[0] + if array.ndim > 1: + num_cols = array.shape[1] + else: + num_cols = 1 + array = array[:, None] + counters = [] + mfe_list = [] + for col in range(num_cols): + counter = {} + for row in range(num_rows): + element = array[row,col] + if element in counter: + counter[element] += 1 + else: + counter[element] = 1 + max_count = 0 + for element in counter: + if counter[element] > max_count: + max_count = counter[element] + mfe = element + counters.append(counter) + mfe_list.append(mfe) + return counters, mfe_list + +# Convert all arguments to strings +def ltos(*args): + outputs = [] + for arg in args: + if type(arg) == list: + out = ' '.join(['%.3f' % e for e in arg]) + if len(arg) == 1: + outputs.append(out) + else: + outputs.append('[' + out + ']') + else: + outputs.append(str(arg)) + return tuple(outputs) + +#-----------------------------------------------------------------------------------------------------------# + +import re + +class BColors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + WHITE = '\033[37m' + YELLOW = '\033[33m' + GREEN = '\033[32m' + BLUE = '\033[34m' + CYAN = '\033[36m' + RED = '\033[31m' + MAGENTA = '\033[35m' + BLACK = '\033[30m' + BHEADER = BOLD + '\033[95m' + BOKBLUE = BOLD + '\033[94m' + BOKGREEN = BOLD + '\033[92m' + BWARNING = BOLD + '\033[93m' + BFAIL = BOLD + '\033[91m' + BUNDERLINE = BOLD + '\033[4m' + BWHITE = BOLD + '\033[37m' + BYELLOW = BOLD + '\033[33m' + BGREEN = BOLD + '\033[32m' + BBLUE = BOLD + '\033[34m' + BCYAN = BOLD + '\033[36m' + BRED = BOLD + '\033[31m' + BMAGENTA = BOLD + '\033[35m' + BBLACK = BOLD + '\033[30m' + + @staticmethod + def cleared(s): + return re.sub("\033\[[0-9][0-9]?m", "", s) + +def red(message): + return BColors.RED + str(message) + BColors.ENDC + +def b_red(message): + return BColors.BRED + str(message) + BColors.ENDC + +def blue(message): + return BColors.BLUE + str(message) + BColors.ENDC + +def b_yellow(message): + return BColors.BYELLOW + str(message) + BColors.ENDC + +def green(message): + return BColors.GREEN + str(message) + BColors.ENDC + +def b_green(message): + return BColors.BGREEN + str(message) + BColors.ENDC + +#-----------------------------------------------------------------------------------------------------------# + +def print_args(args, path=None): + if path: + output_file = open(path, 'w') + logger = logging.getLogger(__name__) + logger.info("Arguments:") + args.command = ' '.join(sys.argv) + items = vars(args) + for key in sorted(items.keys(), key=lambda s: s.lower()): + value = items[key] + if not value: + value = "None" + logger.info(" " + key + ": " + str(items[key])) + if path is not None: + output_file.write(" " + key + ": " + str(items[key]) + "\n") + if path: + output_file.close() + del args.command + +def get_args(args): + items = vars(args) + output_string = '' + for key in sorted(items.keys(), key=lambda s: s.lower()): + value = items[key] + if not value: + value = "None" + output_string += " " + key + ": " + str(items[key] + "\n") + return output_string + diff --git a/code/w2vEmbReader.py b/code/w2vEmbReader.py index 5c90c7d..b446d60 100644 --- a/code/w2vEmbReader.py +++ b/code/w2vEmbReader.py @@ -1,71 +1,71 @@ -import codecs -import logging -import numpy as np -import gensim -from sklearn.cluster import KMeans -import pickle - - -logging.basicConfig(level=logging.INFO, - format='%(asctime)s %(levelname)s %(message)s') -logger = logging.getLogger(__name__) - -class W2VEmbReader: - - def __init__(self, emb_path, emb_dim=None): - - logger.info('Loading embeddings from: ' + emb_path) - self.embeddings = {} - emb_matrix = [] - - model = gensim.models.Word2Vec.load(emb_path) - self.emb_dim = emb_dim - for word in model.vocab: - self.embeddings[word] = list(model[word]) - emb_matrix.append(list(model[word])) - - if emb_dim != None: - assert self.emb_dim == len(self.embeddings['nice']) - - self.vector_size = len(self.embeddings) - self.emb_matrix = np.asarray(emb_matrix) - - logger.info(' #vectors: %i, #dimensions: %i' % (self.vector_size, self.emb_dim)) - - - def get_emb_given_word(self, word): - try: - return self.embeddings[word] - except KeyError: - return None - - def get_emb_matrix_given_vocab(self, vocab, emb_matrix): - counter = 0. - for word, index in vocab.iteritems(): - try: - emb_matrix[index] = self.embeddings[word] - counter += 1 - except KeyError: - pass - - logger.info('%i/%i word vectors initialized (hit rate: %.2f%%)' % (counter, len(vocab), 100*counter/len(vocab))) - # L2 normalization - norm_emb_matrix = emb_matrix / np.linalg.norm(emb_matrix, axis=-1, keepdims=True) - return norm_emb_matrix - - - def get_aspect_matrix(self, n_clusters): - km = KMeans(n_clusters=n_clusters) - km.fit(self.emb_matrix) - clusters = km.cluster_centers_ - - # L2 normalization - norm_aspect_matrix = clusters / np.linalg.norm(clusters, axis=-1, keepdims=True) - return norm_aspect_matrix.astype(np.float32) - - def get_emb_dim(self): - return self.emb_dim - - - - +import codecs +import logging +import numpy as np +import gensim +from sklearn.cluster import KMeans +import pickle + + +logging.basicConfig(level=logging.INFO, + format='%(asctime)s %(levelname)s %(message)s') +logger = logging.getLogger(__name__) + +class W2VEmbReader: + + def __init__(self, emb_path, emb_dim=None): + + logger.info('Loading embeddings from: ' + emb_path) + self.embeddings = {} + emb_matrix = [] + + model = gensim.models.Word2Vec.load(emb_path) + self.emb_dim = emb_dim + for word in model.vocab: + self.embeddings[word] = list(model[word]) + emb_matrix.append(list(model[word])) + + if emb_dim != None: + assert self.emb_dim == len(self.embeddings['nice']) + + self.vector_size = len(self.embeddings) + self.emb_matrix = np.asarray(emb_matrix) + + logger.info(' #vectors: %i, #dimensions: %i' % (self.vector_size, self.emb_dim)) + + + def get_emb_given_word(self, word): + try: + return self.embeddings[word] + except KeyError: + return None + + def get_emb_matrix_given_vocab(self, vocab, emb_matrix): + counter = 0. + for word, index in vocab.iteritems(): + try: + emb_matrix[index] = self.embeddings[word] + counter += 1 + except KeyError: + pass + + logger.info('%i/%i word vectors initialized (hit rate: %.2f%%)' % (counter, len(vocab), 100*counter/len(vocab))) + # L2 normalization + norm_emb_matrix = emb_matrix / np.linalg.norm(emb_matrix, axis=-1, keepdims=True) + return norm_emb_matrix + + + def get_aspect_matrix(self, n_clusters): + km = KMeans(n_clusters=n_clusters) + km.fit(self.emb_matrix) + clusters = km.cluster_centers_ + + # L2 normalization + norm_aspect_matrix = clusters / np.linalg.norm(clusters, axis=-1, keepdims=True) + return norm_aspect_matrix.astype(np.float32) + + def get_emb_dim(self): + return self.emb_dim + + + + diff --git a/code/word2vec.py b/code/word2vec.py old mode 100755 new mode 100644 index 4b524ff..a083e05 --- a/code/word2vec.py +++ b/code/word2vec.py @@ -1,26 +1,26 @@ -import gensim -import codecs - -class MySentences(object): - def __init__(self, filename): - self.filename = filename - - def __iter__(self): - for line in codecs.open(self.filename, 'r', 'utf-8'): - yield line.split() - - -def main(domain): - source = '../preprocessed_data/%s/train.txt' % (domain) - model_file = '../preprocessed_data/%s/w2v_embedding' % (domain) - sentences = MySentences(source) - model = gensim.models.Word2Vec(sentences, size=200, window=5, min_count=10, workers=4) - model.save(model_file) - - -print 'Pre-training word embeddings ...' -main('restaurant') -main('beer') - - - +import gensim +import codecs + +class MySentences(object): + def __init__(self, filename): + self.filename = filename + + def __iter__(self): + for line in codecs.open(self.filename, 'r', 'utf-8'): + yield line.split() + + +def main(domain): + source = '../preprocessed_data/%s/train.txt' % (domain) + model_file = '../preprocessed_data/%s/w2v_embedding' % (domain) + sentences = MySentences(source) + model = gensim.models.Word2Vec(sentences, size=200, window=5, min_count=10, workers=4) + model.save(model_file) + + +print('Pre-training word embeddings ...') +main('restaurant') +main('beer') + + +