HN:
# uname -a
Linux test_host 2.6.18-194.8.1.el5.028stab070.2 #1 SMP Tue Jul 6 15:26:41 MSD 2010 i686 i686 i386 GNU/Linux
# iptables -A FORWARD -p tcp --syn --dport smtp -m connlimit --connlimit-above 1 -j DROP
# iptables -L
Chain INPUT (policy ACCEPT)
target     prot opt source               destination

Chain FORWARD (policy ACCEPT)
target     prot opt source               destination
DROP       tcp  --  anywhere             anywhere            tcp dpt:smtp flags:FIN,SYN,RST,ACK/SYN #conn/32 > 1

Chain OUTPUT (policy ACCEPT)
target     prot opt source               destination
# lsmod |grep limit
xt_connlimit            8072  1
ip_conntrack           60612  4 xt_connlimit,vzrst,ip_nat,vzcpt
xt_limit                6656  0
x_tables               19204  11 xt_connlimit,xt_tcpudp,xt_length,ipt_ttl,xt_tcpmss,ipt_TCPMSS,xt_multiport,xt_limit,ipt_tos,ipt_REJECT,ip_tables
# vzctl restart 103
<skipped>
# vzctl enter 103
# iptables -A FORWARD -p tcp --syn --dport smtp -m connlimit --connlimit-above 1 -j DROP
# # iptables -L
Chain INPUT (policy ACCEPT)
target     prot opt source               destination

Chain FORWARD (policy ACCEPT)
target     prot opt source               destination
DROP       tcp  --  anywhere             anywhere            tcp dpt:smtp flags:FIN,SYN,RST,ACK/SYN #conn/32 > 1

Chain OUTPUT (policy ACCEPT)
target     prot opt source               destination