# block more than 10 connections per subnet by mask 24, for example port 443
iptables -I INPUT -p tcp --syn --dport 443 -m connlimit --connlimit-above 10 --connlimit-mask 24 -j DROP

# log if needed
iptables -I INPUT -p tcp --syn --dport 443 -m connlimit --connlimit-above 10 --connlimit-mask 24 -j LOG