trigger BillingAddressMod on Account (after update) {
public static boolean flag=true;
If(flag==true) {
account acc = [select id, BillingPostalCode from account where id in : trigger.new];
List<contact> contList = [Select id, MailingPostalCode from Contact];
Set <Id> setContact = new Set<id>();
for(Contact cont: contList){
setContact.add(cont.MailingPostalCode);
}
Map<id,Contact> mapAccount = [Select MailingPostalCode from Contact where AccountId in: acc];
for (contact con : contList)
{
if(acc.BillingPostalCode != con.MailingPostalCode)
{
contList.add(con);
}
}
If(contList.size()>=1)
{
acc.OutofZip__c = true;
}
flag=false;
update acc;
}
}

you are right my bad, this should be:
for(Contact con : accsWithContactMap.get(acc.Id).Contacts){