If you want to restrict on the front end i.e within the form then you need to add some dummy filter which doesn't return any results. Below is a sample script which filters contacts by account
=========
function filterContactsByAccount() {
if (Xrm.Page.getControl("new_contact") !== null) {
Xrm.Page.getControl("new_contact").addPreSearch(addFilter);
}
}
function addFilter() {
var AccountId = Xrm.Page.data.entity.attributes.get("new_account").getValue();
if (AccountId !== null) {
AccountId = AccountId[0].id.replace("{", "").replace("}", "");
var filter = "<filter type='and'>" +
"<condition attribute='parentcustomerid' operator='eq' value='" + AccountId + "'/>" +
"</filter>";
Xrm.Page.getControl("new_contact").addCustomFilter(filter);
}
else {
var filterDummy = "<filter type='and'>" +
"<condition attribute='contactid' operator='null' />" +
"</filter>";
Xrm.Page.getControl("new_contact").addCustomFilter(filterDummy);
}
}
===============
Hope this helps.