You can use .length after the selector to see if it matched any elements, like this:

if($("#" + name).length == 0) {
  //it doesn't exist
}

The full version:

$("li.friend").live('click', function(){
  name = $(this).text();
  if($("#" + name).length == 0) {
    $("div#chatbar").append("<div class='labels'><div id='" + name + "' style='display:none;'></div>" + name + "</div>");
  } else {
    alert('this record already exists');
  }
});

Or, the non-jQuery version for this part (since it's an ID):

$("li.friend").live('click', function(){
  name = $(this).text();
  if(document.getElementById(name) == null) {
    $("div#chatbar").append("<div class='labels'><div id='" + name + "' style='display:none;'></div>" + name + "</div>");
  } else {
    alert('this record already exists');
  }
});