No complaints from SplObjectStorage if you try to detach an object that isn't in the collection; it's a no-op.
<?php
$o = new StdClass;
$t = new StdClass;
$s = new SplObjectStorage();
$s->attach($o);
var_dump(count($s));
$s->detach($t); // Didn't attach this one.
var_dump(count($s));
?>