JAXB HashMap @XmlIDRef unmarshall

I'm using JAXB to save a large object graph. I have one HashMap that contains objects from class A and B. I only want to save the ID from the objects in my XML. Thats works fine with a XML Adapter. The XMLElements from B are at the bottom of my document. But when I try to unmarshall this, I get often only Null for A and B. A short example is here:

@XmlRootElement(name="DataContainer")
@XmlAccessorType(XmlAccessType.FIELD)
public class DataContainer {

    @XmlElementWrapper(name="allAs")
    @XmlElement(name="A")
    private List<A> allA_Elements;

    @XmlElementWrapper(name="allBs")
    @XmlElement(name="B")
    private List<B> allB_Elements;
}

@XmlRootElement(name="A")
@XmlAccessorType(XmlAccessType.FIELD)
public class A {

    @XmlAttribute
    String name;

    @XmlID
    String id;

    @XmlJavaTypeAdapter(ReferenceMapAdapter.class)
    @XmlElement(name="allReferences")
    HashMap<A, B> references;
}
@XmlRootElement(name="B")
@XmlAccessorType(XmlAccessType.FIELD)
public class B {

    @XmlAttribute
    String name;

    @XmlID
    String id;
}

public class ReferenceMapAdapter extends XmlAdapter<AdaptedReferenceMap, HashMap<A, B>> {

    public class ReferenceMapEntry {
        @XmlIDREF
        public A keyA;
        @XmlIDREF
        public B valueB;
    }

    public class AdaptedReferenceMap{
        @XmlElement(name="reference")
        public List<ReferenceMapEntry>reference = new ArrayList<ReferenceMapEntry>();
    }

    @Override
    public HashMap<A, B> unmarshal(AdaptedReferenceMap adaptedMap) throws Exception {
        HashMap<A, B> map = new HashMap<A, B>();
        for(ReferenceMapEntry entry : adaptedMap.reference) {
            map.put(entry.keyA, entry.valueB);
        }
        return map;
    }

    @Override
    public AdaptedReferenceMap marshal(HashMap<A, B> map) throws Exception {
        AdaptedReferenceMap adaptedMap = new AdaptedReferenceMap();
        Set<A> elements = map.keySet();
        for (A element : elements) {
            ReferenceMapEntry entry = new ReferenceMapEntry();
            entry.keyA = element;
            entry.valueB = map.get(element);
            adaptedMap.reference.add(entry);
        }
        return adaptedMap;
    }
}

Here is the generated XML

<DataContainer>
    <allAs>
        <A name="nameA0" id="ida_00">
            <alleReferences>
                <reference>
                    <keyA>ida_03</keyA>
                    <valueB>idb_02</valueB>
                </reference>
                <reference>
                    <keyA>ida_01</keyA>
                    <valueB>idb_01</valueB>
                </reference>
            </alleReferences>
        </A>
        <A name="nameA1" id="ida_01">
            <alleReferences>
                <reference>
                    <keyA>ida_00</keyA>
                    <valueB>idb_02</valueB>
                </reference>
                <reference>
                    <keyA>ida_02</keyA>
                    <valueB>idb_01</valueB>
                </reference>
            </alleReferences>
        </A>
        <A name="nameA2" id="ida_02">
            <alleReferences>
                <reference>
                    <keyA>ida_01</keyA>
                    <valueB>idb_01</valueB>
                </reference>
                <reference>
                    <keyA>ida_03</keyA>
                    <valueB>idb_00</valueB>
                </reference>
            </alleReferences>
        </A>
        <A name="nameA3" id="ida_03">
            <alleReferences>
                <reference>
                    <keyA>ida_02</keyA>
                    <valueB>idb_02</valueB>
                </reference>
                <reference>
                    <keyA>ida_00</keyA>
                    <valueB>idb_01</valueB>
                </reference>
            </alleReferences>
        </A>
    </allAs>
    <allBs>
        <B name="nameB0" id="idb_00"/>
        <B name="nameB1" id="idb_01"/>
        <B name="nameB2" id="idb_02"/>
        <B name="nameB3" id="idb_03"/>
    </allBs>
</DataContainer>

So what I have to do to unmarshal it correct??