Changeset View
Changeset View
Standalone View
Standalone View
java/src/main/java/org/softwareheritage/graph/maps/NodeIdMap.java
/* | /* | ||||
* Copyright (c) 2019-2022 The Software Heritage developers | * Copyright (c) 2019-2022 The Software Heritage developers | ||||
* See the AUTHORS file at the top-level directory of this distribution | * See the AUTHORS file at the top-level directory of this distribution | ||||
* License: GNU General Public License version 3, or any later version | * License: GNU General Public License version 3, or any later version | ||||
* See top-level LICENSE file for more information | * See top-level LICENSE file for more information | ||||
*/ | */ | ||||
package org.softwareheritage.graph.maps; | package org.softwareheritage.graph.maps; | ||||
import it.unimi.dsi.fastutil.Size64; | import it.unimi.dsi.fastutil.Size64; | ||||
import it.unimi.dsi.fastutil.bytes.ByteBigList; | import it.unimi.dsi.fastutil.bytes.ByteBigList; | ||||
import it.unimi.dsi.fastutil.bytes.ByteMappedBigList; | import it.unimi.dsi.fastutil.bytes.ByteMappedBigList; | ||||
import it.unimi.dsi.fastutil.io.BinIO; | import it.unimi.dsi.fastutil.io.BinIO; | ||||
import it.unimi.dsi.fastutil.longs.LongBigList; | import it.unimi.dsi.fastutil.longs.LongBigList; | ||||
import it.unimi.dsi.fastutil.longs.LongMappedBigList; | import it.unimi.dsi.fastutil.longs.LongMappedBigList; | ||||
import it.unimi.dsi.fastutil.objects.Object2LongFunction; | import it.unimi.dsi.fastutil.objects.Object2LongFunction; | ||||
import it.unimi.dsi.lang.FlyweightPrototype; | |||||
import org.softwareheritage.graph.SWHID; | import org.softwareheritage.graph.SWHID; | ||||
import org.softwareheritage.graph.compress.NodeMapBuilder; | import org.softwareheritage.graph.compress.NodeMapBuilder; | ||||
import java.io.File; | import java.io.File; | ||||
import java.io.IOException; | import java.io.IOException; | ||||
import java.io.RandomAccessFile; | import java.io.RandomAccessFile; | ||||
import java.nio.charset.StandardCharsets; | import java.nio.charset.StandardCharsets; | ||||
/** | /** | ||||
* Mapping between internal long node id and external SWHID. | * Mapping between internal long node id and external SWHID. | ||||
* <p> | * <p> | ||||
* The SWHID -> node mapping is obtained from hashing the SWHID with a MPH, then permuting it using | * The SWHID -> node mapping is obtained from hashing the SWHID with a MPH, then permuting it using | ||||
* an mmap()-ed .order file containing the graph permutation. | * an mmap()-ed .order file containing the graph permutation. | ||||
* | * | ||||
* The node -> SWHID reverse mapping is pre-computed and dumped on disk in the | * The node -> SWHID reverse mapping is pre-computed and dumped on disk in the | ||||
* {@link NodeMapBuilder} class, then it is loaded here using mmap(). | * {@link NodeMapBuilder} class, then it is loaded here using mmap(). | ||||
* | * | ||||
* @author The Software Heritage developers | * @author The Software Heritage developers | ||||
* @see NodeMapBuilder | * @see NodeMapBuilder | ||||
*/ | */ | ||||
public class NodeIdMap implements Size64 { | public class NodeIdMap implements Size64, FlyweightPrototype<NodeIdMap> { | ||||
/** Fixed length of binary SWHID buffer */ | /** Fixed length of binary SWHID buffer */ | ||||
public static final int SWHID_BIN_SIZE = 22; | public static final int SWHID_BIN_SIZE = 22; | ||||
/** File extension for the long node id to SWHID map */ | /** File extension for the long node id to SWHID map */ | ||||
public static final String NODE_TO_SWHID = ".node2swhid.bin"; | public static final String NODE_TO_SWHID = ".node2swhid.bin"; | ||||
/** Graph path and basename */ | /** Graph path and basename */ | ||||
String graphPath; | String graphPath; | ||||
Show All 21 Lines | public NodeIdMap(String graphPath) throws IOException { | ||||
// SWHID -> node | // SWHID -> node | ||||
this.mph = loadMph(graphPath + ".mph"); | this.mph = loadMph(graphPath + ".mph"); | ||||
try (RandomAccessFile mapFile = new RandomAccessFile(new File(graphPath + ".order"), "r")) { | try (RandomAccessFile mapFile = new RandomAccessFile(new File(graphPath + ".order"), "r")) { | ||||
this.orderMap = LongMappedBigList.map(mapFile.getChannel()); | this.orderMap = LongMappedBigList.map(mapFile.getChannel()); | ||||
} | } | ||||
} | } | ||||
protected NodeIdMap(String graphPath, ByteBigList nodeToSwhMap, Object2LongFunction<byte[]> mph, | |||||
LongBigList orderMap) { | |||||
this.graphPath = graphPath; | |||||
this.nodeToSwhMap = nodeToSwhMap; | |||||
this.mph = mph; | |||||
this.orderMap = orderMap; | |||||
} | |||||
/** Return the number of nodes in the map. */ | |||||
@Override | |||||
public long size64() { | |||||
return nodeToSwhMap.size64(); | |||||
} | |||||
@Override | |||||
public NodeIdMap copy() { | |||||
return new NodeIdMap(graphPath, | |||||
((nodeToSwhMap instanceof ByteMappedBigList) | |||||
? ((ByteMappedBigList) nodeToSwhMap).copy() | |||||
: nodeToSwhMap), | |||||
mph, ((orderMap instanceof LongMappedBigList) ? ((LongMappedBigList) orderMap).copy() : orderMap)); | |||||
} | |||||
@SuppressWarnings("unchecked") | @SuppressWarnings("unchecked") | ||||
public static Object2LongFunction<byte[]> loadMph(String path) throws IOException { | public static Object2LongFunction<byte[]> loadMph(String path) throws IOException { | ||||
Object obj; | Object obj; | ||||
try { | try { | ||||
obj = BinIO.loadObject(path); | obj = BinIO.loadObject(path); | ||||
} catch (ClassNotFoundException e) { | } catch (ClassNotFoundException e) { | ||||
throw new IOException(e.getMessage()); | throw new IOException(e.getMessage()); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 98 Lines • ▼ Show 20 Lines | public SWHID getSWHID(long nodeId) { | ||||
throw new IllegalArgumentException( | throw new IllegalArgumentException( | ||||
"Node id " + nodeId + " should be between 0 and " + nodeToSwhMap.size64()); | "Node id " + nodeId + " should be between 0 and " + nodeToSwhMap.size64()); | ||||
} | } | ||||
byte[] swhid = new byte[SWHID_BIN_SIZE]; | byte[] swhid = new byte[SWHID_BIN_SIZE]; | ||||
nodeToSwhMap.getElements(nodeId * SWHID_BIN_SIZE, swhid, 0, SWHID_BIN_SIZE); | nodeToSwhMap.getElements(nodeId * SWHID_BIN_SIZE, swhid, 0, SWHID_BIN_SIZE); | ||||
return SWHID.fromBytes(swhid); | return SWHID.fromBytes(swhid); | ||||
} | } | ||||
/** Return the number of nodes in the map. */ | |||||
@Override | |||||
public long size64() { | |||||
return nodeToSwhMap.size64(); | |||||
} | |||||
} | } |