Changeset View
Changeset View
Standalone View
Standalone View
java/src/main/java/org/softwareheritage/graph/maps/NodeIdMap.java
package org.softwareheritage.graph.maps; | package org.softwareheritage.graph.maps; | ||||
import it.unimi.dsi.fastutil.Size64; | import it.unimi.dsi.fastutil.Size64; | ||||
import it.unimi.dsi.fastutil.bytes.ByteBigList; | |||||
import it.unimi.dsi.fastutil.bytes.ByteMappedBigList; | |||||
import it.unimi.dsi.fastutil.io.BinIO; | import it.unimi.dsi.fastutil.io.BinIO; | ||||
import it.unimi.dsi.fastutil.longs.LongBigList; | import it.unimi.dsi.fastutil.longs.LongBigList; | ||||
import it.unimi.dsi.fastutil.longs.LongMappedBigList; | |||||
import it.unimi.dsi.fastutil.objects.Object2LongFunction; | import it.unimi.dsi.fastutil.objects.Object2LongFunction; | ||||
import it.unimi.dsi.util.ByteBufferLongBigList; | |||||
import org.softwareheritage.graph.SWHID; | import org.softwareheritage.graph.SWHID; | ||||
import org.softwareheritage.graph.compress.NodeMapBuilder; | import org.softwareheritage.graph.compress.NodeMapBuilder; | ||||
import java.io.File; | import java.io.File; | ||||
import java.io.IOException; | import java.io.IOException; | ||||
import java.io.RandomAccessFile; | import java.io.RandomAccessFile; | ||||
import java.nio.channels.FileChannel; | |||||
import java.nio.charset.StandardCharsets; | import java.nio.charset.StandardCharsets; | ||||
/** | /** | ||||
* Mapping between internal long node id and external SWHID. | * Mapping between internal long node id and external SWHID. | ||||
* <p> | * <p> | ||||
* The SWHID -> node mapping is obtained from hashing the SWHID with a MPH, then permuting it using | * The SWHID -> node mapping is obtained from hashing the SWHID with a MPH, then permuting it using | ||||
* an mmap()-ed .order file containing the graph permutation. | * an mmap()-ed .order file containing the graph permutation. | ||||
* | * | ||||
Show All 10 Lines | public class NodeIdMap implements Size64 { | ||||
/** File extension for the long node id to SWHID map */ | /** File extension for the long node id to SWHID map */ | ||||
public static final String NODE_TO_SWHID = ".node2swhid.bin"; | public static final String NODE_TO_SWHID = ".node2swhid.bin"; | ||||
/** Graph path and basename */ | /** Graph path and basename */ | ||||
String graphPath; | String graphPath; | ||||
/** mmap()-ed NODE_TO_SWHID file */ | /** mmap()-ed NODE_TO_SWHID file */ | ||||
MapFile nodeToSwhMap; | ByteBigList nodeToSwhMap; | ||||
/** Minimal perfect hash (MPH) function SWHID -> initial order */ | /** Minimal perfect hash (MPH) function SWHID -> initial order */ | ||||
Object2LongFunction<byte[]> mph; | Object2LongFunction<byte[]> mph; | ||||
/** mmap()-ed long list with the permutation initial order -> graph order */ | /** mmap()-ed long list with the permutation initial order -> graph order */ | ||||
LongBigList orderMap; | LongBigList orderMap; | ||||
/** | /** | ||||
* Constructor. | * Constructor. | ||||
* | * | ||||
* @param graphPath full graph path | * @param graphPath full graph path | ||||
*/ | */ | ||||
public NodeIdMap(String graphPath) throws IOException { | public NodeIdMap(String graphPath) throws IOException { | ||||
this.graphPath = graphPath; | this.graphPath = graphPath; | ||||
// node -> SWHID | // node -> SWHID | ||||
this.nodeToSwhMap = new MapFile(graphPath + NODE_TO_SWHID, SWHID_BIN_SIZE); | try (RandomAccessFile raf = new RandomAccessFile(graphPath + NODE_TO_SWHID, "r")) { | ||||
this.nodeToSwhMap = ByteMappedBigList.map(raf.getChannel()); | |||||
} | |||||
// SWHID -> node | // SWHID -> node | ||||
this.mph = loadMph(graphPath + ".mph"); | this.mph = loadMph(graphPath + ".mph"); | ||||
try (RandomAccessFile mapFile = new RandomAccessFile(new File(graphPath + ".order"), "r")) { | try (RandomAccessFile mapFile = new RandomAccessFile(new File(graphPath + ".order"), "r")) { | ||||
FileChannel fileChannel = mapFile.getChannel(); | this.orderMap = LongMappedBigList.map(mapFile.getChannel()); | ||||
this.orderMap = ByteBufferLongBigList.map(fileChannel); | |||||
} | } | ||||
} | } | ||||
@SuppressWarnings("unchecked") | @SuppressWarnings("unchecked") | ||||
public static Object2LongFunction<byte[]> loadMph(String path) throws IOException { | public static Object2LongFunction<byte[]> loadMph(String path) throws IOException { | ||||
Object obj; | Object obj; | ||||
try { | try { | ||||
obj = BinIO.loadObject(path); | obj = BinIO.loadObject(path); | ||||
Show All 17 Lines | public static Object2LongFunction<byte[]> loadMph(String path) throws IOException { | ||||
} | } | ||||
@Override | @Override | ||||
public long getLong(Object o) { | public long getLong(Object o) { | ||||
byte[] bi = (byte[]) o; | byte[] bi = (byte[]) o; | ||||
return legacyFunction.getLong(new String(bi, StandardCharsets.UTF_8)); | return legacyFunction.getLong(new String(bi, StandardCharsets.UTF_8)); | ||||
} | } | ||||
@SuppressWarnings("deprecation") | |||||
@Override | @Override | ||||
public int size() { | public int size() { | ||||
return legacyFunction.size(); | return legacyFunction.size(); | ||||
} | } | ||||
@Override | @Override | ||||
public long size64() { | public long size64() { | ||||
return (legacyFunction instanceof Size64) | return (legacyFunction instanceof Size64) | ||||
▲ Show 20 Lines • Show All 58 Lines • ▼ Show 20 Lines | public class NodeIdMap implements Size64 { | ||||
* @return corresponding node as a {@link SWHID} | * @return corresponding node as a {@link SWHID} | ||||
* @see SWHID | * @see SWHID | ||||
*/ | */ | ||||
public SWHID getSWHID(long nodeId) { | public SWHID getSWHID(long nodeId) { | ||||
/* | /* | ||||
* Each line in NODE_TO_SWHID is formatted as: swhid The file is ordered by nodeId, meaning node0's | * Each line in NODE_TO_SWHID is formatted as: swhid The file is ordered by nodeId, meaning node0's | ||||
* swhid is at line 0, hence we can read the nodeId-th line to get corresponding swhid | * swhid is at line 0, hence we can read the nodeId-th line to get corresponding swhid | ||||
*/ | */ | ||||
if (nodeId < 0 || nodeId >= nodeToSwhMap.size()) { | if (nodeId < 0 || nodeId >= nodeToSwhMap.size64()) { | ||||
throw new IllegalArgumentException("Node id " + nodeId + " should be between 0 and " + nodeToSwhMap.size()); | throw new IllegalArgumentException( | ||||
} | "Node id " + nodeId + " should be between 0 and " + nodeToSwhMap.size64()); | ||||
return SWHID.fromBytes(nodeToSwhMap.readAtLine(nodeId)); | |||||
} | } | ||||
/** | byte[] swhid = new byte[SWHID_BIN_SIZE]; | ||||
* Closes the mapping files. | nodeToSwhMap.getElements(nodeId * SWHID_BIN_SIZE, swhid, 0, SWHID_BIN_SIZE); | ||||
*/ | return SWHID.fromBytes(swhid); | ||||
public void close() throws IOException { | |||||
nodeToSwhMap.close(); | |||||
} | } | ||||
/** Return the number of nodes in the map. */ | /** Return the number of nodes in the map. */ | ||||
@Override | @Override | ||||
public long size64() { | public long size64() { | ||||
return nodeToSwhMap.size(); | return nodeToSwhMap.size64(); | ||||
} | } | ||||
} | } |