Differential D7890 Diff 28990 java/src/main/java/org/softwareheritage/graph/maps/NodeIdMap.java

Changeset View

Standalone View

java/src/main/java/org/softwareheritage/graph/maps/NodeIdMap.java

package org.softwareheritage.graph.maps;		package org.softwareheritage.graph.maps;

import it.unimi.dsi.fastutil.Size64;		import it.unimi.dsi.fastutil.Size64;
		import it.unimi.dsi.fastutil.bytes.ByteBigList;
		import it.unimi.dsi.fastutil.bytes.ByteMappedBigList;
import it.unimi.dsi.fastutil.io.BinIO;		import it.unimi.dsi.fastutil.io.BinIO;
import it.unimi.dsi.fastutil.longs.LongBigList;		import it.unimi.dsi.fastutil.longs.LongBigList;
		import it.unimi.dsi.fastutil.longs.LongMappedBigList;
import it.unimi.dsi.fastutil.objects.Object2LongFunction;		import it.unimi.dsi.fastutil.objects.Object2LongFunction;
import it.unimi.dsi.util.ByteBufferLongBigList;
import org.softwareheritage.graph.SWHID;		import org.softwareheritage.graph.SWHID;
import org.softwareheritage.graph.compress.NodeMapBuilder;		import org.softwareheritage.graph.compress.NodeMapBuilder;

import java.io.File;		import java.io.File;
import java.io.IOException;		import java.io.IOException;
import java.io.RandomAccessFile;		import java.io.RandomAccessFile;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;		import java.nio.charset.StandardCharsets;

/**		/**
* Mapping between internal long node id and external SWHID.		* Mapping between internal long node id and external SWHID.
* <p>		* <p>
* The SWHID -> node mapping is obtained from hashing the SWHID with a MPH, then permuting it using		* The SWHID -> node mapping is obtained from hashing the SWHID with a MPH, then permuting it using
* an mmap()-ed .order file containing the graph permutation.		* an mmap()-ed .order file containing the graph permutation.
*		*
Show All 10 Lines	public class NodeIdMap implements Size64 {

/** File extension for the long node id to SWHID map */		/** File extension for the long node id to SWHID map */
public static final String NODE_TO_SWHID = ".node2swhid.bin";		public static final String NODE_TO_SWHID = ".node2swhid.bin";

/** Graph path and basename */		/** Graph path and basename */
String graphPath;		String graphPath;

/** mmap()-ed NODE_TO_SWHID file */		/** mmap()-ed NODE_TO_SWHID file */
MapFile nodeToSwhMap;		ByteBigList nodeToSwhMap;

/** Minimal perfect hash (MPH) function SWHID -> initial order */		/** Minimal perfect hash (MPH) function SWHID -> initial order */
Object2LongFunction<byte[]> mph;		Object2LongFunction<byte[]> mph;
/** mmap()-ed long list with the permutation initial order -> graph order */		/** mmap()-ed long list with the permutation initial order -> graph order */
LongBigList orderMap;		LongBigList orderMap;

/**		/**
* Constructor.		* Constructor.
*		*
* @param graphPath full graph path		* @param graphPath full graph path
*/		*/
public NodeIdMap(String graphPath) throws IOException {		public NodeIdMap(String graphPath) throws IOException {
this.graphPath = graphPath;		this.graphPath = graphPath;

// node -> SWHID		// node -> SWHID
this.nodeToSwhMap = new MapFile(graphPath + NODE_TO_SWHID, SWHID_BIN_SIZE);		try (RandomAccessFile raf = new RandomAccessFile(graphPath + NODE_TO_SWHID, "r")) {
		this.nodeToSwhMap = ByteMappedBigList.map(raf.getChannel());
		}

// SWHID -> node		// SWHID -> node
this.mph = loadMph(graphPath + ".mph");		this.mph = loadMph(graphPath + ".mph");

try (RandomAccessFile mapFile = new RandomAccessFile(new File(graphPath + ".order"), "r")) {		try (RandomAccessFile mapFile = new RandomAccessFile(new File(graphPath + ".order"), "r")) {
FileChannel fileChannel = mapFile.getChannel();		this.orderMap = LongMappedBigList.map(mapFile.getChannel());
this.orderMap = ByteBufferLongBigList.map(fileChannel);
}		}
}		}

@SuppressWarnings("unchecked")		@SuppressWarnings("unchecked")
public static Object2LongFunction<byte[]> loadMph(String path) throws IOException {		public static Object2LongFunction<byte[]> loadMph(String path) throws IOException {
Object obj;		Object obj;
try {		try {
obj = BinIO.loadObject(path);		obj = BinIO.loadObject(path);
Show All 17 Lines	public static Object2LongFunction<byte[]> loadMph(String path) throws IOException {
}		}

@Override		@Override
public long getLong(Object o) {		public long getLong(Object o) {
byte[] bi = (byte[]) o;		byte[] bi = (byte[]) o;
return legacyFunction.getLong(new String(bi, StandardCharsets.UTF_8));		return legacyFunction.getLong(new String(bi, StandardCharsets.UTF_8));
}		}

		@SuppressWarnings("deprecation")
@Override		@Override
public int size() {		public int size() {
return legacyFunction.size();		return legacyFunction.size();
}		}

@Override		@Override
public long size64() {		public long size64() {
return (legacyFunction instanceof Size64)		return (legacyFunction instanceof Size64)
▲ Show 20 Lines • Show All 58 Lines • ▼ Show 20 Lines	public class NodeIdMap implements Size64 {
* @return corresponding node as a {@link SWHID}		* @return corresponding node as a {@link SWHID}
* @see SWHID		* @see SWHID
*/		*/
public SWHID getSWHID(long nodeId) {		public SWHID getSWHID(long nodeId) {
/*		/*
* Each line in NODE_TO_SWHID is formatted as: swhid The file is ordered by nodeId, meaning node0's		* Each line in NODE_TO_SWHID is formatted as: swhid The file is ordered by nodeId, meaning node0's
* swhid is at line 0, hence we can read the nodeId-th line to get corresponding swhid		* swhid is at line 0, hence we can read the nodeId-th line to get corresponding swhid
*/		*/
if (nodeId < 0 \|\| nodeId >= nodeToSwhMap.size()) {		if (nodeId < 0 \|\| nodeId >= nodeToSwhMap.size64()) {
throw new IllegalArgumentException("Node id " + nodeId + " should be between 0 and " + nodeToSwhMap.size());		throw new IllegalArgumentException(
}		"Node id " + nodeId + " should be between 0 and " + nodeToSwhMap.size64());

return SWHID.fromBytes(nodeToSwhMap.readAtLine(nodeId));
}		}

/**		byte[] swhid = new byte[SWHID_BIN_SIZE];
* Closes the mapping files.		nodeToSwhMap.getElements(nodeId * SWHID_BIN_SIZE, swhid, 0, SWHID_BIN_SIZE);
*/		return SWHID.fromBytes(swhid);
public void close() throws IOException {
nodeToSwhMap.close();
}		}

/** Return the number of nodes in the map. */		/** Return the number of nodes in the map. */
@Override		@Override
public long size64() {		public long size64() {
return nodeToSwhMap.size();		return nodeToSwhMap.size64();
}		}
}		}