Differential D5427 Diff 19580 java/src/main/java/org/softwareheritage/graph/utils/ExportSubdataset.java
Changeset View
Changeset View
Standalone View
Standalone View
java/src/main/java/org/softwareheritage/graph/utils/ExportSubdataset.java
package org.softwareheritage.graph.utils; | package org.softwareheritage.graph.utils; | ||||
import com.google.common.primitives.Longs; | import com.google.common.primitives.Longs; | ||||
import it.unimi.dsi.big.webgraph.LazyLongIterator; | import it.unimi.dsi.big.webgraph.LazyLongIterator; | ||||
import it.unimi.dsi.bits.LongArrayBitVector; | import it.unimi.dsi.bits.LongArrayBitVector; | ||||
import it.unimi.dsi.fastutil.Arrays; | import it.unimi.dsi.fastutil.Arrays; | ||||
import it.unimi.dsi.fastutil.io.BinIO; | |||||
import it.unimi.dsi.fastutil.objects.Object2LongFunction; | import it.unimi.dsi.fastutil.objects.Object2LongFunction; | ||||
import it.unimi.dsi.io.ByteDiskQueue; | import it.unimi.dsi.io.ByteDiskQueue; | ||||
import it.unimi.dsi.io.FastBufferedReader; | import it.unimi.dsi.io.FastBufferedReader; | ||||
import it.unimi.dsi.io.LineIterator; | import it.unimi.dsi.io.LineIterator; | ||||
import org.softwareheritage.graph.Graph; | import org.softwareheritage.graph.Graph; | ||||
import org.softwareheritage.graph.SWHID; | import org.softwareheritage.graph.SWHID; | ||||
import org.softwareheritage.graph.experiments.topology.ConnectedComponents; | import org.softwareheritage.graph.experiments.topology.ConnectedComponents; | ||||
import org.softwareheritage.graph.maps.NodeIdMap; | |||||
import java.io.File; | import java.io.File; | ||||
import java.io.IOException; | import java.io.IOException; | ||||
import java.io.InputStreamReader; | import java.io.InputStreamReader; | ||||
import java.nio.charset.StandardCharsets; | import java.nio.charset.StandardCharsets; | ||||
public class ExportSubdataset { | public class ExportSubdataset { | ||||
@SuppressWarnings("unchecked") // Suppress warning for Object2LongFunction cast | |||||
static Object2LongFunction<byte[]> loadMPH(String mphPath) throws IOException, ClassNotFoundException { | |||||
return (Object2LongFunction<byte[]>) BinIO.loadObject(mphPath); | |||||
} | |||||
public static void main(String[] args) throws IOException, ClassNotFoundException { | public static void main(String[] args) throws IOException, ClassNotFoundException { | ||||
System.err.print("Loading everything..."); | System.err.print("Loading everything..."); | ||||
String graphPath = args[0]; | String graphPath = args[0]; | ||||
Graph graph = new Graph(graphPath); | Graph graph = new Graph(graphPath); | ||||
Object2LongFunction<byte[]> mphMap = loadMPH(graphPath + ".mph"); | Object2LongFunction<byte[]> mphMap = NodeIdMap.loadMph(graphPath + ".mph"); | ||||
System.err.println(" done."); | System.err.println(" done."); | ||||
final long n = graph.numNodes(); | final long n = graph.numNodes(); | ||||
// Allow enough memory to behave like in-memory queue | // Allow enough memory to behave like in-memory queue | ||||
int bufferSize = (int) Math.min(Arrays.MAX_ARRAY_SIZE & ~0x7, 8L * n); | int bufferSize = (int) Math.min(Arrays.MAX_ARRAY_SIZE & ~0x7, 8L * n); | ||||
// Use a disk based queue to store BFS frontier | // Use a disk based queue to store BFS frontier | ||||
▲ Show 20 Lines • Show All 42 Lines • Show Last 20 Lines |