public final class HighDFWordsPruner extends Object
Modifier and Type | Field and Description |
---|---|
static String |
MAX_DF |
static String |
MIN_DF |
static String |
STD_CALC_DIR |
Modifier and Type | Method and Description |
---|---|
static void |
mergePartialVectors(Iterable<org.apache.hadoop.fs.Path> partialVectorPaths,
org.apache.hadoop.fs.Path output,
org.apache.hadoop.conf.Configuration baseConf,
float normPower,
boolean logNormalize,
int numReducers) |
static void |
pruneVectors(org.apache.hadoop.fs.Path tfDir,
org.apache.hadoop.fs.Path prunedTFDir,
org.apache.hadoop.fs.Path prunedPartialTFDir,
long maxDF,
long minDF,
org.apache.hadoop.conf.Configuration baseConf,
Pair<Long[],List<org.apache.hadoop.fs.Path>> docFrequenciesFeatures,
float normPower,
boolean logNormalize,
int numReducers) |
public static final String STD_CALC_DIR
public static final String MAX_DF
public static final String MIN_DF
public static void pruneVectors(org.apache.hadoop.fs.Path tfDir, org.apache.hadoop.fs.Path prunedTFDir, org.apache.hadoop.fs.Path prunedPartialTFDir, long maxDF, long minDF, org.apache.hadoop.conf.Configuration baseConf, Pair<Long[],List<org.apache.hadoop.fs.Path>> docFrequenciesFeatures, float normPower, boolean logNormalize, int numReducers) throws IOException, InterruptedException, ClassNotFoundException
public static void mergePartialVectors(Iterable<org.apache.hadoop.fs.Path> partialVectorPaths, org.apache.hadoop.fs.Path output, org.apache.hadoop.conf.Configuration baseConf, float normPower, boolean logNormalize, int numReducers) throws IOException, InterruptedException, ClassNotFoundException
Copyright © 2008–2017 The Apache Software Foundation. All rights reserved.