Constructor Detail

• TextPipeline

  public TextPipeline()

• TextPipeline

  public TextPipeline(org.apache.spark.api.java.JavaRDD<String> corpusRDD,
                      org.apache.spark.broadcast.Broadcast<Map<String,Object>> broadcasTokenizerVarMap)
               throws Exception

  Throws:

  Exception

Method Detail

• setRDDVarMap

  public void setRDDVarMap(org.apache.spark.api.java.JavaRDD<String> corpusRDD,
                           org.apache.spark.broadcast.Broadcast<Map<String,Object>> broadcasTokenizerVarMap)

• tokenize

  public org.apache.spark.api.java.JavaRDD<List<String>> tokenize()

• updateAndReturnAccumulatorVal

  public org.apache.spark.api.java.JavaRDD<Pair<List<String>,AtomicLong>> updateAndReturnAccumulatorVal(org.apache.spark.api.java.JavaRDD<List<String>> tokenizedRDD)

• filterMinWordAddVocab

  public void filterMinWordAddVocab(Counter<String> wordFreq)

• buildVocabCache

  public void buildVocabCache()

• buildVocabWordListRDD

  public void buildVocabWordListRDD()

• getWordFreqAcc

  public org.apache.spark.Accumulator<Counter<String>> getWordFreqAcc()

• getBroadCastVocabCache

  public org.apache.spark.broadcast.Broadcast<VocabCache<VocabWord>> getBroadCastVocabCache()
                                                                                     throws IllegalStateException

  Throws:

  IllegalStateException

• getVocabCache

  public VocabCache<VocabWord> getVocabCache()
                                      throws IllegalStateException

  Throws:

  IllegalStateException

• getSentenceWordsCountRDD

  public org.apache.spark.api.java.JavaRDD<Pair<List<String>,AtomicLong>> getSentenceWordsCountRDD()

• getVocabWordListRDD

  public org.apache.spark.api.java.JavaRDD<List<VocabWord>> getVocabWordListRDD()
                                                                         throws IllegalStateException

  Throws:

  IllegalStateException

• getSentenceCountRDD

  public org.apache.spark.api.java.JavaRDD<AtomicLong> getSentenceCountRDD()
                                                                    throws IllegalStateException

  Throws:

  IllegalStateException

• getTotalWordCount

  public Long getTotalWordCount()