分词器

分词器（Tokenization）将文本划分为独立个体（通常为单词）。

package org.apache.spark.examples.ml

import org.apache.spark.ml.feature.{RegexTokenizer, Tokenizer}
import org.apache.spark.sql.functions._
import org.apache.spark.sql.SparkSession

object TokenizerExample {
    def main(args: Array[String]): Unit = {
        val spark = SparkSession
                .builder
                .master("local[*]")
                .appName("TokenizerExample")
                .getOrCreate()

        // $example on$
        val sentenceDataFrame = spark.createDataFrame(Seq(
            (0, "Hi I heard about Spark"),
            (1, "I wish Java could use case classes"),
            (2, "Logistic,regression,models,are,neat")
        )).toDF("id", "sentence")

        val countTokens = udf { (words: Seq[String]) => words.length }

        // 分词器
        val tokenizer = new Tokenizer().setInputCol("sentence").setOutputCol("words")
        val tokenized = tokenizer.transform(sentenceDataFrame)
        tokenized.select("sentence", "words")
                .withColumn("tokens", countTokens(col("words"))).show(false)

        // 正则分词器
        val regexTokenizer = new RegexTokenizer()
                .setInputCol("sentence")
                .setOutputCol("words")
                .setPattern("\\W") // alternatively .setPattern("\\w+").setGaps(false)
        val regexTokenized = regexTokenizer.transform(sentenceDataFrame)
        regexTokenized.select("sentence", "words")
                .withColumn("tokens", countTokens(col("words"))).show(false)

        spark.stop()
    }
}