总体

• 生成aggregate ，及scala的TypeColumn
• 执行
• 分析，包括取执行结果

生成TypeColumn

没一个check、constraint 的入口差不多都是要生成一个 Constraint 并使用addFilterableConstraint 进行添加。

  def hasSize(assertion: Long => Boolean, hint: Option[String] = None): CheckWithLastConstraintFilterable = {addFilterableConstraint { filter => Constraint.sizeConstraint(assertion, filter, hint) }}/** Size is the number of rows in a DataFrame. */
case class Size(where: Option[String] = None)extends StandardScanShareableAnalyzer[NumMatches]("Size", "*", Entity.Dataset)with FilterableAnalyzer {override def aggregationFunctions(): Seq[Column] = {conditionalCount(where) :: Nil}override def fromAggregationResult(result: Row, offset: Int): Option[NumMatches] = {ifNoNullsIn(result, offset) { _ =>NumMatches(result.getLong(offset))}}override def filterCondition: Option[String] = where
}//返回spark count ，一个TypeColumn 类型，用于aggregate
def conditionalCount(where: Option[String]): Column = {where.map { filter => sum(expr(filter).cast(LongType)) }.getOrElse(count("*"))
}

注意 case class Size 有一个where，代表可以输入where条件 ，hasSize的生成类似如下计算

//生成如下计算
val column: TypedColumn[Any,Long] =count(“*”);
val cc = data.agg(column).collect().head;

再看containsURL，也是类似。

  def containsURL(column: String,assertion: Double => Boolean = Check.IsOne,hint: Option[String] = None): CheckWithLastConstraintFilterable = {hasPattern(column, Patterns.URL, assertion, Some(s"containsURL($column)"), hint)}def patternMatchConstraint(column: String,pattern: Regex,assertion: Double => Boolean,where: Option[String] = None,name: Option[String] = None,hint: Option[String] = None): Constraint = {
//这里返回2个 Typecolumnval patternMatch = PatternMatch(column, pattern, where)val constraint = AnalysisBasedConstraint[NumMatchesAndCount, Double, Double](patternMatch, assertion, hint = hint)val constraintName = name match {case Some(aName) => aNamecase _ => s"PatternMatchConstraint($column, $pattern)"}new NamedConstraint(constraint, constraintName)}case class PatternMatch(column: String, pattern: Regex, where: Option[String] = None)extends StandardScanShareableAnalyzer[NumMatchesAndCount]("PatternMatch", column)with FilterableAnalyzer {override def fromAggregationResult(result: Row, offset: Int): Option[NumMatchesAndCount] = {ifNoNullsIn(result, offset, howMany = 2) { _ =>NumMatchesAndCount(result.getLong(offset), result.getLong(offset + 1))}}override def aggregationFunctions(): Seq[Column] = {val expression = when(regexp_extract(col(column), pattern.toString(), 0) =!= lit(""), 1).otherwise(0)val summation = sum(conditionalSelection(expression, where).cast(IntegerType))summation :: conditionalCount(where) :: Nil}override def filterCondition: Option[String] = whereoverride protected def additionalPreconditions(): Seq[StructType => Unit] = {hasColumn(column) :: isString(column) :: Nil}

执行

核心执行代码如下：

private[this] def runScanningAnalyzers(data: DataFrame,analyzers: Seq[Analyzer[State[_], Metric[_]]],aggregateWith: Option[StateLoader] = None,saveStatesTo: Option[StatePersister] = None): AnalyzerContext = {/* Identify shareable analyzers */val (shareable, others) = analyzers.partition { _.isInstanceOf[ScanShareableAnalyzer[_, _]] }val shareableAnalyzers =shareable.map { _.asInstanceOf[ScanShareableAnalyzer[State[_], Metric[_]]] }/* Compute aggregation functions of shareable analyzers in a single pass over the data */// 生成可用使用agg 一起计算的函数 seqval sharedResults = if (shareableAnalyzers.nonEmpty) {val metricsByAnalyzer = try {val aggregations = shareableAnalyzers.flatMap { _.aggregationFunctions() }/* Compute offsets so that the analyzers can correctly pick their results from the row */val offsets = shareableAnalyzers.scanLeft(0) { case (current, analyzer) =>current + analyzer.aggregationFunctions().length}//实际计算、一个constraint可能有多个agg，比如containsURL就有2个aggregateval results = data.agg(aggregations.head, aggregations.tail: _*).collect().headshareableAnalyzers.zip(offsets).map { case (analyzer, offset) =>//判断成功还是失败analyzer ->successOrFailureMetricFrom(analyzer, results, offset, aggregateWith, saveStatesTo)}} catch {case error: Exception =>shareableAnalyzers.map { analyzer => analyzer -> analyzer.toFailureMetric(error) }}AnalyzerContext(metricsByAnalyzer.toMap[Analyzer[_, Metric[_]], Metric[_]])} else {AnalyzerContext.empty}/* Run non-shareable analyzers separately */val otherMetrics = others.map { analyzer => analyzer -> analyzer.calculate(data, aggregateWith, saveStatesTo) }.toMap[Analyzer[_, Metric[_]], Metric[_]]sharedResults ++ AnalyzerContext(otherMetrics)}

获取metric（结果）和判断结果

前面讲了执行，再看AnalysisBasedConstraint的定义

/* * @param analyzer    Analyzer to be run on the data frame* @param assertion   Assertion function* @param valuePicker Optional function to pick the interested part of the metric value that the*                    assertion will be running on. Absence of such function means the metric*                    value would be used in the assertion as it is.* @param hint A hint to provide additional context why a constraint could have failed* @tparam M : Type of the metric value generated by the Analyzer* @tparam V : Type of the value being used in assertion function**/
private[deequ] case class AnalysisBasedConstraint[S <: State[S], M, V](analyzer: Analyzer[S, Metric[M]],private[deequ] val assertion: V => Boolean,private[deequ] val valuePicker: Option[M => V] = None,private[deequ] val hint: Option[String] = None)

注意取值是有valuePicker 类实现，判断是由 assertion负责

• .hasSize(_ == 5).where(“id = 1”)
• .containsURL(“description”, _ >= 0.5)
  的_ == 5 和 _ >= 0.5 就是assertion。

以containsURL为列，在生成patternMatchConstraint的时候，创建的case class PatternMatch类有如下代码：

  override def fromAggregationResult(result: Row, offset: Int): Option[NumMatchesAndCount] = {ifNoNullsIn(result, offset, howMany = 2) { _ =>NumMatchesAndCount(result.getLong(offset), result.getLong(offset + 1))}}

注意这个NumMatchesAndCount

//计算metric 值，不负责和目标比较
case class NumMatchesAndCount(numMatches: Long, count: Long)extends DoubleValuedState[NumMatchesAndCount] {override def sum(other: NumMatchesAndCount): NumMatchesAndCount = {NumMatchesAndCount(numMatches + other.numMatches, count + other.count)}
// 注意numMatches.toDouble / countoverride def metricValue(): Double = {if (count == 0L) {Double.NaN} else {numMatches.toDouble / count}}
}

HasCount的对应的 case class Size类也有如下代码：

 override def fromAggregationResult(result: Row, offset: Int): Option[NumMatches] = {ifNoNullsIn(result, offset) { _ =>NumMatches(result.getLong(offset))}}

进行判断的核心代码在case class AnalysisBasedConstraint[S <: State[S], M, V] 如下函数中：

private[this] def pickValueAndAssert(metric: Metric[M]): ConstraintResult = {metric.value match {// Analysis done successfully and result metric is therecase Success(metricValue) =>try {val assertOn = runPickerOnMetric(metricValue)val assertionOk = runAssertion(assertOn)if (assertionOk) {ConstraintResult(this, ConstraintStatus.Success, metric = Some(metric))} else {var errorMessage = s"Value: $assertOn does not meet the constraint requirement!"hint.foreach(hint => errorMessage += s" $hint")ConstraintResult(this, ConstraintStatus.Failure, Some(errorMessage), Some(metric))}} catch {case AnalysisBasedConstraint.ConstraintAssertionException(msg) =>ConstraintResult(this, ConstraintStatus.Failure,Some(s"${AnalysisBasedConstraint.AssertionException}: $msg!"), Some(metric))case AnalysisBasedConstraint.ValuePickerException(msg) =>ConstraintResult(this, ConstraintStatus.Failure,Some(s"${AnalysisBasedConstraint.ProblematicMetricPicker}: $msg!"), Some(metric))}// An exception occurred during analysiscase Failure(e) => ConstraintResult(this,ConstraintStatus.Failure, Some(e.getMessage), Some(metric))}
}private def runAssertion(assertOn: V): Boolean =try {assertion(assertOn)} catch {case e: Exception => throw AnalysisBasedConstraint.ConstraintAssertionException(e.getMessage)}