Skip to content

API reference

Measures

dqmeasure.measures.accuracy_range.DataAccuracyRange

Bases: PositionalMeasure

ISO/IEC 25024 Acc-I-7 "Data accuracy range".

Column measure, tier 1, positional: unit = cell, subject = the column.

Null cells are out of scope and count in neither A nor B. A column of nulls scores nan rather than 0.

Parameters:

Name Type Description Default
column str

The numeric column the measure applies to.

 required
low float | None

The interval bounds, or None to learn them from the clean data at fit. Specify both to skip fit entirely.

 None
high float | None

The interval bounds, or None to learn them from the clean data at fit. Specify both to skip fit entirely.

 None
method Literal['minmax']

How the reference interval is derived from the clean data. Currently only "minmax" (the observed minimum and maximum) is implemented, which is the default.

 'minmax'
inclusive bool

Whether the interval bounds count as in-range (low <= v <= high). When False the bounds are treated as out-of-range (low < v < high).

 True
Source code in src/dqmeasure/measures/accuracy_range.py 
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
class DataAccuracyRange(PositionalMeasure):
    """ISO/IEC 25024 `Acc-I-7` "Data accuracy range".

    Column measure, tier 1, positional: unit = cell, subject = the column.

    Null cells are out of scope and count in neither ``A`` nor ``B``. A column of nulls scores
    ``nan`` rather than 0.

    Parameters
    ----------
    column:
        The numeric column the measure applies to.
    low, high:
        The interval bounds, or ``None`` to learn them from the clean data at
        [`fit`][dqmeasure.base.BaseMeasure.fit]. Specify both to skip ``fit`` entirely.
    method:
        How the reference interval is derived from the clean data. Currently only ``"minmax"`` (the observed
        minimum and maximum) is implemented, which is the default.
    inclusive:
        Whether the interval bounds count as in-range (``low <= v <= high``). When ``False`` the bounds are
        treated as out-of-range (``low < v < high``).
    """

    iso_5259_id = "Acc-ML-6"
    iso_25024_id = "Acc-I-7"
    reference_params = ("low", "high")

    low_: float
    high_: float

    def __init__(
        self,
        column: str,
        low: float | None = None,
        high: float | None = None,
        method: Literal["minmax"] = "minmax",
        inclusive: bool = True,
    ) -> None:
        super().__init__(column=column)
        self.low = low
        self.high = high
        self.method = method
        self.inclusive = inclusive

    def _validate(self, frame: nw.DataFrame[Any]) -> None:
        _require_column(frame, self.column, numeric=True)

    def _fit_reference(self, frame: nw.DataFrame[Any]) -> dict[str, Any]:
        if self.method != "minmax":
            raise ValueError(f"Unsupported method: {self.method}")
        col = frame[self.column]
        return {"low": float(col.min()), "high": float(col.max())}

    def _measure_units(self, frame: nw.DataFrame[Any]) -> nw.Series[Any]:
        # Per-cell, return 1.0 if in range, 0.0 if out of range. If a value is missing, return null, which
        # excludes missing values as the measure defines.
        # Encoding it as a null-preserving float rather than a boolean keeps the result identical across
        # backends: pandas' numpy-backed boolean columns cannot hold null and silently turns missing values
        # into False, which would inflate the score() output. Missing values are excluded from the measure.
        closed: Literal["both", "none"] = "both" if self.inclusive else "none"
        expr = (
            nw.when(~nw.col(self.column).is_null())
            .then(nw.col(self.column).is_between(self.low_, self.high_, closed=closed).cast(nw.Float64))
            .otherwise(nw.lit(None))
            .alias(self.column)
        )
        return frame.select(expr)[self.column]

dqmeasure.measures.empty_records.EmptyRecords

Bases: PositionalMeasure

ISO/IEC 25024 Com-I-5 "Empty records in a data file".

Table measure, tier 1, positional: unit = record (row), subject = the whole table.

Counts records where all data items are empty A over all records B and reports X = 1 - A/B, the fraction of records that carry any data ("records exist but are empty"). predict reports 1.0 for a record with at least one non-null cell, and its mean is exactly the standard's 1 - A/B. The output carries no nulls. There is no reference to learn and the measure works without fit.

Source code in src/dqmeasure/measures/empty_records.py 
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
class EmptyRecords(PositionalMeasure):
    """ISO/IEC 25024 `Com-I-5` "Empty records in a data file".

    Table measure, tier 1, positional: unit = record (row), subject = the whole table.

    Counts records where all data items are empty ``A`` over all records ``B`` and reports
    ``X = 1 - A/B``, the fraction of records that carry any data ("records exist but are empty").
    [`predict`][dqmeasure.base.PositionalMeasure.predict] reports ``1.0`` for a record with at least one
    non-null cell, and its mean is exactly the standard's ``1 - A/B``. The output carries no nulls. There
    is no reference to learn and the measure works without [`fit`][dqmeasure.base.BaseMeasure.fit].
    """

    iso_5259_id = None
    iso_25024_id = "Com-I-5"
    scope = "table"

    def __init__(self) -> None:
        pass

    def _measure_units(self, frame: nw.DataFrame[Any]) -> nw.Series[Any]:
        # Per row, returns 1.0 if at least one cell of the row is non-null, 0.0 if the record is empty.
        expr = nw.max_horizontal(*((~nw.col(c).is_null()).cast(nw.Float64) for c in frame.columns))
        return frame.select(expr.alias("record"))["record"]

dqmeasure.measures.feature_completeness.FeatureCompleteness

Bases: PositionalMeasure

ISO/IEC 25024 Com-I-2 "Attribute completeness" (Com-ML-3 "Feature completeness" in ISO/IEC 5259-2).

Column measure, tier 1, positional: unit = cell, subject = the column.

The ratio of non-null values in the column. Every cell is a unit, and nulls are what's being measured. There is no reference to learn, so the measure works without fit.

The table-wide Com-ML-1 "Value completeness" is the table-scoped ValueCompleteness; since every column contributes the same B, its value equals mean(FeatureCompleteness(c).score(df) for c in df.columns).

Parameters:

Name Type Description Default
column str

The column the measure applies to; any dtype works.

 required
Source code in src/dqmeasure/measures/feature_completeness.py 
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
class FeatureCompleteness(PositionalMeasure):
    """ISO/IEC 25024 `Com-I-2` "Attribute completeness" (`Com-ML-3` "Feature completeness" in ISO/IEC 5259-2).

    Column measure, tier 1, positional: unit = cell, subject = the column.

    The ratio of non-null values in the column. Every cell is a unit, and nulls are what's being measured.
    There is no reference to learn, so the measure works without [`fit`][dqmeasure.base.BaseMeasure.fit].

    The table-wide `Com-ML-1` "Value completeness" is the table-scoped
    [`ValueCompleteness`][dqmeasure.measures.value_completeness.ValueCompleteness]; since every column
    contributes the same ``B``, its value equals
    ``mean(FeatureCompleteness(c).score(df) for c in df.columns)``.

    Parameters
    ----------
    column:
        The column the measure applies to; any dtype works.
    """

    iso_5259_id = "Com-ML-3"
    iso_25024_id = "Com-I-2"

    def _measure_units(self, frame: nw.DataFrame[Any]) -> nw.Series[Any]:
        # Per-cell condition: 1.0 if the value is present, 0.0 if null. Every cell is in scope, so the output
        # carries no nulls and score() divides by the row count.
        expr = (~nw.col(self.column).is_null()).cast(nw.Float64).alias(self.column)
        return frame.select(expr)[self.column]

dqmeasure.measures.feature_currentness.FeatureCurrentness

Bases: PositionalMeasure

ISO/IEC 5259-2 Cur-ML-1 "Feature currentness".

Column measure, tier 1, positional: unit = cell, subject = the column. The column holds each data item's timestamp indicating when it was last updated. A column's currentness is measured through the timestamp column that dates it. The condition checks that the cell's age, defined as reference time minus the timestamp, lies within the required age range. A counts the items of the right age and B the non-null timestamps.

The reference time is a measurement-time input, not part of the reference. With reference_time=None it is the wall clock, read once per fit/predict/score call, so the acceptable timestamp window moves with time. Pin reference_time for reproducible results. Time-zone-naive and -aware columns both work as long as the column and the reference time are consistent. Note that the datetime.now() default is naive.

The ISO standard calls for a date range, which includes min_age and max_age. That's somewhat unintuitive, a learned min_age makes data fresher than any clean item fail the range. To measure only a freshness ceiling, specify min_age=timedelta(0).

Parameters:

Name Type Description Default
column str

The datetime column the measure applies to.

 required
min_age timedelta | None

The required age range, or None to learn the bounds from the ages observed in the clean data at fit. Specify both to skip fit entirely. Specify min_age=timedelta(0) to not punish data that's fresher than the reference data's freshest row.

 None
max_age timedelta | None

The required age range, or None to learn the bounds from the ages observed in the clean data at fit. Specify both to skip fit entirely. Specify min_age=timedelta(0) to not punish data that's fresher than the reference data's freshest row.

 None
reference_time datetime | None

The instant ages are computed against, or None for the current wall clock at each call.

 None
Source code in src/dqmeasure/measures/feature_currentness.py 
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
class FeatureCurrentness(PositionalMeasure):
    """ISO/IEC 5259-2 `Cur-ML-1` "Feature currentness".

    Column measure, tier 1, positional: unit = cell, subject = the column. The column holds
    each data item's timestamp indicating when it was last updated. A column's currentness is measured through
    the timestamp column that dates it. The condition checks that the cell's age, defined as reference time minus
    the timestamp, lies within the required age range. ``A`` counts the items of the right age and ``B`` the
    non-null timestamps.

    The reference time is a measurement-time input, not part of the reference. With
    ``reference_time=None`` it is the wall clock, read once per ``fit``/``predict``/``score`` call, so the
    acceptable timestamp window moves with time. Pin ``reference_time`` for reproducible results. Time-zone-naive
    and -aware columns both work as long as the column and the reference time are consistent.  Note that the
    ``datetime.now()`` default is naive.

    The ISO standard calls for a date range, which includes `min_age` and `max_age`. That's somewhat unintuitive,
    a learned ``min_age`` makes data *fresher* than any clean item fail the range. To measure only a freshness ceiling,
    specify ``min_age=timedelta(0)``.

    Parameters
    ----------
    column:
        The datetime column the measure applies to.
    min_age, max_age:
        The required age range, or ``None`` to learn the bounds from the ages observed in the clean data at
        [`fit`][dqmeasure.base.BaseMeasure.fit]. Specify both to skip ``fit`` entirely. Specify ``min_age=timedelta(0)``
        to not punish data that's fresher than the reference data's freshest row.
    reference_time:
        The instant ages are computed against, or ``None`` for the current wall clock at each call.
    """

    iso_5259_id = "Cur-ML-1"
    iso_25024_id = None
    reference_params = ("min_age", "max_age")

    min_age_: timedelta
    max_age_: timedelta

    def __init__(
        self,
        column: str,
        min_age: timedelta | None = None,
        max_age: timedelta | None = None,
        reference_time: datetime | None = None,
    ) -> None:
        super().__init__(column=column)
        self.min_age = min_age
        self.max_age = max_age
        self.reference_time = reference_time

    def _reference_time(self) -> datetime:
        return self.reference_time if self.reference_time is not None else datetime.now()

    def _validate(self, frame: nw.DataFrame[Any]) -> None:
        _require_column(frame, self.column, temporal=True)

    def _fit_reference(self, frame: nw.DataFrame[Any]) -> dict[str, Any]:
        ref = self._reference_time()
        ages = frame.select((nw.lit(ref) - nw.col(self.column)).alias(self.column))[self.column].drop_nulls()
        if len(ages) == 0:
            raise ValueError(
                f"{type(self).__name__}: column {self.column!r} has no non-null timestamps in the clean data"
            )
        return {"min_age": ages.min(), "max_age": ages.max()}

    def _measure_units(self, frame: nw.DataFrame[Any]) -> nw.Series[Any]:
        # Null-preserving float, as in DataAccuracyRange. The explicit null guard is required: In pandas,
        # a duration comparison against a missing timestamp returns False rather than null,
        # which messes up the measure.
        age = nw.lit(self._reference_time()) - nw.col(self.column)
        expr = (
            nw.when(~nw.col(self.column).is_null())
            .then(((age >= self.min_age_) & (age <= self.max_age_)).cast(nw.Float64))
            .otherwise(nw.lit(None))
            .alias(self.column)
        )
        return frame.select(expr)[self.column]

dqmeasure.measures.format_consistency.DataFormatConsistency

Bases: PositionalMeasure

ISO/IEC 25024 Con-I-2 "Data format consistency" (Con-ML-3 in ISO/IEC 5259-2).

Column measure, tier 1, positional: unit = cell, subject = the column.

A value is format-consistent when its shape is one of the column's admissible format shapes. Here, a shape is a string where d indicates a digit, a a letter, and every other character is kept literally. For example, "202401" -> "dddddd", "2024-01" -> "dddd-dd".

The measure applies to string-encoded columns only.

Parameters:

Name Type Description Default
column str

The column the measure applies to (string, categorical, or enum).

 required
formats Collection[str] | None

The admissible format shapes, written in the shape alphabet (e.g. {"dddd-dd"}), or None to learn the set of shapes observed in the clean data at fit.

 None
Source code in src/dqmeasure/measures/format_consistency.py 
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
class DataFormatConsistency(PositionalMeasure):
    """ISO/IEC 25024 `Con-I-2` "Data format consistency" (`Con-ML-3` in ISO/IEC 5259-2).

    Column measure, tier 1, positional: unit = cell, subject = the column.

    A value is format-consistent when its shape is one of the column's admissible format shapes. Here,
    a shape is a string where ``d`` indicates a digit, ``a`` a letter, and every other character is kept
    literally. For example, ``"202401" -> "dddddd"``, ``"2024-01" -> "dddd-dd"``.

    The measure applies to string-encoded columns only.

    Parameters
    ----------
    column:
        The column the measure applies to (string, categorical, or enum).
    formats:
        The admissible format shapes, written in the shape alphabet (e.g. ``{"dddd-dd"}``), or ``None`` to
        learn the set of shapes observed in the clean data at [`fit`][dqmeasure.base.BaseMeasure.fit].
    """

    iso_5259_id = "Con-ML-3"
    iso_25024_id = "Con-I-2"
    reference_params = ("formats",)

    formats_: Collection[str]

    def __init__(self, column: str, formats: Collection[str] | None = None) -> None:
        super().__init__(column=column)
        self.formats = formats

    def _validate(self, frame: nw.DataFrame[Any]) -> None:
        super()._validate(frame)
        dtype = frame.schema[self.column]
        if not isinstance(dtype, (nw.String, nw.Categorical, nw.Enum)):
            raise ValueError(
                f"Column {self.column!r} has dtype {dtype}; format consistency applies to string-encoded "
                "columns only (string, categorical, or enum). Typed columns have their format enforced by the schema"
            )

    def _fit_reference(self, frame: nw.DataFrame[Any]) -> dict[str, Any]:
        values = frame[self.column].drop_nulls().to_list()
        return {"formats": {_shape(value) for value in values}}

    def _measure_units(self, frame: nw.DataFrame[Any]) -> nw.Series[Any]:
        # Per-cell condition: 1.0 if the value's shape is admissible, 0.0 if not, null if the value is missing
        # (out of scope, a null has no format). We use the same to_list/new_series bridge as SemanticDataAccuracy
        # to enable same behavior in pandas and polars.
        formats = set(self.formats_)
        nulls = frame[self.column].is_null().to_list()
        values = frame[self.column].to_list()
        data = [None if nulls[i] else float(_shape(values[i]) in formats) for i in range(len(frame))]
        return nw.new_series(self.column, data, nw.Float64, backend=frame.implementation)

dqmeasure.measures.inaccuracy_risk.RiskOfDataSetInaccuracy

Bases: PositionalMeasure

ISO/IEC 25024 Acc-I-4 "Risk of data set inaccuracy" (Acc-ML-4 in ISO/IEC 5259-2).

Column measure, tier 1, positional: unit = cell, subject = the column. The standard defines Acc-I-4 as the risk of inaccuracy, counting outliers, so we report 1 - X to keep every measure higher-is-better: X is the ratio of values that are not outliers.

The standard leaves the outlier criterion open, and this implementation uses the robust z-score: a value is an outlier when |value - center| > threshold * scale, with center and scale learned from clean data as the median and the sigma-scaled median absolute deviation (see https://en.wikipedia.org/wiki/Robust_measures_of_scale and https://en.wikipedia.org/wiki/Median_absolute_deviation).

When a column is constant in the clean data (scale = 0), every deviating value counts as an outlier.

Parameters:

Name Type Description Default
column str

The numeric column the measure applies to.

 required
center float | None

The reference location and dispersion, or None to learn them from the clean data at fit. Specify both to skip fit entirely.

 None
scale float | None

The reference location and dispersion, or None to learn them from the clean data at fit. Specify both to skip fit entirely.

 None
method Literal['mad']

How the reference is estimated from the clean data. Currently "mad" (median and sigma-scaled median absolute deviation), which is the default.

 'mad'
threshold float

How many scale units a value may deviate from the center before it counts as an outlier. The default 3.5 is the customary cutoff for robust z-scores (Iglewicz-Hoaglin).

 3.5
Source code in src/dqmeasure/measures/inaccuracy_risk.py 
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
class RiskOfDataSetInaccuracy(PositionalMeasure):
    """ISO/IEC 25024 `Acc-I-4` "Risk of data set inaccuracy" (`Acc-ML-4` in ISO/IEC 5259-2).

    Column measure, tier 1, positional: unit = cell, subject = the column. The standard
    defines `Acc-I-4` as the risk of inaccuracy, counting outliers, so we report ``1 - X`` to keep every
    measure higher-is-better: ``X`` is the ratio of values that are not outliers.

    The standard leaves the outlier criterion open, and this implementation uses the robust z-score: a value is
    an outlier when ``|value - center| > threshold * scale``, with ``center`` and ``scale`` learned from clean data
    as the median and the sigma-scaled median absolute deviation (see https://en.wikipedia.org/wiki/Robust_measures_of_scale
    and https://en.wikipedia.org/wiki/Median_absolute_deviation).

    When a column is constant in the clean data (``scale = 0``), every deviating value counts as an outlier.

    Parameters
    ----------
    column:
        The numeric column the measure applies to.
    center, scale:
        The reference location and dispersion, or ``None`` to learn them from the clean data at
        [`fit`][dqmeasure.base.BaseMeasure.fit]. Specify both to skip ``fit`` entirely.
    method:
        How the reference is estimated from the clean data. Currently ``"mad"`` (median and sigma-scaled median
        absolute deviation), which is the default.
    threshold:
        How many scale units a value may deviate from the center before it counts as an outlier. The default
        ``3.5`` is the customary cutoff for robust z-scores (Iglewicz-Hoaglin).
    """

    iso_5259_id = "Acc-ML-4"
    iso_25024_id = "Acc-I-4"
    reference_params = ("center", "scale")

    center_: float
    scale_: float

    def __init__(
        self,
        column: str,
        center: float | None = None,
        scale: float | None = None,
        method: Literal["mad"] = "mad",
        threshold: float = 3.5,
    ) -> None:
        super().__init__(column=column)
        self.center = center
        self.scale = scale
        self.method = method
        self.threshold = threshold

    def _validate(self, frame: nw.DataFrame[Any]) -> None:
        _require_column(frame, self.column, numeric=True)

    def _fit_reference(self, frame: nw.DataFrame[Any]) -> dict[str, Any]:
        if self.method != "mad":
            raise ValueError(f"Unsupported method: {self.method!r}")
        col = frame[self.column].drop_nulls().cast(nw.Float64)
        median = float(col.median())
        return {"center": median, "scale": float((col - median).abs().median()) * _MAD_TO_SIGMA}

    def _measure_units(self, frame: nw.DataFrame[Any]) -> nw.Series[Any]:
        # Per-cell condition: 1.0 if the value is not an outlier, 0.0 if it is, null if the value is missing
        # (the same null-preserving float encoding as DataAccuracyRange, and for the same backend-parity reasons).
        expr = (
            nw.when(~nw.col(self.column).is_null())
            .then(((nw.col(self.column) - self.center_).abs() <= self.threshold * self.scale_).cast(nw.Float64))
            .otherwise(nw.lit(None))
            .alias(self.column)
        )
        return frame.select(expr)[self.column]

dqmeasure.measures.inconsistency_risk.RiskOfDataInconsistency

Bases: PositionalMeasure

ISO/IEC 25024 Con-I-3 "Risk of data inconsistency".

Column measure, tier 1, positional: unit = cell, subject = the column.

A cell counts as a duplication when its value occurs more than once in the column. The standard defines Con-I-3 as the risk of inconsistency, i.e. the ratio of duplicated cells. We report 1 - X to keep every measure higher-is-better, where X is the standard's share of duplicated cells. The score is thus the ratio of cells holding a value unique in the column. Nulls are out of scope, as two nulls are not duplicates of each other.

This measure concerns duplicate values in one column. The table-scoped DataRecordConsistency addresses entire rows.

There is no reference to learn, and the measure works without fit. The user should apply this measure to columns where a repeated value signals redundant storage (identifiers, names of entities stored once), not where repetition is natural (categories).

Parameters:

Name Type Description Default
column str

The column the measure applies to; any dtype works.

 required
Source code in src/dqmeasure/measures/inconsistency_risk.py 
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
class RiskOfDataInconsistency(PositionalMeasure):
    """ISO/IEC 25024 `Con-I-3` "Risk of data inconsistency".

    Column measure, tier 1, positional: unit = cell, subject = the column.

    A cell counts as a duplication when its value occurs more than once in the column. The standard defines
    `Con-I-3` as the risk of inconsistency, i.e. the ratio of duplicated cells. We report ``1 - X`` to keep
    every measure higher-is-better, where ``X`` is the standard's share of duplicated cells. The score is
    thus the ratio of cells holding a value unique in the column. Nulls are out of scope, as two nulls are not
    duplicates of each other.

    This measure concerns duplicate values in one column. The table-scoped
    [`DataRecordConsistency`][dqmeasure.measures.record_consistency.DataRecordConsistency] addresses entire rows.

    There is no reference to learn, and the measure works without [`fit`][dqmeasure.base.BaseMeasure.fit].
    The user should apply this measure to columns where a repeated value signals redundant storage (identifiers,
    names of entities stored once), not where repetition is natural (categories).

    Parameters
    ----------
    column:
        The column the measure applies to; any dtype works.
    """

    iso_5259_id = None
    iso_25024_id = "Con-I-3"

    def _measure_units(self, frame: nw.DataFrame[Any]) -> nw.Series[Any]:
        # We need two helper columns, this ensures that names don't collide with actual column names
        index, count = "_dqm_index", "_dqm_count"
        if self.column in (index, count):
            index, count = index + "_", count + "_"

        # Use a group_by to get value counts and restore order through the index column.
        counts = frame.select(nw.col(self.column)).group_by(self.column).agg(nw.len().alias(count))
        counted = (
            frame.select(nw.col(self.column)).with_row_index(index).join(counts, on=self.column, how="left").sort(index)
        )

        # Per-cell: 1.0 if the value is unique in the column, 0.0 if it occurs more than once, null
        # if the value is missing (out of scope).
        expr = (
            nw.when(~nw.col(self.column).is_null())
            .then((nw.col(count) == 1).cast(nw.Float64))
            .otherwise(nw.lit(None))
            .alias(self.column)
        )
        return counted.select(expr)[self.column]

dqmeasure.measures.label_completeness.LabelCompleteness

Bases: PositionalMeasure

ISO/IEC 5259-2 Com-ML-5 "Label completeness".

Column measure, tier 1, positional: unit = sample (row), subject = the label column.

Counts unlabelled or incompletely labelled samples A over all samples B and reports X = 1 - A/B, the fraction of fully labelled samples. A sample counts as unlabelled when its label is null: missing labels are assumed to be null-encoded (see the model doc's simplifying assumptions). predict reports 1.0 for a labelled sample, and its mean is exactly the standard's 1 - A/B. There is no reference to learn, the measure works without fit.

Com-ML-5 coincides numerically with Com-ML-3 feature completeness on the label column; the measures stay distinct in the role of the column and their measurement function.

Parameters:

Name Type Description Default
column str

The label column the measure applies to; any dtype works.

 required
Source code in src/dqmeasure/measures/label_completeness.py 
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
class LabelCompleteness(PositionalMeasure):
    """ISO/IEC 5259-2 `Com-ML-5` "Label completeness".

    Column measure, tier 1, positional: unit = sample (row), subject = the label column.

    Counts unlabelled or incompletely labelled samples ``A`` over all samples ``B`` and reports
    ``X = 1 - A/B``, the fraction of fully labelled samples. A sample counts as unlabelled when its label is
    null: missing labels are assumed to be null-encoded (see the model doc's simplifying
    assumptions). [`predict`][dqmeasure.base.PositionalMeasure.predict] reports ``1.0`` for a labelled sample,
    and its mean is exactly the standard's ``1 - A/B``. There is no
    reference to learn, the measure works without [`fit`][dqmeasure.base.BaseMeasure.fit].

    `Com-ML-5` coincides numerically with `Com-ML-3` feature completeness on the label column; the measures
    stay distinct in the role of the column and their measurement function.

    Parameters
    ----------
    column:
        The label column the measure applies to; any dtype works.
    """

    iso_5259_id = "Com-ML-5"
    iso_25024_id = None

    def _measure_units(self, frame: nw.DataFrame[Any]) -> nw.Series[Any]:
        # Per-sample condition: 1.0 if labelled, 0.0 if the label is null. Every sample is in scope, so the
        # output carries no nulls and score() divides by the row count.
        expr = (~nw.col(self.column).is_null()).cast(nw.Float64).alias(self.column)
        return frame.select(expr)[self.column]

dqmeasure.measures.record_completeness.RecordCompleteness

Bases: PositionalMeasure

ISO/IEC 5259-2 Com-ML-4 "Record completeness".

Table measure, tier 1, positional: unit = record (row), subject = the whole table.

The ratio of rows that have no empty cell over all rows. nulls are the thing being measured, and a row with null(s) scores 0.0. There is no reference to learn, the measure works without fit.

Source code in src/dqmeasure/measures/record_completeness.py 
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
class RecordCompleteness(PositionalMeasure):
    """ISO/IEC 5259-2 `Com-ML-4` "Record completeness".

    Table measure, tier 1, positional: unit = record (row), subject = the whole table.

    The ratio of rows that have no empty cell over all rows. nulls are the thing
    being measured, and a row with null(s) scores ``0.0``. There is no reference to learn, the
    measure works without [`fit`][dqmeasure.base.BaseMeasure.fit].
    """

    iso_5259_id = "Com-ML-4"
    iso_25024_id = None
    scope = "table"

    def __init__(self) -> None:
        pass

    def _measure_units(self, frame: nw.DataFrame[Any]) -> nw.Series[Any]:
        # Per-record condition: 1.0 if no cell of the row is null, else 0.0. Every record is in scope, the
        # output carries no nulls and score() divides by the row count.
        expr = nw.min_horizontal(*((~nw.col(c).is_null()).cast(nw.Float64) for c in frame.columns))
        return frame.select(expr.alias("record"))["record"]

dqmeasure.measures.record_consistency.DataRecordConsistency

Bases: PositionalMeasure

ISO/IEC 5259-2 Con-ML-1 "Data record consistency".

Table measure, tier 1, positional: unit = record (row), subject = the whole table.

The ratio of records that occur exactly once in the dataset. The standard defines Con-ML-1 as the ratio of duplicate records. We report 1 - X to keep every measure higher-is-better, where X is the ratio of duplicate rows in the table. A record counts as a duplicate when the full row occurs more than once. This measure is the row-level case of RiskOfDataInconsistency. Records containing one or more null values are out of scope, because two nulls are not duplicates of each other.

There is no reference to learn, so the measure works without fit.

Source code in src/dqmeasure/measures/record_consistency.py 
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
class DataRecordConsistency(PositionalMeasure):
    """ISO/IEC 5259-2 `Con-ML-1` "Data record consistency".

    Table measure, tier 1, positional: unit = record (row), subject = the whole table.

    The ratio of records that occur exactly once in the dataset. The standard defines `Con-ML-1` as the
    ratio of *duplicate* records. We report ``1 - X`` to keep every measure higher-is-better, where ``X``
    is the ratio of duplicate rows in the table. A record counts as a duplicate when the full row occurs
    more than once. This measure is the row-level case of
    [`RiskOfDataInconsistency`][dqmeasure.measures.inconsistency_risk.RiskOfDataInconsistency]. Records
    containing one or more null values are out of scope, because two nulls are not duplicates of each other.

    There is no reference to learn, so the measure works without [`fit`][dqmeasure.base.BaseMeasure.fit].
    """

    iso_5259_id = "Con-ML-1"
    iso_25024_id = None
    scope = "table"

    def __init__(self) -> None:
        pass

    def _measure_units(self, frame: nw.DataFrame[Any]) -> nw.Series[Any]:
        # Per row, returns 1.0 if the full row is unique, 0.0 if it occurs more than once, and null if the
        # row contains a null (out of scope).
        unique = (~frame.is_duplicated()).cast(nw.Float64).alias("record")
        no_null = frame.select(
            nw.min_horizontal(*((~nw.col(c).is_null()).cast(nw.Float64) for c in frame.columns)).alias("record")
        )["record"]
        nulls = nw.new_series("record", [None] * len(frame), dtype=nw.Float64, backend=frame.implementation)
        return unique.zip_with(no_null == 1.0, nulls)

dqmeasure.measures.record_currentness.RecordCurrentness

Bases: PositionalMeasure

ISO/IEC 5259-2 Cur-ML-2 "Record currentness".

Table measure, tier 1, positional: unit = record (row), subject = the whole table.

The ratio of records where all data items fall within the required age range. The measure considers columns of datatype date or datetime to derive one or more ages per row. We call these columns "temporal" columns. For example, a tables' temporal columns could be inserted_at and last_updated_at, and they may have different required age ranges.

This measure is the per-column analog of Cur-ML-1 feature currentness.

A record conforms when every one of its non-null temporal cells is of the right age. Null cells are ignored. The frame must have at least one temporal column, and every temporal column of the measured frame must be covered by the reference.

The reference time is a measurement-time input: with reference_time=None it is the wall clock, read once per fit/predict/score call. Pin reference_time for reproducible results.

Parameters:

Name Type Description Default
age_ranges dict[str, tuple[timedelta, timedelta]] | None

The required age range per temporal column, {column: (min_age, max_age)}, or None to learn the ranges from the ages observed in the clean data at fit.

 None
reference_time datetime | None

The instant ages are computed against, or None for the current wall clock at each call.

 None
Source code in src/dqmeasure/measures/record_currentness.py 
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
class RecordCurrentness(PositionalMeasure):
    """ISO/IEC 5259-2 `Cur-ML-2` "Record currentness".

    Table measure, tier 1, positional: unit = record (row), subject = the whole table.

    The ratio of records where all data items fall within the required age range. The measure considers
    columns of datatype date or datetime to derive one or more ages per row. We call these columns "temporal"
    columns. For example, a tables' temporal columns could be `inserted_at` and `last_updated_at`, and they
    may have different required age ranges.

    This measure is the per-column analog of `Cur-ML-1` feature currentness.

    A record conforms when every one of its non-null temporal cells is of the right age. Null cells are ignored.
    The frame must have at least one temporal column, and every temporal column of the measured frame
    must be covered by the reference.

    The reference time is a measurement-time input: with ``reference_time=None`` it is the wall clock, read
    once per ``fit``/``predict``/``score`` call. Pin ``reference_time`` for reproducible results.

    Parameters
    ----------
    age_ranges:
        The required age range per temporal column, ``{column: (min_age, max_age)}``, or ``None`` to learn
        the ranges from the ages observed in the clean data at [`fit`][dqmeasure.base.BaseMeasure.fit].
    reference_time:
        The instant ages are computed against, or ``None`` for the current wall clock at each call.
    """

    iso_5259_id = "Cur-ML-2"
    iso_25024_id = None
    scope = "table"
    reference_params = ("age_ranges",)

    age_ranges_: dict[str, tuple[timedelta, timedelta]]

    def __init__(
        self,
        *,
        age_ranges: dict[str, tuple[timedelta, timedelta]] | None = None,
        reference_time: datetime | None = None,
    ) -> None:
        self.age_ranges = age_ranges
        self.reference_time = reference_time

    def _reference_time(self) -> datetime:
        return self.reference_time if self.reference_time is not None else datetime.now()

    @staticmethod
    def _temporal_columns(frame: nw.DataFrame[Any]) -> list[str]:
        # Not dtype.is_temporal(), which also admits Duration and Time — timestamps only.
        return [c for c, dtype in frame.schema.items() if isinstance(dtype, (nw.Datetime, nw.Date))]

    def _validate(self, frame: nw.DataFrame[Any]) -> None:
        if not self._temporal_columns(frame):
            raise ValueError(f"{type(self).__name__} needs a frame with at least one datetime or date column")

    def _check_coverage(self, frame: nw.DataFrame[Any]) -> None:
        # Ensures all date and datetype columns are age checked.
        uncovered = [c for c in self._temporal_columns(frame) if c not in self.age_ranges_]
        if uncovered:
            raise ValueError(
                f"{type(self).__name__}: temporal columns {uncovered} are not covered by the reference "
                f"(covered: {sorted(self.age_ranges_)})"
                "If you want to exclude date or datetime columns from the evaluation, consider subsetting "
                "the frame or using Cur-ML-1 instead."
            )
        for column in self.age_ranges_:
            _require_column(frame, column, temporal=True)

    def _fit_reference(self, frame: nw.DataFrame[Any]) -> dict[str, Any]:
        ref = self._reference_time()
        age_ranges: dict[str, tuple[timedelta, timedelta]] = {}
        for column in self._temporal_columns(frame):
            ages = frame.select((nw.lit(ref) - nw.col(column)).alias(column))[column].drop_nulls()
            if len(ages) == 0:
                raise ValueError(
                    f"{type(self).__name__}: column {column!r} has no non-null timestamps in the clean data"
                )
            age_ranges[column] = (ages.min(), ages.max())
        return {"age_ranges": age_ranges}

    def _measure_units(self, frame: nw.DataFrame[Any]) -> nw.Series[Any]:
        self._check_coverage(frame)
        ref = self._reference_time()
        # Per temporal cell: in-range as null-preserving float, exactly as in FeatureCurrentness (the null
        # guard is mandatory on the pandas backend, where a duration comparison against a missing timestamp
        # yields False rather than null). Per record: 0.0 if one or more temporal columns are outside the age
        # range, 1.0 otherwise. A record is judged on its non-null timestamps and is out of scope only when
        # all of them are null.
        indicators = []
        for column, (min_age, max_age) in self.age_ranges_.items():
            age = nw.lit(ref) - nw.col(column)
            indicators.append(
                nw.when(~nw.col(column).is_null())
                .then(((age >= min_age) & (age <= max_age)).cast(nw.Float64))
                .otherwise(nw.lit(None))
                .alias(column)
            )
        expr = nw.min_horizontal(*indicators)
        return frame.select(expr.alias("record"))["record"]

dqmeasure.measures.semantic_accuracy.SemanticDataAccuracy

Bases: PositionalMeasure

ISO/IEC 25024 Acc-I-2 "Semantic data accuracy" (Acc-ML-2 in ISO/IEC 5259-2).

Column measure, tier 1, positional: unit = cell, subject = the column. The measure is scoped to one column, but its condition reads the whole row, which goes into the prompt as context.

An LLM judges whether each value is semantically accurate given the rest of its record and its real-world knowledge. The LLM is prompted with example records sampled from the clean data (few-shot serialization inspired by mimir's llm_master, see https://github.com/calgo-lab/mimir).

The measure sends requests to any OpenAI-compatible chat-completions endpoint, sending one request per record.

The reference (the sampled example records) cannot be specified in the constructor: fit on clean data is always required.

Parameters:

Name Type Description Default
column str

The column the measure applies to. All other columns of the frame go into the prompt as context.

 required
llm_model str

Model name and base URL (up to and including /v1) of an OpenAI-compatible chat-completions endpoint. The defaults target a local Ollama server with a model small enough for a laptop.

 'llama3.2:3b'
llm_url str

Model name and base URL (up to and including /v1) of an OpenAI-compatible chat-completions endpoint. The defaults target a local Ollama server with a model small enough for a laptop.

 'llama3.2:3b'
llm_api_key str | None

Bearer token for the endpoint. None (default) falls back to the OPENAI_API_KEY environment variable.

 None
n_examples int

Number of clean records sampled at fit time as few-shot examples in the prompt.

 5
random_state int

Seed for the example sampling, making the measurement procedure reproducible.

 0
n_jobs int

Number of concurrent requests. 1 (default) sends them sequentially.

 1
provider str | None

Pin every request to one upstream provider (routers such as OpenRouter otherwise pick per request, which harms reproducibility). None (default) leaves routing to the endpoint.

 None
Source code in src/dqmeasure/measures/semantic_accuracy.py 
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
class SemanticDataAccuracy(PositionalMeasure):
    """ISO/IEC 25024 `Acc-I-2` "Semantic data accuracy" (`Acc-ML-2` in ISO/IEC 5259-2).

    Column measure, tier 1, positional: unit = cell, subject = the column. The measure is
    scoped to one column, but its condition reads the whole row, which goes into the prompt as context.

    An LLM judges whether each value is semantically accurate given the rest of its record and its
    real-world knowledge. The LLM is prompted with example records sampled from the clean data
    (few-shot serialization inspired by mimir's ``llm_master``, see https://github.com/calgo-lab/mimir).

    The measure sends requests to any OpenAI-compatible chat-completions endpoint, sending one request per record.

    The reference (the sampled example records) cannot be specified in the constructor:
    [`fit`][dqmeasure.base.BaseMeasure.fit] on clean data is always required.

    Parameters
    ----------
    column:
        The column the measure applies to. All other columns of the frame go into the prompt as context.
    llm_model, llm_url:
        Model name and base URL (up to and including ``/v1``) of an OpenAI-compatible chat-completions
        endpoint. The defaults target a local Ollama server with a model small enough for a laptop.
    llm_api_key:
        Bearer token for the endpoint. ``None`` (default) falls back to the ``OPENAI_API_KEY`` environment
        variable.
    n_examples:
        Number of clean records sampled at fit time as few-shot examples in the prompt.
    random_state:
        Seed for the example sampling, making the measurement procedure reproducible.
    n_jobs:
        Number of concurrent requests. ``1`` (default) sends them sequentially.
    provider:
        Pin every request to one upstream provider (routers such as OpenRouter otherwise pick per request,
        which harms reproducibility). ``None`` (default) leaves routing to the endpoint.
    """

    iso_5259_id = "Acc-ML-2"
    iso_25024_id = "Acc-I-2"

    examples_: list[dict[str, Any]]

    def __init__(
        self,
        column: str,
        llm_model: str = "llama3.2:3b",
        llm_url: str = "http://localhost:11434/v1",
        llm_api_key: str | None = None,
        n_examples: int = 5,
        random_state: int = 0,
        n_jobs: int = 1,
        provider: str | None = None,
    ) -> None:
        super().__init__(column=column)
        self.llm_model = llm_model
        self.llm_url = llm_url
        self.llm_api_key = llm_api_key
        self.n_examples = n_examples
        self.random_state = random_state
        self.n_jobs = n_jobs
        self.provider = provider

    def fit(self, X: IntoDataFrame) -> Self:
        """Sample the few-shot example records from a clean (training) dataframe. Mandatory for this measure."""
        frame = nw.from_native(X, eager_only=True)
        self._validate(frame)
        k = min(self.n_examples, len(frame))
        self.examples_ = list(frame.sample(n=k, seed=self.random_state).iter_rows(named=True))
        self._resolved = True
        return self

    def _check_is_resolved(self) -> None:
        if not getattr(self, "_resolved", False):
            raise NotResolvedError(
                f"{type(self).__name__}: the reference cannot be specified in the constructor; "
                "call fit() on clean data first."
            )

    def _measure_units(self, frame: nw.DataFrame[Any]) -> nw.Series[Any]:
        # Per-cell condition: 1.0 if the model judges the value accurate, 0.0 if not, null if the value is
        # missing or the model's response could not be parsed.
        llm = openai_completion(self.llm_model, self.llm_url, self.llm_api_key, provider=self.provider)
        nulls = frame[self.column].is_null().to_list()
        rows = list(frame.iter_rows(named=True))
        row_prompts: list[str | None] = [None if nulls[i] else self._prompt(row) for i, row in enumerate(rows)]

        # Every distinct prompt is asked once
        distinct_prompts = list(dict.fromkeys(prompt for prompt in row_prompts if prompt is not None))
        responses = complete_many(llm, distinct_prompts, n_jobs=self.n_jobs)

        verdicts: dict[str, float | None] = {}
        failures: list[str] = []
        for prompt, response in zip(distinct_prompts, responses, strict=True):
            verdict = _parse_verdict(response)
            verdicts[prompt] = verdict
            if verdict is None:
                failures.append(response)
        if failures:
            warnings.warn(
                f"Could not parse {len(failures)} of {len(distinct_prompts)} LLM responses as a verdict "
                f"(example: {failures[0]!r}). Counting them as missing, not as accurate.",
                stacklevel=2,
            )

        data: list[float | None] = [None if prompt is None else verdicts[prompt] for prompt in row_prompts]
        return nw.new_series(self.column, data, nw.Float64, backend=frame.implementation)

    def _prompt(self, row: dict[str, Any]) -> str:
        header = " | ".join(row.keys())
        examples = "\n".join(render_record(example) for example in self.examples_)
        value = "<missing>" if is_missing(row[self.column]) else row[self.column]
        return (
            "You judge whether a value in a table record is semantically accurate: whether the value makes "
            "sense for its column, given the rest of the record and real-world knowledge.\n"
            "Fields are pipe-separated in the order given by the header. A missing value is "
            "shown as <missing>.\n"
            'Respond with JSON only, in the form {"accurate": true} or {"accurate": false}.\n\n'
            f"Columns: {header}\n\n"
            f"Records from the same table that are known to be accurate:\n{examples}\n\n"
            f"Record: {render_record(row)}\n"
            f"Column: {self.column}\n"
            f"Value: {value}\n"
            "Is the value (not the row) semantically accurate? Respond with JSON only:"
        )

fit 

fit(X)

Sample the few-shot example records from a clean (training) dataframe. Mandatory for this measure.

Source code in src/dqmeasure/measures/semantic_accuracy.py 
83
84
85
86
87
88
89
90
def fit(self, X: IntoDataFrame) -> Self:
    """Sample the few-shot example records from a clean (training) dataframe. Mandatory for this measure."""
    frame = nw.from_native(X, eager_only=True)
    self._validate(frame)
    k = min(self.n_examples, len(frame))
    self.examples_ = list(frame.sample(n=k, seed=self.random_state).iter_rows(named=True))
    self._resolved = True
    return self

dqmeasure.measures.semantic_consistency.SemanticConsistency

Bases: PositionalMeasure

ISO/IEC 25024 Con-I-6 "Semantic consistency" (Con-ML-4 in ISO/IEC 5259-2).

Column measure, tier 1, positional: unit = cell, subject = the column. The measure is scoped to the column whose values the rules constrain, and each rule reads the rest of the row as context.

The semantic rules are narwhals boolean expressions. Per row, the condition is 1 if every evaluable rule holds, 0 if any evaluable rule fails, and null when no rule is evaluable.

The rules are either specified in the constructor, e.g. rules=[nw.col("recruited") > nw.col("born")], or learned from clean data at fit. The rules origin doesn't matter: the DQM evaluates the expressions and counts the rows that satisfy them.

Parameters:

Name Type Description Default
column str

The column the measure applies to. Rules constrain this column's values.

 required
rules Sequence[Expr] | None

The semantic rules as narwhals boolean expressions, or None to mine them from the clean data at fit.

 None
Source code in src/dqmeasure/measures/semantic_consistency.py 
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
class SemanticConsistency(PositionalMeasure):
    """ISO/IEC 25024 `Con-I-6` "Semantic consistency" (`Con-ML-4` in ISO/IEC 5259-2).

    Column measure, tier 1, positional: unit = cell, subject = the column. The measure is
    scoped to the column whose values the rules constrain, and each rule reads the rest of the row as
    context.

    The semantic rules are narwhals boolean expressions. Per row, the condition is 1 if every evaluable rule holds,
    0 if any evaluable rule fails, and null when no rule is evaluable.

    The rules are either specified in the constructor, e.g.
    ``rules=[nw.col("recruited") > nw.col("born")]``, or learned from clean data at
    [`fit`][dqmeasure.base.BaseMeasure.fit]. The rules origin doesn't matter: the DQM evaluates the expressions
    and counts the rows that satisfy them.

    Parameters
    ----------
    column:
        The column the measure applies to. Rules constrain this column's values.
    rules:
        The semantic rules as narwhals boolean expressions, or ``None`` to mine them from the clean data at
        [`fit`][dqmeasure.base.BaseMeasure.fit].
    """

    iso_5259_id = "Con-ML-4"
    iso_25024_id = "Con-I-6"
    reference_params = ("rules",)

    rules_: Sequence[nw.Expr]
    rule_descriptions_: list[str]
    """Human-readable forms of the mined rules; set when ``fit`` mined the reference."""

    def __init__(self, column: str, rules: Sequence[nw.Expr] | None = None) -> None:
        super().__init__(column=column)
        self.rules = rules

    def _fit_reference(self, frame: nw.DataFrame[Any]) -> dict[str, Any]:
        mined = self._mine(frame)
        if not mined:
            warnings.warn(
                f"{type(self).__name__}: no rule survived mining on the clean data; score() will return NaN.",
                stacklevel=2,
            )
        self.rule_descriptions_ = [description for description, _ in mined]
        return {"rules": [expr for _, expr in mined]}

    def _measure_units(self, frame: nw.DataFrame[Any]) -> nw.Series[Any]:
        data: list[float | None]
        if not self.rules_:
            data = [None] * len(frame)
        else:
            results = frame.select([rule.alias(f"rule_{i}") for i, rule in enumerate(self.rules_)])
            columns = [results[f"rule_{i}"].to_list() for i in range(len(self.rules_))]
            data = []
            for row in zip(*columns, strict=True):
                evaluable = [v for v in row if v is not None and v == v]
                data.append(None if not evaluable else float(all(bool(v) for v in evaluable)))
        return nw.new_series(self.column, data, nw.Float64, backend=frame.implementation)

    def _mine(self, frame: nw.DataFrame[Any]) -> list[tuple[str, nw.Expr]]:
        """One rule per column that determines this one on the clean data."""
        rules: list[tuple[str, nw.Expr]] = []
        for context in frame.columns:
            if context == self.column:
                continue
            pairs = frame.select(context, self.column).drop_nulls().unique()
            keys, values = pairs[context].to_list(), pairs[self.column].to_list()
            # A key with two different values determines nothing.
            if not keys or len(set(keys)) < len(keys):
                continue
            # when/then evaluates its branch over every row, so unseen keys reach replace_strict regardless.
            expected = nw.col(context).replace_strict(
                keys, values, default=None, return_dtype=frame.schema[self.column]
            )
            # Null cells and unseen keys are out of scope, spelled out because pandas reads a null
            # comparison as False where Polars keeps it null.
            scope = ~nw.col(self.column).is_null() & ~nw.col(context).is_null() & nw.col(context).is_in(keys)
            rule = nw.when(scope).then(nw.col(self.column) == expected).otherwise(nw.lit(None))
            rules.append((f"{context} -> {self.column}", rule))
        return rules

rule_descriptions_ instance-attribute 

rule_descriptions_

Human-readable forms of the mined rules; set when fit mined the reference.

_mine 

_mine(frame)

One rule per column that determines this one on the clean data.

Source code in src/dqmeasure/measures/semantic_consistency.py 
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
def _mine(self, frame: nw.DataFrame[Any]) -> list[tuple[str, nw.Expr]]:
    """One rule per column that determines this one on the clean data."""
    rules: list[tuple[str, nw.Expr]] = []
    for context in frame.columns:
        if context == self.column:
            continue
        pairs = frame.select(context, self.column).drop_nulls().unique()
        keys, values = pairs[context].to_list(), pairs[self.column].to_list()
        # A key with two different values determines nothing.
        if not keys or len(set(keys)) < len(keys):
            continue
        # when/then evaluates its branch over every row, so unseen keys reach replace_strict regardless.
        expected = nw.col(context).replace_strict(
            keys, values, default=None, return_dtype=frame.schema[self.column]
        )
        # Null cells and unseen keys are out of scope, spelled out because pandas reads a null
        # comparison as False where Polars keeps it null.
        scope = ~nw.col(self.column).is_null() & ~nw.col(context).is_null() & nw.col(context).is_in(keys)
        rule = nw.when(scope).then(nw.col(self.column) == expected).otherwise(nw.lit(None))
        rules.append((f"{context} -> {self.column}", rule))
    return rules

dqmeasure.measures.syntactic_accuracy.SyntacticDataAccuracy

Bases: PositionalMeasure

ISO/IEC 25024 Acc-I-1 "Syntactic data accuracy" (Acc-ML-1 in ISO/IEC 5259-2).

Column measure, tier 1, positional: unit = cell, subject = the column.

A value is syntactically accurate when it equals a member of the column's domain. "The same as one from an identified source of validated information" (ISO/IEC 25024, Table 1, note 1). The clean data acts as that source: fit learns the domain as the set of distinct non-null values it observes.

Checking values against the column's data type (the ISO/IEC 5259-2 reading of syntactic correctness) is deliberately not part of this measure: a typed dataframe already enforces its schema on load, and format conformance is a measure of its own (Con-I-2 data format consistency).

Parameters:

Name Type Description Default
column str

The column the measure applies to. Typically categorical-like, but any dtype works.

 required
domain Collection[Any] | None

The admissible values, or None to learn the domain from the clean data at fit.

 None
method Literal['observed']

How the domain is derived from the clean data. Currently "observed" (the set of distinct non-null values), which is the default.

 'observed'
Source code in src/dqmeasure/measures/syntactic_accuracy.py 
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
class SyntacticDataAccuracy(PositionalMeasure):
    """ISO/IEC 25024 `Acc-I-1` "Syntactic data accuracy" (`Acc-ML-1` in ISO/IEC 5259-2).

    Column measure, tier 1, positional: unit = cell, subject = the column.

    A value is syntactically accurate when it equals a member of the column's domain. "The same as one from an
    identified source of validated information" (ISO/IEC 25024, Table 1, note 1). The clean data acts as that
    source: [`fit`][dqmeasure.base.BaseMeasure.fit] learns the domain as the set of distinct non-null values it
    observes.

    Checking values against the column's *data type* (the ISO/IEC 5259-2 reading of syntactic correctness) is
    deliberately not part of this measure: a typed dataframe already enforces its schema on load, and format
    conformance is a measure of its own (`Con-I-2` data format consistency).

    Parameters
    ----------
    column:
        The column the measure applies to. Typically categorical-like, but any dtype works.
    domain:
        The admissible values, or ``None`` to learn the domain from the clean data at
        [`fit`][dqmeasure.base.BaseMeasure.fit].
    method:
        How the domain is derived from the clean data. Currently ``"observed"`` (the set of distinct non-null
        values), which is the default.
    """

    iso_5259_id = "Acc-ML-1"
    iso_25024_id = "Acc-I-1"
    reference_params = ("domain",)

    domain_: Collection[Any]

    def __init__(
        self,
        column: str,
        domain: Collection[Any] | None = None,
        method: Literal["observed"] = "observed",
    ) -> None:
        super().__init__(column=column)
        self.domain = domain
        self.method = method

    def _fit_reference(self, frame: nw.DataFrame[Any]) -> dict[str, Any]:
        if self.method != "observed":
            raise ValueError(f"Unsupported method: {self.method!r}")
        return {"domain": set(frame[self.column].drop_nulls().unique().to_list())}

    def _measure_units(self, frame: nw.DataFrame[Any]) -> nw.Series[Any]:
        # Per-cell condition: 1.0 if the value is a domain member, 0.0 if not, null if the value is missing (the
        # same null-preserving float encoding as DataAccuracyRange, and for the same backend-parity reasons).
        # Missing values are excluded from the measure: a null is incomplete, not syntactically inaccurate.
        expr = (
            nw.when(~nw.col(self.column).is_null())
            .then(nw.col(self.column).is_in(list(self.domain_)).cast(nw.Float64))
            .otherwise(nw.lit(None))
            .alias(self.column)
        )
        return frame.select(expr)[self.column]

dqmeasure.measures.timeliness.TimelinessOfDataItems

Bases: PositionalMeasure

ISO/IEC 5259-2 Tml-ML-1 "Timeliness of data items".

Column measure, tier 1, positional: unit = row (a data item), subject = the availability-timestamp column. The standard defines timeliness as the latency between the time a phenomenon occurs and the time the data recorded for it becomes available for use — as opposed to currentness (Cur-ML-1), the age of recorded data relative to its use. event_column names when each phenomenon occurred; it is context, not scope. A row is in scope iff its event time is set; the condition checks that the data became available within max_latency of the event. Data that never became available (null availability timestamp with an event time set) is a failure, not out of scope.

Data available before its event has a negative latency and is always timely. Time-zone-naive and -aware columns both work as long as the two columns are consistent with each other.

Tml-ML-1 coincides numerically with Cur-I-2 timeliness of update under a relabeling of the columns; the measures stay distinct in the role of the columns: Cur-I-2 counts items needing updating against a due time, Tml-ML-1 counts every data item against its event time.

Parameters:

Name Type Description Default
column str

The datetime column holding when each data item became available (was recorded).

 required
event_column str

The datetime column holding when the phenomenon each data item records occurred.

 required
max_latency timedelta | None

The allowed latency between event and availability, or None to learn it from the clean data at fit.

 None
method Literal['max']

How the latency requirement is derived from the clean data. Currently "max" (the worst latency observed in rows with both timestamps set), which is the default.

 'max'
Source code in src/dqmeasure/measures/timeliness.py 
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
class TimelinessOfDataItems(PositionalMeasure):
    """ISO/IEC 5259-2 `Tml-ML-1` "Timeliness of data items".

    Column measure, tier 1, positional: unit = row (a data item), subject = the
    availability-timestamp column. The standard defines timeliness as the latency
    between the time a phenomenon occurs and the time the data recorded for it becomes available for
    use — as opposed to currentness (`Cur-ML-1`), the age of recorded data relative to its use.
    ``event_column`` names when each phenomenon occurred; it is context, not scope. A row is in
    scope iff its event time is set; the condition checks that the data became available within
    ``max_latency`` of the event. Data that never became available (null availability timestamp
    with an event time set) is a failure, not out of scope.

    Data available before its event has a negative latency and is always timely. Time-zone-naive
    and -aware columns both work as long as the two columns are consistent with each other.

    `Tml-ML-1` coincides numerically with `Cur-I-2` timeliness of update under a relabeling of the
    columns; the measures stay distinct in the role of the columns: `Cur-I-2` counts items *needing
    updating* against a due time, `Tml-ML-1` counts every data item against its event time.

    Parameters
    ----------
    column:
        The datetime column holding when each data item became available (was recorded).
    event_column:
        The datetime column holding when the phenomenon each data item records occurred.
    max_latency:
        The allowed latency between event and availability, or ``None`` to learn it from the clean
        data at [`fit`][dqmeasure.base.BaseMeasure.fit].
    method:
        How the latency requirement is derived from the clean data. Currently ``"max"`` (the worst
        latency observed in rows with both timestamps set), which is the default.
    """

    iso_5259_id = "Tml-ML-1"
    iso_25024_id = None
    reference_params = ("max_latency",)

    max_latency_: timedelta

    def __init__(
        self,
        column: str,
        event_column: str,
        max_latency: timedelta | None = None,
        method: Literal["max"] = "max",
    ) -> None:
        super().__init__(column=column)
        self.event_column = event_column
        self.max_latency = max_latency
        self.method = method

    def _validate(self, frame: nw.DataFrame[Any]) -> None:
        _require_column(frame, self.column, temporal=True)
        _require_column(frame, self.event_column, temporal=True)

    def _fit_reference(self, frame: nw.DataFrame[Any]) -> dict[str, Any]:
        if self.method != "max":
            raise ValueError(f"Unsupported method: {self.method!r}")
        # Datetime subtraction null-propagates, so this restricts to rows with both timestamps set.
        latencies = frame.select((nw.col(self.column) - nw.col(self.event_column)).alias(self.column))[
            self.column
        ].drop_nulls()
        if len(latencies) == 0:
            raise ValueError(
                f"{type(self).__name__}: no rows with both {self.column!r} and {self.event_column!r} set in "
                "the clean data"
            )
        return {"max_latency": latencies.max()}

    def _measure_units(self, frame: nw.DataFrame[Any]) -> nw.Series[Any]:
        available, event = nw.col(self.column), nw.col(self.event_column)
        # Null-preserving float, as in DataAccuracyRange; the explicit guards are mandatory because on the
        # pandas backend a duration comparison against a missing timestamp yields False rather than null.
        timely = (
            nw.when(available.is_null())
            .then(nw.lit(0.0))
            .otherwise(((available - event) <= self.max_latency_).cast(nw.Float64))
        )
        expr = nw.when(~event.is_null()).then(timely).otherwise(nw.lit(None)).alias(self.column)
        return frame.select(expr)[self.column]

dqmeasure.measures.update_frequency.UpdateFrequency

Bases: PositionalMeasure

ISO/IEC 25024 Cur-I-1 "Update frequency".

Column measure, tier 1, positional: unit = update event, subject = the column. The frame is read as the event log of one update stream: rows are update events and the column holds their timestamps (100 stock prices that should update every minute are 100 rows of one stream). The condition reads the temporally preceding event as context and checks that the event arrived within max_interval of it, so A counts the events keeping up the required frequency and B the events with a predecessor. The earliest event (no predecessor) and null timestamps are out of scope.

Row order does not matter — events are ordered by timestamp internally, and predict returns the results in the input's row order. Duplicate timestamps have a gap of zero and conform; exactly one of the tied earliest events is the out-of-scope first event.

Parameters:

Name Type Description Default
column str

The datetime column holding the update events' timestamps.

 required
max_interval timedelta | None

The required maximum time between consecutive updates, or None to learn it from the clean data at fit.

 None
method Literal['max']

How the interval is derived from the clean data. Currently "max" (the largest inter-event gap observed in the clean stream), which is the default.

 'max'
Source code in src/dqmeasure/measures/update_frequency.py 
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
class UpdateFrequency(PositionalMeasure):
    """ISO/IEC 25024 `Cur-I-1` "Update frequency".

    Column measure, tier 1, positional: unit = update event, subject = the column. The
    frame is read as the event log of **one** update stream: rows are update events and the column holds
    their timestamps (100 stock prices that should update every minute are 100 rows of one stream). The
    condition reads the temporally preceding event as context and checks that the event arrived within
    ``max_interval`` of it, so ``A`` counts the events keeping up the required frequency and ``B`` the events
    with a predecessor. The earliest event (no predecessor) and null timestamps are out of scope.

    Row order does not matter — events are ordered by timestamp internally, and
    [`predict`][dqmeasure.base.PositionalMeasure.predict] returns the results in the input's row order.
    Duplicate timestamps have a gap of zero and conform; exactly one of the tied earliest events is the
    out-of-scope first event.

    Parameters
    ----------
    column:
        The datetime column holding the update events' timestamps.
    max_interval:
        The required maximum time between consecutive updates, or ``None`` to learn it from the clean data at
        [`fit`][dqmeasure.base.BaseMeasure.fit].
    method:
        How the interval is derived from the clean data. Currently ``"max"`` (the largest inter-event gap
        observed in the clean stream), which is the default.
    """

    iso_5259_id = None
    iso_25024_id = "Cur-I-1"
    reference_params = ("max_interval",)

    max_interval_: timedelta

    def __init__(
        self,
        column: str,
        max_interval: timedelta | None = None,
        method: Literal["max"] = "max",
    ) -> None:
        super().__init__(column=column)
        self.max_interval = max_interval
        self.method = method

    def _validate(self, frame: nw.DataFrame[Any]) -> None:
        _require_column(frame, self.column, temporal=True)

    def _fit_reference(self, frame: nw.DataFrame[Any]) -> dict[str, Any]:
        if self.method != "max":
            raise ValueError(f"Unsupported method: {self.method!r}")
        gaps = (
            frame.select(nw.col(self.column))
            .drop_nulls()
            .sort(self.column)
            .select(nw.col(self.column).diff().alias(self.column))[self.column]
            .drop_nulls()
        )
        if len(gaps) == 0:
            raise ValueError(
                f"{type(self).__name__}: needs at least two non-null timestamps in the clean data to learn max_interval"
            )
        return {"max_interval": gaps.max()}

    def _measure_units(self, frame: nw.DataFrame[Any]) -> nw.Series[Any]:
        # The frame is reduced to the measured column, so the helper names can only clash with it.
        index, gap = "_dqm_index", "_dqm_gap"
        while self.column in (index, gap):
            index, gap = index + "_", gap + "_"
        # Order by timestamp to take gaps, then restore the input's row order via the row index. The index
        # tiebreak makes the choice of "first event" among duplicate timestamps deterministic (neither
        # backend guarantees a stable sort), and nulls_last keeps null timestamps from corrupting real gaps:
        # their diffs, and the diff of whatever follows them, stay null.
        gaps = (
            frame.select(nw.col(self.column))
            .with_row_index(index)
            .sort([self.column, index], nulls_last=True)
            .with_columns(nw.col(self.column).diff().alias(gap))
            .sort(index)
        )
        # Null-preserving float, as in DataAccuracyRange; the guard is mandatory because on the pandas
        # backend a duration comparison against a missing gap yields False rather than null.
        expr = (
            nw.when(~nw.col(gap).is_null())
            .then((nw.col(gap) <= self.max_interval_).cast(nw.Float64))
            .otherwise(nw.lit(None))
            .alias(self.column)
        )
        return gaps.select(expr)[self.column]

dqmeasure.measures.update_timeliness.TimelinessOfUpdate

Bases: PositionalMeasure

ISO/IEC 25024 Cur-I-2 "Timeliness of update".

Column measure, tier 1, positional: unit = row (a data item needing updating), subject = the update-timestamp column. due_column names when each update was due or requested; it is context, not scope (scope and context are independent). A row is in scope iff its due time is set, so B counts the items needing updating; the condition checks that the update landed within sla of the due time. A needed update that never happened (null update timestamp with a due time set) is a failure, not out of scope.

An update before its due time has a negative delay and is always timely. Time-zone-naive and -aware columns both work as long as the two columns are consistent with each other.

Parameters:

Name Type Description Default
column str

The datetime column holding when each item was actually updated.

 required
due_column str

The datetime column holding when each item's update was due; null means no update was needed.

 required
sla timedelta | None

The allowed delay between due time and update, or None to learn it from the clean data at fit.

 None
method Literal['max']

How the SLA is derived from the clean data. Currently "max" (the worst delay observed in rows with both timestamps set), which is the default.

 'max'
Source code in src/dqmeasure/measures/update_timeliness.py 
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
class TimelinessOfUpdate(PositionalMeasure):
    """ISO/IEC 25024 `Cur-I-2` "Timeliness of update".

    Column measure, tier 1, positional: unit = row (a data item needing updating), subject = the
    update-timestamp column. ``due_column`` names when each update was due or requested; it
    is context, not scope (scope and context are independent). A row is in scope iff its due time is set, so
    ``B`` counts the items needing updating; the condition checks that the update landed within ``sla`` of
    the due time. A needed update that never happened (null update timestamp with a due time set) is a
    failure, not out of scope.

    An update before its due time has a negative delay and is always timely. Time-zone-naive and -aware
    columns both work as long as the two columns are consistent with each other.

    Parameters
    ----------
    column:
        The datetime column holding when each item was actually updated.
    due_column:
        The datetime column holding when each item's update was due; null means no update was needed.
    sla:
        The allowed delay between due time and update, or ``None`` to learn it from the clean data at
        [`fit`][dqmeasure.base.BaseMeasure.fit].
    method:
        How the SLA is derived from the clean data. Currently ``"max"`` (the worst delay observed in rows
        with both timestamps set), which is the default.
    """

    iso_5259_id = None
    iso_25024_id = "Cur-I-2"
    reference_params = ("sla",)

    sla_: timedelta

    def __init__(
        self,
        column: str,
        due_column: str,
        sla: timedelta | None = None,
        method: Literal["max"] = "max",
    ) -> None:
        super().__init__(column=column)
        self.due_column = due_column
        self.sla = sla
        self.method = method

    def _validate(self, frame: nw.DataFrame[Any]) -> None:
        _require_column(frame, self.column, temporal=True)
        _require_column(frame, self.due_column, temporal=True)

    def _fit_reference(self, frame: nw.DataFrame[Any]) -> dict[str, Any]:
        if self.method != "max":
            raise ValueError(f"Unsupported method: {self.method!r}")
        # Datetime subtraction null-propagates, so this restricts to rows with both timestamps set.
        delays = frame.select((nw.col(self.column) - nw.col(self.due_column)).alias(self.column))[
            self.column
        ].drop_nulls()
        if len(delays) == 0:
            raise ValueError(
                f"{type(self).__name__}: no rows with both {self.column!r} and {self.due_column!r} set in "
                "the clean data"
            )
        return {"sla": delays.max()}

    def _measure_units(self, frame: nw.DataFrame[Any]) -> nw.Series[Any]:
        updated, due = nw.col(self.column), nw.col(self.due_column)
        # Null-preserving float, as in DataAccuracyRange; the explicit guards are mandatory because on the
        # pandas backend a duration comparison against a missing timestamp yields False rather than null.
        timely = nw.when(updated.is_null()).then(nw.lit(0.0)).otherwise(((updated - due) <= self.sla_).cast(nw.Float64))
        expr = nw.when(~due.is_null()).then(timely).otherwise(nw.lit(None)).alias(self.column)
        return frame.select(expr)[self.column]

dqmeasure.measures.value_completeness.ValueCompleteness

Bases: PositionalMeasure

ISO/IEC 5259-2 Com-ML-1 "Value completeness".

Table measure, tier 1, positional: unit = record (row), subject = the whole table.

The ratio of non-null cells over all cells of the table. We implement this by calculating the fraction of non-null cells per row, then averaging them over the entire table. That per-record fraction is itself another measure: predict reports ISO/IEC 25024 Com-I-1 "Record completeness" for each record.

There is no reference to learn, so the measure works without fit.

The table-wide value is also the mean of the per-column Com-ML-3 scores, mean(FeatureCompleteness(c).score(df) for c in df.columns), since every column contributes the same B.

Source code in src/dqmeasure/measures/value_completeness.py 
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
class ValueCompleteness(PositionalMeasure):
    """ISO/IEC 5259-2 `Com-ML-1` "Value completeness".

    Table measure, tier 1, positional: unit = record (row), subject = the whole table.

    The ratio of non-null cells over all cells of the table. We implement this by calculating the fraction of
    non-null cells per row, then averaging them over the entire table. That per-record fraction is itself another
    measure: [`predict`][dqmeasure.base.PositionalMeasure.predict] reports ISO/IEC 25024 `Com-I-1`
    "Record completeness" for each record.

    There is no reference to learn, so the measure works without [`fit`][dqmeasure.base.BaseMeasure.fit].

    The table-wide value is also the mean of the per-column `Com-ML-3` scores,
    ``mean(FeatureCompleteness(c).score(df) for c in df.columns)``, since every column contributes the same
    ``B``.
    """

    iso_5259_id = "Com-ML-1"
    iso_25024_id = None
    scope = "table"

    def __init__(self) -> None:
        pass

    def _measure_units(self, frame: nw.DataFrame[Any]) -> nw.Series[Any]:
        # Per-row: the fraction of non-null cells in the row.
        expr = nw.mean_horizontal(*((~nw.col(c).is_null()).cast(nw.Float64) for c in frame.columns))
        return frame.select(expr.alias("record"))["record"]

    def _score(self, frame: nw.DataFrame[Any]) -> float:
        # Don't reuse _measure_units()' outputs, because rouding errors may compound.
        # Instead, calculate a and b directly.
        b = len(frame) * len(frame.columns)
        if b == 0:
            return float("nan")
        a = b - sum(frame[c].null_count() for c in frame.columns)
        return a / b

dqmeasure.measures.value_distribution.DataValueDistribution

Bases: BaseMeasure

ISO/IEC 5259-2 Con-ML-2 "Distribution of data values".

Column measure, tier 2 (statistic): the QMEs are the reference distribution learned from clean data and the observed distribution of the measured frame, and X is the distance between them — no per-unit value exists, so the measure is score()-only. The standard's X is the distance itself, so we report 1 - X to keep every measure higher-is-better: X = 1 means the distributions agree, X = 0 that they are disjoint.

The standard delegates the choice of distribution measure ("determined according to the ML task"). We resolve it with a single principle instead of a catalogue of tests: X is the worst-case disagreement in probability over the column type's natural events, sup |P(A) - Q(A)|.

  • Ordered columns (numeric, dates, datetimes): the natural events are the half-lines (-∞, x], and the sup over them is the two-sample Kolmogorov-Smirnov statistic sup |F - G| of the two empirical CDFs.
  • Unordered columns (string, categorical, enum, boolean): with no order to exploit, the natural events are all subsets of values, and the sup evaluates to the total variation distance ½ Σ |p - q|. Values unseen in the reference contribute their full observed mass.

Both instantiations are parameter-free — no bins, no kernel, no significance level — which is why there is no method parameter. Nulls are dropped on both sides: missingness is completeness' business, not distribution drift.

Parameters:

Name Type Description Default
column str

The column the measure applies to (numeric, date, datetime, string, categorical, enum, or boolean).

 required
expected Mapping[Any, float] | Sequence[Any] | None

The reference distribution: a {value: proportion} mapping for an unordered column, or a reference sample (sequence of values whose empirical CDF is the reference) for an ordered column. None (default) learns it from the clean data at fit.

 None
Source code in src/dqmeasure/measures/value_distribution.py 
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
class DataValueDistribution(BaseMeasure):
    """ISO/IEC 5259-2 `Con-ML-2` "Distribution of data values".

    Column measure, tier 2 (statistic): the QMEs are the reference distribution learned from clean data and
    the observed distribution of the measured frame, and ``X`` is the distance between them — no per-unit
    value exists, so the measure is ``score()``-only. The standard's ``X`` is the distance itself, so we
    report ``1 - X`` to keep every measure higher-is-better: ``X = 1`` means the distributions agree,
    ``X = 0`` that they are disjoint.

    The standard delegates the choice of distribution measure ("determined according to the ML task"). We
    resolve it with a single principle instead of a catalogue of tests: ``X`` is the worst-case disagreement
    in probability over the column type's natural events, ``sup |P(A) - Q(A)|``.

    * Ordered columns (numeric, dates, datetimes): the natural events are the half-lines ``(-∞, x]``, and the
      sup over them is the two-sample **Kolmogorov-Smirnov statistic** ``sup |F - G|`` of the two empirical
      CDFs.
    * Unordered columns (string, categorical, enum, boolean): with no order to exploit, the natural events
      are all subsets of values, and the sup evaluates to the **total variation distance**
      ``½ Σ |p - q|``. Values unseen in the reference contribute their full observed mass.

    Both instantiations are parameter-free — no bins, no kernel, no significance level — which is why there
    is no ``method`` parameter. Nulls are dropped on both sides: missingness is completeness' business, not
    distribution drift.

    Parameters
    ----------
    column:
        The column the measure applies to (numeric, date, datetime, string, categorical, enum, or boolean).
    expected:
        The reference distribution: a ``{value: proportion}`` mapping for an unordered column, or a reference
        sample (sequence of values whose empirical CDF is the reference) for an ordered column. ``None``
        (default) learns it from the clean data at [`fit`][dqmeasure.base.BaseMeasure.fit].
    """

    iso_5259_id = "Con-ML-2"
    iso_25024_id = None
    reference_params = ("expected",)

    expected_: Mapping[Any, float] | Sequence[Any]

    def __init__(self, column: str, expected: Mapping[Any, float] | Sequence[Any] | None = None) -> None:
        super().__init__(column=column)
        self.expected = expected

    def _validate(self, frame: nw.DataFrame[Any]) -> None:
        super()._validate(frame)
        dtype = frame.schema[self.column]
        if not _is_ordered(dtype) and not _is_unordered(dtype):
            raise ValueError(
                f"Column {self.column!r} has dtype {dtype}; this measure needs a numeric, date, datetime, "
                "string, categorical, enum, or boolean column"
            )

    def _fit_reference(self, frame: nw.DataFrame[Any]) -> dict[str, Any]:
        col = frame[self.column].drop_nulls()
        if _is_ordered(frame.schema[self.column]):
            return {"expected": sorted(_drop_nan(col.to_list()))}
        n = len(col)
        return {"expected": {value: count / n for value, count in col.value_counts().iter_rows()} if n else {}}

    def _score(self, frame: nw.DataFrame[Any]) -> float:
        observed = _drop_nan(frame[self.column].drop_nulls().to_list())
        if _is_ordered(frame.schema[self.column]):
            if isinstance(self.expected_, Mapping):
                raise ValueError(
                    f"Column {self.column!r} is ordered, so `expected` must be a reference sample "
                    "(sequence of values), not a mapping of proportions"
                )
            reference = sorted(_drop_nan(list(self.expected_)))
            if not reference or not observed:
                return float("nan")
            return 1.0 - _ks_statistic(reference, sorted(observed))
        if not isinstance(self.expected_, Mapping):
            raise ValueError(
                f"Column {self.column!r} is unordered, so `expected` must be a {{value: proportion}} "
                "mapping, not a sequence"
            )
        total = float(sum(self.expected_.values()))
        if not self.expected_ or total <= 0 or not observed:
            return float("nan")
        proportions = {value: count / len(observed) for value, count in _value_counts(observed).items()}
        values = set(self.expected_) | set(proportions)
        return 1.0 - 0.5 * sum(abs(proportions.get(v, 0.0) - self.expected_.get(v, 0.0) / total) for v in values)

dqmeasure.measures.value_occurrence.ValueOccurrenceCompleteness

Bases: BaseMeasure

ISO/IEC 5259-2 Com-ML-2 "Value occurrence completeness".

Column measure, tier 1, non-positional: the unit is the expected occurrence of a domain value, which cannot be attached to a position in the frame, so the measure is score()-only. Our interpretation of choices the standard leaves open:

  • We store occurrence proportions rather than raw counts, and expect that the proportions are the same on the measured frame.
  • Counted occurrences are capped at the number of expectation per value. This way, over-represented values cannot compensate for missing ones or push X past 1.

Values outside the observed domain contribute to neither A nor B; null values are not part of the domain.

Parameters:

Name Type Description Default
column str

The column the measure applies to. Typically categorical-like, but any dtype works.

 required
expected dict[Any, float] | None

Expected occurrence proportions as a {value: proportion} dict, or None to learn them from the clean data at fit.

 None
Source code in src/dqmeasure/measures/value_occurrence.py 
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
class ValueOccurrenceCompleteness(BaseMeasure):
    """ISO/IEC 5259-2 `Com-ML-2` "Value occurrence completeness".

    Column measure, tier 1, non-positional: the unit is the expected occurrence of a domain value, which cannot
    be attached to a position in the frame, so the measure is ``score()``-only. Our interpretation of choices
    the standard leaves open:

    * We store occurrence proportions rather than raw counts, and expect that the proportions are the same on
      the measured frame.
    * Counted occurrences are capped at the number of expectation per value. This way, over-represented values
      cannot compensate for missing ones or push ``X`` past 1.

    Values outside the observed domain contribute to neither ``A`` nor ``B``; null values are not part of the
    domain.

    Parameters
    ----------
    column:
        The column the measure applies to. Typically categorical-like, but any dtype works.
    expected:
        Expected occurrence proportions as a ``{value: proportion}`` dict, or ``None`` to learn them from the
        clean data at [`fit`][dqmeasure.base.BaseMeasure.fit].
    """

    iso_5259_id = "Com-ML-2"
    iso_25024_id = None
    reference_params = ("expected",)

    expected_: dict[Any, float]

    def __init__(self, column: str, expected: dict[Any, float] | None = None) -> None:
        super().__init__(column=column)
        self.expected = expected

    def _fit_reference(self, frame: nw.DataFrame[Any]) -> dict[str, Any]:
        # The domain with each value's occurrence proportion, relative to the clean frame's total row count.
        # Nulls in the clean data lower the expected occupancy and aren't part of the domain.
        n = len(frame)
        counts = frame[self.column].drop_nulls().value_counts()
        return {"expected": {value: count / n for value, count in counts.iter_rows()} if n else {}}

    def _score(self, frame: nw.DataFrame[Any]) -> float:
        n = len(frame)
        observed = dict(frame[self.column].drop_nulls().value_counts().iter_rows()) if n else {}
        a = 0.0
        b = 0.0
        for value, proportion in self.expected_.items():
            expected = proportion * n
            a += min(float(observed.get(value, 0)), expected)
            b += expected
        return a / b if b else float("nan")

Base classes

dqmeasure.base.BaseMeasure

Data-quality measure base class.

Every measure has one of two scopes, exposed as the scope class attribute: a column measure is constructed for exactly one column, named by the column constructor parameter. And a table measure applies to all columns of the frame, not requiring a column parameter. Either way score yields exactly one quality measure value X.

A table-scoped subclass sets scope = "table" and defines its own __init__ without a column parameter.

Subclasses set iso_5259_id and iso_25024_id (the measure's IDs in the two standards), and implement two hooks:

  • _fit_reference: learn the reference from clean data.
  • _score: compute the quality measure value X on a (dirty) dataframe.

Measures whose units are positions in the dataframe should inherit from PositionalMeasure instead, which adds predict() and derives _score from it.

Parameters:

Name Type Description Default
column str

The column the measure applies to.

 required
Source code in src/dqmeasure/base.py 
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
class BaseMeasure:
    """Data-quality measure base class.

    Every measure has one of two scopes, exposed as the ``scope`` class attribute: a **column** measure
    is constructed for exactly one column, named by the ``column`` constructor parameter. And a **table**
    measure applies to all columns of the frame, not requiring a ``column`` parameter. Either way
    [`score`][dqmeasure.base.BaseMeasure.score] yields exactly one quality measure value ``X``.

    A table-scoped subclass sets ``scope = "table"`` and defines its own ``__init__`` without a ``column``
    parameter.

    Subclasses set ``iso_5259_id`` and ``iso_25024_id`` (the measure's IDs in the two standards),
    and implement two hooks:

    * [`_fit_reference`][dqmeasure.base.BaseMeasure._fit_reference]: learn the reference from clean data.
    * [`_score`][dqmeasure.base.BaseMeasure._score]: compute the quality measure value ``X`` on a (dirty)
      dataframe.

    Measures whose units are positions in the dataframe should inherit from
    [`PositionalMeasure`][dqmeasure.base.PositionalMeasure] instead, which adds ``predict()`` and derives
    ``_score`` from it.

    Parameters
    ----------
    column:
        The column the measure applies to.
    """

    iso_5259_id: ClassVar[str | None]
    """The measure's ID in ISO/IEC 5259-2, or ``None`` if that standard has no counterpart."""

    iso_25024_id: ClassVar[str | None]
    """The measure's ID in ISO/IEC 25024, or ``None`` if that standard has no counterpart."""

    scope: ClassVar[Scope] = "column"
    """The measure's subject: one named column, or the whole table. Fixed by the ISO definition."""

    reference_params: ClassVar[tuple[str, ...]] = ()
    """Names of the constructor parameters that hold the measure's reference.

    Each may be specified in the constructor or left as ``None`` to be learned at
    [`fit`][dqmeasure.base.BaseMeasure.fit]. After resolution each appears as a fitted ``<name>_`` attribute.
    """

    def __init__(self, column: str) -> None:
        if self.scope != "column":
            raise TypeError(
                f"{type(self).__name__} is table-scoped and takes no column; "
                "table-scoped measures define their own __init__"
            )
        self.column = column

    def fit(self, X: IntoDataFrame) -> Self:
        """Learn the reference from a clean (training) dataframe.

        Sets one fitted ``<name>_`` attribute per reference parameter and returns ``self``. Parameters specified
        in the constructor are kept; only the rest is estimated from ``X``.
        """
        frame = nw.from_native(X, eager_only=True)
        self._validate(frame)
        self._resolve(frame)
        return self

    def score(self, X: IntoDataFrame) -> float:
        """Compute the quality measure value ``X`` for a (dirty) dataframe.

        Returns one value for the measure's subject (its column, or the whole table). Every measure is
        oriented so that **higher is better**. Where the standard defines ``X`` in the opposite direction,
        the measure reports ``1 - X``. When the subject has no units in scope (``B = 0``), the value is ``NaN``.
        """
        self._check_is_resolved()
        frame = nw.from_native(X, eager_only=True)
        self._validate(frame)
        return self._score(frame)

    # hooks for subclasses

    def _validate(self, frame: nw.DataFrame[Any]) -> None:
        """Check that ``frame`` supports this measure. Default: the column exists (column scope) or the frame
        has at least one column (table scope); subclasses may add dtype checks."""
        if self.scope == "column":
            _require_column(frame, self.column)
        elif not frame.columns:
            raise ValueError(f"{type(self).__name__} is table-scoped and needs a frame with at least one column")

    def _fit_reference(self, frame: nw.DataFrame[Any]) -> dict[str, Any]:
        """Estimate every reference parameter from clean data.

        Returns ``{param_name: value}``. The default has nothing to learn; measures with a non-empty
        ``reference_params`` override it.
        """
        return {}

    def _score(self, frame: nw.DataFrame[Any]) -> float:
        """Compute ``X`` for a (dirty) dataframe. Must be overridden."""
        raise NotImplementedError

    # helpers

    def _resolve(self, frame: nw.DataFrame[Any] | None) -> None:
        """Resolve the ``<name>_`` attributes from spec plus estimate.

        ``frame`` is the clean data when learning, or ``None`` when the reference is fully specified and no
        estimation is needed.
        """
        missing = [name for name in self.reference_params if getattr(self, name) is None]
        estimated: dict[str, Any] = {}
        if missing:
            if frame is None:
                raise NotResolvedError(
                    f"{type(self).__name__}: the reference is not fully specified ({', '.join(missing)}); "
                    "either give every parameter in the constructor or call fit() on clean data."
                )
            estimated = self._fit_reference(frame)
        for name in self.reference_params:
            value = getattr(self, name)
            setattr(self, f"{name}_", estimated[name] if value is None else value)
        self._resolved = True

    def _check_is_resolved(self) -> None:
        if not getattr(self, "_resolved", False):
            self._resolve(None)

    # minimal sklearn-style param protocol

    def get_params(self) -> dict[str, Any]:
        """Return the constructor parameters, introspected from ``__init__``."""
        params: dict[str, Any] = {}
        for name, param in inspect.signature(type(self).__init__).parameters.items():
            if name == "self" or param.kind in (param.VAR_POSITIONAL, param.VAR_KEYWORD):
                continue
            params[name] = getattr(self, name)
        return params

    def set_params(self, **params: Any) -> Self:
        for key, value in params.items():
            setattr(self, key, value)
        return self

    def __repr__(self) -> str:
        params = ", ".join(f"{k}={v!r}" for k, v in self.get_params().items())
        return f"{type(self).__name__}({params})"

iso_5259_id class-attribute 

iso_5259_id

The measure's ID in ISO/IEC 5259-2, or None if that standard has no counterpart.

iso_25024_id class-attribute 

iso_25024_id

The measure's ID in ISO/IEC 25024, or None if that standard has no counterpart.

scope class-attribute 

scope = 'column'

The measure's subject: one named column, or the whole table. Fixed by the ISO definition.

reference_params class-attribute 

reference_params = ()

Names of the constructor parameters that hold the measure's reference.

Each may be specified in the constructor or left as None to be learned at fit. After resolution each appears as a fitted <name>_ attribute.

fit 

fit(X)

Learn the reference from a clean (training) dataframe.

Sets one fitted <name>_ attribute per reference parameter and returns self. Parameters specified in the constructor are kept; only the rest is estimated from X.

Source code in src/dqmeasure/base.py 
83
84
85
86
87
88
89
90
91
92
def fit(self, X: IntoDataFrame) -> Self:
    """Learn the reference from a clean (training) dataframe.

    Sets one fitted ``<name>_`` attribute per reference parameter and returns ``self``. Parameters specified
    in the constructor are kept; only the rest is estimated from ``X``.
    """
    frame = nw.from_native(X, eager_only=True)
    self._validate(frame)
    self._resolve(frame)
    return self

score 

score(X)

Compute the quality measure value X for a (dirty) dataframe.

Returns one value for the measure's subject (its column, or the whole table). Every measure is oriented so that higher is better. Where the standard defines X in the opposite direction, the measure reports 1 - X. When the subject has no units in scope (B = 0), the value is NaN.

Source code in src/dqmeasure/base.py 
 94
 95
 96
 97
 98
 99
100
101
102
103
104
def score(self, X: IntoDataFrame) -> float:
    """Compute the quality measure value ``X`` for a (dirty) dataframe.

    Returns one value for the measure's subject (its column, or the whole table). Every measure is
    oriented so that **higher is better**. Where the standard defines ``X`` in the opposite direction,
    the measure reports ``1 - X``. When the subject has no units in scope (``B = 0``), the value is ``NaN``.
    """
    self._check_is_resolved()
    frame = nw.from_native(X, eager_only=True)
    self._validate(frame)
    return self._score(frame)

_validate 

_validate(frame)

Check that frame supports this measure. Default: the column exists (column scope) or the frame has at least one column (table scope); subclasses may add dtype checks.

Source code in src/dqmeasure/base.py 
108
109
110
111
112
113
114
def _validate(self, frame: nw.DataFrame[Any]) -> None:
    """Check that ``frame`` supports this measure. Default: the column exists (column scope) or the frame
    has at least one column (table scope); subclasses may add dtype checks."""
    if self.scope == "column":
        _require_column(frame, self.column)
    elif not frame.columns:
        raise ValueError(f"{type(self).__name__} is table-scoped and needs a frame with at least one column")

_fit_reference 

_fit_reference(frame)

Estimate every reference parameter from clean data.

Returns {param_name: value}. The default has nothing to learn; measures with a non-empty reference_params override it.

Source code in src/dqmeasure/base.py 
116
117
118
119
120
121
122
def _fit_reference(self, frame: nw.DataFrame[Any]) -> dict[str, Any]:
    """Estimate every reference parameter from clean data.

    Returns ``{param_name: value}``. The default has nothing to learn; measures with a non-empty
    ``reference_params`` override it.
    """
    return {}

_score 

_score(frame)

Compute X for a (dirty) dataframe. Must be overridden.

Source code in src/dqmeasure/base.py 
124
125
126
def _score(self, frame: nw.DataFrame[Any]) -> float:
    """Compute ``X`` for a (dirty) dataframe. Must be overridden."""
    raise NotImplementedError

_resolve 

_resolve(frame)

Resolve the <name>_ attributes from spec plus estimate.

frame is the clean data when learning, or None when the reference is fully specified and no estimation is needed.

Source code in src/dqmeasure/base.py 
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
def _resolve(self, frame: nw.DataFrame[Any] | None) -> None:
    """Resolve the ``<name>_`` attributes from spec plus estimate.

    ``frame`` is the clean data when learning, or ``None`` when the reference is fully specified and no
    estimation is needed.
    """
    missing = [name for name in self.reference_params if getattr(self, name) is None]
    estimated: dict[str, Any] = {}
    if missing:
        if frame is None:
            raise NotResolvedError(
                f"{type(self).__name__}: the reference is not fully specified ({', '.join(missing)}); "
                "either give every parameter in the constructor or call fit() on clean data."
            )
        estimated = self._fit_reference(frame)
    for name in self.reference_params:
        value = getattr(self, name)
        setattr(self, f"{name}_", estimated[name] if value is None else value)
    self._resolved = True

get_params 

get_params()

Return the constructor parameters, introspected from __init__.

Source code in src/dqmeasure/base.py 
156
157
158
159
160
161
162
163
def get_params(self) -> dict[str, Any]:
    """Return the constructor parameters, introspected from ``__init__``."""
    params: dict[str, Any] = {}
    for name, param in inspect.signature(type(self).__init__).parameters.items():
        if name == "self" or param.kind in (param.VAR_POSITIONAL, param.VAR_KEYWORD):
            continue
        params[name] = getattr(self, name)
    return params

dqmeasure.base.PositionalMeasure

Bases: BaseMeasure

Base class for tier-1 measures with positional units (cells or rows).

A positional unit is a position in the dataframe you can point at and attach a score to. Such measures gain predict, and score() is its aggregation. Subclasses implement _measure_units instead of _score.

Measures with non-positional units and tier-2 statistic measures have no per-unit output. They derive from BaseMeasure directly and are score()-only.

Source code in src/dqmeasure/base.py 
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
class PositionalMeasure(BaseMeasure):
    """Base class for tier-1 measures with positional units (cells or rows).

    A positional unit is a position in the dataframe you can point at and attach a score to. Such measures gain
    [`predict`][dqmeasure.base.PositionalMeasure.predict], and ``score()`` is its aggregation. Subclasses
    implement [`_measure_units`][dqmeasure.base.PositionalMeasure._measure_units] instead of ``_score``.

    Measures with non-positional units and tier-2 statistic measures have no per-unit output. They derive from
    [`BaseMeasure`][dqmeasure.base.BaseMeasure] directly and are ``score()``-only.
    """

    def predict(self, X: IntoDataFrame) -> IntoSeries:
        """Evaluate the condition per unit on a (dirty) dataframe.

        Returns a series with one entry per input row, holding the per-unit condition result
        ``condition(u) ∈ [0, 1]`` as a null-preserving float (null = unit out of scope). The return type matches
        the backend of ``X``.
        """
        self._check_is_resolved()
        frame = nw.from_native(X, eager_only=True)
        self._validate(frame)
        units = self._measure_units(frame)
        # _measure_units builds a fresh series, so narwhals' input type parameter is erased; cast back to the
        # caller's backend type that predict promises to return.
        return cast(IntoSeries, units.to_native())

    # -- hooks for subclasses ---------------------------------------------------------

    def _measure_units(self, frame: nw.DataFrame[Any]) -> nw.Series[Any]:
        """Return the per-unit condition results as a series. Must be overridden."""
        raise NotImplementedError

    def _score(self, frame: nw.DataFrame[Any]) -> float:
        """Aggregate the per-unit condition results to ``X``.

        Default: the mean ignoring nulls, i.e. the ISO ratio ``A / B`` where ``A`` sums the condition results
        and ``B`` counts the units in scope (non-null entries).
        """
        units = self._measure_units(frame).cast(nw.Float64)
        total = units.count()  # non-null count
        conforming = units.sum()
        return float(conforming / total) if total else float("nan")

predict 

predict(X)

Evaluate the condition per unit on a (dirty) dataframe.

Returns a series with one entry per input row, holding the per-unit condition result condition(u) ∈ [0, 1] as a null-preserving float (null = unit out of scope). The return type matches the backend of X.

Source code in src/dqmeasure/base.py 
186
187
188
189
190
191
192
193
194
195
196
197
198
199
def predict(self, X: IntoDataFrame) -> IntoSeries:
    """Evaluate the condition per unit on a (dirty) dataframe.

    Returns a series with one entry per input row, holding the per-unit condition result
    ``condition(u) ∈ [0, 1]`` as a null-preserving float (null = unit out of scope). The return type matches
    the backend of ``X``.
    """
    self._check_is_resolved()
    frame = nw.from_native(X, eager_only=True)
    self._validate(frame)
    units = self._measure_units(frame)
    # _measure_units builds a fresh series, so narwhals' input type parameter is erased; cast back to the
    # caller's backend type that predict promises to return.
    return cast(IntoSeries, units.to_native())

_measure_units 

_measure_units(frame)

Return the per-unit condition results as a series. Must be overridden.

Source code in src/dqmeasure/base.py 
203
204
205
def _measure_units(self, frame: nw.DataFrame[Any]) -> nw.Series[Any]:
    """Return the per-unit condition results as a series. Must be overridden."""
    raise NotImplementedError

_score 

_score(frame)

Aggregate the per-unit condition results to X.

Default: the mean ignoring nulls, i.e. the ISO ratio A / B where A sums the condition results and B counts the units in scope (non-null entries).

Source code in src/dqmeasure/base.py 
207
208
209
210
211
212
213
214
215
216
def _score(self, frame: nw.DataFrame[Any]) -> float:
    """Aggregate the per-unit condition results to ``X``.

    Default: the mean ignoring nulls, i.e. the ISO ratio ``A / B`` where ``A`` sums the condition results
    and ``B`` counts the units in scope (non-null entries).
    """
    units = self._measure_units(frame).cast(nw.Float64)
    total = units.count()  # non-null count
    conforming = units.sum()
    return float(conforming / total) if total else float("nan")

dqmeasure.base.NotResolvedError

Bases: RuntimeError

Raised when a measure is used before its reference is resolved (specified in the constructor or learned via fit).

Source code in src/dqmeasure/base.py 
13
14
15
class NotResolvedError(RuntimeError):
    """Raised when a measure is used before its reference is resolved (specified in the constructor or learned via
    [`fit`][dqmeasure.base.BaseMeasure.fit])."""