diff --git a/docs/source/conf.py b/docs/source/conf.py index 9a05c43b9..972bef662 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -13,7 +13,7 @@ project = 'semantic-link-labs' copyright = '2026, Microsoft and community' author = 'Microsoft and community' -release = '0.14.3' +release = '0.15.0' # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/pyproject.toml b/pyproject.toml index 3f6c77670..d1d1b4772 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name="semantic-link-labs" authors = [ { name = "Microsoft Corporation" }, ] -version="0.14.3" +version="0.15.0" description="Semantic Link Labs for Microsoft Fabric" readme="README.md" requires-python=">=3.10,<3.13" diff --git a/src/sempy_labs/_a_lib_info.py b/src/sempy_labs/_a_lib_info.py index 677768abd..410321514 100644 --- a/src/sempy_labs/_a_lib_info.py +++ b/src/sempy_labs/_a_lib_info.py @@ -4,7 +4,7 @@ from pathlib import Path lib_name = "semanticlinklabs" -lib_version = "0.14.3" +lib_version = "0.15.0" NUGET_BASE_URL = "https://www.nuget.org/api/v2/package" current_dir = Path(__file__).parent diff --git a/src/sempy_labs/_generate_semantic_model.py b/src/sempy_labs/_generate_semantic_model.py index de0d60baa..bbd190c73 100644 --- a/src/sempy_labs/_generate_semantic_model.py +++ b/src/sempy_labs/_generate_semantic_model.py @@ -404,30 +404,30 @@ def deploy_semantic_model( ) # Remove mini model annotations from the master model if they exist (cleanup) - #ann_to_remove = [ + # ann_to_remove = [ # a.Name # for a in tom.model.Annotations # if a.Name.startswith(icons.prefix_mini) - #] - #for ann in ann_to_remove: + # ] + # for ann in ann_to_remove: # tom.remove_annotation(object=tom.model, name=ann) # Set annotations to the master model - #if filters is not None or perspective is not None: + # if filters is not None or perspective is not None: # with connect_semantic_model( # dataset=source_dataset_id, workspace=source_workspace_id, readonly=False # ) as tom: # ann_name = f"{icons.prefix_mini}_{perspective}" - # --- Get existing annotation safely --- + # --- Get existing annotation safely --- # try: # ann_value = tom.get_annotation_value(object=tom.model, name=ann_name) # ann_list = ast.literal_eval(ann_value) if ann_value else [] # except Exception: # ann_list = [] - # --- Build lookup (faster than loop) --- + # --- Build lookup (faster than loop) --- # index = {a.get("datasetId"): a for a in ann_list} # if target_dataset_id in index: @@ -455,7 +455,7 @@ def deploy_semantic_model( # } # ) - # --- Save once --- + # --- Save once --- # tom.set_annotation(object=tom.model, name=ann_name, value=str(ann_list)) if refresh_target_dataset: refresh_semantic_model(dataset=target_dataset_id, workspace=target_workspace_id) diff --git a/src/sempy_labs/_model_bpa_rules.py b/src/sempy_labs/_model_bpa_rules.py index 1eaaa70cc..aad4b2974 100644 --- a/src/sempy_labs/_model_bpa_rules.py +++ b/src/sempy_labs/_model_bpa_rules.py @@ -3,6 +3,12 @@ import re from typing import Optional from sempy._utils._log import log +from sempy_labs.dax._analysis import ( + find_fully_qualified_measures, + find_non_numeric_aggregations, + find_unqualified_columns, + uses_function, +) @log @@ -434,20 +440,26 @@ def model_bpa_rules( "Measure", "Warning", "Avoid using the IFERROR function", - lambda obj, tom: re.search( - r"iferror\s*\(", obj.Expression, flags=re.IGNORECASE - ), + lambda obj, tom: uses_function(obj.Expression, "IFERROR"), "Avoid using the IFERROR function as it may cause performance degradation. If you are concerned about a divide-by-zero error, use the DIVIDE function as it naturally resolves such errors as blank (or you can customize what should be shown in case of such an error).", "https://www.elegantbi.com/post/top10bestpractices", ), + ( + "DAX Expressions", + "Measure", + "Error", + "Avoid aggregating non-numeric columns", + lambda obj, tom: any( + find_non_numeric_aggregations(obj.Expression, tom) + ), + "Numeric aggregation functions (SUM, SUMX, AVERAGE, AVERAGEX, MIN, MINX, MAX, MAXX, PRODUCT, PRODUCTX) should be applied to numeric columns (Int64, Decimal, Double). Aggregating a non-numeric column will either fail at query time or force an implicit conversion, both of which usually indicate a modeling mistake.", + ), ( "DAX Expressions", "Measure", "Warning", "Use the TREATAS function instead of INTERSECT for virtual relationships", - lambda obj, tom: re.search( - r"intersect\s*\(", obj.Expression, flags=re.IGNORECASE - ), + lambda obj, tom: uses_function(obj.Expression, "INTERSECT"), "The TREATAS function is more efficient and provides better performance than the INTERSECT function when used in virutal relationships.", "https://www.sqlbi.com/articles/propagate-filters-using-treatas-in-dax", ), @@ -456,11 +468,7 @@ def model_bpa_rules( "Measure", "Warning", "The EVALUATEANDLOG function should not be used in production models", - lambda obj, tom: re.search( - r"evaluateandlog\s*\(", - obj.Expression, - flags=re.IGNORECASE, - ), + lambda obj, tom: uses_function(obj.Expression, "EVALUATEANDLOG"), "The EVALUATEANDLOG function is meant to be used only in development/test environments and should not be used in production models.", "https://pbidax.wordpress.com/2022/08/16/introduce-the-dax-evaluateandlog-function", ), @@ -576,7 +584,7 @@ def model_bpa_rules( "Error", "Column references should be fully qualified", lambda obj, tom: any( - tom.unqualified_columns(object=obj, dependencies=dependencies) + find_unqualified_columns(tom._get_expression(obj), tom) ), "Using fully qualified column references makes it easier to distinguish between column and measure references, and also helps avoid certain errors. When referencing a column in DAX, first specify the table name, then specify the column name in square brackets.", "https://www.elegantbi.com/post/top10bestpractices", @@ -592,7 +600,7 @@ def model_bpa_rules( "Error", "Measure references should be unqualified", lambda obj, tom: any( - tom.fully_qualified_measures(object=obj, dependencies=dependencies) + find_fully_qualified_measures(tom._get_expression(obj), tom) ), "Using unqualified measure references makes it easier to distinguish between column and measure references, and also helps avoid certain errors. When referencing a measure using DAX, do not specify the table name. Use only the measure name in square brackets.", "https://www.elegantbi.com/post/top10bestpractices", diff --git a/src/sempy_labs/dax/__init__.py b/src/sempy_labs/dax/__init__.py new file mode 100644 index 000000000..61fcc4e2b --- /dev/null +++ b/src/sempy_labs/dax/__init__.py @@ -0,0 +1,11 @@ +from ._parser import ( + parse_dax, +) +from ._format import ( + format_dax, +) + +__all__ = [ + "parse_dax", + "format_dax", +] diff --git a/src/sempy_labs/dax/_analysis.py b/src/sempy_labs/dax/_analysis.py new file mode 100644 index 000000000..7fe200fd2 --- /dev/null +++ b/src/sempy_labs/dax/_analysis.py @@ -0,0 +1,276 @@ +"""Static analysis helpers over DAX expressions. + +These helpers operate purely on the parsed AST and do not require a TOM +connection, which makes them straightforward to unit-test. +""" + +from typing import Iterator, List, Tuple, Union + +from ._expressions import Column, Function, Measure +from ._parser import parse_dax + + +# Functions that require their target column(s) to be numeric. +# Value is the argument index that contains the column to aggregate: +# 0 -> the first arg is the column itself (e.g. SUM()) +# 1 -> the second arg is an expression that should be scanned for any +# Column references (e.g. SUMX(, )) +NUMERIC_AGGREGATIONS = { + "SUM": 0, + "AVERAGE": 0, + "MIN": 0, + "MAX": 0, + "PRODUCT": 0, + "SUMX": 1, + "AVERAGEX": 1, + "MINX": 1, + "MAXX": 1, + "PRODUCTX": 1, +} + + +def find_numeric_aggregation_columns( + expression: str, +) -> Iterator[Tuple[str, str, str]]: + """ + Walk a DAX expression and yield every column referenced by a numeric + aggregation function. + + Yields + ------ + (function_name, table_name, column_name) + One tuple per column reference found inside a numeric-aggregation + function call. The function name is upper-cased. + + Notes + ----- + Silently yields nothing if the expression cannot be parsed. + """ + + if not expression: + return + + try: + tree = parse_dax(expression) + except SyntaxError: + return + + for fn in tree.find_all(Function): + + fn_name = fn.args.get("this", "").upper() + + if fn_name not in NUMERIC_AGGREGATIONS: + continue + + idx = NUMERIC_AGGREGATIONS[fn_name] + args = fn.args.get("expressions", []) + + if idx >= len(args): + continue + + target = args[idx] + + # For SUM/AVERAGE/etc. the first arg IS the column; for the *X + # variants the column lives somewhere inside the expression arg. + if isinstance(target, Column): + columns = [target] + else: + columns = list(target.find_all(Column)) + + for col in columns: + yield fn_name, col.args["table"], col.args["this"] + + +def uses_function( + expression: str, function_name: Union[str, List[str]] +) -> bool: + """ + Returns True if the DAX expression contains a call to any of the given + function(s) (case-insensitive). Built on the DAX parser so it ignores + occurrences inside string literals or comments that a regex would + falsely match. + + Parameters + ---------- + expression : str + The DAX expression to analyze. + function_name : str | list[str] + A single DAX function name (e.g. ``"IFERROR"``) or a list of names + (e.g. ``["IFERROR", "ERROR"]``). Matching is case-insensitive. + + Returns + ------- + bool + True if any of the named functions is called in ``expression``. + False if the expression is empty or cannot be parsed. + """ + + if not expression: + return False + + try: + tree = parse_dax(expression) + except SyntaxError: + return False + + if isinstance(function_name, str): + targets = {function_name.upper()} + else: + targets = {name.upper() for name in function_name} + + for fn in tree.find_all(Function): + if fn.args.get("this", "").upper() in targets: + return True + + return False + + +def find_non_numeric_aggregations( + expression: str, tom +) -> Iterator[Tuple[str, str, str, str]]: + """ + Yield every column reference inside a numeric-aggregation function + (SUM, SUMX, AVERAGE, AVERAGEX, MIN, MINX, MAX, MAXX, PRODUCT, PRODUCTX) + whose corresponding TOM column is **not** numeric (Int64/Decimal/Double). + + Parameters + ---------- + expression : str + The DAX expression to analyze. + tom : TOMWrapper + Used to resolve table/column data types. Columns that cannot be + resolved against the model are silently skipped. + + Yields + ------ + (function_name, table_name, column_name, data_type) + One tuple per offending reference. ``data_type`` is the string + form of the TOM ``DataType`` enum. + + Notes + ----- + Designed to be called inline from a BPA rule lambda:: + + lambda obj, tom: any( + find_non_numeric_aggregations(obj.Expression, tom) + ) + """ + + import Microsoft.AnalysisServices.Tabular as TOM + + numeric_types = { + TOM.DataType.Int64, + TOM.DataType.Decimal, + TOM.DataType.Double, + } + + tables = tom.model.Tables + + for fn_name, table_name, column_name in find_numeric_aggregation_columns( + expression + ): + table = next((t for t in tables if t.Name == table_name), None) + if table is None: + continue + column = next( + (c for c in table.Columns if c.Name == column_name), None + ) + if column is None: + continue + if column.DataType in numeric_types: + continue + + yield fn_name, table_name, column_name, str(column.DataType) + + +def find_unqualified_columns(expression: str, tom) -> Iterator[str]: + """ + Yield the name of every unqualified column reference in a DAX + expression — i.e. a bracketed reference like ``[Amount]`` whose name + matches a column in the supplied TOM model (and not a measure). + + Disambiguation requires the model schema, which is why ``tom`` is + required: a bare ``[Foo]`` could be a measure reference or an + unqualified column reference depending on what exists in the model. + + Parameters + ---------- + expression : str + The DAX expression to analyze. + tom : TOMWrapper + The TOM wrapper for the semantic model. + + Yields + ------ + str + The column name of each unqualified column reference found. + Yields nothing if the expression is empty or cannot be parsed. + + Notes + ----- + Designed to be called inline from a BPA rule lambda:: + + lambda obj, tom: any( + find_unqualified_columns(obj.Expression, tom) + ) + """ + + if not expression: + return + + try: + tree = parse_dax(expression, tom=tom) + except SyntaxError: + return + + for col in tree.find_all(Column): + if col.args.get("table") is None: + yield col.args["this"] + + +def find_fully_qualified_measures( + expression: str, tom +) -> Iterator[Tuple[str, str]]: + """ + Yield every fully-qualified measure reference in a DAX expression — + i.e. a bracketed reference like ``'Table'[MeasureName]`` whose + bracketed name matches a measure in the supplied TOM model. + + Best practice is to reference measures by their unqualified name + (``[MeasureName]``) and to reserve the ``Table[Name]`` syntax for + columns. Disambiguation requires the model schema, which is why + ``tom`` is required. + + Parameters + ---------- + expression : str + The DAX expression to analyze. + tom : TOMWrapper + The TOM wrapper for the semantic model. + + Yields + ------ + (table_name, measure_name) + One tuple per fully-qualified measure reference found. Yields + nothing if the expression is empty or cannot be parsed. + + Notes + ----- + Designed to be called inline from a BPA rule lambda:: + + lambda obj, tom: any( + find_fully_qualified_measures(obj.Expression, tom) + ) + """ + + if not expression: + return + + try: + tree = parse_dax(expression, tom=tom) + except SyntaxError: + return + + for m in tree.find_all(Measure): + if m.args.get("table") is not None: + yield m.args["table"], m.args["this"] diff --git a/src/sempy_labs/dax/_example.txt b/src/sempy_labs/dax/_example.txt new file mode 100644 index 000000000..114b56a42 --- /dev/null +++ b/src/sempy_labs/dax/_example.txt @@ -0,0 +1,64 @@ +VAR MaxDate = + MAX ( 'Date'[Date] ) + +VAR RollingWindow = + DATESINPERIOD ( + 'Date'[Date], + MaxDate, + -90, + DAY + ) + +VAR CustomerSalesTable = + ADDCOLUMNS ( + SUMMARIZE ( + FILTER ( + ALL ( Sales ), + Sales[OrderDate] IN RollingWindow + ), + Customer[CustomerKey] + ), + "RollingSales", + CALCULATE ( + SUMX ( + Sales, + Sales[ExtendedAmount] - Sales[DiscountAmount] + ) + ), + "OrderCount", + CALCULATE ( + DISTINCTCOUNT ( Sales[OrderNumber] ) + ) + ) + +VAR FilteredCustomers = + FILTER ( + CustomerSalesTable, + [RollingSales] > 5000 + && [OrderCount] >= 3 + ) + +VAR RankedCustomers = + ADDCOLUMNS ( + FilteredCustomers, + "SalesRank", + RANKX ( + FilteredCustomers, + [RollingSales], + , + DESC, + DENSE + ) + ) + +VAR Top10Customers = + FILTER ( + RankedCustomers, + [SalesRank] <= 10 + ) + +RETURN +SUMX ( + Top10Customers, + [RollingSales] +) \ No newline at end of file diff --git a/src/sempy_labs/dax/_expressions.py b/src/sempy_labs/dax/_expressions.py new file mode 100644 index 000000000..d1b0f8379 --- /dev/null +++ b/src/sempy_labs/dax/_expressions.py @@ -0,0 +1,126 @@ +from dataclasses import dataclass, field + + +@dataclass +class Expression: + args: dict = field(default_factory=dict) + + def walk(self): + yield self + + for value in self.args.values(): + if isinstance(value, Expression): + yield from value.walk() + + elif isinstance(value, list): + for item in value: + if isinstance(item, Expression): + yield from item.walk() + + def find_all(self, expression_type): + for node in self.walk(): + if isinstance(node, expression_type): + yield node + + def transform(self, func): + new_node = func(self) + + for key, value in list(new_node.args.items()): + + if isinstance(value, Expression): + new_node.args[key] = value.transform(func) + + elif isinstance(value, list): + + new_list = [] + + for item in value: + + if isinstance(item, Expression): + item = item.transform(func) + + new_list.append(item) + + new_node.args[key] = new_list + + return new_node + + def dump(self, level=0): + + indent = " " * level + + print(f"{indent}{self.__class__.__name__}") + + for key, value in self.args.items(): + + if isinstance(value, Expression): + + print(f"{indent} {key}:") + value.dump(level + 2) + + elif isinstance(value, list): + + print(f"{indent} {key}:") + + for item in value: + + if isinstance(item, Expression): + item.dump(level + 2) + else: + print(f"{indent} {item}") + + else: + print(f"{indent} {key}: {value}") + + +class Function(Expression): + pass + + +class Column(Expression): + pass + + +class Measure(Expression): + pass + + +class Table(Expression): + pass + + +class Literal(Expression): + pass + + +class Binary(Expression): + pass + + +class Unary(Expression): + pass + + +class Var(Expression): + pass + + +class VariableReference(Expression): + pass + + +class Keyword(Expression): + """A bare-identifier DAX enum constant (e.g. DAY, DESC, DENSE, ASC).""" + + pass + + +class VirtualColumn(Expression): + """A bracketed reference to a column introduced by ADDCOLUMNS / + SELECTCOLUMNS (rather than an existing model measure).""" + + pass + + +class Return(Expression): + pass diff --git a/src/sempy_labs/dax/_format.py b/src/sempy_labs/dax/_format.py new file mode 100644 index 000000000..c201fd1a2 --- /dev/null +++ b/src/sempy_labs/dax/_format.py @@ -0,0 +1,178 @@ +import html as _html +from typing import Optional + +from sempy._utils._log import log + +from ._expressions import Var +from ._parser import Parser +from ._tokenizer import tokenize +from ._tokens import TokenType + + +# Soft, Apple-inspired palette tuned to read well on both light and dark +# backgrounds (loosely based on Apple's "vivid" system colors, lightened +# slightly so they don't burn on a light background). +_COLORS = { + "function": "#5E9EFF", # soft blue - functions & keywords + "keyword": "#5E9EFF", + "variable": "#5AC8B8", # soft teal - VAR-defined names + "number": "#FF9F45", # soft orange - numeric literals + "virtual_column": "#FF7A8A", # soft pink - ADDCOLUMNS/SELECTCOLUMNS cols + "string": "#9BB87A", # muted green - string literals (default) + "operator": "#A6A6A6", # neutral gray + "punctuation": "#A6A6A6", + "default": "inherit", +} + + +def _classify_tokens(dax_expression: str): + """Tokenize the DAX text and attach a semantic 'kind' to each token. + + Returns a list of ``(token, kind)`` tuples in source order. ``kind`` is + one of the keys in ``_COLORS`` or ``None`` for tokens that should be + rendered with the default color. + """ + + tokens = list(tokenize(dax_expression)) + + # Best-effort parse to pick up VAR names and virtual columns. If parsing + # fails (e.g. partial input), fall back to a structural-only highlight. + var_names: set[str] = set() + virtual_columns: set[str] = set() + try: + parser = Parser(dax_expression) + tree = parser.parse() + virtual_columns = set(parser.virtual_columns) + for node in tree.walk(): + if isinstance(node, Var): + name = node.args.get("this") + if isinstance(name, str): + var_names.add(name) + except Exception: + pass + + classified = [] + + for i, token in enumerate(tokens): + kind: Optional[str] = None + tt = token.token_type + + if tt == TokenType.EOF: + continue + + if tt in (TokenType.VAR, TokenType.RETURN): + kind = "keyword" + elif tt == TokenType.IDENTIFIER: + # Function call if the next non-EOF token is '('. + next_token = tokens[i + 1] if i + 1 < len(tokens) else None + if next_token is not None and next_token.token_type == TokenType.LPAREN: + kind = "function" + elif token.text in var_names: + kind = "variable" + elif token.text.upper() in Parser.KEYWORDS: + kind = "keyword" + else: + kind = None + elif tt == TokenType.NUMBER: + kind = "number" + elif tt == TokenType.COLUMN: + inner = token.text[1:-1] + if inner in virtual_columns: + kind = "virtual_column" + else: + kind = None + elif tt == TokenType.STRING: + kind = "string" + elif tt == TokenType.OPERATOR: + kind = "operator" + elif tt in (TokenType.LPAREN, TokenType.RPAREN, TokenType.COMMA): + kind = "punctuation" + + classified.append((token, kind)) + + return classified + + +def _render_html(dax_expression: str) -> str: + """Render the DAX expression as an HTML ``
`` block with inline
+    color spans."""
+
+    classified = _classify_tokens(dax_expression)
+
+    parts = []
+    cursor = 0
+
+    for token, kind in classified:
+        # Emit any whitespace / unmatched chars between tokens verbatim so
+        # the original formatting (indentation, line breaks) is preserved.
+        if token.position > cursor:
+            parts.append(_html.escape(dax_expression[cursor : token.position]))
+
+        text = _html.escape(token.text)
+        color = _COLORS.get(kind or "default", "inherit")
+
+        if color == "inherit":
+            parts.append(text)
+        else:
+            parts.append(f'{text}')
+
+        cursor = token.position + len(token.text)
+
+    # Trailing whitespace after the last token.
+    if cursor < len(dax_expression):
+        parts.append(_html.escape(dax_expression[cursor:]))
+
+    body = "".join(parts)
+
+    return (
+        '
'
+        f"{body}"
+        "
"
+    )
+
+
+@log
+def format_dax(dax_expression: str, display: bool = True):
+    """
+    Color-codes a DAX expression for display in a notebook.
+
+    Uses the DAX parser/tokenizer to classify tokens and renders them with
+    a soft, Apple-inspired palette that reads well on both light and dark
+    backgrounds:
+
+    * Functions and keywords - soft blue
+    * VAR-declared variables - soft teal
+    * Numeric literals - soft orange
+    * Virtual columns (introduced by ``ADDCOLUMNS`` / ``SELECTCOLUMNS``) - soft pink
+
+    Parameters
+    ----------
+    dax_expression : str
+        The DAX expression to color-code.
+    display : bool, default=True
+        If True, displays the formatted DAX in the current notebook and
+        returns ``None``. If False, returns an ``IPython.display.HTML``
+        object instead.
+
+    Returns
+    -------
+    IPython.display.HTML | None
+        The HTML object when ``display=False``; otherwise ``None``.
+    """
+
+    from IPython.display import HTML, display as _display
+
+    rendered = HTML(_render_html(dax_expression))
+
+    if display:
+        _display(rendered)
+        return None
+
+    return rendered
diff --git a/src/sempy_labs/dax/_generator.py b/src/sempy_labs/dax/_generator.py
new file mode 100644
index 000000000..71322856a
--- /dev/null
+++ b/src/sempy_labs/dax/_generator.py
@@ -0,0 +1,42 @@
+from ._expressions import *
+
+
+class Generator:
+
+    def generate(self, expression):
+
+        if isinstance(expression, Function):
+
+            args = ", ".join(
+                self.generate(arg) for arg in expression.args["expressions"]
+            )
+
+            return f"{expression.args['this']}({args})"
+
+        elif isinstance(expression, Column):
+
+            return f"'{expression.args['table']}'" f"[{expression.args['this']}]"
+
+        elif isinstance(expression, Measure):
+
+            return f"[{expression.args['this']}]"
+
+        elif isinstance(expression, Literal):
+
+            return str(expression.args["this"])
+
+        elif isinstance(expression, Binary):
+
+            left = self.generate(expression.args["this"])
+
+            right = self.generate(expression.args["expression"])
+
+            op = expression.args["operator"]
+
+            return f"{left} {op} {right}"
+
+        raise ValueError(f"Unsupported expression: {type(expression)}")
+
+
+def to_dax(expression):
+    return Generator().generate(expression)
diff --git a/src/sempy_labs/dax/_lineage.py b/src/sempy_labs/dax/_lineage.py
new file mode 100644
index 000000000..03e86c8be
--- /dev/null
+++ b/src/sempy_labs/dax/_lineage.py
@@ -0,0 +1,28 @@
+from ._expressions import *
+
+
+def extract_lineage(tree):
+
+    results = {
+        "functions": set(),
+        "columns": set(),
+        "measures": set(),
+    }
+
+    for node in tree.walk():
+
+        if isinstance(node, Function):
+            results["functions"].add(node.args["this"])
+
+        elif isinstance(node, Column):
+            results["columns"].add(
+                (
+                    node.args["table"],
+                    node.args["this"],
+                )
+            )
+
+        elif isinstance(node, Measure):
+            results["measures"].add(node.args["this"])
+
+    return {k: sorted(v) for k, v in results.items()}
diff --git a/src/sempy_labs/dax/_parser.py b/src/sempy_labs/dax/_parser.py
new file mode 100644
index 000000000..995448d65
--- /dev/null
+++ b/src/sempy_labs/dax/_parser.py
@@ -0,0 +1,788 @@
+from ._expressions import *
+from ._tokenizer import tokenize
+from ._tokens import TokenType
+
+
+class Parser:
+
+    PRECEDENCE = {
+        "||": 1,
+        "&&": 2,
+        "=": 5,
+        "<>": 5,
+        "<": 5,
+        "<=": 5,
+        ">": 5,
+        ">=": 5,
+        "&": 6,
+        "+": 10,
+        "-": 10,
+        "*": 20,
+        "/": 20,
+    }
+
+    # Bare-identifier DAX enum constants. These are arguments to specific
+    # functions (e.g. DATESINPERIOD's interval, RANKX's order/ties) and must
+    # not be treated as table references. Matched case-insensitively.
+    KEYWORDS = {
+        # Date intervals (DATESINPERIOD, DATEADD, PARALLELPERIOD, ...)
+        "DAY",
+        "WEEK",
+        "MONTH",
+        "QUARTER",
+        "YEAR",
+        # Sort order (RANKX, TOPN, ORDERBY, ...)
+        "ASC",
+        "DESC",
+        # Rank ties (RANKX)
+        "SKIP",
+        "DENSE",
+        "LAST",
+        "FIRST",
+        # Boolean / blank literals
+        "TRUE",
+        "FALSE",
+        "BLANK",
+        # Crossfilter direction (CROSSFILTER)
+        "BOTH",
+        "ONEWAY",
+        "ONEWAY_LEFTFILTERSRIGHT",
+        "ONEWAY_RIGHTFILTERSLEFT",
+        "NONE",
+        # Match modes (LOOKUPVALUE-related)
+        "ABSOLUTE",
+        "RELATIVE",
+        "INTEGER",
+        "STRING",
+        "DOUBLE",
+        "BOOLEAN",
+        "DATETIME",
+        "VARIANT",
+        "TEXT",
+        "ALPHABETICAL",
+        "DEFINE",
+        "ORDER",
+        "BY",
+        "EVALUATE",
+        "EVALUATEANDLOG",
+        "OFFSET",
+        "VAR",
+        "RETURN",
+        "PARTITIONBY",
+        "RANK",
+        "ROWNUMBER",
+        "INDEX",
+        "WINDOW",
+        "ORDERBY",
+        "IN",
+        "NOT",
+        "AND",
+        "OR",
+        "ABS",
+        "ACOS",
+        "ACOSH",
+        "ACOT",
+        "ACOTH",
+        "ASIN",
+        "ASINH",
+        "ATAN",
+        "ATANH",
+        "CEILING",
+        "CONVERT",
+        "COS",
+        "COSH",
+        "COT",
+        "COTH",
+        "CURRENCY",
+        "DEGREES",
+        "DIVIDE",
+        "EVEN",
+        "EXP",
+        "FACT",
+        "FLOOR",
+        "GCD",
+        "ISO.CEILING",
+        "LCM",
+        "LN",
+        "LOG",
+        "LOG10",
+        "MOD",
+        "MROUND",
+        "ODD",
+        "PI",
+        "POWER",
+        "QUOTIENT",
+        "RADIANS",
+        "RAND",
+        "RANDBETWEEN",
+        "ROUND",
+        "ROUNDDOWN",
+        "ROUNDUP",
+        "SIGN",
+        "SIN",
+        "SINH",
+        "SQRT",
+        "SQRTPI",
+        "TAN",
+        "TANH",
+        "TRUNC",
+        "APPROXIMATEDISTINCTCOUNT",
+        "AVERAGE",
+        "AVERAGEA",
+        "AVERAGEX",
+        "COUNT",
+        "COUNTA",
+        "COUNTX",
+        "COUNTAX",
+        "COUNTBLANK",
+        "COUNTROWS",
+        "DISTINCTCOUNT",
+        "DISTINCTCOUNTNOBLANK",
+        "MAX",
+        "MAXA",
+        "MAXX",
+        "MIN",
+        "MINA",
+        "MINX",
+        "PRODUCT",
+        "PRODUCTX",
+        "SUM",
+        "SUMX",
+        "CALENDAR",
+        "CALENDARAUTO",
+        "DATE",
+        "DATEDIFF",
+        "DATEVALUE",
+        "EDATE",
+        "EOMONTH",
+        "HOUR",
+        "MINUTE",
+        "NETWORKDAYS",
+        "NOW",
+        "SECOND",
+        "TIME",
+        "UTCNOW",
+        "UTCTODAY",
+        "WEEKDAY",
+        "WEEKNUM",
+        "YEARFRAC",
+        "ALL",
+        "ALLCROSSFILTERED",
+        "ALLEXCEPT",
+        "ALLNOBLANKROW",
+        "ALLSELECTED",
+        "CALCULATE",
+        "CALCULATETABLE",
+        "EARLIER",
+        "EARLIEST",
+        "FILTER",
+        "FIRSTNONBLANK",
+        "FIRSTNONBLANKVALUE",
+        "KEEPFILTERS",
+        "LASTNONBLANK",
+        "LASTNONBLANKVALUE",
+        "LOOKUP",
+        "LOOKUPVALUE",
+        "LOOKUPWITHTOTALS",
+        "MATCHBY",
+        "MOVINGAVERAGE",
+        "RANGE",
+        "REMOVEFILTERS",
+        "RUNNINGSUM",
+        "SELECTEDVALUE",
+        "ACCRINT",
+        "ACCRINTM",
+        "AMORDEGRC",
+        "AMORLINC",
+        "COUPDAYBS",
+        "COUPDAYS",
+        "COUPDAYSNC",
+        "COUPNCD",
+        "COUPNUM",
+        "COUPPCD",
+        "CUMIPMT",
+        "CUMPRINC",
+        "DB",
+        "DDB",
+        "DISC",
+        "DOLLARDE",
+        "DOLLARFR",
+        "DURATION",
+        "EFFECT",
+        "FV",
+        "INTRATE",
+        "IPMT",
+        "ISPMT",
+        "MDURATION",
+        "NOMINAL",
+        "NPER",
+        "ODDFPRICE",
+        "ODDFYIELD",
+        "ODDLPRICE",
+        "ODDLYIELD",
+        "PDURATION",
+        "PMT",
+        "PPMT",
+        "PRICE",
+        "PRICEDISC",
+        "PRICEMAT",
+        "PV",
+        "RATE",
+        "RECEIVED",
+        "RRI",
+        "SLN",
+        "SYD",
+        "TBILLEQ",
+        "TBILLPRICE",
+        "TBILLYIELD",
+        "VDB",
+        "XIRR",
+        "XNPV",
+        "YIELD",
+        "YIELDDISC",
+        "YIELDMAT",
+        "COLUMNSTATISTICS",
+        "CONTAINS",
+        "CONTAINSROW",
+        "CONTAINSSTRING",
+        "CONTAINSSTRINGEXACT",
+        "CUSTOMDATA",
+        "HASONEFILTER",
+        "HASONEVALUE",
+        "ISAFTER",
+        "ISBLANK",
+        "ISBOOLEAN",
+        "ISCROSSFILTERED",
+        "ISCURRENCY",
+        "ISDATETIME",
+        "ISDECIMAL",
+        "ISDOUBLE",
+        "ISEMPTY",
+        "ISERROR",
+        "ISEVEN",
+        "ISFILTERED",
+        "ISINSCOPE",
+        "ISINT64",
+        "ISLOGICAL",
+        "ISNONTEXT",
+        "ISNUMBER",
+        "ISNUMERIC",
+        "ISODD",
+        "ISONORAFTER",
+        "ISSELECTEDMEASURE",
+        "ISSTRING",
+        "ISSUBTOTAL",
+        "ISTEXT",
+        "NAMEOF",
+        "NONVISUAL",
+        "SELECTEDMEASURE",
+        "SELECTEDMEASUREFORMATSTRING",
+        "SELECTEDMEASURENAME",
+        "TABLEOF",
+        "USERCULTURE",
+        "USERNAME",
+        "USEROBJECTID",
+        "USERPRINCIPALNAME",
+        "BITAND",
+        "BITLSHIFT",
+        "BITOR",
+        "BITRSHIFT",
+        "BITXOR",
+        "COALESCE",
+        "IF",
+        "IF.EAGER",
+        "IFERROR",
+        "SWITCH",
+        "ERROR",
+        "EXTERNALMEASURE",
+        "TOCSV",
+        "TOJSON",
+        "PATH",
+        "PATHCONTAINS",
+        "PATHITEM",
+        "PATHITEMREVERSE",
+        "PATHLENGTH",
+        "CROSSFILTER",
+        "RELATED",
+        "RELATEDTABLE",
+        "USERELATIONSHIP",
+        "BETA.DIST",
+        "BETA.INV",
+        "CHISQ.DIST",
+        "CHISQ.DIST.RT",
+        "CHISQ.INV",
+        "CHISQ.INV.RT",
+        "COMBIN",
+        "COMBINA",
+        "CONFIDENCE.NORM",
+        "CONFIDENCE.T",
+        "EXPON.DIST",
+        "GEOMEAN",
+        "GEOMEANX",
+        "LINEST",
+        "LINESTX",
+        "MEDIAN",
+        "MEDIANX",
+        "NORM.DIST",
+        "NORM.INV",
+        "NORM.S.DIST",
+        "NORM.S.INV",
+        "PERCENTILE.EXC",
+        "PERCENTILE.INC",
+        "PERCENTILEX.EXC",
+        "PERCENTILEX.INC",
+        "PERMUT",
+        "POISSON.DIST",
+        "RANK.EX",
+        "RANKX",
+        "SAMPLE",
+        "SAMPLECARTESIANPOINTSBYCOVER",
+        "STDEV.S",
+        "STDEV.P",
+        "STDEVX.S",
+        "STDEVX.P",
+        "T.DIST",
+        "T.DIST.2T",
+        "T.DIST.RT",
+        "T.INV",
+        "T.INV.2T",
+        "VAR.S",
+        "VAR.P",
+        "VARX.S",
+        "VARX.P",
+        "ADDCOLUMNS",
+        "ADDMISSINGITEMS",
+        "CROSSJOIN",
+        "CURRENTGROUP",
+        "DATATABLE",
+        "DETAILROWS",
+        "DISTINCT",
+        "EXCEPT",
+        "FILTERS",
+        "GENERATE",
+        "GENERATEALL",
+        "GENERATESERIES",
+        "GROUPBY",
+        "IGNORE",
+        "INTERSECT",
+        "NATURALINNERJOIN",
+        "NATURALLEFTOUTERJOIN",
+        "ROLLUP",
+        "ROLLUPADDISSUBTOTAL",
+        "ROLLUPGROUP",
+        "ROLLUPISSUBTOTAL",
+        "ROW",
+        "SELECTCOLUMNS",
+        "SUBSTITUTEWITHINDEX",
+        "SUMMARIZE",
+        "SUMMARIZECOLUMNS",
+        "TOPN",
+        "TREATAS",
+        "UNION",
+        "VALUES",
+        "COMBINEVALUES",
+        "CONCATENATE",
+        "CONCATENATEX",
+        "EXACT",
+        "FIND",
+        "FIXED",
+        "FORMAT",
+        "LEFT",
+        "LEN",
+        "LOWER",
+        "MID",
+        "REPLACE",
+        "REPT",
+        "RIGHT",
+        "SEARCH",
+        "SUBSTITUTE",
+        "TRIM",
+        "UNICHAR",
+        "UNICODE",
+        "UPPER",
+        "VALUE",
+        # TIME INTELLIGENCE
+        "CLOSINGBALANCEWEEK",
+        "CLOSINGBALANCEMONTH",
+        "CLOSINGBALANCEQUARTER",
+        "CLOSINGBALANCEYEAR",
+        "DATEADD",
+        "DATESBETWEEN",
+        "DATESINPERIOD",
+        "DATESWTD",
+        "DATESMTD",
+        "DATESQTD",
+        "DATESYTD",
+        "ENDOFWEEK",
+        "ENDOFMONTH",
+        "ENDOFQUARTER",
+        "ENDOFYEAR",
+        "FIRSTDATE",
+        "LASTDATE",
+        "NEXTDAY",
+        "NEXTWEEK",
+        "NEXTMONTH",
+        "NEXTQUARTER",
+        "NEXTYEAR",
+        "OPENINGBALANCEWEEK",
+        "OPENINGBALANCEMONTH",
+        "OPENINGBALANCEQUARTER",
+        "OPENINGBALANCEYEAR",
+        "PARALLELPERIOD",
+        "PREVIOUSDAY",
+        "PREVIOUSWEEK",
+        "PREVIOUSMONTH",
+        "PREVIOUSYEAR",
+        "SAMEPERIODLASTYEAR",
+        "STARTOFWEEK",
+        "STARTOFMONTH",
+        "STARTOFQUARTER",
+        "STARTOFYEAR",
+        "TOTALWTD",
+        "TOTALMTD",
+        "TOTALQTD",
+        "TOTALYTD",
+        # DAX STATEMENTS
+        "FUNCTION",
+        "MEASURE",
+    }
+
+    def __init__(self, text, tom=None):
+
+        self.tokens = list(tokenize(text))
+        self.index = 0
+        # Stack of sets of variable names currently in scope. Used to
+        # distinguish references to VAR-declared names from table refs.
+        self.scopes = []
+        # Names of virtual columns introduced via ADDCOLUMNS / SELECTCOLUMNS
+        # anywhere in the input. Bracketed references to these names are
+        # emitted as VirtualColumn instead of Measure.
+        self.virtual_columns = set()
+        # Optional TOM model. When supplied, the parser uses it to
+        # disambiguate bare bracketed references (`[Name]`) between
+        # measures and columns. Without a model every `[Name]` defaults
+        # to a Measure node (backward-compatible behavior).
+        self._measure_names = set()
+        self._column_names = set()
+        if tom is not None:
+            model = getattr(tom, "model", tom)
+            for t in model.Tables:
+                for m in t.Measures:
+                    self._measure_names.add(m.Name)
+                for c in t.Columns:
+                    self._column_names.add(c.Name)
+
+    @property
+    def current(self):
+        return self.tokens[self.index]
+
+    def advance(self):
+        self.index += 1
+
+    def _is_variable(self, name):
+        return any(name in scope for scope in self.scopes)
+
+    def parse(self):
+        return self.statement()
+
+    def statement(self):
+
+        # VAR ... [VAR ...]* RETURN 
+        if self.current.token_type == TokenType.VAR:
+
+            variables = []
+            self.scopes.append(set())
+
+            try:
+                while self.current.token_type == TokenType.VAR:
+
+                    self.advance()
+
+                    if self.current.token_type != TokenType.IDENTIFIER:
+                        raise SyntaxError(
+                            f"Expected variable name after VAR, got: {self.current}"
+                        )
+
+                    name = self.current.text
+                    self.advance()
+
+                    if not (
+                        self.current.token_type == TokenType.OPERATOR
+                        and self.current.text == "="
+                    ):
+                        raise SyntaxError(
+                            f"Expected '=' after VAR name, got: {self.current}"
+                        )
+                    self.advance()
+
+                    value = self.statement()
+
+                    # Make the new variable visible to subsequent VARs and to
+                    # the RETURN expression. (DAX allows later VARs to
+                    # reference earlier ones in the same block.)
+                    self.scopes[-1].add(name)
+
+                    variables.append(
+                        Var(
+                            args={
+                                "this": name,
+                                "expression": value,
+                            }
+                        )
+                    )
+
+                if self.current.token_type != TokenType.RETURN:
+                    raise SyntaxError(
+                        f"Expected RETURN after VAR block, got: {self.current}"
+                    )
+                self.advance()
+
+                ret_expr = self.expression()
+
+                return Return(
+                    args={
+                        "variables": variables,
+                        "expression": ret_expr,
+                    }
+                )
+            finally:
+                self.scopes.pop()
+
+        return self.expression()
+
+    def expression(self, precedence=0):
+
+        left = self.primary()
+
+        while True:
+
+            token = self.current
+
+            if token.token_type != TokenType.OPERATOR:
+                break
+
+            token_precedence = self.PRECEDENCE.get(
+                token.text,
+                0,
+            )
+
+            if token_precedence < precedence:
+                break
+
+            operator = token.text
+
+            self.advance()
+
+            right = self.expression(token_precedence + 1)
+
+            left = Binary(
+                args={
+                    "this": left,
+                    "expression": right,
+                    "operator": operator,
+                }
+            )
+
+        return left
+
+    def primary(self):
+
+        token = self.current
+
+        # Unary +/-
+        if token.token_type == TokenType.OPERATOR and token.text in ("-", "+"):
+            self.advance()
+            operand = self.primary()
+            return Unary(
+                args={
+                    "this": operand,
+                    "operator": token.text,
+                }
+            )
+
+        if token.token_type == TokenType.IDENTIFIER:
+
+            name = token.text
+
+            self.advance()
+
+            if self.current.token_type == TokenType.LPAREN:
+
+                self.advance()
+
+                args = []
+
+                while self.current.token_type != TokenType.RPAREN:
+
+                    # Empty argument (e.g. RANKX(..., , DESC, DENSE))
+                    if self.current.token_type == TokenType.COMMA:
+                        args.append(Literal(args={"this": None}))
+                        self.advance()
+                        continue
+
+                    args.append(self.statement())
+
+                    if self.current.token_type == TokenType.COMMA:
+                        self.advance()
+                        # Trailing comma immediately before RPAREN -> empty arg
+                        if self.current.token_type == TokenType.RPAREN:
+                            args.append(Literal(args={"this": None}))
+
+                self.advance()
+
+                # Record virtual column names introduced by ADDCOLUMNS /
+                # SELECTCOLUMNS so later bracketed references resolve to
+                # VirtualColumn rather than Measure. Both have the shape:
+                #   FN (
, , [, , ]... ) + if name.upper() in ("ADDCOLUMNS", "SELECTCOLUMNS"): + for i in range(1, len(args), 2): + arg = args[i] + if isinstance(arg, Literal): + value = arg.args.get("this") + if ( + isinstance(value, str) + and len(value) >= 2 + and value.startswith('"') + and value.endswith('"') + ): + self.virtual_columns.add(value[1:-1]) + + return Function( + args={ + "this": name, + "expressions": args, + } + ) + + # Bare identifier (not a function call): a reference to a VAR + # in scope, a known DAX enum keyword, or otherwise a table + # reference. + if self._is_variable(name): + return VariableReference(args={"this": name}) + + if name.upper() in self.KEYWORDS: + return Keyword(args={"this": name}) + + return Table(args={"this": name}) + + elif token.token_type == TokenType.TABLE_COLUMN: + + text = token.text + + self.advance() + + table, column = text.split("[") + + table = table.strip("'") + + column = column[:-1] + + # When a TOM model is supplied, a `Table[Name]` token where + # `Name` resolves to a measure (not a column) is a + # fully-qualified measure reference. Emit a Measure node with + # the table preserved so downstream analyses can flag it. + if self._measure_names and column in self._measure_names: + if column not in self._column_names: + return Measure( + args={ + "table": table, + "this": column, + } + ) + + return Column( + args={ + "table": table, + "this": column, + } + ) + + elif token.token_type == TokenType.QUOTED_IDENTIFIER: + + name = token.text.strip("'") + + self.advance() + + return Table( + args={ + "this": name, + } + ) + + elif token.token_type == TokenType.COLUMN: + + name = token.text[1:-1] + + self.advance() + + if name in self.virtual_columns: + return VirtualColumn(args={"this": name}) + + # When a TOM model is available, disambiguate bare bracketed + # references: prefer measure resolution, then fall back to an + # unqualified column reference (Column with table=None) if a + # column of that name exists in the model. Without a model we + # keep the legacy default of emitting a Measure node. + if self._measure_names or self._column_names: + if name in self._measure_names: + return Measure(args={"this": name}) + if name in self._column_names: + return Column(args={"table": None, "this": name}) + + return Measure( + args={ + "this": name, + } + ) + + elif token.token_type == TokenType.NUMBER: + + self.advance() + + return Literal( + args={ + "this": token.text, + } + ) + + elif token.token_type == TokenType.STRING: + + self.advance() + + return Literal( + args={ + "this": token.text, + } + ) + + elif token.token_type == TokenType.LPAREN: + + self.advance() + expr = self.statement() + + if self.current.token_type != TokenType.RPAREN: + raise SyntaxError("Expected closing parenthesis") + + self.advance() + + return expr + + raise SyntaxError(f"Unexpected token: {token}") + + +def parse_dax(text: str, tom=None): + """Parse a DAX expression and return its AST. + + Parameters + ---------- + text : str + The DAX expression. + tom : TOMWrapper | Microsoft.AnalysisServices.Tabular.Model, optional + When supplied, the parser uses the model to disambiguate bare + bracketed references. A ``[Name]`` reference resolves to a + ``Measure`` node if a measure with that name exists, otherwise to + a ``Column`` node with ``table=None`` (an unqualified column) if a + column with that name exists, otherwise it falls back to + ``Measure`` (the default when no model is provided). + """ + return Parser(text, tom=tom).parse() diff --git a/src/sempy_labs/dax/_tokenizer.py b/src/sempy_labs/dax/_tokenizer.py new file mode 100644 index 000000000..b6b3b3c82 --- /dev/null +++ b/src/sempy_labs/dax/_tokenizer.py @@ -0,0 +1,64 @@ +import re +from dataclasses import dataclass +from ._tokens import TokenType + + +@dataclass +class Token: + token_type: TokenType + text: str + position: int + + +TOKEN_REGEX = [ + # Table[Column] reference: table may be quoted ('Sales') or unquoted (Sales) + (TokenType.TABLE_COLUMN, r"(?:'[^']+'|[A-Za-z_][A-Za-z0-9_]*)\[[^\]]+\]"), + # NEW + (TokenType.QUOTED_IDENTIFIER, r"'[^']+'"), + (TokenType.COLUMN, r"\[[^\]]+\]"), + (TokenType.STRING, r'"([^"]|"")*"'), + (TokenType.NUMBER, r"\d+(\.\d+)?"), + (TokenType.IDENTIFIER, r"[A-Za-z_][A-Za-z0-9_]*"), + (TokenType.OPERATOR, r"<=|>=|<>|&&|\|\||[-+*/=<>&]"), + (TokenType.LPAREN, r"\("), + (TokenType.RPAREN, r"\)"), + (TokenType.COMMA, r","), +] + +MASTER_REGEX = re.compile( + "|".join(f"(?P<{t.name}>{r})" for t, r in TOKEN_REGEX), + re.IGNORECASE, +) + + +def tokenize(text): + + position = 0 + + while position < len(text): + + if text[position].isspace(): + position += 1 + continue + + match = MASTER_REGEX.match(text, position) + + if not match: + raise SyntaxError(f"Unexpected character: {text[position]}") + + group = match.lastgroup + token_type = TokenType[group] + token_text = match.group() + + if token_type == TokenType.IDENTIFIER: + upper = token_text.upper() + if upper == "VAR": + token_type = TokenType.VAR + elif upper == "RETURN": + token_type = TokenType.RETURN + + yield Token(token_type, token_text, position) + + position = match.end() + + yield Token(TokenType.EOF, "", position) diff --git a/src/sempy_labs/dax/_tokens.py b/src/sempy_labs/dax/_tokens.py new file mode 100644 index 000000000..aefd1fd2b --- /dev/null +++ b/src/sempy_labs/dax/_tokens.py @@ -0,0 +1,24 @@ +from enum import Enum, auto + + +class TokenType(Enum): + + IDENTIFIER = auto() + QUOTED_IDENTIFIER = auto() + + NUMBER = auto() + STRING = auto() + + COLUMN = auto() + TABLE_COLUMN = auto() + + LPAREN = auto() + RPAREN = auto() + COMMA = auto() + + OPERATOR = auto() + + VAR = auto() + RETURN = auto() + + EOF = auto() diff --git a/src/sempy_labs/semantic_model/_perspective_editor.py b/src/sempy_labs/semantic_model/_perspective_editor.py index 59d561ca1..eeb9c7c1b 100644 --- a/src/sempy_labs/semantic_model/_perspective_editor.py +++ b/src/sempy_labs/semantic_model/_perspective_editor.py @@ -1150,9 +1150,7 @@ def perspective_editor( measures = sorted([m.Name for m in table.Measures]) hierarchies = sorted([h.Name for h in table.Hierarchies]) hidden_columns = [ - c.Name - for c in tom.all_columns() - if c.Parent == table and c.IsHidden + c.Name for c in tom.all_columns() if c.Parent == table and c.IsHidden ] hidden_measures = [m.Name for m in table.Measures if m.IsHidden] hidden_hierarchies = [h.Name for h in table.Hierarchies if h.IsHidden] @@ -1175,7 +1173,9 @@ def perspective_editor( tbl = pt.Table.Name members[tbl] = { "columns": sorted(pc.Column.Name for pc in pt.PerspectiveColumns), - "measures": sorted(pm.Measure.Name for pm in pt.PerspectiveMeasures), + "measures": sorted( + pm.Measure.Name for pm in pt.PerspectiveMeasures + ), "hierarchies": sorted( ph.Hierarchy.Name for ph in pt.PerspectiveHierarchies ), @@ -1268,9 +1268,7 @@ def _on_run(change): obj = tom.model.Perspectives[perspective_name] tom.remove_object(object=obj) tom.model.SaveChanges() - new_list = [ - x for x in widget.perspectives if x != perspective_name - ] + new_list = [x for x in widget.perspectives if x != perspective_name] new_members = dict(widget.perspective_members) new_members.pop(perspective_name, None) widget.perspective_members = new_members diff --git a/src/sempy_labs/tom/_model.py b/src/sempy_labs/tom/_model.py index 5054ae53f..810a9092e 100644 --- a/src/sempy_labs/tom/_model.py +++ b/src/sempy_labs/tom/_model.py @@ -282,6 +282,77 @@ def all_measures(self): for m in t.Measures: yield m + def find_non_numeric_aggregations(self) -> pd.DataFrame: + """ + Identifies measures whose DAX expression aggregates a non-numeric + column via a numeric aggregation function (SUM, SUMX, AVERAGE, + AVERAGEX, MIN, MINX, MAX, MAXX, PRODUCT, PRODUCTX). + + A column is considered numeric when its TOM DataType is one of + Int64, Decimal or Double. + + Returns + ------- + pandas.DataFrame + One row per offending column reference with the columns: + Measure, Table, Function, Column Table, Column, Data Type. + Returns an empty dataframe (with the same schema) if nothing + is found. + """ + import Microsoft.AnalysisServices.Tabular as TOM + from sempy_labs.dax._analysis import find_numeric_aggregation_columns + + numeric_types = { + TOM.DataType.Int64, + TOM.DataType.Decimal, + TOM.DataType.Double, + } + + # (table_name, column_name) -> DataType + column_data_types = { + (t.Name, c.Name): c.DataType for t in self.model.Tables for c in t.Columns + } + + columns = { + "Measure": "string", + "Table": "string", + "Function": "string", + "Column Table": "string", + "Column": "string", + "Data Type": "string", + } + + rows = [] + + for m in self.all_measures(): + + for fn_name, table_name, column_name in find_numeric_aggregation_columns( + m.Expression + ): + + data_type = column_data_types.get((table_name, column_name)) + + # Skip unresolved (e.g. virtual columns, typos) and numeric + # columns - we only want non-numeric model columns. + if data_type is None or data_type in numeric_types: + continue + + rows.append( + { + "Measure": m.Name, + "Table": m.Parent.Name, + "Function": fn_name, + "Column Table": table_name, + "Column": column_name, + "Data Type": str(data_type), + } + ) + + if rows: + return pd.DataFrame(rows, columns=list(columns.keys())) + + return _create_dataframe(columns=columns) + def all_partitions(self): """ Outputs a list of all partitions in the semantic model. diff --git a/tests/format_dax_preview.html b/tests/format_dax_preview.html new file mode 100644 index 000000000..5a3c68d61 --- /dev/null +++ b/tests/format_dax_preview.html @@ -0,0 +1,127 @@ +format_dax preview

Dark background

VAR MaxDate =
+    MAX ( 'Date'[Date] )
+
+VAR RollingWindow =
+    DATESINPERIOD (
+        'Date'[Date],
+        MaxDate,
+        -90,
+        DAY
+    )
+
+VAR CustomerSalesTable =
+    ADDCOLUMNS (
+        SUMMARIZE (
+            FILTER (
+                ALL ( Sales ),
+                Sales[OrderDate] IN RollingWindow
+            ),
+            Customer[CustomerKey]
+        ),
+        "RollingSales",
+            CALCULATE (
+                SUMX (
+                    Sales,
+                    Sales[ExtendedAmount] - Sales[DiscountAmount]
+                )
+            ),
+        "OrderCount",
+            CALCULATE (
+                DISTINCTCOUNT ( Sales[OrderNumber] )
+            )
+    )
+
+VAR FilteredCustomers =
+    FILTER (
+        CustomerSalesTable,
+        [RollingSales] > 5000
+            && [OrderCount] >= 3
+    )
+
+VAR RankedCustomers =
+    ADDCOLUMNS (
+        FilteredCustomers,
+        "SalesRank",
+            RANKX (
+                FilteredCustomers,
+                [RollingSales],
+                ,
+                DESC,
+                DENSE
+            )
+    )
+
+VAR Top10Customers =
+    FILTER (
+        RankedCustomers,
+        [SalesRank] <= 10
+    )
+
+RETURN
+SUMX (
+    Top10Customers,
+    [RollingSales]
+)

Light background

VAR MaxDate =
+    MAX ( 'Date'[Date] )
+
+VAR RollingWindow =
+    DATESINPERIOD (
+        'Date'[Date],
+        MaxDate,
+        -90,
+        DAY
+    )
+
+VAR CustomerSalesTable =
+    ADDCOLUMNS (
+        SUMMARIZE (
+            FILTER (
+                ALL ( Sales ),
+                Sales[OrderDate] IN RollingWindow
+            ),
+            Customer[CustomerKey]
+        ),
+        "RollingSales",
+            CALCULATE (
+                SUMX (
+                    Sales,
+                    Sales[ExtendedAmount] - Sales[DiscountAmount]
+                )
+            ),
+        "OrderCount",
+            CALCULATE (
+                DISTINCTCOUNT ( Sales[OrderNumber] )
+            )
+    )
+
+VAR FilteredCustomers =
+    FILTER (
+        CustomerSalesTable,
+        [RollingSales] > 5000
+            && [OrderCount] >= 3
+    )
+
+VAR RankedCustomers =
+    ADDCOLUMNS (
+        FilteredCustomers,
+        "SalesRank",
+            RANKX (
+                FilteredCustomers,
+                [RollingSales],
+                ,
+                DESC,
+                DENSE
+            )
+    )
+
+VAR Top10Customers =
+    FILTER (
+        RankedCustomers,
+        [SalesRank] <= 10
+    )
+
+RETURN
+SUMX (
+    Top10Customers,
+    [RollingSales]
+)
\ No newline at end of file diff --git a/tests/test_dax_analysis.py b/tests/test_dax_analysis.py new file mode 100644 index 000000000..2f718a408 --- /dev/null +++ b/tests/test_dax_analysis.py @@ -0,0 +1,77 @@ +"""Tests for `sempy_labs.dax._analysis.find_numeric_aggregation_columns`. + +These tests exercise the parser-driven detector that +`TOMWrapper.find_non_numeric_aggregations` is built on. They do not +require a TOM connection. +""" + +from sempy_labs.dax import find_numeric_aggregation_columns + + +def _findings(expr): + return list(find_numeric_aggregation_columns(expr)) + + +def test_sum_first_arg_column(): + # SUM() -- the column is the first arg + assert _findings("SUM ( Sales[Quantity] )") == [ + ("SUM", "Sales", "Quantity"), + ] + + +def test_average_first_arg_column(): + assert _findings("AVERAGE ( 'Sales'[Net Price] )") == [ + ("AVERAGE", "Sales", "Net Price"), + ] + + +def test_sumx_scans_expression_arg(): + # SUMX(
, ) -- column(s) live in the second arg + found = _findings("SUMX ( Sales, Sales[Quantity] * Sales[Net Price] )") + assert ("SUMX", "Sales", "Quantity") in found + assert ("SUMX", "Sales", "Net Price") in found + assert len(found) == 2 + + +def test_averagex_scans_expression_arg(): + found = _findings( + "AVERAGEX ( 'Date', 'Date'[Year] - 'Date'[StartYear] )" + ) + assert ("AVERAGEX", "Date", "Year") in found + assert ("AVERAGEX", "Date", "StartYear") in found + + +def test_nested_inside_calculate(): + # find_all walks the whole tree, so nesting under CALCULATE is fine + found = _findings("CALCULATE ( SUM ( Sales[Amount] ) )") + assert found == [("SUM", "Sales", "Amount")] + + +def test_non_aggregation_functions_ignored(): + # DIVIDE / IF aren't numeric aggregations -> no findings + assert _findings("DIVIDE ( Sales[A], Sales[B] )") == [] + assert _findings("IF ( Sales[A] > 0, 1, 0 )") == [] + + +def test_pure_measure_reference_yields_nothing(): + # [Total Sales] is a measure ref, not a column ref, so nothing to flag + assert _findings("SUMX ( Sales, [Total Sales] )") == [] + + +def test_unparseable_expression_returns_empty(): + # Should not raise -- just return nothing. + assert _findings("THIS IS NOT VALID DAX ((") == [] + + +def test_empty_or_none_expression(): + assert _findings("") == [] + assert _findings(None) == [] + + +def test_multiple_aggregations_in_one_expression(): + found = _findings( + "SUM ( Sales[Amount] ) + AVERAGE ( Sales[Discount] )" + ) + assert ("SUM", "Sales", "Amount") in found + assert ("AVERAGE", "Sales", "Discount") in found + assert len(found) == 2 diff --git a/tests/test_format_dax.py b/tests/test_format_dax.py new file mode 100644 index 000000000..b31d3c546 --- /dev/null +++ b/tests/test_format_dax.py @@ -0,0 +1,149 @@ +"""Interactive visual test for ``sempy_labs.dax.format_dax``. + +Usage: + # Run interactively (prompts for DAX input; end with Ctrl-D / EOF): + pytest -s tests/test_format_dax.py -k interactive + + # Or pass DAX via env var: + DAX_EXPR="EVALUATE { 1 + 2 }" pytest -s tests/test_format_dax.py -k env + + # Or run the file directly: + python tests/test_format_dax.py "EVALUATE { 1 + 2 }" + python tests/test_format_dax.py path/to/expression.dax + python tests/test_format_dax.py # then type DAX, end with Ctrl-D + +Prints the DAX to the terminal using ANSI 24-bit colors (the same palette +``format_dax`` uses for HTML) and also writes an HTML preview file next +to this script so you can open it in a browser. +""" + +from __future__ import annotations + +import os +import sys + +import pytest + +# Make ``src/`` importable when running this file directly without +# installing the package. +_HERE = os.path.dirname(os.path.abspath(__file__)) +_SRC = os.path.join(os.path.dirname(_HERE), "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +from sempy_labs.dax._format import ( # noqa: E402 + _COLORS, + _classify_tokens, + _render_html, +) + + +def _hex_to_rgb(hex_color: str) -> tuple[int, int, int]: + h = hex_color.lstrip("#") + return int(h[0:2], 16), int(h[2:4], 16), int(h[4:6], 16) + + +def _ansi(hex_color: str) -> str: + if not hex_color or hex_color == "inherit": + return "" + r, g, b = _hex_to_rgb(hex_color) + return f"\x1b[38;2;{r};{g};{b}m" + + +_RESET = "\x1b[0m" + + +def render_ansi(dax_expression: str) -> str: + """Render ``dax_expression`` as an ANSI-colored string for terminal output.""" + + classified = _classify_tokens(dax_expression) + parts: list[str] = [] + cursor = 0 + + for token, kind in classified: + if token.position > cursor: + parts.append(dax_expression[cursor:token.position]) + + color = _COLORS.get(kind or "default", "inherit") + prefix = _ansi(color) + if prefix: + parts.append(f"{prefix}{token.text}{_RESET}") + else: + parts.append(token.text) + + cursor = token.position + len(token.text) + + if cursor < len(dax_expression): + parts.append(dax_expression[cursor:]) + + return "".join(parts) + + +def _read_stdin_dax() -> str: + print("Enter DAX statement (end with Ctrl-D / EOF):", file=sys.stderr) + return sys.stdin.read() + + +def _show(text: str, *, write_html: bool = True) -> None: + text = text.strip() + assert text, "No DAX expression provided." + + print("\n--- Input DAX ---") + print(text) + + print("\n--- format_dax (ANSI terminal preview) ---") + print(render_ansi(text)) + + if write_html: + out_path = os.path.join(_HERE, "format_dax_preview.html") + html = _render_html(text) + with open(out_path, "w", encoding="utf-8") as f: + f.write( + "" + "format_dax preview" + "" + "" + "

" + "Dark background

" + f"
" + f"{html}
" + "

" + "Light background

" + f"
{html}
" + "" + ) + print(f"\nHTML preview written to: {out_path}") + + +@pytest.mark.skipif( + not sys.stdin.isatty(), + reason="Interactive test; run with `pytest -s` from a terminal.", +) +def test_format_dax_interactive(): + _show(_read_stdin_dax()) + + +@pytest.mark.skipif( + "DAX_EXPR" not in os.environ, + reason="Set the DAX_EXPR environment variable to run this test.", +) +def test_format_dax_env(): + _show(os.environ["DAX_EXPR"]) + + +if __name__ == "__main__": + if len(sys.argv) > 1: + arg = sys.argv[1] + # If the single argument is a path to an existing file, read it; + # otherwise treat the joined arguments as the DAX expression itself. + if len(sys.argv) == 2 and os.path.isfile(arg): + with open(arg, "r", encoding="utf-8") as f: + dax = f.read() + else: + dax = " ".join(sys.argv[1:]) + else: + dax = _read_stdin_dax() + + _show(dax) diff --git a/tests/test_parse_dax.py b/tests/test_parse_dax.py new file mode 100644 index 000000000..ba806f46f --- /dev/null +++ b/tests/test_parse_dax.py @@ -0,0 +1,55 @@ +"""Interactive test for parse_dax().dump(). + +Usage: + # Run interactively (prompts for DAX input; end with Ctrl-D / EOF): + pytest -s tests/test_parse_dax.py -k interactive + + # Or pass DAX via env var: + DAX_EXPR="EVALUATE { 1 + 2 }" pytest -s tests/test_parse_dax.py -k env + + # Or run the file directly: + python tests/test_parse_dax.py "EVALUATE { 1 + 2 }" + python tests/test_parse_dax.py # then type DAX, end with Ctrl-D +""" + +import os +import sys + +import pytest + +from sempy_labs.dax._parser import parse_dax + + +def _read_stdin_dax() -> str: + print("Enter DAX statement (end with Ctrl-D / EOF):", file=sys.stderr) + return sys.stdin.read() + + +def _dump(text: str) -> None: + text = text.strip() + assert text, "No DAX expression provided." + print("\n--- Input DAX ---") + print(text) + print("\n--- parse_dax().dump() ---") + parse_dax(text).dump() + + +@pytest.mark.skipif( + not sys.stdin.isatty(), + reason="Interactive test; run with `pytest -s` from a terminal.", +) +def test_parse_dax_interactive(): + _dump(_read_stdin_dax()) + + +@pytest.mark.skipif( + "DAX_EXPR" not in os.environ, + reason="Set the DAX_EXPR environment variable to run this test.", +) +def test_parse_dax_env(): + _dump(os.environ["DAX_EXPR"]) + + +if __name__ == "__main__": + dax = " ".join(sys.argv[1:]) if len(sys.argv) > 1 else _read_stdin_dax() + _dump(dax)