diff --git a/.agents/skills/java-checkstyle/SKILL.md b/.agents/skills/java-checkstyle/SKILL.md new file mode 100644 index 000000000000..30b46ec063b6 --- /dev/null +++ b/.agents/skills/java-checkstyle/SKILL.md @@ -0,0 +1,58 @@ +--- +name: java-checkstyle +description: Run `mvn spotless:apply` to fix Java checkstyle / formatting failures and verify the result. Run after authoring or modifying any `.java` files, or when CI reports a "Java checkstyle failed" / "Fix Java checkstyle" issue on a PR. +--- + +# Java Checkstyle / Spotless (Codex agent) + +OpenMetadata enforces Java formatting via the Spotless Maven plugin. Every CI +build runs `mvn spotless:check` and fails the PR if any file is not formatted. + +## When to activate + +- The user asks to "fix checkstyle", "fix Java formatting", "apply spotless", + "run spotless", "format Java", or similar. +- CI posts a `Java checkstyle failed` / `Fix Java checkstyle` comment on a PR + (the bot's exact phrasing is "Please run `mvn spotless:apply` in the root of + your repository and commit the changes to this PR"). +- After you have finished authoring or editing any `.java` files — before + opening a PR or pushing a commit that touches Java. + +## Procedure + +1. From the repo root run Spotless: + + ```bash + mvn spotless:apply # formats everything + # or + mvn -pl spotless:apply # scope to a single Maven module for speed + # or + mvn spotless:check # verify only, without rewriting files + ``` + + Spotless is fast (seconds, no compilation). If it fails with a plugin error + rather than a formatting diff, surface the error and stop — do not try to + hand-edit formatting around the failure. + +2. Inspect the diff: + + ```bash + git status --short + git diff --stat + ``` + + Expect changes only in `.java` (and possibly `pom.xml`) files. If Spotless + keeps rewriting a change you just made, re-read the root `pom.xml`'s + `spotless-maven-plugin` config — Spotless is the source of truth, not the + IDE. + +3. Only commit if the user asked to. Report the changed-file list first so the + user can decide whether to fold the reformat into the in-progress commit or + make a separate "Fix Java checkstyle" commit (matches the repo's existing + history for bot-triggered formatting-only commits). + +## Out of scope + +- UI / TypeScript formatting — use `yarn pretty` / ESLint flow (see AGENTS.md + UI section). +- Python formatting — use `make py_format` (black + isort + pycln). diff --git a/.agents/skills/ui-checkstyle/SKILL.md b/.agents/skills/ui-checkstyle/SKILL.md new file mode 100644 index 000000000000..bbdd4fdc7b99 --- /dev/null +++ b/.agents/skills/ui-checkstyle/SKILL.md @@ -0,0 +1,86 @@ +--- +name: ui-checkstyle +description: Run the ESLint + Prettier + organize-imports sequence that CI's `UI Checkstyle` jobs (`lint-src`, `lint-playwright`, `lint-core-components`) run — on just the files the PR changed — and fail if any file ends up with a diff. Run after authoring or modifying any `.ts`/`.tsx`/`.js`/`.jsx`/`.json` under `openmetadata-ui/src/main/resources/ui/src/`, `.../playwright/`, or `openmetadata-ui-core-components/src/main/resources/ui/src/`, or when CI reports a `UI Checkstyle` failure on a PR. +--- + +# UI Checkstyle / ESLint + Prettier + organize-imports (Codex agent) + +The `UI Checkstyle` workflow (`.github/workflows/ui-checkstyle.yml`) has three +per-area jobs — `lint-src`, `lint-playwright`, `lint-core-components`. Each +reformats the files changed in the PR and fails if the reformat produces a +diff, so the committed tree must already be formatted. + +## When to activate + +- The user asks to "fix UI checkstyle", "fix UI lint", "run prettier", "run + eslint", "fix UI format", or similar. +- CI posts a `UI Checkstyle / lint-src|lint-playwright|lint-core-components` + failure (the bot surfaces the modified files in the job summary). +- After you have finished authoring or editing any `.ts`/`.tsx`/`.js`/ + `.jsx`/`.json` under the three UI trees — before opening a PR or pushing + a commit that touches UI. + +## Procedure + +1. Build the file list for each affected area: + + ```bash + # repo root + git diff --name-only origin/main...HEAD -- \ + 'openmetadata-ui/src/main/resources/ui/src/**/*.{ts,tsx,js,jsx,json}' \ + | sed 's|openmetadata-ui/src/main/resources/ui/||' > /tmp/src_files.txt + + git diff --name-only origin/main...HEAD -- \ + 'openmetadata-ui/src/main/resources/ui/playwright/**/*.{ts,tsx,js,jsx}' \ + | sed 's|openmetadata-ui/src/main/resources/ui/||' > /tmp/pw_files.txt + + git diff --name-only origin/main...HEAD -- \ + 'openmetadata-ui-core-components/**/*.{ts,tsx,js,jsx,json}' \ + | sed 's|openmetadata-ui-core-components/src/main/resources/ui/||' \ + > /tmp/core_files.txt + ``` + + Skip any empty list — CI won't run that area's job either. + +2. From the matching working directory (`openmetadata-ui/src/main/resources/ui` + or `openmetadata-ui-core-components/src/main/resources/ui`), run the + three-step sequence that CI runs: + + ```bash + # 1) imports first + cat /tmp/src_files.txt | xargs ./node_modules/.bin/organize-imports-cli + + # 2) ESLint --fix + NODE_OPTIONS='--max-old-space-size=8192' cat /tmp/src_files.txt \ + | xargs ./node_modules/.bin/eslint --no-error-on-unmatched-pattern --fix + + # 3) prettier --write — MUST be last, because organize-imports-cli uses + # 4-space indentation and drops trailing commas; prettier restores them + # to the repo's 2-space + trailing-comma style. Reversing the order + # leaves CI with a dirty diff. + cat /tmp/src_files.txt \ + | xargs ./node_modules/.bin/prettier \ + --config './.prettierrc.yaml' --ignore-path './.prettierignore' \ + --write + ``` + + Core-components has no `organize-imports-cli` wired up — skip step 1 there. + +3. Check the diff from the repo root: + + ```bash + git status --short + git diff --stat + ``` + + If `git status --short` is empty you're done. Otherwise commit the + reformatting diff as its own `Fix UI checkstyle` commit, matching the + existing history for bot-triggered formatting-only commits — unless the + user asked you to fold it into the in-progress commit. + +## Out of scope + +- TypeScript type-check errors (`tsc`) — different jobs, different failure + modes, not auto-fixable by this skill. +- Java formatting — use the `java-checkstyle` skill (`mvn spotless:apply`). +- Python formatting — use `make py_format` (black + isort + pycln). diff --git a/.claude/skills/java-checkstyle/SKILL.md b/.claude/skills/java-checkstyle/SKILL.md new file mode 100644 index 000000000000..06958118ad48 --- /dev/null +++ b/.claude/skills/java-checkstyle/SKILL.md @@ -0,0 +1,83 @@ +--- +name: java-checkstyle +description: Run `mvn spotless:apply` to fix Java checkstyle / formatting failures and verify the result. Invoke after authoring or modifying any `.java` files, or when CI reports a "Java checkstyle failed" or "Fix Java checkstyle" issue on a PR. +user-invocable: true +argument-hint: "[-pl ] [--check]" +allowed-tools: + - Bash + - Read + - Grep + - Glob +--- + +# Java Checkstyle / Spotless + +OpenMetadata enforces Java formatting via the Spotless Maven plugin. Every CI +build runs `mvn spotless:check` and fails the PR if any file is not formatted. +This skill keeps the fix on a single, consistent command so reviewers never have +to ask for it manually again. + +## When to activate + +- The user asks to "fix checkstyle", "fix Java formatting", "apply spotless", + "run spotless", "format Java", or similar. +- CI posts a `Java checkstyle failed` / `Fix Java checkstyle` comment on a PR + (the project's bot phrases the instruction as "Please run + `mvn spotless:apply` in the root of your repository and commit the changes"). +- After the assistant has finished authoring or editing any `.java` files — + before opening a PR or pushing a commit that touches Java. + +## Arguments + +- No arguments: run `mvn spotless:apply` at the repo root across all modules. +- `-pl `: scope to a single Maven module (e.g. + `-pl openmetadata-service`). Useful when only one module changed and you want + a faster run. +- `--check`: run `mvn spotless:check` instead of `apply`. Use to confirm the + tree is clean without touching files (e.g. to verify before push). + +## Process + +### Step 1: Run Spotless + +From the repo root: + +```bash +mvn spotless:apply # default — formats everything +# or +mvn -pl spotless:apply # scoped to one module +# or +mvn spotless:check # verify only, don't write +``` + +Spotless is fast (seconds, no compilation). If it fails with a plugin error +(not a formatting diff), surface the error and stop — do not try to hand-edit +formatting around the failure. + +### Step 2: Check what changed + +```bash +git status --short +git diff --stat +``` + +Expect reformatting in `.java` files only. If Spotless touches `pom.xml` or +other non-Java files, that's also fine — Spotless is configured for those too +in this repo. + +### Step 3: Stage and commit (only if the user asked to commit) + +Do NOT auto-commit. Report the changed file list to the user and let them +decide whether to fold the formatting into the in-progress commit or make a +separate "Fix Java checkstyle" commit. Follow the repo convention: the +existing branch history already uses `Fix Java checkstyle` as the commit title +for bot-triggered formatting-only commits. + +## Notes + +- Spotless config lives in the root `pom.xml` (`spotless-maven-plugin` + section). Do not redefine formatting rules inline in source files. +- If Spotless keeps rewriting a change the user just made, re-read the config + — Spotless is the source of truth, not the IDE. +- The analogous UI command is `yarn pretty` (see the `test-locally` skill / + CLAUDE.md for the UI lint flow); this skill is Java-only. diff --git a/.claude/skills/ui-checkstyle/SKILL.md b/.claude/skills/ui-checkstyle/SKILL.md new file mode 100644 index 000000000000..34735e2b1549 --- /dev/null +++ b/.claude/skills/ui-checkstyle/SKILL.md @@ -0,0 +1,132 @@ +--- +name: ui-checkstyle +description: Run the exact ESLint + Prettier + organize-imports sequence that CI's `UI Checkstyle` jobs (`lint-src`, `lint-playwright`, `lint-core-components`) run — on just the files the PR changed — and fail the task if any file ends up with a diff. Invoke after authoring or modifying any `.ts`, `.tsx`, `.js`, `.jsx`, or `.json` file under `openmetadata-ui/src/main/resources/ui/src/`, `.../playwright/`, or `openmetadata-ui-core-components/src/main/resources/ui/src/`, or when CI reports a "UI Checkstyle" job failure on the PR. +user-invocable: true +argument-hint: "[--src] [--playwright] [--core-components] [--all] [--check]" +allowed-tools: + - Bash + - Read + - Grep + - Glob +--- + +# UI Checkstyle / ESLint + Prettier + organize-imports + +The `UI Checkstyle` GitHub workflow +(`.github/workflows/ui-checkstyle.yml`) runs three per-area jobs: +`lint-src` (`openmetadata-ui/src/main/resources/ui/src/...`), +`lint-playwright` (`.../playwright/...`), +`lint-core-components` +(`openmetadata-ui-core-components/src/main/resources/ui/src/...`). Each job +reformats only the files changed in the PR and fails if the reformat produces +any diff — i.e. the committed tree must already be formatted. + +This skill runs the same sequence locally so the CI never has to ask. + +## When to activate + +- The user asks to "fix UI checkstyle", "fix UI lint", "run prettier", "run + eslint", "fix the UI format", "apply UI format", or similar. +- CI posts a `UI Checkstyle / lint-src|lint-playwright|lint-core-components` + failure (the bot lists the modified files in the job summary). +- After the assistant has finished authoring or editing any `.ts`/`.tsx`/ + `.js`/`.jsx`/`.json` under the three UI trees — before opening a PR or + pushing a commit that touches UI. + +## Arguments + +- `--src` (default for files under `openmetadata-ui/.../ui/src/`) +- `--playwright` (files under `.../ui/playwright/`) +- `--core-components` (files under `openmetadata-ui-core-components/...`) +- `--all` — run all three areas +- `--check` — verify only: run the sequence in a dry-run pass and report + which files are still dirty, without writing. Useful before push. + +If invoked with no flag, auto-detect the affected areas from +`git diff --name-only origin/main...HEAD` and run only those. + +## Process + +### Step 1: Compute the file list + +For each area you are running against: + +```bash +# from the repo root +git diff --name-only origin/main...HEAD -- \ + 'openmetadata-ui/src/main/resources/ui/src/**/*.{ts,tsx,js,jsx,json}' \ + | sed 's|openmetadata-ui/src/main/resources/ui/||' > /tmp/src_files.txt + +git diff --name-only origin/main...HEAD -- \ + 'openmetadata-ui/src/main/resources/ui/playwright/**/*.{ts,tsx,js,jsx}' \ + | sed 's|openmetadata-ui/src/main/resources/ui/||' > /tmp/pw_files.txt + +git diff --name-only origin/main...HEAD -- \ + 'openmetadata-ui-core-components/**/*.{ts,tsx,js,jsx,json}' \ + | sed 's|openmetadata-ui-core-components/src/main/resources/ui/||' \ + > /tmp/core_files.txt +``` + +Skip any list that is empty — that area has no changes so the CI job for it +wouldn't run anyway. + +### Step 2: Run the CI sequence + +From the corresponding working directory: + +```bash +cd openmetadata-ui/src/main/resources/ui # or .../openmetadata-ui-core-components/src/main/resources/ui + +# 1) imports first — organize-imports-cli only exists for the ui module +cat /tmp/src_files.txt | xargs ./node_modules/.bin/organize-imports-cli + +# 2) eslint --fix (same flags CI uses) +NODE_OPTIONS='--max-old-space-size=8192' cat /tmp/src_files.txt \ + | xargs ./node_modules/.bin/eslint --no-error-on-unmatched-pattern --fix + +# 3) prettier --write — this MUST run after organize-imports because +# organize-imports uses 4-space indentation / drops trailing commas, +# and prettier then puts them back to the repo's 2-space + trailing-comma +# style. Running them in the other order leaves a dirty diff. +cat /tmp/src_files.txt \ + | xargs ./node_modules/.bin/prettier \ + --config './.prettierrc.yaml' --ignore-path './.prettierignore' \ + --write +``` + +For playwright, use the same three commands on `/tmp/pw_files.txt`. +For core-components, the organize-imports step is skipped (no CLI there) — +just eslint + prettier. + +### Step 3: Report what changed + +```bash +cd +git status --short # should list only .ts/.tsx/.js/.jsx/.json files +git diff --stat +``` + +If `git status --short` is empty, the tree is already clean — tell the user +and stop. + +### Step 4: Commit (only if the user asked to) + +Do NOT auto-commit. Surface the list of modified files to the user; they +decide whether to fold the reformat into the in-progress commit or create a +dedicated "Fix UI checkstyle" commit (matches the repo's existing history for +bot-triggered formatting-only commits). + +## Notes + +- The `--check` mode mirrors CI's behavior: run the three commands and then + verify `git status --short` is empty. Revert any writes before exiting so + the user's working tree isn't touched. +- If ESLint reports hard errors (not warnings, not auto-fixable), stop and + surface them — they need a real code change, not a format pass. Warnings + (e.g. `playwright/no-wait-for-selector`) don't fail CI and can be left. +- The analogous Java command is `mvn spotless:apply` — see the + `java-checkstyle` skill. +- TypeScript type-check errors (`tsc`) are a separate concern and are + *not* fixed by this skill — the `tsc-src` / `tsc-playwright` jobs are + currently either skipped or have their own failures surfaced via the CI + report. diff --git a/AGENTS.md b/AGENTS.md index 6ca342f0dce2..0219b1f3d56c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -193,7 +193,12 @@ yarn parse-schema # Parse JSON schemas for frontend (connection and - If the code needs a comment to be understood, refactor the code to be clearer instead ### Java Code Requirements -- **Always mention** running `mvn spotless:apply` when generating/modifying .java files +- **Always run `mvn spotless:apply`** before finishing any task that touched + `.java` files. CI runs `mvn spotless:check` and will fail the PR otherwise + (bot's exact phrasing: "Please run `mvn spotless:apply` in the root of your + repository and commit the changes to this PR"). Scope with `-pl ` + for speed if only one module changed. A reusable procedure is written up at + `.agents/skills/java-checkstyle/SKILL.md`. - Use clear, descriptive variable and method names instead of comments - Follow existing project patterns and conventions - Generate production-ready code, not tutorial code @@ -202,6 +207,14 @@ yarn parse-schema # Parse JSON schemas for frontend (connection and - Do not import wild-card packages instead import exactly required packages ### TypeScript/Frontend Code Requirements +- **Always run the UI checkstyle sequence** before finishing any task that + touched `.ts`/`.tsx`/`.js`/`.jsx`/`.json` under + `openmetadata-ui/src/main/resources/ui/src/`, `.../playwright/`, or + `openmetadata-ui-core-components/src/main/resources/ui/src/`. CI's + `UI Checkstyle / lint-src|lint-playwright|lint-core-components` jobs fail + the PR otherwise. Order matters: `organize-imports-cli` → `eslint --fix` → + `prettier --write`. A reusable procedure lives at + `.agents/skills/ui-checkstyle/SKILL.md`. - **NEVER use `any` type** in TypeScript code - always use proper types - Use `unknown` when the type is truly unknown and add type guards - Import types from existing type definitions (e.g., `RJSFSchema` from `@rjsf/utils`) diff --git a/CLAUDE.md b/CLAUDE.md index b87f85edeeee..2aef0dd61836 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -283,7 +283,14 @@ yarn parse-schema # Parse JSON schemas for frontend (connection and ### Java Code Requirements -**Always run `mvn spotless:apply` when generating/modifying .java files.** +**Always run `mvn spotless:apply` before you finish any task that touched +`.java` files.** CI runs `mvn spotless:check` and will fail the PR otherwise — +the bot's exact suggestion is "Please run `mvn spotless:apply` in the root of +your repository and commit the changes to this PR." Scope the run with +`-pl ` for speed if only one module changed. When asked to "fix +checkstyle" / "fix Java formatting" / "apply spotless", invoke the +`java-checkstyle` skill (see `.claude/skills/java-checkstyle/`) rather than +hand-editing formatting. #### Method Size and Complexity (Kafka-Grade Standards) - **Methods must be 15 lines or fewer** (excluding blank lines and braces). If a method is longer, break it into smaller focused methods with descriptive names. @@ -421,6 +428,19 @@ yarn parse-schema # Parse JSON schemas for frontend (connection and - One statement per line — no `if (x) return y;` on one line ### TypeScript/Frontend Code Requirements + +**Always run the UI checkstyle sequence before you finish any task that +touched `.ts`/`.tsx`/`.js`/`.jsx`/`.json` under +`openmetadata-ui/src/main/resources/ui/src/`, `.../playwright/`, or +`openmetadata-ui-core-components/src/main/resources/ui/src/`.** CI's +`UI Checkstyle / lint-src|lint-playwright|lint-core-components` jobs fail the +PR otherwise. The order matters — run `organize-imports-cli`, then +`eslint --fix`, then `prettier --write`; reversing organize-imports and +prettier leaves a dirty diff (organize-imports uses 4-space indentation, +prettier uses 2 + trailing commas). When asked to "fix UI checkstyle" / "run +prettier" / "fix UI lint", invoke the `ui-checkstyle` skill (see +`.claude/skills/ui-checkstyle/`) rather than hand-editing formatting. + - **NEVER use `any` type** in TypeScript code - always use proper types - Use `unknown` when the type is truly unknown and add type guards - Import types from existing type definitions (e.g., `RJSFSchema` from `@rjsf/utils`) diff --git a/bootstrap/sql/migrations/native/2.0.0/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/2.0.0/mysql/schemaChanges.sql index 576f7b739682..85c8362622b9 100644 --- a/bootstrap/sql/migrations/native/2.0.0/mysql/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/2.0.0/mysql/schemaChanges.sql @@ -135,3 +135,166 @@ CREATE TABLE IF NOT EXISTS task_form_schema_entity ( KEY idx_task_form_schema_task_type (taskType), KEY idx_task_form_schema_deleted (deleted) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- ===================================================== +-- KNOWLEDGE CENTER + CONTEXT CENTER DRIVE (Collate → OM port) +-- Appended below the Task Redesign tables to preserve main's +-- migration order when merging. +-- ===================================================== + +-- MCP tables are created in 1.13.0 migration. + +-- Knowledge Center: page entity table (Article, QuickLink). +-- Existing Collate customers already have this table from 1.2.0-collate with +-- subsequent shape changes through 1.6.0-collate (nameHash -> fqnHash VARCHAR(756), +-- pageType generated column, composite deleted index). CREATE TABLE IF NOT EXISTS +-- is a no-op for them and creates the final shape for fresh OpenMetadata installs. +CREATE TABLE IF NOT EXISTS knowledge_center ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, + fqnHash VARCHAR(756) NOT NULL COLLATE ascii_bin, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') STORED NOT NULL, + json JSON NOT NULL, + updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') STORED NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') STORED NOT NULL, + deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted') STORED, + pageType VARCHAR(16) GENERATED ALWAYS AS (json ->> '$.pageType') STORED NOT NULL, + PRIMARY KEY (id), + UNIQUE (fqnHash), + INDEX knowledge_center_name_index (name), + INDEX index_knowledge_center_deleted (fqnHash, deleted) +); + +-- Context Center Drive: Folder entity table. +CREATE TABLE IF NOT EXISTS drive_folder ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') STORED NOT NULL, + nameHash VARCHAR(256) NOT NULL COLLATE ascii_bin, + json JSON NOT NULL, + updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') STORED NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') STORED NOT NULL, + deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted') STORED, + PRIMARY KEY (id), + UNIQUE KEY unique_drive_folder_name (nameHash), + INDEX idx_drive_folder_updated_at (updatedAt) +) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- Context Center Drive: File entity table (uploaded PDF/image/spreadsheet/office docs). +CREATE TABLE IF NOT EXISTS context_file ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') STORED NOT NULL, + nameHash VARCHAR(256) NOT NULL COLLATE ascii_bin, + json JSON NOT NULL, + updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') STORED NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') STORED NOT NULL, + deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted') STORED, + PRIMARY KEY (id), + UNIQUE KEY unique_context_file_name (nameHash), + INDEX idx_context_file_updated_at (updatedAt) +) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- Attachments: Asset entity table for uploaded file blobs referenced by ContextFiles, Pages, etc. +-- Existing Collate customers have this from 1.7.0-collate. CREATE TABLE IF NOT EXISTS is a no-op for them. +CREATE TABLE IF NOT EXISTS asset_entity ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fileName') STORED NOT NULL, + url VARCHAR(1024) GENERATED ALWAYS AS (json ->> '$.url') STORED NOT NULL, + fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.fullyQualifiedName') STORED NOT NULL, + assetType VARCHAR(100) GENERATED ALWAYS AS (json ->> '$.assetType') STORED NOT NULL, + json JSON NOT NULL, + updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') STORED NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') STORED NOT NULL, + fqnHash VARCHAR(768) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL, + deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted') STORED, + PRIMARY KEY (id), + INDEX fqnhash_index (fqnHash), + INDEX asset_type_index (assetType), + INDEX idx_asset_deleted (deleted) +); + +-- Context Center Drive: File content snapshot table (revisions, extracted text). +CREATE TABLE IF NOT EXISTS context_file_content ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') STORED NOT NULL, + nameHash VARCHAR(256) NOT NULL COLLATE ascii_bin, + json JSON NOT NULL, + updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') STORED NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') STORED NOT NULL, + deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted') STORED, + PRIMARY KEY (id), + UNIQUE KEY unique_context_file_content_name (nameHash), + INDEX idx_context_file_content_updated_at (updatedAt) +) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- Add tag_usage.metadata column if missing (newer tag usage payloads carry metadata). +SET @ddl = ( + SELECT IF( + EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = DATABASE() + AND table_name = 'tag_usage' + AND column_name = 'metadata' + ), + 'SELECT 1', + 'ALTER TABLE tag_usage ADD COLUMN metadata JSON NULL' + ) +); +PREPARE stmt FROM @ddl; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +-- Add audit_log_event.search_text column if missing (searchable audit log text). +SET @ddl = ( + SELECT IF( + EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = DATABASE() + AND table_name = 'audit_log_event' + AND column_name = 'search_text' + ), + 'SELECT 1', + 'ALTER TABLE audit_log_event ADD COLUMN search_text LONGTEXT NULL' + ) +); +PREPARE stmt FROM @ddl; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +-- Distributed reindex job tracking. +CREATE TABLE IF NOT EXISTS search_index_job ( + id VARCHAR(64) NOT NULL, + status VARCHAR(64) NOT NULL, + jobConfiguration JSON NOT NULL, + targetIndexPrefix VARCHAR(256) NOT NULL, + stagedIndexMapping JSON DEFAULT NULL, + totalRecords BIGINT NOT NULL DEFAULT 0, + processedRecords BIGINT NOT NULL DEFAULT 0, + successRecords BIGINT NOT NULL DEFAULT 0, + failedRecords BIGINT NOT NULL DEFAULT 0, + stats JSON NOT NULL, + createdBy VARCHAR(256) NOT NULL, + createdAt BIGINT NOT NULL, + startedAt BIGINT DEFAULT NULL, + completedAt BIGINT DEFAULT NULL, + updatedAt BIGINT NOT NULL, + errorMessage LONGTEXT DEFAULT NULL, + registrationDeadline BIGINT DEFAULT NULL, + registeredServerCount INT DEFAULT NULL, + PRIMARY KEY (id), + KEY idx_search_index_job_status_created_at (status, createdAt DESC) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- Retry queue for failed search-index writes. +CREATE TABLE IF NOT EXISTS search_index_retry_queue ( + entityId VARCHAR(64) NOT NULL, + entityFqn VARCHAR(700) NOT NULL, + failureReason LONGTEXT DEFAULT NULL, + status VARCHAR(64) NOT NULL, + entityType VARCHAR(128) NOT NULL, + retryCount INT NOT NULL DEFAULT 0, + claimedAt TIMESTAMP NULL DEFAULT NULL, + PRIMARY KEY (entityId, entityFqn), + KEY idx_search_index_retry_queue_status (status), + KEY idx_search_index_retry_queue_claimed_at (claimedAt) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; diff --git a/bootstrap/sql/migrations/native/2.0.0/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/2.0.0/postgres/schemaChanges.sql index 288caf8cf0be..5b8b6a953f97 100644 --- a/bootstrap/sql/migrations/native/2.0.0/postgres/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/2.0.0/postgres/schemaChanges.sql @@ -140,3 +140,140 @@ CREATE TABLE IF NOT EXISTS task_form_schema_entity ( CREATE INDEX IF NOT EXISTS idx_task_form_schema_name ON task_form_schema_entity (name); CREATE INDEX IF NOT EXISTS idx_task_form_schema_tasktype ON task_form_schema_entity (tasktype); CREATE INDEX IF NOT EXISTS idx_task_form_schema_deleted ON task_form_schema_entity (deleted); + +-- ===================================================== +-- KNOWLEDGE CENTER + CONTEXT CENTER DRIVE (Collate → OM port) +-- Appended below the Task Redesign tables to preserve main's +-- migration order when merging. +-- ===================================================== + +-- MCP tables are created in 1.13.0 migration. + +-- Knowledge Center: page entity table (Article, QuickLink). +-- Existing Collate customers already have this table from 1.2.0-collate with +-- subsequent shape changes through 1.6.0-collate (nameHash -> fqnHash VARCHAR(756), +-- pageType generated column, composite deleted index). CREATE TABLE IF NOT EXISTS +-- is a no-op for them and creates the final shape for fresh OpenMetadata installs. +CREATE TABLE IF NOT EXISTS knowledge_center ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, + fqnHash VARCHAR(756) NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, + json JSONB NOT NULL, + updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, + deleted BOOLEAN GENERATED ALWAYS AS (COALESCE((json ->> 'deleted')::boolean, false)) STORED, + pageType VARCHAR(16) GENERATED ALWAYS AS (json ->> 'pageType') STORED NOT NULL, + PRIMARY KEY (id), + UNIQUE (fqnHash) +); +CREATE INDEX IF NOT EXISTS knowledge_center_name_index ON knowledge_center (name); +CREATE INDEX IF NOT EXISTS index_knowledge_center_deleted ON knowledge_center (fqnHash, deleted); + +-- Context Center Drive: Folder entity table. +CREATE TABLE IF NOT EXISTS drive_folder ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, + nameHash VARCHAR(256) NOT NULL, + json JSONB NOT NULL, + updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, + deleted BOOLEAN GENERATED ALWAYS AS (COALESCE((json ->> 'deleted')::boolean, false)) STORED, + PRIMARY KEY (id), + UNIQUE (nameHash) +); +CREATE INDEX IF NOT EXISTS idx_drive_folder_updated_at ON drive_folder (updatedAt); + +-- Context Center Drive: File entity table (uploaded PDF/image/spreadsheet/office docs). +CREATE TABLE IF NOT EXISTS context_file ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, + nameHash VARCHAR(256) NOT NULL, + json JSONB NOT NULL, + updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, + deleted BOOLEAN GENERATED ALWAYS AS (COALESCE((json ->> 'deleted')::boolean, false)) STORED, + PRIMARY KEY (id), + UNIQUE (nameHash) +); +CREATE INDEX IF NOT EXISTS idx_context_file_updated_at ON context_file (updatedAt); + +-- Attachments: Asset entity table for uploaded file blobs referenced by ContextFiles, Pages, etc. +-- Existing Collate customers have this from 1.7.0-collate. CREATE TABLE IF NOT EXISTS is a no-op for them. +CREATE TABLE IF NOT EXISTS asset_entity ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fileName') STORED NOT NULL, + url VARCHAR(1024) GENERATED ALWAYS AS (json ->> 'url') STORED NOT NULL, + fullyQualifiedName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'fullyQualifiedName') STORED NOT NULL, + assetType VARCHAR(100) GENERATED ALWAYS AS (json ->> 'assetType') STORED NOT NULL, + json JSONB NOT NULL, + updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, + fqnHash VARCHAR(768) NOT NULL, + deleted BOOLEAN GENERATED ALWAYS AS (COALESCE(CAST(json ->> 'deleted' AS BOOLEAN), false)) STORED, + PRIMARY KEY (id) +); +CREATE INDEX IF NOT EXISTS fqnhash_index ON asset_entity (fqnHash); +CREATE INDEX IF NOT EXISTS asset_type_index ON asset_entity (assetType); +CREATE INDEX IF NOT EXISTS idx_asset_deleted ON asset_entity (deleted); + +-- Context Center Drive: File content snapshot table (revisions, extracted text). +CREATE TABLE IF NOT EXISTS context_file_content ( + id VARCHAR(36) GENERATED ALWAYS AS (json ->> 'id') STORED NOT NULL, + name VARCHAR(256) GENERATED ALWAYS AS (json ->> 'name') STORED NOT NULL, + nameHash VARCHAR(256) NOT NULL, + json JSONB NOT NULL, + updatedAt BIGINT GENERATED ALWAYS AS ((json ->> 'updatedAt')::bigint) STORED NOT NULL, + updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> 'updatedBy') STORED NOT NULL, + deleted BOOLEAN GENERATED ALWAYS AS (COALESCE((json ->> 'deleted')::boolean, false)) STORED, + PRIMARY KEY (id), + UNIQUE (nameHash) +); +CREATE INDEX IF NOT EXISTS idx_context_file_content_updated_at ON context_file_content (updatedAt); + +-- Add tag_usage.metadata column if missing (newer tag usage payloads carry metadata). +ALTER TABLE IF EXISTS tag_usage + ADD COLUMN IF NOT EXISTS metadata JSONB; + +-- Add audit_log_event.search_text column if missing (searchable audit log text). +ALTER TABLE IF EXISTS audit_log_event + ADD COLUMN IF NOT EXISTS search_text TEXT; + +-- Distributed reindex job tracking. +CREATE TABLE IF NOT EXISTS search_index_job ( + id VARCHAR(64) PRIMARY KEY, + status VARCHAR(64) NOT NULL, + jobConfiguration JSONB NOT NULL, + targetIndexPrefix VARCHAR(256) NOT NULL, + stagedIndexMapping JSONB NULL, + totalRecords BIGINT NOT NULL DEFAULT 0, + processedRecords BIGINT NOT NULL DEFAULT 0, + successRecords BIGINT NOT NULL DEFAULT 0, + failedRecords BIGINT NOT NULL DEFAULT 0, + stats JSONB NOT NULL DEFAULT '{}'::jsonb, + createdBy VARCHAR(256) NOT NULL, + createdAt BIGINT NOT NULL, + startedAt BIGINT NULL, + completedAt BIGINT NULL, + updatedAt BIGINT NOT NULL, + errorMessage TEXT NULL, + registrationDeadline BIGINT NULL, + registeredServerCount INTEGER NULL +); +CREATE INDEX IF NOT EXISTS idx_search_index_job_status_created_at + ON search_index_job (status, createdAt DESC); + +-- Retry queue for failed search-index writes. +CREATE TABLE IF NOT EXISTS search_index_retry_queue ( + entityId VARCHAR(64) NOT NULL, + entityFqn VARCHAR(768) NOT NULL, + failureReason TEXT NULL, + status VARCHAR(64) NOT NULL, + entityType VARCHAR(128) NOT NULL, + retryCount INTEGER NOT NULL DEFAULT 0, + claimedAt TIMESTAMP NULL, + PRIMARY KEY (entityId, entityFqn) +); +CREATE INDEX IF NOT EXISTS idx_search_index_retry_queue_status + ON search_index_retry_queue (status); +CREATE INDEX IF NOT EXISTS idx_search_index_retry_queue_claimed_at + ON search_index_retry_queue (claimedAt); diff --git a/openmetadata-integration-tests/pom.xml b/openmetadata-integration-tests/pom.xml index 4f802f987024..be778a39420f 100644 --- a/openmetadata-integration-tests/pom.xml +++ b/openmetadata-integration-tests/pom.xml @@ -28,6 +28,15 @@ + + + org.apache.commons + commons-compress + 1.27.1 + test + org.open-metadata @@ -160,6 +169,48 @@ 4.4.0 test + + + jakarta.ws.rs + jakarta.ws.rs-api + 3.1.0 + test + + + org.glassfish.jersey.core + jersey-client + 3.1.9 + test + + + org.glassfish.jersey.connectors + jersey-apache-connector + 3.1.9 + test + + + org.apache.httpcomponents + httpclient + 4.5.14 + test + + + jakarta.json + jakarta.json-api + 2.1.3 + test + + + org.eclipse.parsson + parsson + 1.1.7 + test + org.apache.jena diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/bootstrap/TestSuiteBootstrap.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/bootstrap/TestSuiteBootstrap.java index e5507bee3ad0..0ad8a0953aed 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/bootstrap/TestSuiteBootstrap.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/bootstrap/TestSuiteBootstrap.java @@ -136,6 +136,7 @@ public class TestSuiteBootstrap implements LauncherSessionListener { private static GenericContainer FUSEKI_CONTAINER; private static GenericContainer REDIS_CONTAINER; private static K3sContainer K3S_CONTAINER; + private static GenericContainer MINIO_CONTAINER; private static DropwizardAppExtension APP; private static Jdbi jdbi; @@ -556,6 +557,17 @@ private void startApplication() throws Exception { createIndices(); + // Start MinIO before app boot if object storage is configured to use S3 so that the + // S3AssetService picks up the correct endpoint. + if (config.getObjectStorage() != null + && config.getObjectStorage().isEnabled() + && "s3".equalsIgnoreCase(config.getObjectStorage().getProvider())) { + setupMinIO(); + if (config.getObjectStorage().getS3Configuration() != null) { + config.getObjectStorage().getS3Configuration().setEndpoint(getMinIOEndpoint()); + } + } + // Start the application APP.before(); @@ -820,6 +832,79 @@ private void cleanup() { } catch (Exception e) { LOG.warn("Error stopping database container", e); } + + try { + if (MINIO_CONTAINER != null) { + MINIO_CONTAINER.stop(); + } + } catch (Exception e) { + LOG.warn("Error stopping MinIO container", e); + } + } + + // === On-demand MinIO container for object-storage tests === + + public static synchronized void setupMinIO() { + if (MINIO_CONTAINER != null && MINIO_CONTAINER.isRunning()) { + LOG.info("MinIO already running at {}", getMinIOEndpoint()); + return; + } + LOG.info("Starting MinIO Testcontainer on-demand..."); + // Pin the MinIO image to a known-good release so a newly-published :latest tag + // cannot break integration tests without a code change. + MINIO_CONTAINER = + new GenericContainer<>("minio/minio:RELEASE.2024-01-16T16-07-38Z") + .withExposedPorts(9000) + .withEnv("MINIO_ROOT_USER", "minio") + .withEnv("MINIO_ROOT_PASSWORD", "minio123") + .withCommand("server /data") + .waitingFor( + Wait.forHttp("/minio/health/live") + .forPort(9000) + .forStatusCode(200) + .withStartupTimeout(java.time.Duration.ofSeconds(60))); + MINIO_CONTAINER.start(); + + String endpoint = getMinIOEndpoint(); + + // Create the default test bucket so tests can upload immediately. + software.amazon.awssdk.services.s3.S3Client s3 = + software.amazon.awssdk.services.s3.S3Client.builder() + .region(software.amazon.awssdk.regions.Region.US_EAST_1) + .credentialsProvider( + software.amazon.awssdk.auth.credentials.StaticCredentialsProvider.create( + software.amazon.awssdk.auth.credentials.AwsBasicCredentials.create( + "minio", "minio123"))) + .endpointOverride(java.net.URI.create(endpoint)) + .serviceConfiguration( + software.amazon.awssdk.services.s3.S3Configuration.builder() + .pathStyleAccessEnabled(true) + .build()) + .build(); + try { + boolean exists = + s3.listBuckets().buckets().stream().anyMatch(b -> b.name().equals("test-bucket")); + if (!exists) { + s3.createBucket( + software.amazon.awssdk.services.s3.model.CreateBucketRequest.builder() + .bucket("test-bucket") + .build()); + } + } finally { + s3.close(); + } + + // Expose endpoint to tests that read a system property / env var. + System.setProperty("IT_MINIO_ENDPOINT", endpoint); + + LOG.info("MinIO started at {}", endpoint); + } + + public static String getMinIOEndpoint() { + if (MINIO_CONTAINER == null || !MINIO_CONTAINER.isRunning()) { + throw new IllegalStateException("MinIO container not running. Call setupMinIO() first."); + } + return "http://" + MINIO_CONTAINER.getHost() + ":" + MINIO_CONTAINER.getMappedPort(9000); } // === Static accessor methods for tests === diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/drive/ContextFileIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/drive/ContextFileIT.java new file mode 100644 index 000000000000..7ed7440e07e7 --- /dev/null +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/drive/ContextFileIT.java @@ -0,0 +1,408 @@ +package org.openmetadata.it.drive; + +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import jakarta.ws.rs.core.Response; +import java.time.Duration; +import java.util.List; +import java.util.UUID; +import org.apache.http.client.HttpResponseException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.openmetadata.schema.api.data.CreateContextFile; +import org.openmetadata.schema.api.data.CreateFolder; +import org.openmetadata.schema.entity.data.ContextFile; +import org.openmetadata.schema.entity.data.ContextFileSourceType; +import org.openmetadata.schema.entity.data.ContextFileType; +import org.openmetadata.schema.entity.data.Folder; +import org.openmetadata.schema.entity.data.ProcessingStatus; +import org.openmetadata.schema.entity.teams.User; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.sdk.test.util.RestClient; +import org.openmetadata.sdk.test.util.TestNamespace; +import org.openmetadata.sdk.test.util.TestNamespaceExtension; + +@ExtendWith(TestNamespaceExtension.class) +class ContextFileIT { + + private static final String FILE_PATH = "v1/drive/files"; + private static final String FOLDER_PATH = "v1/drive/folders"; + + private ContextFile createFile(RestClient rest, CreateContextFile request) + throws HttpResponseException { + return rest.create(FILE_PATH, request, ContextFile.class); + } + + private ContextFile getFile(RestClient rest, UUID id, String fields) + throws HttpResponseException { + return rest.getById(FILE_PATH, id, fields, ContextFile.class); + } + + private Folder createFolder(RestClient rest, CreateFolder request) throws HttpResponseException { + return rest.create(FOLDER_PATH, request, Folder.class); + } + + // --- CRUD --- + + @Test + void testCreateContextFile(TestNamespace ns) throws HttpResponseException { + RestClient rest = RestClient.admin(); + + CreateContextFile create = + new CreateContextFile() + .withName(ns.prefix("report-pdf")) + .withDisplayName("Annual Report 2023") + .withFileType(ContextFileType.PDF) + .withFileSize(4200000) + .withContentType("application/pdf") + .withFileExtension("pdf") + .withProcessingStatus(ProcessingStatus.Uploaded); + + ContextFile file = createFile(rest, create); + assertNotNull(file.getId()); + assertEquals("Annual Report 2023", file.getDisplayName()); + assertEquals(ContextFileType.PDF, file.getFileType()); + assertEquals(4200000, file.getFileSize().intValue()); + } + + @Test + void testCreateSpreadsheet(TestNamespace ns) throws HttpResponseException { + RestClient rest = RestClient.admin(); + + CreateContextFile create = + new CreateContextFile() + .withName(ns.prefix("pricing-xlsx")) + .withDisplayName("Product Pricing") + .withFileType(ContextFileType.Spreadsheet) + .withFileSize(128000) + .withContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") + .withFileExtension("xlsx") + .withProcessingStatus(ProcessingStatus.Uploaded); + + ContextFile file = createFile(rest, create); + assertEquals(ContextFileType.Spreadsheet, file.getFileType()); + } + + @Test + void testGetFileById(TestNamespace ns) throws HttpResponseException { + RestClient rest = RestClient.admin(); + + ContextFile created = + createFile( + rest, + new CreateContextFile() + .withName(ns.prefix("get-test")) + .withFileType(ContextFileType.CSV) + .withProcessingStatus(ProcessingStatus.Uploaded)); + + ContextFile fetched = getFile(rest, created.getId(), ""); + assertEquals(created.getId(), fetched.getId()); + assertEquals(ContextFileType.CSV, fetched.getFileType()); + } + + @Test + void testDeleteFile(TestNamespace ns) throws HttpResponseException { + RestClient rest = RestClient.admin(); + + ContextFile file = + createFile( + rest, + new CreateContextFile() + .withName(ns.prefix("delete-test")) + .withProcessingStatus(ProcessingStatus.Uploaded)); + + rest.delete(FILE_PATH, file.getId()); + + HttpResponseException ex = + assertThrows(HttpResponseException.class, () -> getFile(rest, file.getId(), "")); + assertEquals(404, ex.getStatusCode()); + + try (Response deletedResponse = rest.rawGet(FILE_PATH + "/" + file.getId() + "?include=all")) { + assertEquals(200, deletedResponse.getStatus()); + assertTrue(deletedResponse.readEntity(String.class).contains("\"deleted\":true")); + } + } + + @Test + void testRestoreSoftDeletedFile(TestNamespace ns) throws HttpResponseException { + RestClient rest = RestClient.admin(); + + ContextFile file = + createFile( + rest, + new CreateContextFile() + .withName(ns.prefix("restore-test")) + .withProcessingStatus(ProcessingStatus.Uploaded)); + + rest.delete(FILE_PATH, file.getId()); + ContextFile restored = rest.restore(FILE_PATH, file.getId(), ContextFile.class); + + assertEquals(file.getId(), restored.getId()); + assertTrue(!Boolean.TRUE.equals(restored.getDeleted())); + assertEquals(file.getId(), getFile(rest, file.getId(), "").getId()); + } + + @Test + void testHardDeleteFileIsAsync(TestNamespace ns) throws HttpResponseException { + RestClient rest = RestClient.admin(); + + ContextFile file = + createFile( + rest, + new CreateContextFile() + .withName(ns.prefix("perm-delete-test")) + .withProcessingStatus(ProcessingStatus.Uploaded)); + + try (Response deleteResponse = + rest.rawDelete(FILE_PATH + "/" + file.getId() + "?hardDelete=true")) { + assertEquals(202, deleteResponse.getStatus()); + assertTrue(deleteResponse.readEntity(String.class).contains("\"hardDelete\":true")); + } + + await() + .atMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + try (Response deletedResponse = + rest.rawGet(FILE_PATH + "/" + file.getId() + "?include=all")) { + assertEquals(404, deletedResponse.getStatus()); + } + }); + } + + // --- File in Folder --- + + @Test + void testFileInFolder(TestNamespace ns) throws HttpResponseException { + RestClient rest = RestClient.admin(); + + Folder folder = createFolder(rest, new CreateFolder().withName(ns.prefix("docs-folder"))); + + ContextFile file = + createFile( + rest, + new CreateContextFile() + .withName(ns.prefix("file-in-folder")) + .withDisplayName("Report in Folder") + .withFileType(ContextFileType.PDF) + .withFolder(folder.getFullyQualifiedName()) + .withProcessingStatus(ProcessingStatus.Uploaded)); + + ContextFile fetched = getFile(rest, file.getId(), "folder"); + assertNotNull(fetched.getFolder()); + assertEquals(folder.getId(), fetched.getFolder().getId()); + + // FQN should include folder name + assertTrue( + fetched.getFullyQualifiedName().contains(folder.getName()), + "File FQN should include folder name"); + } + + @Test + void testFileInNestedFolder(TestNamespace ns) throws HttpResponseException { + RestClient rest = RestClient.admin(); + + Folder root = createFolder(rest, new CreateFolder().withName(ns.prefix("root"))); + Folder child = + createFolder( + rest, + new CreateFolder() + .withName(ns.prefix("child")) + .withParent(root.getFullyQualifiedName())); + + ContextFile file = + createFile( + rest, + new CreateContextFile() + .withName(ns.prefix("deep-file")) + .withFolder(child.getFullyQualifiedName()) + .withProcessingStatus(ProcessingStatus.Uploaded)); + + ContextFile fetched = getFile(rest, file.getId(), "folder"); + assertTrue( + fetched.getFullyQualifiedName().contains(root.getName()), + "File FQN should contain root folder"); + assertTrue( + fetched.getFullyQualifiedName().contains(child.getName()), + "File FQN should contain child folder"); + } + + // --- Source Provenance --- + + @Test + void testFileSourceProvenance(TestNamespace ns) throws HttpResponseException { + RestClient rest = RestClient.admin(); + + CreateContextFile create = + new CreateContextFile() + .withName(ns.prefix("synced-file")) + .withFileType(ContextFileType.Document) + .withSourceType(ContextFileSourceType.Confluence) + .withSourceId("page-12345") + .withSourceUrl(java.net.URI.create("https://wiki.example.com/page/12345")) + .withProcessingStatus(ProcessingStatus.Processed); + + ContextFile file = createFile(rest, create); + assertEquals(ContextFileSourceType.Confluence, file.getSourceType()); + assertEquals("page-12345", file.getSourceId()); + } + + // --- Processing Status Update --- + + @Test + void testUpdateProcessingStatus(TestNamespace ns) throws HttpResponseException { + RestClient rest = RestClient.admin(); + + ContextFile file = + createFile( + rest, + new CreateContextFile() + .withName(ns.prefix("status-test")) + .withFileType(ContextFileType.PDF) + .withProcessingStatus(ProcessingStatus.Uploaded)); + + assertEquals(ProcessingStatus.Uploaded, file.getProcessingStatus()); + + // Patch to Processed + String original = JsonUtils.pojoToJson(file); + file.setProcessingStatus(ProcessingStatus.Processed); + ContextFile updated = rest.patch(FILE_PATH, file.getId(), original, file, ContextFile.class); + + assertEquals(ProcessingStatus.Processed, updated.getProcessingStatus()); + } + + // --- Permissions --- + + @Test + void testUnprivilegedUserCannotDeleteFile(TestNamespace ns) throws HttpResponseException { + RestClient adminRest = RestClient.admin(); + User owner = DriveTestUsers.createUser(ns, "file-owner"); + + ContextFile file = + createFile( + adminRest, + new CreateContextFile() + .withName(ns.prefix("perm-delete")) + .withFileType(ContextFileType.PDF) + .withProcessingStatus(ProcessingStatus.Uploaded) + .withOwners(List.of(owner.getEntityReference()))); + + RestClient consumerRest = RestClient.forUser("test@open-metadata.org", new String[] {}); + + HttpResponseException ex = + assertThrows( + HttpResponseException.class, () -> consumerRest.hardDelete(FILE_PATH, file.getId())); + + assertTrue( + ex.getStatusCode() == 403 || ex.getStatusCode() == 401, + "Expected 403/401, got " + ex.getStatusCode()); + } + + @Test + void testUnprivilegedUserCannotUpdateOthersFile(TestNamespace ns) throws HttpResponseException { + RestClient adminRest = RestClient.admin(); + User owner = DriveTestUsers.createUser(ns, "file-editor"); + + ContextFile file = + createFile( + adminRest, + new CreateContextFile() + .withName(ns.prefix("perm-update")) + .withDisplayName("Admin's File") + .withFileType(ContextFileType.PDF) + .withProcessingStatus(ProcessingStatus.Uploaded) + .withOwners(List.of(owner.getEntityReference()))); + + RestClient consumerRest = RestClient.forUser("test@open-metadata.org", new String[] {}); + + String original = JsonUtils.pojoToJson(file); + file.setDisplayName("Hacked"); + + HttpResponseException ex = + assertThrows( + HttpResponseException.class, + () -> consumerRest.patch(FILE_PATH, file.getId(), original, file, ContextFile.class)); + + assertTrue( + ex.getStatusCode() == 403 || ex.getStatusCode() == 401, + "Expected 403/401, got " + ex.getStatusCode()); + } + + @Test + void testOwnerCanUpdateOwnFile(TestNamespace ns) throws HttpResponseException { + RestClient adminRest = RestClient.admin(); + User owner = DriveTestUsers.createUser(ns, "file-self-owner"); + + ContextFile file = + createFile( + adminRest, + new CreateContextFile() + .withName(ns.prefix("owner-update")) + .withDisplayName("Owner's File") + .withFileType(ContextFileType.PDF) + .withProcessingStatus(ProcessingStatus.Uploaded) + .withOwners(List.of(owner.getEntityReference()))); + + // The explicit owner should be able to update the file. + RestClient ownerRest = RestClient.forUser(owner.getEmail(), new String[] {}); + + String original = JsonUtils.pojoToJson(file); + file.setDisplayName("Updated by Owner"); + + ContextFile updated = + ownerRest.patch(FILE_PATH, file.getId(), original, file, ContextFile.class); + assertEquals("Updated by Owner", updated.getDisplayName()); + } + + // --- Search --- + + @Test + void testFileAppearsInSearch(TestNamespace ns) throws Exception { + RestClient rest = RestClient.admin(); + + String uniqueName = ns.prefix("searchable-file"); + ContextFile file = + createFile( + rest, + new CreateContextFile() + .withName(uniqueName) + .withDisplayName("Searchable PDF") + .withFileType(ContextFileType.PDF) + .withProcessingStatus(ProcessingStatus.Processed)); + + // ES indexing is async. Poll the direct get-by-id endpoint, which performs a real-time + // ES GET (no query_string parsing, no analyzer involvement) and is the most reliable + // signal that the document was indexed. The previous version of this test issued a + // free-text q= search using the namespaced unique name, but the prefix contains '-' + // which the query_string parser treats as a NOT operator and can produce a 500 on + // ES 9.x — yielding a flaky 30s-timeout failure even when the document is indexed. + await() + .pollDelay(Duration.ZERO) + .pollInterval(Duration.ofMillis(200)) + .atMost(Duration.ofSeconds(60)) + .untilAsserted( + () -> { + try (Response getResp = + rest.rawGet("v1/search/get/context_file_search_index/doc/" + file.getId())) { + int status = getResp.getStatus(); + String body = getResp.readEntity(String.class); + if (status != 200) { + throw new AssertionError( + "Expected 200 from search-by-id for file " + + file.getId() + + " but got " + + status + + " body=" + + body); + } + assertTrue( + body.contains(file.getId().toString()), + "Expected file " + file.getId() + " in search-by-id response: " + body); + } + }); + } +} diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/drive/DriveFileUploadIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/drive/DriveFileUploadIT.java new file mode 100644 index 000000000000..02ea575fdf69 --- /dev/null +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/drive/DriveFileUploadIT.java @@ -0,0 +1,729 @@ +package org.openmetadata.it.drive; + +import static jakarta.ws.rs.core.Response.Status.CREATED; +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.dropwizard.jackson.Jackson; +import io.dropwizard.jersey.jackson.JacksonFeature; +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.ClientBuilder; +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.client.WebTarget; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.MultivaluedHashMap; +import jakarta.ws.rs.core.MultivaluedMap; +import jakarta.ws.rs.core.Response; +import java.awt.Color; +import java.awt.Font; +import java.awt.Graphics2D; +import java.awt.RenderingHints; +import java.awt.image.BufferedImage; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Comparator; +import java.util.UUID; +import javax.imageio.ImageIO; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.PDPageContentStream; +import org.apache.pdfbox.pdmodel.font.PDType1Font; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.glassfish.jersey.client.ClientProperties; +import org.glassfish.jersey.media.multipart.FormDataMultiPart; +import org.glassfish.jersey.media.multipart.MultiPartFeature; +import org.glassfish.jersey.media.multipart.file.StreamDataBodyPart; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.openmetadata.schema.api.data.CreateFolder; +import org.openmetadata.schema.entity.data.ContextFile; +import org.openmetadata.schema.entity.data.ContextFileType; +import org.openmetadata.schema.entity.data.Folder; +import org.openmetadata.schema.entity.data.ProcessingStatus; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.sdk.test.util.RestClient; +import org.openmetadata.sdk.test.util.SdkClients; +import org.openmetadata.sdk.test.util.TestNamespace; +import org.openmetadata.sdk.test.util.TestNamespaceExtension; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Configuration; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.S3Object; + +/** + * Integration test for Context Center Drive file upload with MinIO-backed S3 storage using + * fixture files from src/test/resources. + */ +@ExtendWith(TestNamespaceExtension.class) +class DriveFileUploadIT { + + private static final String MINIO_BUCKET = "test-bucket"; + private static final String TIKA_TESSERACT_PATH_PROPERTY = "collate.tika.tesseract.path"; + private static String serverBaseUrl; + private static Client multipartClient; + private static WebTarget uploadTarget; + + @BeforeAll + static void setup() { + String itBaseUrl = + System.getProperty( + "IT_BASE_URL", + System.getenv().getOrDefault("IT_BASE_URL", "http://localhost:8585/api")); + if (itBaseUrl.endsWith("/api")) { + serverBaseUrl = itBaseUrl.substring(0, itBaseUrl.length() - 4); + } else { + serverBaseUrl = itBaseUrl; + } + + multipartClient = ClientBuilder.newClient(); + multipartClient.register(MultiPartFeature.class); + multipartClient.register(new JacksonFeature(Jackson.newObjectMapper())); + + uploadTarget = + multipartClient + .target(serverBaseUrl + "/api/v1/drive/files/upload") + .property(ClientProperties.CONNECT_TIMEOUT, 30000) + .property(ClientProperties.READ_TIMEOUT, 30000); + } + + @AfterAll + static void tearDown() { + if (multipartClient != null) { + multipartClient.close(); + multipartClient = null; + } + } + + private static MultivaluedMap adminAuthHeaders() { + String token = SdkClients.getAdminToken(); + MultivaluedMap headers = new MultivaluedHashMap<>(); + headers.add("Authorization", "Bearer " + token); + return headers; + } + + private byte[] readFixture(String resourcePath) throws IOException { + try (InputStream inputStream = getClass().getResourceAsStream(resourcePath)) { + assertNotNull(inputStream, "Missing drive fixture: " + resourcePath); + return inputStream.readAllBytes(); + } + } + + private Response uploadFile(String fileName, byte[] content, String displayName, String folderFqn) + throws IOException { + try (FormDataMultiPart multipart = new FormDataMultiPart()) { + if (displayName != null) { + multipart.field("displayName", displayName); + } + if (folderFqn != null) { + multipart.field("folder", folderFqn); + } + multipart.bodyPart( + new StreamDataBodyPart( + "file", + new ByteArrayInputStream(content), + fileName, + MediaType.APPLICATION_OCTET_STREAM_TYPE)); + + return uploadTarget + .request() + .headers(adminAuthHeaders()) + .post(Entity.entity(multipart, multipart.getMediaType())); + } + } + + private Response uploadFixture(String resourcePath, String displayName) throws IOException { + String fileName = resourcePath.substring(resourcePath.lastIndexOf('/') + 1); + return uploadFixture(resourcePath, fileName, displayName, null); + } + + private Response uploadFixture( + String resourcePath, String uploadedFileName, String displayName, String folderFqn) + throws IOException { + return uploadFile(uploadedFileName, readFixture(resourcePath), displayName, folderFqn); + } + + private String resolveStoredObjectKey(S3Client s3Client, String assetId) { + return s3Client + .listObjectsV2Paginator(ListObjectsV2Request.builder().bucket(MINIO_BUCKET).build()) + .contents() + .stream() + .map(S3Object::key) + .filter(key -> key.equals(assetId) || key.endsWith(assetId) || key.contains(assetId)) + .findFirst() + .orElse(null); + } + + private S3Client buildMinioClient() { + return S3Client.builder() + .region(Region.US_EAST_1) + .credentialsProvider( + StaticCredentialsProvider.create(AwsBasicCredentials.create("minio", "minio123"))) + .endpointOverride( + URI.create( + System.getProperty( + "IT_MINIO_ENDPOINT", + System.getenv().getOrDefault("IT_MINIO_ENDPOINT", "http://localhost:9000")))) + .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()) + .build(); + } + + private void assertStoredInMinIO(String assetId, byte[] expectedBytes) { + try (S3Client s3Client = buildMinioClient()) { + // atMost must stay above the global Awaitility pollInterval that + // K8sOMJobOperatorIT raises to 5s; otherwise Awaitility rejects with + // "Timeout must be greater than the poll delay". + await() + .pollDelay(Duration.ZERO) + .pollInterval(Duration.ofMillis(200)) + .atMost(Duration.ofSeconds(20)) + .untilAsserted( + () -> { + String objectKey = resolveStoredObjectKey(s3Client, assetId); + assertNotNull(objectKey, "Expected uploaded object for asset " + assetId); + try (ResponseInputStream objectStream = + s3Client.getObject( + GetObjectRequest.builder().bucket(MINIO_BUCKET).key(objectKey).build())) { + assertArrayEquals(expectedBytes, objectStream.readAllBytes()); + } + }); + } + } + + private void assertRemovedFromMinIO(String assetId) { + try (S3Client s3Client = buildMinioClient()) { + await() + .atMost(Duration.ofSeconds(10)) + .untilAsserted(() -> assertTrue(resolveStoredObjectKey(s3Client, assetId) == null)); + } + } + + private ContextFile fetchFile(UUID fileId) { + try { + return RestClient.admin().getById("v1/drive/files", fileId, "", ContextFile.class); + } catch (Exception e) { + throw new AssertionError("Failed to fetch uploaded file " + fileId, e); + } + } + + private void assertSearchContainsFile(String query, UUID fileId) { + RestClient rest = RestClient.admin(); + String encodedQuery = URLEncoder.encode(query, StandardCharsets.UTF_8); + + await() + .atMost(Duration.ofSeconds(20)) + .untilAsserted( + () -> { + try (Response searchResponse = + rest.rawGet( + "v1/search/query?q=" + + encodedQuery + + "&index=context_file_search_index&from=0&size=10")) { + assertEquals(200, searchResponse.getStatus()); + assertTrue(searchResponse.readEntity(String.class).contains(fileId.toString())); + } + }); + } + + private byte[] createPdf(String text) throws IOException { + try (PDDocument document = new PDDocument(); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + PDPage page = new PDPage(); + document.addPage(page); + try (PDPageContentStream contentStream = new PDPageContentStream(document, page)) { + contentStream.beginText(); + contentStream.setFont(PDType1Font.HELVETICA_BOLD, 12); + contentStream.newLineAtOffset(72, 720); + contentStream.showText(text); + contentStream.endText(); + } + document.save(outputStream); + return outputStream.toByteArray(); + } + } + + private byte[] createWorkbook(String sheetName, String key, String value) throws IOException { + try (Workbook workbook = new XSSFWorkbook(); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + var sheet = workbook.createSheet(sheetName); + var header = sheet.createRow(0); + header.createCell(0).setCellValue("Key"); + header.createCell(1).setCellValue("Value"); + var row = sheet.createRow(1); + row.createCell(0).setCellValue(key); + row.createCell(1).setCellValue(value); + workbook.write(outputStream); + return outputStream.toByteArray(); + } + } + + private byte[] createPngWithText(String text) throws IOException { + BufferedImage image = new BufferedImage(1400, 240, BufferedImage.TYPE_INT_RGB); + Graphics2D graphics = image.createGraphics(); + try { + graphics.setColor(Color.WHITE); + graphics.fillRect(0, 0, image.getWidth(), image.getHeight()); + graphics.setColor(Color.BLACK); + graphics.setRenderingHint( + RenderingHints.KEY_TEXT_ANTIALIASING, RenderingHints.VALUE_TEXT_ANTIALIAS_ON); + graphics.setFont(new Font("Monospaced", Font.BOLD, 56)); + graphics.drawString(text, 40, 140); + } finally { + graphics.dispose(); + } + + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + ImageIO.write(image, "png", outputStream); + return outputStream.toByteArray(); + } + } + + private Path createFakeTesseractHome(String extractedText) throws IOException { + Path home = Files.createTempDirectory("fake-tesseract-home-"); + Path executable = home.resolve("tesseract"); + Files.writeString( + executable, + "#!/bin/sh\n" + + "if [ $# -eq 0 ] || [ \"$1\" = \"--version\" ]; then\n" + + " echo \"tesseract 5.0.0\"\n" + + " exit 0\n" + + "fi\n" + + "output_base=\"$2\"\n" + + "printf '%s\\n' \"" + + extractedText + + "\" > \"${output_base}.txt\"\n", + StandardCharsets.UTF_8); + executable.toFile().setExecutable(true); + return home; + } + + private void deleteRecursively(Path root) throws IOException { + if (root == null || Files.notExists(root)) { + return; + } + try (var paths = Files.walk(root)) { + paths.sorted(Comparator.reverseOrder()).forEach(path -> path.toFile().delete()); + } + } + + @Test + void testUploadPdfToMinIO(TestNamespace ns) throws Exception { + byte[] content = readFixture("/drive/sample-report.pdf"); + ContextFile file; + try (Response response = uploadFixture("/drive/sample-report.pdf", "Annual Report")) { + String body = response.readEntity(String.class); + assertEquals( + CREATED.getStatusCode(), response.getStatus(), "Upload to MinIO failed: " + body); + + file = JsonUtils.readValue(body, ContextFile.class); + assertNotNull(file.getId()); + assertNotNull(file.getAssetId(), "File should have assetId from S3 upload"); + assertNotNull(file.getHeadContentId(), "File should point at a current content snapshot"); + assertEquals("Annual Report", file.getDisplayName()); + assertEquals(content.length, file.getFileSize().intValue()); + assertStoredInMinIO(file.getAssetId(), content); + } + + await() + .atMost(Duration.ofSeconds(20)) + .untilAsserted( + () -> { + ContextFile refreshed = fetchFile(file.getId()); + assertEquals(ProcessingStatus.Processed, refreshed.getProcessingStatus()); + assertTrue(refreshed.getExtractedText().contains("Context Center PDF Fixture")); + assertEquals(1, refreshed.getPageCount()); + }); + } + + @Test + void testUploadSpreadsheetToMinIO(TestNamespace ns) throws Exception { + byte[] content = readFixture("/drive/sample-pricing.xlsx"); + Response response = uploadFixture("/drive/sample-pricing.xlsx", "Pricing Sheet"); + + String body = response.readEntity(String.class); + assertEquals(CREATED.getStatusCode(), response.getStatus(), "Upload failed: " + body); + + ContextFile file = JsonUtils.readValue(body, ContextFile.class); + assertNotNull(file.getAssetId()); + assertNotNull(file.getHeadContentId()); + assertEquals("Pricing Sheet", file.getDisplayName()); + assertEquals(content.length, file.getFileSize().intValue()); + } + + @Test + void testUploadCsvToMinIO(TestNamespace ns) throws Exception { + byte[] content = readFixture("/drive/sample-data.csv"); + Response response = uploadFixture("/drive/sample-data.csv", null); + + String body = response.readEntity(String.class); + assertEquals(CREATED.getStatusCode(), response.getStatus(), "Upload failed: " + body); + + ContextFile file = JsonUtils.readValue(body, ContextFile.class); + assertNotNull(file.getAssetId()); + assertNotNull(file.getHeadContentId()); + assertEquals("sample-data.csv", file.getDisplayName()); + assertEquals(content.length, file.getFileSize().intValue()); + } + + @Test + void testUploadVerifyFileSize(TestNamespace ns) throws Exception { + byte[] contentBytes = readFixture("/drive/sample-notes.txt"); + try (Response response = uploadFixture("/drive/sample-notes.txt", "Sized File")) { + String body = response.readEntity(String.class); + assertEquals(CREATED.getStatusCode(), response.getStatus(), "Upload failed: " + body); + + ContextFile file = JsonUtils.readValue(body, ContextFile.class); + assertEquals( + contentBytes.length, + file.getFileSize().intValue(), + "File size should match uploaded bytes"); + assertEquals("txt", file.getFileExtension()); + assertEquals(ProcessingStatus.Uploaded, file.getProcessingStatus()); + assertNotNull(file.getHeadContentId()); + } + } + + @Test + void testUploadedTextFileIsSearchableByExtractedText(TestNamespace ns) throws Exception { + String uniqueToken = "contextneedle" + UUID.randomUUID().toString().replace("-", ""); + byte[] content = + ("User supplied context that should be searchable " + uniqueToken) + .getBytes(StandardCharsets.UTF_8); + + ContextFile file; + try (Response response = uploadFile("search-fixture.txt", content, "Search Fixture", null)) { + String body = response.readEntity(String.class); + assertEquals(CREATED.getStatusCode(), response.getStatus(), "Upload failed: " + body); + file = JsonUtils.readValue(body, ContextFile.class); + } + + await() + .atMost(Duration.ofSeconds(20)) + .untilAsserted( + () -> { + ContextFile refreshed = fetchFile(file.getId()); + assertEquals(ProcessingStatus.Processed, refreshed.getProcessingStatus()); + assertTrue(refreshed.getExtractedText().contains(uniqueToken)); + }); + + assertSearchContainsFile(uniqueToken, file.getId()); + } + + @Test + void testUploadedPdfIsSearchableByExtractedText(TestNamespace ns) throws Exception { + String uniqueToken = "pdfneedle" + UUID.randomUUID().toString().replace("-", ""); + byte[] content = createPdf("Quarterly context for " + uniqueToken); + + ContextFile file; + try (Response response = + uploadFile("search-fixture.pdf", content, ns.shortPrefix("PDF Search"), null)) { + String body = response.readEntity(String.class); + assertEquals(CREATED.getStatusCode(), response.getStatus(), "Upload failed: " + body); + file = JsonUtils.readValue(body, ContextFile.class); + } + + await() + .atMost(Duration.ofSeconds(20)) + .untilAsserted( + () -> { + ContextFile refreshed = fetchFile(file.getId()); + assertEquals(ProcessingStatus.Processed, refreshed.getProcessingStatus()); + assertTrue(refreshed.getExtractedText().contains(uniqueToken)); + }); + + assertSearchContainsFile(uniqueToken, file.getId()); + } + + @Test + void testUploadedSpreadsheetIsSearchableByExtractedText(TestNamespace ns) throws Exception { + String uniqueToken = "sheetneedle" + UUID.randomUUID().toString().replace("-", ""); + byte[] content = createWorkbook("Pricing", "SearchToken", uniqueToken); + + ContextFile file; + try (Response response = + uploadFile("search-fixture.xlsx", content, ns.shortPrefix("Spreadsheet Search"), null)) { + String body = response.readEntity(String.class); + assertEquals(CREATED.getStatusCode(), response.getStatus(), "Upload failed: " + body); + file = JsonUtils.readValue(body, ContextFile.class); + } + + await() + .atMost(Duration.ofSeconds(20)) + .untilAsserted( + () -> { + ContextFile refreshed = fetchFile(file.getId()); + assertEquals(ProcessingStatus.Processed, refreshed.getProcessingStatus()); + assertTrue(refreshed.getExtractedText().contains(uniqueToken)); + }); + + assertSearchContainsFile(uniqueToken, file.getId()); + } + + @Test + void testUploadedImageIsSearchableByOcrExtractedText(TestNamespace ns) throws Exception { + String uniqueToken = + "IMAGENEEDLE" + + UUID.randomUUID().toString().replace("-", "").substring(0, 10).toUpperCase(); + Path fakeTesseractHome = createFakeTesseractHome("Revenue chart " + uniqueToken); + String originalPath = System.getProperty(TIKA_TESSERACT_PATH_PROPERTY); + + try { + System.setProperty(TIKA_TESSERACT_PATH_PROPERTY, fakeTesseractHome.toString()); + byte[] content = createPngWithText(uniqueToken); + + ContextFile file; + try (Response response = + uploadFile("search-fixture.png", content, ns.shortPrefix("Image Search"), null)) { + String body = response.readEntity(String.class); + assertEquals(CREATED.getStatusCode(), response.getStatus(), "Upload failed: " + body); + file = JsonUtils.readValue(body, ContextFile.class); + } + + await() + .atMost(Duration.ofSeconds(20)) + .untilAsserted( + () -> { + ContextFile refreshed = fetchFile(file.getId()); + assertEquals(ProcessingStatus.Processed, refreshed.getProcessingStatus()); + assertTrue(refreshed.getExtractedText().contains(uniqueToken)); + }); + + assertSearchContainsFile(uniqueToken, file.getId()); + } finally { + if (originalPath == null) { + System.clearProperty(TIKA_TESSERACT_PATH_PROPERTY); + } else { + System.setProperty(TIKA_TESSERACT_PATH_PROPERTY, originalPath); + } + deleteRecursively(fakeTesseractHome); + } + } + + @Test + void testUploadFileIntoFolder(TestNamespace ns) throws Exception { + RestClient rest = RestClient.admin(); + Folder folder = + rest.create( + "v1/drive/folders", + new CreateFolder().withName(ns.prefix("upload-target-folder")), + Folder.class); + + Response response = + uploadFixture( + "/drive/sample-report.pdf", + "nested.pdf", + "File In Folder", + folder.getFullyQualifiedName()); + + String body = response.readEntity(String.class); + assertEquals(CREATED.getStatusCode(), response.getStatus(), "Upload failed: " + body); + + ContextFile file = JsonUtils.readValue(body, ContextFile.class); + assertNotNull(file.getAssetId()); + assertNotNull(file.getHeadContentId()); + + ContextFile fetched = rest.getById("v1/drive/files", file.getId(), "folder", ContextFile.class); + assertNotNull(fetched.getFolder(), "File should be in folder"); + assertEquals(folder.getId(), fetched.getFolder().getId()); + } + + @Test + void testUploadMultipleFilesUniqueness(TestNamespace ns) throws Exception { + byte[] content = readFixture("/drive/sample-report.pdf"); + + Response resp1 = uploadFile("duplicate.pdf", content, "First Upload", null); + Response resp2 = uploadFile("duplicate.pdf", content, "Second Upload", null); + + String body1 = resp1.readEntity(String.class); + String body2 = resp2.readEntity(String.class); + + assertEquals(CREATED.getStatusCode(), resp1.getStatus(), "First upload failed: " + body1); + assertEquals(CREATED.getStatusCode(), resp2.getStatus(), "Second upload failed: " + body2); + + ContextFile file1 = JsonUtils.readValue(body1, ContextFile.class); + ContextFile file2 = JsonUtils.readValue(body2, ContextFile.class); + + assertTrue( + !file1.getId().equals(file2.getId()), "Two uploads of same filename should get unique IDs"); + assertTrue( + !file1.getName().equals(file2.getName()), + "Two uploads of same filename should get unique names"); + assertNotNull(file1.getHeadContentId()); + assertNotNull(file2.getHeadContentId()); + } + + @Test + void testUploadLargeFileRejected(TestNamespace ns) throws Exception { + Response response = + uploadFile("too_large.jpg", readFixture("/2mb-jpg-example-file.jpg"), "Too Large", null); + + assertTrue( + response.getStatus() >= 400, + "Oversized upload should be rejected, got " + response.getStatus()); + } + + @Test + void testUploadDetectsFileType(TestNamespace ns) throws Exception { + Response pdfResp = uploadFixture("/drive/sample-report.pdf", "PDF Test"); + ContextFile pdf = JsonUtils.readValue(pdfResp.readEntity(String.class), ContextFile.class); + + Response csvResp = uploadFixture("/drive/sample-data.csv", "CSV Test"); + ContextFile csv = JsonUtils.readValue(csvResp.readEntity(String.class), ContextFile.class); + + Response spreadsheetResp = uploadFixture("/drive/sample-pricing.xlsx", "Spreadsheet Test"); + ContextFile spreadsheet = + JsonUtils.readValue(spreadsheetResp.readEntity(String.class), ContextFile.class); + + Response textResp = uploadFixture("/drive/sample-notes.txt", "Text Test"); + ContextFile text = JsonUtils.readValue(textResp.readEntity(String.class), ContextFile.class); + + assertEquals(ContextFileType.PDF, pdf.getFileType()); + assertEquals(ContextFileType.CSV, csv.getFileType()); + assertEquals(ContextFileType.Spreadsheet, spreadsheet.getFileType()); + assertEquals(ContextFileType.Text, text.getFileType()); + } + + @Test + void testDownloadUploadedFileThroughContextFileEndpoint(TestNamespace ns) throws Exception { + byte[] content = readFixture("/drive/sample-notes.txt"); + + Response uploadResponse = uploadFixture("/drive/sample-notes.txt", "Download Test"); + String body = uploadResponse.readEntity(String.class); + assertEquals(CREATED.getStatusCode(), uploadResponse.getStatus(), "Upload failed: " + body); + + ContextFile file = JsonUtils.readValue(body, ContextFile.class); + + await() + .pollDelay(Duration.ZERO) + .pollInterval(Duration.ofMillis(200)) + .atMost(Duration.ofSeconds(20)) + .untilAsserted( + () -> { + try (Response downloadResponse = + multipartClient + .target( + serverBaseUrl + + "/api/v1/drive/files/" + + file.getId() + + "/download?redirect=false") + .request() + .headers(adminAuthHeaders()) + .get(); + InputStream downloaded = downloadResponse.readEntity(InputStream.class)) { + assertEquals(200, downloadResponse.getStatus()); + assertArrayEquals(content, downloaded.readAllBytes()); + } + }); + } + + @Test + void testDownloadUploadedFileThroughSignedRedirect(TestNamespace ns) throws Exception { + byte[] content = readFixture("/drive/sample-notes.txt"); + + Response uploadResponse = uploadFixture("/drive/sample-notes.txt", "Redirect Download"); + String body = uploadResponse.readEntity(String.class); + assertEquals(CREATED.getStatusCode(), uploadResponse.getStatus(), "Upload failed: " + body); + + ContextFile file = JsonUtils.readValue(body, ContextFile.class); + + await() + .pollDelay(Duration.ZERO) + .pollInterval(Duration.ofMillis(200)) + .atMost(Duration.ofSeconds(20)) + .untilAsserted( + () -> { + try (Response redirectResponse = + multipartClient + .target( + serverBaseUrl + "/api/v1/drive/files/" + file.getId() + "/download") + .property(ClientProperties.FOLLOW_REDIRECTS, false) + .request() + .headers(adminAuthHeaders()) + .get(); + Client signedUrlClient = ClientBuilder.newClient()) { + assertEquals(307, redirectResponse.getStatus()); + String signedUrl = redirectResponse.getHeaderString("Location"); + assertNotNull(signedUrl); + + try (Response signedDownload = signedUrlClient.target(signedUrl).request().get(); + InputStream downloaded = signedDownload.readEntity(InputStream.class)) { + assertEquals(200, signedDownload.getStatus()); + assertArrayEquals(content, downloaded.readAllBytes()); + } + } + }); + } + + @Test + void testSoftDeletedFileCanDownloadFromTrash(TestNamespace ns) throws Exception { + byte[] content = readFixture("/drive/sample-notes.txt"); + RestClient rest = RestClient.admin(); + + Response uploadResponse = uploadFixture("/drive/sample-notes.txt", "Trash Download"); + String body = uploadResponse.readEntity(String.class); + assertEquals(CREATED.getStatusCode(), uploadResponse.getStatus(), "Upload failed: " + body); + + ContextFile file = JsonUtils.readValue(body, ContextFile.class); + rest.delete("v1/drive/files", file.getId()); + + try (Response downloadResponse = + multipartClient + .target( + serverBaseUrl + + "/api/v1/drive/files/" + + file.getId() + + "/download?include=all&redirect=false") + .request() + .headers(adminAuthHeaders()) + .get(); + InputStream downloaded = downloadResponse.readEntity(InputStream.class)) { + assertEquals(200, downloadResponse.getStatus()); + assertArrayEquals(content, downloaded.readAllBytes()); + } + } + + @Test + void testHardDeleteRemovesObjectFromMinIO(TestNamespace ns) throws Exception { + byte[] content = readFixture("/drive/sample-notes.txt"); + RestClient rest = RestClient.admin(); + + Response uploadResponse = uploadFixture("/drive/sample-notes.txt", "Hard Delete"); + String body = uploadResponse.readEntity(String.class); + assertEquals(CREATED.getStatusCode(), uploadResponse.getStatus(), "Upload failed: " + body); + + ContextFile file = JsonUtils.readValue(body, ContextFile.class); + assertStoredInMinIO(file.getAssetId(), content); + + rest.hardDelete("v1/drive/files", file.getId()); + + await() + .atMost(Duration.ofSeconds(10)) + .untilAsserted( + () -> { + try (Response deletedResponse = + rest.rawGet("v1/drive/files/" + file.getId() + "?include=all")) { + assertEquals(404, deletedResponse.getStatus()); + } + }); + assertRemovedFromMinIO(file.getAssetId()); + } +} diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/drive/DriveTestUsers.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/drive/DriveTestUsers.java new file mode 100644 index 000000000000..13b59f914e0a --- /dev/null +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/drive/DriveTestUsers.java @@ -0,0 +1,23 @@ +package org.openmetadata.it.drive; + +import org.openmetadata.schema.api.teams.CreateUser; +import org.openmetadata.schema.entity.teams.User; +import org.openmetadata.sdk.services.teams.UserService; +import org.openmetadata.sdk.test.util.SdkClients; +import org.openmetadata.sdk.test.util.TestNamespace; + +final class DriveTestUsers { + + private DriveTestUsers() {} + + static User createUser(TestNamespace ns, String suffix) { + String base = (ns.shortPrefix("drive") + suffix).replaceAll("[^a-zA-Z0-9]", "").toLowerCase(); + String name = base.substring(0, Math.min(base.length(), 48)); + CreateUser createUser = + new CreateUser() + .withName(name) + .withDisplayName(name) + .withEmail(name + "@test.openmetadata.org"); + return new UserService(SdkClients.adminClient().getHttpClient()).create(createUser); + } +} diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/drive/FolderIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/drive/FolderIT.java new file mode 100644 index 000000000000..e791a3174099 --- /dev/null +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/drive/FolderIT.java @@ -0,0 +1,384 @@ +package org.openmetadata.it.drive; + +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import jakarta.ws.rs.core.Response; +import java.time.Duration; +import java.util.List; +import java.util.UUID; +import org.apache.http.client.HttpResponseException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.openmetadata.schema.api.data.CreateContextFile; +import org.openmetadata.schema.api.data.CreateFolder; +import org.openmetadata.schema.entity.data.ContextFile; +import org.openmetadata.schema.entity.data.ContextFileType; +import org.openmetadata.schema.entity.data.Folder; +import org.openmetadata.schema.entity.data.ProcessingStatus; +import org.openmetadata.schema.entity.teams.User; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.sdk.client.OpenMetadataClient; +import org.openmetadata.sdk.services.teams.UserService; +import org.openmetadata.sdk.test.util.RestClient; +import org.openmetadata.sdk.test.util.SdkClients; +import org.openmetadata.sdk.test.util.TestNamespace; +import org.openmetadata.sdk.test.util.TestNamespaceExtension; + +@ExtendWith(TestNamespaceExtension.class) +class FolderIT { + + private static final String PATH = "v1/drive/folders"; + + private Folder createFolder(RestClient rest, CreateFolder request) throws HttpResponseException { + return rest.create(PATH, request, Folder.class); + } + + private Folder getFolder(RestClient rest, UUID id, String fields) throws HttpResponseException { + return rest.getById(PATH, id, fields, Folder.class); + } + + private Folder patchFolder(RestClient rest, UUID id, String origJson, Folder updated) + throws HttpResponseException { + return rest.patch(PATH, id, origJson, updated, Folder.class); + } + + // --- CRUD --- + + @Test + void testCreateFolder(TestNamespace ns) throws HttpResponseException { + RestClient rest = RestClient.admin(); + + CreateFolder create = + new CreateFolder().withName(ns.prefix("my-folder")).withDisplayName("My Folder"); + + Folder folder = createFolder(rest, create); + assertNotNull(folder.getId()); + assertEquals("My Folder", folder.getDisplayName()); + } + + @Test + void testGetFolderById(TestNamespace ns) throws HttpResponseException { + RestClient rest = RestClient.admin(); + + Folder created = createFolder(rest, new CreateFolder().withName(ns.prefix("get-test"))); + + Folder fetched = getFolder(rest, created.getId(), ""); + assertEquals(created.getId(), fetched.getId()); + assertEquals(created.getName(), fetched.getName()); + } + + @Test + void testUpdateFolderDisplayName(TestNamespace ns) throws HttpResponseException { + RestClient rest = RestClient.admin(); + + Folder folder = + createFolder( + rest, + new CreateFolder().withName(ns.prefix("update-test")).withDisplayName("Original Name")); + + String original = JsonUtils.pojoToJson(folder); + folder.setDisplayName("Updated Name"); + Folder updated = patchFolder(rest, folder.getId(), original, folder); + + assertEquals("Updated Name", updated.getDisplayName()); + } + + @Test + void testDeleteFolder(TestNamespace ns) throws HttpResponseException { + RestClient rest = RestClient.admin(); + + Folder folder = createFolder(rest, new CreateFolder().withName(ns.prefix("delete-test"))); + + rest.delete(PATH, folder.getId()); + + HttpResponseException ex = + assertThrows(HttpResponseException.class, () -> getFolder(rest, folder.getId(), "")); + assertEquals(404, ex.getStatusCode()); + + try (Response deletedResponse = rest.rawGet(PATH + "/" + folder.getId() + "?include=all")) { + assertEquals(200, deletedResponse.getStatus()); + assertTrue(deletedResponse.readEntity(String.class).contains("\"deleted\":true")); + } + } + + @Test + void testRestoreSoftDeletedFolder(TestNamespace ns) throws HttpResponseException { + RestClient rest = RestClient.admin(); + + Folder folder = createFolder(rest, new CreateFolder().withName(ns.prefix("restore-folder"))); + rest.delete(PATH, folder.getId()); + + Folder restored = rest.restore(PATH, folder.getId(), Folder.class); + assertEquals(folder.getId(), restored.getId()); + assertTrue(!Boolean.TRUE.equals(restored.getDeleted())); + } + + @Test + void testHardDeleteFolderIsAsync(TestNamespace ns) throws HttpResponseException { + RestClient rest = RestClient.admin(); + + Folder folder = + createFolder(rest, new CreateFolder().withName(ns.prefix("hard-delete-folder"))); + + try (Response deleteResponse = + rest.rawDelete(PATH + "/" + folder.getId() + "?hardDelete=true&recursive=true")) { + assertEquals(202, deleteResponse.getStatus()); + assertTrue(deleteResponse.readEntity(String.class).contains("\"hardDelete\":true")); + } + + await() + .atMost(Duration.ofSeconds(20)) + .untilAsserted( + () -> { + try (Response deletedResponse = + rest.rawGet(PATH + "/" + folder.getId() + "?include=all")) { + assertEquals(404, deletedResponse.getStatus()); + } + }); + } + + // --- Nested Folder Hierarchy --- + + @Test + void testNestedFolders(TestNamespace ns) throws HttpResponseException { + RestClient rest = RestClient.admin(); + + Folder root = + createFolder( + rest, new CreateFolder().withName(ns.prefix("root")).withDisplayName("Root Folder")); + + Folder child = + createFolder( + rest, + new CreateFolder() + .withName(ns.prefix("child")) + .withDisplayName("Child Folder") + .withParent(root.getFullyQualifiedName())); + + Folder grandchild = + createFolder( + rest, + new CreateFolder() + .withName(ns.prefix("grandchild")) + .withDisplayName("Grandchild Folder") + .withParent(child.getFullyQualifiedName())); + + // Verify parent-child + Folder fetchedChild = getFolder(rest, child.getId(), "parent"); + assertNotNull(fetchedChild.getParent()); + assertEquals(root.getId(), fetchedChild.getParent().getId()); + + // Verify FQN includes full path + Folder fetchedGrandchild = getFolder(rest, grandchild.getId(), "parent"); + assertTrue( + fetchedGrandchild.getFullyQualifiedName().contains(root.getName()), + "Grandchild FQN should contain root folder name"); + assertTrue( + fetchedGrandchild.getFullyQualifiedName().contains(child.getName()), + "Grandchild FQN should contain child folder name"); + } + + @Test + void testFolderWithChildren(TestNamespace ns) throws HttpResponseException { + RestClient rest = RestClient.admin(); + + Folder parent = createFolder(rest, new CreateFolder().withName(ns.prefix("parent-list"))); + + createFolder( + rest, + new CreateFolder() + .withName(ns.prefix("child-1")) + .withParent(parent.getFullyQualifiedName())); + createFolder( + rest, + new CreateFolder() + .withName(ns.prefix("child-2")) + .withParent(parent.getFullyQualifiedName())); + + Folder fetched = getFolder(rest, parent.getId(), "children"); + assertNotNull(fetched.getChildren()); + assertEquals(2, fetched.getChildren().size()); + } + + // --- Ownership (personal vs team folder) --- + + @Test + void testFolderWithUserOwner(TestNamespace ns) throws HttpResponseException { + RestClient rest = RestClient.admin(); + OpenMetadataClient adminClient = SdkClients.adminClient(); + UserService userSvc = new UserService(adminClient.getHttpClient()); + User admin = userSvc.getByName("admin", null); + + Folder folder = + createFolder( + rest, + new CreateFolder() + .withName(ns.prefix("personal")) + .withDisplayName("My Personal Docs") + .withOwners(List.of(admin.getEntityReference()))); + + Folder fetched = getFolder(rest, folder.getId(), "owners"); + assertNotNull(fetched.getOwners()); + assertEquals(1, fetched.getOwners().size()); + assertEquals(admin.getId(), fetched.getOwners().get(0).getId()); + } + + // --- Permissions --- + + @Test + void testUnprivilegedUserCannotDeleteFolder(TestNamespace ns) throws HttpResponseException { + RestClient adminRest = RestClient.admin(); + User owner = DriveTestUsers.createUser(ns, "folder-owner"); + + Folder folder = + createFolder( + adminRest, + new CreateFolder() + .withName(ns.prefix("perm-delete")) + .withOwners(List.of(owner.getEntityReference()))); + + RestClient consumerRest = RestClient.forUser("test@open-metadata.org", new String[] {}); + + HttpResponseException ex = + assertThrows( + HttpResponseException.class, () -> consumerRest.hardDelete(PATH, folder.getId())); + + assertTrue( + ex.getStatusCode() == 403 || ex.getStatusCode() == 401, + "Expected 403 or 401, got " + ex.getStatusCode()); + } + + @Test + void testUnprivilegedUserCannotUpdateOthersFolder(TestNamespace ns) throws HttpResponseException { + RestClient adminRest = RestClient.admin(); + User owner = DriveTestUsers.createUser(ns, "folder-editor"); + + Folder folder = + createFolder( + adminRest, + new CreateFolder() + .withName(ns.prefix("perm-update")) + .withDisplayName("Original") + .withOwners(List.of(owner.getEntityReference()))); + + RestClient consumerRest = RestClient.forUser("test@open-metadata.org", new String[] {}); + + String original = JsonUtils.pojoToJson(folder); + folder.setDisplayName("Hacked Name"); + + HttpResponseException ex = + assertThrows( + HttpResponseException.class, + () -> consumerRest.patch(PATH, folder.getId(), original, folder, Folder.class)); + + assertTrue( + ex.getStatusCode() == 403 || ex.getStatusCode() == 401, + "Expected 403 or 401, got " + ex.getStatusCode()); + } + + @Test + @Execution(ExecutionMode.SAME_THREAD) + void testDeleteFolderRecursive(TestNamespace ns) throws HttpResponseException { + RestClient rest = RestClient.admin(); + + Folder parent = createFolder(rest, new CreateFolder().withName(ns.prefix("recursive-parent"))); + + Folder child = + createFolder( + rest, + new CreateFolder() + .withName(ns.prefix("recursive-child")) + .withParent(parent.getFullyQualifiedName())); + + // Delete parent recursively + try (Response deleteResponse = + rest.rawDelete(PATH + "/" + parent.getId() + "?recursive=true&hardDelete=true")) { + assertEquals(202, deleteResponse.getStatus()); + String responseBody = deleteResponse.readEntity(String.class); + assertTrue(responseBody.contains("\"hardDelete\":true")); + assertTrue(responseBody.contains("\"recursive\":true")); + } + + // Both should be gone. Close each Response before opening the next so the Apache HTTP + // client's connection pool doesn't hold two concurrent requests — under parallel-test load + // the second GET can otherwise block waiting for a free connection. + await() + .atMost(Duration.ofMinutes(2)) + .pollInterval(Duration.ofSeconds(1)) + .untilAsserted( + () -> { + int parentStatus; + try (Response parentResponse = + rest.rawGet(PATH + "/" + parent.getId() + "?include=all")) { + parentStatus = parentResponse.getStatus(); + } + int childStatus; + try (Response childResponse = + rest.rawGet(PATH + "/" + child.getId() + "?include=all")) { + childStatus = childResponse.getStatus(); + } + assertEquals(404, parentStatus); + assertEquals(404, childStatus); + }); + } + + @Test + void testFolderContentsIncludesFoldersAndFiles(TestNamespace ns) throws Exception { + RestClient rest = RestClient.admin(); + OpenMetadataClient adminClient = SdkClients.adminClient(); + UserService userSvc = new UserService(adminClient.getHttpClient()); + User admin = userSvc.getByName("admin", null); + + Folder parent = createFolder(rest, new CreateFolder().withName(ns.prefix("contents-parent"))); + Folder child = + createFolder( + rest, + new CreateFolder() + .withName(ns.prefix("child-folder")) + .withParent(parent.getFullyQualifiedName()) + .withOwners(List.of(admin.getEntityReference()))); + + ContextFile file = + rest.create( + "v1/drive/files", + new CreateContextFile() + .withName(ns.prefix("contents-file")) + .withDisplayName("Contents File") + .withFileType(ContextFileType.PDF) + .withFolder(parent.getFullyQualifiedName()) + .withOwners(List.of(admin.getEntityReference())) + .withProcessingStatus(ProcessingStatus.Uploaded), + ContextFile.class); + + String json; + try (Response response = rest.rawGet(PATH + "/" + parent.getId() + "/contents")) { + assertEquals(200, response.getStatus()); + json = response.readEntity(String.class); + } + jakarta.json.JsonObject contents = + jakarta.json.Json.createReader(new java.io.StringReader(json)).readObject(); + jakarta.json.JsonObject folderJson = contents.getJsonArray("folders").getJsonObject(0); + jakarta.json.JsonObject fileJson = contents.getJsonArray("files").getJsonObject(0); + + assertEquals(1, contents.getInt("childrenFolderCount")); + assertEquals(1, contents.getInt("childrenFileCount")); + assertEquals(2, contents.getInt("itemCount")); + assertEquals(1, contents.getJsonArray("folders").size()); + assertEquals(1, contents.getJsonArray("files").size()); + assertEquals(child.getName(), folderJson.getString("name")); + assertEquals(parent.getId().toString(), folderJson.getJsonObject("parent").getString("id")); + assertEquals( + admin.getId().toString(), + folderJson.getJsonArray("owners").getJsonObject(0).getString("id")); + assertEquals(file.getName(), fileJson.getString("name")); + assertEquals(parent.getId().toString(), fileJson.getJsonObject("folder").getString("id")); + assertEquals( + admin.getId().toString(), fileJson.getJsonArray("owners").getJsonObject(0).getString("id")); + } +} diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/knowledge/KnowledgeCenterIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/knowledge/KnowledgeCenterIT.java new file mode 100644 index 000000000000..1022bfadd9a7 --- /dev/null +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/knowledge/KnowledgeCenterIT.java @@ -0,0 +1,237 @@ +package org.openmetadata.it.knowledge; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; +import java.util.UUID; +import org.apache.http.client.HttpResponseException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.openmetadata.schema.api.data.CreatePage; +import org.openmetadata.schema.api.domains.CreateDataProduct; +import org.openmetadata.schema.api.domains.CreateDomain; +import org.openmetadata.schema.entity.data.Article; +import org.openmetadata.schema.entity.data.Page; +import org.openmetadata.schema.entity.data.PageType; +import org.openmetadata.schema.entity.domains.DataProduct; +import org.openmetadata.schema.entity.domains.Domain; +import org.openmetadata.schema.entity.teams.Team; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.sdk.client.OpenMetadataClient; +import org.openmetadata.sdk.services.domains.DataProductService; +import org.openmetadata.sdk.services.domains.DomainService; +import org.openmetadata.sdk.services.teams.TeamService; +import org.openmetadata.sdk.test.util.RestClient; +import org.openmetadata.sdk.test.util.SdkClients; +import org.openmetadata.sdk.test.util.TestNamespace; +import org.openmetadata.sdk.test.util.TestNamespaceExtension; + +@ExtendWith(TestNamespaceExtension.class) +public class KnowledgeCenterIT { + + private static final String KC_PATH = "v1/knowledgeCenter"; + + private Page createPage(RestClient rest, CreatePage request) throws HttpResponseException { + return rest.create(KC_PATH, request, Page.class); + } + + private Page getPage(RestClient rest, UUID id, String fields) throws HttpResponseException { + return rest.getById(KC_PATH, id, fields, Page.class); + } + + private Page patchPage(RestClient rest, UUID id, String originalJson, Page updated) + throws HttpResponseException { + return rest.patch(KC_PATH, id, originalJson, updated, Page.class); + } + + private CreatePage buildCreateRequest(String name, EntityReference relatedEntity) { + return new CreatePage() + .withName(name) + .withPageType(PageType.ARTICLE) + .withDescription("This is a test Description.") + .withPage(new Article()) + .withRelatedEntities(List.of(relatedEntity)); + } + + private EntityReference getOrganizationRef() { + OpenMetadataClient adminClient = SdkClients.adminClient(); + TeamService teamService = new TeamService(adminClient.getHttpClient()); + Team org = teamService.getByName("Organization", null); + return org.getEntityReference(); + } + + @Test + void testRelatedEntitiesExcludesDomainsAndDataProducts(TestNamespace ns) + throws HttpResponseException { + RestClient rest = RestClient.admin(); + OpenMetadataClient adminClient = SdkClients.adminClient(); + DomainService domainSvc = new DomainService(adminClient.getHttpClient()); + + EntityReference orgRef = getOrganizationRef(); + CreatePage createPageReq = buildCreateRequest(ns.prefix("pageExcludesDomains"), orgRef); + Page page = createPage(rest, createPageReq); + + CreateDomain createDomain = + new CreateDomain() + .withName(ns.prefix("testDomain")) + .withDomainType(CreateDomain.DomainType.AGGREGATE) + .withDescription("Test domain"); + Domain domain = domainSvc.create(createDomain); + + String original = JsonUtils.pojoToJson(page); + page.withDomains(List.of(domain.getEntityReference())); + page = patchPage(rest, page.getId(), original, page); + + Page fetchedPage = getPage(rest, page.getId(), "relatedEntities,domains,dataProducts"); + + assertEquals(1, fetchedPage.getDomains().size()); + assertEquals(domain.getName(), fetchedPage.getDomains().get(0).getName()); + + boolean domainInRelatedEntities = + fetchedPage.getRelatedEntities().stream().anyMatch(ref -> "domain".equals(ref.getType())); + assertEquals(false, domainInRelatedEntities, "Domains should not appear in relatedEntities"); + + boolean dataProductInRelatedEntities = + fetchedPage.getRelatedEntities().stream() + .anyMatch(ref -> "dataProduct".equals(ref.getType())); + assertEquals( + false, dataProductInRelatedEntities, "DataProducts should not appear in relatedEntities"); + } + + @Test + void testDomainAddUpdateRemove(TestNamespace ns) throws HttpResponseException { + RestClient rest = RestClient.admin(); + OpenMetadataClient adminClient = SdkClients.adminClient(); + DomainService domainSvc = new DomainService(adminClient.getHttpClient()); + + EntityReference orgRef = getOrganizationRef(); + CreatePage createPageReq = buildCreateRequest(ns.prefix("pageDomainCrud"), orgRef); + Page page = createPage(rest, createPageReq); + + CreateDomain createDomain1 = + new CreateDomain() + .withName(ns.prefix("testDomain1")) + .withDomainType(CreateDomain.DomainType.AGGREGATE) + .withDescription("Test domain 1"); + Domain domain1 = domainSvc.create(createDomain1); + + CreateDomain createDomain2 = + new CreateDomain() + .withName(ns.prefix("testDomain2")) + .withDomainType(CreateDomain.DomainType.AGGREGATE) + .withDescription("Test domain 2"); + Domain domain2 = domainSvc.create(createDomain2); + + String original = JsonUtils.pojoToJson(page); + page.withDomains(List.of(domain1.getEntityReference())); + page = patchPage(rest, page.getId(), original, page); + + Page fetchedPage = getPage(rest, page.getId(), "domains,relatedEntities"); + assertEquals(1, fetchedPage.getDomains().size()); + assertEquals(domain1.getName(), fetchedPage.getDomains().get(0).getName()); + + original = JsonUtils.pojoToJson(page); + page.withDomains(List.of(domain2.getEntityReference())); + page = patchPage(rest, page.getId(), original, page); + + fetchedPage = getPage(rest, page.getId(), "domains,relatedEntities"); + assertEquals(1, fetchedPage.getDomains().size()); + assertEquals(domain2.getName(), fetchedPage.getDomains().get(0).getName()); + + boolean domain1InDomains = + fetchedPage.getDomains().stream().anyMatch(ref -> domain1.getName().equals(ref.getName())); + assertEquals(false, domain1InDomains, "Old domain should be removed after update"); + + original = JsonUtils.pojoToJson(page); + page.withDomains(null); + page = patchPage(rest, page.getId(), original, page); + + fetchedPage = getPage(rest, page.getId(), "domains,relatedEntities"); + int domainCount = fetchedPage.getDomains() == null ? 0 : fetchedPage.getDomains().size(); + assertEquals(0, domainCount, "Domain should be removed"); + + boolean anyDomainInRelatedEntities = + fetchedPage.getRelatedEntities().stream().anyMatch(ref -> "domain".equals(ref.getType())); + assertEquals( + false, anyDomainInRelatedEntities, "No domains should ever appear in relatedEntities"); + } + + @Test + void testDataProductAddUpdateRemove(TestNamespace ns) throws HttpResponseException { + RestClient rest = RestClient.admin(); + OpenMetadataClient adminClient = SdkClients.adminClient(); + DomainService domainSvc = new DomainService(adminClient.getHttpClient()); + DataProductService dpSvc = new DataProductService(adminClient.getHttpClient()); + + EntityReference orgRef = getOrganizationRef(); + CreatePage createPageReq = buildCreateRequest(ns.prefix("pageDpCrud"), orgRef); + Page page = createPage(rest, createPageReq); + + CreateDomain createDomain = + new CreateDomain() + .withName(ns.prefix("testDomainDP")) + .withDomainType(CreateDomain.DomainType.AGGREGATE) + .withDescription("Test domain for data products"); + Domain domain = domainSvc.create(createDomain); + + String original = JsonUtils.pojoToJson(page); + page.withDomains(List.of(domain.getEntityReference())); + page = patchPage(rest, page.getId(), original, page); + + page = getPage(rest, page.getId(), "domains,relatedEntities"); + + CreateDataProduct createDataProduct1 = + new CreateDataProduct() + .withName(ns.prefix("testDP1")) + .withDomains(List.of(domain.getFullyQualifiedName())) + .withDescription("Test data product 1"); + DataProduct dataProduct1 = dpSvc.create(createDataProduct1); + + CreateDataProduct createDataProduct2 = + new CreateDataProduct() + .withName(ns.prefix("testDP2")) + .withDomains(List.of(domain.getFullyQualifiedName())) + .withDescription("Test data product 2"); + DataProduct dataProduct2 = dpSvc.create(createDataProduct2); + + original = JsonUtils.pojoToJson(page); + page.withDataProducts(List.of(dataProduct1.getEntityReference())); + page = patchPage(rest, page.getId(), original, page); + + Page fetchedPage = getPage(rest, page.getId(), "dataProducts,relatedEntities,domains"); + assertEquals(1, fetchedPage.getDataProducts().size()); + assertEquals(dataProduct1.getName(), fetchedPage.getDataProducts().get(0).getName()); + + original = JsonUtils.pojoToJson(page); + page.withDataProducts(List.of(dataProduct2.getEntityReference())); + page = patchPage(rest, page.getId(), original, page); + + fetchedPage = getPage(rest, page.getId(), "dataProducts,relatedEntities,domains"); + assertEquals(1, fetchedPage.getDataProducts().size()); + assertEquals(dataProduct2.getName(), fetchedPage.getDataProducts().get(0).getName()); + + boolean dataProduct1InDataProducts = + fetchedPage.getDataProducts().stream() + .anyMatch(ref -> dataProduct1.getName().equals(ref.getName())); + assertEquals( + false, dataProduct1InDataProducts, "Old dataProduct should be removed after update"); + + original = JsonUtils.pojoToJson(page); + page.withDataProducts(null); + page = patchPage(rest, page.getId(), original, page); + + fetchedPage = getPage(rest, page.getId(), "dataProducts,relatedEntities,domains"); + int dataProductCount = + fetchedPage.getDataProducts() == null ? 0 : fetchedPage.getDataProducts().size(); + assertEquals(0, dataProductCount, "DataProduct should be removed"); + + boolean anyDataProductInRelatedEntities = + fetchedPage.getRelatedEntities().stream() + .anyMatch(ref -> "dataProduct".equals(ref.getType())); + assertEquals( + false, + anyDataProductInRelatedEntities, + "No dataProducts should ever appear in relatedEntities"); + } +} diff --git a/openmetadata-integration-tests/src/test/resources/2mb-jpg-example-file.jpg b/openmetadata-integration-tests/src/test/resources/2mb-jpg-example-file.jpg new file mode 100644 index 000000000000..5bf279a9d931 Binary files /dev/null and b/openmetadata-integration-tests/src/test/resources/2mb-jpg-example-file.jpg differ diff --git a/openmetadata-integration-tests/src/test/resources/drive/sample-data.csv b/openmetadata-integration-tests/src/test/resources/drive/sample-data.csv new file mode 100644 index 000000000000..1a2457b37da4 --- /dev/null +++ b/openmetadata-integration-tests/src/test/resources/drive/sample-data.csv @@ -0,0 +1,4 @@ +name,value,category +alpha,1,finance +beta,2,ops +gamma,3,marketing diff --git a/openmetadata-integration-tests/src/test/resources/drive/sample-notes.txt b/openmetadata-integration-tests/src/test/resources/drive/sample-notes.txt new file mode 100644 index 000000000000..9e35c5e9d974 --- /dev/null +++ b/openmetadata-integration-tests/src/test/resources/drive/sample-notes.txt @@ -0,0 +1,2 @@ +Context Center upload fixture +This text file is used to verify upload, download, and file-size handling. diff --git a/openmetadata-integration-tests/src/test/resources/drive/sample-pricing.xlsx b/openmetadata-integration-tests/src/test/resources/drive/sample-pricing.xlsx new file mode 100644 index 000000000000..05020acaa480 Binary files /dev/null and b/openmetadata-integration-tests/src/test/resources/drive/sample-pricing.xlsx differ diff --git a/openmetadata-integration-tests/src/test/resources/drive/sample-report.pdf b/openmetadata-integration-tests/src/test/resources/drive/sample-report.pdf new file mode 100644 index 000000000000..db1210770179 Binary files /dev/null and b/openmetadata-integration-tests/src/test/resources/drive/sample-report.pdf differ diff --git a/openmetadata-integration-tests/src/test/resources/openmetadata-secure-test.yaml b/openmetadata-integration-tests/src/test/resources/openmetadata-secure-test.yaml index 159105f6fce0..7ff24e907b97 100644 --- a/openmetadata-integration-tests/src/test/resources/openmetadata-secure-test.yaml +++ b/openmetadata-integration-tests/src/test/resources/openmetadata-secure-test.yaml @@ -106,9 +106,15 @@ pipelineServiceClientConfiguration: fernetConfiguration: fernetKey: ihZpp5gmmDvVsgoOG6OVivKWwC9vd5JQ objectStorage: - enabled: false - provider: NOOP - maxFileSize: 5242880 + enabled: true + provider: s3 + maxFileSize: 1048576 + s3: + bucketName: test-bucket + region: us-east-1 + accessKey: minio + secretKey: minio123 + endpoint: http://placeholder:9000 # RDF Configuration - will be dynamically configured by TestSuiteBootstrap rdf: diff --git a/openmetadata-sdk/README.md b/openmetadata-sdk/README.md index 975dfcd0ff71..c1ee63470380 100644 --- a/openmetadata-sdk/README.md +++ b/openmetadata-sdk/README.md @@ -342,6 +342,76 @@ TableCollection tables = Table.list(params); The OpenMetadataClient is thread-safe and can be shared across multiple threads. The static API methods use a shared default client instance. +## Test utilities + +The SDK ships a small set of helpers under `org.openmetadata.sdk.test.*` +(e.g. `JwtAuthProvider`, `RestClient`, `SdkClients`, `TestNamespace`) for +projects that run integration tests against a real OpenMetadata server. + +Their dependencies — `java-jwt`, `jersey-client`, `jersey-apache-connector`, +`jakarta.ws.rs-api`, `httpclient`, `jakarta.json-api`, `parsson`, and +`junit-jupiter-api` — are declared on the SDK as `true` +so projects that only use the core SDK don't inherit the full JAX-RS / +JUnit stack transitively. + +If your module uses any of the `org.openmetadata.sdk.test.*` classes, add +these deps to your own pom (typically with `test`): + +```xml + + com.auth0 + java-jwt + ${jwt.version} + test + + + jakarta.ws.rs + jakarta.ws.rs-api + 3.1.0 + test + + + org.glassfish.jersey.core + jersey-client + 3.1.9 + test + + + org.glassfish.jersey.connectors + jersey-apache-connector + 3.1.9 + test + + + org.apache.httpcomponents + httpclient + 4.5.14 + test + + + jakarta.json + jakarta.json-api + 2.1.3 + test + + + org.eclipse.parsson + parsson + 1.1.7 + test + + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + +``` + +Without these, classes like `RestClient` fail to initialize with +`NoClassDefFoundError: org/glassfish/jersey/apache/connector/ApacheConnectorProvider` +at test time. + ## Examples See the [examples](examples/) directory for complete working examples: diff --git a/openmetadata-sdk/pom.xml b/openmetadata-sdk/pom.xml index f6d3f4536def..a0442247b31a 100644 --- a/openmetadata-sdk/pom.xml +++ b/openmetadata-sdk/pom.xml @@ -84,6 +84,66 @@ ${json.version} + + + + + com.auth0 + java-jwt + ${jwt.version} + true + + + + + jakarta.ws.rs + jakarta.ws.rs-api + 3.1.0 + true + + + org.glassfish.jersey.core + jersey-client + 3.1.9 + true + + + org.glassfish.jersey.connectors + jersey-apache-connector + 3.1.9 + true + + + org.apache.httpcomponents + httpclient + 4.5.14 + true + + + jakarta.json + jakarta.json-api + 2.1.3 + true + + + org.eclipse.parsson + parsson + 1.1.7 + true + + + + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + true + + org.junit.jupiter @@ -91,14 +151,14 @@ ${junit.version} test - + org.junit.jupiter junit-jupiter-engine ${junit.version} test - + org.mockito mockito-core diff --git a/openmetadata-sdk/src/main/java/org/openmetadata/sdk/test/auth/JwtAuthProvider.java b/openmetadata-sdk/src/main/java/org/openmetadata/sdk/test/auth/JwtAuthProvider.java new file mode 100644 index 000000000000..defeb207364c --- /dev/null +++ b/openmetadata-sdk/src/main/java/org/openmetadata/sdk/test/auth/JwtAuthProvider.java @@ -0,0 +1,79 @@ +/* + * Copyright 2025 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.sdk.test.auth; + +import com.auth0.jwt.JWT; +import com.auth0.jwt.algorithms.Algorithm; +import java.io.InputStream; +import java.security.KeyFactory; +import java.security.PrivateKey; +import java.security.interfaces.RSAPrivateKey; +import java.security.spec.PKCS8EncodedKeySpec; +import java.time.Instant; +import java.util.Date; + +/** + * Issues short-lived RSA256 JWTs suitable for integration tests against a local OpenMetadata + * server. Loads a test-only private key from the classpath at {@code private_key.der}. The + * caller's test harness is responsible for configuring the server with a matching public key. + */ +public final class JwtAuthProvider { + + private static final String DEFAULT_ISSUER = "open-metadata.org"; + private static final String DEFAULT_KEY_ID = "test-key"; + private static final String DEFAULT_KEY_RESOURCE = "private_key.der"; + + private static volatile PrivateKey cachedKey; + + private JwtAuthProvider() {} + + public static String tokenFor(String subject, String email, String[] roles, long ttlSeconds) { + return tokenFor(subject, email, roles, ttlSeconds, DEFAULT_ISSUER, DEFAULT_KEY_ID); + } + + public static String tokenFor( + String subject, String email, String[] roles, long ttlSeconds, String issuer, String keyId) { + Algorithm alg = Algorithm.RSA256(null, (RSAPrivateKey) loadPrivateKey()); + Instant now = Instant.now(); + var builder = + JWT.create() + .withIssuer(issuer) + .withKeyId(keyId) + .withIssuedAt(Date.from(now)) + .withExpiresAt(Date.from(now.plusSeconds(ttlSeconds))) + .withSubject(subject) + .withClaim("email", email); + if (roles != null && roles.length > 0) { + builder.withArrayClaim("roles", roles); + } + return builder.sign(alg); + } + + private static synchronized PrivateKey loadPrivateKey() { + if (cachedKey != null) { + return cachedKey; + } + try (InputStream is = + JwtAuthProvider.class.getClassLoader().getResourceAsStream(DEFAULT_KEY_RESOURCE)) { + if (is == null) { + throw new IllegalStateException(DEFAULT_KEY_RESOURCE + " not found on the test classpath"); + } + byte[] keyBytes = is.readAllBytes(); + PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(keyBytes); + cachedKey = KeyFactory.getInstance("RSA").generatePrivate(spec); + return cachedKey; + } catch (Exception e) { + throw new IllegalStateException("Failed to load test private key", e); + } + } +} diff --git a/openmetadata-sdk/src/main/java/org/openmetadata/sdk/test/util/RestClient.java b/openmetadata-sdk/src/main/java/org/openmetadata/sdk/test/util/RestClient.java new file mode 100644 index 000000000000..ec9894bd5dca --- /dev/null +++ b/openmetadata-sdk/src/main/java/org/openmetadata/sdk/test/util/RestClient.java @@ -0,0 +1,200 @@ +/* + * Copyright 2025 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.sdk.test.util; + +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.ClientBuilder; +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.client.Invocation; +import jakarta.ws.rs.client.WebTarget; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import java.time.Duration; +import java.util.Map; +import java.util.UUID; +import org.apache.http.client.HttpResponseException; +import org.glassfish.jersey.apache.connector.ApacheConnectorProvider; +import org.glassfish.jersey.client.ClientConfig; +import org.glassfish.jersey.client.ClientProperties; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.sdk.test.auth.JwtAuthProvider; + +/** + * JAX-RS REST client for integration tests targeting endpoints that the main + * {@link org.openmetadata.sdk.client.OpenMetadataClient} does not (yet) cover. Useful for raw + * REST interactions — arbitrary paths, custom query params, PATCH diff requests, and the + * {@code hardDelete=true&recursive=true} flavor of delete. + * + *

All requests are authenticated via the bearer token attached when the client is built. + */ +public class RestClient { + + private static final int CONNECT_TIMEOUT_MILLIS = (int) Duration.ofSeconds(10).toMillis(); + private static final int READ_TIMEOUT_MILLIS = (int) Duration.ofSeconds(60).toMillis(); + private static final Client SHARED_CLIENT; + + static { + ClientConfig clientConfig = + new ClientConfig() + .connectorProvider(new ApacheConnectorProvider()) + .property(ClientProperties.CONNECT_TIMEOUT, CONNECT_TIMEOUT_MILLIS) + .property(ClientProperties.READ_TIMEOUT, READ_TIMEOUT_MILLIS); + SHARED_CLIENT = ClientBuilder.newBuilder().withConfig(clientConfig).build(); + } + + private final Client client; + private final String baseUrl; + private final Map authHeaders; + + private RestClient(String baseUrl, Map authHeaders) { + this.baseUrl = baseUrl; + this.authHeaders = authHeaders; + this.client = SHARED_CLIENT; + } + + public static RestClient admin() { + String url = SdkClients.getServerUrl(); + String token = SdkClients.getAdminToken(); + return new RestClient(url, Map.of("Authorization", "Bearer " + token)); + } + + public static RestClient forUser(String email, String[] roles) { + String url = SdkClients.getServerUrl(); + String token = JwtAuthProvider.tokenFor(email.split("@")[0], email, roles, 3600); + return new RestClient(url, Map.of("Authorization", "Bearer " + token)); + } + + public T create(String path, Object request, Class responseType) + throws HttpResponseException { + Response response = + target(path).post(Entity.entity(JsonUtils.pojoToJson(request), MediaType.APPLICATION_JSON)); + return handleResponse(response, responseType); + } + + public T get(String path, Class responseType) throws HttpResponseException { + Response response = target(path).get(); + return handleResponse(response, responseType); + } + + public T getById(String path, UUID id, String fields, Class responseType) + throws HttpResponseException { + WebTarget t = webTarget(path + "/" + id); + if (fields != null && !fields.isEmpty()) { + t = t.queryParam("fields", fields); + } + Response response = addHeaders(t).get(); + return handleResponse(response, responseType); + } + + public T update(String path, Object request, Class responseType) + throws HttpResponseException { + Response response = + target(path).put(Entity.entity(JsonUtils.pojoToJson(request), MediaType.APPLICATION_JSON)); + return handleResponse(response, responseType); + } + + public T patch( + String path, UUID id, String originalJson, Object updated, Class responseType) + throws HttpResponseException { + String updatedJson = JsonUtils.pojoToJson(updated); + jakarta.json.JsonPatch patch = + jakarta.json.Json.createDiff( + jakarta.json.Json.createReader(new java.io.StringReader(originalJson)).readObject(), + jakarta.json.Json.createReader(new java.io.StringReader(updatedJson)).readObject()); + + Response response = + addHeaders(webTarget(path + "/" + id)) + .method( + "PATCH", Entity.entity(patch.toString(), MediaType.APPLICATION_JSON_PATCH_JSON)); + return handleResponse(response, responseType); + } + + public void delete(String path, UUID id) throws HttpResponseException { + Response response = target(path + "/" + id).delete(); + try { + if (response.getStatus() >= 400) { + throw new HttpResponseException(response.getStatus(), response.readEntity(String.class)); + } + } finally { + response.close(); + } + } + + public void hardDelete(String path, UUID id) throws HttpResponseException { + WebTarget t = + webTarget(path + "/" + id).queryParam("hardDelete", true).queryParam("recursive", true); + Response response = addHeaders(t).delete(); + try { + if (response.getStatus() >= 400) { + throw new HttpResponseException(response.getStatus(), response.readEntity(String.class)); + } + } finally { + response.close(); + } + } + + public T restore(String path, UUID id, Class responseType) throws HttpResponseException { + Response response = + target(path + "/restore") + .put(Entity.entity("{\"id\":\"" + id + "\"}", MediaType.APPLICATION_JSON)); + return handleResponse(response, responseType); + } + + public Response rawGet(String path) { + return target(path).get(); + } + + public Response rawPost(String path, Object body) { + return target(path).post(Entity.entity(JsonUtils.pojoToJson(body), MediaType.APPLICATION_JSON)); + } + + public Response rawPut(String path, Object body) { + return target(path).put(Entity.entity(JsonUtils.pojoToJson(body), MediaType.APPLICATION_JSON)); + } + + public Response rawDelete(String path) { + return target(path).delete(); + } + + public WebTarget webTarget(String path) { + String p = path.startsWith("/") ? path : "/" + path; + // baseUrl already ends with /api, paths start with /v1/... + return client.target(baseUrl + p); + } + + private Invocation.Builder target(String path) { + return addHeaders(webTarget(path)); + } + + private Invocation.Builder addHeaders(WebTarget target) { + Invocation.Builder builder = target.request(MediaType.APPLICATION_JSON); + for (Map.Entry header : authHeaders.entrySet()) { + builder = builder.header(header.getKey(), header.getValue()); + } + return builder; + } + + private T handleResponse(Response response, Class type) throws HttpResponseException { + try (response) { + if (response.getStatus() >= 400) { + String body = response.readEntity(String.class); + throw new HttpResponseException(response.getStatus(), body); + } + if (type == String.class) { + return type.cast(response.readEntity(String.class)); + } + String json = response.readEntity(String.class); + return JsonUtils.readValue(json, type); + } + } +} diff --git a/openmetadata-sdk/src/main/java/org/openmetadata/sdk/test/util/SdkClients.java b/openmetadata-sdk/src/main/java/org/openmetadata/sdk/test/util/SdkClients.java new file mode 100644 index 000000000000..ae01945f88bd --- /dev/null +++ b/openmetadata-sdk/src/main/java/org/openmetadata/sdk/test/util/SdkClients.java @@ -0,0 +1,168 @@ +/* + * Copyright 2025 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.sdk.test.util; + +import org.openmetadata.sdk.client.OpenMetadataClient; +import org.openmetadata.sdk.config.OpenMetadataConfig; +import org.openmetadata.sdk.test.auth.JwtAuthProvider; + +/** + * Lazily-cached {@link OpenMetadataClient} factory for integration tests. Each static accessor + * returns a client authenticated as a distinct well-known test subject (admin, ingestion-bot, + * data-steward, shared_user1, etc.), so tests can verify authorization and sharing without + * managing JWTs themselves. + * + *

Base URL is resolved from the {@code IT_BASE_URL} system property or environment variable, + * defaulting to {@code http://localhost:8585/api}. + */ +public final class SdkClients { + + private static final String BASE_URL = + System.getProperty( + "IT_BASE_URL", System.getenv().getOrDefault("IT_BASE_URL", "http://localhost:8585/api")); + + private static volatile OpenMetadataClient ADMIN_CLIENT; + private static volatile OpenMetadataClient TEST_USER_CLIENT; + private static volatile OpenMetadataClient BOT_CLIENT; + private static volatile OpenMetadataClient USER1_CLIENT; + private static volatile OpenMetadataClient USER2_CLIENT; + private static volatile OpenMetadataClient USER3_CLIENT; + private static volatile OpenMetadataClient DATA_STEWARD_CLIENT; + private static volatile OpenMetadataClient DATA_CONSUMER_CLIENT; + + private SdkClients() {} + + public static OpenMetadataClient adminClient() { + if (ADMIN_CLIENT == null) { + synchronized (SdkClients.class) { + if (ADMIN_CLIENT == null) { + ADMIN_CLIENT = createClient("admin", "admin@open-metadata.org", new String[] {"admin"}); + } + } + } + return ADMIN_CLIENT; + } + + public static OpenMetadataClient testUserClient() { + if (TEST_USER_CLIENT == null) { + synchronized (SdkClients.class) { + if (TEST_USER_CLIENT == null) { + TEST_USER_CLIENT = createClient("test", "test@open-metadata.org", new String[] {}); + } + } + } + return TEST_USER_CLIENT; + } + + public static OpenMetadataClient botClient() { + if (BOT_CLIENT == null) { + synchronized (SdkClients.class) { + if (BOT_CLIENT == null) { + BOT_CLIENT = + createClient( + "ingestion-bot", "ingestion-bot@open-metadata.org", new String[] {"bot"}); + } + } + } + return BOT_CLIENT; + } + + public static OpenMetadataClient ingestionBotClient() { + return botClient(); + } + + public static OpenMetadataClient dataStewardClient() { + if (DATA_STEWARD_CLIENT == null) { + synchronized (SdkClients.class) { + if (DATA_STEWARD_CLIENT == null) { + DATA_STEWARD_CLIENT = + createClient( + "data-steward", "data-steward@open-metadata.org", new String[] {"DataSteward"}); + } + } + } + return DATA_STEWARD_CLIENT; + } + + public static OpenMetadataClient dataConsumerClient() { + if (DATA_CONSUMER_CLIENT == null) { + synchronized (SdkClients.class) { + if (DATA_CONSUMER_CLIENT == null) { + DATA_CONSUMER_CLIENT = + createClient( + "data-consumer", + "data-consumer@open-metadata.org", + new String[] {"DataConsumer"}); + } + } + } + return DATA_CONSUMER_CLIENT; + } + + public static OpenMetadataClient user1Client() { + if (USER1_CLIENT == null) { + synchronized (SdkClients.class) { + if (USER1_CLIENT == null) { + USER1_CLIENT = + createClient("shared_user1", "shared_user1@test.openmetadata.org", new String[] {}); + } + } + } + return USER1_CLIENT; + } + + public static OpenMetadataClient user2Client() { + if (USER2_CLIENT == null) { + synchronized (SdkClients.class) { + if (USER2_CLIENT == null) { + USER2_CLIENT = + createClient("shared_user2", "shared_user2@test.openmetadata.org", new String[] {}); + } + } + } + return USER2_CLIENT; + } + + public static OpenMetadataClient user3Client() { + if (USER3_CLIENT == null) { + synchronized (SdkClients.class) { + if (USER3_CLIENT == null) { + USER3_CLIENT = + createClient("shared_user3", "shared_user3@test.openmetadata.org", new String[] {}); + } + } + } + return USER3_CLIENT; + } + + public static OpenMetadataClient createClient(String subject, String email, String[] roles) { + String token = JwtAuthProvider.tokenFor(subject, email, roles, 3600); + OpenMetadataConfig cfg = + OpenMetadataConfig.builder() + .serverUrl(BASE_URL) + .accessToken(token) + .readTimeout(300000) + .writeTimeout(300000) + .build(); + return new OpenMetadataClient(cfg); + } + + public static String getServerUrl() { + return BASE_URL; + } + + public static String getAdminToken() { + return JwtAuthProvider.tokenFor( + "admin", "admin@open-metadata.org", new String[] {"admin"}, 3600); + } +} diff --git a/openmetadata-sdk/src/main/java/org/openmetadata/sdk/test/util/TestNamespace.java b/openmetadata-sdk/src/main/java/org/openmetadata/sdk/test/util/TestNamespace.java new file mode 100644 index 000000000000..2755d1fed4b5 --- /dev/null +++ b/openmetadata-sdk/src/main/java/org/openmetadata/sdk/test/util/TestNamespace.java @@ -0,0 +1,70 @@ +/* + * Copyright 2025 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.sdk.test.util; + +import java.util.UUID; + +/** + * Per-test-method namespace for entity names. Combines a process-wide run id, a test class id, + * and the current test method id to produce collision-free prefixes when multiple tests (or + * multiple processes running in parallel) hit the same server. Call {@link #prefix(String)} or + * {@link #shortPrefix(String)} when naming entities; call {@link #uniqueShortId()} when a fresh + * unique id is needed on every call. + */ +public class TestNamespace { + private static final String RUN_ID = UUID.randomUUID().toString().replaceAll("-", ""); + private final String classId; + private String methodId; + private String cachedShortPrefix; + + public TestNamespace(String classId) { + this.classId = classId; + } + + public void setMethodId(String methodId) { + this.methodId = methodId; + this.cachedShortPrefix = null; + } + + public String prefix(String base) { + return base + "__" + RUN_ID + "__" + classId + (methodId != null ? ("__" + methodId) : ""); + } + + /** + * Returns a short prefix suitable for database entity names with length constraints. The result + * is cached per method — calling this multiple times within the same test method returns the + * same value. Use {@link #uniqueShortId()} if you need a fresh unique id on every call. + */ + public String shortPrefix() { + if (cachedShortPrefix == null) { + String shortRun = RUN_ID.substring(0, 8); + String methodHash = + methodId != null ? Integer.toHexString(Math.abs(methodId.hashCode()) % 0xFFFF) : "0"; + String uniqueSuffix = UUID.randomUUID().toString().substring(0, 4); + cachedShortPrefix = shortRun + methodHash + uniqueSuffix; + } + return cachedShortPrefix; + } + + public String shortPrefix(String base) { + return shortPrefix() + "_" + base; + } + + public String uniqueShortId() { + String shortRun = RUN_ID.substring(0, 8); + String methodHash = + methodId != null ? Integer.toHexString(Math.abs(methodId.hashCode()) % 0xFFFF) : "0"; + String uniqueSuffix = UUID.randomUUID().toString().substring(0, 4); + return shortRun + methodHash + uniqueSuffix; + } +} diff --git a/openmetadata-sdk/src/main/java/org/openmetadata/sdk/test/util/TestNamespaceExtension.java b/openmetadata-sdk/src/main/java/org/openmetadata/sdk/test/util/TestNamespaceExtension.java new file mode 100644 index 000000000000..b955013d0f73 --- /dev/null +++ b/openmetadata-sdk/src/main/java/org/openmetadata/sdk/test/util/TestNamespaceExtension.java @@ -0,0 +1,51 @@ +/* + * Copyright 2025 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.sdk.test.util; + +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ExtensionContext.Namespace; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolver; + +/** + * JUnit 5 extension that provides a fresh {@link TestNamespace} instance to every test method. + * Annotate a test class with {@code @ExtendWith(TestNamespaceExtension.class)} and declare a + * {@code TestNamespace} parameter on any test method to receive an auto-populated namespace. + */ +public class TestNamespaceExtension implements BeforeEachCallback, ParameterResolver { + + private static final Namespace NAMESPACE = Namespace.create(TestNamespaceExtension.class); + private static final String NS_KEY = "testNamespace"; + + @Override + public void beforeEach(ExtensionContext context) { + String classId = context.getRequiredTestClass().getSimpleName(); + String methodId = context.getRequiredTestMethod().getName(); + TestNamespace ns = new TestNamespace(classId); + ns.setMethodId(methodId); + context.getStore(NAMESPACE).put(NS_KEY, ns); + } + + @Override + public boolean supportsParameter( + ParameterContext parameterContext, ExtensionContext extensionContext) { + return parameterContext.getParameter().getType().equals(TestNamespace.class); + } + + @Override + public Object resolveParameter( + ParameterContext parameterContext, ExtensionContext extensionContext) { + return extensionContext.getStore(NAMESPACE).get(NS_KEY, TestNamespace.class); + } +} diff --git a/openmetadata-service/pom.xml b/openmetadata-service/pom.xml index c18bb9c65ef1..93a0e4113dce 100644 --- a/openmetadata-service/pom.xml +++ b/openmetadata-service/pom.xml @@ -31,6 +31,7 @@ 1.5.25 2.3.0 24.0.0 + 3.2.3 @@ -1017,6 +1018,51 @@ owasp-java-html-sanitizer ${owasp-html-sanitizer.version} + + + com.azure + azure-storage-blob + 12.31.1 + + + software.amazon.awssdk + cloudfront + + + software.amazon.awssdk + checksums + + + + org.apache.pdfbox + pdfbox + 2.0.31 + + + org.apache.poi + poi + 5.4.1 + + + org.apache.poi + poi-ooxml + 5.4.1 + + + org.apache.poi + poi-scratchpad + 5.4.1 + + + org.apache.tika + tika-core + ${tika.version} + + + org.apache.tika + tika-parser-ocr-module + ${tika.version} + diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index eaded403c79e..fbdc582784a7 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -201,6 +201,9 @@ public final class Entity { public static final String FILE = "file"; public static final String SPREADSHEET = "spreadsheet"; public static final String WORKSHEET = "worksheet"; + public static final String FOLDER = "folder"; + public static final String CONTEXT_FILE = "contextFile"; + public static final String CONTEXT_FILE_CONTENT = "contextFileContent"; public static final String GLOSSARY = "glossary"; public static final String GLOSSARY_TERM = "glossaryTerm"; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedSearchIndexExecutor.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedSearchIndexExecutor.java index 427cc4ad4625..0950a0c490eb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedSearchIndexExecutor.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedSearchIndexExecutor.java @@ -1102,8 +1102,10 @@ private void initializeEntityTracker(UUID jobId, boolean recreateIndex) { // Set up per-entity promotion callback if recreating indices if (recreateIndex && recreateContext != null) { this.recreateIndexHandler = Entity.getSearchRepository().createReindexHandler(); - // Wire job configuration so applyLiveServingSettings can revert bulk-build overrides - // (refresh=-1, replicas=0, async translog) before the per-entity alias swap. + // Wire jobData into the handler so applyLiveServingSettings can revert bulk-build + // overrides (refresh_interval=-1, replicas=0, async translog) before the per-entity + // alias swap. Without this, buildRevertJson returns null and the bulk overrides + // silently become the live settings. if (recreateIndexHandler instanceof DefaultRecreateHandler defaultHandler && currentJob != null && currentJob.getJobConfiguration() != null) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/attachments/AssetService.java b/openmetadata-service/src/main/java/org/openmetadata/service/attachments/AssetService.java new file mode 100644 index 000000000000..bcd26e015ead --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/attachments/AssetService.java @@ -0,0 +1,42 @@ +package org.openmetadata.service.attachments; + +import java.io.InputStream; +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import org.openmetadata.schema.attachments.Asset; + +public interface AssetService extends AutoCloseable { + CompletableFuture upload(Asset asset, InputStream content); + + CompletableFuture read(Asset asset); + + CompletableFuture delete(Asset asset); + + default String generateDownloadURL(Asset asset) { + return asset.getUrl(); + } + + String generateDownloadUrlWithExpiry(Asset asset, Duration expiry); + + /** + * Default no-op for providers that hold no closeable resources (in-memory, no-op, + * Azure — whose BlobServiceClient has no explicit close). Providers that own + * SDK clients with connection pools (e.g. S3) should override to release them on + * application shutdown. + */ + @Override + default void close() {} + + default String determineBasePathPrefix(String[] pathParts) { + if (pathParts.length <= 1) { + return ""; + } + + String prefix = pathParts[1]; + if (!prefix.endsWith("/")) { + prefix += "/"; + } + + return prefix; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/attachments/AssetServiceFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/attachments/AssetServiceFactory.java new file mode 100644 index 000000000000..7186b164f7c7 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/attachments/AssetServiceFactory.java @@ -0,0 +1,132 @@ +package org.openmetadata.service.attachments; + +import java.util.Locale; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.service.OpenMetadataApplicationConfig; +import org.openmetadata.service.config.ObjectStorageConfiguration; + +@Slf4j +public class AssetServiceFactory { + private static AssetService instance; + private static boolean shutdownHookRegistered; + + public static synchronized void init(OpenMetadataApplicationConfig config) { + registerShutdownHook(); + ObjectStorageConfiguration objectStorageConfiguration = config.getObjectStorage(); + if (objectStorageConfiguration == null || !objectStorageConfiguration.isEnabled()) { + // Storage disabled — always swap to a fresh NoOp provider. If a previous init + // wired up S3/Azure/InMemory, leaving that instance live after a reload to the + // disabled state would keep serving real uploads/downloads against the old + // backend, which hides the misconfiguration and leaks connections in tests. + if (!(instance instanceof NoOpAssetService)) { + closeCurrent(); + instance = new NoOpAssetService(); + } + return; + } + + String provider = validateProvider(objectStorageConfiguration.getProvider()); + if (isInitializedForProvider(provider)) { + return; + } + closeCurrent(); + + AssetService delegate; + String normalizedProvider = provider.toLowerCase(Locale.ROOT); + if ("s3".equals(normalizedProvider)) { + delegate = new S3AssetService(objectStorageConfiguration.getS3Configuration()); + } else if ("azure".equals(normalizedProvider)) { + delegate = new AzureAssetService(objectStorageConfiguration.getAzureConfiguration()); + } else if ("inmemory".equals(normalizedProvider) || "in-memory".equals(normalizedProvider)) { + LOG.info("Using InMemoryAssetService for local testing"); + delegate = new InMemoryAssetService(); + } else if ("noop".equals(normalizedProvider)) { + delegate = new NoOpAssetService(); + } else { + throw new IllegalArgumentException("Unsupported asset uploader provider: " + provider); + } + instance = new QueuedDeleteAssetService(delegate, ObjectDeleteQueueService.getInstance()); + } + + private static String validateProvider(String provider) { + if (provider == null || provider.isBlank()) { + throw new IllegalArgumentException( + "Object storage provider must be configured when object storage is enabled."); + } + return provider.trim(); + } + + private static boolean isInitializedForProvider(String provider) { + if (instance == null || provider == null || provider.isBlank()) { + return false; + } + AssetService unwrapped = unwrap(instance); + return switch (provider.toLowerCase(Locale.ROOT)) { + case "s3" -> unwrapped instanceof S3AssetService; + case "azure" -> unwrapped instanceof AzureAssetService; + case "inmemory", "in-memory" -> unwrapped instanceof InMemoryAssetService; + case "noop" -> unwrapped instanceof NoOpAssetService; + default -> false; + }; + } + + /** + * Returns the concrete {@link AssetService} implementation, stripping any wrapper layers such as + * {@link QueuedDeleteAssetService}. Callers that need to inspect provider capabilities + * (e.g. {@code instanceof S3AssetService}) should go through this helper because the wrapper + * hides the delegate from direct type checks. + */ + public static AssetService unwrap(AssetService service) { + AssetService current = service; + while (current instanceof QueuedDeleteAssetService queuedService) { + current = queuedService.getDelegate(); + } + return current; + } + + public static AssetService getService() { + if (instance == null) { + throw new IllegalStateException( + "AssetService not initialized. Please make sure ObjectStorage is configured."); + } + return instance; + } + + /** + * Close the current instance if it owns lifecycle resources (e.g. S3Client / S3Presigner + * connection pools). Safe to call with no instance or an already-closed instance. + */ + public static synchronized void shutdown() { + AssetService current = instance; + if (current == null) { + return; + } + try { + current.close(); + } catch (Exception e) { + LOG.warn("Failed to close AssetService cleanly", e); + } + instance = null; + } + + private static void closeCurrent() { + AssetService current = instance; + if (current == null) { + return; + } + try { + current.close(); + } catch (Exception e) { + LOG.warn("Failed to close previous AssetService cleanly", e); + } + } + + private static void registerShutdownHook() { + if (shutdownHookRegistered) { + return; + } + Runtime.getRuntime() + .addShutdownHook(new Thread(AssetServiceFactory::shutdown, "asset-service-shutdown")); + shutdownHookRegistered = true; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/attachments/AzureAssetService.java b/openmetadata-service/src/main/java/org/openmetadata/service/attachments/AzureAssetService.java new file mode 100644 index 000000000000..cd08f0f96afc --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/attachments/AzureAssetService.java @@ -0,0 +1,184 @@ +package org.openmetadata.service.attachments; + +import com.azure.identity.DefaultAzureCredentialBuilder; +import com.azure.storage.blob.*; +import com.azure.storage.blob.models.*; +import com.azure.storage.blob.sas.*; +import com.azure.storage.common.sas.SasProtocol; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.time.Duration; +import java.time.OffsetDateTime; +import java.util.concurrent.CompletableFuture; +import org.apache.commons.io.IOUtils; +import org.openmetadata.schema.attachments.Asset; +import org.openmetadata.sdk.exception.AssetServiceException; +import org.openmetadata.service.config.AzureConfiguration; +import org.openmetadata.service.util.AsyncService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AzureAssetService implements AssetService { + private static final Logger LOG = LoggerFactory.getLogger(AzureAssetService.class); + + private final AzureConfiguration config; + private final BlobServiceClient blobServiceClient; + private final BlobContainerClient containerClient; + private final String basePathPrefix; + + public AzureAssetService(AzureConfiguration config) { + this.config = config; + this.basePathPrefix = formatPrefix(config.getPrefixPath()); + + if (config.getBlobEndpoint() == null || config.getBlobEndpoint().isEmpty()) { + throw new IllegalArgumentException("blobEndpoint must be provided in Azure configuration"); + } + + this.blobServiceClient = + new BlobServiceClientBuilder() + .endpoint(config.getBlobEndpoint()) + .credential(new DefaultAzureCredentialBuilder().build()) + .buildClient(); + + this.containerClient = blobServiceClient.getBlobContainerClient(config.getContainerName()); + initializeContainer(); + } + + /** + * Normalize a configured prefix so that a null/blank prefix becomes an empty string + * (not the literal "null/") and any non-empty prefix ends with exactly one "/". + * Matches {@link S3AssetService#formatPrefix(String)} so both providers lay out + * blobs the same way. + */ + private static String formatPrefix(String rawPrefix) { + if (rawPrefix == null || rawPrefix.isBlank()) { + return ""; + } + String trimmed = rawPrefix.trim(); + return trimmed.endsWith("/") ? trimmed : trimmed + "/"; + } + + private void initializeContainer() { + try { + if (!containerClient.exists()) { + containerClient.create(); + LOG.info("Created Azure blob container: {}", containerClient.getBlobContainerName()); + } + createDirectoryMarker(); + } catch (Exception e) { + LOG.error("Failed to initialize Azure blob container: {}", e.getMessage(), e); + throw new RuntimeException("Failed to initialize Azure blob container", e); + } + } + + private void createDirectoryMarker() { + String markerPath = basePathPrefix + ".directory"; + BlobClient blobClient = containerClient.getBlobClient(markerPath); + if (!blobClient.exists()) { + blobClient.upload(new ByteArrayInputStream(new byte[0]), 0, true); + } + } + + @Override + public CompletableFuture upload(Asset asset, InputStream content) { + return AsyncService.executeAsync( + () -> { + String fullPath = basePathPrefix + asset.getId(); + BlobClient blobClient = containerClient.getBlobClient(fullPath); + + // Stream the upload straight through to Azure using the known size on the + // Asset. Previously we read the whole payload into a byte[] via + // IOUtils.toByteArray, which put full-file pressure on heap for every + // upload and risked OOM for larger files. Fall back to buffering only + // when the upload hasn't populated a size (shouldn't happen in the + // production path, but keeps this resilient to unusual callers). + try { + Long size = asset.getSize() == null ? null : asset.getSize().longValue(); + if (size != null && size >= 0) { + blobClient.upload(content, size, true); + } else { + byte[] bytes = IOUtils.toByteArray(content); + blobClient.upload(new ByteArrayInputStream(bytes), bytes.length, true); + } + blobClient.setHttpHeaders(new BlobHttpHeaders().setContentType(asset.getContentType())); + return generateDownloadUrlWithExpiry(asset, Duration.ofMinutes(15)); + } catch (IOException e) { + throw AssetServiceException.byMessage( + "Failed to upload asset: " + asset.getId(), e.getMessage()); + } + }, + "Upload", + asset.getId()); + } + + @Override + public CompletableFuture read(Asset asset) { + // Open the blob on the caller's thread (see S3AssetService.read for the + // full rationale) — every read() caller immediately joins, so wrapping + // through AsyncService only added scheduling overhead and a starvation + // path when AsyncService was saturated. + try { + LOG.debug("Reading asset {} from Azure blob storage", asset.getId()); + BlobClient blobClient = containerClient.getBlobClient(basePathPrefix + asset.getId()); + InputStream inputStream = blobClient.openInputStream(); + LOG.debug("Successfully opened input stream for asset {}", asset.getId()); + return CompletableFuture.completedFuture(inputStream); + } catch (Exception e) { + CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally( + AssetServiceException.byMessage( + "Failed to read asset: " + asset.getId(), e.getMessage())); + return failed; + } + } + + @Override + public CompletableFuture delete(Asset asset) { + return AsyncService.executeAsync( + () -> { + try { + BlobClient blobClient = containerClient.getBlobClient(basePathPrefix + asset.getId()); + blobClient.delete(); + LOG.debug("Successfully deleted asset {}", asset.getId()); + return null; + } catch (Exception e) { + throw AssetServiceException.byMessage( + "Failed to delete asset: " + asset.getId(), e.getMessage()); + } + }, + "Delete", + asset.getId()); + } + + @Override + public String generateDownloadUrlWithExpiry(Asset asset, Duration expiry) { + try { + String blobName = basePathPrefix + asset.getId(); + BlobClient blobClient = containerClient.getBlobClient(blobName); + + OffsetDateTime start = OffsetDateTime.now().minusMinutes(5); + OffsetDateTime end = OffsetDateTime.now().plus(expiry); + UserDelegationKey userDelegationKey = blobServiceClient.getUserDelegationKey(start, end); + + BlobSasPermission permission = new BlobSasPermission().setReadPermission(true); + BlobServiceSasSignatureValues sasValues = + new BlobServiceSasSignatureValues(end, permission) + .setStartTime(start) + .setProtocol(SasProtocol.HTTPS_ONLY) + .setBlobName(blobName) + .setContainerName(containerClient.getBlobContainerName()); + + String sasToken = blobClient.generateUserDelegationSas(sasValues, userDelegationKey); + return blobClient.getBlobUrl() + "?" + sasToken; + } catch (Exception e) { + LOG.error("Failed to generate SAS token for asset: {}", asset.getId(), e); + throw new RuntimeException("Could not generate SAS token", e); + } + } + + @Override + public String generateDownloadURL(Asset asset) { + return generateDownloadUrlWithExpiry(asset, Duration.ofMinutes(15)); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/attachments/InMemoryAssetService.java b/openmetadata-service/src/main/java/org/openmetadata/service/attachments/InMemoryAssetService.java new file mode 100644 index 000000000000..d77280136a49 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/attachments/InMemoryAssetService.java @@ -0,0 +1,155 @@ +package org.openmetadata.service.attachments; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.attachments.Asset; +import org.openmetadata.service.util.AsyncService; + +/** + * In-memory implementation of AssetService for local testing and development. + * Stores asset contents in memory using a ConcurrentHashMap. + * + * WARNING: This implementation is NOT suitable for production use as: + * - Data is lost on restart + * - Memory usage grows with asset size + * - Not distributed/shared across instances + */ +@Slf4j +public class InMemoryAssetService implements AssetService { + private final ConcurrentHashMap assetStore; + private final String baseUrl; + + public InMemoryAssetService() { + this("http://localhost:8585/api/v1/assets"); + } + + public InMemoryAssetService(String baseUrl) { + this.assetStore = new ConcurrentHashMap<>(); + this.baseUrl = baseUrl; + LOG.info("Initialized InMemoryAssetService for local testing (base URL: {})", baseUrl); + } + + /** + * Run async work on the shared OM {@link AsyncService} executor so server-side + * concurrency is bounded and observable, rather than falling back to the JVM + * common ForkJoinPool. + */ + private static Executor executor() { + return AsyncService.getInstance().getExecutorService(); + } + + @Override + public CompletableFuture upload(Asset asset, InputStream content) { + return CompletableFuture.supplyAsync( + () -> { + try { + // Read the input stream into a byte array + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + byte[] data = new byte[8192]; + int bytesRead; + while ((bytesRead = content.read(data, 0, data.length)) != -1) { + buffer.write(data, 0, bytesRead); + } + byte[] assetBytes = buffer.toByteArray(); + + // Store in memory + assetStore.put(asset.getId(), assetBytes); + + LOG.debug( + "Uploaded asset {} ({} bytes) to in-memory storage", + asset.getId(), + assetBytes.length); + + return "success"; + } catch (Exception e) { + LOG.error("Failed to upload asset {}: {}", asset.getId(), e.getMessage(), e); + throw new RuntimeException("Failed to upload asset", e); + } + }, + executor()); + } + + @Override + public CompletableFuture read(Asset asset) { + // Return synchronously — the in-memory fetch is trivial and every caller + // immediately joins on the returned future. Matches S3AssetService.read + // and AzureAssetService.read so none of the providers route read traffic + // through AsyncService (callers already block, no benefit to queueing). + byte[] assetBytes = assetStore.get(asset.getId()); + if (assetBytes == null) { + LOG.warn("Asset {} not found in in-memory storage", asset.getId()); + return CompletableFuture.completedFuture(null); + } + LOG.debug( + "Retrieved asset {} ({} bytes) from in-memory storage", asset.getId(), assetBytes.length); + return CompletableFuture.completedFuture(new ByteArrayInputStream(assetBytes)); + } + + @Override + public CompletableFuture delete(Asset asset) { + return CompletableFuture.runAsync( + () -> { + byte[] removed = assetStore.remove(asset.getId()); + if (removed != null) { + LOG.debug( + "Deleted asset {} ({} bytes) from in-memory storage", + asset.getId(), + removed.length); + } else { + LOG.warn("Attempted to delete non-existent asset {}", asset.getId()); + } + }, + executor()); + } + + @Override + public String generateDownloadUrlWithExpiry(Asset asset, Duration expiry) { + // For in-memory storage, we just return a mock URL + // In a real implementation, this would require a separate endpoint to serve the assets + String url = baseUrl + "/" + asset.getId() + "?expiry=" + expiry.toSeconds(); + LOG.debug("Generated mock download URL for asset {}: {}", asset.getId(), url); + return url; + } + + /** + * Match S3/Azure providers by delegating the no-expiry entry point to the expiry + * variant. The default in {@link AssetService} returns {@code asset.getUrl()} which + * for in-memory assets is never set (the stored URL is always empty), leading to + * broken download links for callers that use the non-expiry API. + */ + @Override + public String generateDownloadURL(Asset asset) { + return generateDownloadUrlWithExpiry(asset, Duration.ofMinutes(15)); + } + + /** + * Get the current size of the in-memory store (for debugging/monitoring) + * @return number of assets stored + */ + public int getStoreSize() { + return assetStore.size(); + } + + /** + * Get the total memory used by stored assets (approximate) + * @return total bytes stored + */ + public long getTotalBytesStored() { + return assetStore.values().stream().mapToLong(bytes -> bytes.length).sum(); + } + + /** + * Clear all assets from memory (useful for testing) + */ + public void clear() { + int size = assetStore.size(); + assetStore.clear(); + LOG.info("Cleared {} assets from in-memory storage", size); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/attachments/NoOpAssetService.java b/openmetadata-service/src/main/java/org/openmetadata/service/attachments/NoOpAssetService.java new file mode 100644 index 000000000000..3916786dd68b --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/attachments/NoOpAssetService.java @@ -0,0 +1,51 @@ +package org.openmetadata.service.attachments; + +import java.io.InputStream; +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import org.openmetadata.schema.attachments.Asset; + +public class NoOpAssetService implements AssetService { + @Override + public CompletableFuture upload(Asset asset, InputStream content) { + return CompletableFuture.completedFuture(""); + } + + @Override + public CompletableFuture read(Asset asset) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture delete(Asset asset) { + return CompletableFuture.completedFuture(null); + } + + /** + * Return the asset's own URL when present, otherwise an empty string. We deliberately + * avoid returning a synthetic CDN URL here — a fake URL would let clients issue + * downloads that can never succeed and would mask the "storage disabled" + * misconfiguration. {@link org.openmetadata.schema.attachments.Asset#getUrl()} is + * optional in the schema, so normalize null/blank to "" to preserve the non-null + * contract callers rely on. + */ + @Override + public String generateDownloadUrlWithExpiry(Asset asset, Duration expiry) { + if (asset == null) { + return ""; + } + String url = asset.getUrl(); + return url == null || url.isBlank() ? "" : url; + } + + /** + * Keep {@link #generateDownloadURL(Asset)} aligned with the expiry variant so the two + * entry points never disagree. The default {@code AssetService} implementation returns + * {@code asset.getUrl()} as-is (potentially {@code null}); delegating ensures NoOp + * always satisfies the non-null contract. + */ + @Override + public String generateDownloadURL(Asset asset) { + return generateDownloadUrlWithExpiry(asset, Duration.ofMinutes(15)); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/attachments/ObjectDeleteQueueService.java b/openmetadata-service/src/main/java/org/openmetadata/service/attachments/ObjectDeleteQueueService.java new file mode 100644 index 000000000000..837f7404f762 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/attachments/ObjectDeleteQueueService.java @@ -0,0 +1,198 @@ +package org.openmetadata.service.attachments; + +import io.dropwizard.lifecycle.Managed; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.Semaphore; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ObjectDeleteQueueService implements Managed { + static final int DEFAULT_WORKER_COUNT = + Integer.getInteger( + "collate.object.delete.workers", + Math.max(2, Math.min(4, Runtime.getRuntime().availableProcessors()))); + static final int DEFAULT_QUEUE_CAPACITY = + Integer.getInteger("collate.object.delete.queue.capacity", 128); + static final long DEFAULT_ENQUEUE_TIMEOUT_MILLIS = + Long.getLong("collate.object.delete.enqueue.timeout.ms", 5000L); + static final long DEFAULT_KEEP_ALIVE_MILLIS = + Long.getLong("collate.object.delete.keepalive.ms", 5000L); + + private static final ObjectDeleteQueueService INSTANCE = createInstance(); + + private static ObjectDeleteQueueService createInstance() { + ObjectDeleteQueueService service = + new ObjectDeleteQueueService( + DEFAULT_WORKER_COUNT, DEFAULT_QUEUE_CAPACITY, DEFAULT_ENQUEUE_TIMEOUT_MILLIS); + // Ensure the non-daemon worker threads are drained on JVM exit even when Dropwizard's + // Managed.stop() wasn't invoked (e.g. when the server is run outside of a full + // application lifecycle, or if the stop hook is missed). Without this, ungracefully + // terminated servers can leave orphan threads that prevent the JVM from exiting. + Runtime.getRuntime() + .addShutdownHook( + new Thread( + () -> { + try { + service.stop(); + } catch (Exception e) { + // Best-effort shutdown — log at JVM-exit-time is fine here. + LOG.warn("Failed to cleanly stop ObjectDeleteQueueService on JVM exit", e); + } + }, + "object-delete-queue-shutdown")); + return service; + } + + private final ThreadPoolExecutor executorService; + private final Semaphore capacitySemaphore; + private final int workerCount; + private final int queueCapacity; + private final long enqueueTimeoutMillis; + + ObjectDeleteQueueService(int workerCount, int queueCapacity, long enqueueTimeoutMillis) { + if (workerCount <= 0) { + throw new IllegalArgumentException("workerCount must be > 0"); + } + if (queueCapacity < 0) { + throw new IllegalArgumentException("queueCapacity must be >= 0"); + } + if (enqueueTimeoutMillis < 0) { + throw new IllegalArgumentException("enqueueTimeoutMillis must be >= 0"); + } + + this.workerCount = workerCount; + this.queueCapacity = queueCapacity; + this.enqueueTimeoutMillis = enqueueTimeoutMillis; + this.capacitySemaphore = new Semaphore(workerCount + queueCapacity, true); + // queueCapacity == 0 means "reject when all workers are busy, no buffering". + // SynchronousQueue preserves that semantic; ArrayBlockingQueue(1) would silently + // buffer one task past the semaphore's accounting. + BlockingQueue workQueue = + queueCapacity == 0 ? new SynchronousQueue<>() : new ArrayBlockingQueue<>(queueCapacity); + this.executorService = + new ThreadPoolExecutor( + workerCount, + workerCount, + DEFAULT_KEEP_ALIVE_MILLIS, + TimeUnit.MILLISECONDS, + workQueue, + new DeleteThreadFactory(), + new ThreadPoolExecutor.AbortPolicy()); + this.executorService.allowCoreThreadTimeOut(true); + } + + public static ObjectDeleteQueueService getInstance() { + return INSTANCE; + } + + public CompletableFuture submit(String jobLabel, Runnable task) { + try { + if (!capacitySemaphore.tryAcquire(enqueueTimeoutMillis, TimeUnit.MILLISECONDS)) { + throw buildQueueSaturatedException(jobLabel); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RejectedExecutionException( + "Interrupted while waiting for delete queue capacity", e); + } + + CompletableFuture result = new CompletableFuture<>(); + try { + executorService.execute( + () -> { + try { + task.run(); + result.complete(null); + } catch (Throwable t) { + result.completeExceptionally(t); + } finally { + capacitySemaphore.release(); + } + }); + } catch (RejectedExecutionException e) { + capacitySemaphore.release(); + if (executorService.isShutdown()) { + throw new RejectedExecutionException( + "Delete queue is shutting down, cannot accept job: " + jobLabel); + } + throw buildQueueSaturatedException(jobLabel); + } + + return result; + } + + public int getWorkerCount() { + return workerCount; + } + + public int getQueueCapacity() { + return queueCapacity; + } + + public long getEnqueueTimeoutMillis() { + return enqueueTimeoutMillis; + } + + public int getActiveCount() { + return executorService.getActiveCount(); + } + + public int getQueueDepth() { + return executorService.getQueue().size(); + } + + public int getTotalCapacity() { + return workerCount + queueCapacity; + } + + @Override + public void start() { + // Executor is initialized eagerly. + } + + @Override + public void stop() { + executorService.shutdown(); + try { + if (!executorService.awaitTermination(30, TimeUnit.SECONDS)) { + LOG.warn("Delete queue did not terminate within 30s, forcing shutdown"); + executorService.shutdownNow(); + } + } catch (InterruptedException e) { + executorService.shutdownNow(); + Thread.currentThread().interrupt(); + } + } + + private RejectedExecutionException buildQueueSaturatedException(String jobLabel) { + LOG.warn( + "Object delete queue is full for job {}. active={}, queued={}, capacity={}", + jobLabel, + getActiveCount(), + getQueueDepth(), + getTotalCapacity()); + return new RejectedExecutionException( + String.format( + "Object delete queue is full. active=%d queued=%d capacity=%d", + getActiveCount(), getQueueDepth(), getTotalCapacity())); + } + + private static final class DeleteThreadFactory implements ThreadFactory { + private final AtomicInteger counter = new AtomicInteger(1); + + @Override + public Thread newThread(Runnable runnable) { + Thread thread = new Thread(runnable, "object-delete-worker-" + counter.getAndIncrement()); + thread.setDaemon(false); + return thread; + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/attachments/QueuedDeleteAssetService.java b/openmetadata-service/src/main/java/org/openmetadata/service/attachments/QueuedDeleteAssetService.java new file mode 100644 index 000000000000..786cb646946d --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/attachments/QueuedDeleteAssetService.java @@ -0,0 +1,91 @@ +package org.openmetadata.service.attachments; + +import java.io.InputStream; +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.openmetadata.schema.attachments.Asset; + +public class QueuedDeleteAssetService implements AssetService { + static final long DEFAULT_DELETE_WAIT_MILLIS = + Long.getLong("collate.object.delete.task.timeout.ms", 60000L); + + private final AssetService delegate; + private final ObjectDeleteQueueService deleteQueueService; + private final long deleteWaitMillis; + + public QueuedDeleteAssetService( + AssetService delegate, ObjectDeleteQueueService deleteQueueService) { + this(delegate, deleteQueueService, DEFAULT_DELETE_WAIT_MILLIS); + } + + QueuedDeleteAssetService( + AssetService delegate, ObjectDeleteQueueService deleteQueueService, long deleteWaitMillis) { + this.delegate = delegate; + this.deleteQueueService = deleteQueueService; + if (deleteWaitMillis <= 0) { + throw new IllegalArgumentException("deleteWaitMillis must be > 0"); + } + this.deleteWaitMillis = deleteWaitMillis; + } + + AssetService getDelegate() { + return delegate; + } + + @Override + public CompletableFuture upload(Asset asset, InputStream content) { + return delegate.upload(asset, content); + } + + @Override + public CompletableFuture read(Asset asset) { + return delegate.read(asset); + } + + @Override + public CompletableFuture delete(Asset asset) { + return deleteQueueService.submit( + "asset:" + asset.getId(), + () -> { + CompletableFuture deleteFuture = delegate.delete(asset); + if (deleteFuture != null) { + waitForDelete(deleteFuture, asset.getId()); + } + }); + } + + private void waitForDelete(CompletableFuture deleteFuture, String assetId) { + try { + deleteFuture.get(deleteWaitMillis, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + deleteFuture.cancel(true); + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while deleting asset " + assetId, e); + } catch (TimeoutException e) { + deleteFuture.cancel(true); + throw new IllegalStateException( + "Timed out deleting asset %s after %d ms".formatted(assetId, deleteWaitMillis), e); + } catch (ExecutionException e) { + Throwable cause = e.getCause() != null ? e.getCause() : e; + throw new IllegalStateException("Delete failed for asset " + assetId, cause); + } + } + + @Override + public String generateDownloadURL(Asset asset) { + return delegate.generateDownloadURL(asset); + } + + @Override + public String generateDownloadUrlWithExpiry(Asset asset, Duration expiry) { + return delegate.generateDownloadUrlWithExpiry(asset, expiry); + } + + @Override + public void close() { + delegate.close(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/attachments/S3AssetService.java b/openmetadata-service/src/main/java/org/openmetadata/service/attachments/S3AssetService.java new file mode 100644 index 000000000000..735459ee4777 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/attachments/S3AssetService.java @@ -0,0 +1,247 @@ +package org.openmetadata.service.attachments; + +import java.io.InputStream; +import java.net.URI; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.common.utils.CommonUtil; +import org.openmetadata.schema.attachments.Asset; +import org.openmetadata.service.config.S3Configuration; +import org.openmetadata.service.util.AsyncService; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.cloudfront.CloudFrontUtilities; +import software.amazon.awssdk.services.cloudfront.model.CustomSignerRequest; +import software.amazon.awssdk.services.cloudfront.url.SignedUrl; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; +import software.amazon.awssdk.services.s3.model.*; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; +import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; +import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; + +@Slf4j +public class S3AssetService implements AssetService { + private final S3Configuration config; + private final S3Client s3Client; + private final S3Presigner presigner; + private final CloudFrontUtilities cloudFrontUtilities; + private final String actualBucketName; + private final String prefixPath; + + public S3AssetService(S3Configuration config) { + this.config = config; + this.actualBucketName = config.getBucketName(); + this.prefixPath = formatPrefix(config.getPrefixPath()); + + AwsCredentialsProvider credentialsProvider = resolveCredentials(config); + URI endpointOverride = + CommonUtil.nullOrEmpty(config.getEndpoint()) ? null : URI.create(config.getEndpoint()); + software.amazon.awssdk.services.s3.S3Configuration serviceConfiguration = + software.amazon.awssdk.services.s3.S3Configuration.builder() + .pathStyleAccessEnabled(endpointOverride != null) + .build(); + + S3ClientBuilder builder = + S3Client.builder() + .region(Region.of(config.getRegion())) + .credentialsProvider(credentialsProvider) + .serviceConfiguration(serviceConfiguration); + + if (endpointOverride != null) { + builder.endpointOverride(endpointOverride); + } + + this.s3Client = builder.build(); + S3Presigner.Builder presignerBuilder = + S3Presigner.builder() + .region(Region.of(config.getRegion())) + .credentialsProvider(credentialsProvider) + .serviceConfiguration(serviceConfiguration); + if (endpointOverride != null) { + presignerBuilder.endpointOverride(endpointOverride); + } + this.presigner = presignerBuilder.build(); + + this.cloudFrontUtilities = CloudFrontUtilities.create(); + } + + @Override + public void close() { + // S3Client and S3Presigner both hold HTTP connection pools backed by the AWS SDK — + // release them on shutdown so the JVM doesn't leak pool threads / sockets. + try { + s3Client.close(); + } catch (Exception e) { + LOG.warn("Failed to close S3 client cleanly", e); + } + try { + presigner.close(); + } catch (Exception e) { + LOG.warn("Failed to close S3 presigner cleanly", e); + } + } + + private AwsCredentialsProvider resolveCredentials(S3Configuration config) { + if (config.getEndpoint() != null && !config.getEndpoint().isEmpty()) { + LOG.info("Custom endpoint detected, using StaticCredentialsProvider"); + return StaticCredentialsProvider.create( + AwsBasicCredentials.create(config.getAccessKey(), config.getSecretKey())); + } + try { + AwsCredentialsProvider defaultProvider = DefaultCredentialsProvider.create(); + defaultProvider.resolveCredentials(); // Triggers validation + LOG.info("Using AWS DefaultCredentialsProvider"); + return defaultProvider; + } catch (Exception e) { + LOG.warn( + "Default credentials not found. Falling back to static credentials. Reason: {}", + e.getMessage()); + return StaticCredentialsProvider.create( + AwsBasicCredentials.create(config.getAccessKey(), config.getSecretKey())); + } + } + + private String formatPrefix(String rawPrefix) { + if (CommonUtil.nullOrEmpty(rawPrefix)) return ""; + return rawPrefix.endsWith("/") ? rawPrefix : rawPrefix + "/"; + } + + private String resolveKey(String assetId) { + return prefixPath + assetId; + } + + @Override + public CompletableFuture upload(Asset asset, InputStream content) { + return AsyncService.executeAsync( + () -> { + try { + String key = resolveKey(asset.getId()); + PutObjectRequest.Builder putBuilder = + PutObjectRequest.builder() + .bucket(actualBucketName) + .key(key) + .contentType(asset.getContentType()); + + if (config.getSseAlgorithm() != null && !config.getSseAlgorithm().isEmpty()) { + if ("AES256".equals(config.getSseAlgorithm())) { + putBuilder.serverSideEncryption(ServerSideEncryption.AES256); + } else if ("aws:kms".equals(config.getSseAlgorithm())) { + putBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS); + if (config.getKmsKeyId() != null && !config.getKmsKeyId().isEmpty()) { + putBuilder.ssekmsKeyId(config.getKmsKeyId()); + } + } + } + + PutObjectRequest putRequest = putBuilder.build(); + s3Client.putObject( + putRequest, RequestBody.fromInputStream(content, asset.getSize().longValue())); + return "success"; + } catch (Exception e) { + throw new CompletionException(e); + } + }, + "Upload", + asset.getId()); + } + + @Override + public CompletableFuture read(Asset asset) { + // Open the S3 object on the caller's thread rather than hopping through + // AsyncService. Every caller of read() immediately joins on the returned + // future, so routing the blocking getObject through AsyncService's bounded + // pool just added scheduling overhead and created a starvation path — when + // a caller already running on AsyncService (or a caller that can monopolize + // AsyncService throughput) blocks on join(), the submitted read task has to + // fight for a worker before it can run. + try { + LOG.debug("Reading asset {} from S3 bucket {}", asset.getId(), actualBucketName); + String key = resolveKey(asset.getId()); + GetObjectRequest getRequest = + GetObjectRequest.builder().bucket(actualBucketName).key(key).build(); + InputStream inputStream = s3Client.getObject(getRequest); + LOG.debug("Successfully opened input stream for asset {}", asset.getId()); + return CompletableFuture.completedFuture(inputStream); + } catch (Exception e) { + CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally(e); + return failed; + } + } + + @Override + public CompletableFuture delete(Asset asset) { + return AsyncService.executeAsync( + () -> { + try { + String key = resolveKey(asset.getId()); + DeleteObjectRequest deleteRequest = + DeleteObjectRequest.builder().bucket(actualBucketName).key(key).build(); + + s3Client.deleteObject(deleteRequest); + LOG.debug("Successfully deleted asset {}", asset.getId()); + return null; + } catch (Exception e) { + throw new CompletionException(e); + } + }, + "Delete", + asset.getId()); + } + + @Override + public String generateDownloadURL(Asset asset) { + // The stored asset.url points at the S3 object key, not a signed URL. Return a + // short-lived presigned URL instead so the caller can actually fetch the object. + // Matches AzureAssetService.generateDownloadURL which does the same thing. + return generateDownloadUrlWithExpiry(asset, Duration.ofMinutes(15)); + } + + @Override + public String generateDownloadUrlWithExpiry(Asset asset, Duration expiry) { + String cloudFrontUrl = config.getCloudFrontUrl(); + String key = resolveKey(asset.getId()); + + if (cloudFrontUrl != null + && !cloudFrontUrl.isEmpty() + && config.getCloudFrontKeyPairId() != null + && config.getCloudFrontPrivateKeyPath() != null) { + try { + String resourceUrl = cloudFrontUrl + "/" + key; + Path privateKeyPath = Paths.get(config.getCloudFrontPrivateKeyPath()); + + CustomSignerRequest signerRequest = + CustomSignerRequest.builder() + .resourceUrl(resourceUrl) + .keyPairId(config.getCloudFrontKeyPairId()) + .privateKey(privateKeyPath) + .expirationDate(Instant.now().plus(expiry)) + .build(); + + SignedUrl signedUrl = cloudFrontUtilities.getSignedUrlWithCustomPolicy(signerRequest); + return signedUrl.url(); + } catch (Exception e) { + LOG.error("Failed to generate CloudFront signed URL: {}", e.getMessage(), e); + } + } + + GetObjectPresignRequest presignRequest = + GetObjectPresignRequest.builder() + .signatureDuration(expiry) + .getObjectRequest(req -> req.bucket(actualBucketName).key(key)) + .build(); + + PresignedGetObjectRequest presignedRequest = presigner.presignGetObject(presignRequest); + return presignedRequest.url().toString(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/context/ContextEntityPromptLoader.java b/openmetadata-service/src/main/java/org/openmetadata/service/context/ContextEntityPromptLoader.java new file mode 100644 index 000000000000..ab27d6756e0f --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/context/ContextEntityPromptLoader.java @@ -0,0 +1,11 @@ +package org.openmetadata.service.context; + +import jakarta.ws.rs.core.SecurityContext; +import java.util.Optional; +import org.openmetadata.schema.type.EntityReference; + +/** Resolves an entity reference into prompt-ready structured context. */ +@FunctionalInterface +interface ContextEntityPromptLoader { + Optional load(SecurityContext securityContext, EntityReference reference); +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/context/ContextEntityPromptService.java b/openmetadata-service/src/main/java/org/openmetadata/service/context/ContextEntityPromptService.java new file mode 100644 index 000000000000..5c7fdb33f37e --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/context/ContextEntityPromptService.java @@ -0,0 +1,305 @@ +package org.openmetadata.service.context; + +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; + +import jakarta.ws.rs.core.SecurityContext; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Pattern; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.service.security.Authorizer; + +/** Builds prompt-safe structured context from files and pages attached to a chat request. */ +public class ContextEntityPromptService { + static final int TOTAL_TOKEN_BUDGET = 2500; + static final int MAX_ENTITIES = 5; + static final int MAX_TOKENS_PER_ENTITY = 900; + private static final int MAX_RELEVANT_CHUNKS = 3; + private static final int CHUNK_TARGET_CHARS = 1600; + private static final int CHUNK_OVERLAP_CHARS = 250; + private static final Pattern NON_WORD = Pattern.compile("[^a-z0-9]+"); + private static final Set STOP_WORDS = + Set.of( + "a", "an", "and", "are", "as", "at", "be", "by", "can", "do", "for", "from", "how", "i", + "in", "is", "it", "of", "on", "or", "that", "the", "this", "to", "what", "when", "where", + "which", "who", "why", "with"); + + private final ContextEntityPromptLoader loader; + + public ContextEntityPromptService(Authorizer authorizer) { + this(new DefaultContextEntityPromptLoader(authorizer)); + } + + ContextEntityPromptService(ContextEntityPromptLoader loader) { + this.loader = loader; + } + + public ContextPromptInjectionResult assemble( + SecurityContext securityContext, List contextEntities) { + return assemble(securityContext, contextEntities, null); + } + + public ContextPromptInjectionResult assemble( + SecurityContext securityContext, List contextEntities, String query) { + if (contextEntities == null || contextEntities.isEmpty()) { + return ContextPromptInjectionResult.empty(); + } + + List deduplicated = deduplicate(contextEntities); + List usedEntityRefs = new ArrayList<>(); + StringBuilder prompt = new StringBuilder(); + int totalTokens = 0; + + for (EntityReference reference : deduplicated) { + if (usedEntityRefs.size() >= MAX_ENTITIES || totalTokens >= TOTAL_TOKEN_BUDGET) { + break; + } + + Optional resolved = loader.load(securityContext, reference); + if (resolved.isEmpty()) { + continue; + } + + String section = + buildSection( + resolved.get(), + query, + Math.min(TOTAL_TOKEN_BUDGET - totalTokens, MAX_TOKENS_PER_ENTITY)); + if (nullOrEmpty(section)) { + continue; + } + + prompt.append(section).append("\n\n"); + usedEntityRefs.add(resolved.get().reference()); + totalTokens += TokenCounter.countTokens(section); + } + + if (prompt.isEmpty()) { + return ContextPromptInjectionResult.empty(); + } + + String formatted = "\n" + prompt.toString().trim() + "\n"; + return new ContextPromptInjectionResult(formatted, List.copyOf(usedEntityRefs), totalTokens); + } + + private List deduplicate(List contextEntities) { + LinkedHashMap deduplicated = new LinkedHashMap<>(); + for (EntityReference reference : contextEntities) { + if (reference == null || reference.getId() == null || nullOrEmpty(reference.getType())) { + continue; + } + deduplicated.putIfAbsent(reference.getType() + ":" + reference.getId(), reference); + } + return new ArrayList<>(deduplicated.values()); + } + + private String buildSection(ResolvedContextEntity entity, String query, int maxTokens) { + if (maxTokens <= 0) { + return ""; + } + + StringBuilder header = new StringBuilder(); + header.append("### ").append(entity.label()).append(": ").append(entity.title()).append("\n"); + if (!nullOrEmpty(entity.location())) { + header.append("Reference: ").append(entity.location()).append("\n"); + } + if (!nullOrEmpty(entity.summary())) { + header.append("Summary: ").append(entity.summary()).append("\n"); + } + + String headerText = header.toString(); + int headerTokens = TokenCounter.countTokens(headerText); + if (headerTokens >= maxTokens) { + return truncateToTokens(headerText, maxTokens); + } + + String body = selectRelevantBody(entity.body(), query, maxTokens - headerTokens); + if (nullOrEmpty(body)) { + return headerText.trim(); + } + return (headerText + "Content:\n" + body).trim(); + } + + private String selectRelevantBody(String body, String query, int maxTokens) { + if (nullOrEmpty(body) || maxTokens <= 0) { + return ""; + } + if (TokenCounter.countTokens(body) <= maxTokens) { + return body; + } + List queryTerms = extractQueryTerms(query); + if (queryTerms.isEmpty()) { + return truncateToTokens(body, maxTokens); + } + + List chunks = buildChunks(body); + if (chunks.isEmpty()) { + return truncateToTokens(body, maxTokens); + } + + List ranked = + chunks.stream() + .map(chunk -> chunk.withScore(scoreChunk(chunk.text(), query, queryTerms))) + .filter(chunk -> chunk.score() > 0) + .sorted( + Comparator.comparingInt(ChunkCandidate::score) + .reversed() + .thenComparingInt(ChunkCandidate::index)) + .limit(MAX_RELEVANT_CHUNKS) + .toList(); + + if (ranked.isEmpty()) { + return truncateToTokens(body, maxTokens); + } + + List ordered = + ranked.stream().sorted(Comparator.comparingInt(ChunkCandidate::index)).toList(); + StringBuilder builder = new StringBuilder(); + for (ChunkCandidate chunk : ordered) { + if (builder.length() > 0) { + builder.append("\n...\n"); + } + builder.append(chunk.text()); + String assembled = truncateToTokens(builder.toString(), maxTokens); + if (!assembled.isEmpty() && !assembled.endsWith("[truncated]")) { + builder = new StringBuilder(assembled); + continue; + } + return assembled; + } + return truncateToTokens(builder.toString(), maxTokens); + } + + private List extractQueryTerms(String query) { + if (nullOrEmpty(query)) { + return List.of(); + } + LinkedHashSet terms = new LinkedHashSet<>(); + for (String raw : NON_WORD.split(query.toLowerCase())) { + if (raw.length() < 3 || STOP_WORDS.contains(raw)) { + continue; + } + terms.add(raw); + } + return List.copyOf(terms); + } + + private List buildChunks(String body) { + String normalized = body.trim(); + if (normalized.isEmpty()) { + return List.of(); + } + + List chunks = new ArrayList<>(); + int index = 0; + int start = 0; + int overlap = Math.min(CHUNK_OVERLAP_CHARS, CHUNK_TARGET_CHARS / 2); + while (start < normalized.length()) { + int end = Math.min(normalized.length(), start + CHUNK_TARGET_CHARS); + if (end < normalized.length()) { + int paragraphBreak = normalized.lastIndexOf("\n\n", end); + if (paragraphBreak > start + (CHUNK_TARGET_CHARS / 2)) { + end = paragraphBreak; + } else { + int lineBreak = normalized.lastIndexOf('\n', end); + if (lineBreak > start + (CHUNK_TARGET_CHARS / 2)) { + end = lineBreak; + } + } + } + + String chunk = normalized.substring(start, end).trim(); + if (!chunk.isEmpty()) { + chunks.add(new ChunkCandidate(index++, chunk, 0)); + } + if (end >= normalized.length()) { + break; + } + start = Math.max(end - overlap, start + 1); + } + return chunks; + } + + private int scoreChunk(String chunk, String query, List queryTerms) { + String lowerChunk = chunk.toLowerCase(); + String lowerQuery = query == null ? "" : query.toLowerCase().trim(); + int score = 0; + + if (!lowerQuery.isEmpty() && lowerChunk.contains(lowerQuery)) { + score += 20; + } + + int matchedTerms = 0; + for (String term : queryTerms) { + int count = countOccurrences(lowerChunk, term); + if (count > 0) { + matchedTerms++; + score += Math.min(count, 4) * 4; + } + } + + if (matchedTerms == queryTerms.size() && !queryTerms.isEmpty()) { + score += 10; + } else { + score += matchedTerms * 2; + } + + return score; + } + + private int countOccurrences(String text, String term) { + int count = 0; + int start = 0; + while (start >= 0) { + start = text.indexOf(term, start); + if (start < 0) { + break; + } + count++; + start += term.length(); + } + return count; + } + + static String truncateToTokens(String text, int maxTokens) { + if (nullOrEmpty(text) || maxTokens <= 0) { + return ""; + } + if (TokenCounter.countTokens(text) <= maxTokens) { + return text; + } + + String suffix = "\n[truncated]"; + int low = 0; + int high = text.length(); + String best = ""; + while (low <= high) { + int mid = (low + high) >>> 1; + String candidate = text.substring(0, mid).trim(); + if (candidate.isEmpty()) { + low = mid + 1; + continue; + } + + String candidateWithSuffix = candidate + suffix; + if (TokenCounter.countTokens(candidateWithSuffix) <= maxTokens) { + best = candidateWithSuffix; + low = mid + 1; + } else { + high = mid - 1; + } + } + return best; + } + + private record ChunkCandidate(int index, String text, int score) { + private ChunkCandidate withScore(int newScore) { + return new ChunkCandidate(index, text, newScore); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/context/ContextPromptInjectionResult.java b/openmetadata-service/src/main/java/org/openmetadata/service/context/ContextPromptInjectionResult.java new file mode 100644 index 000000000000..b63fa5f700a4 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/context/ContextPromptInjectionResult.java @@ -0,0 +1,13 @@ +package org.openmetadata.service.context; + +import java.util.List; +import org.openmetadata.schema.type.EntityReference; + +/** Result of assembling structured entity context for AskCollate prompt injection. */ +public record ContextPromptInjectionResult( + String formattedContext, List usedEntityRefs, int totalTokens) { + + public static ContextPromptInjectionResult empty() { + return new ContextPromptInjectionResult("", List.of(), 0); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/context/DefaultContextEntityPromptLoader.java b/openmetadata-service/src/main/java/org/openmetadata/service/context/DefaultContextEntityPromptLoader.java new file mode 100644 index 000000000000..0e9894bee1fe --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/context/DefaultContextEntityPromptLoader.java @@ -0,0 +1,195 @@ +package org.openmetadata.service.context; + +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.service.jdbi3.ContextFileRepository.CONTEXT_FILE_ENTITY; +import static org.openmetadata.service.jdbi3.KnowledgePageRepository.KNOWLEDGE_PAGE_ENTITY; + +import jakarta.ws.rs.core.SecurityContext; +import java.util.Optional; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.entity.data.ContextFile; +import org.openmetadata.schema.entity.data.ContextFileContent; +import org.openmetadata.schema.entity.data.Page; +import org.openmetadata.schema.entity.data.PageType; +import org.openmetadata.schema.entity.data.QuickLink; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.MetadataOperation; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.ContextFileContentRepository; +import org.openmetadata.service.jdbi3.ContextFileRepository; +import org.openmetadata.service.jdbi3.KnowledgePageRepository; +import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.security.policyevaluator.OperationContext; +import org.openmetadata.service.security.policyevaluator.ResourceContext; +import org.openmetadata.service.util.EntityUtil; + +@Slf4j +class DefaultContextEntityPromptLoader implements ContextEntityPromptLoader { + private record LoaderDependencies( + ContextFileRepository contextFileRepository, + ContextFileContentRepository contextFileContentRepository, + KnowledgePageRepository knowledgeCenterRepository) {} + + private final Authorizer authorizer; + private final ContextFileRepository contextFileRepository; + private final ContextFileContentRepository contextFileContentRepository; + private final KnowledgePageRepository knowledgeCenterRepository; + + DefaultContextEntityPromptLoader(Authorizer authorizer) { + this(authorizer, defaultDependencies()); + } + + private DefaultContextEntityPromptLoader(Authorizer authorizer, LoaderDependencies dependencies) { + this( + authorizer, + dependencies.contextFileRepository(), + dependencies.contextFileContentRepository(), + dependencies.knowledgeCenterRepository()); + } + + private static LoaderDependencies defaultDependencies() { + ContextFileRepository contextFileRepository = + (ContextFileRepository) Entity.getEntityRepository(CONTEXT_FILE_ENTITY); + return new LoaderDependencies( + contextFileRepository, + contextFileRepository == null ? null : contextFileRepository.getContentRepository(), + (KnowledgePageRepository) Entity.getEntityRepository(KNOWLEDGE_PAGE_ENTITY)); + } + + DefaultContextEntityPromptLoader( + Authorizer authorizer, + ContextFileRepository contextFileRepository, + ContextFileContentRepository contextFileContentRepository, + KnowledgePageRepository knowledgeCenterRepository) { + this.authorizer = authorizer; + this.contextFileRepository = contextFileRepository; + this.contextFileContentRepository = contextFileContentRepository; + this.knowledgeCenterRepository = knowledgeCenterRepository; + } + + @Override + public Optional load( + SecurityContext securityContext, EntityReference reference) { + if (reference == null || reference.getId() == null || nullOrEmpty(reference.getType())) { + return Optional.empty(); + } + + try { + return switch (reference.getType()) { + case CONTEXT_FILE_ENTITY -> loadContextFile(securityContext, reference); + case KNOWLEDGE_PAGE_ENTITY -> loadPage(securityContext, reference); + default -> Optional.empty(); + }; + } catch (Exception e) { + LOG.debug("Skipping context entity {} due to load failure", reference, e); + return Optional.empty(); + } + } + + private Optional loadContextFile( + SecurityContext securityContext, EntityReference reference) { + authorizeView(securityContext, reference); + + ContextFile file = + contextFileRepository.get( + null, + reference.getId(), + contextFileRepository.getFields("folder"), + Include.NON_DELETED, + false); + + String extractedText = resolveExtractedText(file); + String summary = normalize(file.getDescription()); + if (nullOrEmpty(extractedText) && nullOrEmpty(summary)) { + return Optional.empty(); + } + + return Optional.of( + new ResolvedContextEntity( + file.getEntityReference(), + file.getFileType() == null ? "File" : "File (" + file.getFileType() + ")", + firstNonBlank(file.getDisplayName(), file.getName()), + firstNonBlank(file.getFullyQualifiedName(), reference.getFullyQualifiedName()), + summary, + normalize(extractedText))); + } + + private Optional loadPage( + SecurityContext securityContext, EntityReference reference) { + authorizeView(securityContext, reference); + + Page page = + knowledgeCenterRepository.get( + null, reference.getId(), EntityUtil.Fields.EMPTY_FIELDS, Include.NON_DELETED, false); + + StringBuilder body = new StringBuilder(); + String description = normalize(page.getDescription()); + if (!nullOrEmpty(description)) { + body.append(description); + } + + if (page.getPageType() == PageType.QUICK_LINK && page.getPage() != null) { + QuickLink quickLink = JsonUtils.convertValue(page.getPage(), QuickLink.class); + if (quickLink != null && !nullOrEmpty(quickLink.getUrl())) { + if (body.length() > 0) { + body.append("\n"); + } + body.append("Quick link URL: ").append(quickLink.getUrl()); + } + } + + if (body.isEmpty()) { + return Optional.empty(); + } + + return Optional.of( + new ResolvedContextEntity( + page.getEntityReference(), + page.getPageType() == PageType.QUICK_LINK ? "Quick Link" : "Page", + firstNonBlank(page.getDisplayName(), page.getName()), + firstNonBlank(page.getFullyQualifiedName(), reference.getFullyQualifiedName()), + null, + body.toString())); + } + + private String resolveExtractedText(ContextFile file) { + UUID contentId = parseUuid(file.getHeadContentId()); + if (contentId != null && contextFileContentRepository != null) { + ContextFileContent content = contextFileContentRepository.getById(contentId); + if (content != null && !nullOrEmpty(content.getExtractedText())) { + return content.getExtractedText(); + } + } + return file.getExtractedText(); + } + + private void authorizeView(SecurityContext securityContext, EntityReference reference) { + authorizer.authorize( + securityContext, + new OperationContext(reference.getType(), MetadataOperation.VIEW_BASIC), + new ResourceContext<>( + reference.getType(), reference.getId(), reference.getFullyQualifiedName())); + } + + private String firstNonBlank(String primary, String fallback) { + return nullOrEmpty(primary) ? fallback : primary; + } + + private String normalize(String value) { + return nullOrEmpty(value) ? null : value.trim(); + } + + private UUID parseUuid(String value) { + if (nullOrEmpty(value)) { + return null; + } + try { + return UUID.fromString(value); + } catch (IllegalArgumentException e) { + return null; + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/context/ResolvedContextEntity.java b/openmetadata-service/src/main/java/org/openmetadata/service/context/ResolvedContextEntity.java new file mode 100644 index 000000000000..67f48a019492 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/context/ResolvedContextEntity.java @@ -0,0 +1,12 @@ +package org.openmetadata.service.context; + +import org.openmetadata.schema.type.EntityReference; + +/** Canonical prompt-ready representation of a context entity. */ +record ResolvedContextEntity( + EntityReference reference, + String label, + String title, + String location, + String summary, + String body) {} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/context/TokenCounter.java b/openmetadata-service/src/main/java/org/openmetadata/service/context/TokenCounter.java new file mode 100644 index 000000000000..58d81b708172 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/context/TokenCounter.java @@ -0,0 +1,27 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.context; + +public final class TokenCounter { + private TokenCounter() {} + + public static int countTokens(String text) { + if (text == null || text.isEmpty()) { + return 0; + } + // Approximation: 1 token ≈ 4 characters for English text. Good enough for budget + // enforcement in prompt assembly. A jtokkit-based implementation can replace this + // if more accurate tokenization is required. + return Math.max(1, (text.length() + 3) / 4); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/drive/ContextFileExtractionService.java b/openmetadata-service/src/main/java/org/openmetadata/service/drive/ContextFileExtractionService.java new file mode 100644 index 000000000000..e96c2f813def --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/drive/ContextFileExtractionService.java @@ -0,0 +1,268 @@ +package org.openmetadata.service.drive; + +import static org.openmetadata.service.Entity.ADMIN_USER_NAME; + +import java.io.InputStream; +import java.util.UUID; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.attachments.Asset; +import org.openmetadata.schema.entity.data.ContextFile; +import org.openmetadata.schema.entity.data.ContextFileContent; +import org.openmetadata.schema.entity.data.ProcessingStatus; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.attachments.AssetService; +import org.openmetadata.service.attachments.AssetServiceFactory; +import org.openmetadata.service.jdbi3.ContextFileRepository; + +@Slf4j +public class ContextFileExtractionService { + private final ContextFileRepository repository; + private final Supplier assetServiceSupplier; + private final Executor executor; + private final ContextFileTextExtractor textExtractor; + + public ContextFileExtractionService(ContextFileRepository repository) { + this( + repository, + AssetServiceFactory::getService, + DEFAULT_EXECUTOR, + new ContextFileTextExtractor()); + } + + /** + * Single shared thread pool for text extraction. Kept separate from + * {@code AsyncService.getExecutorService()} because {@link #process(UUID, UUID)} + * blocks on {@code AssetService.read(...).join()} for S3/Azure reads, which are + * themselves scheduled on AsyncService — sharing the pool would starve those read + * tasks (and potentially deadlock) once every thread is busy running extractions. + * + *

Held {@code static final} so every production {@link ContextFileExtractionService} + * instance reuses one pool — tests that instantiate the service repeatedly no longer + * leak a new pool each construction. Threads are daemons, so the pool never blocks + * JVM shutdown; explicit lifecycle management isn't required. + */ + private static final Executor DEFAULT_EXECUTOR = createDefaultExtractionExecutor(); + + private static Executor createDefaultExtractionExecutor() { + int threads = Math.max(2, Runtime.getRuntime().availableProcessors() / 2); + ThreadFactory threadFactory = + new ThreadFactory() { + private final AtomicInteger counter = new AtomicInteger(); + + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(r, "context-file-extraction-" + counter.incrementAndGet()); + t.setDaemon(true); + return t; + } + }; + // Bounded queue + AbortPolicy so an overloaded server rejects new extractions + // rather than accumulating an unbounded backlog on the heap. The RejectedExecutionException + // handling in submit(...) below turns the rejection into a Failed processing status + // on the content, so callers see a clear "retry later" signal instead of silent buildup. + int queueCapacity = Math.max(64, threads * 8); + return new ThreadPoolExecutor( + threads, + threads, + 0L, + TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(queueCapacity), + threadFactory, + new ThreadPoolExecutor.AbortPolicy()); + } + + ContextFileExtractionService( + ContextFileRepository repository, + Supplier assetServiceSupplier, + Executor executor, + ContextFileTextExtractor textExtractor) { + this.repository = repository; + this.assetServiceSupplier = assetServiceSupplier; + this.executor = executor; + this.textExtractor = textExtractor; + } + + public void submit(UUID fileId, UUID contentId) { + try { + executor.execute(() -> process(fileId, contentId)); + } catch (RejectedExecutionException e) { + LOG.warn( + "Skipping text extraction for file {} because the async executor rejected it", fileId, e); + applyFailure(fileId, contentId, "Text extraction queue is full. Please retry later."); + } + } + + void process(UUID fileId, UUID contentId) { + ContextFile file = getFile(fileId); + if (file == null || !contentId.toString().equals(file.getHeadContentId())) { + return; + } + + updateFile( + fileId, + current -> { + if (!contentId.toString().equals(current.getHeadContentId())) { + return null; + } + ContextFile updated = JsonUtils.deepCopy(current, ContextFile.class); + updated.setProcessingStatus(ProcessingStatus.Analyzing); + return updated; + }); + updateContent( + contentId, + current -> { + // Re-read the file inside the content updater so we don't mark an + // older content "Analyzing" when headContentId changed concurrently. + // Without this guard, a no-op updateFile above would still be followed + // by a status update on the now-stale content, leaving it stuck once + // the later head-check early-returns. + ContextFile currentHead = getFile(fileId); + if (currentHead == null || !contentId.toString().equals(currentHead.getHeadContentId())) { + return null; + } + ContextFileContent updated = JsonUtils.deepCopy(current, ContextFileContent.class); + updated.setProcessingStatus(ProcessingStatus.Analyzing); + updated.setProcessingError(null); + return updated; + }); + + try { + ContextFile currentFile = getFile(fileId); + ContextFileContent currentContent = getContent(contentId); + if (currentFile == null + || currentContent == null + || !contentId.toString().equals(currentFile.getHeadContentId())) { + return; + } + + AssetService assetService = assetServiceSupplier.get(); + if (assetService == null) { + applyFailure(fileId, contentId, "Object storage is not configured for text extraction"); + return; + } + + Asset asset = repository.getAssetRepository().getById(currentContent.getAssetId()); + try (InputStream inputStream = assetService.read(asset).join()) { + if (inputStream == null) { + applyFailure(fileId, contentId, "Unable to read file content from object storage"); + return; + } + ContextFileTextExtractor.ExtractionResult result = + textExtractor.extract(inputStream, currentFile); + applyResult(fileId, contentId, result); + } + } catch (Throwable t) { + if (t instanceof VirtualMachineError vmError) { + throw vmError; + } + LOG.error("Failed to extract text for file {} content {}", fileId, contentId, t); + applyFailure(fileId, contentId, describeFailure(t)); + } + } + + private String describeFailure(Throwable t) { + return t.getMessage() == null || t.getMessage().isBlank() ? t.toString() : t.getMessage(); + } + + private void applyResult( + UUID fileId, UUID contentId, ContextFileTextExtractor.ExtractionResult result) { + updateContent( + contentId, + current -> { + ContextFileContent updated = JsonUtils.deepCopy(current, ContextFileContent.class); + updated.setProcessingStatus(result.processingStatus()); + updated.setProcessingError(result.processingError()); + updated.setExtractedText(result.extractedText()); + return updated; + }); + + updateFile( + fileId, + current -> { + if (!contentId.toString().equals(current.getHeadContentId())) { + return null; + } + ContextFile updated = JsonUtils.deepCopy(current, ContextFile.class); + updated.setProcessingStatus(result.processingStatus()); + updated.setExtractedText(result.indexedText()); + updated.setPageCount(result.pageCount()); + return updated; + }); + } + + private void applyFailure(UUID fileId, UUID contentId, String reason) { + updateContent( + contentId, + current -> { + ContextFileContent updated = JsonUtils.deepCopy(current, ContextFileContent.class); + updated.setProcessingStatus(ProcessingStatus.Failed); + updated.setProcessingError(reason); + updated.setExtractedText(null); + return updated; + }); + + updateFile( + fileId, + current -> { + if (!contentId.toString().equals(current.getHeadContentId())) { + return null; + } + ContextFile updated = JsonUtils.deepCopy(current, ContextFile.class); + updated.setProcessingStatus(ProcessingStatus.Failed); + updated.setExtractedText(null); + updated.setPageCount(null); + return updated; + }); + } + + private ContextFile getFile(UUID fileId) { + try { + return repository.get(null, fileId, repository.getFields(""), Include.NON_DELETED, false); + } catch (Exception e) { + return null; + } + } + + private ContextFileContent getContent(UUID contentId) { + try { + return repository.getContentRepository().getById(contentId); + } catch (Exception e) { + return null; + } + } + + private void updateFile( + UUID fileId, java.util.function.Function updater) { + ContextFile current = getFile(fileId); + if (current == null) { + return; + } + ContextFile updated = updater.apply(current); + if (updated == null) { + return; + } + repository.update(null, current, updated, ADMIN_USER_NAME); + } + + private void updateContent( + UUID contentId, java.util.function.Function updater) { + ContextFileContent current = getContent(contentId); + if (current == null) { + return; + } + ContextFileContent updated = updater.apply(current); + if (updated == null) { + return; + } + repository.getContentRepository().update(null, current, updated, ADMIN_USER_NAME); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/drive/ContextFileTextExtractor.java b/openmetadata-service/src/main/java/org/openmetadata/service/drive/ContextFileTextExtractor.java new file mode 100644 index 000000000000..a4341e6a28a9 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/drive/ContextFileTextExtractor.java @@ -0,0 +1,360 @@ +package org.openmetadata.service.drive; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.StringJoiner; +import lombok.Builder; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.text.PDFTextStripper; +import org.apache.poi.extractor.ExtractorFactory; +import org.apache.poi.extractor.POITextExtractor; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.usermodel.WorkbookFactory; +import org.apache.tika.exception.TikaConfigException; +import org.apache.tika.exception.TikaException; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.ocr.TesseractOCRConfig; +import org.apache.tika.parser.ocr.TesseractOCRParser; +import org.apache.tika.sax.BodyContentHandler; +import org.openmetadata.schema.entity.data.ContextFile; +import org.openmetadata.schema.entity.data.ContextFileType; +import org.openmetadata.schema.entity.data.ProcessingStatus; +import org.xml.sax.SAXException; + +public class ContextFileTextExtractor { + static final int MAX_CANONICAL_TEXT_LENGTH = 1_000_000; + static final int MAX_INDEXED_TEXT_LENGTH = 200_000; + public static final String TIKA_TESSERACT_PATH_PROPERTY = "collate.tika.tesseract.path"; + public static final String TIKA_TESSERACT_PATH_ENV = "COLLATE_TIKA_TESSERACT_PATH"; + public static final String TIKA_TESSDATA_PATH_PROPERTY = "collate.tika.tessdata.path"; + public static final String TIKA_TESSDATA_PATH_ENV = "COLLATE_TIKA_TESSDATA_PATH"; + @Deprecated public static final String TESSERACT_COMMAND_PROPERTY = "collate.tesseract.command"; + @Deprecated public static final String TESSERACT_COMMAND_ENV = "COLLATE_TESSERACT_COMMAND"; + private static final long OCR_TIMEOUT_SECONDS = 60; + + private final ImageOcrEngine imageOcrEngine; + + public ContextFileTextExtractor() { + this(new TesseractImageOcrEngine()); + } + + ContextFileTextExtractor(ImageOcrEngine imageOcrEngine) { + this.imageOcrEngine = imageOcrEngine; + } + + public ExtractionResult extract(InputStream inputStream, ContextFile file) throws IOException { + if (inputStream == null) { + throw new IOException("No file stream available for extraction"); + } + + ContextFileType fileType = + file.getFileType() == null ? ContextFileType.Other : file.getFileType(); + return switch (fileType) { + case PDF -> extractPdf(inputStream, file.getFileExtension()); + case Spreadsheet -> extractSpreadsheet(inputStream, file.getFileExtension()); + case Document, Presentation -> extractOfficeDocument(inputStream, file.getFileExtension()); + case CSV, Text -> extractPlainText(inputStream); + case Image -> extractImage(inputStream, file.getFileExtension()); + case Archive, Other -> ExtractionResult.unsupported( + "Text extraction is not supported for file type " + fileType); + }; + } + + private ExtractionResult extractPlainText(InputStream inputStream) throws IOException { + String text = readText(inputStream, MAX_CANONICAL_TEXT_LENGTH); + return ExtractionResult.processed(text, null); + } + + private ExtractionResult extractPdf(InputStream inputStream, String fileExtension) + throws IOException { + Path tempFile = spoolToTempFile(inputStream, fileExtension); + try (PDDocument document = PDDocument.load(tempFile.toFile())) { + String text = new PDFTextStripper().getText(document); + return ExtractionResult.processed(text, document.getNumberOfPages()); + } finally { + Files.deleteIfExists(tempFile); + } + } + + private ExtractionResult extractSpreadsheet(InputStream inputStream, String fileExtension) + throws IOException { + Path tempFile = spoolToTempFile(inputStream, fileExtension); + try (Workbook workbook = WorkbookFactory.create(tempFile.toFile())) { + DataFormatter formatter = new DataFormatter(); + StringBuilder text = new StringBuilder(); + for (int i = 0; i < workbook.getNumberOfSheets(); i++) { + Sheet sheet = workbook.getSheetAt(i); + if (text.length() > 0) { + text.append('\n'); + } + text.append("Sheet: ").append(sheet.getSheetName()).append('\n'); + for (Row row : sheet) { + StringJoiner joiner = new StringJoiner("\t"); + for (Cell cell : row) { + String formatted = formatter.formatCellValue(cell); + if (formatted != null && !formatted.isBlank()) { + joiner.add(formatted.trim()); + } + } + String rowText = joiner.toString(); + if (!rowText.isBlank()) { + text.append(rowText).append('\n'); + } + if (text.length() >= MAX_CANONICAL_TEXT_LENGTH) { + break; + } + } + if (text.length() >= MAX_CANONICAL_TEXT_LENGTH) { + break; + } + } + return ExtractionResult.processed(text.toString(), workbook.getNumberOfSheets()); + } finally { + Files.deleteIfExists(tempFile); + } + } + + private ExtractionResult extractOfficeDocument(InputStream inputStream, String fileExtension) + throws IOException { + Path tempFile = spoolToTempFile(inputStream, fileExtension); + try (POITextExtractor extractor = ExtractorFactory.createExtractor(tempFile.toFile())) { + return ExtractionResult.processed(extractor.getText(), null); + } finally { + Files.deleteIfExists(tempFile); + } + } + + private ExtractionResult extractImage(InputStream inputStream, String fileExtension) + throws IOException { + Path tempFile = spoolToTempFile(inputStream, fileExtension); + try { + if (!imageOcrEngine.isAvailable()) { + return ExtractionResult.unsupported( + "Image OCR requires tesseract to be installed and configured for Apache Tika"); + } + return ExtractionResult.processed(imageOcrEngine.extract(tempFile), 1); + } finally { + Files.deleteIfExists(tempFile); + } + } + + private Path spoolToTempFile(InputStream inputStream, String fileExtension) throws IOException { + String suffix = fileExtension == null || fileExtension.isBlank() ? ".bin" : "." + fileExtension; + Path tempFile = Files.createTempFile("context-file-extract-", suffix); + try (OutputStream outputStream = Files.newOutputStream(tempFile)) { + inputStream.transferTo(outputStream); + } catch (IOException | RuntimeException e) { + Files.deleteIfExists(tempFile); + throw e; + } + return tempFile; + } + + private String readText(InputStream inputStream, int maxChars) throws IOException { + StringBuilder builder = new StringBuilder(Math.min(maxChars, 8192)); + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + char[] buffer = new char[4096]; + int read; + while ((read = reader.read(buffer)) != -1) { + int remaining = maxChars - builder.length(); + if (remaining <= 0) { + break; + } + builder.append(buffer, 0, Math.min(read, remaining)); + } + } + return normalize(builder.toString()); + } + + static String normalize(String text) { + if (text == null || text.isBlank()) { + return ""; + } + String normalized = text.replace("\u0000", "").replace("\r\n", "\n").replace('\r', '\n'); + return normalized.trim(); + } + + static String truncate(String text, int maxLength) { + if (text == null || text.length() <= maxLength) { + return text; + } + return text.substring(0, maxLength); + } + + @Builder + public record ExtractionResult( + ProcessingStatus processingStatus, + String extractedText, + String indexedText, + Integer pageCount, + String processingError) { + static ExtractionResult processed(String text, Integer pageCount) { + String normalized = normalize(text); + return new ExtractionResult( + ProcessingStatus.Processed, + truncate(normalized, MAX_CANONICAL_TEXT_LENGTH), + truncate(normalized, MAX_INDEXED_TEXT_LENGTH), + pageCount, + null); + } + + static ExtractionResult unsupported(String reason) { + return new ExtractionResult(ProcessingStatus.Unsupported, null, null, null, reason); + } + } + + interface ImageOcrEngine { + boolean isAvailable(); + + String extract(Path imagePath) throws IOException; + } + + static class TesseractImageOcrEngine implements ImageOcrEngine { + private volatile Boolean available; + private volatile String availableForConfiguration; + + @Override + public boolean isAvailable() { + String configuration = resolveAvailabilityConfiguration(); + Boolean cached = available; + if (cached != null && configuration.equals(availableForConfiguration)) { + return cached; + } + synchronized (this) { + configuration = resolveAvailabilityConfiguration(); + if (available != null && configuration.equals(availableForConfiguration)) { + return available; + } + available = detectAvailability(); + availableForConfiguration = configuration; + return available; + } + } + + @Override + public String extract(Path imagePath) throws IOException { + try { + TesseractOCRParser parser = createParser(); + TesseractOCRConfig config = createConfig(); + ParseContext parseContext = new ParseContext(); + parseContext.set(TesseractOCRConfig.class, config); + // Bound the handler at MAX_CANONICAL_TEXT_LENGTH so a very large or malicious image + // cannot drive Tika to accumulate unbounded OCR output on the heap (OOM risk). + BodyContentHandler handler = new BodyContentHandler(MAX_CANONICAL_TEXT_LENGTH); + Metadata metadata = new Metadata(); + + try (InputStream stream = Files.newInputStream(imagePath)) { + parser.parse(stream, handler, metadata, parseContext); + } + return handler.toString(); + } catch (TikaConfigException e) { + throw new IOException("Invalid Apache Tika OCR configuration", e); + } catch (TikaException | SAXException e) { + throw new IOException("Apache Tika OCR failed", e); + } + } + + private boolean detectAvailability() { + try { + return createParser().hasTesseract(); + } catch (TikaConfigException e) { + return false; + } + } + + private TesseractOCRParser createParser() throws TikaConfigException { + TesseractOCRParser parser = new TesseractOCRParser(); + String tesseractPath = resolveTesseractPath(); + if (!tesseractPath.isBlank()) { + parser.setTesseractPath(tesseractPath); + } + String tessdataPath = resolveTessdataPath(); + if (!tessdataPath.isBlank()) { + parser.setTessdataPath(tessdataPath); + } + parser.initialize(Collections.emptyMap()); + return parser; + } + + private TesseractOCRConfig createConfig() { + TesseractOCRConfig config = new TesseractOCRConfig(); + config.setTimeoutSeconds((int) OCR_TIMEOUT_SECONDS); + return config; + } + + private String resolveAvailabilityConfiguration() { + return resolveTesseractPath() + "|" + resolveTessdataPath(); + } + + private String resolveTesseractPath() { + String configuredValue = + firstNonBlankPropertyOrEnv( + TIKA_TESSERACT_PATH_PROPERTY, + TIKA_TESSERACT_PATH_ENV, + TESSERACT_COMMAND_PROPERTY, + TESSERACT_COMMAND_ENV); + if (configuredValue == null) { + return ""; + } + return normalizeTesseractPath(configuredValue); + } + + private String resolveTessdataPath() { + String configuredValue = + firstNonBlankPropertyOrEnv(TIKA_TESSDATA_PATH_PROPERTY, TIKA_TESSDATA_PATH_ENV); + if (configuredValue == null) { + return ""; + } + return Path.of(configuredValue.trim()).normalize().toString(); + } + + private String firstNonBlankPropertyOrEnv( + String propertyName, String envName, String fallbackPropertyName, String fallbackEnvName) { + String configuredValue = firstNonBlankPropertyOrEnv(propertyName, envName); + if (configuredValue != null) { + return configuredValue; + } + return firstNonBlankPropertyOrEnv(fallbackPropertyName, fallbackEnvName); + } + + private String firstNonBlankPropertyOrEnv(String propertyName, String envName) { + String propertyValue = System.getProperty(propertyName); + if (propertyValue != null && !propertyValue.isBlank()) { + return propertyValue.trim(); + } + + String envValue = System.getenv(envName); + if (envValue != null && !envValue.isBlank()) { + return envValue.trim(); + } + + return null; + } + + private String normalizeTesseractPath(String configuredValue) { + Path path = Path.of(configuredValue.trim()).normalize(); + Path fileName = path.getFileName(); + if (fileName != null) { + String lastSegment = fileName.toString(); + if ("tesseract".equals(lastSegment) || "tesseract.exe".equalsIgnoreCase(lastSegment)) { + Path parent = path.getParent(); + return parent == null ? "" : parent.toString(); + } + } + return path.toString(); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AssetRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AssetRepository.java new file mode 100644 index 000000000000..02ea3194b83e --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AssetRepository.java @@ -0,0 +1,129 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.jdbi3; + +import java.util.List; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.attachments.Asset; +import org.openmetadata.schema.attachments.AssetType; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.exception.CatalogExceptionMessage; +import org.openmetadata.service.exception.EntityNotFoundException; +import org.openmetadata.service.resources.feeds.MessageParser; + +@Slf4j +public class AssetRepository { + private final CollectionDAO.AssetDAO dao; + private static final String ENTITY_TYPE = "Asset"; + + public AssetRepository(CollectionDAO.AssetDAO dao) { + this.dao = dao; + } + + public Asset create(Asset asset) { + if (asset.getId() == null || asset.getId().isEmpty()) { + asset.setId(UUID.randomUUID().toString()); + } + + MessageParser.EntityLink entityLink = MessageParser.EntityLink.parse(asset.getEntityLink()); + String json = JsonUtils.pojoToJson(asset); + try { + dao.insert(entityLink.getEntityFQN(), json); + LOG.info("Created asset with id {}", asset.getId()); + } catch (Exception e) { + LOG.error("Failed to create asset with id {}: {}", asset.getId(), e.getMessage(), e); + throw e; + } + return asset; + } + + public List getByFQN(String fqn, AssetType assetType) { + try { + List json = dao.getByFqnExact(assetType.value(), fqn); + // Treat null and empty identically (matches getByFqnPrefix) so callers cannot + // silently receive an empty list when they expect "not found". + if (json == null || json.isEmpty()) { + throw EntityNotFoundException.byMessage( + CatalogExceptionMessage.entityNotFound(ENTITY_TYPE, fqn)); + } + return JsonUtils.readObjects(json, Asset.class); + } catch (Exception e) { + LOG.error("Failed to read asset with FQN {}: {}", fqn, e.getMessage(), e); + throw e; + } + } + + public Asset getById(String id) { + try { + String json = dao.getById(id); + if (json == null) { + throw EntityNotFoundException.byMessage( + CatalogExceptionMessage.entityNotFound(ENTITY_TYPE, id)); + } + return JsonUtils.readValue(json, Asset.class); + } catch (Exception e) { + LOG.error("Failed to get asset with id {}: {}", id, e.getMessage(), e); + throw e; + } + } + + public List getByFqnPrefix(String fqnPrefix, AssetType assetType) { + try { + List jsonList = dao.getByFqnPrefix(assetType.value(), fqnPrefix); + if (jsonList == null || jsonList.isEmpty()) { + throw EntityNotFoundException.byMessage( + CatalogExceptionMessage.entityNotFound(ENTITY_TYPE, fqnPrefix)); + } + return JsonUtils.readObjects(jsonList, Asset.class); + } catch (Exception e) { + LOG.error("Failed to get assets with fqnPrefix {}: {}", fqnPrefix, e.getMessage(), e); + throw e; + } + } + + public Asset update(Asset asset) { + String json = JsonUtils.pojoToJson(asset); + try { + // Update by id — multiple assets can share a fullyQualifiedName (e.g. revisions + // of the same context file), so an fqnHash-based update would silently touch + // sibling rows. + dao.update(json, asset.getId()); + LOG.info("Updated asset with id {}", asset.getId()); + } catch (Exception e) { + LOG.error("Failed to update asset with id {}: {}", asset.getId(), e.getMessage(), e); + throw e; + } + return asset; + } + + public void markDeleted(String fqnPrefix) { + try { + dao.markDeletedByFqnPrefix(fqnPrefix); + LOG.info("Marked asset {} as deleted", fqnPrefix); + } catch (Exception e) { + LOG.error("Failed to mark asset {} as deleted: {}", fqnPrefix, e.getMessage(), e); + throw e; + } + } + + public void delete(String id) { + try { + dao.delete(id); + LOG.info("Deleted asset {}", id); + } catch (Exception e) { + LOG.error("Failed to delete asset {}: {}", id, e.getMessage(), e); + throw e; + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 67c88b54a628..0cb047c7095f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -15,11 +15,15 @@ import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.schema.type.Relationship.CONTAINS; +import static org.openmetadata.schema.type.Relationship.HAS; import static org.openmetadata.schema.type.Relationship.MENTIONED_IN; +import static org.openmetadata.schema.type.Relationship.OWNS; import static org.openmetadata.service.Entity.APPLICATION; import static org.openmetadata.service.Entity.GLOSSARY_TERM; import static org.openmetadata.service.Entity.ORGANIZATION_NAME; import static org.openmetadata.service.Entity.QUERY; +import static org.openmetadata.service.Entity.TEAM; +import static org.openmetadata.service.Entity.USER; import static org.openmetadata.service.jdbi3.ListFilter.escapeApostrophe; import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL; import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES; @@ -378,6 +382,21 @@ public interface CollectionDAO { @CreateSqlObject WorksheetDAO worksheetDAO(); + @CreateSqlObject + FolderDAO folderDAO(); + + @CreateSqlObject + ContextFileDAO contextFileDAO(); + + @CreateSqlObject + ContextFileContentDAO contextFileContentDAO(); + + @CreateSqlObject + KnowledgePageDAO knowledgePageDAO(); + + @CreateSqlObject + AssetDAO assetDAO(); + @CreateSqlObject FeedDAO feedDAO(); @@ -14264,4 +14283,391 @@ public OAuthRecords.McpPendingAuthRequest map(ResultSet rs, StatementContext ctx rs.getLong("expires_at")); } } + + interface FolderDAO extends EntityDAO { + @Override + default String getTableName() { + return "drive_folder"; + } + + @Override + default Class getEntityClass() { + return org.openmetadata.schema.entity.data.Folder.class; + } + + @Override + default String getNameHashColumn() { + return "nameHash"; + } + } + + interface ContextFileDAO extends EntityDAO { + @Override + default String getTableName() { + return "context_file"; + } + + @Override + default Class getEntityClass() { + return org.openmetadata.schema.entity.data.ContextFile.class; + } + + @Override + default String getNameHashColumn() { + return "nameHash"; + } + } + + interface ContextFileContentDAO + extends EntityDAO { + @Override + default String getTableName() { + return "context_file_content"; + } + + @Override + default Class getEntityClass() { + return org.openmetadata.schema.entity.data.ContextFileContent.class; + } + + @Override + default String getNameHashColumn() { + return "nameHash"; + } + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM context_file_content " + + "WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.contextFile.id')) = :contextFileId", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM context_file_content " + + "WHERE json->'contextFile'->>'id' = :contextFileId", + connectionType = POSTGRES) + List listByContextFileId(@Bind("contextFileId") String contextFileId); + } + + interface KnowledgePageDAO extends EntityDAO { + String KNOWLEDGE_PAGE_ENTITY = "page"; + + @Override + default String getTableName() { + return "knowledge_center"; + } + + @Override + default Class getEntityClass() { + return org.openmetadata.schema.entity.data.Page.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + @Override + default boolean supportsSoftDelete() { + return false; + } + + /** + * When the caller supplies {@code entityId} + {@code entityType} (e.g. from a data-asset + * page that wants the list of knowledge pages referencing it), join against + * {@code entity_relationship} so that only pages whose {@code relatedEntities} contains + * the target entity are returned. Without this override, the base {@code EntityDAO.listAfter} + * ignores those params and returns every knowledge page — breaking the Knowledge + * Articles right-panel widget (and the corresponding playwright assertions). + */ + @Override + default int listCount(ListFilter filter) { + String entityId = filter.getQueryParam("entityId"); + String entityType = filter.getQueryParam("entityType"); + String knowledgePageType = filter.getQueryParam("pageType"); + String tagFQN = filter.getQueryParam("tagFQN"); + String tagListCondition = + "INNER JOIN tag_usage ON knowledge_center.fqnHash = tag_usage.targetFQNHash"; + String tagFilterCondition = "WHERE tag_usage.tagFQN = :tagFQN and "; + if (nullOrEmpty(tagFQN)) { + tagListCondition = ""; + tagFilterCondition = "WHERE"; + } + Map bindMap = new HashMap<>(); + if (!nullOrEmpty(entityId) && !nullOrEmpty(entityType)) { + String knowledgePageTypeQuery = getKnowledgePageTypeQuery("AND", knowledgePageType); + String condition = + String.format( + "INNER JOIN entity_relationship ON knowledge_center.id = entity_relationship.toId %s %s " + + "entity_relationship.fromId IN (%s) %s" + + "and entity_relationship.toEntity = :toEntityType %s", + tagListCondition, + tagFilterCondition, + entityId, + getRelationCondition(entityType), + knowledgePageTypeQuery); + bindMap.put("toEntityType", KNOWLEDGE_PAGE_ENTITY); + bindMap.put("tagFQN", tagFQN); + if (!nullOrEmpty(knowledgePageTypeQuery)) { + bindMap.put("pageType", knowledgePageType); + } + return listKnowledgePageCountByEntity(condition, bindMap); + } else if ((!nullOrEmpty(entityId) && nullOrEmpty(entityType)) + || (nullOrEmpty(entityId) && !nullOrEmpty(entityType))) { + throw new IllegalArgumentException( + "Query Param Entity Id and Entity Type both needs to be provided."); + } + + String knowledgePageQueryClause = + String.format( + "%s %s %s", + tagListCondition, + tagFilterCondition, + getKnowledgePageTypeQuery("", knowledgePageType)); + return listCount( + getTableName(), + getNameHashColumn(), + filter.getQueryParams(), + getKnowledgePageWhereClause(knowledgePageQueryClause)); + } + + @Override + default List listBefore( + ListFilter filter, int limit, String beforeName, String beforeId) { + String entityId = filter.getQueryParam("entityId"); + String entityType = filter.getQueryParam("entityType"); + String knowledgePageType = filter.getQueryParam("pageType"); + String tagFQN = filter.getQueryParam("tagFQN"); + String tagListCondition = + "INNER JOIN tag_usage ON knowledge_center.fqnHash = tag_usage.targetFQNHash"; + String tagFilterCondition = "WHERE tag_usage.tagFQN = :tagFQN and "; + if (nullOrEmpty(tagFQN)) { + tagListCondition = ""; + tagFilterCondition = "WHERE"; + } + Map bindMap = new HashMap<>(); + if (!nullOrEmpty(entityId) && !nullOrEmpty(entityType)) { + String knowledgePageTypeQuery = getKnowledgePageTypeQuery("AND", knowledgePageType); + String condition = + String.format( + "INNER JOIN entity_relationship ON knowledge_center.id = entity_relationship.toId %s %s entity_relationship.fromId IN (%s) " + + "%s and entity_relationship.toEntity = :toEntity %s " + + "and (knowledge_center.name < :beforeName OR (knowledge_center.name = :beforeName AND knowledge_center.id < :beforeId)) order by knowledge_center.name DESC,knowledge_center.id DESC LIMIT :limit", + tagListCondition, + tagFilterCondition, + entityId, + getRelationCondition(entityType), + knowledgePageTypeQuery); + bindMap.put("toEntity", KNOWLEDGE_PAGE_ENTITY); + bindMap.put("beforeName", beforeName); + bindMap.put("beforeId", beforeId); + bindMap.put("limit", limit); + bindMap.put("tagFQN", tagFQN); + if (!nullOrEmpty(knowledgePageTypeQuery)) { + bindMap.put("pageType", knowledgePageType); + } + return listBeforeKnowledgePageByEntityId(condition, bindMap); + } else if ((!nullOrEmpty(entityId) && nullOrEmpty(entityType)) + || (nullOrEmpty(entityId) && !nullOrEmpty(entityType))) { + throw new IllegalArgumentException( + "Query Param Entity Id and Entity Type both needs to be provided."); + } + String knowledgePageQueryClause = + String.format( + "%s %s %s", + tagListCondition, + tagFilterCondition, + getKnowledgePageTypeQuery("", knowledgePageType)); + beforeName = FullyQualifiedName.unquoteName(beforeName); + return listBefore( + getTableName(), + filter.getQueryParams(), + getKnowledgePageWhereClause(knowledgePageQueryClause), + limit, + beforeName, + beforeId); + } + + @Override + default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { + String entityId = filter.getQueryParam("entityId"); + String entityType = filter.getQueryParam("entityType"); + String knowledgePageType = filter.getQueryParam("pageType"); + String tagFQN = filter.getQueryParam("tagFQN"); + String tagListCondition = + "INNER JOIN tag_usage ON knowledge_center.fqnHash = tag_usage.targetFQNHash"; + String tagFilterCondition = "WHERE tag_usage.tagFQN = :tagFQN and "; + if (nullOrEmpty(tagFQN)) { + tagListCondition = ""; + tagFilterCondition = "WHERE"; + } + Map bindMap = new HashMap<>(); + if (!nullOrEmpty(entityId) && !nullOrEmpty(entityType)) { + String knowledgePageTypeQuery = getKnowledgePageTypeQuery("AND", knowledgePageType); + String condition = + String.format( + "INNER JOIN entity_relationship ON knowledge_center.id = entity_relationship.toId %s %s entity_relationship.fromId IN (%s) " + + "%s and entity_relationship.toEntity = :toEntity %s " + + "and (knowledge_center.name > :afterName OR (knowledge_center.name = :afterName AND knowledge_center.id > :afterId)) order by knowledge_center.name ASC,knowledge_center.id ASC LIMIT :limit", + tagListCondition, + tagFilterCondition, + entityId, + getRelationCondition(entityType), + knowledgePageTypeQuery); + bindMap.put("toEntity", KNOWLEDGE_PAGE_ENTITY); + bindMap.put("afterName", afterName); + bindMap.put("afterId", afterId); + bindMap.put("limit", limit); + bindMap.put("tagFQN", tagFQN); + if (!nullOrEmpty(knowledgePageTypeQuery)) { + bindMap.put("pageType", knowledgePageType); + } + return listAfterKnowledgePageByEntityId(condition, bindMap); + } else if ((!nullOrEmpty(entityId) && nullOrEmpty(entityType)) + || (nullOrEmpty(entityId) && !nullOrEmpty(entityType))) { + throw new IllegalArgumentException( + "Query Param Entity Id and Entity Type both needs to be provided."); + } + String knowledgePageQueryClause = + String.format( + "%s %s %s", + tagListCondition, + tagFilterCondition, + getKnowledgePageTypeQuery("", knowledgePageType)); + afterName = FullyQualifiedName.unquoteName(afterName); + return listAfter( + getTableName(), + filter.getQueryParams(), + getKnowledgePageWhereClause(knowledgePageQueryClause), + limit, + afterName, + afterId); + } + + private String getRelationCondition(String entityType) { + // Users/teams "own" pages (membership-based); every other entity type reaches the page + // through a HAS relationship (the page's relatedEntities list). + String owns = String.valueOf(OWNS.ordinal()); + String has = String.valueOf(HAS.ordinal()); + if (entityType.equals(USER) || entityType.equals(TEAM)) { + return String.format(" and entity_relationship.relation = %s ", owns); + } else { + return String.format(" and entity_relationship.relation = %s ", has); + } + } + + private String getKnowledgePageWhereClause(String knowledgePageQueryClause) { + return nullOrEmpty(knowledgePageQueryClause) ? "WHERE TRUE" : knowledgePageQueryClause; + } + + private String getKnowledgePageTypeQuery(String clause, String type) { + if (!nullOrEmpty(type)) { + if (Boolean.TRUE.equals( + org.openmetadata.service.resources.databases.DatasourceConfig.getInstance() + .isMySQL())) { + return String.format( + " %s JSON_EXTRACT(knowledge_center.json, '$.pageType') = :pageType", clause); + } else { + return String.format(" %s knowledge_center.json->>'pageType' = :pageType", clause); + } + } + if ("AND".equals(clause)) { + return ""; + } + return "TRUE"; + } + + @SqlQuery("SELECT knowledge_center.json FROM knowledge_center ") + List listAfterKnowledgePageByEntityId( + @Define("cond") String cond, @BindMap Map bindings); + + @SqlQuery( + "SELECT json FROM (SELECT knowledge_center.name,knowledge_center.id, knowledge_center.json FROM knowledge_center ) last_rows_subquery ORDER BY name,id") + List listBeforeKnowledgePageByEntityId( + @Define("cond") String cond, @BindMap Map bindings); + + @SqlQuery("SELECT count(*) FROM knowledge_center ") + int listKnowledgePageCountByEntity( + @Define("cond") String cond, @BindMap Map bindings); + + @SqlQuery( + "SELECT json " + + "FROM knowledge_center " + + "WHERE id NOT IN (" + + " SELECT toId FROM entity_relationship WHERE (relation = 0 AND toEntity = 'page') OR (relation = 9 AND toEntity = 'page')" + + ")") + List listTopLevelPages(); + + @SqlQuery( + "SELECT kc.json " + + "FROM knowledge_center kc " + + "JOIN entity_relationship er ON kc.id = er.toId " + + "WHERE er.fromId = :parentId " + + "AND (er.relation = 9 or er.relation = 0) " + + "AND er.toEntity = 'page'") + List listChildren(@Bind("parentId") String parentId); + + @ConnectionAwareSqlUpdate( + value = "UPDATE knowledge_center SET json = :json, fqnHash = :fqnHash WHERE id = :id", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE knowledge_center SET json = :json::jsonb, fqnHash = :fqnHash WHERE id = :id", + connectionType = POSTGRES) + void updateFullyQualifiedName( + @Bind("id") String pageId, @Bind("json") String json, @BindFQN("fqnHash") String fqnHash); + } + + interface AssetDAO { + @ConnectionAwareSqlUpdate( + value = "INSERT INTO asset_entity (json, fqnHash) VALUES (:json, :fqnHash)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "INSERT INTO asset_entity (json, fqnHash) VALUES (:json :: jsonb, :fqnHash)", + connectionType = POSTGRES) + void insert(@BindFQN("fqnHash") String fqnHash, @Bind("json") String json); + + @ConnectionAwareSqlUpdate( + value = "UPDATE asset_entity SET json = :json WHERE id = :id", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "UPDATE asset_entity SET json = :json::jsonb WHERE id = :id", + connectionType = POSTGRES) + void update(@Bind("json") String json, @Bind("id") String id); + + @SqlQuery("SELECT json FROM asset_entity WHERE id = :id") + String getById(@Bind("id") String id); + + @SqlQuery( + "SELECT json FROM asset_entity WHERE LOWER(assetType) = LOWER(:assetType) AND fqnHash = :fqnHash") + List getByFqnExact( + @Bind("assetType") String assetType, @BindFQN("fqnHash") String fullyQualifiedName); + + @SqlQuery( + "SELECT json FROM asset_entity WHERE LOWER(assetType) = LOWER(:assetType) AND fqnHash LIKE :concatFqnPrefixHash") + List getByFqnPrefix( + @Bind("assetType") String assetType, + @org.openmetadata.service.util.jdbi.BindConcat( + value = "concatFqnPrefixHash", + parts = {":fqnPrefixHash", "%"}, + hash = true) + String fqnPrefixHash); + + @ConnectionAwareSqlUpdate( + value = + "UPDATE asset_entity SET json = JSON_SET(json, '$.deleted', true) " + + "WHERE fqnHash LIKE :prefix", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE asset_entity SET json = jsonb_set(json, '{deleted}', 'true') " + + "WHERE fqnHash LIKE :prefix", + connectionType = POSTGRES) + void markDeletedByFqnPrefix(@BindFQN("prefix") String prefix); + + @SqlUpdate("DELETE FROM asset_entity WHERE fqnHash LIKE :prefix") + void deleteByFqnPrefix(@BindFQN("prefix") String prefix); + + @SqlUpdate("DELETE FROM asset_entity WHERE id = :id") + void delete(@Bind("id") String id); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ContextFileContentRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ContextFileContentRepository.java new file mode 100644 index 000000000000..4d3556fdacde --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ContextFileContentRepository.java @@ -0,0 +1,110 @@ +package org.openmetadata.service.jdbi3; + +import java.util.UUID; +import org.jdbi.v3.core.Jdbi; +import org.openmetadata.schema.entity.data.ContextFile; +import org.openmetadata.schema.entity.data.ContextFileContent; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.change.ChangeSource; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.Entity; +import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.EntityUtil.RelationIncludes; +import org.openmetadata.service.util.FullyQualifiedName; + +@Repository +public class ContextFileContentRepository extends EntityRepository { + public static final String CONTEXT_FILE_CONTENT_ENTITY = "contextFileContent"; + + public ContextFileContentRepository(Jdbi jdbi) { + super( + null, + CONTEXT_FILE_CONTENT_ENTITY, + ContextFileContent.class, + jdbi.onDemand(CollectionDAO.class).contextFileContentDAO(), + "", + ""); + } + + @Override + public void setFields( + ContextFileContent entity, EntityUtil.Fields fields, RelationIncludes relationIncludes) { + // No relationship-backed fields for now. + } + + @Override + public void clearFields(ContextFileContent entity, EntityUtil.Fields fields) { + // No relationship-backed fields for now. + } + + @Override + public void setFullyQualifiedName(ContextFileContent entity) { + if (entity.getContextFile() == null + || entity.getContextFile().getFullyQualifiedName() == null + || entity.getContextFile().getFullyQualifiedName().isEmpty()) { + entity.setFullyQualifiedName(entity.getName()); + return; + } + entity.setFullyQualifiedName( + FullyQualifiedName.add(entity.getContextFile().getFullyQualifiedName(), entity.getName())); + } + + @Override + public void prepare(ContextFileContent entity, boolean update) { + if (entity.getContextFile() != null) { + ContextFile file = + Entity.getEntity( + ContextFileRepository.CONTEXT_FILE_ENTITY, + entity.getContextFile().getId(), + "", + Include.ALL); + entity.setContextFile(file.getEntityReference()); + } + } + + @Override + public void storeEntity(ContextFileContent entity, boolean update) { + store(entity, update); + } + + @Override + public void storeRelationships(ContextFileContent entity) { + // No relationship-backed fields for now. + } + + @Override + public EntityUpdater getUpdater( + ContextFileContent original, + ContextFileContent updated, + Operation operation, + ChangeSource source) { + return new ContextFileContentUpdater(original, updated, operation); + } + + public ContextFileContent getById(UUID id) { + return get(null, id, getFields(""), Include.NON_DELETED, false); + } + + public java.util.List listByContextFileId(UUID contextFileId) { + return JsonUtils.readObjects( + ((CollectionDAO.ContextFileContentDAO) dao).listByContextFileId(contextFileId.toString()), + ContextFileContent.class); + } + + public class ContextFileContentUpdater extends EntityUpdater { + public ContextFileContentUpdater( + ContextFileContent original, ContextFileContent updated, Operation operation) { + super(original, updated, operation); + } + + @Override + public void entitySpecificUpdate(boolean consolidatingChanges) { + recordChange("assetId", original.getAssetId(), updated.getAssetId()); + recordChange("isCurrent", original.getIsCurrent(), updated.getIsCurrent()); + recordChange( + "processingStatus", original.getProcessingStatus(), updated.getProcessingStatus()); + recordChange("processingError", original.getProcessingError(), updated.getProcessingError()); + recordChange("extractedText", original.getExtractedText(), updated.getExtractedText()); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ContextFileRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ContextFileRepository.java new file mode 100644 index 000000000000..212985983c5f --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ContextFileRepository.java @@ -0,0 +1,236 @@ +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.service.Entity.ADMIN_USER_NAME; +import static org.openmetadata.service.jdbi3.FolderRepository.FOLDER_ENTITY; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.RejectedExecutionException; +import lombok.extern.slf4j.Slf4j; +import org.jdbi.v3.core.Jdbi; +import org.openmetadata.schema.attachments.Asset; +import org.openmetadata.schema.entity.data.ContextFile; +import org.openmetadata.schema.entity.data.ContextFileContent; +import org.openmetadata.schema.entity.data.Folder; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.Relationship; +import org.openmetadata.schema.type.change.ChangeSource; +import org.openmetadata.service.Entity; +import org.openmetadata.service.attachments.AssetService; +import org.openmetadata.service.attachments.AssetServiceFactory; +import org.openmetadata.service.resources.drive.ContextFileResource; +import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.EntityUtil.RelationIncludes; +import org.openmetadata.service.util.FullyQualifiedName; + +@Slf4j +@Repository +public class ContextFileRepository extends EntityRepository { + public static final String CONTEXT_FILE_ENTITY = "contextFile"; + private final AssetRepository assetRepository; + private final ContextFileContentRepository contentRepository; + + public ContextFileRepository(Jdbi jdbi) { + super( + ContextFileResource.COLLECTION_PATH, + CONTEXT_FILE_ENTITY, + ContextFile.class, + jdbi.onDemand(CollectionDAO.class).contextFileDAO(), + "", + ""); + supportsSearch = true; + // NOTE: SearchIndexFactory registration handled by OpenMetadata core + CollectionDAO dao = jdbi.onDemand(CollectionDAO.class); + this.assetRepository = new AssetRepository(dao.assetDAO()); + this.contentRepository = new ContextFileContentRepository(jdbi); + } + + public AssetRepository getAssetRepository() { + return assetRepository; + } + + public ContextFileContentRepository getContentRepository() { + return contentRepository; + } + + @Override + public void setFields( + ContextFile file, EntityUtil.Fields fields, RelationIncludes relationIncludes) { + file.setFolder(fields.contains("folder") ? getFolder(file) : file.getFolder()); + } + + @Override + public void clearFields(ContextFile file, EntityUtil.Fields fields) { + file.setFolder(fields.contains("folder") ? file.getFolder() : null); + } + + @Override + public void setFieldsInBulk(EntityUtil.Fields fields, List entities) { + if (entities == null || entities.isEmpty()) { + return; + } + + if (fields.contains("folder")) { + var folderMap = batchFetchFromIdsAndRelationSingleRelation(entities, Relationship.CONTAINS); + entities.forEach(file -> file.setFolder(folderMap.get(file.getId()))); + } + + fetchAndSetFields(entities, fields); + setInheritedFields(entities, fields); + entities.forEach(entity -> clearFieldsInternal(entity, fields)); + } + + @Override + public void setFullyQualifiedName(ContextFile file) { + if (file.getFolder() == null) { + file.setFullyQualifiedName(file.getName()); + } else { + Folder folder = Entity.getEntity(FOLDER_ENTITY, file.getFolder().getId(), "", Include.ALL); + file.setFullyQualifiedName( + FullyQualifiedName.add(folder.getFullyQualifiedName(), file.getName())); + } + } + + @Override + public void prepare(ContextFile file, boolean update) { + if (file.getFolder() != null) { + Folder folder = Entity.getEntity(file.getFolder(), "", Include.NON_DELETED); + file.setFolder(folder.getEntityReference()); + } + } + + @Override + public void storeEntity(ContextFile file, boolean update) { + EntityReference folder = file.getFolder(); + file.withFolder(null); + store(file, update); + file.withFolder(folder); + } + + @Override + public void storeRelationships(ContextFile file) { + if (file.getFolder() != null) { + addRelationship( + file.getFolder().getId(), + file.getId(), + FOLDER_ENTITY, + CONTEXT_FILE_ENTITY, + Relationship.CONTAINS); + } + } + + @Override + public void restorePatchAttributes(ContextFile original, ContextFile updated) { + updated.withFolder(original.getFolder()); + } + + @Override + public EntityUpdater getUpdater( + ContextFile original, ContextFile updated, Operation operation, ChangeSource source) { + return new ContextFileUpdater(original, updated, operation); + } + + private EntityReference getFolder(ContextFile file) { + return getFromEntityRef(file.getId(), Relationship.CONTAINS, FOLDER_ENTITY, false); + } + + public class ContextFileUpdater extends EntityUpdater { + public ContextFileUpdater(ContextFile original, ContextFile updated, Operation operation) { + super(original, updated, operation); + } + + @Override + public void entitySpecificUpdate(boolean consolidatingChanges) { + recordChange("fileType", original.getFileType(), updated.getFileType()); + recordChange( + "processingStatus", original.getProcessingStatus(), updated.getProcessingStatus()); + recordChange("extractedText", original.getExtractedText(), updated.getExtractedText()); + recordChange("pageCount", original.getPageCount(), updated.getPageCount()); + } + } + + @Override + protected void entitySpecificCleanup(ContextFile entityInterface) { + List contents = + new ArrayList<>(contentRepository.listByContextFileId(entityInterface.getId())); + if (contents.isEmpty()) { + UUID headContentId = parseUuid(entityInterface.getHeadContentId()); + if (headContentId != null) { + try { + ContextFileContent headContent = contentRepository.getById(headContentId); + if (headContent != null) { + contents.add(headContent); + } + } catch (Exception ignored) { + // Fall through to legacy asset cleanup when the content row was never persisted. + } + } + } + + for (ContextFileContent content : contents) { + deleteContentSnapshot(content); + } + + if (contents.isEmpty() + && entityInterface.getAssetId() != null + && !entityInterface.getAssetId().isEmpty()) { + deleteAsset(entityInterface.getAssetId()); + } + } + + public ContextFileContent getContentById(String id) { + UUID contentId = parseUuid(id); + return contentId == null ? null : contentRepository.getById(contentId); + } + + private UUID parseUuid(String value) { + if (value == null || value.isEmpty()) { + return null; + } + try { + return UUID.fromString(value); + } catch (IllegalArgumentException ex) { + return null; + } + } + + private void deleteContentSnapshot(ContextFileContent content) { + if (content.getAssetId() != null && !content.getAssetId().isEmpty()) { + deleteAsset(content.getAssetId()); + } + + contentRepository.delete(ADMIN_USER_NAME, content.getId(), false, true); + } + + private void deleteAsset(String assetId) { + AssetService assetService = AssetServiceFactory.getService(); + Asset asset = null; + try { + asset = assetRepository.getById(assetId); + } catch (Exception ignored) { + // If the asset metadata is already gone, continue deleting any remaining references. + } + if (asset != null && assetService != null) { + try { + assetService + .delete(asset) + .thenRun(() -> assetRepository.delete(assetId)) + .exceptionally( + ex -> { + LOG.error( + "Failed to delete asset {} from storage, metadata retained", assetId, ex); + return null; + }); + } catch (RejectedExecutionException e) { + LOG.warn( + "Object delete queue is full for asset {}. Storage cleanup deferred and metadata retained", + assetId, + e); + } + } else { + assetRepository.delete(assetId); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DaoListFilter.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DaoListFilter.java new file mode 100644 index 000000000000..1ecd8d02a3f9 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DaoListFilter.java @@ -0,0 +1,57 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.jdbi3; + +import java.util.ArrayList; +import java.util.List; +import org.openmetadata.schema.type.Include; + +public class DaoListFilter extends ListFilter { + public DaoListFilter() { + super(Include.NON_DELETED); + } + + public DaoListFilter(Include include) { + super(include); + } + + @Override + public String getCondition() { + return this.getCondition(null); + } + + @Override + public String getCondition(String tableName) { + List conditions = new ArrayList<>(); + String baseConditions = super.getCondition(tableName); + conditions.add(baseConditions); + conditions.add(getPageTypeCondition(tableName)); + return addCondition(conditions); + } + + public String getPageTypeCondition(String tableName) { + String pageType = this.queryParams.get("pageType"); + if (pageType == null) { + return ""; + } + String qualifiedColumn = + (tableName == null || tableName.isBlank()) ? "pageType" : tableName + ".pageType"; + return qualifiedColumn + " = :pageType"; + } + + /** @deprecated use {@link #getPageTypeCondition(String)} instead. */ + @Deprecated + public String getArticleCondition() { + return getPageTypeCondition(null); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FolderRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FolderRepository.java new file mode 100644 index 000000000000..6db5cc8ca833 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FolderRepository.java @@ -0,0 +1,174 @@ +package org.openmetadata.service.jdbi3; + +import java.util.Comparator; +import java.util.List; +import java.util.UUID; +import org.jdbi.v3.core.Jdbi; +import org.openmetadata.schema.entity.data.ContextFile; +import org.openmetadata.schema.entity.data.Folder; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.Relationship; +import org.openmetadata.schema.type.change.ChangeSource; +import org.openmetadata.service.Entity; +import org.openmetadata.service.resources.drive.ContextFileResource; +import org.openmetadata.service.resources.drive.FolderResource; +import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.EntityUtil.RelationIncludes; +import org.openmetadata.service.util.FullyQualifiedName; + +@Repository +public class FolderRepository extends EntityRepository { + public static final String FOLDER_ENTITY = "folder"; + + public FolderRepository(Jdbi jdbi) { + super( + FolderResource.COLLECTION_PATH, + FOLDER_ENTITY, + Folder.class, + jdbi.onDemand(CollectionDAO.class).folderDAO(), + "", + ""); + supportsSearch = true; + // NOTE: SearchIndexFactory registration handled by OpenMetadata core + } + + @Override + public void setFields( + Folder folder, EntityUtil.Fields fields, RelationIncludes relationIncludes) { + folder.setParent(fields.contains("parent") ? getParentFolder(folder) : folder.getParent()); + folder.setChildren( + fields.contains("children") ? getChildFolders(folder) : folder.getChildren()); + } + + @Override + public void clearFields(Folder folder, EntityUtil.Fields fields) { + folder.setParent(fields.contains("parent") ? folder.getParent() : null); + folder.setChildren(fields.contains("children") ? folder.getChildren() : null); + } + + @Override + public void setFieldsInBulk(EntityUtil.Fields fields, List entities) { + if (entities == null || entities.isEmpty()) { + return; + } + + if (fields.contains("parent")) { + var parentMap = batchFetchFromIdsAndRelationSingleRelation(entities, Relationship.CONTAINS); + entities.forEach(folder -> folder.setParent(parentMap.get(folder.getId()))); + } + + if (fields.contains("children")) { + var childrenMap = batchFetchToIdsOneToMany(entities, Relationship.CONTAINS, FOLDER_ENTITY); + entities.forEach( + folder -> folder.setChildren(childrenMap.getOrDefault(folder.getId(), List.of()))); + } + + fetchAndSetFields(entities, fields); + setInheritedFields(entities, fields); + entities.forEach(entity -> clearFieldsInternal(entity, fields)); + } + + @Override + public void setFullyQualifiedName(Folder folder) { + if (folder.getParent() == null) { + folder.setFullyQualifiedName(folder.getName()); + } else { + Folder parentFolder = + Entity.getEntity(FOLDER_ENTITY, folder.getParent().getId(), "", Include.ALL); + folder.setFullyQualifiedName( + FullyQualifiedName.add(parentFolder.getFullyQualifiedName(), folder.getName())); + } + } + + @Override + public void prepare(Folder folder, boolean update) { + // Resolve parent folder reference if provided + if (folder.getParent() != null) { + Folder parent = Entity.getEntity(folder.getParent(), "", Include.NON_DELETED); + folder.setParent(parent.getEntityReference()); + } + } + + @Override + public void storeEntity(Folder folder, boolean update) { + EntityReference parent = folder.getParent(); + List children = folder.getChildren(); + folder.withParent(null).withChildren(null); + store(folder, update); + folder.withParent(parent).withChildren(children); + } + + @Override + public void storeRelationships(Folder folder) { + if (folder.getParent() != null) { + addRelationship( + folder.getParent().getId(), + folder.getId(), + FOLDER_ENTITY, + FOLDER_ENTITY, + Relationship.CONTAINS); + } + } + + @Override + public EntityUpdater getUpdater( + Folder original, Folder updated, Operation operation, ChangeSource source) { + return new FolderUpdater(original, updated, operation); + } + + private EntityReference getParentFolder(Folder folder) { + return getFromEntityRef(folder.getId(), Relationship.CONTAINS, FOLDER_ENTITY, false); + } + + private List getChildFolders(Folder folder) { + return findTo(folder.getId(), FOLDER_ENTITY, Relationship.CONTAINS, FOLDER_ENTITY); + } + + @SuppressWarnings("unchecked") + public List getChildFolderEntities(Folder folder) { + List childIds = getChildFolders(folder).stream().map(EntityReference::getId).toList(); + if (childIds.isEmpty()) { + return List.of(); + } + return get(null, childIds, getFields(FolderResource.FIELDS), Include.NON_DELETED).stream() + .sorted(Comparator.comparing(Folder::getName)) + .toList(); + } + + @SuppressWarnings("unchecked") + public List getChildFileEntities(Folder folder) { + List childIds = + findTo( + folder.getId(), + FOLDER_ENTITY, + Relationship.CONTAINS, + ContextFileRepository.CONTEXT_FILE_ENTITY) + .stream() + .map(EntityReference::getId) + .toList(); + if (childIds.isEmpty()) { + return List.of(); + } + ContextFileRepository fileRepo = + (ContextFileRepository) + Entity.getEntityRepository(ContextFileRepository.CONTEXT_FILE_ENTITY); + return fileRepo + .get(null, childIds, fileRepo.getFields(ContextFileResource.FIELDS), Include.NON_DELETED) + .stream() + .sorted(Comparator.comparing(ContextFile::getName)) + .toList(); + } + + public class FolderUpdater extends EntityUpdater { + public FolderUpdater(Folder original, Folder updated, Operation operation) { + super(original, updated, operation); + } + + @Override + public void entitySpecificUpdate(boolean consolidatingChanges) { + recordChange("icon", original.getIcon(), updated.getIcon()); + recordChange("color", original.getColor(), updated.getColor()); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/KnowledgePageRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/KnowledgePageRepository.java new file mode 100644 index 000000000000..06e868f6313a --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/KnowledgePageRepository.java @@ -0,0 +1,705 @@ +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.schema.type.EventType.ENTITY_FIELDS_CHANGED; +import static org.openmetadata.schema.type.Relationship.EDITED_BY; +import static org.openmetadata.schema.type.Relationship.HAS; +import static org.openmetadata.schema.type.Relationship.RELATED_TO; +import static org.openmetadata.service.Entity.TEAM; +import static org.openmetadata.service.Entity.USER; +import static org.openmetadata.service.Entity.getEntity; +import static org.openmetadata.service.exception.CatalogExceptionMessage.notReviewer; +import static org.openmetadata.service.governance.workflows.Workflow.RESULT_VARIABLE; +import static org.openmetadata.service.governance.workflows.Workflow.UPDATED_BY_VARIABLE; +import static org.openmetadata.service.util.EntityUtil.entityReferenceMatch; +import static org.openmetadata.service.util.EntityUtil.getId; + +import jakarta.json.JsonPatch; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriInfo; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.jdbi.v3.core.Jdbi; +import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.api.feed.CloseTask; +import org.openmetadata.schema.api.feed.ResolveTask; +import org.openmetadata.schema.attachments.Asset; +import org.openmetadata.schema.attachments.AssetType; +import org.openmetadata.schema.entity.data.Article; +import org.openmetadata.schema.entity.data.Page; +import org.openmetadata.schema.entity.data.PageHierarchy; +import org.openmetadata.schema.entity.data.PageType; +import org.openmetadata.schema.entity.data.QuickLink; +import org.openmetadata.schema.entity.feed.Thread; +import org.openmetadata.schema.entity.teams.Team; +import org.openmetadata.schema.type.ChangeDescription; +import org.openmetadata.schema.type.ChangeEvent; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.EntityStatus; +import org.openmetadata.schema.type.EventType; +import org.openmetadata.schema.type.FieldChange; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.Relationship; +import org.openmetadata.schema.type.TaskStatus; +import org.openmetadata.schema.type.TaskType; +import org.openmetadata.schema.type.change.ChangeSource; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.schema.utils.ResultList; +import org.openmetadata.service.Entity; +import org.openmetadata.service.exception.EntityNotFoundException; +import org.openmetadata.service.governance.workflows.WorkflowHandler; +import org.openmetadata.service.resources.feeds.MessageParser; +import org.openmetadata.service.resources.knowledge.KnowledgePageResource; +import org.openmetadata.service.search.PropagationDescriptor; +import org.openmetadata.service.security.AuthorizationException; +import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.EntityUtil.RelationIncludes; +import org.openmetadata.service.util.FullyQualifiedName; +import org.openmetadata.service.util.RestUtil; +import org.openmetadata.service.util.WebsocketNotificationHandler; + +@Slf4j +@Repository +public class KnowledgePageRepository extends EntityRepository { + public static final String KNOWLEDGE_PAGE_ENTITY = "page"; + private static final String KNOWLEDGE_PATCH_FIELDS = "page,relatedEntities,parent,children"; + private static final String KNOWLEDGE_UPDATE_FIELDS = "page,relatedEntities,parent,children"; + public static final String RELATED_ENTITIES = "relatedEntities"; + public static final String KNOWLEDGE_PAGE_TERM_SEARCH_INDEX = "page"; + private final CollectionDAO.KnowledgePageDAO daoExtension; + private final CollectionDAO.AssetDAO assetDAO; + + /** + * IMPORTANT: relatedEntities excludes domains and dataProducts as they use the HAS relationship + * and are managed separately in EntityRepository. Always use filterOutDomainsAndDataProducts() + * when working with relatedEntities to prevent duplicate assignments. + */ + public KnowledgePageRepository(Jdbi jdbi) { + super( + KnowledgePageResource.COLLECTION_PATH, + KNOWLEDGE_PAGE_ENTITY, + Page.class, + (jdbi.onDemand(CollectionDAO.class)).knowledgePageDAO(), + KNOWLEDGE_PATCH_FIELDS, + KNOWLEDGE_UPDATE_FIELDS); + supportsSearch = true; + // NOTE: SearchIndexFactory registration handled by OpenMetadata core + this.daoExtension = jdbi.onDemand(CollectionDAO.class).knowledgePageDAO(); + this.assetDAO = jdbi.onDemand(CollectionDAO.class).assetDAO(); + } + + @Override + public List getSearchPropagationDescriptors() { + List descriptors = + new ArrayList<>(super.getSearchPropagationDescriptors()); + descriptors.add( + new PropagationDescriptor( + "parent", PropagationDescriptor.PropagationType.ENTITY_REFERENCE, null)); + return descriptors; + } + + @Override + public void setFields( + Page knowledgePage, EntityUtil.Fields fields, RelationIncludes relationIncludes) { + knowledgePage.setRelatedEntities( + fields.contains(RELATED_ENTITIES) + ? getRelatedEntities(knowledgePage) + : knowledgePage.getRelatedEntities()); + knowledgePage.setEditors( + fields.contains("editors") ? getEditors(knowledgePage) : knowledgePage.getEditors()); + knowledgePage.setParent( + fields.contains("parent") ? getParent(knowledgePage) : knowledgePage.getParent()); + knowledgePage.setChildren( + fields.contains("children") ? getChildren(knowledgePage) : knowledgePage.getChildren()); + if (knowledgePage.getPageType().equals(PageType.ARTICLE)) { + Article article = new Article(); + if (knowledgePage.getPage() != null) { + article = JsonUtils.convertValue(knowledgePage.getPage(), Article.class); + } + article.setRelatedArticles( + fields.contains(RELATED_ENTITIES) + ? getRelatedArticles(knowledgePage) + : article.getRelatedArticles()); + knowledgePage.setPage(article); + knowledgePage.setAttachments( + fields.contains("attachments") + ? getAttachments(knowledgePage) + : knowledgePage.getAttachments()); + } + } + + @Override + public void setFullyQualifiedName(Page page) { + if (page.getParent() == null) { + page.setFullyQualifiedName(page.getName()); + } else { + EntityReference parent = page.getParent(); + Page parentPage = Entity.getEntity(parent, "", Include.ALL); + page.setFullyQualifiedName( + FullyQualifiedName.add(parentPage.getFullyQualifiedName(), page.getName())); + } + } + + @Override + public void restorePatchAttributes(Page original, Page updated) { + // Patch can't update Children + super.restorePatchAttributes(original, updated); + updated.withChildren(original.getChildren()); + } + + private List filterOutDomainsAndDataProducts(List entities) { + if (nullOrEmpty(entities)) { + return Collections.emptyList(); + } + return entities.stream() + .filter( + ref -> + !Entity.DOMAIN.equals(ref.getType()) && !Entity.DATA_PRODUCT.equals(ref.getType())) + .collect(Collectors.toList()); + } + + private List getRelatedEntities(Page entity) { + if (entity == null) { + return Collections.emptyList(); + } + List allRelated = findFrom(entity.getId(), KNOWLEDGE_PAGE_ENTITY, HAS, null); + return filterOutDomainsAndDataProducts(allRelated); + } + + private List getEditors(Page entity) { + return entity == null + ? Collections.emptyList() + : findTo(entity.getId(), KNOWLEDGE_PAGE_ENTITY, EDITED_BY, USER); + } + + private List getRelatedArticles(Page entity) { + return findFrom(entity.getId(), KNOWLEDGE_PAGE_ENTITY, RELATED_TO, KNOWLEDGE_PAGE_ENTITY); + } + + private List getAttachments(Page page) { + List json = + assetDAO.getByFqnExact(AssetType.External.value(), page.getFullyQualifiedName()); + if (json == null || json.isEmpty()) { + return Collections.emptyList(); + } + return JsonUtils.readObjects(json, Asset.class); + } + + @Override + protected List getChildren(Page knowledgePage) { + return findTo( + knowledgePage.getId(), + KNOWLEDGE_PAGE_ENTITY, + Relationship.PARENT_OF, + KNOWLEDGE_PAGE_ENTITY); + } + + @Override + public void clearFields(Page entity, EntityUtil.Fields fields) { + entity.withRelatedEntities( + fields.contains(RELATED_ENTITIES) ? entity.getRelatedEntities() : null); + entity.withEditors(fields.contains("editors") ? entity.getEditors() : null); + entity.setParent(fields.contains("parent") ? entity.getParent() : null); + entity.setChildren(fields.contains("children") ? entity.getChildren() : null); + if (entity.getPageType().equals(PageType.ARTICLE)) { + Article article = new Article(); + if (entity.getPage() != null) { + article = JsonUtils.convertValue(entity.getPage(), Article.class); + } + article.withRelatedArticles( + fields.contains(RELATED_ENTITIES) ? article.getRelatedArticles() : null); + entity.withPage(article); + } + } + + @Override + public void prepare(Page knowledgePage, boolean b) { + // Validate Related Entities + List relatedEntities = knowledgePage.getRelatedEntities(); + if (!nullOrEmpty(relatedEntities)) { + List filtered = filterOutDomainsAndDataProducts(relatedEntities); + knowledgePage.withRelatedEntities(filtered); + } + EntityUtil.populateEntityReferences(knowledgePage.getRelatedEntities()); + + if (knowledgePage.getPageType().equals(PageType.ARTICLE)) { + Article article = JsonUtils.convertValue(knowledgePage.getPage(), Article.class); + + // Validate Related Articles + EntityUtil.populateEntityReferences(article.getRelatedArticles()); + + knowledgePage.setPage(article); + } + } + + public ResultList getHierarchyWithSearch( + String parent, PageType pageType, int offset, int limit) { + String pageTypeValue = pageType != null ? pageType.value() : null; + return searchRepository + .getSearchClient() + .listPageHierarchy(parent, pageTypeValue, offset, limit); + } + + public ResultList getHierarchyWithSearchForActivePage( + String activeFqn, PageType pageType, int offset, int limit) { + String pageTypeValue = pageType != null ? pageType.value() : null; + return searchRepository + .getSearchClient() + .listPageHierarchyForActivePage(activeFqn, pageTypeValue, offset, limit); + } + + public List listHierarchy(ListFilter filter, int limit) { + List pageHierarchyList = new ArrayList<>(); + EntityUtil.Fields fields = getFields("parent,children"); + + ResultList resultList = listAfter(null, fields, filter, limit, null); + Map lookUp = + resultList.getData().stream().collect(Collectors.toMap(Page::getId, p -> p)); + List topLevelPages = + resultList.getData().stream().filter(p -> p.getParent() == null).toList(); + + for (Page page : topLevelPages) { + pageHierarchyList.add(getHierarchy(lookUp, page)); + } + + return pageHierarchyList; + } + + public PageHierarchy getHierarchy(Map lookUp, Page topLevelPage) { + PageHierarchy topLevelHierarchy = getPageHierarchy(topLevelPage); + int childrenCount = countChildren(lookUp, topLevelPage); + topLevelHierarchy.withChildrenCount(childrenCount); + return topLevelHierarchy; + } + + private int countChildren(Map lookUp, Page parentPage) { + int childCount = 0; + // For each child reference, we check if the page exists in the lookup map + for (EntityReference childRef : listOrEmpty(parentPage.getChildren())) { + Page childPage = lookUp.get(childRef.getId()); + if (childPage != null) { + childCount++; + } + } + return childCount; + } + + private PageHierarchy getPageHierarchy(Page page) { + // Build a PageHierarchy object from the given Page object + return new PageHierarchy() + .withId(page.getId()) + .withPageType(page.getPageType()) + .withName(page.getName()) + .withDisplayName(page.getDisplayName()) + .withHref(page.getHref()) + .withFullyQualifiedName(page.getFullyQualifiedName()) + .withDescription(page.getDescription()); + } + + @Override + public void storeEntity(Page knowledgePage, boolean update) { + // Related Entities + List relatedEntities = knowledgePage.getRelatedEntities(); + EntityReference parent = knowledgePage.getParent(); + List children = knowledgePage.getChildren(); + knowledgePage.withRelatedEntities(null).withParent(null).withChildren(null); + + if (knowledgePage.getPageType().equals(PageType.ARTICLE)) { + Article article = JsonUtils.convertValue(knowledgePage.getPage(), Article.class); + List relatedArticles = article.getRelatedArticles(); + article.withRelatedArticles(null); + store(knowledgePage, update); + article.withRelatedArticles(relatedArticles); + knowledgePage.withRelatedEntities(relatedEntities).withParent(parent).withChildren(children); + return; + } + + store(knowledgePage, update); + knowledgePage.withRelatedEntities(relatedEntities).withParent(parent).withChildren(children); + } + + @Override + public void storeRelationships(Page knowledgePage) { + // Add Parent for this entity + if (knowledgePage.getParent() != null) { + addRelationship( + knowledgePage.getParent().getId(), + knowledgePage.getId(), + KNOWLEDGE_PAGE_ENTITY, + KNOWLEDGE_PAGE_ENTITY, + Relationship.CONTAINS); + } + + for (EntityReference child : listOrEmpty(knowledgePage.getChildren())) { + addRelationship( + knowledgePage.getId(), + child.getId(), + KNOWLEDGE_PAGE_ENTITY, + KNOWLEDGE_PAGE_ENTITY, + Relationship.CONTAINS); + } + // Add Related Entities + for (EntityReference relatedEntity : listOrEmpty(knowledgePage.getRelatedEntities())) { + addRelationship( + relatedEntity.getId(), + knowledgePage.getId(), + relatedEntity.getType(), + KNOWLEDGE_PAGE_ENTITY, + HAS); + } + + if (knowledgePage.getPageType().equals(PageType.ARTICLE)) { + Article article = JsonUtils.convertValue(knowledgePage.getPage(), Article.class); + for (EntityReference relatedArticle : listOrEmpty(article.getRelatedArticles())) { + addRelationship( + relatedArticle.getId(), + knowledgePage.getId(), + KNOWLEDGE_PAGE_ENTITY, + KNOWLEDGE_PAGE_ENTITY, + RELATED_TO); + } + } + } + + public RestUtil.PutResponse addKnowledgePageUsage( + UriInfo uriInfo, String updatedBy, UUID knowledgePageId, List entityIds) { + Page page = + getEntity(KNOWLEDGE_PAGE_ENTITY, knowledgePageId, RELATED_ENTITIES, Include.NON_DELETED); + List oldValue = page.getRelatedEntities(); + // Create Relationships + List validEntities = filterOutDomainsAndDataProducts(entityIds); + validEntities.forEach( + entityRef -> + addRelationship( + entityRef.getId(), + knowledgePageId, + entityRef.getType(), + KNOWLEDGE_PAGE_ENTITY, + HAS)); + + // Populate Fields + setFieldsInternal(page, new EntityUtil.Fields(allowedFields, RELATED_ENTITIES)); + Entity.withHref(uriInfo, page.getRelatedEntities()); + ChangeEvent changeEvent = + getKnowledgeChangeEvent( + updatedBy, + RELATED_ENTITIES, + oldValue, + page.getRelatedEntities(), + withHref(uriInfo, page)); + return new RestUtil.PutResponse<>(Response.Status.CREATED, changeEvent, ENTITY_FIELDS_CHANGED); + } + + public RestUtil.PutResponse removeKnowledgePageUsedIn( + UriInfo uriInfo, String updatedBy, UUID knowledgePageId, List entityIds) { + Page page = + getEntity(KNOWLEDGE_PAGE_ENTITY, knowledgePageId, RELATED_ENTITIES, Include.NON_DELETED); + List oldValue = page.getRelatedEntities(); + List validEntities = filterOutDomainsAndDataProducts(entityIds); + for (EntityReference ref : validEntities) { + deleteRelationship(ref.getId(), ref.getType(), knowledgePageId, KNOWLEDGE_PAGE_ENTITY, HAS); + } + + // Populate Fields + setFieldsInternal(page, new EntityUtil.Fields(allowedFields, RELATED_ENTITIES)); + Entity.withHref(uriInfo, page.getRelatedEntities()); + ChangeEvent changeEvent = + getKnowledgeChangeEvent( + updatedBy, + RELATED_ENTITIES, + oldValue, + page.getRelatedEntities(), + withHref(uriInfo, page)); + return new RestUtil.PutResponse<>(Response.Status.CREATED, changeEvent, ENTITY_FIELDS_CHANGED); + } + + private ChangeEvent getKnowledgeChangeEvent( + String updatedBy, String fieldUpdated, Object oldValue, Object newValue, Page updatedPage) { + FieldChange fieldChange = + new FieldChange().withName(fieldUpdated).withNewValue(newValue).withOldValue(oldValue); + ChangeDescription change = + new ChangeDescription().withPreviousVersion(updatedPage.getVersion()); + change.getFieldsUpdated().add(fieldChange); + return new ChangeEvent() + .withEntity(updatedPage) + .withChangeDescription(change) + .withEventType(EventType.ENTITY_UPDATED) + .withEntityType(entityType) + .withEntityId(updatedPage.getId()) + .withEntityFullyQualifiedName(updatedPage.getFullyQualifiedName()) + .withUserName(updatedBy) + .withTimestamp(System.currentTimeMillis()) + .withCurrentVersion(updatedPage.getVersion()) + .withPreviousVersion(updatedPage.getVersion()); + } + + @Override + public EntityUpdater getUpdater( + Page original, Page updated, Operation operation, ChangeSource source) { + return new KnowledgePageUpdater(original, updated, operation); + } + + public class KnowledgePageUpdater extends EntityUpdater { + public KnowledgePageUpdater(Page original, Page updated, Operation operation) { + super(original, updated, operation); + } + + @Override + public void entitySpecificUpdate(boolean consolidatingChanges) { + // Update Related Terms + updateRelatedEntities(original, updated); + + // Updated Quick Link + if (original.getPageType().equals(PageType.QUICK_LINK)) { + QuickLink originalLink = JsonUtils.convertValue(original.getPage(), QuickLink.class); + QuickLink updatedLink = JsonUtils.convertValue(updated.getPage(), QuickLink.class); + recordChange("quickLink", originalLink, updatedLink); + } + + // Updated Article + if (original.getPageType().equals(PageType.ARTICLE)) { + updateArticles(original, updated); + } + + // Add Editor + if (fieldsChanged() && updatingUser.getId() != null) { + addRelationship( + original.getId(), updatingUser.getId(), KNOWLEDGE_PAGE_ENTITY, USER, EDITED_BY); + } + + updateParent(original, updated); + } + + private void updateParent(Page original, Page updated) { + UUID oldParentId = getId(original.getParent()); + UUID newParentId = getId(updated.getParent()); + final boolean parentChanged = !Objects.equals(oldParentId, newParentId); + if (parentChanged) { + if (oldParentId != null) { + deleteRelationship( + oldParentId, + KNOWLEDGE_PAGE_ENTITY, + original.getId(), + KNOWLEDGE_PAGE_ENTITY, + Relationship.CONTAINS); + } + if (newParentId != null) { + setFullyQualifiedName(updated); + daoExtension.updateFqn(original.getFullyQualifiedName(), updated.getFullyQualifiedName()); + addRelationship( + newParentId, + original.getId(), + KNOWLEDGE_PAGE_ENTITY, + KNOWLEDGE_PAGE_ENTITY, + Relationship.CONTAINS); + } else { + setFullyQualifiedName(updated); + daoExtension.updateFqn(original.getFullyQualifiedName(), updated.getFullyQualifiedName()); + } + recordChange( + "parent", original.getParent(), updated.getParent(), true, entityReferenceMatch); + } + } + + private void updateChildren(Page original, Page updated) { + List origChildren = listOrEmpty(original.getChildren()); + List updatedChildren = listOrEmpty(updated.getChildren()); + updateToRelationships( + "children", + KNOWLEDGE_PAGE_ENTITY, + original.getId(), + Relationship.PARENT_OF, + KNOWLEDGE_PAGE_ENTITY, + origChildren, + updatedChildren, + false); + } + + private void updateRelatedEntities(Page original, Page updated) { + List origRelatedEntities = + filterOutDomainsAndDataProducts(listOrEmpty(original.getRelatedEntities())); + List updatedRelatedEntities = + filterOutDomainsAndDataProducts(listOrEmpty(updated.getRelatedEntities())); + List added = new ArrayList<>(); + List deleted = new ArrayList<>(); + if (!recordListChange( + RELATED_ENTITIES, + origRelatedEntities, + updatedRelatedEntities, + added, + deleted, + entityReferenceMatch)) { + return; // No changes between original and updated. + } + // Remove relationships from original + for (EntityReference ref : origRelatedEntities) { + deleteRelationship( + ref.getId(), ref.getType(), original.getId(), KNOWLEDGE_PAGE_ENTITY, HAS); + } + + // Add relationships from updated + for (EntityReference ref : updatedRelatedEntities) { + addRelationship(ref.getId(), original.getId(), ref.getType(), KNOWLEDGE_PAGE_ENTITY, HAS); + } + updatedRelatedEntities.sort(EntityUtil.compareEntityReference); + origRelatedEntities.sort(EntityUtil.compareEntityReference); + } + + private void updateArticles(Page original, Page updated) { + Article oldArticle = JsonUtils.convertValue(original.getPage(), Article.class); + Article updateArticle = JsonUtils.convertValue(updated.getPage(), Article.class); + + // Related Articles + List origRelatedArticles = listOrEmpty(oldArticle.getRelatedArticles()); + List updatedRelatedArticles = + listOrEmpty(updateArticle.getRelatedArticles()); + updateFromRelationships( + RELATED_ENTITIES, + KNOWLEDGE_PAGE_ENTITY, + origRelatedArticles, + updatedRelatedArticles, + RELATED_TO, + KNOWLEDGE_PAGE_ENTITY, + original.getId()); + } + } + + protected void updateTaskWithNewReviewers(Page page) { + try { + MessageParser.EntityLink about = + new MessageParser.EntityLink(KNOWLEDGE_PAGE_ENTITY, page.getFullyQualifiedName()); + FeedRepository feedRepository = Entity.getFeedRepository(); + Thread originalTask = + feedRepository.getTask(about, TaskType.RequestApproval, TaskStatus.Open); + page = + Entity.getEntityByName( + KNOWLEDGE_PAGE_ENTITY, + page.getFullyQualifiedName(), + "id,fullyQualifiedName,reviewers", + Include.ALL); + + Thread updatedTask = JsonUtils.deepCopy(originalTask, Thread.class); + updatedTask.getTask().withAssignees(new ArrayList<>(page.getReviewers())); + JsonPatch patch = JsonUtils.getJsonPatch(originalTask, updatedTask); + RestUtil.PatchResponse thread = + feedRepository.patchThread(null, originalTask.getId(), updatedTask.getUpdatedBy(), patch); + + // Send WebSocket Notification + WebsocketNotificationHandler.handleTaskNotification(thread.entity()); + } catch (EntityNotFoundException e) { + // Task may not be present + LOG.debug("Task not found for page {}", page.getFullyQualifiedName()); + } + } + + @Override + public FeedRepository.TaskWorkflow getTaskWorkflow(FeedRepository.ThreadContext threadContext) { + validateTaskThread(threadContext); + TaskType taskType = threadContext.getThread().getTask().getType(); + return new ApprovalTaskWorkflow(threadContext); + } + + public static class ApprovalTaskWorkflow extends FeedRepository.TaskWorkflow { + ApprovalTaskWorkflow(FeedRepository.ThreadContext threadContext) { + super(threadContext); + } + + @Override + public EntityInterface performTask(String user, ResolveTask resolveTask) { + Page page = (Page) threadContext.getAboutEntity(); + checkUpdatedByReviewer(page, user); + + UUID taskId = threadContext.getThread().getId(); + Map variables = new HashMap<>(); + variables.put(RESULT_VARIABLE, resolveTask.getNewValue().equalsIgnoreCase("approved")); + variables.put(UPDATED_BY_VARIABLE, user); + WorkflowHandler workflowHandler = WorkflowHandler.getInstance(); + workflowHandler.resolveTask( + taskId, workflowHandler.transformToNodeVariables(taskId, variables)); + + return page; + } + } + + @Override + public void postUpdate(Page original, Page updated) { + super.postUpdate(original, updated); + if (EntityStatus.IN_REVIEW.equals(original.getEntityStatus())) { + if (EntityStatus.APPROVED.equals(updated.getEntityStatus())) { + closeApprovalTask(updated, "Approved the page"); + } else if (EntityStatus.REJECTED.equals(updated.getEntityStatus())) { + closeApprovalTask(updated, "Rejected the page"); + } + } + + // TODO: It might happen that a task went from DRAFT to IN_REVIEW to DRAFT fairly quickly + // Due to ChangesConsolidation, the postUpdate will be called as from DRAFT to DRAFT, but there + // will be a Task created. + // This if handles this case scenario, by guaranteeing that we are any Approval Task if the + // Tag goes back to DRAFT. + if (EntityStatus.DRAFT.equals(updated.getEntityStatus())) { + try { + closeApprovalTask(updated, "Closed due to page going back to DRAFT."); + } catch (EntityNotFoundException ignored) { + } // No ApprovalTask is present, and thus we don't need to worry about this. + } + } + + private void closeApprovalTask(Page entity, String comment) { + MessageParser.EntityLink about = + new MessageParser.EntityLink(KNOWLEDGE_PAGE_ENTITY, entity.getFullyQualifiedName()); + FeedRepository feedRepository = Entity.getFeedRepository(); + + // Skip closing tasks if updatedBy is null (e.g., during tests) + if (entity.getUpdatedBy() == null) { + LOG.debug( + "Skipping task closure for page {} - updatedBy is null", entity.getFullyQualifiedName()); + return; + } + + // Close User Tasks + try { + Thread taskThread = feedRepository.getTask(about, TaskType.RequestApproval, TaskStatus.Open); + feedRepository.closeTask( + taskThread, entity.getUpdatedBy(), new CloseTask().withComment(comment)); + } catch (EntityNotFoundException ex) { + LOG.info("No approval task found for page {}", entity.getFullyQualifiedName()); + } + } + + public static void checkUpdatedByReviewer(Page page, String updatedBy) { + // Only list of allowed reviewers can change the status from DRAFT to APPROVED + List reviewers = page.getReviewers(); + if (!nullOrEmpty(reviewers)) { + // Updating user must be one of the reviewers + boolean isReviewer = + reviewers.stream() + .anyMatch( + e -> { + if (e.getType().equals(TEAM)) { + Team team = + Entity.getEntityByName(TEAM, e.getName(), "users", Include.NON_DELETED); + return team.getUsers().stream() + .anyMatch( + u -> + u.getName().equals(updatedBy) + || u.getFullyQualifiedName().equals(updatedBy)); + } else { + return e.getName().equals(updatedBy) + || e.getFullyQualifiedName().equals(updatedBy); + } + }); + if (!isReviewer) { + throw new AuthorizationException(notReviewer(updatedBy)); + } + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/attachments/AttachmentResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/attachments/AttachmentResource.java new file mode 100644 index 000000000000..88090f0e2bfd --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/attachments/AttachmentResource.java @@ -0,0 +1,394 @@ +package org.openmetadata.service.resources.attachments; + +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.ws.rs.*; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.SecurityContext; +import jakarta.ws.rs.core.UriInfo; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URLConnection; +import java.time.Duration; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeoutException; +import lombok.extern.slf4j.Slf4j; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.glassfish.jersey.media.multipart.FormDataParam; +import org.jdbi.v3.core.Jdbi; +import org.openmetadata.schema.api.attachments.CreateAsset; +import org.openmetadata.schema.attachments.Asset; +import org.openmetadata.schema.attachments.AssetType; +import org.openmetadata.schema.type.MetadataOperation; +import org.openmetadata.sdk.exception.AssetServiceException; +import org.openmetadata.sdk.exception.AttachmentException; +import org.openmetadata.service.OpenMetadataApplicationConfig; +import org.openmetadata.service.attachments.AssetService; +import org.openmetadata.service.attachments.AssetServiceFactory; +import org.openmetadata.service.jdbi3.AssetRepository; +import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.resources.Collection; +import org.openmetadata.service.resources.drive.ContextFileUploadSupport; +import org.openmetadata.service.resources.drive.ContextFileUploadSupport.BufferedUpload; +import org.openmetadata.service.resources.drive.ContextFileUploadSupport.MaxFileSizeExceededException; +import org.openmetadata.service.resources.feeds.MessageParser; +import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.security.policyevaluator.OperationContext; +import org.openmetadata.service.security.policyevaluator.ResourceContext; +import org.openmetadata.service.security.policyevaluator.ResourceContextInterface; + +@Slf4j +@Path("/v1/attachments") +@Tag(name = "Attachments", description = "APIs related to uploading attachments.") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +@Collection(name = "Attachments") +public class AttachmentResource { + // 5 MiB — conservative default when object storage is configured but didn't specify a + // max file size, and the "storage disabled" case where initialize() still leaves the + // field at a valid number (so uploads fail with a clear validation error rather than a + // silent "size > 0" mismatch). + private static final long DEFAULT_MAX_FILE_SIZE = 5L * 1024 * 1024; + + private final AssetRepository assetRepository; + private AssetService assetService; + private final Authorizer authorizer; + private long maxFileSize = DEFAULT_MAX_FILE_SIZE; + private String cdnUrl; + + public AttachmentResource(Jdbi jdbi, Authorizer authorizer) { + CollectionDAO extension = jdbi.onDemand(CollectionDAO.class); + this.assetRepository = new AssetRepository(extension.assetDAO()); + this.authorizer = authorizer; + } + + public void initialize(OpenMetadataApplicationConfig config) { + // Object storage is optional — deployments that don't configure it must not crash + // the server at startup with an NPE on config.getObjectStorage().getMaxFileSize(). + // Mirrors the guarded ContextFileResource.initialize() flow. + if (config.getObjectStorage() == null) { + LOG.info("Object storage is not configured; attachments API will not accept uploads"); + return; + } + this.maxFileSize = + config.getObjectStorage().getMaxFileSize() > 0 + ? config.getObjectStorage().getMaxFileSize() + : DEFAULT_MAX_FILE_SIZE; + this.cdnUrl = + config.getObjectStorage().getAzureConfiguration() != null + && config.getObjectStorage().getAzureConfiguration().getCdnUrl() != null + ? config.getObjectStorage().getAzureConfiguration().getCdnUrl() + : config.getObjectStorage().getS3Configuration() != null + ? config.getObjectStorage().getS3Configuration().getCloudFrontUrl() + : null; + AssetServiceFactory.init(config); + this.assetService = AssetServiceFactory.getService(); + } + + @GET + @Path("/{id}") + public Response getAssetById( + @PathParam("id") String id, @Context SecurityContext securityContext) { + Asset asset = assetRepository.getById(id); + if (asset == null) { + return Response.status(Response.Status.NOT_FOUND).build(); + } + return Response.ok(asset).build(); + } + + @POST + @Path("/upload") + @Consumes(MediaType.MULTIPART_FORM_DATA) + public Response uploadAttachment( + @FormDataParam("file") InputStream fileInputStream, + @FormDataParam("file") FormDataContentDisposition fileDetail, + @FormDataParam("entityLink") String entityLink, + @FormDataParam("assetType") @DefaultValue("Inline") AssetType assetType, + @Context UriInfo uriInfo, + @Context SecurityContext securityContext) + throws IOException { + MessageParser.EntityLink parsedLink = MessageParser.EntityLink.parse(entityLink); + ResourceContextInterface resourceContext = + new ResourceContext<>(parsedLink.getEntityType(), null, parsedLink.getEntityFQN()); + OperationContext operationContext = + new OperationContext(parsedLink.getEntityType(), MetadataOperation.EDIT_DESCRIPTION); + authorizer.authorize(securityContext, operationContext, resourceContext); + + Asset asset = + createAssetFromUpload(fileInputStream, fileDetail, entityLink, assetType, securityContext); + + String proxyUrl; + if (asset.getAssetType() == AssetType.Inline) { + if (cdnUrl != null && !cdnUrl.isEmpty()) { + proxyUrl = cdnUrl + "/assets/" + asset.getId(); + } else { + proxyUrl = + uriInfo + .getBaseUriBuilder() + .path(AttachmentResource.class) + .path(asset.getId() + "/download") + .queryParam("direct", true) + .build() + .toString(); + } + } else { + proxyUrl = + uriInfo + .getBaseUriBuilder() + .path(AttachmentResource.class) + .path(asset.getId() + "/download") + .queryParam("direct", false) + .build() + .toString(); + } + asset.setUrl(proxyUrl); + try { + assetRepository.create(asset); + } catch (Exception e) { + try { + assetService.delete(asset); + } catch (Exception ignored) { + LOG.warn("Failed to enqueue cleanup for asset {}", asset.getId(), ignored); + } + throw AttachmentException.byMessage( + "Failed to create asset in the database. Upload has been rolled back.", e.getMessage()); + } + return Response.status(Response.Status.CREATED).entity(asset).build(); + } + + @GET + @Path("/{id}/download") + public Response downloadAsset( + @PathParam("id") String id, + @QueryParam("expiry") @DefaultValue("3600") int expirySeconds, + @QueryParam("direct") @DefaultValue("false") boolean direct, + @Context SecurityContext securityContext) { + Asset asset = assetRepository.getById(id); + if (asset == null) { + return Response.status(Response.Status.NOT_FOUND).build(); + } + + // Authorization check + MessageParser.EntityLink parsedLink = MessageParser.EntityLink.parse(asset.getEntityLink()); + ResourceContextInterface resourceContext = + new ResourceContext<>(parsedLink.getEntityType(), null, parsedLink.getEntityFQN()); + OperationContext operationContext = + new OperationContext(parsedLink.getEntityType(), MetadataOperation.VIEW_BASIC); + authorizer.authorize(securityContext, operationContext, resourceContext); + + boolean isImage = asset.getContentType() != null && asset.getContentType().startsWith("image/"); + boolean useCdn = cdnUrl != null && !cdnUrl.isEmpty(); + + if (useCdn) { + try { + String signedUrl = + assetService.generateDownloadUrlWithExpiry(asset, Duration.ofSeconds(expirySeconds)); + + if (signedUrl != null) { + if (isImage && direct) { + return Response.ok(signedUrl).build(); + } else { + return Response.temporaryRedirect(URI.create(signedUrl)).build(); + } + } + } catch (Exception e) { + LOG.error("Error generating CDN URL: {}", e.getMessage(), e); + } + } + + // Fallback to direct serving + LOG.debug( + useCdn + ? "Falling back to direct serving after CDN URL generation failed" + : "Serving asset {} directly", + asset.getId()); + + try { + InputStream fileStream = assetService.read(asset).join(); + if (isImage && direct) { + return Response.ok(fileStream, asset.getContentType()).build(); + } else { + // Use the shared RFC-5987-aware Content-Disposition builder to prevent header + // injection via quotes/CRLF in asset.getFileName() and to round-trip non-ASCII + // filenames safely. Matches ContextFileResource's download flow. + return Response.ok(fileStream, asset.getContentType()) + .header( + "Content-Disposition", + ContextFileUploadSupport.buildContentDisposition(asset.getFileName())) + .build(); + } + } catch (java.util.concurrent.CompletionException e) { + // Handle timeout and other async exceptions + Throwable cause = e.getCause(); + + // Check if it's a timeout by examining the cause chain + if (isTimeoutException(cause)) { + LOG.error("Timeout reading asset {}", asset.getId()); + return Response.status(Response.Status.GATEWAY_TIMEOUT) + .entity("{\"message\":\"Asset download timed out. Please try again later.\"}") + .type(MediaType.APPLICATION_JSON) + .build(); + } + + if (cause instanceof AssetServiceException ase) { + // Log full details server-side, but return sanitized message to client + LOG.error("Failed to read asset {}: {}", asset.getId(), ase.getMessage()); + + return Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity( + "{\"message\":\"Failed to download asset. Please contact support if the problem persists.\"}") + .type(MediaType.APPLICATION_JSON) + .build(); + } + + LOG.error("Unexpected error reading asset {}: {}", asset.getId(), e.getMessage(), e); + return Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity("{\"message\":\"Unexpected error downloading asset\"}") + .type(MediaType.APPLICATION_JSON) + .build(); + } + } + + @DELETE + @Path("/{id}") + public Response deleteAttachment( + @PathParam("id") String id, + @QueryParam("hardDelete") @DefaultValue("false") boolean hardDelete, + @Context SecurityContext securityContext) { + Asset asset = assetRepository.getById(id); + if (asset == null) { + return Response.status(Response.Status.NOT_FOUND).build(); + } + MessageParser.EntityLink parsedLink = MessageParser.EntityLink.parse(asset.getEntityLink()); + ResourceContextInterface resourceContext = + new ResourceContext<>(parsedLink.getEntityType(), null, parsedLink.getEntityFQN()); + OperationContext operationContext = + new OperationContext(parsedLink.getEntityType(), MetadataOperation.EDIT_DESCRIPTION); + authorizer.authorize(securityContext, operationContext, resourceContext); + + if (hardDelete) { + try { + assetService.delete(asset); + } catch (RejectedExecutionException e) { + return Response.status(Response.Status.TOO_MANY_REQUESTS) + .entity( + "{\"message\":\"Object delete queue is full. Please retry the attachment delete.\"}") + .type(MediaType.APPLICATION_JSON) + .build(); + } + assetRepository.delete(asset.getId()); + } else { + assetRepository.markDeleted(asset.getEntityLink()); + } + return Response.ok().build(); + } + + @GET + @Path("/fqn/{fqn}/{assetType}") + public Response listAttachmentsByFqn( + @PathParam("fqn") String fqn, @PathParam("assetType") AssetType assetType) { + List assets = assetRepository.getByFQN(fqn, assetType); + if (assets == null) { + return Response.status(Response.Status.NOT_FOUND).build(); + } + return Response.ok(assets).build(); + } + + private Asset buildAsset(CreateAsset createAsset, String url, String updatedBy) { + MessageParser.EntityLink assetLink = + MessageParser.EntityLink.parse(createAsset.getEntityLink()); + Asset asset = new Asset(); + asset.setId(UUID.randomUUID().toString()); + asset.setFileName(createAsset.getFileName()); + asset.setContentType(createAsset.getContentType()); + asset.setSize(createAsset.getSize()); + asset.setEntityLink(createAsset.getEntityLink()); + asset.setFullyQualifiedName(assetLink.getEntityFQN()); + asset.setUrl(url); + asset.setAssetType(createAsset.getAssetType()); + asset.setUpdatedBy(updatedBy); + asset.setUpdatedAt(System.currentTimeMillis()); + asset.setDeleted(false); + return asset; + } + + private Asset createAssetFromUpload( + InputStream fileInputStream, + FormDataContentDisposition fileDetail, + String entityLink, + AssetType assetType, + SecurityContext securityContext) + throws IOException { + + // Stream into a bounded temp file instead of IOUtils.toByteArray so an attacker + // sending an arbitrarily large body cannot exhaust heap before the size check runs. + // The helper throws MaxFileSizeExceededException the moment totalBytes passes + // maxFileSize, which we translate to a 4xx-style AttachmentException below. + try (BufferedUpload buffered = + ContextFileUploadSupport.bufferUpload(fileInputStream, maxFileSize)) { + String originalFileName = + fileDetail.getFileName() != null ? fileDetail.getFileName() : fileDetail.getName(); + String extension = ""; + int dotIndex = originalFileName == null ? -1 : originalFileName.lastIndexOf('.'); + if (dotIndex != -1) { + extension = originalFileName.substring(dotIndex); + } + + String contentType = URLConnection.guessContentTypeFromName(originalFileName); + if (contentType == null) { + contentType = "application/octet-stream"; + } + + CreateAsset createAsset = new CreateAsset(); + createAsset.setEntityLink(entityLink); + createAsset.setAssetType(assetType); + + Asset asset = buildAsset(createAsset, "", securityContext.getUserPrincipal().getName()); + asset.setFileName(originalFileName); + asset.setSize(Math.toIntExact(buffered.getSize())); + asset.setContentType(contentType); + asset.setAssetType(assetType); + asset.setExtension(extension); + if (assetService == null) { + throw AssetServiceException.byMessage( + "Asset Service is unavailable", "Please reach out to administrator."); + } + try (InputStream bodyStream = buffered.newInputStream()) { + assetService.upload(asset, bodyStream).join(); + } + return asset; + } catch (MaxFileSizeExceededException tooBig) { + throw AttachmentException.byMessage( + "File Size Validation", + String.format( + "File size (%s) exceeds maximum allowed size of %s", + formatFileSize(tooBig.getActualSize()), formatFileSize(tooBig.getMaxFileSize()))); + } + } + + private String formatFileSize(long bytes) { + if (bytes < 1024) return bytes + " B"; + if (bytes < 1024 * 1024) return String.format("%.2f KB", bytes / 1024.0); + if (bytes < 1024 * 1024 * 1024) return String.format("%.2f MB", bytes / (1024.0 * 1024)); + return String.format("%.2f GB", bytes / (1024.0 * 1024 * 1024)); + } + + /** + * Checks if the exception or any of its causes is a TimeoutException. + * More robust than string matching on exception messages. + */ + private boolean isTimeoutException(Throwable throwable) { + Throwable current = throwable; + while (current != null) { + if (current instanceof TimeoutException) { + return true; + } + current = current.getCause(); + } + return false; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/drive/ContextFileMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/drive/ContextFileMapper.java new file mode 100644 index 000000000000..3b36846f5e9b --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/drive/ContextFileMapper.java @@ -0,0 +1,28 @@ +package org.openmetadata.service.resources.drive; + +import static org.openmetadata.service.util.EntityUtil.getEntityReference; + +import org.openmetadata.schema.api.data.CreateContextFile; +import org.openmetadata.schema.entity.data.ContextFile; +import org.openmetadata.schema.type.Votes; +import org.openmetadata.service.jdbi3.FolderRepository; +import org.openmetadata.service.mapper.EntityMapper; + +public class ContextFileMapper implements EntityMapper { + @Override + public ContextFile createToEntity(CreateContextFile create, String user) { + return copy(new ContextFile(), create, user) + .withTags(create.getTags()) + .withVotes(new Votes().withUpVotes(0).withDownVotes(0)) + .withFileType(create.getFileType()) + .withFileSize(create.getFileSize()) + .withContentType(create.getContentType()) + .withFileExtension(create.getFileExtension()) + .withFolder(getEntityReference(FolderRepository.FOLDER_ENTITY, create.getFolder())) + .withAssetId(create.getAssetId()) + .withProcessingStatus(create.getProcessingStatus()) + .withSourceType(create.getSourceType()) + .withSourceId(create.getSourceId()) + .withSourceUrl(create.getSourceUrl()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/drive/ContextFileResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/drive/ContextFileResource.java new file mode 100644 index 000000000000..f0bc652fc838 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/drive/ContextFileResource.java @@ -0,0 +1,465 @@ +package org.openmetadata.service.resources.drive; + +import static org.openmetadata.service.jdbi3.ContextFileRepository.CONTEXT_FILE_ENTITY; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.validation.Valid; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.DefaultValue; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.PATCH; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.WebApplicationException; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.SecurityContext; +import jakarta.ws.rs.core.StreamingOutput; +import jakarta.ws.rs.core.UriInfo; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URLConnection; +import java.time.Duration; +import java.util.List; +import java.util.UUID; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.glassfish.jersey.media.multipart.FormDataParam; +import org.openmetadata.schema.api.data.CreateContextFile; +import org.openmetadata.schema.api.data.RestoreEntity; +import org.openmetadata.schema.attachments.Asset; +import org.openmetadata.schema.entity.data.ContextFile; +import org.openmetadata.schema.entity.data.ContextFileContent; +import org.openmetadata.schema.entity.data.ContextFileType; +import org.openmetadata.schema.entity.data.ProcessingStatus; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.MetadataOperation; +import org.openmetadata.schema.utils.ResultList; +import org.openmetadata.service.Entity; +import org.openmetadata.service.OpenMetadataApplicationConfig; +import org.openmetadata.service.attachments.AssetService; +import org.openmetadata.service.attachments.AssetServiceFactory; +import org.openmetadata.service.attachments.AzureAssetService; +import org.openmetadata.service.attachments.S3AssetService; +import org.openmetadata.service.drive.ContextFileExtractionService; +import org.openmetadata.service.jdbi3.ContextFileRepository; +import org.openmetadata.service.jdbi3.ListFilter; +import org.openmetadata.service.limits.Limits; +import org.openmetadata.service.resources.Collection; +import org.openmetadata.service.resources.EntityResource; +import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.security.ImpersonationContext; + +@Tag(name = "Drive Files", description = "APIs for managing files in the Context Center Drive.") +@Path("/v1/drive/files") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +@Collection(name = "driveFiles") +public class ContextFileResource extends EntityResource { + public static final String COLLECTION_PATH = "v1/drive/files/"; + public static final String FIELDS = "owners,tags,folder,domains,followers,votes"; + private final ContextFileMapper mapper = new ContextFileMapper(); + private final ContextFileExtractionService extractionService; + private long maxFileSize = 5 * 1024 * 1024L; + + public ContextFileResource(Authorizer authorizer, Limits limits) { + super(CONTEXT_FILE_ENTITY, authorizer, limits); + this.extractionService = new ContextFileExtractionService(repository); + } + + @Override + public void initialize(OpenMetadataApplicationConfig config) { + AssetServiceFactory.init(config); + if (config.getObjectStorage() != null) { + maxFileSize = config.getObjectStorage().getMaxFileSize(); + } + } + + public static class ContextFileList extends ResultList {} + + @Override + protected List getEntitySpecificOperations() { + addViewOperation("folder", MetadataOperation.VIEW_BASIC); + return List.of(); + } + + @Override + public ContextFile addHref(UriInfo uriInfo, ContextFile file) { + super.addHref(uriInfo, file); + Entity.withHref(uriInfo, file.getFolder()); + return file; + } + + @GET + @Operation( + operationId = "listDriveFiles", + summary = "List files", + responses = { + @ApiResponse( + responseCode = "200", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = ContextFileList.class))) + }) + public ResultList list( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @QueryParam("fields") String fieldsParam, + @QueryParam("limit") @DefaultValue("10") int limit, + @QueryParam("before") String before, + @QueryParam("after") String after, + @QueryParam("include") @DefaultValue("non-deleted") Include include) { + return super.listInternal( + uriInfo, securityContext, fieldsParam, new ListFilter(include), limit, before, after); + } + + @GET + @Path("/{id}") + @Operation(operationId = "getDriveFile", summary = "Get a file by ID") + public ContextFile get( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @PathParam("id") UUID id, + @QueryParam("fields") String fieldsParam, + @QueryParam("include") @DefaultValue("non-deleted") Include include) { + return getInternal(uriInfo, securityContext, id, fieldsParam, include); + } + + @GET + @Path("/name/{fqn}") + @Operation(operationId = "getDriveFileByFqn", summary = "Get a file by FQN") + public ContextFile getByName( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @PathParam("fqn") String fqn, + @QueryParam("fields") String fieldsParam, + @QueryParam("include") @DefaultValue("non-deleted") Include include) { + return getByNameInternal(uriInfo, securityContext, fqn, fieldsParam, include); + } + + @POST + @Operation(operationId = "createDriveFile", summary = "Create a file entry") + public Response create( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Valid CreateContextFile createFile) { + ContextFile file = + mapper.createToEntity(createFile, securityContext.getUserPrincipal().getName()); + return create(uriInfo, securityContext, file); + } + + @PUT + @Operation(operationId = "createOrUpdateDriveFile", summary = "Create or update a file") + public Response createOrUpdate( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Valid CreateContextFile createFile) { + ContextFile file = + mapper.createToEntity(createFile, securityContext.getUserPrincipal().getName()); + return createOrUpdate(uriInfo, securityContext, file); + } + + @PATCH + @Path("/{id}") + @Consumes(MediaType.APPLICATION_JSON_PATCH_JSON) + @Operation(operationId = "patchDriveFile", summary = "Update a file via JSON Patch") + public Response patch( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @PathParam("id") UUID id, + @Valid jakarta.json.JsonPatch patch) { + return patchInternal(uriInfo, securityContext, id, patch); + } + + @POST + @Path("/upload") + @Consumes(MediaType.MULTIPART_FORM_DATA) + @Operation( + operationId = "uploadDriveFile", + summary = "Upload a file to Drive", + description = "Uploads a file to S3 and creates a ContextFile entity.", + responses = { + @ApiResponse( + responseCode = "201", + description = "File uploaded", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = ContextFile.class))) + }) + public Response uploadFile( + @FormDataParam("file") InputStream fileInputStream, + @FormDataParam("file") FormDataContentDisposition fileDetail, + @FormDataParam("displayName") String displayName, + @FormDataParam("description") String description, + @FormDataParam("folder") String folderFqn, + @Context UriInfo uriInfo, + @Context SecurityContext securityContext) + throws IOException { + String user = securityContext.getUserPrincipal().getName(); + String originalFileName = + fileDetail.getFileName() != null ? fileDetail.getFileName() : fileDetail.getName(); + String contentType = URLConnection.guessContentTypeFromName(originalFileName); + if (contentType == null) { + contentType = "application/octet-stream"; + } + String fileExtension = ""; + int dotIdx = originalFileName.lastIndexOf('.'); + if (dotIdx != -1) { + fileExtension = originalFileName.substring(dotIdx + 1).toLowerCase(); + } + + AssetService assetService = AssetServiceFactory.getService(); + if (assetService == null) { + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .entity("{\"message\":\"Object storage is not configured\"}") + .type(MediaType.APPLICATION_JSON) + .build(); + } + + String pageName = ContextFileUploadSupport.sanitizeEntityName(originalFileName); + ContextFileType fileType = ContextFileUploadSupport.detectFileType(contentType); + + CreateContextFile createFile = new CreateContextFile(); + createFile.setName(pageName); + createFile.setDisplayName(displayName != null ? displayName : originalFileName); + createFile.setDescription(description); + createFile.setFileType(fileType); + createFile.setContentType(contentType); + createFile.setFileExtension(fileExtension); + createFile.setProcessingStatus(ProcessingStatus.Uploaded); + if (folderFqn != null && !folderFqn.isEmpty()) { + createFile.setFolder(folderFqn); + } + + try (ContextFileUploadSupport.BufferedUpload bufferedUpload = + ContextFileUploadSupport.bufferUpload(fileInputStream, maxFileSize)) { + createFile.setFileSize(Math.toIntExact(bufferedUpload.getSize())); + + ContextFile file = mapper.createToEntity(createFile, user); + repository.prepareInternal(file, false); + + Asset asset = + ContextFileUploadSupport.buildAsset( + file, originalFileName, contentType, fileExtension, bufferedUpload.getSize(), user); + ContextFileContent content = + ContextFileUploadSupport.buildContent(file, asset, bufferedUpload.getChecksum(), user); + file.setAssetId(asset.getId()); + file.setHeadContentId(content.getId().toString()); + + boolean assetUploaded = false; + boolean assetPersisted = false; + boolean contentPersisted = false; + ContextFile createdFile = null; + try { + try (InputStream uploadStream = bufferedUpload.newInputStream()) { + assetService.upload(asset, uploadStream).join(); + } + assetUploaded = true; + repository.getAssetRepository().create(asset); + assetPersisted = true; + + Response createResponse = create(uriInfo, securityContext, file); + createdFile = (ContextFile) createResponse.getEntity(); + + repository + .getContentRepository() + .create(null, content, user, ImpersonationContext.getImpersonatedBy()); + contentPersisted = true; + extractionService.submit(createdFile.getId(), content.getId()); + return createResponse; + } catch (Exception e) { + if (contentPersisted) { + try { + repository.getContentRepository().delete(user, content.getId(), false, true); + } catch (Exception ignored) { + // Best-effort cleanup. + } + } + if (createdFile != null) { + cleanupFailedUpload(user, createdFile.getId()); + } + if (assetPersisted) { + try { + repository.getAssetRepository().delete(asset.getId()); + } catch (Exception ignored) { + // Best-effort cleanup. + } + } + if (assetUploaded) { + try { + assetService.delete(asset).join(); + } catch (Exception ignored) { + // Best-effort cleanup. + } + } + throw e; + } + } catch (ContextFileUploadSupport.MaxFileSizeExceededException e) { + return Response.status(Response.Status.REQUEST_ENTITY_TOO_LARGE) + .entity( + String.format( + "{\"message\":\"File size %d exceeds configured limit %d bytes\"}", + e.getActualSize(), e.getMaxFileSize())) + .type(MediaType.APPLICATION_JSON) + .build(); + } + } + + @GET + @Path("/{id}/download") + @Operation(operationId = "downloadDriveFile", summary = "Download a file by ID") + public Response downloadFile( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @PathParam("id") UUID id, + @QueryParam("include") @DefaultValue("non-deleted") Include include, + @QueryParam("redirect") @DefaultValue("true") boolean redirect, + @QueryParam("expiry") @DefaultValue("300") int expirySeconds) { + ContextFile file = getInternal(uriInfo, securityContext, id, "", include); + Asset asset = resolveAsset(file); + if (asset == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity("{\"message\":\"No current content found for this file\"}") + .type(MediaType.APPLICATION_JSON) + .build(); + } + + AssetService assetService = AssetServiceFactory.getService(); + if (assetService == null) { + return Response.status(Response.Status.SERVICE_UNAVAILABLE) + .entity("{\"message\":\"Object storage is not configured\"}") + .type(MediaType.APPLICATION_JSON) + .build(); + } + + try { + if (redirect && supportsRedirectDownload(assetService)) { + String signedUrl = + assetService.generateDownloadUrlWithExpiry( + asset, Duration.ofSeconds(clampExpiry(expirySeconds))); + if (signedUrl != null && !signedUrl.isEmpty()) { + return Response.temporaryRedirect(URI.create(signedUrl)).build(); + } + } + + InputStream fileStream = assetService.read(asset).join(); + if (fileStream == null) { + return Response.status(Response.Status.NOT_FOUND) + .entity("{\"message\":\"No current content found for this file\"}") + .type(MediaType.APPLICATION_JSON) + .build(); + } + + StreamingOutput output = + stream -> { + try (InputStream input = fileStream) { + input.transferTo(stream); + } catch (IOException e) { + throw new WebApplicationException("Failed to stream file content", e); + } + }; + + return Response.ok(output, asset.getContentType()) + .header("Content-Disposition", buildContentDisposition(asset.getFileName())) + .header("Content-Length", asset.getSize().longValue()) + .build(); + } catch (Exception e) { + return Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity("{\"message\":\"Failed to download file content\"}") + .type(MediaType.APPLICATION_JSON) + .build(); + } + } + + @DELETE + @Path("/{id}") + @Operation(operationId = "deleteDriveFile", summary = "Delete a file") + public Response delete( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @PathParam("id") UUID id, + @Parameter(description = "Permanently delete the file asynchronously.") + @QueryParam("hardDelete") + @DefaultValue("false") + boolean hardDelete) { + if (hardDelete) { + ContextFile file = getInternal(uriInfo, securityContext, id, "", Include.ALL); + if (!Boolean.TRUE.equals(file.getDeleted())) { + super.delete(uriInfo, securityContext, id, false, false); + } + return deleteByIdAsync(uriInfo, securityContext, id, false, true); + } + return super.delete(uriInfo, securityContext, id, false, false); + } + + @PUT + @Path("/restore") + @Operation( + operationId = "restoreDriveFile", + summary = "Restore a soft deleted drive file", + description = "Restore a drive file from the trash.") + public Response restore( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Valid RestoreEntity restore) { + return restoreEntity(uriInfo, securityContext, restore.getId()); + } + + private Asset resolveAsset(ContextFile file) { + if (file.getHeadContentId() != null && !file.getHeadContentId().isEmpty()) { + ContextFileContent content = repository.getContentById(file.getHeadContentId()); + if (content != null && content.getAssetId() != null && !content.getAssetId().isEmpty()) { + return repository.getAssetRepository().getById(content.getAssetId()); + } + } + if (file.getAssetId() != null && !file.getAssetId().isEmpty()) { + return repository.getAssetRepository().getById(file.getAssetId()); + } + return null; + } + + private void cleanupFailedUpload(String user, UUID fileId) { + try { + repository.delete(user, fileId, false, true); + } catch (Exception ignored) { + // Best-effort cleanup after a partially completed upload. + } + } + + private boolean supportsRedirectDownload(AssetService assetService) { + // The configured service is wrapped by QueuedDeleteAssetService, so unwrap to inspect the + // real provider when deciding whether to issue a signed-URL redirect. + AssetService unwrapped = AssetServiceFactory.unwrap(assetService); + return unwrapped instanceof S3AssetService || unwrapped instanceof AzureAssetService; + } + + static final int MAX_EXPIRY_SECONDS = 3600; + + /** Delegate to {@link ContextFileUploadSupport#sanitizeFileName(String)}. */ + static String sanitizeFileName(String fileName) { + return ContextFileUploadSupport.sanitizeFileName(fileName); + } + + /** Delegate to {@link ContextFileUploadSupport#buildContentDisposition(String)}. */ + static String buildContentDisposition(String fileName) { + return ContextFileUploadSupport.buildContentDisposition(fileName); + } + + /** Clamp expiry to [1, MAX_EXPIRY_SECONDS]. */ + static int clampExpiry(int expirySeconds) { + return Math.max(1, Math.min(expirySeconds, MAX_EXPIRY_SECONDS)); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/drive/ContextFileUploadSupport.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/drive/ContextFileUploadSupport.java new file mode 100644 index 000000000000..bbce62ce8c0f --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/drive/ContextFileUploadSupport.java @@ -0,0 +1,241 @@ +package org.openmetadata.service.resources.drive; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; +import java.util.UUID; +import org.openmetadata.schema.attachments.Asset; +import org.openmetadata.schema.attachments.AssetType; +import org.openmetadata.schema.entity.data.ContextFile; +import org.openmetadata.schema.entity.data.ContextFileContent; +import org.openmetadata.schema.entity.data.ContextFileType; +import org.openmetadata.schema.entity.data.ProcessingStatus; +import org.openmetadata.service.resources.feeds.MessageParser; + +/** + * Shared helpers for attachment/asset upload flows. Promoted to {@code public} so the + * attachments resource (a different package) can reuse the streaming upload buffer and + * the Content-Disposition sanitization without duplicating them. + */ +public final class ContextFileUploadSupport { + private static final String CONTEXT_FILE_ENTITY = "contextFile"; + + public static final class MaxFileSizeExceededException extends IOException { + private final long actualSize; + private final long maxFileSize; + + MaxFileSizeExceededException(long actualSize, long maxFileSize) { + super( + String.format("File size %d exceeds configured limit %d bytes", actualSize, maxFileSize)); + this.actualSize = actualSize; + this.maxFileSize = maxFileSize; + } + + public long getActualSize() { + return actualSize; + } + + public long getMaxFileSize() { + return maxFileSize; + } + } + + public static final class BufferedUpload implements AutoCloseable { + private final Path path; + private final long size; + private final String checksum; + + BufferedUpload(Path path, long size, String checksum) { + this.path = path; + this.size = size; + this.checksum = checksum; + } + + public long getSize() { + return size; + } + + public String getChecksum() { + return checksum; + } + + public InputStream newInputStream() throws IOException { + return Files.newInputStream(path); + } + + @Override + public void close() throws IOException { + Files.deleteIfExists(path); + } + } + + private ContextFileUploadSupport() {} + + static boolean exceedsMaxFileSize(long fileSize, long maxFileSize) { + return maxFileSize > 0 && fileSize > maxFileSize; + } + + static String sanitizeEntityName(String originalFileName) { + // Multipart uploads can arrive with missing or blank filename metadata. Fall back + // to a stable base so the upload does not fail with NullPointerException. + String source = + (originalFileName == null || originalFileName.isBlank()) ? "file" : originalFileName; + String sanitized = + source.replaceAll("[^a-zA-Z0-9._-]", "_").replaceAll("_+", "_").toLowerCase(); + if (sanitized.isEmpty()) { + sanitized = "file"; + } + if (sanitized.length() > 180) { + sanitized = sanitized.substring(0, 180); + } + return sanitized + "_" + UUID.randomUUID().toString().substring(0, 8); + } + + static ContextFileType detectFileType(String contentType) { + if (contentType == null) { + return ContextFileType.Other; + } + String ct = contentType.toLowerCase(); + if (ct.equals("application/pdf")) { + return ContextFileType.PDF; + } + if (ct.contains("spreadsheet") || ct.contains("excel")) { + return ContextFileType.Spreadsheet; + } + if (ct.contains("presentation") || ct.contains("powerpoint")) { + return ContextFileType.Presentation; + } + if (ct.startsWith("image/")) { + return ContextFileType.Image; + } + if (ct.equals("text/csv") || ct.equals("application/csv")) { + return ContextFileType.CSV; + } + if (ct.contains("document") || ct.contains("word")) { + return ContextFileType.Document; + } + if (ct.startsWith("text/")) { + return ContextFileType.Text; + } + return ContextFileType.Other; + } + + static String buildEntityLink(ContextFile file) { + return "<#E::" + CONTEXT_FILE_ENTITY + "::" + file.getFullyQualifiedName() + ">"; + } + + /** + * Safe-for-{@code Content-Disposition} rendering of {@code fileName}. Strips the + * characters that would let a hostile filename break out of the header + * ({@code "}, {@code \}, CR, LF) and falls back to {@code "download"} if the + * sanitized form is empty. Shared with the attachments resource so both upload/download + * paths apply the same protection. + */ + public static String sanitizeFileName(String fileName) { + if (fileName == null) { + return "download"; + } + String sanitized = fileName.replaceAll("[\"\\\\\\r\\n]", "_").trim(); + return sanitized.isEmpty() ? "download" : sanitized; + } + + /** + * Build a {@code Content-Disposition} header value that is safe for non-ASCII + * filenames. Emits both the legacy quoted {@code filename=} parameter (for older + * clients) and the RFC 5987 {@code filename*=UTF-8''...} parameter with + * percent-encoded bytes — so international filenames round-trip while remaining + * header-injection safe. + */ + public static String buildContentDisposition(String fileName) { + String safeAscii = sanitizeFileName(fileName); + String encoded = URLEncoder.encode(safeAscii, StandardCharsets.UTF_8).replace("+", "%20"); + return "attachment; filename=\"" + safeAscii + "\"; filename*=UTF-8''" + encoded; + } + + public static BufferedUpload bufferUpload(InputStream inputStream, long maxFileSize) + throws IOException { + Path tempFile = Files.createTempFile("context-file-upload-", ".bin"); + MessageDigest digest = sha256Digest(); + long totalBytes = 0L; + byte[] buffer = new byte[8192]; + + try (OutputStream outputStream = Files.newOutputStream(tempFile)) { + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + digest.update(buffer, 0, bytesRead); + totalBytes += bytesRead; + if (exceedsMaxFileSize(totalBytes, maxFileSize)) { + throw new MaxFileSizeExceededException(totalBytes, maxFileSize); + } + } + return new BufferedUpload(tempFile, totalBytes, HexFormat.of().formatHex(digest.digest())); + } catch (IOException | RuntimeException e) { + Files.deleteIfExists(tempFile); + throw e; + } + } + + static Asset buildAsset( + ContextFile file, + String originalFileName, + String contentType, + String fileExtension, + long fileSize, + String updatedBy) { + Asset asset = new Asset(); + String entityLink = buildEntityLink(file); + MessageParser.EntityLink assetLink = MessageParser.EntityLink.parse(entityLink); + asset.setId(UUID.randomUUID().toString()); + asset.setFileName(originalFileName); + asset.setContentType(contentType); + asset.setSize(Math.toIntExact(fileSize)); + asset.setEntityLink(entityLink); + asset.setFullyQualifiedName(assetLink.getEntityFQN()); + asset.setUrl(""); + asset.setAssetType(AssetType.External); + asset.setExtension(fileExtension); + asset.setUpdatedBy(updatedBy); + asset.setUpdatedAt(System.currentTimeMillis()); + asset.setDeleted(false); + return asset; + } + + static ContextFileContent buildContent( + ContextFile file, Asset asset, String checksum, String updatedBy) { + String suffix = UUID.randomUUID().toString().substring(0, 8); + return new ContextFileContent() + .withId(UUID.randomUUID()) + .withName(file.getName() + "_content_" + suffix) + .withContextFile(file.getEntityReference()) + .withAssetId(asset.getId()) + .withContentType(asset.getContentType()) + .withSize(asset.getSize()) + .withChecksum(checksum) + .withIngestedAt(System.currentTimeMillis()) + .withIsCurrent(true) + .withProcessingStatus(ProcessingStatus.Uploaded) + .withUpdatedBy(updatedBy) + .withUpdatedAt(System.currentTimeMillis()) + .withDeleted(false); + } + + static String sha256(byte[] content) { + return HexFormat.of().formatHex(sha256Digest().digest(content)); + } + + private static MessageDigest sha256Digest() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is required for ContextFile content checksums", e); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/drive/FolderMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/drive/FolderMapper.java new file mode 100644 index 000000000000..f16dc27358f4 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/drive/FolderMapper.java @@ -0,0 +1,19 @@ +package org.openmetadata.service.resources.drive; + +import static org.openmetadata.service.util.EntityUtil.getEntityReference; + +import org.openmetadata.schema.api.data.CreateFolder; +import org.openmetadata.schema.entity.data.Folder; +import org.openmetadata.service.jdbi3.FolderRepository; +import org.openmetadata.service.mapper.EntityMapper; + +public class FolderMapper implements EntityMapper { + @Override + public Folder createToEntity(CreateFolder create, String user) { + return copy(new Folder(), create, user) + .withTags(create.getTags()) + .withIcon(create.getIcon()) + .withColor(create.getColor()) + .withParent(getEntityReference(FolderRepository.FOLDER_ENTITY, create.getParent())); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/drive/FolderResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/drive/FolderResource.java new file mode 100644 index 000000000000..74de4d4137a2 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/drive/FolderResource.java @@ -0,0 +1,222 @@ +package org.openmetadata.service.resources.drive; + +import static org.openmetadata.service.jdbi3.FolderRepository.FOLDER_ENTITY; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.validation.Valid; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.DefaultValue; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.PATCH; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.SecurityContext; +import jakarta.ws.rs.core.UriInfo; +import java.util.List; +import java.util.UUID; +import org.openmetadata.schema.api.data.CreateFolder; +import org.openmetadata.schema.api.data.RestoreEntity; +import org.openmetadata.schema.entity.data.ContextFile; +import org.openmetadata.schema.entity.data.Folder; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.MetadataOperation; +import org.openmetadata.schema.utils.ResultList; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.FolderRepository; +import org.openmetadata.service.jdbi3.ListFilter; +import org.openmetadata.service.limits.Limits; +import org.openmetadata.service.resources.Collection; +import org.openmetadata.service.resources.EntityResource; +import org.openmetadata.service.security.Authorizer; + +@Tag(name = "Drive Folders", description = "APIs for managing folders in the Context Center Drive.") +@Path("/v1/drive/folders") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +@Collection(name = "driveFolders") +public class FolderResource extends EntityResource { + public static final String COLLECTION_PATH = "v1/drive/folders/"; + public static final String FIELDS = "owners,tags,parent,children,domains,followers"; + private final FolderMapper mapper = new FolderMapper(); + + public static class FolderContents { + public Folder folder; + public List folders; + public List files; + public int childrenFolderCount; + public int childrenFileCount; + public int itemCount; + } + + public FolderResource(Authorizer authorizer, Limits limits) { + super(FOLDER_ENTITY, authorizer, limits); + } + + public static class FolderList extends ResultList {} + + @Override + protected List getEntitySpecificOperations() { + addViewOperation("parent,children", MetadataOperation.VIEW_BASIC); + return List.of(); + } + + @Override + public Folder addHref(UriInfo uriInfo, Folder folder) { + super.addHref(uriInfo, folder); + Entity.withHref(uriInfo, folder.getParent()); + Entity.withHref(uriInfo, folder.getChildren()); + return folder; + } + + @GET + @Operation( + operationId = "listDriveFolders", + summary = "List folders", + responses = { + @ApiResponse( + responseCode = "200", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = FolderList.class))) + }) + public ResultList list( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @QueryParam("fields") String fieldsParam, + @QueryParam("limit") @DefaultValue("10") int limit, + @QueryParam("before") String before, + @QueryParam("after") String after, + @QueryParam("include") @DefaultValue("non-deleted") Include include) { + return super.listInternal( + uriInfo, securityContext, fieldsParam, new ListFilter(include), limit, before, after); + } + + @GET + @Path("/{id}") + @Operation(operationId = "getDriveFolder", summary = "Get a folder by ID") + public Folder get( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @PathParam("id") UUID id, + @QueryParam("fields") String fieldsParam, + @QueryParam("include") @DefaultValue("non-deleted") Include include) { + return getInternal(uriInfo, securityContext, id, fieldsParam, include); + } + + @GET + @Path("/name/{fqn}") + @Operation(operationId = "getDriveFolderByFqn", summary = "Get a folder by FQN") + public Folder getByName( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @PathParam("fqn") String fqn, + @QueryParam("fields") String fieldsParam, + @QueryParam("include") @DefaultValue("non-deleted") Include include) { + return getByNameInternal(uriInfo, securityContext, fqn, fieldsParam, include); + } + + @GET + @Path("/{id}/contents") + @Operation( + operationId = "getDriveFolderContents", + summary = "Get the direct contents of a folder") + public FolderContents getContents( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @PathParam("id") UUID id, + @QueryParam("include") @DefaultValue("non-deleted") Include include) { + Folder folder = getInternal(uriInfo, securityContext, id, "parent,children", include); + List folders = repository.getChildFolderEntities(folder); + List files = repository.getChildFileEntities(folder); + + FolderContents response = new FolderContents(); + response.folder = folder; + response.folders = folders; + response.files = files; + response.childrenFolderCount = folders.size(); + response.childrenFileCount = files.size(); + response.itemCount = folders.size() + files.size(); + return response; + } + + @POST + @Operation(operationId = "createDriveFolder", summary = "Create a folder") + public Response create( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Valid CreateFolder create) { + Folder folder = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); + return create(uriInfo, securityContext, folder); + } + + @PUT + @Operation(operationId = "createOrUpdateDriveFolder", summary = "Create or update a folder") + public Response createOrUpdate( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Valid CreateFolder create) { + Folder folder = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); + return createOrUpdate(uriInfo, securityContext, folder); + } + + @PATCH + @Path("/{id}") + @Consumes(MediaType.APPLICATION_JSON_PATCH_JSON) + @Operation(operationId = "patchDriveFolder", summary = "Update a folder via JSON Patch") + public Response patch( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @PathParam("id") UUID id, + @Valid jakarta.json.JsonPatch patch) { + return patchInternal(uriInfo, securityContext, id, patch); + } + + @DELETE + @Path("/{id}") + @Operation(operationId = "deleteDriveFolder", summary = "Delete a folder") + public Response delete( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @PathParam("id") UUID id, + @QueryParam("recursive") @DefaultValue("false") boolean recursive, + @Parameter(description = "Permanently delete the folder asynchronously.") + @QueryParam("hardDelete") + @DefaultValue("false") + boolean hardDelete) { + if (hardDelete) { + Folder folder = getInternal(uriInfo, securityContext, id, "", Include.ALL); + if (!Boolean.TRUE.equals(folder.getDeleted())) { + super.delete(uriInfo, securityContext, id, recursive, false); + } + return deleteByIdAsync(uriInfo, securityContext, id, recursive, true); + } + return super.delete(uriInfo, securityContext, id, recursive, false); + } + + @PUT + @Path("/restore") + @Operation( + operationId = "restoreDriveFolder", + summary = "Restore a soft deleted drive folder", + description = "Restore a folder from the trash.") + public Response restore( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Valid RestoreEntity restore) { + return restoreEntity(uriInfo, securityContext, restore.getId()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/knowledge/KnowledgePageMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/knowledge/KnowledgePageMapper.java new file mode 100644 index 000000000000..1507c7d23c57 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/knowledge/KnowledgePageMapper.java @@ -0,0 +1,36 @@ +package org.openmetadata.service.resources.knowledge; + +import static org.openmetadata.service.Entity.ORGANIZATION_NAME; +import static org.openmetadata.service.Entity.TEAM; + +import java.util.ArrayList; +import java.util.List; +import org.openmetadata.schema.api.data.CreatePage; +import org.openmetadata.schema.entity.data.Page; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.Votes; +import org.openmetadata.service.Entity; +import org.openmetadata.service.mapper.EntityMapper; + +public class KnowledgePageMapper implements EntityMapper { + @Override + public Page createToEntity(CreatePage create, String user) { + // Resolve the effective related-entities list locally without mutating the inbound + // CreatePage. Mutating the request object (previously via create.withRelatedEntities) + // leaked the Organization fallback into the caller's request copy, which is surprising + // if the request is re-used or logged. + List relatedEntities = create.getRelatedEntities(); + if (relatedEntities == null || relatedEntities.isEmpty()) { + relatedEntities = new ArrayList<>(); + relatedEntities.add(Entity.getEntityReferenceByName(TEAM, ORGANIZATION_NAME, Include.ALL)); + } + return copy(new Page(), create, user) + .withTags(create.getTags()) + .withVotes(new Votes().withUpVotes(0).withDownVotes(0)) + .withPageType(create.getPageType()) + .withPage(create.getPage()) + .withParent(create.getParent()) + .withRelatedEntities(relatedEntities); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/knowledge/KnowledgePageResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/knowledge/KnowledgePageResource.java new file mode 100644 index 000000000000..6ef1d106b547 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/knowledge/KnowledgePageResource.java @@ -0,0 +1,768 @@ +package org.openmetadata.service.resources.knowledge; + +import static org.openmetadata.service.Entity.BOT; +import static org.openmetadata.service.Entity.TEST_CASE; +import static org.openmetadata.service.Entity.TEST_SUITE; +import static org.openmetadata.service.Entity.USER; +import static org.openmetadata.service.jdbi3.KnowledgePageRepository.KNOWLEDGE_PAGE_ENTITY; + +import io.swagger.v3.oas.annotations.ExternalDocumentation; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.ExampleObject; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.json.JsonPatch; +import jakarta.validation.Valid; +import jakarta.validation.constraints.Max; +import jakarta.validation.constraints.Min; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.DefaultValue; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.PATCH; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.SecurityContext; +import jakarta.ws.rs.core.UriInfo; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.common.utils.CommonUtil; +import org.openmetadata.schema.api.VoteRequest; +import org.openmetadata.schema.api.data.CreatePage; +import org.openmetadata.schema.api.data.RestoreEntity; +import org.openmetadata.schema.entity.data.Page; +import org.openmetadata.schema.entity.data.PageHierarchy; +import org.openmetadata.schema.entity.data.PageType; +import org.openmetadata.schema.entity.teams.User; +import org.openmetadata.schema.type.ChangeEvent; +import org.openmetadata.schema.type.EntityHistory; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.MetadataOperation; +import org.openmetadata.schema.utils.ResultList; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.DaoListFilter; +import org.openmetadata.service.jdbi3.KnowledgePageRepository; +import org.openmetadata.service.jdbi3.ListFilter; +import org.openmetadata.service.limits.Limits; +import org.openmetadata.service.resources.Collection; +import org.openmetadata.service.resources.EntityResource; +import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.security.policyevaluator.OperationContext; + +@Slf4j +@Tag(name = "Knowledge", description = "APIs related knowledge pages of data assets.") +@Path("/v1/knowledgeCenter") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +@Collection(name = "KnowledgeCenter") +public class KnowledgePageResource extends EntityResource { + public static final String INVALID_ENTITY_MSG = + "Given Entity Type : %s does not support Knowledge Pages."; + public static final Set EXCLUDED_ENTITIES = Set.of(USER, BOT, TEST_SUITE, TEST_CASE); + public static final String COLLECTION_PATH = "v1/knowledgeCenter"; + public static final String FIELDS = + "owners,tags,followers,votes,page,parent,childrenCount,relatedEntities,relatedArticles,attachments,domains,dataProducts"; + private final KnowledgePageMapper mapper = new KnowledgePageMapper(); + + public KnowledgePageResource(Authorizer authorizer, Limits limits) { + super(KNOWLEDGE_PAGE_ENTITY, authorizer, limits); + } + + public static class PageList extends ResultList { + /* Required for serde */ + } + + @Override + protected List getEntitySpecificOperations() { + this.allowedFields.add("relatedArticles"); + addViewOperation( + "pageType,page,parent,children,relatedEntities,relatedArticles", + MetadataOperation.VIEW_BASIC); + return null; + } + + @Override + public Page addHref(UriInfo uriInfo, Page entity) { + super.addHref(uriInfo, entity); + Entity.withHref(uriInfo, entity.getRelatedEntities()); + return entity; + } + + @GET + @Operation( + operationId = "listKnowledgePages", + summary = "Get a list of Knowledge Pages", + description = + "Get a list of Knowledge Pages. Use `fields` " + + "parameter to get only necessary fields. Use cursor-based pagination to limit the number " + + "entries in the list using `limit` and `before` or `after` query params.", + responses = { + @ApiResponse( + responseCode = "200", + description = "Get List of Knowledge Pages", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = PageList.class))) + }) + public ResultList listKnowledgePage( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter( + description = "Type of the entity for which to list the Knowledge Pages", + schema = @Schema(type = "string")) + @QueryParam("entityType") + String entityType, + @Parameter(description = "Knowledge Page Type", schema = @Schema(type = "string")) + @QueryParam("pageType") + PageType knowledgePageType, + @Parameter( + description = "UUID of the entity for which to list the Knowledge Pages", + schema = @Schema(type = "UUID")) + @QueryParam("entityId") + UUID entityId, + @Parameter( + description = + "Limit the number Knowledge Pages returned. " + "(1 to 1000000, default = 10)") + @DefaultValue("10") + @Min(value = 0, message = "must be greater than or equal to 0") + @Max(value = 1000000, message = "must be less than or equal to 1000000") + @QueryParam("limit") + int limitParam, + @Parameter( + description = "UUID of the entity for which to list the Knowledge Pages", + schema = @Schema(type = "UUID")) + @QueryParam("tagFQN") + String tagFQN, + @Parameter( + description = "Returns list of Knowledge Pages before this cursor", + schema = @Schema(type = "string")) + @QueryParam("before") + String before, + @Parameter( + description = "Returns list of Knowledge Pages after this cursor", + schema = @Schema(type = "string")) + @QueryParam("after") + String after, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include) { + ListFilter filter = new ListFilter(include); + if ((!CommonUtil.nullOrEmpty(entityId) && CommonUtil.nullOrEmpty(entityType)) + || (CommonUtil.nullOrEmpty(entityId) && !CommonUtil.nullOrEmpty(entityType))) { + throw new IllegalArgumentException( + "Query Param Entity Id and Entity Type both needs to be provided."); + } else if (!CommonUtil.nullOrEmpty(entityId) && !CommonUtil.nullOrEmpty(entityType)) { + filter.addQueryParam("entityType", entityType); + List fromIds = new ArrayList<>(); + // Add the User + fromIds.add(entityId.toString()); + // Add team and domain if exists + if (entityType.equals(USER)) { + User user = Entity.getEntity(USER, entityId, "domains,teams", include); + // Add Teams + if (user.getTeams() != null) { + user.getTeams().forEach(team -> fromIds.add(team.getId().toString())); + } + // Add Domains + if (user.getDomains() != null) { + user.getDomains().forEach(domain -> fromIds.add(domain.getId().toString())); + } + } + filter.addQueryParam("entityId", getUsersFromIdList(fromIds)); + } + if (knowledgePageType != null) { + filter.addQueryParam("pageType", knowledgePageType.value()); + } + + if (!CommonUtil.nullOrEmpty(tagFQN)) { + filter.addQueryParam("tagFQN", tagFQN); + } + return super.listInternal( + uriInfo, securityContext, fieldsParam, filter, limitParam, before, after); + } + + @GET + @Path("/hierarchy") + @Valid + @Operation( + operationId = "listPageHierarchy", + summary = "List Page with hierarchy", + description = "Get a list of pages with hierarchy.", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of pages with hierarchy", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = KnowledgePageResource.PageList.class))) + }) + public ResultList listHierarchy( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Knowledge Page Type", schema = @Schema(type = "string")) + @QueryParam("pageType") + PageType knowledgePageType, + @Parameter(description = "Limit the number of pages returned. (1 to 1000000, default = 10)") + @DefaultValue("10000") + @Min(value = 0, message = "must be greater than or equal to 0") + @Max(value = 1000000, message = "must be less than or equal to 1000000") + @QueryParam("limit") + int limitParam) { + DaoListFilter filter = new DaoListFilter(Include.NON_DELETED); + if (knowledgePageType != null) { + filter.addQueryParam("pageType", knowledgePageType.value()); + } + return new ResultList<>(repository.listHierarchy(filter, limitParam)); + } + + @GET + @Path("/search/hierarchy") + @Valid + @Operation( + operationId = "listPageHierarchySearch", + summary = "List Page with hierarchy from Search", + description = "Get a list of pages with hierarchy from Search.", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of pages with hierarchy from Search", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = KnowledgePageResource.PageList.class))) + }) + public ResultList listHierarchyWithSearch( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Knowledge Page Type", schema = @Schema(type = "string")) + @QueryParam("pageType") + PageType knowledgePageType, + @Parameter(description = "Offset for pagination") @QueryParam("offset") @DefaultValue("0") + int offset, + @Parameter(description = "Limit the number of pages returned. (1 to 1000000, default = 10)") + @DefaultValue("10") + @QueryParam("limit") + int limit, + @Parameter(description = "Parent Fully Qualified Name") @QueryParam("parent") String parent, + @Parameter( + description = + "FQN of the active page to show the active page correctly in the hierarchy , while showing other root nodes at level 1.") + @QueryParam("activeFqn") + String activeFqn) { + if (!CommonUtil.nullOrEmpty(activeFqn)) { + return repository.getHierarchyWithSearchForActivePage( + activeFqn, knowledgePageType, offset, limit); + } else { + return repository.getHierarchyWithSearch(parent, knowledgePageType, offset, limit); + } + } + + @GET + @Path("/{id}") + @Operation( + operationId = "getKnowledgePageById", + summary = "Get a Knowledge Page", + description = "Get a KnowledgePage by `id`", + responses = { + @ApiResponse( + responseCode = "200", + description = "KnowledgePage", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = Page.class))), + @ApiResponse( + responseCode = "404", + description = "KnowledgePage for instance {id} is not found") + }) + public Page getKnowledgePageById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "KnowledgePage Id", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include, + @Parameter( + description = + "Per-relation include control. Format: field:value,field2:value2. " + + "Example: owners:non-deleted,followers:all. " + + "Valid values: all, deleted, non-deleted. " + + "If not specified for a field, uses the entity's include value.", + schema = @Schema(type = "string", example = "owners:non-deleted,followers:all")) + @QueryParam("includeRelations") + String includeRelations) { + return getInternal(uriInfo, securityContext, id, fieldsParam, include, includeRelations); + } + + @GET + @Path("/name/{fqn}") + @Operation( + operationId = "getKnowledgePageFqn", + summary = "Get a KnowledgePage by name", + description = "Get a KnowledgePage by fully qualified table name.", + responses = { + @ApiResponse( + responseCode = "200", + description = "KnowledgePage", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = Page.class))), + @ApiResponse( + responseCode = "404", + description = "KnowledgePage for instance {id} is not found") + }) + public Page getKnowledgePageByName( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter( + description = "Fully qualified name of the KnowledgePage", + schema = @Schema(type = "string")) + @PathParam("fqn") + String fqn, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @Parameter( + description = "Include all, deleted, or non-deleted entities.", + schema = @Schema(implementation = Include.class)) + @QueryParam("include") + @DefaultValue("non-deleted") + Include include, + @Parameter( + description = + "Per-relation include control. Format: field:value,field2:value2. " + + "Example: owners:non-deleted,followers:all. " + + "Valid values: all, deleted, non-deleted. " + + "If not specified for a field, uses the entity's include value.", + schema = @Schema(type = "string", example = "owners:non-deleted,followers:all")) + @QueryParam("includeRelations") + String includeRelations) { + return getByNameInternal(uriInfo, securityContext, fqn, fieldsParam, include, includeRelations); + } + + @GET + @Path("/{id}/versions") + @Operation( + operationId = "listAllKnowledgePageVersion", + summary = "Get List of all KnowledgePage versions", + description = "Get a list of all the versions of a KnowledgePage identified by `id`", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of KnowledgePage versions", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = EntityHistory.class))) + }) + public EntityHistory listVersions( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "KnowledgePage Id", schema = @Schema(type = "string")) + @PathParam("id") + UUID id) { + return super.listVersionsInternal(securityContext, id); + } + + @GET + @Path("/{id}/versions/{version}") + @Operation( + operationId = "getSpecificKnowledgePageVersion", + summary = "Get a specific version of the KnowledgePage", + description = "Get a version of the KnowledgePage by given `id`", + responses = { + @ApiResponse( + responseCode = "200", + description = "KnowledgePage", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = Page.class))), + @ApiResponse( + responseCode = "404", + description = "KnowledgePage for instance {id} and version {version} is " + "not found") + }) + public Page getVersion( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "KnowledgePage Id", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id, + @Parameter( + description = "KnowledgePage version number in the form `major`.`minor`", + schema = @Schema(type = "string", example = "0.1 or 1.1")) + @PathParam("version") + String version) { + return super.getVersionInternal(securityContext, id, version); + } + + @POST + @Operation( + operationId = "createKnowledgePage", + summary = "Create a Knowledge Page", + description = "Create a Knowledge Page.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The Knowledge Page", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = Page.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response create( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Valid CreatePage create) { + Page page = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); + return create(uriInfo, securityContext, page); + } + + @PUT + @Operation( + operationId = "createOrUpdateKnowledgePage", + summary = "Create or update a Knowledge Page", + description = + "Create a Knowledge Page, if it does not exist. If a knowledge page already exists, update the Knowledge Page.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The Knowledge Page", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = Page.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response createOrUpdate( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Valid CreatePage create) { + Page page = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); + return createOrUpdate(uriInfo, securityContext, page); + } + + @PATCH + @Path("/{id}") + @Operation( + operationId = "patchKnowledgePage", + summary = "Update a Knowledge Page", + description = "Update an existing Knowledge Page using JsonPatch.", + externalDocs = + @ExternalDocumentation( + description = "JsonPatch RFC", + url = "https://tools.ietf.org/html/rfc6902")) + @Consumes(MediaType.APPLICATION_JSON_PATCH_JSON) + public Response patch( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the Knowledge Page", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id, + @RequestBody( + description = "JsonPatch with array of operations", + content = + @Content( + mediaType = MediaType.APPLICATION_JSON_PATCH_JSON, + examples = { + @ExampleObject( + "[" + "{op:remove, path:/a}," + "{op:add, path: /b, value: val}" + "]") + })) + JsonPatch patch) { + return patchInternal(uriInfo, securityContext, id, patch); + } + + @PUT + @Path("/{id}/followers") + @Operation( + operationId = "addFollower", + summary = "Add a follower", + description = "Add a user identified by `userId` as follower of this model", + responses = { + @ApiResponse( + responseCode = "200", + description = "OK", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = ChangeEvent.class))), + @ApiResponse(responseCode = "404", description = "model for instance {id} is not found") + }) + public Response addFollower( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the Knowledge Page", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id, + @Parameter( + description = "Id of the user to be added as follower", + schema = @Schema(type = "UUID")) + UUID userId) { + return repository + .addFollower(securityContext.getUserPrincipal().getName(), id, userId) + .toResponse(); + } + + @PUT + @Path("/{id}/vote") + @Operation( + operationId = "updateVote", + summary = "Update Vote for a this entity", + description = "Update vote for a entity", + responses = { + @ApiResponse( + responseCode = "200", + description = "OK", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = ChangeEvent.class))), + @ApiResponse(responseCode = "404", description = "model for instance {id} is not found") + }) + public Response updateVote( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the Query", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id, + @Valid VoteRequest request) { + return repository + .updateVote(securityContext.getUserPrincipal().getName(), id, request) + .toResponse(); + } + + @DELETE + @Path("/{id}/followers/{userId}") + @Operation( + operationId = "deleteFollower", + summary = "Remove a follower", + description = "Remove the user identified `userId` as a follower of the model.", + responses = { + @ApiResponse( + responseCode = "200", + description = "OK", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = ChangeEvent.class))), + }) + public Response deleteFollower( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the Knowledge Page", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id, + @Parameter( + description = "Id of the user being removed as follower", + schema = @Schema(type = "UUID")) + @PathParam("userId") + UUID userId) { + return repository + .deleteFollower(securityContext.getUserPrincipal().getName(), id, userId) + .toResponse(); + } + + @PUT + @Path("/{id}/usage") + @Operation( + operationId = "addKnowledgePageUsage", + summary = "Add Knowledge Page usage", + description = "Add Knowledge Page usage", + responses = { + @ApiResponse( + responseCode = "200", + description = "OK", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = Page.class))) + }) + public Response addKnowledgePageUsage( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the Knowledge Page", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id, + @Valid List entityIds) { + OperationContext operationContext = + new OperationContext(entityType, MetadataOperation.EDIT_ALL); + authorizer.authorize(securityContext, operationContext, getResourceContextById(id)); + return repository + .addKnowledgePageUsage(uriInfo, securityContext.getUserPrincipal().getName(), id, entityIds) + .toResponse(); + } + + @DELETE + @Path("/{id}/usage") + @Operation( + operationId = "removeKnowledgePageUsage", + summary = "remove Knowledge Page usage", + description = "remove Knowledge Page Usage", + responses = { + @ApiResponse( + responseCode = "200", + description = "OK", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = Page.class))) + }) + public Response removeKnowledgePageUsage( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the knowledge page", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id, + @Valid List entityIds) { + OperationContext operationContext = + new OperationContext(entityType, MetadataOperation.EDIT_ALL); + authorizer.authorize(securityContext, operationContext, getResourceContextById(id)); + return repository + .removeKnowledgePageUsedIn( + uriInfo, securityContext.getUserPrincipal().getName(), id, entityIds) + .toResponse(); + } + + @PUT + @Path("/restore") + @Operation( + operationId = "restore", + summary = "Restore a soft deleted Knowledge Page", + description = "Restore a soft deleted Knowledge Page.", + responses = { + @ApiResponse( + responseCode = "200", + description = "Successfully restored the Knowledge Page ", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = Page.class))) + }) + public Response restoreQuery( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Valid RestoreEntity restore) { + return restoreEntity(uriInfo, securityContext, restore.getId()); + } + + @DELETE + @Path("/{id}") + @Operation( + operationId = "deleteKnowledgePage", + summary = "Delete a Knowledge Page", + description = "Delete a knowledge page by `id`.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse( + responseCode = "404", + description = "Knowledge Page for instance {id} is not found") + }) + public Response delete( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the Knowledge Page", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id, + @Parameter( + description = "Recursively delete this entity and it's children. (Default `false`)") + @QueryParam("recursive") + @DefaultValue("false") + boolean recursive) { + return delete(uriInfo, securityContext, id, recursive, true); + } + + @DELETE + @Path("/async/{id}") + @Operation( + operationId = "deleteKnowledgePageAsync", + summary = "Asynchronously delete a Knowledge Page", + description = "Asynchronously delete a knowledge page by `id`.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse( + responseCode = "404", + description = "Knowledge Page for instance {id} is not found") + }) + public Response deleteByIdAsync( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the Knowledge Page", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id, + @Parameter( + description = "Recursively delete this entity and it's children. (Default `false`)") + @QueryParam("recursive") + @DefaultValue("false") + boolean recursive) { + return deleteByIdAsync(uriInfo, securityContext, id, recursive, true); + } + + @DELETE + @Path("/name/{fqn}") + @Operation( + operationId = "deleteKnowledgePageByFQN", + summary = "Delete a Knowledge Page", + description = "Delete a KnowledgePage by `fullyQualifiedName`.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse( + responseCode = "404", + description = "Knowledge Page for instance {fqn} is not found") + }) + public Response delete( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter( + description = "Fully qualified name of the Knowledge Page", + schema = @Schema(type = "string")) + @PathParam("fqn") + String fqn, + @Parameter( + description = "Recursively delete this entity and it's children. (Default `false`)") + @QueryParam("recursive") + @DefaultValue("false") + boolean recursive) { + return deleteByName(uriInfo, securityContext, fqn, recursive, true); + } + + private String getUsersFromIdList(List fromIds) { + return fromIds.stream().map(item -> "'" + item + "'").collect(Collectors.joining(",")); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/DefaultRecreateHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/DefaultRecreateHandler.java index 20cc570a83fe..1c347a95e390 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/DefaultRecreateHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/DefaultRecreateHandler.java @@ -333,6 +333,15 @@ public void promoteEntityIndex(EntityReindexContext context, boolean reindexSucc return; } + // Restore live serving settings on the staged index before alias swap. The bulk-build + // overrides (refresh=-1, replicas=0, async translog) must NOT survive into live serving — + // otherwise live writes after promotion are buffered indefinitely and only become + // searchable on a manual _refresh, which surfaces as the "create-then-search returns + // nothing until reindex" symptom on knowledge pages. This mirrors the call in + // finalizeReindex; the per-entity distributed promotion path was missing it. + applyLiveServingSettings(searchClient, stagedIndex, entityType); + maybeForceMerge(searchClient, stagedIndex, entityType); + // Always clear staged-index routing on the way out — see the rationale in finalizeReindex. try { // Restore live serving settings on the staged index before alias swap. The bulk-build diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexFactory.java index 33d3908ad22e..1ae808e393e1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexFactory.java @@ -14,16 +14,19 @@ import org.openmetadata.schema.entity.data.APIEndpoint; import org.openmetadata.schema.entity.data.Chart; import org.openmetadata.schema.entity.data.Container; +import org.openmetadata.schema.entity.data.ContextFile; import org.openmetadata.schema.entity.data.Dashboard; import org.openmetadata.schema.entity.data.DashboardDataModel; import org.openmetadata.schema.entity.data.Database; import org.openmetadata.schema.entity.data.DatabaseSchema; import org.openmetadata.schema.entity.data.Directory; import org.openmetadata.schema.entity.data.File; +import org.openmetadata.schema.entity.data.Folder; import org.openmetadata.schema.entity.data.Glossary; import org.openmetadata.schema.entity.data.GlossaryTerm; import org.openmetadata.schema.entity.data.Metric; import org.openmetadata.schema.entity.data.MlModel; +import org.openmetadata.schema.entity.data.Page; import org.openmetadata.schema.entity.data.Pipeline; import org.openmetadata.schema.entity.data.Query; import org.openmetadata.schema.entity.data.QueryCostRecord; @@ -52,6 +55,7 @@ import org.openmetadata.service.search.indexes.ChartIndex; import org.openmetadata.service.search.indexes.ClassificationIndex; import org.openmetadata.service.search.indexes.ContainerIndex; +import org.openmetadata.service.search.indexes.ContextFileIndex; import org.openmetadata.service.search.indexes.DashboardDataModelIndex; import org.openmetadata.service.search.indexes.DashboardIndex; import org.openmetadata.service.search.indexes.DashboardServiceIndex; @@ -64,6 +68,7 @@ import org.openmetadata.service.search.indexes.DriveServiceIndex; import org.openmetadata.service.search.indexes.EntityReportDataIndex; import org.openmetadata.service.search.indexes.FileIndex; +import org.openmetadata.service.search.indexes.FolderIndex; import org.openmetadata.service.search.indexes.GlossaryIndex; import org.openmetadata.service.search.indexes.GlossaryTermIndex; import org.openmetadata.service.search.indexes.IngestionPipelineIndex; @@ -77,6 +82,7 @@ import org.openmetadata.service.search.indexes.MetricIndex; import org.openmetadata.service.search.indexes.MlModelIndex; import org.openmetadata.service.search.indexes.MlModelServiceIndex; +import org.openmetadata.service.search.indexes.PageIndex; import org.openmetadata.service.search.indexes.PipelineExecutionIndex; import org.openmetadata.service.search.indexes.PipelineIndex; import org.openmetadata.service.search.indexes.PipelineServiceIndex; @@ -181,6 +187,9 @@ public SearchIndex buildIndex(String entityType, Object entity) { case Entity.FILE -> new FileIndex((File) entity); case Entity.SPREADSHEET -> new SpreadsheetIndex((Spreadsheet) entity); case Entity.WORKSHEET -> new WorksheetIndex((Worksheet) entity); + case Entity.FOLDER -> new FolderIndex((Folder) entity); + case Entity.CONTEXT_FILE -> new ContextFileIndex((ContextFile) entity); + case Entity.PAGE -> new PageIndex((Page) entity); case Entity.DATA_PRODUCT -> new DataProductIndex((DataProduct) entity); case Entity.METADATA_SERVICE -> new MetadataServiceIndex((MetadataService) entity); case Entity.ENTITY_REPORT_DATA -> new EntityReportDataIndex((ReportData) entity); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index 442ec0f2d99e..c976a56fbfb0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -2491,6 +2491,12 @@ public void deleteOrUpdateChildren(EntityInterface entity, IndexMapping indexMap Entity.DRIVE_SERVICE -> searchClient.deleteEntityByFields( indexMapping.getChildAliases(clusterAlias), List.of(new ImmutablePair<>("service.id", docId))); + // Knowledge Center pages are nested via FQN (parent.fqn -> parent.fqn.child), + // not via a parent.id field on the child doc. A recursive hard-delete on the + // parent must therefore also remove every descendant from search by FQN + // prefix; otherwise stale child docs survive in the index and re-appear in + // hierarchy / search results until a full reindex. + case Entity.PAGE -> deleteEntityByFQNPrefix(entity); default -> { List indexNames = indexMapping.getChildAliases(clusterAlias); if (!indexNames.isEmpty()) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index 097776d5c93f..bf69bed844f9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -1071,4 +1071,349 @@ public void initializeLineageBuilders() { LOG.debug("ESLineageGraphBuilder already initialized or newClient is null"); } } + + // ===================== Knowledge Center page hierarchy ===================== + + @Override + @lombok.SneakyThrows + public org.openmetadata.schema.utils.ResultList + listPageHierarchy(String parentFqn, String pageType, int offset, int limit) { + return getPageHierarchyFromSearch(parentFqn, pageType, offset, limit); + } + + @Override + @lombok.SneakyThrows + public org.openmetadata.schema.utils.ResultList + listPageHierarchyForActivePage(String activeFqn, String pageType, int offset, int limit) { + return getPageHierarchyFromSearchForActivePage(activeFqn, pageType, offset, limit); + } + + private org.openmetadata.schema.utils.ResultList< + org.openmetadata.schema.entity.data.PageHierarchy> + getPageHierarchyFromSearch(String parentFqn, String pageType, int offset, int limit) + throws java.io.IOException { + es.co.elastic.clients.elasticsearch._types.query_dsl.Query boolQuery = + buildPageHierarchyBoolQuery(parentFqn, pageType); + + es.co.elastic.clients.elasticsearch.core.SearchRequest searchRequest = + es.co.elastic.clients.elasticsearch.core.SearchRequest.of( + s -> + s.index( + org.openmetadata.service.Entity.getSearchRepository() + .getIndexOrAliasName( + org.openmetadata.service.jdbi3.KnowledgePageRepository + .KNOWLEDGE_PAGE_TERM_SEARCH_INDEX)) + .query(boolQuery) + // Stable sort so from/size pagination cannot miss/duplicate hits. + // fullyQualifiedName is a keyword field with doc_values and is unique per + // page (name is unique within a parent's children), so no tiebreaker is + // needed. _id cannot be used as a sort field on ES 9.x / OpenSearch 3.x + // without setting indices.id_field_data.enabled=true at the cluster level. + .sort( + sort -> + sort.field( + f -> + f.field("fullyQualifiedName") + .order( + es.co.elastic.clients.elasticsearch._types.SortOrder + .Asc))) + .from(offset) + .size(limit)); + + es.co.elastic.clients.elasticsearch.core.SearchResponse + searchResponse = newClient.search(searchRequest, es.co.elastic.clients.json.JsonData.class); + java.util.List pageHierarchies = + processPageHierarchyHits(searchResponse); + int total = 0; + if (searchResponse != null + && searchResponse.hits() != null + && searchResponse.hits().total() != null) { + total = (int) searchResponse.hits().total().value(); + } + return new org.openmetadata.schema.utils.ResultList<>( + pageHierarchies, offset, pageHierarchies.size(), total); + } + + private org.openmetadata.schema.utils.ResultList< + org.openmetadata.schema.entity.data.PageHierarchy> + getPageHierarchyFromSearchForActivePage( + String activeFqn, String pageType, int offset, int limit) throws java.io.IOException { + es.co.elastic.clients.elasticsearch._types.query_dsl.Query boolQuery = + buildPageHierarchyBoolQueryForActivePage(activeFqn, pageType); + + es.co.elastic.clients.elasticsearch.core.SearchRequest searchRequest = + es.co.elastic.clients.elasticsearch.core.SearchRequest.of( + s -> + s.index( + org.openmetadata.service.Entity.getSearchRepository() + .getIndexOrAliasName( + org.openmetadata.service.jdbi3.KnowledgePageRepository + .KNOWLEDGE_PAGE_TERM_SEARCH_INDEX)) + .query(boolQuery) + // Stable sort by fqn (keyword, unique per page). See note above on _id. + .sort( + sort -> + sort.field( + f -> + f.field("fullyQualifiedName") + .order( + es.co.elastic.clients.elasticsearch._types.SortOrder + .Asc))) + .from(offset) + .size(limit)); + + es.co.elastic.clients.elasticsearch.core.SearchResponse + searchResponse = newClient.search(searchRequest, es.co.elastic.clients.json.JsonData.class); + java.util.List pageHierarchies = + processPageHierarchyHits(searchResponse); + pageHierarchies = buildPageNestedSearchHierarchy(pageHierarchies); + int total = 0; + if (searchResponse != null + && searchResponse.hits() != null + && searchResponse.hits().total() != null) { + total = (int) searchResponse.hits().total().value(); + } + return new org.openmetadata.schema.utils.ResultList<>( + pageHierarchies, offset, pageHierarchies.size(), total); + } + + private es.co.elastic.clients.elasticsearch._types.query_dsl.Query buildPageHierarchyBoolQuery( + String parentFqn, String pageType) { + es.co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery.Builder boolQueryBuilder = + new es.co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery.Builder(); + + if (org.openmetadata.common.utils.CommonUtil.nullOrEmpty(parentFqn)) { + boolQueryBuilder.must( + es.co.elastic.clients.elasticsearch._types.query_dsl.Query.of( + q -> + q.term( + t -> + t.field("fqnDepth") + .value( + es.co.elastic.clients.elasticsearch._types.FieldValue.of(1))))); + } else { + int parentDepth = org.openmetadata.service.util.FullyQualifiedName.split(parentFqn).length; + boolQueryBuilder.must( + es.co.elastic.clients.elasticsearch._types.query_dsl.Query.of( + q -> q.prefix(p -> p.field("fullyQualifiedName").value(parentFqn + ".")))); + boolQueryBuilder.must( + es.co.elastic.clients.elasticsearch._types.query_dsl.Query.of( + q -> + q.term( + t -> + t.field("fqnDepth") + .value( + es.co.elastic.clients.elasticsearch._types.FieldValue.of( + parentDepth + 1))))); + } + + if (!org.openmetadata.common.utils.CommonUtil.nullOrEmpty(pageType)) { + boolQueryBuilder.must( + es.co.elastic.clients.elasticsearch._types.query_dsl.Query.of( + q -> + q.term( + t -> + t.field("pageType") + .value( + es.co.elastic.clients.elasticsearch._types.FieldValue.of( + pageType))))); + } + + return es.co.elastic.clients.elasticsearch._types.query_dsl.Query.of( + q -> q.bool(boolQueryBuilder.build())); + } + + private es.co.elastic.clients.elasticsearch._types.query_dsl.Query + buildPageHierarchyBoolQueryForActivePage(String activeFqn, String pageType) { + es.co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery.Builder boolQueryBuilder = + new es.co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery.Builder(); + + String rootParentFqn = org.openmetadata.service.util.FullyQualifiedName.split(activeFqn)[0]; + boolQueryBuilder.should( + es.co.elastic.clients.elasticsearch._types.query_dsl.Query.of( + q -> + q.term( + t -> + t.field("fqnDepth") + .value(es.co.elastic.clients.elasticsearch._types.FieldValue.of(1))))); + boolQueryBuilder.should( + es.co.elastic.clients.elasticsearch._types.query_dsl.Query.of( + q -> q.prefix(p -> p.field("fullyQualifiedName").value(rootParentFqn + ".")))); + boolQueryBuilder.minimumShouldMatch("1"); + + if (!org.openmetadata.common.utils.CommonUtil.nullOrEmpty(pageType)) { + boolQueryBuilder.must( + es.co.elastic.clients.elasticsearch._types.query_dsl.Query.of( + q -> + q.term( + t -> + t.field("pageType") + .value( + es.co.elastic.clients.elasticsearch._types.FieldValue.of( + pageType))))); + } + + return es.co.elastic.clients.elasticsearch._types.query_dsl.Query.of( + q -> q.bool(boolQueryBuilder.build())); + } + + private java.util.List + processPageHierarchyHits( + es.co.elastic.clients.elasticsearch.core.SearchResponse< + es.co.elastic.clients.json.JsonData> + searchResponse) + throws java.io.IOException { + java.util.List pageHierarchies = + new java.util.ArrayList<>(); + + if (searchResponse != null && searchResponse.hits() != null) { + for (es.co.elastic.clients.elasticsearch.core.search.Hit + hit : searchResponse.hits().hits()) { + if (hit.source() != null) { + java.util.Map sourceMap = EsUtils.jsonDataToMap(hit.source()); + org.openmetadata.schema.entity.data.PageHierarchy page = + org.openmetadata.service.util.SearchUtils.getPageHierarchy(sourceMap); + pageHierarchies.add(page); + } + } + } + + populateChildrenCounts(pageHierarchies); + return pageHierarchies; + } + + /** + * Populate {@code childrenCount} on each page using a single aggregation round-trip + * instead of one search per page (N+1). Uses a filters aggregation keyed by page id, + * where each bucket matches descendants via the page's fullyQualifiedName prefix. + */ + private void populateChildrenCounts( + java.util.List pageHierarchies) + throws java.io.IOException { + if (pageHierarchies.isEmpty()) { + return; + } + + java.util.Map filters = + new java.util.HashMap<>(); + for (org.openmetadata.schema.entity.data.PageHierarchy page : pageHierarchies) { + if (page.getId() == null + || page.getFullyQualifiedName() == null + || page.getFullyQualifiedName().isEmpty()) { + continue; + } + String fqnPrefix = page.getFullyQualifiedName() + "."; + int childDepth = + org.openmetadata.service.util.FullyQualifiedName.split(page.getFullyQualifiedName()) + .length + + 1; + // Match only direct children: FQN starts with "." AND fqnDepth is + // exactly one deeper than the parent. Descendants deeper than that are excluded. + filters.put( + page.getId().toString(), + es.co.elastic.clients.elasticsearch._types.query_dsl.Query.of( + q -> + q.bool( + b -> + b.must( + es.co.elastic.clients.elasticsearch._types.query_dsl.Query.of( + m -> + m.prefix( + p -> p.field("fullyQualifiedName").value(fqnPrefix)))) + .must( + es.co.elastic.clients.elasticsearch._types.query_dsl.Query.of( + m -> + m.term( + t -> + t.field("fqnDepth") + .value( + es.co.elastic.clients.elasticsearch._types + .FieldValue.of(childDepth)))))))); + page.setChildrenCount(0); + } + + if (filters.isEmpty()) { + return; + } + + es.co.elastic.clients.elasticsearch.core.SearchRequest aggregationRequest = + es.co.elastic.clients.elasticsearch.core.SearchRequest.of( + s -> + s.index( + org.openmetadata.service.Entity.getSearchRepository() + .getIndexOrAliasName( + org.openmetadata.service.jdbi3.KnowledgePageRepository + .KNOWLEDGE_PAGE_TERM_SEARCH_INDEX)) + .size(0) + .aggregations( + "children_by_parent", + a -> a.filters(f -> f.filters(fs -> fs.keyed(filters))))); + + es.co.elastic.clients.elasticsearch.core.SearchResponse + aggregationResponse = + newClient.search(aggregationRequest, es.co.elastic.clients.json.JsonData.class); + + if (aggregationResponse == null + || aggregationResponse.aggregations() == null + || aggregationResponse.aggregations().get("children_by_parent") == null) { + return; + } + + java.util.Map + buckets = + aggregationResponse + .aggregations() + .get("children_by_parent") + .filters() + .buckets() + .keyed(); + + for (org.openmetadata.schema.entity.data.PageHierarchy page : pageHierarchies) { + if (page.getId() == null) { + continue; + } + es.co.elastic.clients.elasticsearch._types.aggregations.FiltersBucket bucket = + buckets.get(page.getId().toString()); + if (bucket != null) { + page.setChildrenCount((int) bucket.docCount()); + } + } + } + + private java.util.List + buildPageNestedSearchHierarchy( + java.util.List pageHierarchyList) { + java.util.Map + pageHierarchyMap = + pageHierarchyList.stream() + // Skip hits that lost their id during parsing (SearchUtils returns a + // null id for malformed/missing UUID strings) so Collectors.toMap + // does not throw on the null key. + .filter(p -> p.getId() != null) + .collect( + java.util.stream.Collectors.toMap( + org.openmetadata.schema.entity.data.PageHierarchy::getId, + page -> { + page.setChildren(new java.util.ArrayList<>()); + return page; + }, + (existing, replacement) -> existing, + java.util.LinkedHashMap::new)); + + java.util.List rootPages = + new java.util.ArrayList<>(); + + for (org.openmetadata.schema.entity.data.PageHierarchy page : pageHierarchyMap.values()) { + java.util.UUID parentId = page.getParent() != null ? page.getParent().getId() : null; + org.openmetadata.schema.entity.data.PageHierarchy parentPage = + parentId != null ? pageHierarchyMap.get(parentId) : null; + if (parentPage != null) { + parentPage.getChildren().add(page); + } else { + rootPages.add(page); + } + } + + return rootPages; + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContextFileIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContextFileIndex.java new file mode 100644 index 000000000000..e063943a4192 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContextFileIndex.java @@ -0,0 +1,49 @@ +package org.openmetadata.service.search.indexes; + +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; + +import java.util.Map; +import org.openmetadata.schema.entity.data.ContextFile; +import org.openmetadata.service.Entity; + +public class ContextFileIndex implements TaggableIndex { + final ContextFile file; + + public ContextFileIndex(ContextFile file) { + this.file = file; + } + + @Override + public Object getEntity() { + return file; + } + + @Override + public String getEntityTypeName() { + return Entity.CONTEXT_FILE; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { + doc.put("fileType", file.getFileType()); + doc.put("fileSize", file.getFileSize()); + doc.put("fileExtension", file.getFileExtension()); + doc.put("contentType", file.getContentType()); + doc.put("processingStatus", file.getProcessingStatus()); + doc.put("sourceType", file.getSourceType()); + if (!nullOrEmpty(file.getExtractedText())) { + doc.put("extractedText", file.getExtractedText()); + } + if (file.getFolder() != null) { + doc.put("folder", getEntityWithDisplayName(file.getFolder())); + } + return doc; + } + + public static Map getFields() { + Map fields = SearchIndex.getDefaultFields(); + fields.put("fileExtension", 3.0f); + fields.put("extractedText", 2.0f); + return fields; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/FolderIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/FolderIndex.java new file mode 100644 index 000000000000..6d7391e61bd7 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/FolderIndex.java @@ -0,0 +1,39 @@ +package org.openmetadata.service.search.indexes; + +import java.util.Map; +import org.openmetadata.schema.entity.data.Folder; +import org.openmetadata.service.Entity; + +public class FolderIndex implements TaggableIndex { + final Folder folder; + + public FolderIndex(Folder folder) { + this.folder = folder; + } + + @Override + public Object getEntity() { + return folder; + } + + @Override + public String getEntityTypeName() { + return Entity.FOLDER; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { + if (folder.getParent() != null) { + doc.put("parent", getEntityWithDisplayName(folder.getParent())); + } + // Default to 0 when the entity hasn't had its children recomputed yet (e.g. just-created + // folders). Storing null as a long/integer in ES indexes as `missing` and breaks + // numeric range/sort queries that assume the field is always present. + doc.put("childrenCount", folder.getChildrenCount() != null ? folder.getChildrenCount() : 0); + return doc; + } + + public static Map getFields() { + return SearchIndex.getDefaultFields(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PageIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PageIndex.java new file mode 100644 index 000000000000..40ca5116954d --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PageIndex.java @@ -0,0 +1,57 @@ +package org.openmetadata.service.search.indexes; + +import static org.openmetadata.service.jdbi3.KnowledgePageRepository.KNOWLEDGE_PAGE_ENTITY; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import org.openmetadata.schema.entity.data.Page; +import org.openmetadata.service.util.FullyQualifiedName; + +public class PageIndex implements SearchIndex { + final Page page; + + public PageIndex(Page page) { + this.page = page; + } + + @Override + public Object getEntity() { + return page; + } + + @Override + public String getEntityTypeName() { + return KNOWLEDGE_PAGE_ENTITY; + } + + @Override + public Set getRequiredReindexFields() { + Set fields = new HashSet<>(SearchIndex.super.getRequiredReindexFields()); + fields.add("parent"); + fields.add("children"); + fields.add("editors"); + fields.add("relatedEntities"); + return Collections.unmodifiableSet(fields); + } + + public Map buildSearchIndexDocInternal(Map doc) { + doc.put("fqnDepth", calculateFqnDepth(page.getFullyQualifiedName())); + // Override common deleted field: pages are hard-deleted (not soft-deleted), + // so they should always appear as not-deleted in the search index + doc.put("deleted", Boolean.FALSE); + return doc; + } + + public static int calculateFqnDepth(String fullyQualifiedName) { + if (fullyQualifiedName == null || fullyQualifiedName.isEmpty()) { + return 0; + } + return FullyQualifiedName.split(fullyQualifiedName).length; + } + + public static Map getFields() { + return SearchIndex.getDefaultFields(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index dd046b6dfb48..adaf5556bb0e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -1113,4 +1113,352 @@ public void initializeLineageBuilders() { LOG.debug("OSLineageGraphBuilder already initialized or newClient is null"); } } + + // ===================== Knowledge Center page hierarchy ===================== + + @Override + @lombok.SneakyThrows + public org.openmetadata.schema.utils.ResultList + listPageHierarchy(String parentFqn, String pageType, int offset, int limit) { + return getPageHierarchyFromSearch(parentFqn, pageType, offset, limit); + } + + @Override + @lombok.SneakyThrows + public org.openmetadata.schema.utils.ResultList + listPageHierarchyForActivePage(String activeFqn, String pageType, int offset, int limit) { + return getPageHierarchyFromSearchForActivePage(activeFqn, pageType, offset, limit); + } + + private org.openmetadata.schema.utils.ResultList< + org.openmetadata.schema.entity.data.PageHierarchy> + getPageHierarchyFromSearch(String parentFqn, String pageType, int offset, int limit) + throws java.io.IOException { + os.org.opensearch.client.opensearch._types.query_dsl.Query boolQuery = + buildPageHierarchyBoolQuery(parentFqn, pageType); + + os.org.opensearch.client.opensearch.core.SearchRequest searchRequest = + os.org.opensearch.client.opensearch.core.SearchRequest.of( + s -> + s.index( + org.openmetadata.service.Entity.getSearchRepository() + .getIndexOrAliasName( + org.openmetadata.service.jdbi3.KnowledgePageRepository + .KNOWLEDGE_PAGE_TERM_SEARCH_INDEX)) + .query(boolQuery) + // Stable sort so from/size pagination cannot miss/duplicate hits. + // fullyQualifiedName is a keyword field with doc_values and is unique per + // page (name is unique within a parent's children), so no tiebreaker is + // needed. _id cannot be used as a sort field on ES 9.x / OpenSearch 3.x + // without setting indices.id_field_data.enabled=true at the cluster level. + .sort( + sort -> + sort.field( + f -> + f.field("fullyQualifiedName") + .order( + os.org.opensearch.client.opensearch._types.SortOrder + .Asc))) + .from(offset) + .size(limit)); + + os.org.opensearch.client.opensearch.core.SearchResponse + searchResponse = + newClient.search(searchRequest, os.org.opensearch.client.json.JsonData.class); + java.util.List pageHierarchies = + processPageHierarchyHits(searchResponse); + int total = 0; + if (searchResponse != null + && searchResponse.hits() != null + && searchResponse.hits().total() != null) { + total = (int) searchResponse.hits().total().value(); + } + return new org.openmetadata.schema.utils.ResultList<>( + pageHierarchies, offset, pageHierarchies.size(), total); + } + + private org.openmetadata.schema.utils.ResultList< + org.openmetadata.schema.entity.data.PageHierarchy> + getPageHierarchyFromSearchForActivePage( + String activeFqn, String pageType, int offset, int limit) throws java.io.IOException { + os.org.opensearch.client.opensearch._types.query_dsl.Query boolQuery = + buildPageHierarchyBoolQueryForActivePage(activeFqn, pageType); + + os.org.opensearch.client.opensearch.core.SearchRequest searchRequest = + os.org.opensearch.client.opensearch.core.SearchRequest.of( + s -> + s.index( + org.openmetadata.service.Entity.getSearchRepository() + .getIndexOrAliasName( + org.openmetadata.service.jdbi3.KnowledgePageRepository + .KNOWLEDGE_PAGE_TERM_SEARCH_INDEX)) + .query(boolQuery) + // Stable sort by fqn (keyword, unique per page). See note above on _id. + .sort( + sort -> + sort.field( + f -> + f.field("fullyQualifiedName") + .order( + os.org.opensearch.client.opensearch._types.SortOrder + .Asc))) + .from(offset) + .size(limit)); + + os.org.opensearch.client.opensearch.core.SearchResponse + searchResponse = + newClient.search(searchRequest, os.org.opensearch.client.json.JsonData.class); + java.util.List pageHierarchies = + processPageHierarchyHits(searchResponse); + pageHierarchies = buildPageNestedSearchHierarchy(pageHierarchies); + int total = 0; + if (searchResponse != null + && searchResponse.hits() != null + && searchResponse.hits().total() != null) { + total = (int) searchResponse.hits().total().value(); + } + return new org.openmetadata.schema.utils.ResultList<>( + pageHierarchies, offset, pageHierarchies.size(), total); + } + + private os.org.opensearch.client.opensearch._types.query_dsl.Query buildPageHierarchyBoolQuery( + String parentFqn, String pageType) { + os.org.opensearch.client.opensearch._types.query_dsl.BoolQuery.Builder boolQueryBuilder = + new os.org.opensearch.client.opensearch._types.query_dsl.BoolQuery.Builder(); + + if (org.openmetadata.common.utils.CommonUtil.nullOrEmpty(parentFqn)) { + boolQueryBuilder.must( + os.org.opensearch.client.opensearch._types.query_dsl.Query.of( + q -> + q.term( + t -> + t.field("fqnDepth") + .value( + os.org.opensearch.client.opensearch._types.FieldValue.of(1))))); + } else { + int parentDepth = org.openmetadata.service.util.FullyQualifiedName.split(parentFqn).length; + boolQueryBuilder.must( + os.org.opensearch.client.opensearch._types.query_dsl.Query.of( + q -> q.prefix(p -> p.field("fullyQualifiedName").value(parentFqn + ".")))); + boolQueryBuilder.must( + os.org.opensearch.client.opensearch._types.query_dsl.Query.of( + q -> + q.term( + t -> + t.field("fqnDepth") + .value( + os.org.opensearch.client.opensearch._types.FieldValue.of( + parentDepth + 1))))); + } + + if (!org.openmetadata.common.utils.CommonUtil.nullOrEmpty(pageType)) { + boolQueryBuilder.must( + os.org.opensearch.client.opensearch._types.query_dsl.Query.of( + q -> + q.term( + t -> + t.field("pageType") + .value( + os.org.opensearch.client.opensearch._types.FieldValue.of( + pageType))))); + } + + return os.org.opensearch.client.opensearch._types.query_dsl.Query.of( + q -> q.bool(boolQueryBuilder.build())); + } + + private os.org.opensearch.client.opensearch._types.query_dsl.Query + buildPageHierarchyBoolQueryForActivePage(String activeFqn, String pageType) { + os.org.opensearch.client.opensearch._types.query_dsl.BoolQuery.Builder boolQueryBuilder = + new os.org.opensearch.client.opensearch._types.query_dsl.BoolQuery.Builder(); + + String rootParentFqn = org.openmetadata.service.util.FullyQualifiedName.split(activeFqn)[0]; + boolQueryBuilder.should( + os.org.opensearch.client.opensearch._types.query_dsl.Query.of( + q -> + q.term( + t -> + t.field("fqnDepth") + .value(os.org.opensearch.client.opensearch._types.FieldValue.of(1))))); + boolQueryBuilder.should( + os.org.opensearch.client.opensearch._types.query_dsl.Query.of( + q -> q.prefix(p -> p.field("fullyQualifiedName").value(rootParentFqn + ".")))); + boolQueryBuilder.minimumShouldMatch("1"); + + if (!org.openmetadata.common.utils.CommonUtil.nullOrEmpty(pageType)) { + boolQueryBuilder.must( + os.org.opensearch.client.opensearch._types.query_dsl.Query.of( + q -> + q.term( + t -> + t.field("pageType") + .value( + os.org.opensearch.client.opensearch._types.FieldValue.of( + pageType))))); + } + + return os.org.opensearch.client.opensearch._types.query_dsl.Query.of( + q -> q.bool(boolQueryBuilder.build())); + } + + private java.util.List + processPageHierarchyHits( + os.org.opensearch.client.opensearch.core.SearchResponse< + os.org.opensearch.client.json.JsonData> + searchResponse) + throws java.io.IOException { + java.util.List pageHierarchies = + new java.util.ArrayList<>(); + + if (searchResponse != null && searchResponse.hits() != null) { + for (os.org.opensearch.client.opensearch.core.search.Hit< + os.org.opensearch.client.json.JsonData> + hit : searchResponse.hits().hits()) { + if (hit.source() != null) { + java.util.Map sourceMap = OsUtils.jsonDataToMap(hit.source()); + org.openmetadata.schema.entity.data.PageHierarchy page = + org.openmetadata.service.util.SearchUtils.getPageHierarchy(sourceMap); + pageHierarchies.add(page); + } + } + } + + populateChildrenCounts(pageHierarchies); + return pageHierarchies; + } + + /** + * Populate {@code childrenCount} on each page using a single aggregation round-trip + * instead of one search per page (N+1). Uses a filters aggregation keyed by page id, + * where each bucket matches descendants via the page's fullyQualifiedName prefix. + */ + private void populateChildrenCounts( + java.util.List pageHierarchies) + throws java.io.IOException { + if (pageHierarchies.isEmpty()) { + return; + } + + java.util.Map filters = + new java.util.HashMap<>(); + for (org.openmetadata.schema.entity.data.PageHierarchy page : pageHierarchies) { + if (page.getId() == null + || page.getFullyQualifiedName() == null + || page.getFullyQualifiedName().isEmpty()) { + continue; + } + String fqnPrefix = page.getFullyQualifiedName() + "."; + int childDepth = + org.openmetadata.service.util.FullyQualifiedName.split(page.getFullyQualifiedName()) + .length + + 1; + // Match only direct children: FQN starts with "." AND fqnDepth is + // exactly one deeper than the parent. Descendants deeper than that are excluded. + filters.put( + page.getId().toString(), + os.org.opensearch.client.opensearch._types.query_dsl.Query.of( + q -> + q.bool( + b -> + b.must( + os.org.opensearch.client.opensearch._types.query_dsl.Query.of( + m -> + m.prefix( + p -> p.field("fullyQualifiedName").value(fqnPrefix)))) + .must( + os.org.opensearch.client.opensearch._types.query_dsl.Query.of( + m -> + m.term( + t -> + t.field("fqnDepth") + .value( + os.org.opensearch.client.opensearch._types + .FieldValue.of(childDepth)))))))); + page.setChildrenCount(0); + } + + if (filters.isEmpty()) { + return; + } + + os.org.opensearch.client.opensearch.core.SearchRequest aggregationRequest = + os.org.opensearch.client.opensearch.core.SearchRequest.of( + s -> + s.index( + org.openmetadata.service.Entity.getSearchRepository() + .getIndexOrAliasName( + org.openmetadata.service.jdbi3.KnowledgePageRepository + .KNOWLEDGE_PAGE_TERM_SEARCH_INDEX)) + .size(0) + .aggregations( + "children_by_parent", + a -> a.filters(f -> f.filters(fs -> fs.keyed(filters))))); + + os.org.opensearch.client.opensearch.core.SearchResponse + aggregationResponse = + newClient.search(aggregationRequest, os.org.opensearch.client.json.JsonData.class); + + if (aggregationResponse == null + || aggregationResponse.aggregations() == null + || aggregationResponse.aggregations().get("children_by_parent") == null) { + return; + } + + java.util.Map + buckets = + aggregationResponse + .aggregations() + .get("children_by_parent") + .filters() + .buckets() + .keyed(); + + for (org.openmetadata.schema.entity.data.PageHierarchy page : pageHierarchies) { + if (page.getId() == null) { + continue; + } + os.org.opensearch.client.opensearch._types.aggregations.FiltersBucket bucket = + buckets.get(page.getId().toString()); + if (bucket != null) { + page.setChildrenCount((int) bucket.docCount()); + } + } + } + + private java.util.List + buildPageNestedSearchHierarchy( + java.util.List pageHierarchyList) { + java.util.Map + pageHierarchyMap = + pageHierarchyList.stream() + // Skip hits that lost their id during parsing (SearchUtils returns a + // null id for malformed/missing UUID strings) so Collectors.toMap + // does not throw on the null key. + .filter(p -> p.getId() != null) + .collect( + java.util.stream.Collectors.toMap( + org.openmetadata.schema.entity.data.PageHierarchy::getId, + page -> { + page.setChildren(new java.util.ArrayList<>()); + return page; + }, + (existing, replacement) -> existing, + java.util.LinkedHashMap::new)); + + java.util.List rootPages = + new java.util.ArrayList<>(); + + for (org.openmetadata.schema.entity.data.PageHierarchy page : pageHierarchyMap.values()) { + java.util.UUID parentId = page.getParent() != null ? page.getParent().getId() : null; + org.openmetadata.schema.entity.data.PageHierarchy parentPage = + parentId != null ? pageHierarchyMap.get(parentId) : null; + if (parentPage != null) { + parentPage.getChildren().add(page); + } else { + rootPages.add(page); + } + } + + return rootPages; + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/SearchUtils.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/SearchUtils.java new file mode 100644 index 000000000000..2475c546f571 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/SearchUtils.java @@ -0,0 +1,99 @@ +/* + * Copyright 2025 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.util; + +import java.util.Map; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.entity.data.PageHierarchy; +import org.openmetadata.schema.entity.data.PageType; +import org.openmetadata.schema.type.EntityReference; + +@Slf4j +public final class SearchUtils { + + private SearchUtils() {} + + @SuppressWarnings("unchecked") + public static PageHierarchy getPageHierarchy(Map sourceMap) { + String idStr = (String) sourceMap.get("id"); + String pageTypeStr = (String) sourceMap.get("pageType"); + String name = (String) sourceMap.get("name"); + String displayName = (String) sourceMap.get("displayName"); + String fullyQualifiedName = (String) sourceMap.get("fullyQualifiedName"); + Map parentMap = (Map) sourceMap.get("parent"); + EntityReference parent = null; + + if (parentMap != null) { + parent = new EntityReference(); + parent.setId(parseUuid((String) parentMap.get("id"))); + parent.setType((String) parentMap.get("type")); + parent.setName((String) parentMap.get("name")); + parent.setFullyQualifiedName((String) parentMap.get("fullyQualifiedName")); + parent.setDisplayName((String) parentMap.get("displayName")); + parent.setDescription((String) parentMap.get("description")); + } + + PageHierarchy page = new PageHierarchy(); + + UUID pageId = parseUuid(idStr); + if (pageId != null) { + page.withId(pageId); + } + + PageType pageType = parsePageType(pageTypeStr); + if (pageType != null) { + page.withPageType(pageType); + } + + page.withName(name) + .withDisplayName(displayName) + .withFullyQualifiedName(fullyQualifiedName) + .withParent(parent); + + return page; + } + + /** + * Parse a UUID string safely — returns null for missing or malformed values so a single + * bad hit does not break the entire hierarchy response. + */ + private static UUID parseUuid(String value) { + if (value == null || value.isEmpty()) { + return null; + } + try { + return UUID.fromString(value); + } catch (IllegalArgumentException e) { + LOG.warn("Ignoring malformed UUID in search hit: {}", value); + return null; + } + } + + /** + * Parse a PageType string safely — returns null for missing or unknown values (e.g. an + * index written by a newer server version) so a single bad hit does not break the + * entire hierarchy response. + */ + private static PageType parsePageType(String value) { + if (value == null || value.isEmpty()) { + return null; + } + try { + return PageType.fromValue(value); + } catch (IllegalArgumentException e) { + LOG.warn("Ignoring unknown pageType in search hit: {}", value); + return null; + } + } +} diff --git a/openmetadata-service/src/main/resources/json/data/settings/searchSettings.json b/openmetadata-service/src/main/resources/json/data/settings/searchSettings.json index b4b43e104cfa..4ee15bd2d0ba 100644 --- a/openmetadata-service/src/main/resources/json/data/settings/searchSettings.json +++ b/openmetadata-service/src/main/resources/json/data/settings/searchSettings.json @@ -1919,6 +1919,80 @@ "fuzzyMatchMultiplier": 1.0 } }, + { + "assetType": "contextFile", + "searchFields": [ + { "field": "name.keyword", "boost": 20.0, "matchType": "exact" }, + { "field": "name", "boost": 10.0, "matchType": "phrase" }, + { "field": "name.ngram", "boost": 1.0, "matchType": "fuzzy" }, + { "field": "displayName", "boost": 10.0, "matchType": "phrase" }, + { "field": "displayName.ngram", "boost": 1.0, "matchType": "fuzzy" }, + { "field": "description", "boost": 2.0, "matchType": "standard" }, + { "field": "fullyQualifiedName", "boost": 5.0, "matchType": "standard" }, + { "field": "fqnParts", "boost": 5.0, "matchType": "standard" }, + { "field": "extractedText", "boost": 3.0, "matchType": "standard" }, + { "field": "fileExtension", "boost": 2.0, "matchType": "standard" } + ], + "aggregations": [ + { "name": "fileType", "type": "terms", "field": "fileType" }, + { "name": "processingStatus", "type": "terms", "field": "processingStatus" }, + { "name": "sourceType", "type": "terms", "field": "sourceType" } + ], + "scoreMode": "sum", + "boostMode": "multiply", + "highlightFields": ["name", "displayName", "description", "extractedText"], + "matchTypeBoostMultipliers": { + "exactMatchMultiplier": 2.0, + "phraseMatchMultiplier": 1.5, + "fuzzyMatchMultiplier": 1.0 + } + }, + { + "assetType": "folder", + "searchFields": [ + { "field": "name.keyword", "boost": 20.0, "matchType": "exact" }, + { "field": "name", "boost": 10.0, "matchType": "phrase" }, + { "field": "name.ngram", "boost": 1.0, "matchType": "fuzzy" }, + { "field": "displayName", "boost": 10.0, "matchType": "phrase" }, + { "field": "displayName.ngram", "boost": 1.0, "matchType": "fuzzy" }, + { "field": "description", "boost": 2.0, "matchType": "standard" }, + { "field": "fullyQualifiedName", "boost": 5.0, "matchType": "standard" }, + { "field": "fqnParts", "boost": 5.0, "matchType": "standard" } + ], + "aggregations": [], + "scoreMode": "sum", + "boostMode": "multiply", + "highlightFields": ["name", "displayName", "description"], + "matchTypeBoostMultipliers": { + "exactMatchMultiplier": 2.0, + "phraseMatchMultiplier": 1.5, + "fuzzyMatchMultiplier": 1.0 + } + }, + { + "assetType": "page", + "searchFields": [ + { "field": "name.keyword", "boost": 20.0, "matchType": "exact" }, + { "field": "name", "boost": 10.0, "matchType": "phrase" }, + { "field": "name.ngram", "boost": 1.0, "matchType": "fuzzy" }, + { "field": "displayName", "boost": 10.0, "matchType": "phrase" }, + { "field": "displayName.ngram", "boost": 1.0, "matchType": "fuzzy" }, + { "field": "description", "boost": 2.0, "matchType": "standard" }, + { "field": "fullyQualifiedName", "boost": 5.0, "matchType": "standard" }, + { "field": "fqnParts", "boost": 5.0, "matchType": "standard" } + ], + "aggregations": [ + { "name": "pageType", "type": "terms", "field": "pageType" } + ], + "scoreMode": "sum", + "boostMode": "multiply", + "highlightFields": ["name", "displayName", "description"], + "matchTypeBoostMultipliers": { + "exactMatchMultiplier": 2.0, + "phraseMatchMultiplier": 1.5, + "fuzzyMatchMultiplier": 1.0 + } + }, { "assetType": "file", "searchFields": [ @@ -3823,6 +3897,201 @@ } ] }, + { + "entityType": "contextFile", + "fields": [ + { + "name": "name.keyword", + "description": "Exact match on context file name for precise lookups." + }, + { + "name": "name", + "description": "Standard text analysis on context file name with tokenization and stemming." + }, + { + "name": "name.ngram", + "description": "Partial matching on context file name for finding with incomplete information." + }, + { + "name": "displayName", + "description": "Standard text analysis on the human-readable context file name." + }, + { + "name": "displayName.ngram", + "description": "Partial matching on display name for flexible searching." + }, + { + "name": "description", + "description": "Full-text search on context file descriptions to find by purpose or content." + }, + { + "name": "fullyQualifiedName", + "description": "Search on the complete hierarchical name of the context file including folder path." + }, + { + "name": "fqnParts", + "description": "Search on parts of the hierarchical name for flexible matching." + }, + { + "name": "fileType", + "description": "Exact match on the context file type (e.g., PDF, Image, Spreadsheet, Text)." + }, + { + "name": "contentType", + "description": "Exact match on the MIME content type of the context file." + }, + { + "name": "fileExtension", + "description": "Exact match on the context file extension (e.g., pdf, docx, xlsx)." + }, + { + "name": "processingStatus", + "description": "Exact match on the extraction processing status of the context file." + }, + { + "name": "folder.displayName.keyword", + "description": "Exact match on the folder that contains this context file." + }, + { + "name": "extractedText", + "description": "Full-text search on text extracted from the uploaded context file." + }, + { + "name": "tags.tagFQN.text", + "description": "Search within parts of tag names to find context files with specific tags." + }, + { + "name": "tier.tagFQN.text", + "description": "Search within parts of tier classification names for context files." + }, + { + "name": "domains.displayName.keyword", + "description": "Exact match on domain associated with context file." + }, + { + "name": "dataProducts.displayName.keyword", + "description": "Exact match on dataProducts associated with context file." + } + ] + }, + { + "entityType": "folder", + "fields": [ + { + "name": "name.keyword", + "description": "Exact match on folder name for precise lookups." + }, + { + "name": "name", + "description": "Standard text analysis on folder name with tokenization and stemming." + }, + { + "name": "name.ngram", + "description": "Partial matching on folder name for finding with incomplete information." + }, + { + "name": "displayName", + "description": "Standard text analysis on the human-readable folder name." + }, + { + "name": "displayName.ngram", + "description": "Partial matching on display name for flexible searching." + }, + { + "name": "description", + "description": "Full-text search on folder descriptions to find by purpose or content." + }, + { + "name": "fullyQualifiedName", + "description": "Search on the complete hierarchical name of the folder including parent path." + }, + { + "name": "fqnParts", + "description": "Search on parts of the hierarchical name for flexible matching." + }, + { + "name": "parent.displayName.keyword", + "description": "Exact match on parent folder display name to find subfolders." + }, + { + "name": "tags.tagFQN.text", + "description": "Search within parts of tag names to find folders with specific tags." + }, + { + "name": "tier.tagFQN.text", + "description": "Search within parts of tier classification names for folders." + }, + { + "name": "domains.displayName.keyword", + "description": "Exact match on domain associated with folder." + }, + { + "name": "dataProducts.displayName.keyword", + "description": "Exact match on dataProducts associated with folder." + } + ] + }, + { + "entityType": "page", + "fields": [ + { + "name": "name.keyword", + "description": "Exact match on knowledge page name for precise lookups." + }, + { + "name": "name", + "description": "Standard text analysis on knowledge page name with tokenization and stemming." + }, + { + "name": "name.ngram", + "description": "Partial matching on knowledge page name for finding with incomplete information." + }, + { + "name": "displayName", + "description": "Standard text analysis on the human-readable knowledge page name." + }, + { + "name": "displayName.ngram", + "description": "Partial matching on display name for flexible searching." + }, + { + "name": "description", + "description": "Full-text search on knowledge page descriptions to find by purpose or content." + }, + { + "name": "fullyQualifiedName", + "description": "Search on the complete hierarchical name of the knowledge page including parent path." + }, + { + "name": "fqnParts", + "description": "Search on parts of the hierarchical name for flexible matching." + }, + { + "name": "pageType", + "description": "Exact match on the knowledge page type (Article or QuickLink)." + }, + { + "name": "parent.displayName.keyword", + "description": "Exact match on parent knowledge page display name to find child pages." + }, + { + "name": "tags.tagFQN.text", + "description": "Search within parts of tag names to find knowledge pages with specific tags." + }, + { + "name": "tier.tagFQN.text", + "description": "Search within parts of tier classification names for knowledge pages." + }, + { + "name": "domains.displayName.keyword", + "description": "Exact match on domain associated with knowledge page." + }, + { + "name": "dataProducts.displayName.keyword", + "description": "Exact match on dataProducts associated with knowledge page." + } + ] + }, { "entityType": "default", "fields": [ diff --git a/openmetadata-service/src/main/resources/json/data/tags/knowledgeCenterTags.json b/openmetadata-service/src/main/resources/json/data/tags/knowledgeCenterTags.json new file mode 100644 index 000000000000..6725262af7e5 --- /dev/null +++ b/openmetadata-service/src/main/resources/json/data/tags/knowledgeCenterTags.json @@ -0,0 +1,22 @@ +{ + "createClassification": { + "name": "KnowledgeCenter", + "description": "Category describing the knowledge center articles or quickLinks. E.g., How-To-Guide, Quick-Link etc.", + "provider": "system", + "mutuallyExclusive": false + }, + "createTags": [ + { + "name": "Article", + "description": "Knowledge Article." + }, + { + "name": "QuickLink", + "description": "Knowledge Quick Link." + }, + { + "name": "HowToGuide", + "description": "How To Guide Quick Link or Article Tag." + } + ] +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/context/ContextEntityPromptServiceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/context/ContextEntityPromptServiceTest.java new file mode 100644 index 000000000000..664bca1fc923 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/context/ContextEntityPromptServiceTest.java @@ -0,0 +1,126 @@ +package org.openmetadata.service.context; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import jakarta.ws.rs.core.SecurityContext; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import org.junit.jupiter.api.Test; +import org.openmetadata.schema.type.EntityReference; + +class ContextEntityPromptServiceTest { + + @Test + void assembleDeduplicatesEntitiesAndFormatsPrompt() { + EntityReference fileRef = reference("contextFile", "q3-report"); + EntityReference pageRef = reference("page", "distribution-guidelines"); + ContextEntityPromptService service = + new ContextEntityPromptService( + (securityContext, reference) -> + switch (reference.getType()) { + case "contextFile" -> Optional.of( + new ResolvedContextEntity( + fileRef, + "File (PDF)", + "Q3 Report", + "finance.q3-report", + "Quarterly planning document", + "Revenue grew materially year over year.")); + case "page" -> Optional.of( + new ResolvedContextEntity( + pageRef, + "Page", + "Distribution Guidelines", + "knowledge.distribution-guidelines", + null, + "Check skewness and percentiles before quoting averages.")); + default -> Optional.empty(); + }); + + ContextPromptInjectionResult result = + service.assemble(null, List.of(fileRef, fileRef, pageRef)); + + assertEquals(2, result.usedEntityRefs().size()); + assertTrue(result.formattedContext().contains("")); + assertTrue(result.formattedContext().contains("Q3 Report")); + assertTrue(result.formattedContext().contains("Distribution Guidelines")); + assertTrue(result.formattedContext().contains("Content:")); + assertTrue(result.totalTokens() > 0); + } + + @Test + void assembleRespectsBudgetByTruncatingLongBodies() { + EntityReference fileRef = reference("contextFile", "long-file"); + String longBody = "token ".repeat(5000); + ContextEntityPromptService service = + new ContextEntityPromptService( + (securityContext, reference) -> + Optional.of( + new ResolvedContextEntity( + fileRef, "File (Text)", "Long File", "drive.long-file", null, longBody))); + + ContextPromptInjectionResult result = service.assemble(null, List.of(fileRef)); + + assertFalse(result.formattedContext().isEmpty()); + assertTrue(result.formattedContext().contains("[truncated]")); + assertTrue(result.totalTokens() <= ContextEntityPromptService.TOTAL_TOKEN_BUDGET); + } + + @Test + void assembleReturnsEmptyWhenNothingResolves() { + EntityReference ref = reference("contextFile", "missing"); + ContextEntityPromptService service = + new ContextEntityPromptService( + (SecurityContext sc, EntityReference reference) -> Optional.empty()); + + ContextPromptInjectionResult result = service.assemble(null, List.of(ref)); + + assertTrue(result.formattedContext().isEmpty()); + assertTrue(result.usedEntityRefs().isEmpty()); + assertEquals(0, result.totalTokens()); + } + + @Test + void assembleSelectsRelevantChunkForQueryInsteadOfDocumentPrefix() { + EntityReference fileRef = reference("contextFile", "analytics-playbook"); + String longIntro = "intro ".repeat(1500); + String relevantSection = + "When the revenue distribution is skewed, do not rely only on averages. " + + "Use median, percentiles, and outlier review before making claims."; + String longTail = "tail ".repeat(1500); + String longBody = longIntro + "\n\n" + relevantSection + "\n\n" + longTail; + + ContextEntityPromptService service = + new ContextEntityPromptService( + (securityContext, reference) -> + Optional.of( + new ResolvedContextEntity( + fileRef, + "File (PDF)", + "Analytics Playbook", + "drive.analytics-playbook", + null, + longBody))); + + ContextPromptInjectionResult result = + service.assemble( + null, + List.of(fileRef), + "What does the playbook say about skewed revenue distributions and percentiles?"); + + assertFalse(result.formattedContext().isEmpty()); + assertTrue(result.formattedContext().contains(relevantSection)); + } + + private EntityReference reference(String type, String name) { + return new EntityReference() + .withId(UUID.randomUUID()) + .withType(type) + .withName(name) + .withFullyQualifiedName(type + "." + name) + .withDisplayName(name); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/context/DefaultContextEntityPromptLoaderTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/context/DefaultContextEntityPromptLoaderTest.java new file mode 100644 index 000000000000..c4dc10bd3be0 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/context/DefaultContextEntityPromptLoaderTest.java @@ -0,0 +1,122 @@ +package org.openmetadata.service.context; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Method; +import java.util.Optional; +import java.util.UUID; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.openmetadata.schema.entity.data.ContextFile; +import org.openmetadata.schema.entity.data.ContextFileContent; +import org.openmetadata.schema.entity.data.ContextFileType; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Include; +import org.openmetadata.service.jdbi3.ContextFileContentRepository; +import org.openmetadata.service.jdbi3.ContextFileRepository; +import org.openmetadata.service.jdbi3.KnowledgePageRepository; +import org.openmetadata.service.security.Authorizer; + +class DefaultContextEntityPromptLoaderTest { + + @Test + void resolveExtractedTextPrefersCanonicalContentSnapshot() throws Exception { + Authorizer authorizer = mock(Authorizer.class); + ContextFileRepository contextFileRepository = mock(ContextFileRepository.class); + ContextFileContentRepository contentRepository = mock(ContextFileContentRepository.class); + KnowledgePageRepository knowledgeCenterRepository = mock(KnowledgePageRepository.class); + + UUID contentId = UUID.randomUUID(); + + ContextFile file = + new ContextFile() + .withId(UUID.randomUUID()) + .withName("revenue-chart") + .withDisplayName("Revenue Chart") + .withFullyQualifiedName("drive.revenue-chart") + .withFileType(ContextFileType.Image) + .withDescription("Quarterly snapshot") + .withHeadContentId(contentId.toString()) + .withExtractedText("Indexed excerpt only"); + ContextFileContent content = + new ContextFileContent() + .withId(contentId) + .withExtractedText("Canonical OCR text with full numeric callouts"); + + when(contentRepository.getById(contentId)).thenReturn(content); + + DefaultContextEntityPromptLoader loader = + new DefaultContextEntityPromptLoader( + authorizer, contextFileRepository, contentRepository, knowledgeCenterRepository); + Method resolveExtractedText = + DefaultContextEntityPromptLoader.class.getDeclaredMethod( + "resolveExtractedText", ContextFile.class); + resolveExtractedText.setAccessible(true); + + String extractedText = (String) resolveExtractedText.invoke(loader, file); + + assertEquals("Canonical OCR text with full numeric callouts", extractedText); + } + + @Disabled( + "Requires Entity registry initialized with ContextFileRepository; authorizeView " + + "calls new ResourceContext(...) which looks up Entity.getEntityRepository(\"contextFile\"). " + + "Integration test coverage verifies this end-to-end.") + @Test + void loadContextFileBuildsPromptEntityFromCanonicalContentSnapshot() { + Authorizer authorizer = mock(Authorizer.class); + ContextFileRepository contextFileRepository = mock(ContextFileRepository.class); + ContextFileContentRepository contentRepository = mock(ContextFileContentRepository.class); + KnowledgePageRepository knowledgeCenterRepository = mock(KnowledgePageRepository.class); + + UUID fileId = UUID.randomUUID(); + UUID contentId = UUID.randomUUID(); + EntityReference reference = + new EntityReference() + .withId(fileId) + .withType("contextFile") + .withName("revenue-playbook") + .withFullyQualifiedName("drive.revenue-playbook") + .withDisplayName("Revenue Playbook"); + + ContextFile file = + new ContextFile() + .withId(fileId) + .withName("revenue-playbook") + .withDisplayName("Revenue Playbook") + .withFullyQualifiedName("drive.revenue-playbook") + .withDescription("Reusable guidance for AskCollate") + .withFileType(ContextFileType.PDF) + .withHeadContentId(contentId.toString()); + ContextFileContent content = + new ContextFileContent() + .withId(contentId) + .withExtractedText("Use median and percentiles when the distribution is skewed."); + + when(contextFileRepository.get(isNull(), eq(fileId), any(), eq(Include.NON_DELETED), eq(false))) + .thenReturn(file); + when(contentRepository.getById(contentId)).thenReturn(content); + + DefaultContextEntityPromptLoader loader = + new DefaultContextEntityPromptLoader( + authorizer, contextFileRepository, contentRepository, knowledgeCenterRepository); + + Optional resolved = loader.load(null, reference); + + assertTrue(resolved.isPresent()); + assertEquals("File (PDF)", resolved.get().label()); + assertEquals("Revenue Playbook", resolved.get().title()); + assertEquals("drive.revenue-playbook", resolved.get().location()); + assertEquals("Reusable guidance for AskCollate", resolved.get().summary()); + assertEquals( + "Use median and percentiles when the distribution is skewed.", resolved.get().body()); + verify(authorizer).authorize(isNull(), any(), any()); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/drive/ContextFileExtractionServiceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/drive/ContextFileExtractionServiceTest.java new file mode 100644 index 000000000000..03e4b8f05a47 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/drive/ContextFileExtractionServiceTest.java @@ -0,0 +1,216 @@ +package org.openmetadata.service.drive; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.same; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.concurrent.RejectedExecutionException; +import java.util.function.Supplier; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.openmetadata.schema.attachments.Asset; +import org.openmetadata.schema.entity.data.ContextFile; +import org.openmetadata.schema.entity.data.ContextFileContent; +import org.openmetadata.schema.entity.data.ContextFileType; +import org.openmetadata.schema.entity.data.ProcessingStatus; +import org.openmetadata.schema.type.Include; +import org.openmetadata.service.attachments.AssetService; +import org.openmetadata.service.jdbi3.AssetRepository; +import org.openmetadata.service.jdbi3.ContextFileContentRepository; +import org.openmetadata.service.jdbi3.ContextFileRepository; + +@ExtendWith(MockitoExtension.class) +class ContextFileExtractionServiceTest { + + @Mock private ContextFileRepository repository; + @Mock private ContextFileContentRepository contentRepository; + @Mock private AssetRepository assetRepository; + @Mock private AssetService assetService; + @Mock private ContextFileTextExtractor textExtractor; + + @Captor private ArgumentCaptor updatedFileCaptor; + @Captor private ArgumentCaptor updatedContentCaptor; + + private UUID fileId; + private UUID contentId; + private ContextFile file; + private ContextFileContent content; + private Asset asset; + + @BeforeEach + void setUp() { + fileId = UUID.randomUUID(); + contentId = UUID.randomUUID(); + + file = + new ContextFile() + .withId(fileId) + .withName("report") + .withFileType(ContextFileType.PDF) + .withFileExtension("pdf") + .withHeadContentId(contentId.toString()) + .withProcessingStatus(ProcessingStatus.Uploaded); + + content = + new ContextFileContent() + .withId(contentId) + .withName("v1") + .withAssetId("asset-1") + .withContextFile(file.getEntityReference()) + .withProcessingStatus(ProcessingStatus.Uploaded); + + asset = new Asset(); + asset.setId("asset-1"); + + lenient().when(repository.getContentRepository()).thenReturn(contentRepository); + lenient().when(repository.getAssetRepository()).thenReturn(assetRepository); + when(repository.get(isNull(), eq(fileId), any(), eq(Include.NON_DELETED), eq(false))) + .thenReturn(file); + lenient().when(contentRepository.getById(contentId)).thenReturn(content); + lenient().when(assetRepository.getById("asset-1")).thenReturn(asset); + } + + @Test + void processSuccessMarksAnalyzingThenProcessed() throws Exception { + when(assetService.read(asset)) + .thenReturn( + CompletableFuture.completedFuture( + new ByteArrayInputStream("Quarterly results".getBytes()))); + when(textExtractor.extract(any(InputStream.class), same(file))) + .thenReturn(ContextFileTextExtractor.ExtractionResult.processed("Quarterly results", 3)); + + service(Runnable::run, () -> assetService).process(fileId, contentId); + + verify(repository, times(2)) + .update(isNull(), same(file), updatedFileCaptor.capture(), anyString()); + verify(contentRepository, times(2)) + .update(isNull(), same(content), updatedContentCaptor.capture(), anyString()); + + List fileUpdates = updatedFileCaptor.getAllValues(); + assertEquals(ProcessingStatus.Analyzing, fileUpdates.get(0).getProcessingStatus()); + assertEquals(ProcessingStatus.Processed, fileUpdates.get(1).getProcessingStatus()); + assertEquals("Quarterly results", fileUpdates.get(1).getExtractedText()); + assertEquals(3, fileUpdates.get(1).getPageCount()); + + List contentUpdates = updatedContentCaptor.getAllValues(); + assertEquals(ProcessingStatus.Analyzing, contentUpdates.get(0).getProcessingStatus()); + assertNull(contentUpdates.get(0).getProcessingError()); + assertEquals(ProcessingStatus.Processed, contentUpdates.get(1).getProcessingStatus()); + assertEquals("Quarterly results", contentUpdates.get(1).getExtractedText()); + } + + @Test + void processMarksFailureWhenObjectStorageIsUnavailable() { + service(Runnable::run, () -> null).process(fileId, contentId); + + verifyFailedWith("Object storage is not configured for text extraction"); + } + + @Test + void processMarksFailureWhenStorageReadReturnsNullStream() { + when(assetService.read(asset)).thenReturn(CompletableFuture.completedFuture(null)); + + service(Runnable::run, () -> assetService).process(fileId, contentId); + + verifyFailedWith("Unable to read file content from object storage"); + } + + @Test + void submitMarksFailureWhenExecutorRejectsWork() { + Executor rejectingExecutor = + task -> { + throw new RejectedExecutionException("queue full"); + }; + + service(rejectingExecutor, () -> assetService).submit(fileId, contentId); + + verifyImmediateFailureWith("Text extraction queue is full. Please retry later."); + verify(assetService, never()).read(any()); + } + + @Test + void processSkipsWhenHeadContentNoLongerMatches() { + file.setHeadContentId(UUID.randomUUID().toString()); + + service(Runnable::run, () -> assetService).process(fileId, contentId); + + verify(repository, never()).update(any(), any(), any(), anyString()); + verify(contentRepository, never()).update(any(), any(), any(), anyString()); + verify(assetService, never()).read(any()); + } + + @Test + void processRethrowsVirtualMachineErrors() throws Exception { + when(assetService.read(asset)) + .thenReturn( + CompletableFuture.completedFuture(new ByteArrayInputStream(new byte[] {1, 2, 3}))); + when(textExtractor.extract(any(InputStream.class), same(file))) + .thenThrow(new InternalError("fatal")); + + assertThrows( + InternalError.class, + () -> service(Runnable::run, () -> assetService).process(fileId, contentId)); + } + + private void verifyFailedWith(String expectedReason) { + verify(repository, times(2)) + .update(isNull(), same(file), updatedFileCaptor.capture(), anyString()); + verify(contentRepository, times(2)) + .update(isNull(), same(content), updatedContentCaptor.capture(), anyString()); + + List fileUpdates = updatedFileCaptor.getAllValues(); + assertEquals(ProcessingStatus.Analyzing, fileUpdates.get(0).getProcessingStatus()); + assertEquals(ProcessingStatus.Failed, fileUpdates.get(1).getProcessingStatus()); + assertNull(fileUpdates.get(1).getExtractedText()); + assertNull(fileUpdates.get(1).getPageCount()); + + List contentUpdates = updatedContentCaptor.getAllValues(); + assertEquals(ProcessingStatus.Analyzing, contentUpdates.get(0).getProcessingStatus()); + assertEquals(ProcessingStatus.Failed, contentUpdates.get(1).getProcessingStatus()); + assertEquals(expectedReason, contentUpdates.get(1).getProcessingError()); + assertNull(contentUpdates.get(1).getExtractedText()); + } + + private void verifyImmediateFailureWith(String expectedReason) { + verify(repository).update(isNull(), same(file), updatedFileCaptor.capture(), anyString()); + verify(contentRepository) + .update(isNull(), same(content), updatedContentCaptor.capture(), anyString()); + + ContextFile fileUpdate = updatedFileCaptor.getValue(); + assertEquals(ProcessingStatus.Failed, fileUpdate.getProcessingStatus()); + assertNull(fileUpdate.getExtractedText()); + assertNull(fileUpdate.getPageCount()); + + ContextFileContent contentUpdate = updatedContentCaptor.getValue(); + assertEquals(ProcessingStatus.Failed, contentUpdate.getProcessingStatus()); + assertEquals(expectedReason, contentUpdate.getProcessingError()); + assertNull(contentUpdate.getExtractedText()); + } + + private ContextFileExtractionService service( + Executor executor, Supplier assetServiceSupplier) { + return new ContextFileExtractionService( + repository, assetServiceSupplier, executor, textExtractor); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/drive/ContextFileTextExtractorTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/drive/ContextFileTextExtractorTest.java new file mode 100644 index 000000000000..c23e02279084 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/drive/ContextFileTextExtractorTest.java @@ -0,0 +1,248 @@ +package org.openmetadata.service.drive; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.UUID; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.PDPageContentStream; +import org.apache.pdfbox.pdmodel.font.PDType1Font; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.junit.jupiter.api.Test; +import org.openmetadata.schema.entity.data.ContextFile; +import org.openmetadata.schema.entity.data.ContextFileType; +import org.openmetadata.schema.entity.data.ProcessingStatus; + +class ContextFileTextExtractorTest { + + private final ContextFileTextExtractor extractor = new ContextFileTextExtractor(); + + @Test + void extractPlainTextMarksFileProcessed() throws Exception { + ContextFile file = + new ContextFile() + .withId(UUID.randomUUID()) + .withName("notes") + .withFileType(ContextFileType.Text); + byte[] content = "Context Center remembers this note".getBytes(StandardCharsets.UTF_8); + + ContextFileTextExtractor.ExtractionResult result = + extractor.extract(new ByteArrayInputStream(content), file); + + assertEquals(ProcessingStatus.Processed, result.processingStatus()); + assertEquals("Context Center remembers this note", result.extractedText()); + assertEquals(result.extractedText(), result.indexedText()); + assertNull(result.pageCount()); + } + + @Test + void extractPdfReturnsTextAndPageCount() throws Exception { + ContextFile file = + new ContextFile() + .withId(UUID.randomUUID()) + .withName("report") + .withFileType(ContextFileType.PDF) + .withFileExtension("pdf"); + + ContextFileTextExtractor.ExtractionResult result = + extractor.extract(new ByteArrayInputStream(createPdf("Quarterly PDF Fixture")), file); + + assertEquals(ProcessingStatus.Processed, result.processingStatus()); + assertTrue(result.extractedText().contains("Quarterly PDF Fixture")); + assertEquals(1, result.pageCount()); + } + + @Test + void extractSpreadsheetReturnsSheetTextAndCount() throws Exception { + ContextFile file = + new ContextFile() + .withId(UUID.randomUUID()) + .withName("pricing") + .withFileType(ContextFileType.Spreadsheet) + .withFileExtension("xlsx"); + + ContextFileTextExtractor.ExtractionResult result = + extractor.extract(new ByteArrayInputStream(createWorkbook()), file); + + assertEquals(ProcessingStatus.Processed, result.processingStatus()); + assertTrue(result.extractedText().contains("Sheet: Pricing")); + assertTrue(result.extractedText().contains("Widget")); + assertEquals(1, result.pageCount()); + } + + @Test + void extractImageUsesConfiguredOcrEngine() throws Exception { + ContextFileTextExtractor extractor = + new ContextFileTextExtractor( + new ContextFileTextExtractor.ImageOcrEngine() { + @Override + public boolean isAvailable() { + return true; + } + + @Override + public String extract(java.nio.file.Path imagePath) { + return "Revenue chart shows regional growth"; + } + }); + ContextFile file = + new ContextFile() + .withId(UUID.randomUUID()) + .withName("diagram") + .withFileType(ContextFileType.Image) + .withFileExtension("png"); + + ContextFileTextExtractor.ExtractionResult result = + extractor.extract(new ByteArrayInputStream(new byte[] {1, 2, 3}), file); + + assertEquals(ProcessingStatus.Processed, result.processingStatus()); + assertEquals("Revenue chart shows regional growth", result.extractedText()); + assertEquals(result.extractedText(), result.indexedText()); + assertEquals(1, result.pageCount()); + } + + @Test + void extractImageReturnsUnsupportedWhenOcrUnavailable() throws Exception { + ContextFileTextExtractor extractor = + new ContextFileTextExtractor( + new ContextFileTextExtractor.ImageOcrEngine() { + @Override + public boolean isAvailable() { + return false; + } + + @Override + public String extract(java.nio.file.Path imagePath) { + throw new UnsupportedOperationException("OCR should not run when unavailable"); + } + }); + ContextFile file = + new ContextFile() + .withId(UUID.randomUUID()) + .withName("diagram") + .withFileType(ContextFileType.Image) + .withFileExtension("png"); + + ContextFileTextExtractor.ExtractionResult result = + extractor.extract(new ByteArrayInputStream(new byte[] {1, 2, 3}), file); + + assertEquals(ProcessingStatus.Unsupported, result.processingStatus()); + assertNull(result.extractedText()); + assertTrue(result.processingError().contains("OCR")); + } + + @Test + void extractImageUsesConfiguredTikaTesseractPathOverride() throws Exception { + String originalPath = System.getProperty(ContextFileTextExtractor.TIKA_TESSERACT_PATH_PROPERTY); + Path fakeTesseractHome = createFakeTesseractHome("Revenue chart shows regional growth"); + System.setProperty( + ContextFileTextExtractor.TIKA_TESSERACT_PATH_PROPERTY, fakeTesseractHome.toString()); + + try { + ContextFileTextExtractor extractor = + new ContextFileTextExtractor(new ContextFileTextExtractor.TesseractImageOcrEngine()); + ContextFile file = + new ContextFile() + .withId(UUID.randomUUID()) + .withName("diagram") + .withFileType(ContextFileType.Image) + .withFileExtension("png"); + + ContextFileTextExtractor.ExtractionResult result = + extractor.extract(new ByteArrayInputStream(new byte[] {1, 2, 3}), file); + + assertEquals(ProcessingStatus.Processed, result.processingStatus()); + assertEquals("Revenue chart shows regional growth", result.extractedText()); + assertEquals(result.extractedText(), result.indexedText()); + } finally { + if (originalPath == null) { + System.clearProperty(ContextFileTextExtractor.TIKA_TESSERACT_PATH_PROPERTY); + } else { + System.setProperty(ContextFileTextExtractor.TIKA_TESSERACT_PATH_PROPERTY, originalPath); + } + deleteRecursively(fakeTesseractHome); + } + } + + @Test + void processedResultsTruncateIndexedTextBeforeCanonicalText() { + String text = "x".repeat(ContextFileTextExtractor.MAX_CANONICAL_TEXT_LENGTH + 100); + + ContextFileTextExtractor.ExtractionResult result = + ContextFileTextExtractor.ExtractionResult.processed(text, null); + + assertEquals( + ContextFileTextExtractor.MAX_CANONICAL_TEXT_LENGTH, result.extractedText().length()); + assertEquals(ContextFileTextExtractor.MAX_INDEXED_TEXT_LENGTH, result.indexedText().length()); + } + + private byte[] createPdf(String text) throws IOException { + try (PDDocument document = new PDDocument(); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + PDPage page = new PDPage(); + document.addPage(page); + try (PDPageContentStream contentStream = new PDPageContentStream(document, page)) { + contentStream.beginText(); + contentStream.setFont(PDType1Font.HELVETICA_BOLD, 12); + contentStream.newLineAtOffset(72, 720); + contentStream.showText(text); + contentStream.endText(); + } + document.save(outputStream); + return outputStream.toByteArray(); + } + } + + private byte[] createWorkbook() throws IOException { + try (Workbook workbook = new XSSFWorkbook(); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + var sheet = workbook.createSheet("Pricing"); + var header = sheet.createRow(0); + header.createCell(0).setCellValue("Item"); + header.createCell(1).setCellValue("Price"); + var row = sheet.createRow(1); + row.createCell(0).setCellValue("Widget"); + row.createCell(1).setCellValue(42); + workbook.write(outputStream); + return outputStream.toByteArray(); + } + } + + private Path createFakeTesseractHome(String extractedText) throws IOException { + Path home = Files.createTempDirectory("fake-tesseract-home-"); + Path executable = home.resolve("tesseract"); + Files.writeString( + executable, + "#!/bin/sh\n" + + "if [ $# -eq 0 ] || [ \"$1\" = \"--version\" ]; then\n" + + " echo \"tesseract 5.0.0\"\n" + + " exit 0\n" + + "fi\n" + + "output_base=\"$2\"\n" + + "printf '%s\\n' \"" + + extractedText + + "\" > \"${output_base}.txt\"\n", + StandardCharsets.UTF_8); + executable.toFile().setExecutable(true); + return home; + } + + private void deleteRecursively(Path root) throws IOException { + if (root == null || Files.notExists(root)) { + return; + } + try (var paths = Files.walk(root)) { + paths.sorted(Comparator.reverseOrder()).forEach(path -> path.toFile().delete()); + } + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/drive/ContextFileResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/drive/ContextFileResourceTest.java new file mode 100644 index 000000000000..fb0859ba038e --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/drive/ContextFileResourceTest.java @@ -0,0 +1,119 @@ +package org.openmetadata.service.resources.drive; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class ContextFileResourceTest { + + // ------------------------------------------------------------------ + // sanitizeFileName + // ------------------------------------------------------------------ + + @Test + void testSanitizeFileName_normalName() { + assertEquals("report.pdf", ContextFileResource.sanitizeFileName("report.pdf")); + } + + @Test + void testSanitizeFileName_removesDoubleQuotes() { + assertEquals("file_name_.pdf", ContextFileResource.sanitizeFileName("file\"name\".pdf")); + } + + @Test + void testSanitizeFileName_removesBackslashes() { + assertEquals("path_to_file.txt", ContextFileResource.sanitizeFileName("path\\to\\file.txt")); + } + + @Test + void testSanitizeFileName_removesNewlines() { + assertEquals("file_name.txt", ContextFileResource.sanitizeFileName("file\nname.txt")); + } + + @Test + void testSanitizeFileName_removesCarriageReturns() { + assertEquals("file_name.txt", ContextFileResource.sanitizeFileName("file\rname.txt")); + } + + @Test + void testSanitizeFileName_combinedInjection() { + assertEquals("a_b_c_d_e.txt", ContextFileResource.sanitizeFileName("a\"b\\c\rd\ne.txt")); + } + + @Test + void testSanitizeFileName_nullFallback() { + assertEquals("download", ContextFileResource.sanitizeFileName(null)); + } + + @Test + void testSanitizeFileName_blankFallback() { + assertEquals("download", ContextFileResource.sanitizeFileName(" ")); + } + + // ------------------------------------------------------------------ + // buildContentDisposition + // ------------------------------------------------------------------ + + @Test + void testBuildContentDisposition_asciiName() { + assertEquals( + "attachment; filename=\"report.pdf\"; filename*=UTF-8''report.pdf", + ContextFileResource.buildContentDisposition("report.pdf")); + } + + @Test + void testBuildContentDisposition_encodesUnicode() { + // Non-ASCII characters must be percent-encoded per RFC 5987. + assertEquals( + "attachment; filename=\"héllo.txt\"; filename*=UTF-8''h%C3%A9llo.txt", + ContextFileResource.buildContentDisposition("héllo.txt")); + } + + @Test + void testBuildContentDisposition_encodesSpacesAsPercent20() { + assertEquals( + "attachment; filename=\"my file.txt\"; filename*=UTF-8''my%20file.txt", + ContextFileResource.buildContentDisposition("my file.txt")); + } + + @Test + void testBuildContentDisposition_stripsInjectionCharacters() { + assertEquals( + "attachment; filename=\"_evil_.txt\"; filename*=UTF-8''_evil_.txt", + ContextFileResource.buildContentDisposition("\"evil\".txt")); + } + + // ------------------------------------------------------------------ + // clampExpiry + // ------------------------------------------------------------------ + + @Test + void testClampExpiry_normalValue() { + assertEquals(300, ContextFileResource.clampExpiry(300)); + } + + @Test + void testClampExpiry_zeroClampedToOne() { + assertEquals(1, ContextFileResource.clampExpiry(0)); + } + + @Test + void testClampExpiry_negativeClampedToOne() { + assertEquals(1, ContextFileResource.clampExpiry(-100)); + } + + @Test + void testClampExpiry_exactMax() { + assertEquals(3600, ContextFileResource.clampExpiry(3600)); + } + + @Test + void testClampExpiry_exceedsMaxClampedToMax() { + assertEquals(3600, ContextFileResource.clampExpiry(999999999)); + } + + @Test + void testClampExpiry_intMaxClampedToMax() { + assertEquals(3600, ContextFileResource.clampExpiry(Integer.MAX_VALUE)); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/drive/ContextFileUploadSupportTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/drive/ContextFileUploadSupportTest.java new file mode 100644 index 000000000000..a58338332b8d --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/drive/ContextFileUploadSupportTest.java @@ -0,0 +1,97 @@ +package org.openmetadata.service.resources.drive; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.UUID; +import org.junit.jupiter.api.Test; +import org.openmetadata.schema.attachments.Asset; +import org.openmetadata.schema.entity.data.ContextFile; +import org.openmetadata.schema.entity.data.ContextFileContent; +import org.openmetadata.schema.entity.data.ContextFileType; + +class ContextFileUploadSupportTest { + + @Test + void detectFileTypeUsesMimeMappings() { + assertEquals(ContextFileType.PDF, ContextFileUploadSupport.detectFileType("application/pdf")); + assertEquals( + ContextFileType.Spreadsheet, + ContextFileUploadSupport.detectFileType( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")); + assertEquals(ContextFileType.Image, ContextFileUploadSupport.detectFileType("image/png")); + assertEquals(ContextFileType.CSV, ContextFileUploadSupport.detectFileType("text/csv")); + assertEquals( + ContextFileType.Other, ContextFileUploadSupport.detectFileType("application/octet-stream")); + } + + @Test + void sanitizeEntityNameProducesBoundedUniqueName() { + String name = ContextFileUploadSupport.sanitizeEntityName("Quarterly Report (Final).pdf"); + assertTrue(name.startsWith("quarterly_report_final_.pdf_")); + assertTrue(name.length() <= 189); + } + + @Test + void exceedsMaxFileSizeHonorsConfiguredLimit() { + assertTrue(ContextFileUploadSupport.exceedsMaxFileSize(1025, 1024)); + assertTrue(!ContextFileUploadSupport.exceedsMaxFileSize(1024, 1024)); + assertTrue(!ContextFileUploadSupport.exceedsMaxFileSize(2048, 0)); + } + + @Test + void buildAssetAndContentCarryCanonicalFileIdentity() { + ContextFile file = + new ContextFile() + .withId(UUID.randomUUID()) + .withName("q1-report") + .withFullyQualifiedName("finance.q1-report"); + byte[] bytes = "hello world".getBytes(StandardCharsets.UTF_8); + + Asset asset = + ContextFileUploadSupport.buildAsset( + file, "Q1 Report.pdf", "application/pdf", "pdf", bytes.length, "admin"); + ContextFileContent content = + ContextFileUploadSupport.buildContent( + file, asset, ContextFileUploadSupport.sha256(bytes), "admin"); + + assertNotNull(asset.getId()); + assertEquals("<#E::contextFile::finance.q1-report>", asset.getEntityLink()); + assertEquals(file.getEntityReference(), content.getContextFile()); + assertEquals(asset.getId(), content.getAssetId()); + assertEquals(ContextFileUploadSupport.sha256(bytes), content.getChecksum()); + assertTrue(content.getName().startsWith("q1-report_content_")); + } + + @Test + void bufferUploadStreamsToTempFileAndComputesChecksum() throws Exception { + byte[] bytes = "streamed payload".getBytes(StandardCharsets.UTF_8); + + try (ContextFileUploadSupport.BufferedUpload bufferedUpload = + ContextFileUploadSupport.bufferUpload(new ByteArrayInputStream(bytes), 1024)) { + assertEquals(bytes.length, bufferedUpload.getSize()); + assertEquals(ContextFileUploadSupport.sha256(bytes), bufferedUpload.getChecksum()); + try (var inputStream = bufferedUpload.newInputStream()) { + assertArrayEquals(bytes, inputStream.readAllBytes()); + } + } + } + + @Test + void bufferUploadRejectsOversizedFiles() { + byte[] bytes = "too-large".getBytes(StandardCharsets.UTF_8); + + ContextFileUploadSupport.MaxFileSizeExceededException ex = + assertThrows( + ContextFileUploadSupport.MaxFileSizeExceededException.class, + () -> ContextFileUploadSupport.bufferUpload(new ByteArrayInputStream(bytes), 3)); + + assertEquals(bytes.length, ex.getActualSize()); + assertEquals(3, ex.getMaxFileSize()); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/drive/DriveMapperTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/drive/DriveMapperTest.java new file mode 100644 index 000000000000..0f60b001930f --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/drive/DriveMapperTest.java @@ -0,0 +1,45 @@ +package org.openmetadata.service.resources.drive; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.openmetadata.service.jdbi3.FolderRepository.FOLDER_ENTITY; + +import org.junit.jupiter.api.Test; +import org.openmetadata.schema.api.data.CreateContextFile; +import org.openmetadata.schema.api.data.CreateFolder; +import org.openmetadata.schema.entity.data.ContextFile; +import org.openmetadata.schema.entity.data.ContextFileType; +import org.openmetadata.schema.entity.data.Folder; +import org.openmetadata.schema.entity.data.ProcessingStatus; + +class DriveMapperTest { + + @Test + void folderMapperCarriesParentReference() { + Folder folder = + new FolderMapper() + .createToEntity( + new CreateFolder().withName("child-folder").withParent("root-folder"), "admin"); + + assertNotNull(folder.getParent()); + assertEquals(FOLDER_ENTITY, folder.getParent().getType()); + assertEquals("root-folder", folder.getParent().getFullyQualifiedName()); + } + + @Test + void contextFileMapperCarriesFolderReference() { + ContextFile file = + new ContextFileMapper() + .createToEntity( + new CreateContextFile() + .withName("report") + .withFolder("root-folder.child-folder") + .withFileType(ContextFileType.PDF) + .withProcessingStatus(ProcessingStatus.Uploaded), + "admin"); + + assertNotNull(file.getFolder()); + assertEquals(FOLDER_ENTITY, file.getFolder().getType()); + assertEquals("root-folder.child-folder", file.getFolder().getFullyQualifiedName()); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/IndexMappingNestedFieldConsistencyTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/IndexMappingNestedFieldConsistencyTest.java index 90a532f6deb1..a4d0a8ac6080 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/search/IndexMappingNestedFieldConsistencyTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/IndexMappingNestedFieldConsistencyTest.java @@ -68,6 +68,37 @@ void extensionFieldMustBeFlattenedInAllIndices() { + violations); } + @Test + void taggableIndexFieldsMustAppearTogether() { + List violations = new ArrayList<>(); + for (Map.Entry entry : allMappings.entrySet()) { + String entity = entry.getKey(); + JsonNode properties = getTopLevelProperties(entry.getValue()); + assertNotNull( + properties, + "Index mapping for '" + entity + "' has no properties — mapping file may be malformed."); + boolean hasClassificationTags = properties.has("classificationTags"); + boolean hasGlossaryTags = properties.has("glossaryTags"); + if (hasClassificationTags != hasGlossaryTags) { + violations.add( + entity + + " (classificationTags=" + + hasClassificationTags + + ", glossaryTags=" + + hasGlossaryTags + + ")"); + } + } + assertTrue( + violations.isEmpty(), + "Indexes whose backing index class implements TaggableIndex must define both " + + "'classificationTags' and 'glossaryTags' as top-level keyword fields. " + + "TaggableIndex.applyTagFields() writes both into every doc; if the mapping omits " + + "one, OpenSearch dynamic-maps it as text and aggregations/sorts/scripts on it fail " + + "at reindex time. Violations: " + + violations); + } + @Test void ownersFieldMustBeNestedInAllIndices() { List violations = new ArrayList<>(); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/ContextFileIndexTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/ContextFileIndexTest.java new file mode 100644 index 000000000000..6d7d5d0813bc --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/ContextFileIndexTest.java @@ -0,0 +1,89 @@ +package org.openmetadata.service.search.indexes; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.Mockito.mock; +import static org.openmetadata.service.jdbi3.ContextFileRepository.CONTEXT_FILE_ENTITY; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.openmetadata.schema.entity.data.ContextFile; +import org.openmetadata.schema.entity.data.ContextFileType; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Votes; +import org.openmetadata.service.Entity; +import org.openmetadata.service.search.SearchRepository; + +class ContextFileIndexTest { + + private static MockedStatic entityStaticMock; + + @BeforeAll + static void setUp() { + SearchRepository mockSearchRepo = mock(SearchRepository.class, Mockito.RETURNS_DEEP_STUBS); + entityStaticMock = Mockito.mockStatic(Entity.class); + entityStaticMock.when(Entity::getSearchRepository).thenReturn(mockSearchRepo); + } + + @AfterAll + static void tearDown() { + entityStaticMock.close(); + } + + @Test + void testGetEntityTypeName() { + ContextFile file = new ContextFile().withId(UUID.randomUUID()).withName("file"); + assertEquals(CONTEXT_FILE_ENTITY, new ContextFileIndex(file).getEntityTypeName()); + } + + @Test + void testGetEntity() { + ContextFile file = new ContextFile().withId(UUID.randomUUID()).withName("file"); + assertEquals(file, new ContextFileIndex(file).getEntity()); + } + + @Test + void testBuildSearchIndexDocInternal_setsEntitySpecificFieldsOnly() { + EntityReference owner = + new EntityReference().withId(UUID.randomUUID()).withType("user").withName("admin"); + EntityReference folder = + new EntityReference() + .withId(UUID.randomUUID()) + .withType("folder") + .withName("docs") + .withDisplayName("Docs"); + + ContextFile file = + new ContextFile() + .withId(UUID.randomUUID()) + .withName("quarterly-report") + .withFullyQualifiedName("docs.quarterly-report") + .withOwners(List.of(owner)) + .withFolder(folder) + .withFileType(ContextFileType.PDF) + .withVotes(new Votes().withUpVotes(3).withDownVotes(1)); + + Map result = + new ContextFileIndex(file).buildSearchIndexDocInternal(new HashMap<>()); + + // Common fields (entityType, deleted, owners, totalVotes) are handled by + // populateCommonFields in the SearchIndex template method, not in + // buildSearchIndexDocInternal. See PageIndexTest for the same convention. + assertFalse(result.containsKey("entityType")); + assertFalse(result.containsKey("deleted")); + assertFalse(result.containsKey("owners")); + assertFalse(result.containsKey("totalVotes")); + + // Entity-specific fields + assertEquals(ContextFileType.PDF, result.get("fileType")); + assertNotNull(result.get("folder")); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/FolderIndexTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/FolderIndexTest.java new file mode 100644 index 000000000000..cc8be6444c57 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/FolderIndexTest.java @@ -0,0 +1,85 @@ +package org.openmetadata.service.search.indexes; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.Mockito.mock; +import static org.openmetadata.service.jdbi3.FolderRepository.FOLDER_ENTITY; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.openmetadata.schema.entity.data.Folder; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.service.Entity; +import org.openmetadata.service.search.SearchRepository; + +class FolderIndexTest { + + private static MockedStatic entityStaticMock; + + @BeforeAll + static void setUp() { + SearchRepository mockSearchRepo = mock(SearchRepository.class, Mockito.RETURNS_DEEP_STUBS); + entityStaticMock = Mockito.mockStatic(Entity.class); + entityStaticMock.when(Entity::getSearchRepository).thenReturn(mockSearchRepo); + } + + @AfterAll + static void tearDown() { + entityStaticMock.close(); + } + + @Test + void testGetEntityTypeName() { + Folder folder = new Folder().withId(UUID.randomUUID()).withName("folder"); + assertEquals(FOLDER_ENTITY, new FolderIndex(folder).getEntityTypeName()); + } + + @Test + void testGetEntity() { + Folder folder = new Folder().withId(UUID.randomUUID()).withName("folder"); + assertEquals(folder, new FolderIndex(folder).getEntity()); + } + + @Test + void testBuildSearchIndexDocInternal_setsEntitySpecificFieldsOnly() { + EntityReference owner = + new EntityReference().withId(UUID.randomUUID()).withType("user").withName("admin"); + EntityReference parent = + new EntityReference() + .withId(UUID.randomUUID()) + .withType(FOLDER_ENTITY) + .withName("parent") + .withDisplayName("Parent"); + + Folder folder = + new Folder() + .withId(UUID.randomUUID()) + .withName("child") + .withFullyQualifiedName("parent.child") + .withOwners(List.of(owner)) + .withParent(parent) + .withChildrenCount(2); + + Map result = + new FolderIndex(folder).buildSearchIndexDocInternal(new HashMap<>()); + + // Common fields (entityType, deleted, owners) are handled by populateCommonFields in the + // SearchIndex template method, not in buildSearchIndexDocInternal. See PageIndexTest for + // the same convention. + assertFalse(result.containsKey("entityType")); + assertFalse(result.containsKey("deleted")); + assertFalse(result.containsKey("owners")); + + // Entity-specific fields + assertEquals(2, result.get("childrenCount")); + assertNotNull(result.get("parent")); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/PageIndexTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/PageIndexTest.java new file mode 100644 index 000000000000..5f43dbaba360 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/PageIndexTest.java @@ -0,0 +1,114 @@ +package org.openmetadata.service.search.indexes; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.mockito.Mockito.mock; +import static org.openmetadata.service.jdbi3.KnowledgePageRepository.KNOWLEDGE_PAGE_ENTITY; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.openmetadata.schema.entity.data.Page; +import org.openmetadata.service.Entity; +import org.openmetadata.service.search.SearchRepository; + +class PageIndexTest { + + private static MockedStatic entityStaticMock; + + @BeforeAll + static void setUp() { + SearchRepository mockSearchRepo = mock(SearchRepository.class, Mockito.RETURNS_DEEP_STUBS); + entityStaticMock = Mockito.mockStatic(Entity.class); + entityStaticMock.when(Entity::getSearchRepository).thenReturn(mockSearchRepo); + } + + @AfterAll + static void tearDown() { + entityStaticMock.close(); + } + + @Test + void testGetEntityTypeName() { + Page page = new Page().withId(UUID.randomUUID()).withName("p"); + assertEquals(KNOWLEDGE_PAGE_ENTITY, new PageIndex(page).getEntityTypeName()); + } + + @Test + void testGetEntity() { + Page page = new Page().withId(UUID.randomUUID()).withName("p"); + assertEquals(page, new PageIndex(page).getEntity()); + } + + @Test + void testBuildSearchIndexDocInternal_setsFqnDepth() { + Page page = + new Page() + .withId(UUID.randomUUID()) + .withName("child") + .withFullyQualifiedName("root.parent.child"); + + Map doc = new HashMap<>(); + Map result = new PageIndex(page).buildSearchIndexDocInternal(doc); + + assertEquals(3, result.get("fqnDepth")); + } + + @Test + void testBuildSearchIndexDocInternal_setsDeletedFalse() { + Page page = new Page().withId(UUID.randomUUID()).withName("p").withFullyQualifiedName("root.p"); + + Map doc = new HashMap<>(); + Map result = new PageIndex(page).buildSearchIndexDocInternal(doc); + + assertEquals(Boolean.FALSE, result.get("deleted")); + } + + @Test + void testBuildSearchIndexDocInternal_doesNotSetCommonFields() { + Page page = new Page().withId(UUID.randomUUID()).withName("p").withFullyQualifiedName("root.p"); + + Map doc = new HashMap<>(); + Map result = new PageIndex(page).buildSearchIndexDocInternal(doc); + + // Common fields are now auto-handled by the template method + assertFalse(result.containsKey("owners")); + assertFalse(result.containsKey("entityType")); + assertFalse(result.containsKey("followers")); + assertFalse(result.containsKey("totalVotes")); + // Only entity-specific fields + assertEquals(2, result.size()); // fqnDepth + deleted + } + + @Test + void testFqnDepth_singlePart() { + Page page = + new Page().withId(UUID.randomUUID()).withName("root").withFullyQualifiedName("root"); + + Map doc = new HashMap<>(); + Map result = new PageIndex(page).buildSearchIndexDocInternal(doc); + + assertEquals(1, result.get("fqnDepth")); + } + + @Test + void testFqnDepth_nullFqn() { + Page page = new Page().withId(UUID.randomUUID()).withName("p"); + + PageIndex index = new PageIndex(page); + assertEquals(0, index.calculateFqnDepth(null)); + } + + @Test + void testFqnDepth_emptyFqn() { + Page page = new Page().withId(UUID.randomUUID()).withName("p"); + + PageIndex index = new PageIndex(page); + assertEquals(0, index.calculateFqnDepth("")); + } +} diff --git a/openmetadata-spec/src/main/antlr4/org/openmetadata/schema/EntityLink.g4 b/openmetadata-spec/src/main/antlr4/org/openmetadata/schema/EntityLink.g4 index 62ac7a9b4c51..b07dff497711 100644 --- a/openmetadata-spec/src/main/antlr4/org/openmetadata/schema/EntityLink.g4 +++ b/openmetadata-spec/src/main/antlr4/org/openmetadata/schema/EntityLink.g4 @@ -99,6 +99,9 @@ ENTITY_TYPE | 'query' | 'directory' | 'file' + | 'folder' + | 'contextFile' + | 'contextFileContent' | 'type' | 'aiApplication' | 'llmModel' diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/context_file_search_index.json b/openmetadata-spec/src/main/resources/elasticsearch/en/context_file_search_index.json new file mode 100644 index 000000000000..3763cfa244e9 --- /dev/null +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/context_file_search_index.json @@ -0,0 +1,859 @@ +{ + "settings": { + "index": { + "max_ngram_diff": 17 + }, + "analysis": { + "tokenizer": { + "n_gram_tokenizer": { + "type": "ngram", + "min_gram": 3, + "max_gram": 20, + "token_chars": [ + "letter", + "digit" + ] + } + }, + "normalizer": { + "lowercase_normalizer": { + "type": "custom", + "char_filter": [], + "filter": [ + "lowercase" + ] + } + }, + "analyzer": { + "om_analyzer": { + "tokenizer": "standard", + "filter": [ + "lowercase", + "word_delimiter_filter", + "om_stemmer" + ] + }, + "om_ngram": { + "type": "custom", + "tokenizer": "n_gram_tokenizer", + "filter": [ + "lowercase" + ] + }, + "om_compound_analyzer": { + "tokenizer": "standard", + "filter": [ + "lowercase", + "compound_word_delimiter_graph", + "flatten_graph" + ] + } + }, + "filter": { + "om_stemmer": { + "type": "stemmer", + "name": "kstem" + }, + "word_delimiter_filter": { + "type": "word_delimiter", + "preserve_original": true + }, + "compound_word_delimiter_graph": { + "type": "word_delimiter_graph", + "generate_word_parts": true, + "generate_number_parts": true, + "split_on_case_change": true, + "split_on_numerics": true, + "catenate_words": false, + "catenate_numbers": false, + "catenate_all": false, + "preserve_original": true, + "stem_english_possessive": true + } + } + } + }, + "mappings": { + "properties": { + "changeDescription": { + "enabled": false + }, + "incrementalChangeDescription": { + "enabled": false + }, + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "name": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256, + "normalizer": "lowercase_normalizer" + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + }, + "compound": { + "type": "text", + "analyzer": "om_compound_analyzer" + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "displayName": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + }, + "actualCase": { + "type": "keyword", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + }, + "compound": { + "type": "text", + "analyzer": "om_compound_analyzer" + } + } + }, + "description": { + "type": "text", + "analyzer": "om_analyzer", + "similarity": "boolean", + "term_vector": "with_positions_offsets" + }, + "serviceType": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "service": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "directory": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "version": { + "type": "float" + }, + "dataProducts": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "updatedAt": { + "type": "date", + "format": "epoch_second" + }, + "updatedBy": { + "type": "text" + }, + "href": { + "type": "text" + }, + "sourceUrl": { + "type": "text" + }, + "fileType": { + "type": "keyword" + }, + "mimeType": { + "type": "keyword" + }, + "fileExtension": { + "type": "keyword" + }, + "processingStatus": { + "type": "keyword" + }, + "sourceType": { + "type": "keyword" + }, + "extractedText": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { "type": "keyword", "ignore_above": 256 } + } + }, + "folder": { + "properties": { + "id": { "type": "keyword" }, + "type": { "type": "keyword" }, + "name": { "type": "keyword", "normalizer": "lowercase_normalizer" }, + "displayName": { "type": "keyword" }, + "fullyQualifiedName": { "type": "keyword", "normalizer": "lowercase_normalizer" } + } + }, + "extension": { + "type": "flattened" + }, + "path": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "driveFileId": { + "type": "keyword" + }, + "size": { + "type": "long" + }, + "checksum": { + "type": "keyword" + }, + "isShared": { + "type": "boolean" + }, + "fileVersion": { + "type": "keyword" + }, + "createdTime": { + "type": "date" + }, + "modifiedTime": { + "type": "date" + }, + "lastModifiedBy": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "entityType": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "entityStatus": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "owners": { + "type": "nested", + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "domains": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "followers": { + "type": "keyword" + }, + "totalVotes": { + "type": "long", + "null_value": 0 + }, + "votes": { + "type": "object", + "dynamic": false, + "properties": { + "upVotes": { + "type": "integer" + }, + "downVotes": { + "type": "integer" + } + } + }, + "descriptionStatus": { + "type": "keyword" + }, + "tags": { + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "text": { + "type": "text", + "analyzer": "om_analyzer" + } + } + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "text": { + "type": "text", + "analyzer": "om_analyzer" + } + } + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "classificationTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "glossaryTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "deleted": { + "type": "boolean" + }, + "fqnParts": { + "type": "keyword" + }, + "descriptionSources": { + "type": "object", + "dynamic": false + }, + "tagSources": { + "type": "object", + "dynamic": false + }, + "tierSources": { + "type": "object", + "dynamic": false + }, + "upstreamLineage": { + "properties": { + "fromEntity": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "toEntity": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "pipeline": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "columns": { + "properties": { + "fromColumns": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "toColumn": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "docId": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "sqlQueryKey": { + "type": "keyword" + } + } + }, + "certification": { + "type": "object", + "properties": { + "tagLabel": { + "type": "object", + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "text": { + "type": "text", + "analyzer": "om_analyzer" + } + } + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "appliedDate": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + }, + "expiryDate": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + } + } + }, + "suggest": { + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] + }, + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "customPropertiesTyped": { + "type": "nested", + "properties": { + "name": { + "type": "keyword" + }, + "propertyType": { + "type": "keyword" + }, + "stringValue": { + "type": "keyword" + }, + "textValue": { + "type": "text", + "analyzer": "om_analyzer" + }, + "longValue": { + "type": "long" + }, + "doubleValue": { + "type": "double" + }, + "start": { + "type": "long" + }, + "end": { + "type": "long" + }, + "refId": { + "type": "keyword" + }, + "refType": { + "type": "keyword" + }, + "refName": { + "type": "keyword" + }, + "refFqn": { + "type": "keyword" + } + } + }, + "fingerprint": { + "type": "keyword" + }, + "textToEmbed": { + "type": "text" + }, + "chunkIndex": { + "type": "integer" + }, + "chunkCount": { + "type": "integer" + }, + "parentId": { + "type": "keyword" + }, + "ownerDisplayName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "ownerName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false + } + } + } +} diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/folder_search_index.json b/openmetadata-spec/src/main/resources/elasticsearch/en/folder_search_index.json new file mode 100644 index 000000000000..7f8da963da1b --- /dev/null +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/folder_search_index.json @@ -0,0 +1,785 @@ +{ + "settings": { + "index": { + "max_ngram_diff": 17 + }, + "analysis": { + "tokenizer": { + "n_gram_tokenizer": { + "type": "ngram", + "min_gram": 3, + "max_gram": 20, + "token_chars": [ + "letter", + "digit" + ] + } + }, + "normalizer": { + "lowercase_normalizer": { + "type": "custom", + "char_filter": [], + "filter": [ + "lowercase" + ] + } + }, + "analyzer": { + "om_analyzer": { + "tokenizer": "standard", + "filter": [ + "lowercase", + "word_delimiter_filter", + "om_stemmer" + ] + }, + "om_ngram": { + "type": "custom", + "tokenizer": "n_gram_tokenizer", + "filter": [ + "lowercase" + ] + }, + "om_compound_analyzer": { + "tokenizer": "standard", + "filter": [ + "lowercase", + "compound_word_delimiter_graph", + "flatten_graph" + ] + } + }, + "filter": { + "om_stemmer": { + "type": "stemmer", + "name": "kstem" + }, + "word_delimiter_filter": { + "type": "word_delimiter", + "preserve_original": true + }, + "compound_word_delimiter_graph": { + "type": "word_delimiter_graph", + "generate_word_parts": true, + "generate_number_parts": true, + "split_on_case_change": true, + "split_on_numerics": true, + "catenate_words": false, + "catenate_numbers": false, + "catenate_all": false, + "preserve_original": true, + "stem_english_possessive": true + } + } + } + }, + "mappings": { + "properties": { + "changeDescription": { + "enabled": false + }, + "incrementalChangeDescription": { + "enabled": false + }, + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "name": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256, + "normalizer": "lowercase_normalizer" + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + }, + "compound": { + "type": "text", + "analyzer": "om_compound_analyzer" + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "displayName": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + }, + "actualCase": { + "type": "keyword", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + }, + "compound": { + "type": "text", + "analyzer": "om_compound_analyzer" + } + } + }, + "description": { + "type": "text", + "analyzer": "om_analyzer", + "similarity": "boolean", + "term_vector": "with_positions_offsets" + }, + "serviceType": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "service": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "version": { + "type": "float" + }, + "dataProducts": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "updatedAt": { + "type": "date", + "format": "epoch_second" + }, + "updatedBy": { + "type": "text" + }, + "href": { + "type": "text" + }, + "sourceUrl": { + "type": "text" + }, + "extension": { + "type": "flattened" + }, + "parent": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "directoryType": { + "type": "keyword" + }, + "path": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "driveId": { + "type": "keyword" + }, + "isShared": { + "type": "boolean" + }, + "numberOfFiles": { + "type": "long" + }, + "numberOfSubDirectories": { + "type": "long" + }, + "totalSize": { + "type": "long" + }, + "entityType": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "entityStatus": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "owners": { + "type": "nested", + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "domains": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "followers": { + "type": "keyword" + }, + "totalVotes": { + "type": "long", + "null_value": 0 + }, + "votes": { + "type": "object", + "dynamic": false, + "properties": { + "upVotes": { + "type": "integer" + }, + "downVotes": { + "type": "integer" + } + } + }, + "descriptionStatus": { + "type": "keyword" + }, + "tags": { + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "text": { + "type": "text", + "analyzer": "om_analyzer" + } + } + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "text": { + "type": "text", + "analyzer": "om_analyzer" + } + } + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "classificationTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "glossaryTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "deleted": { + "type": "boolean" + }, + "fqnParts": { + "type": "keyword" + }, + "descriptionSources": { + "type": "object", + "dynamic": false + }, + "tagSources": { + "type": "object", + "dynamic": false + }, + "tierSources": { + "type": "object", + "dynamic": false + }, + "upstreamLineage": { + "properties": { + "fromEntity": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "toEntity": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "pipeline": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "columns": { + "properties": { + "fromColumns": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "toColumn": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "docId": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "sqlQueryKey": { + "type": "keyword" + } + } + }, + "certification": { + "type": "object", + "properties": { + "tagLabel": { + "type": "object", + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "text": { + "type": "text", + "analyzer": "om_analyzer" + } + } + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "appliedDate": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + }, + "expiryDate": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + } + } + }, + "suggest": { + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] + }, + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "customPropertiesTyped": { + "type": "nested", + "properties": { + "name": { + "type": "keyword" + }, + "propertyType": { + "type": "keyword" + }, + "stringValue": { + "type": "keyword" + }, + "textValue": { + "type": "text", + "analyzer": "om_analyzer" + }, + "longValue": { + "type": "long" + }, + "doubleValue": { + "type": "double" + }, + "start": { + "type": "long" + }, + "end": { + "type": "long" + }, + "refId": { + "type": "keyword" + }, + "refType": { + "type": "keyword" + }, + "refName": { + "type": "keyword" + }, + "refFqn": { + "type": "keyword" + } + } + }, + "fingerprint": { + "type": "keyword" + }, + "textToEmbed": { + "type": "text" + }, + "chunkIndex": { + "type": "integer" + }, + "chunkCount": { + "type": "integer" + }, + "parentId": { + "type": "keyword" + }, + "ownerDisplayName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "ownerName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false + } + } + } +} diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/knowledge_page_search_index.json b/openmetadata-spec/src/main/resources/elasticsearch/en/knowledge_page_search_index.json new file mode 100644 index 000000000000..fa3510aa424f --- /dev/null +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/knowledge_page_search_index.json @@ -0,0 +1,482 @@ +{ + "settings": { + "index": { + "max_ngram_diff": 7 + }, + "analysis": { + "normalizer": { + "lowercase_normalizer": { + "type": "custom", + "char_filter": [], + "filter": ["lowercase"] + } + }, + "tokenizer": { + "om_ngram_tokenizer": { + "type": "ngram", + "min_gram": 3, + "max_gram": 10, + "token_chars": ["letter", "digit"] + } + }, + "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": ["lowercase", "om_stemmer"] + }, + "om_ngram": { + "type": "custom", + "tokenizer": "om_ngram_tokenizer", + "filter": ["lowercase"] + } + }, + "filter": { + "om_stemmer": { + "type": "stemmer", + "name": "english" + } + } + } + }, + "mappings": { + "properties": { + "id": { + "type": "text" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "displayName": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + }, + "actualCase": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "entityType": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "description": { + "type": "text" + }, + "version": { + "type": "float" + }, + "updatedAt": { + "type": "date", + "format": "epoch_second" + }, + "updatedBy": { + "type": "text" + }, + "href": { + "type": "text" + }, + "fqnDepth": { + "type": "integer" + }, + "deleted": { + "type": "boolean" + }, + "owners": { + "type": "nested", + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "reviewers": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "entityStatus": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "followers": { + "type": "keyword" + }, + "tags": { + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "classificationTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "glossaryTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "glossaryTerms": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "pageType" : { + "type": "keyword" + }, + "relatedEntities": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "editors": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "parent" : { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "children" : { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "ownerDisplayName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "ownerName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "fingerprint": { + "type": "keyword" + }, + "textToEmbed": { + "type": "text" + }, + "chunkIndex": { + "type": "integer" + }, + "chunkCount": { + "type": "integer" + }, + "parentId": { + "type": "keyword" + } + } + } +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/elasticsearch/indexMapping.json b/openmetadata-spec/src/main/resources/elasticsearch/indexMapping.json index a9e9513e331c..c444328c7d43 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/indexMapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/indexMapping.json @@ -589,5 +589,26 @@ "mcpServer" ], "childAliases": [] + }, + "page": { + "indexName": "knowledge_page_search_index", + "indexMappingFile": "/elasticsearch/%s/knowledge_page_search_index.json", + "alias": "page", + "parentAliases": ["all", "dataAsset", "dataAssetEmbeddings"], + "childAliases": [] + }, + "folder": { + "indexName": "folder_search_index", + "indexMappingFile": "/elasticsearch/%s/folder_search_index.json", + "alias": "folder", + "parentAliases": ["all"], + "childAliases": [] + }, + "contextFile": { + "indexName": "context_file_search_index", + "indexMappingFile": "/elasticsearch/%s/context_file_search_index.json", + "alias": "contextFile", + "parentAliases": ["all"], + "childAliases": [] } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/context_file_search_index.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/context_file_search_index.json new file mode 100644 index 000000000000..3763cfa244e9 --- /dev/null +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/context_file_search_index.json @@ -0,0 +1,859 @@ +{ + "settings": { + "index": { + "max_ngram_diff": 17 + }, + "analysis": { + "tokenizer": { + "n_gram_tokenizer": { + "type": "ngram", + "min_gram": 3, + "max_gram": 20, + "token_chars": [ + "letter", + "digit" + ] + } + }, + "normalizer": { + "lowercase_normalizer": { + "type": "custom", + "char_filter": [], + "filter": [ + "lowercase" + ] + } + }, + "analyzer": { + "om_analyzer": { + "tokenizer": "standard", + "filter": [ + "lowercase", + "word_delimiter_filter", + "om_stemmer" + ] + }, + "om_ngram": { + "type": "custom", + "tokenizer": "n_gram_tokenizer", + "filter": [ + "lowercase" + ] + }, + "om_compound_analyzer": { + "tokenizer": "standard", + "filter": [ + "lowercase", + "compound_word_delimiter_graph", + "flatten_graph" + ] + } + }, + "filter": { + "om_stemmer": { + "type": "stemmer", + "name": "kstem" + }, + "word_delimiter_filter": { + "type": "word_delimiter", + "preserve_original": true + }, + "compound_word_delimiter_graph": { + "type": "word_delimiter_graph", + "generate_word_parts": true, + "generate_number_parts": true, + "split_on_case_change": true, + "split_on_numerics": true, + "catenate_words": false, + "catenate_numbers": false, + "catenate_all": false, + "preserve_original": true, + "stem_english_possessive": true + } + } + } + }, + "mappings": { + "properties": { + "changeDescription": { + "enabled": false + }, + "incrementalChangeDescription": { + "enabled": false + }, + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "name": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256, + "normalizer": "lowercase_normalizer" + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + }, + "compound": { + "type": "text", + "analyzer": "om_compound_analyzer" + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "displayName": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + }, + "actualCase": { + "type": "keyword", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + }, + "compound": { + "type": "text", + "analyzer": "om_compound_analyzer" + } + } + }, + "description": { + "type": "text", + "analyzer": "om_analyzer", + "similarity": "boolean", + "term_vector": "with_positions_offsets" + }, + "serviceType": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "service": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "directory": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "version": { + "type": "float" + }, + "dataProducts": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "updatedAt": { + "type": "date", + "format": "epoch_second" + }, + "updatedBy": { + "type": "text" + }, + "href": { + "type": "text" + }, + "sourceUrl": { + "type": "text" + }, + "fileType": { + "type": "keyword" + }, + "mimeType": { + "type": "keyword" + }, + "fileExtension": { + "type": "keyword" + }, + "processingStatus": { + "type": "keyword" + }, + "sourceType": { + "type": "keyword" + }, + "extractedText": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { "type": "keyword", "ignore_above": 256 } + } + }, + "folder": { + "properties": { + "id": { "type": "keyword" }, + "type": { "type": "keyword" }, + "name": { "type": "keyword", "normalizer": "lowercase_normalizer" }, + "displayName": { "type": "keyword" }, + "fullyQualifiedName": { "type": "keyword", "normalizer": "lowercase_normalizer" } + } + }, + "extension": { + "type": "flattened" + }, + "path": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "driveFileId": { + "type": "keyword" + }, + "size": { + "type": "long" + }, + "checksum": { + "type": "keyword" + }, + "isShared": { + "type": "boolean" + }, + "fileVersion": { + "type": "keyword" + }, + "createdTime": { + "type": "date" + }, + "modifiedTime": { + "type": "date" + }, + "lastModifiedBy": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "entityType": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "entityStatus": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "owners": { + "type": "nested", + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "domains": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "followers": { + "type": "keyword" + }, + "totalVotes": { + "type": "long", + "null_value": 0 + }, + "votes": { + "type": "object", + "dynamic": false, + "properties": { + "upVotes": { + "type": "integer" + }, + "downVotes": { + "type": "integer" + } + } + }, + "descriptionStatus": { + "type": "keyword" + }, + "tags": { + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "text": { + "type": "text", + "analyzer": "om_analyzer" + } + } + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "text": { + "type": "text", + "analyzer": "om_analyzer" + } + } + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "classificationTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "glossaryTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "deleted": { + "type": "boolean" + }, + "fqnParts": { + "type": "keyword" + }, + "descriptionSources": { + "type": "object", + "dynamic": false + }, + "tagSources": { + "type": "object", + "dynamic": false + }, + "tierSources": { + "type": "object", + "dynamic": false + }, + "upstreamLineage": { + "properties": { + "fromEntity": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "toEntity": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "pipeline": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "columns": { + "properties": { + "fromColumns": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "toColumn": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "docId": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "sqlQueryKey": { + "type": "keyword" + } + } + }, + "certification": { + "type": "object", + "properties": { + "tagLabel": { + "type": "object", + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "text": { + "type": "text", + "analyzer": "om_analyzer" + } + } + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "appliedDate": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + }, + "expiryDate": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + } + } + }, + "suggest": { + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] + }, + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "customPropertiesTyped": { + "type": "nested", + "properties": { + "name": { + "type": "keyword" + }, + "propertyType": { + "type": "keyword" + }, + "stringValue": { + "type": "keyword" + }, + "textValue": { + "type": "text", + "analyzer": "om_analyzer" + }, + "longValue": { + "type": "long" + }, + "doubleValue": { + "type": "double" + }, + "start": { + "type": "long" + }, + "end": { + "type": "long" + }, + "refId": { + "type": "keyword" + }, + "refType": { + "type": "keyword" + }, + "refName": { + "type": "keyword" + }, + "refFqn": { + "type": "keyword" + } + } + }, + "fingerprint": { + "type": "keyword" + }, + "textToEmbed": { + "type": "text" + }, + "chunkIndex": { + "type": "integer" + }, + "chunkCount": { + "type": "integer" + }, + "parentId": { + "type": "keyword" + }, + "ownerDisplayName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "ownerName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false + } + } + } +} diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/folder_search_index.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/folder_search_index.json new file mode 100644 index 000000000000..8ada88658170 --- /dev/null +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/folder_search_index.json @@ -0,0 +1,767 @@ +{ + "settings": { + "analysis": { + "normalizer": { + "lowercase_normalizer": { + "type": "custom", + "char_filter": [], + "filter": [ + "lowercase" + ] + } + }, + "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, + "om_analyzer_jp": { + "tokenizer": "kuromoji_tokenizer", + "type": "custom", + "filter": [ + "kuromoji_baseform", + "kuromoji_part_of_speech", + "kuromoji_number", + "kuromoji_stemmer" + ] + }, + "om_ngram": { + "type": "custom", + "tokenizer": "n_gram_tokenizer", + "filter": [ + "lowercase" + ] + }, + "om_compound_analyzer": { + "tokenizer": "standard", + "filter": [ + "lowercase", + "compound_word_delimiter_graph", + "flatten_graph" + ] + } + }, + "filter": { + "om_stemmer": { + "type": "stemmer", + "name": "english" + }, + "compound_word_delimiter_graph": { + "type": "word_delimiter_graph", + "generate_word_parts": true, + "generate_number_parts": true, + "split_on_case_change": true, + "split_on_numerics": true, + "catenate_words": false, + "catenate_numbers": false, + "catenate_all": false, + "preserve_original": true, + "stem_english_possessive": true + } + }, + "tokenizer": { + "n_gram_tokenizer": { + "type": "ngram", + "min_gram": 1, + "max_gram": 2, + "token_chars": [ + "letter", + "digit" + ] + } + } + }, + "index": { + "max_ngram_diff": 1 + } + }, + "mappings": { + "properties": { + "changeDescription": { + "enabled": false + }, + "incrementalChangeDescription": { + "enabled": false + }, + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "name": { + "type": "text", + "analyzer": "om_analyzer_jp", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256, + "normalizer": "lowercase_normalizer" + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + }, + "compound": { + "type": "text", + "analyzer": "om_compound_analyzer" + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "fqnParts": { + "type": "keyword" + }, + "displayName": { + "type": "text", + "analyzer": "om_analyzer_jp", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + }, + "actualCase": { + "type": "keyword", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + }, + "compound": { + "type": "text", + "analyzer": "om_compound_analyzer" + } + } + }, + "description": { + "type": "text", + "analyzer": "om_analyzer_jp" + }, + "version": { + "type": "float" + }, + "updatedAt": { + "type": "date", + "format": "epoch_second" + }, + "updatedBy": { + "type": "text" + }, + "href": { + "type": "text" + }, + "directoryType": { + "type": "keyword" + }, + "path": { + "type": "keyword" + }, + "isShared": { + "type": "boolean" + }, + "parent": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "service": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "owners": { + "type": "nested", + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "domains": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "dataProducts": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "usageSummary": { + "properties": { + "dailyStats": { + "properties": { + "count": { + "type": "long" + }, + "percentileRank": { + "type": "long" + } + } + }, + "weeklyStats": { + "properties": { + "count": { + "type": "long" + }, + "percentileRank": { + "type": "long" + } + } + }, + "monthlyStats": { + "properties": { + "count": { + "type": "long" + }, + "percentileRank": { + "type": "long" + } + } + }, + "date": { + "type": "date", + "format": "strict_date_optional_time||yyyy-MM-dd HH:mm:ss||epoch_millis" + } + } + }, + "deleted": { + "type": "boolean" + }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword", + "fields": { + "text": { + "type": "text", + "analyzer": "om_analyzer" + } + } + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "classificationTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "glossaryTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "tags": { + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "text": { + "type": "text", + "analyzer": "om_analyzer" + } + } + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "serviceType": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "entityType": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "entityStatus": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "totalVotes": { + "type": "long", + "null_value": 0 + }, + "descriptionStatus": { + "type": "keyword" + }, + "certification": { + "type": "object", + "properties": { + "tagLabel": { + "type": "object", + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "text": { + "type": "text", + "analyzer": "om_analyzer" + } + } + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "updatedBy": { + "type": "keyword" + }, + "updatedAt": { + "type": "date" + } + } + }, + "upstreamLineage": { + "properties": { + "fromEntity": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "toEntity": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "pipeline": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "columns": { + "properties": { + "fromColumns": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "toColumn": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "docId": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "sqlQueryKey": { + "type": "keyword" + } + } + }, + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "customPropertiesTyped": { + "type": "nested", + "properties": { + "name": { + "type": "keyword" + }, + "propertyType": { + "type": "keyword" + }, + "stringValue": { + "type": "keyword" + }, + "textValue": { + "type": "text", + "analyzer": "om_analyzer" + }, + "longValue": { + "type": "long" + }, + "doubleValue": { + "type": "double" + }, + "start": { + "type": "long" + }, + "end": { + "type": "long" + }, + "refId": { + "type": "keyword" + }, + "refType": { + "type": "keyword" + }, + "refName": { + "type": "keyword" + }, + "refFqn": { + "type": "keyword" + } + } + }, + "fingerprint": { + "type": "keyword" + }, + "textToEmbed": { + "type": "text" + }, + "chunkIndex": { + "type": "integer" + }, + "chunkCount": { + "type": "integer" + }, + "parentId": { + "type": "keyword" + }, + "ownerDisplayName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "ownerName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false + } + } + } +} diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/knowledge_page_search_index.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/knowledge_page_search_index.json new file mode 100644 index 000000000000..fa3510aa424f --- /dev/null +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/knowledge_page_search_index.json @@ -0,0 +1,482 @@ +{ + "settings": { + "index": { + "max_ngram_diff": 7 + }, + "analysis": { + "normalizer": { + "lowercase_normalizer": { + "type": "custom", + "char_filter": [], + "filter": ["lowercase"] + } + }, + "tokenizer": { + "om_ngram_tokenizer": { + "type": "ngram", + "min_gram": 3, + "max_gram": 10, + "token_chars": ["letter", "digit"] + } + }, + "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": ["lowercase", "om_stemmer"] + }, + "om_ngram": { + "type": "custom", + "tokenizer": "om_ngram_tokenizer", + "filter": ["lowercase"] + } + }, + "filter": { + "om_stemmer": { + "type": "stemmer", + "name": "english" + } + } + } + }, + "mappings": { + "properties": { + "id": { + "type": "text" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "displayName": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + }, + "actualCase": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "entityType": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "description": { + "type": "text" + }, + "version": { + "type": "float" + }, + "updatedAt": { + "type": "date", + "format": "epoch_second" + }, + "updatedBy": { + "type": "text" + }, + "href": { + "type": "text" + }, + "fqnDepth": { + "type": "integer" + }, + "deleted": { + "type": "boolean" + }, + "owners": { + "type": "nested", + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "reviewers": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "entityStatus": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "followers": { + "type": "keyword" + }, + "tags": { + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "classificationTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "glossaryTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "glossaryTerms": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "pageType" : { + "type": "keyword" + }, + "relatedEntities": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "editors": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "parent" : { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "children" : { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "ownerDisplayName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "ownerName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "fingerprint": { + "type": "keyword" + }, + "textToEmbed": { + "type": "text" + }, + "chunkIndex": { + "type": "integer" + }, + "chunkCount": { + "type": "integer" + }, + "parentId": { + "type": "keyword" + } + } + } +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/context_file_search_index.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/context_file_search_index.json new file mode 100644 index 000000000000..3763cfa244e9 --- /dev/null +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/context_file_search_index.json @@ -0,0 +1,859 @@ +{ + "settings": { + "index": { + "max_ngram_diff": 17 + }, + "analysis": { + "tokenizer": { + "n_gram_tokenizer": { + "type": "ngram", + "min_gram": 3, + "max_gram": 20, + "token_chars": [ + "letter", + "digit" + ] + } + }, + "normalizer": { + "lowercase_normalizer": { + "type": "custom", + "char_filter": [], + "filter": [ + "lowercase" + ] + } + }, + "analyzer": { + "om_analyzer": { + "tokenizer": "standard", + "filter": [ + "lowercase", + "word_delimiter_filter", + "om_stemmer" + ] + }, + "om_ngram": { + "type": "custom", + "tokenizer": "n_gram_tokenizer", + "filter": [ + "lowercase" + ] + }, + "om_compound_analyzer": { + "tokenizer": "standard", + "filter": [ + "lowercase", + "compound_word_delimiter_graph", + "flatten_graph" + ] + } + }, + "filter": { + "om_stemmer": { + "type": "stemmer", + "name": "kstem" + }, + "word_delimiter_filter": { + "type": "word_delimiter", + "preserve_original": true + }, + "compound_word_delimiter_graph": { + "type": "word_delimiter_graph", + "generate_word_parts": true, + "generate_number_parts": true, + "split_on_case_change": true, + "split_on_numerics": true, + "catenate_words": false, + "catenate_numbers": false, + "catenate_all": false, + "preserve_original": true, + "stem_english_possessive": true + } + } + } + }, + "mappings": { + "properties": { + "changeDescription": { + "enabled": false + }, + "incrementalChangeDescription": { + "enabled": false + }, + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "name": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256, + "normalizer": "lowercase_normalizer" + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + }, + "compound": { + "type": "text", + "analyzer": "om_compound_analyzer" + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "displayName": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + }, + "actualCase": { + "type": "keyword", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + }, + "compound": { + "type": "text", + "analyzer": "om_compound_analyzer" + } + } + }, + "description": { + "type": "text", + "analyzer": "om_analyzer", + "similarity": "boolean", + "term_vector": "with_positions_offsets" + }, + "serviceType": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "service": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "directory": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "version": { + "type": "float" + }, + "dataProducts": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "updatedAt": { + "type": "date", + "format": "epoch_second" + }, + "updatedBy": { + "type": "text" + }, + "href": { + "type": "text" + }, + "sourceUrl": { + "type": "text" + }, + "fileType": { + "type": "keyword" + }, + "mimeType": { + "type": "keyword" + }, + "fileExtension": { + "type": "keyword" + }, + "processingStatus": { + "type": "keyword" + }, + "sourceType": { + "type": "keyword" + }, + "extractedText": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { "type": "keyword", "ignore_above": 256 } + } + }, + "folder": { + "properties": { + "id": { "type": "keyword" }, + "type": { "type": "keyword" }, + "name": { "type": "keyword", "normalizer": "lowercase_normalizer" }, + "displayName": { "type": "keyword" }, + "fullyQualifiedName": { "type": "keyword", "normalizer": "lowercase_normalizer" } + } + }, + "extension": { + "type": "flattened" + }, + "path": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "driveFileId": { + "type": "keyword" + }, + "size": { + "type": "long" + }, + "checksum": { + "type": "keyword" + }, + "isShared": { + "type": "boolean" + }, + "fileVersion": { + "type": "keyword" + }, + "createdTime": { + "type": "date" + }, + "modifiedTime": { + "type": "date" + }, + "lastModifiedBy": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "entityType": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "entityStatus": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "owners": { + "type": "nested", + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "domains": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "followers": { + "type": "keyword" + }, + "totalVotes": { + "type": "long", + "null_value": 0 + }, + "votes": { + "type": "object", + "dynamic": false, + "properties": { + "upVotes": { + "type": "integer" + }, + "downVotes": { + "type": "integer" + } + } + }, + "descriptionStatus": { + "type": "keyword" + }, + "tags": { + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "text": { + "type": "text", + "analyzer": "om_analyzer" + } + } + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "text": { + "type": "text", + "analyzer": "om_analyzer" + } + } + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "classificationTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "glossaryTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "deleted": { + "type": "boolean" + }, + "fqnParts": { + "type": "keyword" + }, + "descriptionSources": { + "type": "object", + "dynamic": false + }, + "tagSources": { + "type": "object", + "dynamic": false + }, + "tierSources": { + "type": "object", + "dynamic": false + }, + "upstreamLineage": { + "properties": { + "fromEntity": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "toEntity": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "pipeline": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "columns": { + "properties": { + "fromColumns": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "toColumn": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "docId": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "sqlQueryKey": { + "type": "keyword" + } + } + }, + "certification": { + "type": "object", + "properties": { + "tagLabel": { + "type": "object", + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "text": { + "type": "text", + "analyzer": "om_analyzer" + } + } + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "appliedDate": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + }, + "expiryDate": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + } + } + }, + "suggest": { + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] + }, + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "customPropertiesTyped": { + "type": "nested", + "properties": { + "name": { + "type": "keyword" + }, + "propertyType": { + "type": "keyword" + }, + "stringValue": { + "type": "keyword" + }, + "textValue": { + "type": "text", + "analyzer": "om_analyzer" + }, + "longValue": { + "type": "long" + }, + "doubleValue": { + "type": "double" + }, + "start": { + "type": "long" + }, + "end": { + "type": "long" + }, + "refId": { + "type": "keyword" + }, + "refType": { + "type": "keyword" + }, + "refName": { + "type": "keyword" + }, + "refFqn": { + "type": "keyword" + } + } + }, + "fingerprint": { + "type": "keyword" + }, + "textToEmbed": { + "type": "text" + }, + "chunkIndex": { + "type": "integer" + }, + "chunkCount": { + "type": "integer" + }, + "parentId": { + "type": "keyword" + }, + "ownerDisplayName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "ownerName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false + } + } + } +} diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/folder_search_index.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/folder_search_index.json new file mode 100644 index 000000000000..2c9b3850f051 --- /dev/null +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/folder_search_index.json @@ -0,0 +1,668 @@ +{ + "settings": { + "index": { + "max_ngram_diff": 17 + }, + "analysis": { + "tokenizer": { + "n_gram_tokenizer": { + "type": "ngram", + "min_gram": 3, + "max_gram": 20, + "token_chars": [ + "letter", + "digit" + ] + } + }, + "normalizer": { + "lowercase_normalizer": { + "type": "custom", + "char_filter": [], + "filter": [ + "lowercase", + "asciifolding" + ] + } + }, + "analyzer": { + "om_analyzer": { + "tokenizer": "standard", + "filter": [ + "word_delimiter_filter", + "lowercase", + "asciifolding", + "russian_stop", + "russian_snowball", + "english_stop", + "om_kstem" + ] + }, + "om_ngram": { + "type": "custom", + "tokenizer": "n_gram_tokenizer", + "filter": [ + "lowercase" + ] + }, + "om_compound_analyzer": { + "tokenizer": "standard", + "filter": [ + "compound_word_delimiter_graph", + "lowercase", + "flatten_graph" + ] + } + }, + "filter": { + "word_delimiter_filter": { + "type": "word_delimiter", + "preserve_original": true + }, + "compound_word_delimiter_graph": { + "type": "word_delimiter_graph", + "generate_word_parts": true, + "generate_number_parts": true, + "split_on_case_change": true, + "split_on_numerics": true, + "catenate_words": false, + "catenate_numbers": false, + "catenate_all": false, + "preserve_original": true, + "stem_english_possessive": true + }, + "russian_stop": { + "type": "stop", + "stopwords": "_russian_" + }, + "english_stop": { + "type": "stop", + "stopwords": "_english_" + }, + "russian_snowball": { + "name": "russian", + "type": "stemmer" + }, + "om_kstem": { + "type": "kstem" + }, + "asciifolding": { + "type": "asciifolding" + } + } + } + }, + "mappings": { + "properties": { + "changeDescription": { + "enabled": false + }, + "incrementalChangeDescription": { + "enabled": false + }, + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "name": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256, + "normalizer": "lowercase_normalizer" + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + }, + "compound": { + "type": "text", + "analyzer": "om_compound_analyzer" + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "displayName": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + }, + "actualCase": { + "type": "keyword", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + }, + "compound": { + "type": "text", + "analyzer": "om_compound_analyzer" + } + } + }, + "description": { + "type": "text", + "analyzer": "om_analyzer" + }, + "serviceType": { + "type": "keyword" + }, + "service": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "description": { + "type": "text" + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "parent": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "directoryType": { + "type": "keyword" + }, + "path": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "driveId": { + "type": "keyword" + }, + "isShared": { + "type": "boolean" + }, + "numberOfFiles": { + "type": "long" + }, + "numberOfSubDirectories": { + "type": "long" + }, + "totalSize": { + "type": "long" + }, + "entityType": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "entityStatus": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "owners": { + "type": "nested", + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "domains": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "followers": { + "type": "keyword" + }, + "totalVotes": { + "type": "long" + }, + "descriptionStatus": { + "type": "keyword" + }, + "tags": { + "properties": { + "tagFQN": { + "type": "keyword" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "classificationTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "glossaryTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "deleted": { + "type": "boolean" + }, + "fqnParts": { + "type": "keyword" + }, + "certification": { + "type": "object", + "properties": { + "tagLabel": { + "type": "object", + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "text": { + "type": "text", + "analyzer": "om_analyzer" + } + } + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "updatedBy": { + "type": "keyword" + }, + "updatedAt": { + "type": "date" + } + } + }, + "suggest": { + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] + }, + "upstreamLineage": { + "properties": { + "fromEntity": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "toEntity": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "pipeline": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "columns": { + "properties": { + "fromColumns": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "toColumn": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "docId": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "sqlQueryKey": { + "type": "keyword" + } + } + }, + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "customPropertiesTyped": { + "type": "nested", + "properties": { + "name": { + "type": "keyword" + }, + "propertyType": { + "type": "keyword" + }, + "stringValue": { + "type": "keyword" + }, + "textValue": { + "type": "text", + "analyzer": "om_analyzer" + }, + "longValue": { + "type": "long" + }, + "doubleValue": { + "type": "double" + }, + "start": { + "type": "long" + }, + "end": { + "type": "long" + }, + "refId": { + "type": "keyword" + }, + "refType": { + "type": "keyword" + }, + "refName": { + "type": "keyword" + }, + "refFqn": { + "type": "keyword" + } + } + }, + "fingerprint": { + "type": "keyword" + }, + "textToEmbed": { + "type": "text" + }, + "chunkIndex": { + "type": "integer" + }, + "chunkCount": { + "type": "integer" + }, + "parentId": { + "type": "keyword" + }, + "ownerDisplayName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "ownerName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false + } + } + } +} diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/knowledge_page_search_index.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/knowledge_page_search_index.json new file mode 100644 index 000000000000..fa3510aa424f --- /dev/null +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/knowledge_page_search_index.json @@ -0,0 +1,482 @@ +{ + "settings": { + "index": { + "max_ngram_diff": 7 + }, + "analysis": { + "normalizer": { + "lowercase_normalizer": { + "type": "custom", + "char_filter": [], + "filter": ["lowercase"] + } + }, + "tokenizer": { + "om_ngram_tokenizer": { + "type": "ngram", + "min_gram": 3, + "max_gram": 10, + "token_chars": ["letter", "digit"] + } + }, + "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": ["lowercase", "om_stemmer"] + }, + "om_ngram": { + "type": "custom", + "tokenizer": "om_ngram_tokenizer", + "filter": ["lowercase"] + } + }, + "filter": { + "om_stemmer": { + "type": "stemmer", + "name": "english" + } + } + } + }, + "mappings": { + "properties": { + "id": { + "type": "text" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "displayName": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + }, + "actualCase": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "entityType": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "description": { + "type": "text" + }, + "version": { + "type": "float" + }, + "updatedAt": { + "type": "date", + "format": "epoch_second" + }, + "updatedBy": { + "type": "text" + }, + "href": { + "type": "text" + }, + "fqnDepth": { + "type": "integer" + }, + "deleted": { + "type": "boolean" + }, + "owners": { + "type": "nested", + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "reviewers": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "entityStatus": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "followers": { + "type": "keyword" + }, + "tags": { + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "classificationTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "glossaryTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "glossaryTerms": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "pageType" : { + "type": "keyword" + }, + "relatedEntities": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "editors": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "parent" : { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "children" : { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "ownerDisplayName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "ownerName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "fingerprint": { + "type": "keyword" + }, + "textToEmbed": { + "type": "text" + }, + "chunkIndex": { + "type": "integer" + }, + "chunkCount": { + "type": "integer" + }, + "parentId": { + "type": "keyword" + } + } + } +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/context_file_search_index.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/context_file_search_index.json new file mode 100644 index 000000000000..3763cfa244e9 --- /dev/null +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/context_file_search_index.json @@ -0,0 +1,859 @@ +{ + "settings": { + "index": { + "max_ngram_diff": 17 + }, + "analysis": { + "tokenizer": { + "n_gram_tokenizer": { + "type": "ngram", + "min_gram": 3, + "max_gram": 20, + "token_chars": [ + "letter", + "digit" + ] + } + }, + "normalizer": { + "lowercase_normalizer": { + "type": "custom", + "char_filter": [], + "filter": [ + "lowercase" + ] + } + }, + "analyzer": { + "om_analyzer": { + "tokenizer": "standard", + "filter": [ + "lowercase", + "word_delimiter_filter", + "om_stemmer" + ] + }, + "om_ngram": { + "type": "custom", + "tokenizer": "n_gram_tokenizer", + "filter": [ + "lowercase" + ] + }, + "om_compound_analyzer": { + "tokenizer": "standard", + "filter": [ + "lowercase", + "compound_word_delimiter_graph", + "flatten_graph" + ] + } + }, + "filter": { + "om_stemmer": { + "type": "stemmer", + "name": "kstem" + }, + "word_delimiter_filter": { + "type": "word_delimiter", + "preserve_original": true + }, + "compound_word_delimiter_graph": { + "type": "word_delimiter_graph", + "generate_word_parts": true, + "generate_number_parts": true, + "split_on_case_change": true, + "split_on_numerics": true, + "catenate_words": false, + "catenate_numbers": false, + "catenate_all": false, + "preserve_original": true, + "stem_english_possessive": true + } + } + } + }, + "mappings": { + "properties": { + "changeDescription": { + "enabled": false + }, + "incrementalChangeDescription": { + "enabled": false + }, + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "name": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256, + "normalizer": "lowercase_normalizer" + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + }, + "compound": { + "type": "text", + "analyzer": "om_compound_analyzer" + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "displayName": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + }, + "actualCase": { + "type": "keyword", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + }, + "compound": { + "type": "text", + "analyzer": "om_compound_analyzer" + } + } + }, + "description": { + "type": "text", + "analyzer": "om_analyzer", + "similarity": "boolean", + "term_vector": "with_positions_offsets" + }, + "serviceType": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "service": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "directory": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "version": { + "type": "float" + }, + "dataProducts": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "updatedAt": { + "type": "date", + "format": "epoch_second" + }, + "updatedBy": { + "type": "text" + }, + "href": { + "type": "text" + }, + "sourceUrl": { + "type": "text" + }, + "fileType": { + "type": "keyword" + }, + "mimeType": { + "type": "keyword" + }, + "fileExtension": { + "type": "keyword" + }, + "processingStatus": { + "type": "keyword" + }, + "sourceType": { + "type": "keyword" + }, + "extractedText": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { "type": "keyword", "ignore_above": 256 } + } + }, + "folder": { + "properties": { + "id": { "type": "keyword" }, + "type": { "type": "keyword" }, + "name": { "type": "keyword", "normalizer": "lowercase_normalizer" }, + "displayName": { "type": "keyword" }, + "fullyQualifiedName": { "type": "keyword", "normalizer": "lowercase_normalizer" } + } + }, + "extension": { + "type": "flattened" + }, + "path": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "driveFileId": { + "type": "keyword" + }, + "size": { + "type": "long" + }, + "checksum": { + "type": "keyword" + }, + "isShared": { + "type": "boolean" + }, + "fileVersion": { + "type": "keyword" + }, + "createdTime": { + "type": "date" + }, + "modifiedTime": { + "type": "date" + }, + "lastModifiedBy": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "entityType": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "entityStatus": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "owners": { + "type": "nested", + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "domains": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "followers": { + "type": "keyword" + }, + "totalVotes": { + "type": "long", + "null_value": 0 + }, + "votes": { + "type": "object", + "dynamic": false, + "properties": { + "upVotes": { + "type": "integer" + }, + "downVotes": { + "type": "integer" + } + } + }, + "descriptionStatus": { + "type": "keyword" + }, + "tags": { + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "text": { + "type": "text", + "analyzer": "om_analyzer" + } + } + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "text": { + "type": "text", + "analyzer": "om_analyzer" + } + } + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "classificationTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "glossaryTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "deleted": { + "type": "boolean" + }, + "fqnParts": { + "type": "keyword" + }, + "descriptionSources": { + "type": "object", + "dynamic": false + }, + "tagSources": { + "type": "object", + "dynamic": false + }, + "tierSources": { + "type": "object", + "dynamic": false + }, + "upstreamLineage": { + "properties": { + "fromEntity": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "toEntity": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "pipeline": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "columns": { + "properties": { + "fromColumns": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "toColumn": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "docId": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "sqlQueryKey": { + "type": "keyword" + } + } + }, + "certification": { + "type": "object", + "properties": { + "tagLabel": { + "type": "object", + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "text": { + "type": "text", + "analyzer": "om_analyzer" + } + } + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "appliedDate": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + }, + "expiryDate": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + } + } + }, + "suggest": { + "type": "completion", + "contexts": [ + { + "name": "deleted", + "type": "category", + "path": "deleted" + } + ] + }, + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "customPropertiesTyped": { + "type": "nested", + "properties": { + "name": { + "type": "keyword" + }, + "propertyType": { + "type": "keyword" + }, + "stringValue": { + "type": "keyword" + }, + "textValue": { + "type": "text", + "analyzer": "om_analyzer" + }, + "longValue": { + "type": "long" + }, + "doubleValue": { + "type": "double" + }, + "start": { + "type": "long" + }, + "end": { + "type": "long" + }, + "refId": { + "type": "keyword" + }, + "refType": { + "type": "keyword" + }, + "refName": { + "type": "keyword" + }, + "refFqn": { + "type": "keyword" + } + } + }, + "fingerprint": { + "type": "keyword" + }, + "textToEmbed": { + "type": "text" + }, + "chunkIndex": { + "type": "integer" + }, + "chunkCount": { + "type": "integer" + }, + "parentId": { + "type": "keyword" + }, + "ownerDisplayName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "ownerName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false + } + } + } +} diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/folder_search_index.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/folder_search_index.json new file mode 100644 index 000000000000..c9119fe16ef7 --- /dev/null +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/folder_search_index.json @@ -0,0 +1,740 @@ +{ + "settings": { + "analysis": { + "normalizer": { + "lowercase_normalizer": { + "type": "custom", + "char_filter": [], + "filter": [ + "lowercase" + ] + } + }, + "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, + "om_compound_analyzer": { + "tokenizer": "standard", + "filter": [ + "lowercase", + "compound_word_delimiter_graph", + "flatten_graph" + ] + } + }, + "filter": { + "om_stemmer": { + "type": "stemmer", + "name": "english" + }, + "compound_word_delimiter_graph": { + "type": "word_delimiter_graph", + "generate_word_parts": true, + "generate_number_parts": true, + "split_on_case_change": true, + "split_on_numerics": true, + "catenate_words": false, + "catenate_numbers": false, + "catenate_all": false, + "preserve_original": true, + "stem_english_possessive": true + } + } + }, + "index": {} + }, + "mappings": { + "properties": { + "changeDescription": { + "enabled": false + }, + "incrementalChangeDescription": { + "enabled": false + }, + "id": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "name": { + "type": "text", + "analyzer": "ik_max_word", + "search_analyzer": "ik_smart", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256, + "normalizer": "lowercase_normalizer" + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + }, + "compound": { + "type": "text", + "analyzer": "om_compound_analyzer" + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "fqnParts": { + "type": "keyword" + }, + "displayName": { + "type": "text", + "analyzer": "ik_max_word", + "search_analyzer": "ik_smart", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + }, + "actualCase": { + "type": "keyword", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + }, + "compound": { + "type": "text", + "analyzer": "om_compound_analyzer" + } + } + }, + "description": { + "type": "text", + "analyzer": "ik_max_word", + "search_analyzer": "ik_smart" + }, + "version": { + "type": "float" + }, + "updatedAt": { + "type": "date", + "format": "epoch_second" + }, + "updatedBy": { + "type": "text" + }, + "href": { + "type": "text" + }, + "directoryType": { + "type": "keyword" + }, + "path": { + "type": "keyword" + }, + "isShared": { + "type": "boolean" + }, + "parent": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "service": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "owners": { + "type": "nested", + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "domains": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "dataProducts": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "usageSummary": { + "properties": { + "dailyStats": { + "properties": { + "count": { + "type": "long" + }, + "percentileRank": { + "type": "long" + } + } + }, + "weeklyStats": { + "properties": { + "count": { + "type": "long" + }, + "percentileRank": { + "type": "long" + } + } + }, + "monthlyStats": { + "properties": { + "count": { + "type": "long" + }, + "percentileRank": { + "type": "long" + } + } + }, + "date": { + "type": "date", + "format": "strict_date_optional_time||yyyy-MM-dd HH:mm:ss||epoch_millis" + } + } + }, + "deleted": { + "type": "boolean" + }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword", + "fields": { + "text": { + "type": "text", + "analyzer": "om_analyzer" + } + } + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "classificationTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "glossaryTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "tags": { + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "text": { + "type": "text", + "analyzer": "om_analyzer" + } + } + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "serviceType": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "entityType": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "entityStatus": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "totalVotes": { + "type": "long", + "null_value": 0 + }, + "descriptionStatus": { + "type": "keyword" + }, + "certification": { + "type": "object", + "properties": { + "tagLabel": { + "type": "object", + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "text": { + "type": "text", + "analyzer": "om_analyzer" + } + } + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "updatedBy": { + "type": "keyword" + }, + "updatedAt": { + "type": "date" + } + } + }, + "upstreamLineage": { + "properties": { + "fromEntity": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "toEntity": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "pipeline": { + "properties": { + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "fullyQualifiedName": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "columns": { + "properties": { + "fromColumns": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "toColumn": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + } + } + }, + "docId": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "sqlQueryKey": { + "type": "keyword" + } + } + }, + "fqnHash": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 512 + } + } + }, + "customPropertiesTyped": { + "type": "nested", + "properties": { + "name": { + "type": "keyword" + }, + "propertyType": { + "type": "keyword" + }, + "stringValue": { + "type": "keyword" + }, + "textValue": { + "type": "text", + "analyzer": "om_analyzer" + }, + "longValue": { + "type": "long" + }, + "doubleValue": { + "type": "double" + }, + "start": { + "type": "long" + }, + "end": { + "type": "long" + }, + "refId": { + "type": "keyword" + }, + "refType": { + "type": "keyword" + }, + "refName": { + "type": "keyword" + }, + "refFqn": { + "type": "keyword" + } + } + }, + "fingerprint": { + "type": "keyword" + }, + "textToEmbed": { + "type": "text" + }, + "chunkIndex": { + "type": "integer" + }, + "chunkCount": { + "type": "integer" + }, + "parentId": { + "type": "keyword" + }, + "ownerDisplayName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "ownerName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false + } + } + } +} diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/knowledge_page_search_index.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/knowledge_page_search_index.json new file mode 100644 index 000000000000..fa3510aa424f --- /dev/null +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/knowledge_page_search_index.json @@ -0,0 +1,482 @@ +{ + "settings": { + "index": { + "max_ngram_diff": 7 + }, + "analysis": { + "normalizer": { + "lowercase_normalizer": { + "type": "custom", + "char_filter": [], + "filter": ["lowercase"] + } + }, + "tokenizer": { + "om_ngram_tokenizer": { + "type": "ngram", + "min_gram": 3, + "max_gram": 10, + "token_chars": ["letter", "digit"] + } + }, + "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": ["lowercase", "om_stemmer"] + }, + "om_ngram": { + "type": "custom", + "tokenizer": "om_ngram_tokenizer", + "filter": ["lowercase"] + } + }, + "filter": { + "om_stemmer": { + "type": "stemmer", + "name": "english" + } + } + } + }, + "mappings": { + "properties": { + "id": { + "type": "text" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "displayName": { + "type": "text", + "analyzer": "om_analyzer", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + }, + "ngram": { + "type": "text", + "analyzer": "om_ngram" + }, + "actualCase": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "entityType": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "description": { + "type": "text" + }, + "version": { + "type": "float" + }, + "updatedAt": { + "type": "date", + "format": "epoch_second" + }, + "updatedBy": { + "type": "text" + }, + "href": { + "type": "text" + }, + "fqnDepth": { + "type": "integer" + }, + "deleted": { + "type": "boolean" + }, + "owners": { + "type": "nested", + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "reviewers": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, + "entityStatus": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "followers": { + "type": "keyword" + }, + "tags": { + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "classificationTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "glossaryTags": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "glossaryTerms": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "tier": { + "properties": { + "tagFQN": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "labelType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "source": { + "type": "keyword" + }, + "state": { + "type": "keyword" + } + } + }, + "pageType" : { + "type": "keyword" + }, + "relatedEntities": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "editors": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "parent" : { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "children" : { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "text" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "text" + }, + "href": { + "type": "text" + } + } + }, + "ownerDisplayName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "ownerName": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "fingerprint": { + "type": "keyword" + }, + "textToEmbed": { + "type": "text" + }, + "chunkIndex": { + "type": "integer" + }, + "chunkCount": { + "type": "integer" + }, + "parentId": { + "type": "keyword" + } + } + } +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/api/attachments/createAsset.json b/openmetadata-spec/src/main/resources/json/schema/api/attachments/createAsset.json new file mode 100644 index 000000000000..22cfa0c59b54 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/api/attachments/createAsset.json @@ -0,0 +1,34 @@ +{ + "$id": "https://open-metadata.org/schema/api/attachments/createAsset.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CreateAssetRequest", + "description": "Schema for creating a new asset record after file upload. The asset record will be updated with the URL and status once the upload is complete.", + "type": "object", + "javaType": "org.openmetadata.schema.api.attachments.CreateAsset", + "properties": { + "fileName": { + "type": "string", + "description": "The original file name of the asset." + }, + "contentType": { + "type": "string", + "description": "MIME type of the asset." + }, + "size": { + "type": "integer", + "format": "int64", + "minimum": 0, + "description": "File size in bytes." + }, + "assetType": { + "description": "Type of the asset.", + "$ref": "../../attachments/asset.json#/definitions/assetType" + }, + "entityLink": { + "description": "Link to the entity that this asset belongs to.", + "$ref": "../../type/basic.json#/definitions/entityLink" + } + }, + "required": ["fileName", "entityLink"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createContextFile.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createContextFile.json new file mode 100644 index 000000000000..de1830ea9889 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createContextFile.json @@ -0,0 +1,88 @@ +{ + "$id": "https://open-metadata.org/schema/api/data/createContextFile.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CreateContextFile", + "description": "Request to create a file in the Context Center Drive.", + "type": "object", + "javaType": "org.openmetadata.schema.api.data.CreateContextFile", + "javaInterfaces": ["org.openmetadata.schema.CreateEntity"], + "properties": { + "name": { + "description": "Name of the file.", + "$ref": "../../type/basic.json#/definitions/entityName" + }, + "displayName": { + "description": "Display name (original filename or user-provided title).", + "type": "string" + }, + "description": { + "description": "Description of the file.", + "$ref": "../../type/basic.json#/definitions/markdown" + }, + "fileType": { + "description": "Type of file.", + "$ref": "../../entity/data/contextFile.json#/definitions/fileType" + }, + "fileSize": { + "description": "File size in bytes.", + "type": "integer", + "format": "int64", + "minimum": 0 + }, + "contentType": { + "description": "MIME type.", + "type": "string" + }, + "fileExtension": { + "description": "File extension.", + "type": "string" + }, + "assetId": { + "description": "Legacy reference to Asset entity in object storage (S3, Azure Blob, in-memory, or no-op provider). Prefer headContentId / ContextFileContent for new flows.", + "type": "string" + }, + "processingStatus": { + "description": "Processing status.", + "$ref": "../../entity/data/contextFile.json#/definitions/processingStatus" + }, + "sourceType": { + "description": "How the file was added.", + "$ref": "../../entity/data/contextFile.json#/definitions/sourceType" + }, + "sourceId": { + "description": "ID in external source system.", + "type": "string" + }, + "sourceUrl": { + "description": "URL in external source system.", + "type": "string", + "format": "uri" + }, + "folder": { + "description": "Parent folder fully qualified name.", + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "owners": { + "description": "Owners of this file.", + "$ref": "../../type/entityReferenceList.json", + "default": null + }, + "tags": { + "description": "Tags for this file.", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + }, + "default": null + }, + "domains": { + "description": "Fully qualified names of the domains this file belongs to.", + "type": "array", + "items": { + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + } + } + }, + "required": ["name"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createFolder.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createFolder.json new file mode 100644 index 000000000000..b8179399e82b --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createFolder.json @@ -0,0 +1,57 @@ +{ + "$id": "https://open-metadata.org/schema/api/data/createFolder.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CreateFolder", + "description": "Request to create a folder in the Context Center Drive.", + "type": "object", + "javaType": "org.openmetadata.schema.api.data.CreateFolder", + "javaInterfaces": ["org.openmetadata.schema.CreateEntity"], + "properties": { + "name": { + "description": "Name of the folder.", + "$ref": "../../type/basic.json#/definitions/entityName" + }, + "displayName": { + "description": "Display name of the folder.", + "type": "string" + }, + "description": { + "description": "Description of the folder.", + "$ref": "../../type/basic.json#/definitions/markdown" + }, + "icon": { + "description": "Optional icon identifier.", + "type": "string" + }, + "color": { + "description": "Optional color for folder icon.", + "type": "string" + }, + "parent": { + "description": "Parent folder fully qualified name.", + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "owners": { + "description": "Owners of this folder.", + "$ref": "../../type/entityReferenceList.json", + "default": null + }, + "tags": { + "description": "Tags for this folder.", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + }, + "default": null + }, + "domains": { + "description": "Fully qualified names of the domains this folder belongs to.", + "type": "array", + "items": { + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + } + } + }, + "required": ["name"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createPage.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createPage.json new file mode 100644 index 000000000000..564fe377d02a --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createPage.json @@ -0,0 +1,80 @@ +{ + "$id": "https://open-metadata.org/schema/api/data/createPage.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CreatePage", + "description": "Schema for a Page Request.", + "type": "object", + "javaType": "org.openmetadata.schema.api.data.CreatePage", + "javaInterfaces": [ + "org.openmetadata.schema.CreateEntity" + ], + "properties": { + "name": { + "description": "Name of the Knowledge Page.", + "$ref": "../../type/basic.json#/definitions/entityName" + }, + "displayName": { + "description": "Display name of the Knowledge Page.", + "type": "string" + }, + "description": { + "description": "Description of the Knowledge Page.", + "$ref": "../../type/basic.json#/definitions/markdown" + }, + "owners": { + "description": "Owners of this Knowledge Page.", + "$ref": "../../type/entityReferenceList.json", + "default": null + }, + "reviewers": { + "description": "Reviewers of this Knowledge Page.", + "$ref": "../../type/entityReferenceList.json" + }, + "entityStatus": { + "description": "Status of this Knowledge Page (Draft, In Review, Approved, Rejected).", + "$ref": "../../type/status.json", + "default": "Approved" + }, + "tags": { + "description": "Tags for this Page", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + }, + "default": null + }, + "pageType" : { + "description": "Type of the Page.", + "$ref": "../../entity/data/page.json#/definitions/pageType" + }, + "page" : { + "description": "Knowledge Page Schema", + "oneOf": [ + { + "$ref": "../../entity/data/quickLink.json" + }, + { + "$ref": "../../entity/data/article.json" + } + ] + }, + "relatedEntities": { + "description": "Related Entities for the Knowledge Page", + "$ref": "../../type/entityReferenceList.json" + }, + "parent": { + "description": "Parent Knowledge Page.", + "$ref": "../../type/entityReference.json", + "default": null + }, + "domains" : { + "description": "Fully qualified names of the domains the Knowledge Page belongs to.", + "type": "array", + "items": { + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + } + } + }, + "required": ["name", "pageType", "page"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/attachments/asset.json b/openmetadata-spec/src/main/resources/json/schema/attachments/asset.json new file mode 100644 index 000000000000..1c5805f361a5 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/attachments/asset.json @@ -0,0 +1,84 @@ +{ + "$id": "https://open-metadata.org/schema/attachments/asset.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Asset", + "description": "Represents an uploaded asset record (e.g. an image, pdf or attachment) for an entity.", + "type": "object", + "javaType": "org.openmetadata.schema.attachments.Asset", + "definitions": { + "assetType": { + "javaType": "org.openmetadata.schema.attachments.AssetType", + "description": "This schema defines the type used for describing different types of Attachments.", + "type": "string", + "enum": [ + "Inline", + "External" + ], + "javaEnums": [ + { + "name": "Inline" + }, + { + "name": "External" + } + ], + "default": "Inline", + "additionalProperties": false + } + }, + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the asset." + }, + "fullyQualifiedName": { + "description": "Fully qualified name of a data asset the attachment belongsTo`.", + "$ref": "../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "fileName": { + "type": "string", + "description": "The original file name of the asset." + }, + "url": { + "type": "string", + "description": "URL where the asset is accessible." + }, + "contentType": { + "type": "string", + "description": "MIME type of the asset." + }, + "size": { + "type": "integer", + "format": "int64", + "minimum": 0, + "description": "File size in bytes." + }, + "extension": { + "type": "string", + "description": "File extension of the asset." + }, + "assetType": { + "description": "Type of the asset.", + "$ref": "#/definitions/assetType" + }, + "updatedAt": { + "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.", + "$ref": "../type/basic.json#/definitions/timestamp" + }, + "updatedBy": { + "description": "User who made the update.", + "type": "string" + }, + "deleted": { + "description": "When `true` indicates the entity has been marked for permanent deletion.", + "type": "boolean", + "default": false + }, + "entityLink": { + "description": "Link to the entity that this asset belongs to.", + "$ref": "../type/basic.json#/definitions/entityLink" + } + }, + "required": ["id", "fileName", "entityLink"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/article.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/article.json new file mode 100644 index 000000000000..b65ffe581b42 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/article.json @@ -0,0 +1,19 @@ +{ + "$id": "https://open-metadata.org/schema/entity/data/article.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Article", + "description": "Article Knowledge Page", + "type": "object", + "javaType": "org.openmetadata.schema.entity.data.Article", + "properties": { + "publicationDate": { + "description": "The publication date of the article.", + "$ref": "../../type/basic.json#/definitions/dateTime" + }, + "relatedArticles": { + "description": "An array of related articles.", + "$ref": "../../type/entityReferenceList.json" + } + }, + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/contextFile.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/contextFile.json new file mode 100644 index 000000000000..e8f7a1f039c2 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/contextFile.json @@ -0,0 +1,189 @@ +{ + "$id": "https://open-metadata.org/schema/entity/data/contextFile.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ContextFile", + "$comment": "@om-entity-type", + "description": "An uploaded file (PDF, spreadsheet, document) stored in the Context Center Drive.", + "type": "object", + "javaType": "org.openmetadata.schema.entity.data.ContextFile", + "javaInterfaces": ["org.openmetadata.schema.EntityInterface"], + "definitions": { + "fileType": { + "javaType": "org.openmetadata.schema.entity.data.ContextFileType", + "description": "Type of file based on content.", + "type": "string", + "enum": ["PDF", "Spreadsheet", "Presentation", "Image", "Document", "CSV", "Text", "Archive", "Other"], + "javaEnums": [ + { "name": "PDF" }, + { "name": "Spreadsheet" }, + { "name": "Presentation" }, + { "name": "Image" }, + { "name": "Document" }, + { "name": "CSV" }, + { "name": "Text" }, + { "name": "Archive" }, + { "name": "Other" } + ] + }, + "processingStatus": { + "javaType": "org.openmetadata.schema.entity.data.ProcessingStatus", + "description": "Processing state of the file after upload.", + "type": "string", + "enum": ["Uploaded", "Analyzing", "Processed", "Failed", "Unsupported"], + "javaEnums": [ + { "name": "Uploaded" }, + { "name": "Analyzing" }, + { "name": "Processed" }, + { "name": "Failed" }, + { "name": "Unsupported" } + ], + "default": "Uploaded" + }, + "sourceType": { + "javaType": "org.openmetadata.schema.entity.data.ContextFileSourceType", + "description": "How this file was added to the drive.", + "type": "string", + "enum": ["Upload", "DriveSync", "Confluence", "Notion"], + "javaEnums": [ + { "name": "Upload" }, + { "name": "DriveSync" }, + { "name": "Confluence" }, + { "name": "Notion" } + ], + "default": "Upload" + } + }, + "properties": { + "id": { + "description": "Unique identifier of the file.", + "$ref": "../../type/basic.json#/definitions/uuid" + }, + "name": { + "description": "Name of the file.", + "$ref": "../../type/basic.json#/definitions/entityName" + }, + "fullyQualifiedName": { + "description": "Fully qualified name of the file.", + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "displayName": { + "description": "Display name (original filename or user-provided title).", + "type": "string" + }, + "description": { + "description": "Description of the file.", + "$ref": "../../type/basic.json#/definitions/markdown" + }, + "fileType": { + "description": "Type of file (PDF, Spreadsheet, etc.).", + "$ref": "#/definitions/fileType" + }, + "fileSize": { + "description": "File size in bytes.", + "type": "integer", + "format": "int64", + "minimum": 0 + }, + "contentType": { + "description": "MIME type of the file.", + "type": "string" + }, + "fileExtension": { + "description": "File extension (e.g., pdf, xlsx).", + "type": "string" + }, + "assetId": { + "description": "Legacy reference to the current Asset entity storing the file blob. Prefer headContentId for new flows.", + "type": "string" + }, + "headContentId": { + "description": "Identifier of the current ContextFileContent snapshot for this file.", + "type": "string" + }, + "processingStatus": { + "description": "Current processing state after upload.", + "$ref": "#/definitions/processingStatus" + }, + "extractedText": { + "description": "Full text extracted from the file for search and AI context.", + "type": "string" + }, + "pageCount": { + "description": "Number of pages (PDF) or sheets (spreadsheet).", + "type": "integer" + }, + "sourceType": { + "description": "How this file was added.", + "$ref": "#/definitions/sourceType" + }, + "sourceId": { + "description": "ID of the file in the external source system.", + "type": "string" + }, + "sourceUrl": { + "description": "URL to view the file in the external source system.", + "type": "string", + "format": "uri" + }, + "folder": { + "description": "Parent folder containing this file.", + "$ref": "../../type/entityReference.json" + }, + "owners": { + "description": "Owners of this file.", + "$ref": "../../type/entityReferenceList.json", + "default": null + }, + "tags": { + "description": "Tags associated with this file.", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + }, + "default": null + }, + "version": { + "description": "Metadata version of the entity.", + "$ref": "../../type/entityHistory.json#/definitions/entityVersion" + }, + "updatedAt": { + "description": "Last update time in Unix epoch time milliseconds.", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "updatedBy": { + "description": "User who made the update.", + "type": "string" + }, + "href": { + "description": "Link to this resource.", + "$ref": "../../type/basic.json#/definitions/href" + }, + "changeDescription": { + "description": "Change that led to this version.", + "$ref": "../../type/entityHistory.json#/definitions/changeDescription" + }, + "incrementalChangeDescription": { + "description": "Incremental change that led to this version.", + "$ref": "../../type/entityHistory.json#/definitions/changeDescription" + }, + "deleted": { + "description": "When true indicates the entity has been soft deleted.", + "type": "boolean", + "default": false + }, + "domains": { + "description": "Domains this file belongs to.", + "$ref": "../../type/entityReferenceList.json" + }, + "followers": { + "description": "Followers of this file.", + "$ref": "../../type/entityReferenceList.json" + }, + "votes": { + "description": "Votes on this file.", + "$ref": "../../type/votes.json" + } + }, + "required": ["id", "name"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/contextFileContent.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/contextFileContent.json new file mode 100644 index 000000000000..2c5dad4da3e6 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/contextFileContent.json @@ -0,0 +1,110 @@ +{ + "$id": "https://open-metadata.org/schema/entity/data/contextFileContent.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ContextFileContent", + "$comment": "@om-entity-type", + "description": "A stored content snapshot for a ContextFile.", + "type": "object", + "javaType": "org.openmetadata.schema.entity.data.ContextFileContent", + "javaInterfaces": ["org.openmetadata.schema.EntityInterface"], + "properties": { + "id": { + "description": "Unique identifier of the content snapshot.", + "$ref": "../../type/basic.json#/definitions/uuid" + }, + "name": { + "description": "Name of the content snapshot.", + "$ref": "../../type/basic.json#/definitions/entityName" + }, + "displayName": { + "description": "Display name of the content snapshot.", + "type": "string" + }, + "description": { + "description": "Description of the content snapshot.", + "$ref": "../../type/basic.json#/definitions/markdown" + }, + "fullyQualifiedName": { + "description": "Fully qualified name of the content snapshot.", + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "contextFile": { + "description": "The file this content snapshot belongs to.", + "$ref": "../../type/entityReference.json" + }, + "assetId": { + "description": "Reference to the Asset entity storing the actual file blob.", + "type": "string" + }, + "contentType": { + "description": "MIME type of the stored content.", + "type": "string" + }, + "size": { + "description": "Content size in bytes.", + "type": "integer", + "format": "int64", + "minimum": 0 + }, + "checksum": { + "description": "SHA-256 checksum of the stored content.", + "type": "string" + }, + "sourceVersion": { + "description": "Provider revision or version token for synced files.", + "type": "string" + }, + "ingestedAt": { + "description": "Time the content snapshot was ingested.", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "isCurrent": { + "description": "Whether this is the current content snapshot for the file.", + "type": "boolean", + "default": true + }, + "processingStatus": { + "description": "Processing status for this content snapshot.", + "$ref": "./contextFile.json#/definitions/processingStatus" + }, + "processingError": { + "description": "Processing failure details for this snapshot.", + "type": "string" + }, + "extractedText": { + "description": "Canonical extracted text for this content snapshot.", + "type": "string" + }, + "version": { + "description": "Metadata version of the entity.", + "$ref": "../../type/entityHistory.json#/definitions/entityVersion" + }, + "updatedAt": { + "description": "Last update time in Unix epoch time milliseconds.", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "updatedBy": { + "description": "User who made the update.", + "type": "string" + }, + "href": { + "description": "Link to this resource.", + "$ref": "../../type/basic.json#/definitions/href" + }, + "changeDescription": { + "description": "Change that led to this version.", + "$ref": "../../type/entityHistory.json#/definitions/changeDescription" + }, + "incrementalChangeDescription": { + "description": "Incremental change that led to this version.", + "$ref": "../../type/entityHistory.json#/definitions/changeDescription" + }, + "deleted": { + "description": "When true indicates the entity has been soft deleted.", + "type": "boolean", + "default": false + } + }, + "required": ["id", "name", "contextFile"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/folder.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/folder.json new file mode 100644 index 000000000000..ebcf9f15d714 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/folder.json @@ -0,0 +1,104 @@ +{ + "$id": "https://open-metadata.org/schema/entity/data/folder.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Folder", + "$comment": "@om-entity-type", + "description": "A directory container for organizing files in the Context Center Drive. Folders can nest other folders. Access is determined by owners.", + "type": "object", + "javaType": "org.openmetadata.schema.entity.data.Folder", + "javaInterfaces": ["org.openmetadata.schema.EntityInterface"], + "properties": { + "id": { + "description": "Unique identifier of the folder.", + "$ref": "../../type/basic.json#/definitions/uuid" + }, + "name": { + "description": "Name of the folder.", + "$ref": "../../type/basic.json#/definitions/entityName" + }, + "fullyQualifiedName": { + "description": "Fully qualified name of the folder.", + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "displayName": { + "description": "Display name of the folder.", + "type": "string" + }, + "description": { + "description": "Description of the folder.", + "$ref": "../../type/basic.json#/definitions/markdown" + }, + "icon": { + "description": "Optional icon identifier for UI display.", + "type": "string" + }, + "color": { + "description": "Optional color for the folder icon.", + "type": "string" + }, + "parent": { + "description": "Parent folder (for nested folders).", + "$ref": "../../type/entityReference.json" + }, + "children": { + "description": "Child folders.", + "$ref": "../../type/entityReferenceList.json" + }, + "childrenCount": { + "description": "Count of direct children (folders + files).", + "type": "integer" + }, + "owners": { + "description": "Owners of this folder. User-owned = personal, Team-owned = team folder, Org-owned = org-wide.", + "$ref": "../../type/entityReferenceList.json", + "default": null + }, + "tags": { + "description": "Tags associated with this folder.", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + }, + "default": null + }, + "version": { + "description": "Metadata version of the entity.", + "$ref": "../../type/entityHistory.json#/definitions/entityVersion" + }, + "updatedAt": { + "description": "Last update time in Unix epoch time milliseconds.", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "updatedBy": { + "description": "User who made the update.", + "type": "string" + }, + "href": { + "description": "Link to this resource.", + "$ref": "../../type/basic.json#/definitions/href" + }, + "changeDescription": { + "description": "Change that led to this version.", + "$ref": "../../type/entityHistory.json#/definitions/changeDescription" + }, + "incrementalChangeDescription": { + "description": "Incremental change that led to this version.", + "$ref": "../../type/entityHistory.json#/definitions/changeDescription" + }, + "deleted": { + "description": "When true indicates the entity has been soft deleted.", + "type": "boolean", + "default": false + }, + "domains": { + "description": "Domains this folder belongs to.", + "$ref": "../../type/entityReferenceList.json" + }, + "followers": { + "description": "Followers of this folder.", + "$ref": "../../type/entityReferenceList.json" + } + }, + "required": ["id", "name"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/page.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/page.json new file mode 100644 index 000000000000..b6cffc00139e --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/page.json @@ -0,0 +1,146 @@ +{ + "$id": "https://open-metadata.org/schema/entity/data/page.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Page", + "$comment": "@om-entity-type", + "description": "This schema defines the type of Page.", + "type": "object", + "javaType": "org.openmetadata.schema.entity.data.Page", + "javaInterfaces": ["org.openmetadata.schema.EntityInterface"], + "definitions": { + "pageType": { + "javaType": "org.openmetadata.schema.entity.data.PageType", + "description": "Type of the Knowledge Page.", + "type": "string", + "enum": ["Article", "QuickLink"] + } + }, + "properties": { + "id": { + "description": "Unique identifier of the Knowledge Page.", + "$ref": "../../type/basic.json#/definitions/uuid" + }, + "name": { + "description": "Name of Knowledge Page belongs to", + "$ref": "../../type/basic.json#/definitions/entityName" + }, + "fullyQualifiedName": { + "description": "Fully qualified name of a Knowledge Page.", + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "displayName": { + "description": "Display Name that identifies this Knowledge Page. It could be title or label.", + "type": "string" + }, + "description": { + "description": "Description of a Knowledge Page.", + "$ref": "../../type/basic.json#/definitions/markdown" + }, + "version": { + "description": "Metadata version of the entity.", + "$ref": "../../type/entityHistory.json#/definitions/entityVersion" + }, + "updatedAt": { + "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.", + "$ref": "../../type/basic.json#/definitions/timestamp" + }, + "updatedBy": { + "description": "User who updated the Knowledge Page.", + "type": "string" + }, + "href": { + "description": "Link to this Knowledge Page resource.", + "$ref": "../../type/basic.json#/definitions/href" + }, + "changeDescription": { + "description": "Change that lead to this version of the entity.", + "$ref": "../../type/entityHistory.json#/definitions/changeDescription" + }, + "incrementalChangeDescription": { + "description": "Change that lead to this version of the entity.", + "$ref": "../../type/entityHistory.json#/definitions/changeDescription" + }, + "owners": { + "description": "Owners of this Knowledge Page.", + "$ref": "../../type/entityReferenceList.json", + "default": null + }, + "reviewers": { + "description": "User references of the reviewers for this tag.", + "$ref": "../../type/entityReferenceList.json" + }, + "entityStatus": { + "description": "Status of the tag.", + "$ref": "../../type/status.json" + }, + "followers": { + "description": "Followers of this Knowledge Page.", + "$ref": "../../type/entityReferenceList.json" + }, + "votes" : { + "description": "Votes for this Knowledge Page.", + "$ref": "../../type/votes.json" + }, + "tags": { + "description": "Tags for this SQL query.", + "type": "array", + "items": { + "$ref": "../../type/tagLabel.json" + }, + "default": null + }, + "pageType" : { + "description": "Type of the Knowledge Page.", + "$ref": "#/definitions/pageType" + }, + "page" : { + "description": "Knowledge Page Schema", + "oneOf": [ + { + "$ref": "./article.json" + }, + { + "$ref": "./quickLink.json" + } + ] + }, + "relatedEntities": { + "description": "Related Entities for the Knowledge Page", + "$ref": "../../type/entityReferenceList.json" + }, + "editors": { + "description": "List of users who are updating the entity", + "$ref": "../../type/entityReferenceList.json" + }, + "parent" : { + "description" : "Parent of this Knowledege Center.", + "$ref" : "../../type/entityReference.json" + }, + "children" : { + "description" : "Children of this Knowledge Center.", + "$ref" : "../../type/entityReferenceList.json" + }, + "childrenCount": { + "description": "Count of immediate children glossary terms.", + "type": "integer" + }, + "domains" : { + "description": "Fully qualified name of the domains the Knowledge Page belongs to.", + "$ref" : "../../type/entityReferenceList.json" + }, + "dataProducts" : { + "description": "List of data products this entity is part of.", + "$ref" : "../../type/entityReferenceList.json" + }, + "attachments": { + "description": "Attachments for the Knowledge Page", + "type": "array", + "items": { + "$ref": "../../attachments/asset.json" + }, + "default": null + } + }, + "required": ["name", "pageType", "page"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/pageHierarchy.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/pageHierarchy.json new file mode 100644 index 000000000000..4914b630d1e3 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/pageHierarchy.json @@ -0,0 +1,65 @@ +{ + "$id": "https://open-metadata.org/schema/entity/data/pageHierarchy.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Page Hierarchy", + "description": "This schema defines the Page entity with Hierarchy.", + "type": "object", + "javaType": "org.openmetadata.schema.entity.data.PageHierarchy", + "definitions": { + "pageHierarchyList": { + "type": "array", + "items": { + "$ref": "pageHierarchy.json" + }, + "default": null + } + }, + "properties": { + "id": { + "description": "Unique identifier for the Page.", + "$ref": "../../type/basic.json#/definitions/uuid" + }, + "pageType": { + "description": "Page type", + "$ref": "./page.json#/definitions/pageType" + }, + "name": { + "description": "A unique name of the Page/.", + "$ref": "../../type/basic.json#/definitions/entityName" + }, + "description": { + "description": "Description of the Page.", + "$ref": "../../type/basic.json#/definitions/markdown" + }, + "fullyQualifiedName": { + "description": "FullyQualifiedName same as `name`.", + "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" + }, + "displayName": { + "description": "Name used for display purposes", + "type": "string" + }, + "href": { + "description": "Link to the resource corresponding to this entity.", + "$ref": "../../type/basic.json#/definitions/href" + }, + "parent": { + "description": "Parent Knowledge Page.", + "$ref": "../../type/entityReference.json", + "default": null + }, + "children" : { + "excludedFromEqualsAndHashCode": true, + "description" : "Children of this Knowledge Page.", + "$ref" : "#/definitions/pageHierarchyList" + }, + "childrenCount" : { + "excludedFromEqualsAndHashCode": true, + "description" : "Children Count of the Pages", + "type" : "integer", + "default": 0 + } + }, + "required": ["id", "name"], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/quickLink.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/quickLink.json new file mode 100644 index 000000000000..bb47a409152e --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/quickLink.json @@ -0,0 +1,17 @@ +{ + "$id": "https://open-metadata.org/schema/entity/data/quickLink.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "QuickLink", + "description": "Quick Link Knowledge Page Store.", + "type": "object", + "javaType": "org.openmetadata.schema.entity.data.QuickLink", + "properties": { + "url": { + "description": "The URL or destination of the Quick Link.", + "type": "string", + "format": "uri" + } + }, + "required": ["url"], + "additionalProperties": false +} diff --git a/openmetadata-ui/src/main/resources/ui/playwright/constant/KnowledgeCenter.constant.ts b/openmetadata-ui/src/main/resources/ui/playwright/constant/KnowledgeCenter.constant.ts new file mode 100644 index 000000000000..bcf80a1aeb4c --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/constant/KnowledgeCenter.constant.ts @@ -0,0 +1,49 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// Editor shortcuts use Cmd on macOS and Ctrl on Linux/Windows. Previously hard-coded +// to "Meta+..." which maps to Cmd on macOS but to the Super/Windows key on Linux — +// that's why bold / undo / redo / copy / paste tests all failed on CI (Linux runners) +// while slash-command tests kept working: Linux Super+b is not a bold shortcut, but +// typing "/" followed by a block name does not need any modifier key at all. +const IS_MAC = process.platform === 'darwin'; +const MOD = IS_MAC ? 'Meta' : 'Control'; + +export const SHORTCUTS = { + bold: `${MOD}+b`, + italic: `${MOD}+i`, + code: `${MOD}+e`, + undo: `${MOD}+z`, + redo: `${MOD}+Shift+z`, + selectAll: `${MOD}+a`, + copy: `${MOD}+c`, + paste: `${MOD}+v`, + selectWord: `${MOD}+Shift+ArrowLeft`, + enter: 'Enter', + end: 'End', + backspace: 'Backspace', + tab: 'Tab', +} as const; + +export const SLASH_COMMANDS = { + h1: 'H1', + h2: 'H2', + h3: 'H3', + bullet: 'Bullet', + numbered: 'Numbered', + divider: 'Divider', + quote: 'Quote', + codeBlock: 'CodeBlock', + callout: 'Callout', + table: 'Table', + task: 'Task', +} as const; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/constant/sidebar.ts b/openmetadata-ui/src/main/resources/ui/playwright/constant/sidebar.ts index 0556dfdedb73..d7a7ae884644 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/constant/sidebar.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/constant/sidebar.ts @@ -31,6 +31,7 @@ export enum SidebarItem { LINEAGE = 'lineage', COLUMN_BULK_OPERATIONS = 'column-bulk-operations', DATA_MARKETPLACE = 'data-marketplace', + KNOWLEDGE_CENTER = 'knowledge-center', } export const SIDEBAR_LIST_ITEMS = { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/KnowledgeCenter.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/KnowledgeCenter.spec.ts new file mode 100644 index 000000000000..10d606f5be3a --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/KnowledgeCenter.spec.ts @@ -0,0 +1,1211 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { expect, test } from '@playwright/test'; +// import cryptoRandomString from 'crypto-random-string-with-promisify-polyfill'; +import { SidebarItem } from '../../constant/sidebar'; +import { EntityTypeEndpoint } from '../../support/entity/Entity.interface'; +import { EntityDataClass } from '../../support/entity/EntityDataClass'; +import { KnowledgeCenterClass } from '../../support/entity/KnowledgeCenterClass'; +import { TableClass } from '../../support/entity/TableClass'; +import { TopicClass } from '../../support/entity/TopicClass'; +import { UserClass } from '../../support/user/UserClass'; +import { + assignDataProduct, + assignSingleSelectDomain, + createNewPage, + descriptionBox, + getApiContext, + redirectToHomePage, + removeDataProduct, + // toastNotification, + uuid, +} from '../../utils/common'; +import { + addMultiOwner, + removeOwner, + visitEntityPage, + waitForAllLoadersToDisappear, +} from '../../utils/entity'; +import { + addTitle, + createMentionInConversation, + createQuickLink, + deletePage, + // readArticleData, + // readArticleInHierarchy, + readQuickLink, + updateBody, + updateDataAsset, + updateQuickLink, + updateTags, + verifyNotificationAndClick, +} from '../../utils/KnowledgeCenter'; +import { sidebarClick } from '../../utils/sidebar'; + +const knowledgePageArticle = { + title: `Playwright Article Title-${uuid()}`, + body: 'This is playwright article body here you can add rich text and block, it also support the slash command.', + tag: 'Article', + tagFqn: 'KnowledgeCenter.Article', + updatedBy: 'admin', + entityType: 'knowledgeCenter', +}; + +const knowledgePageQuickLink = { + displayName: `Playwright Quick Link Title-${uuid()}`, + updatedDisplayName: `Playwright Quick Link Title Updated-${uuid()}`, + description: 'This is playwright quick link body here you can add rich text', + updatedDescription: + 'This is playwright quick link body here you can add rich text updated', + url: 'https://docs.open-metadata.org', + updatedUrl: 'https://docs.open-metadata.org/quick-link', + tag: 'QuickLink', + tagFqn: 'KnowledgeCenter.QuickLink', +}; + +const dataAsset = new TopicClass(); +const tableAsset = new TableClass(); +const user = new UserClass(); + +const knowledgeCenter = new KnowledgeCenterClass({}, undefined, tableAsset); +const knowledgeCenter1 = new KnowledgeCenterClass(); +const knowledgeCenter2 = new KnowledgeCenterClass(); + +// use the admin user to login +test.use({ + storageState: 'playwright/.auth/admin.json', +}); + +test.describe('Knowledge Center', () => { + test.beforeAll('Setup pre-requests', async ({ browser }) => { + const { apiContext, afterAction } = await createNewPage(browser); + await user.create(apiContext); + await tableAsset.create(apiContext); + await dataAsset.create(apiContext); + await knowledgeCenter.create(apiContext, 15); + await knowledgeCenter1.create(apiContext, 2); + await afterAction(); + }); + + test.beforeEach(async ({ page }) => { + await redirectToHomePage(page); + }); + + test('Knowledge Center page', async ({ page }) => { + test.slow(true); + + await test.step('Article: Create, Read, Update and Delete', async () => { + const createKnowledgePage = page.waitForResponse( + '/api/v1/knowledgeCenter' + ); + + // visit knowledge center + await sidebarClick(page, SidebarItem.KNOWLEDGE_CENTER); + await page + .locator('[data-testid="left-panel"]') + .getByTestId('add-knowledge-page-btn') + .click(); + await page.getByRole('menuitem', { name: 'Article' }).click(); + await createKnowledgePage; + + // add title + await addTitle(page, knowledgePageArticle.title); + + await assignSingleSelectDomain( + page, + EntityDataClass.domain1.responseData + ); + + await assignDataProduct( + page, + EntityDataClass.domain1.responseData, + [EntityDataClass.dataProduct1.responseData], + 'Add' + ); + + await removeDataProduct(page, EntityDataClass.dataProduct1.responseData); + + // update owner + await addMultiOwner({ + page, + ownerNames: [user.responseData.displayName], + activatorBtnDataTestId: 'edit-owner', + endpoint: knowledgePageArticle.entityType as EntityTypeEndpoint, + type: 'Users', + }); + + // remove owner + await removeOwner({ + page, + endpoint: knowledgePageArticle.entityType as EntityTypeEndpoint, + ownerName: user.responseData.displayName, + type: 'Users', + dataTestId: 'add-owner', + }); + + // update owner + await addMultiOwner({ + page, + ownerNames: [user.responseData.displayName], + activatorBtnDataTestId: 'add-owner', + endpoint: knowledgePageArticle.entityType as EntityTypeEndpoint, + type: 'Users', + }); + + // update tags + await updateTags(page, { + tag: knowledgePageArticle.tag, + tagFqn: knowledgePageArticle.tagFqn, + }); + + // update data assets and view the data asset then navigate to article + await updateDataAsset(page, dataAsset, knowledgePageArticle.title); + + // Update body + await updateBody(page, knowledgePageArticle.body); + + // Read Article + // await readArticleData(page, knowledgePageArticle); + + // verify article in left panel + // await expect( + // page.locator(`[data-testid="page-node-${knowledgePageArticle.title}"]`) + // ).toBeVisible(); + + // verify bookmarked + // await expect( + // page.locator(`[data-testid="bookmarked-${knowledgePageArticle.title}"]`) + // ).toBeVisible(); + + // // verify recent viewed + // await expect( + // page.locator( + // `[data-testid="recent-viewed-${knowledgePageArticle.title}"]` + // ) + // ).toBeVisible(); + + // // verify the tag category + // await expect( + // page.locator( + // `[data-testid="tag-category-${knowledgePageArticle.tagFqn}-${knowledgePageArticle.title}"]` + // ) + // ).toBeVisible(); + + // await page + // .locator( + // `[data-testid="tag-category-${knowledgePageArticle.tagFqn}-${knowledgePageArticle.title}"]` + // ) + // .click(); + await deletePage(page, false, knowledgePageArticle.title); + }); + + await test.step('Quick Links: Create, Read, Update and Delete', async () => { + await sidebarClick(page, SidebarItem.KNOWLEDGE_CENTER); + + // Create Quick Link + await createQuickLink(page, knowledgePageQuickLink, dataAsset); + + // Read Quick Link + await readQuickLink(page, knowledgePageQuickLink); + + // Update Quick Link + await updateQuickLink(page, knowledgePageQuickLink); + + // verify the tag category + await expect( + page.locator( + `[data-testid="tag-category-${knowledgePageQuickLink.tagFqn}-${knowledgePageQuickLink.updatedDisplayName}"]` + ) + ).toBeVisible(); + + await page + .locator(`[data-testid="${knowledgePageQuickLink.updatedDisplayName}"]`) + .locator('[data-testid="delete-quick-link-btn"]') + .click(); + await deletePage(page, true); + }); + + await test.step('Related articles should be visible for an assets', async () => { + const { apiContext, afterAction } = await getApiContext(page); + + for (const page of knowledgeCenter.knowledgePages) { + if (page.id) { + await knowledgeCenter.addDataAssetToPage(apiContext, page.id); + } + } + await afterAction(); + + const waitForRelatedArticles = page.waitForResponse( + `/api/v1/knowledgeCenter?*` + ); + + await tableAsset.visitEntityPage(page); + + await waitForRelatedArticles; + + await expect( + page.locator('[data-testid="knowledge-pages"]') + ).toBeVisible(); + + const knowledgeCenterFilterPagePromise = page.waitForResponse( + `/api/v1/knowledgeCenter?fields=*&limit=15&entityId=${ + (tableAsset.entityResponseData as { id: string }).id + }&entityType=table` + ); + + await page + .locator('[data-testid="view-all-data-asset-related-articles"]') + .click(); + + await knowledgeCenterFilterPagePromise; + }); + + await test.step("Search for an article and navigate to it's page", async () => { + test.slow(true); + + const { knowledgePages } = knowledgeCenter.get(); + + for await (const knowledgePage of knowledgePages) { + await visitEntityPage({ + page, + searchTerm: knowledgePage?.['fullyQualifiedName'], + dataTestId: `${knowledgePage?.['fullyQualifiedName']}-${knowledgePage?.['fullyQualifiedName']}`, + }); + + await expect( + page.locator('[data-testid="entity-header-display-name"]') + ).toHaveValue(knowledgePage?.['fullyQualifiedName']); + } + + await page + .getByTestId('breadcrumb') + .getByRole('link', { name: 'Knowledge Center' }) + .click(); + }); + // TODO: Commented out due to performance issue with infinite scroll + // The readArticleInHierarchy function uses infinite scroll which takes 6-7 minutes + // with large data sets. UI needs to be enhanced to fix this issue. + // await test.step( + // 'Articles should be organized in a hierarchy, and the add and delete functions should work within that hierarchy.', + // async () => { + // test.slow(true); + + // // const { knowledgePages } = knowledgeCenter.get(); + + // // verify the hierarchy is visible + // await expect( + // page.locator('[data-testid="knowledge-pages-hierarchy"]') + // ).toBeVisible(); + + // for await (const knowledgePage of knowledgePages) { + // const articleResponse = page.waitForResponse( + // `/api/v1/knowledgeCenter/name/${knowledgePage?.['fullyQualifiedName']}?**` + // ); + + // await readArticleInHierarchy( + // page, + // knowledgePage?.['fullyQualifiedName'] + // ); + + // // navigate to the article page from the hierarchy node + // await page + // .locator( + // `[data-testid="page-node-${knowledgePage?.['fullyQualifiedName']}"]` + // ) + // .click(); + + // await articleResponse; + + // // verify the article page is visible + // await expect( + // page.locator('[data-testid="entity-header-display-name"]') + // ).toHaveValue(knowledgePage?.['fullyQualifiedName']); + // } + + // // add and delete a page from the hierarchy + + // const { knowledgePages: knowledgePages1 } = knowledgeCenter1.get(); + + // for await (const knowledgePage of knowledgePages1) { + // const createKnowledgePage = page.waitForResponse( + // '/api/v1/knowledgeCenter' + // ); + // const articleResponse = page.waitForResponse( + // `/api/v1/knowledgeCenter/name/${knowledgePage?.['fullyQualifiedName']}?**` + // ); + + // // TODO: Commented out due to performance issue with infinite scroll + // // The readArticleInHierarchy function uses infinite scroll which takes 6-7 minutes + // // with large data sets. UI needs to be enhanced to fix this performance issue. + // // await readArticleInHierarchy( + // // page, + // // knowledgePage?.['fullyQualifiedName'] + // // ); + + // await page + // .locator( + // `[data-testid="page-node-${knowledgePage?.['fullyQualifiedName']}"]` + // ) + // .click(); + + // await articleResponse; + + // // TODO: Commented out due to performance issue with infinite scroll + // // The readArticleInHierarchy function uses infinite scroll which takes 6-7 minutes + // // with large data sets. UI needs to be enhanced to fix this issue. + // // await readArticleInHierarchy( + // // page, + // // knowledgePage?.['fullyQualifiedName'] + // // ); + + // await page + // .locator( + // `[data-testid="page-node-${knowledgePage?.['fullyQualifiedName']}"]` + // ) + // .hover(); + + // // add a page + // await page + // .locator( + // `[data-testid="${knowledgePage?.['fullyQualifiedName']}-add-page-btn"]` + // ) + // .click(); + + // await expect( + // page.getByTestId('entity-header-display-name') + // ).toBeEmpty(); + + // const title = `${knowledgePage?.['fullyQualifiedName']} Article Title`; + + // // update the title of the created page + // await addTitle(page, title); + + // await page.reload(); + // await page.waitForLoadState('networkidle'); + // await page.waitForSelector('[data-testid="loader"]', { + // state: 'detached', + // }); + + // // verify the created page is visible on the hierarchy with the updated title + // // TODO: Commented out due to performance issue with infinite scroll + // // The readArticleInHierarchy function uses infinite scroll which takes 6-7 minutes + // // with large data sets. UI needs to be enhanced to fix this performance issue. + // // await readArticleInHierarchy(page, title); + + // await expect( + // page.locator(`[data-testid="page-node-${title}"]`) + // ).toBeVisible(); + + // await createKnowledgePage; + + // await redirectToHomePage(page); + // await sidebarClick(page, SidebarItem.KNOWLEDGE_CENTER); + + // await expect( + // page.getByTestId('left-panel').getByTestId('add-knowledge-page-btn') + // ).toBeVisible(); + + // const loadHierarchy1 = page.waitForResponse( + // `/api/v1/knowledgeCenter/search/hierarchy?parent=${knowledgePage?.['fullyQualifiedName']}&pageType=Article&offset=0&limit=100` + // ); + + // // TODO: Commented out due to performance issue with infinite scroll + // // The readArticleInHierarchy function uses infinite scroll which takes 6-7 minutes + // // with large data sets. UI needs to be enhanced to fix this performance issue. + // // await readArticleInHierarchy( + // // page, + // // knowledgePage?.['fullyQualifiedName'] + // // ); + + // await page + // .locator( + // `[data-testid="${knowledgePage?.['fullyQualifiedName']}-collapse-icon"]` + // ) + // .click(); + + // const hierarchyResponse = await loadHierarchy1; + + // const hierarchyData = await hierarchyResponse.json(); + + // // verify the hierarchy should have only one child + // expect(hierarchyData.data).toHaveLength(1); + + // await page + // .locator( + // `[data-testid="page-node-${knowledgePage?.['fullyQualifiedName']}"]` + // ) + // .click(); + + // // check if created page is loaded for current hierarchy + + // // TODO: Commented out due to performance issue with infinite scroll + // // The readArticleInHierarchy function uses infinite scroll which takes 6-7 minutes + // // with large data sets. UI needs to be enhanced to fix this performance issue. + // // await readArticleInHierarchy( + // // page, + // // knowledgePage?.['fullyQualifiedName'] + // // ); + + // await page + // .locator( + // `[data-testid="page-node-${knowledgePage?.['fullyQualifiedName']}"]` + // ) + // .hover(); + + // // delete a page + // await page + // .locator( + // `[data-testid="${knowledgePage?.['fullyQualifiedName']}-delete-page-btn"]` + // ) + // .click(); + + // await page.waitForSelector('[role="dialog"].ant-modal'); + + // await expect(page.locator('[role="dialog"].ant-modal')).toBeVisible(); + + // await page.click('[data-testid="hard-delete-option"]'); + // await page.check('[data-testid="hard-delete"]'); + // await page.fill('[data-testid="confirmation-text-input"]', 'DELETE'); + + // const deleteResponse = page.waitForResponse( + // '/api/v1/knowledgeCenter/*?hardDelete=true&recursive=true' + // ); + + // await page.click('[data-testid="confirm-button"]'); + + // await deleteResponse; + + // await toastNotification(page, /deleted successfully!/); + // } + // } + // ); + }); + + test('Verify Left Panel hierarchy pagination functionality', async ({ + page, + browser, + }) => { + test.slow(true); + + try { + const { apiContext } = await createNewPage(browser); + await knowledgeCenter2.create(apiContext, 10); + + await sidebarClick(page, SidebarItem.KNOWLEDGE_CENTER); + + // Get the first element content before scrolling + const firstElementBeforeScroll = page + .locator('[data-testid="knowledge-pages-hierarchy"] .ant-tree-treenode') + .first(); + const paginationResponse = page.waitForResponse( + (response) => + response.url().includes('/api/v1/knowledgeCenter/search/hierarchy') && + response.url().includes(`offset=100`) && + response.url().includes('limit=100') && + response.status() === 200 + ); + + const scrollHeight = await page + .locator( + '[data-testid="knowledge-pages-hierarchy"] .ant-tree-list-holder > div' + ) + .evaluate((element) => element.scrollHeight); + + await page.locator('[data-testid="knowledge-pages-hierarchy"]').hover(); + await page.mouse.wheel(0, scrollHeight); + await paginationResponse; + + // Wait a bit for any potential scroll resets + await page.waitForTimeout(1000); + + // Get the first element content after scrolling + const firstElementAfterScroll = await page + .locator('[data-testid="knowledge-pages-hierarchy"] .ant-tree-treenode') + .first() + .textContent(); + + // Verify that the first element content is not same on before and after scroll + // scroll was being reset in left panel due to use of activeKey in DirectoryTree component + // Verify that the first element content is not same on before and after scroll + // scroll was being reset in left panel due to use of activeKey in DirectoryTree component + expect(firstElementBeforeScroll).not.toBe(firstElementAfterScroll); + } finally { + const { apiContext, afterAction } = await createNewPage(browser); + await knowledgeCenter2.delete(apiContext); + await afterAction(); + } + }); + + test('Activity feed functionality in Knowledge Center page', async ({ + page, + }) => { + const createKnowledgePage = page.waitForResponse('/api/v1/knowledgeCenter'); + + await sidebarClick(page, SidebarItem.KNOWLEDGE_CENTER); + await page + .locator('[data-testid="left-panel"]') + .getByTestId('add-knowledge-page-btn') + .click(); + await page.getByRole('menuitem', { name: 'Article' }).click(); + await createKnowledgePage; + + await addTitle(page, knowledgePageArticle.title); + + await page.waitForSelector('[data-testid="entity-header-display-name"]'); + + await test.step('Create a new conversation thread', async () => { + await page.locator('[data-testid="conversation"]').click(); + + await page.locator('.feed-drawer').waitFor({ state: 'visible' }); + + await page.getByRole('tab', { name: 'Conversations' }).click(); + + await page.waitForSelector('[data-testid="editor-wrapper"]'); + + const conversationMessage = `Test Conversation Message ${uuid()}`; + await page + .locator('[data-testid="editor-wrapper"] [contenteditable="true"]') + .click(); + await page + .locator('[data-testid="editor-wrapper"] [contenteditable="true"]') + .fill(conversationMessage); + + const feedResponse = page.waitForResponse('/api/v1/feed'); + + await page.locator('[data-testid="send-button"]').click(); + + await feedResponse; + + await expect(page.locator(`text=${conversationMessage}`)).toBeVisible(); + }); + + await test.step('Test post update functionality', async () => { + await page.locator('[data-testid="main-message"]').hover(); + + await page.locator('[data-testid="edit-message"]').click(); + + await page.waitForSelector('[data-testid="editor-wrapper"]'); + + const updatedMessage = `Updated Thread Message ${uuid()}`; + await page + .locator('[data-testid="editor-wrapper"] [contenteditable="true"]') + .click(); + await page + .locator('[data-testid="editor-wrapper"] [contenteditable="true"]') + .clear(); + await page + .locator('[data-testid="editor-wrapper"] [contenteditable="true"]') + .fill(updatedMessage); + + const updateThreadPromise = page.waitForResponse( + (response) => + response.url().includes('/api/v1/feed') && + response.request().method() === 'PATCH' + ); + + await page.locator('[data-testid="save-button"]').click(); + + await updateThreadPromise; + + await expect(page.locator(`text=${updatedMessage}`)).toBeVisible(); + }); + + await test.step('Test post deletion functionality', async () => { + await page.locator('[data-testid="main-message"]').hover(); + + await page.locator('[data-testid="delete-message"]').click(); + + const deletePostPromise = page.waitForResponse( + (response) => + response.url().includes('/api/v1/feed') && + response.request().method() === 'DELETE' + ); + + await page.waitForSelector('[role="dialog"].ant-modal', { + state: 'visible', + }); + + await page.locator('[data-testid="save-button"]').click(); + + await deletePostPromise; + + await expect( + page.locator('[data-testid="main-message"]') + ).not.toBeVisible(); + }); + }); + + test('User Mentions in article and redirect should work of Knowledge Center page', async ({ + page, + }) => { + const createKnowledgePage = page.waitForResponse('/api/v1/knowledgeCenter'); + + await sidebarClick(page, SidebarItem.KNOWLEDGE_CENTER); + await page + .locator('[data-testid="left-panel"]') + .getByTestId('add-knowledge-page-btn') + .click(); + await page.getByRole('menuitem', { name: 'Article' }).click(); + await createKnowledgePage; + + await addTitle(page, knowledgePageArticle.title); + + await page.waitForSelector('[data-testid="entity-header-display-name"]'); + + await test.step('Create a new conversation thread with mention', async () => { + const conversationMessage = `Test Conversation Message with mention ${uuid()}`; + + await createMentionInConversation(page, 'admin', conversationMessage); + }); + + await test.step('Verify notification appears and click to navigate', async () => { + await redirectToHomePage(page); + // Verify notification and click to navigate to the article + await verifyNotificationAndClick( + page, + 'admin', + knowledgePageArticle.title + ); + + await waitForAllLoadersToDisappear(page); + + // Verify we are redirected to the correct article page + await expect( + page.locator('[data-testid="entity-header-display-name"]') + ).toHaveValue(knowledgePageArticle.title); + }); + }); + + test('Article mentions in description should working for Knowledge Center', async ({ + page, + }) => { + const { apiContext, afterAction } = await getApiContext(page); + await knowledgeCenter.addDataAssetToPage(apiContext); + await afterAction(); + + const waitForRelatedArticles = page.waitForResponse( + `/api/v1/knowledgeCenter?*` + ); + + await dataAsset.visitEntityPage(page); + + await waitForRelatedArticles; + + await page.getByTestId('edit-description').click(); + await page.locator(descriptionBox).first().click(); + await page.locator(descriptionBox).first().clear(); + + const mentionResponse = page.waitForResponse('/api/v1/search/query?**'); + await page + .locator(descriptionBox) + .first() + .fill(`#${knowledgeCenter.knowledgePages[0].displayName}`); + await mentionResponse; + + await page + .getByTestId( + `hash-mention-${knowledgeCenter.knowledgePages[0].displayName}` + ) + .click(); + await page.getByTestId('save').click(); + + await page.waitForSelector('[role="dialog"].description-markdown-editor', { + state: 'hidden', + }); + + const element = page.locator( + `[data-label=${knowledgeCenter.knowledgePages[0].displayName}]` + ); + + const href = await element.getAttribute('href'); + + expect(href).toMatch( + new RegExp(`/knowledge-center/${knowledgeCenter.knowledgePages[0].name}$`) + ); + }); + + test('Explore Filter by Knowledge Center', async ({ page }) => { + await sidebarClick(page, SidebarItem.EXPLORE); + + await waitForAllLoadersToDisappear(page); + + await expect( + page.getByTestId('explore-tree-title-Knowledge Center') + ).toContainText('Knowledge Center'); + + await page + .locator('div') + .filter({ hasText: /^Knowledge Center$/ }) + .locator('svg') + .first() + .click(); + + await expect( + page.getByTestId('explore-tree-title-Knowledge Page') + ).toContainText('Knowledge Page'); + + const apiRes = page.waitForResponse( + '/api/v1/search/query?q=&index=dataAsset*' + ); + + await page.getByTestId('explore-tree-title-Knowledge Page').click(); + + const response = await apiRes; + const responseData = await response.json(); + + expect(responseData.hits.total.value).toBeGreaterThan(0); + + await expect(page.getByTestId('search-dropdown-Data Assets')).toContainText( + 'Data Assets: page' + ); + + await expect( + page.getByTestId('search-error-placeholder') + ).not.toBeVisible(); + }); + + // TODO: Commented out due to performance issue with infinite scroll it's taking too long to load the data. + // The readArticleInHierarchy function uses infinite scroll which takes 6-7 minutes + // with large data sets. UI needs to be enhanced to fix this performance issue. + // test('New article should be visible in left panel when created using add page button', async ({ + // page, + // browser, + // }) => { + // test.slow(true); + + // const { apiContext } = await createNewPage(browser); + + // // Create a parent article to which we'll add a child + // const parentArticle = await apiContext.post('/api/v1/knowledgeCenter', { + // data: { + // name: `Article_${cryptoRandomString({ + // length: 8, + // type: 'alphanumeric', + // })}`, + // displayName: `Parent_Article_${cryptoRandomString({ + // length: 8, + // type: 'alphanumeric', + // })}`, + // description: 'Parent Article for testing add page', + // pageType: 'Article', + // page: { + // publicationDate: new Date(), + // relatedArticles: [], + // }, + // owners: [ + // { + // type: 'user', + // id: user.responseData.id, + // }, + // ], + // }, + // }); + + // const parentArticleData = await parentArticle.json(); + + // try { + // // Navigate to knowledge center + // await sidebarClick(page, SidebarItem.KNOWLEDGE_CENTER); + + // // Wait for hierarchy to load + // await page.waitForSelector('[data-testid="knowledge-pages-hierarchy"]'); + + // await test.step('Find parent article in hierarchy', async () => { + // // Find the parent article in the hierarchy using the helper function + // // TODO: Commented out due to performance issue with infinite scroll + // // The readArticleInHierarchy function uses infinite scroll which takes 6-7 minutes + // // with large data sets. UI needs to be enhanced to fix this performance issue. + // // await readArticleInHierarchy(page, parentArticleData.displayName); + // // Verify parent article is visible + // // await expect( + // // page.locator( + // // `[data-testid="page-node-${parentArticleData.displayName}"]` + // // ) + // // ).toBeVisible(); + // }); + + // await test.step( + // 'Create new article using add page button and verify visibility', + // async () => { + // // Wait for the create knowledge page API call + // const createKnowledgePagePromise = page.waitForResponse( + // '/api/v1/knowledgeCenter' + // ); + + // // Hover over the parent article to reveal the add button + // await page + // .locator( + // `[data-testid="page-node-${parentArticleData.displayName}"]` + // ) + // .hover(); + + // // Click the add page button for the parent article + // await page + // .locator( + // `[data-testid="${parentArticleData.displayName}-add-page-btn"]` + // ) + // .click(); + + // // Wait for the API call to complete + // await createKnowledgePagePromise; + + // // Verify we're on a new article page with empty title + // await expect( + // page.getByTestId('entity-header-display-name') + // ).toBeEmpty(); + + // // Add a title to the newly created article + // const newArticleTitle = `New Article ${cryptoRandomString({ + // length: 8, + // type: 'alphanumeric', + // })}`; + // await addTitle(page, newArticleTitle); + + // // Verify the newly created article is visible in the left panel hierarchy + // // TODO: Commented out due to performance issue with infinite scroll + // // The readArticleInHierarchy function uses infinite scroll which takes 6-7 minutes + // // with large data sets. UI needs to be enhanced to fix this performance issue. + // // await readArticleInHierarchy(page, newArticleTitle); + + // // await expect( + // // page.locator(`[data-testid="page-node-${newArticleTitle}"]`) + // // ).toBeVisible(); + + // // Verify the article is nested under the parent (check parent is expanded) + // await test.step('Verify article hierarchy structure', async () => { + // // The new article should be visible as a child of the parent + // // Click on the new article to navigate to it + // await page + // .locator(`[data-testid="page-node-${newArticleTitle}"]`) + // .click(); + + // await page.waitForLoadState('networkidle'); + + // // Verify we're on the correct article page + // await expect( + // page.locator('[data-testid="entity-header-display-name"]') + // ).toHaveValue(newArticleTitle); + + // // Navigate back to knowledge center + // await sidebarClick(page, SidebarItem.KNOWLEDGE_CENTER); + + // // Verify parent can be collapsed and the child disappears + // await page + // .locator( + // `[data-testid="${parentArticleData.displayName}-collapse-icon"]` + // ) + // .click(); + + // await page.waitForTimeout(500); + + // // Child should not be visible when parent is collapsed + // await expect( + // page.locator(`[data-testid="page-node-${newArticleTitle}"]`) + // ).not.toBeVisible(); + + // // Expand parent again + // await page + // .locator( + // `[data-testid="${parentArticleData.displayName}-collapse-icon"]` + // ) + // .click(); + + // await page.waitForTimeout(500); + + // // Child should be visible again when parent is expanded + // await expect( + // page.locator(`[data-testid="page-node-${newArticleTitle}"]`) + // ).toBeVisible(); + // }); + + // // Clean up: Delete the created article + // await page + // .locator(`[data-testid="page-node-${newArticleTitle}"]`) + // .click(); + + // await page.waitForLoadState('networkidle'); + + // await deletePage(page); + // } + // ); + // } finally { + // // Clean up: Delete the parent article + // const { apiContext: cleanupContext, afterAction } = await createNewPage( + // browser + // ); + // await cleanupContext.delete( + // `/api/v1/knowledgeCenter/${parentArticleData.id}?hardDelete=true&recursive=true` + // ); + // await afterAction(); + // } + // }); + + // TODO: Commented out due to performance issue with infinite scroll it's taking too long to load the data. + // The readArticleInHierarchy function uses infinite scroll which takes 6-7 minutes + // with large data sets. UI needs to be enhanced to fix this performance issue. + // await readArticleInHierarchy(page, grandChildArticleData.displayName); + + // test('Nested hierarchy navigation with activeFqn should expand parent nodes', async ({ + // page, + // browser, + // }) => { + // test.slow(true); + + // const { apiContext } = await createNewPage(browser); + + // // Create a parent article + // const parentArticle = await apiContext.post('/api/v1/knowledgeCenter', { + // data: { + // name: `Article_${cryptoRandomString({ + // length: 8, + // type: 'alphanumeric', + // })}`, + // displayName: `Parent_Article${cryptoRandomString({ + // length: 8, + // type: 'alphanumeric', + // })}`, + // description: 'Parent Article Description', + // pageType: 'Article', + // page: { + // publicationDate: new Date(), + // relatedArticles: [], + // }, + // owners: [ + // { + // type: 'user', + // id: user.responseData.id, + // }, + // ], + // }, + // }); + + // const parentArticleData = await parentArticle.json(); + + // // Create a child article under parent + // const childArticle = await apiContext.post('/api/v1/knowledgeCenter', { + // data: { + // name: `Article_${cryptoRandomString({ + // length: 8, + // type: 'alphanumeric', + // })}`, + // displayName: `Child_Article${cryptoRandomString({ + // length: 8, + // type: 'alphanumeric', + // })}`, + // description: 'Child Article Description', + // pageType: 'Article', + // page: { + // publicationDate: new Date(), + // relatedArticles: [], + // }, + // owners: [ + // { + // type: 'user', + // id: user.responseData.id, + // }, + // ], + // parent: { + // id: parentArticleData.id, + // type: 'page', + // }, + // }, + // }); + + // const childArticleData = await childArticle.json(); + + // // Create a grandchild article under child + // const grandChildArticle = await apiContext.post('/api/v1/knowledgeCenter', { + // data: { + // name: `Article_${cryptoRandomString({ + // length: 8, + // type: 'alphanumeric', + // })}`, + // displayName: `GrandChild_Article${cryptoRandomString({ + // length: 8, + // type: 'alphanumeric', + // })}`, + // description: 'GrandChild Article Description', + // pageType: 'Article', + // page: { + // publicationDate: new Date(), + // relatedArticles: [], + // }, + // owners: [ + // { + // type: 'user', + // id: user.responseData.id, + // }, + // ], + // parent: { + // id: childArticleData.id, + // type: 'page', + // }, + // }, + // }); + + // const grandChildArticleData = await grandChildArticle.json(); + // const grandChildArticleFqn = grandChildArticleData.fullyQualifiedName; + + // // Navigate to knowledge center + // await sidebarClick(page, SidebarItem.KNOWLEDGE_CENTER); + + // await test.step( + // 'Navigate directly to grandchild article using activeFqn', + // async () => { + // // Wait for the hierarchy API call with activeFqn + // const hierarchyWithActiveFqnResponse = page.waitForResponse( + // (response) => + // response + // .url() + // .includes('/api/v1/knowledgeCenter/search/hierarchy') && + // response.url().includes(`activeFqn=${grandChildArticleFqn}`) + // ); + + // // Navigate directly to grandchild article + // await page.goto(`/knowledge-center/${grandChildArticleFqn}`, { + // waitUntil: 'networkidle', + // }); + + // await hierarchyWithActiveFqnResponse; + + // // Wait for page to load + // await page.waitForSelector('[data-testid="loader"]', { + // state: 'detached', + // }); + + // // Verify grandchild article is loaded + // await expect( + // page.locator('[data-testid="entity-header-display-name"]') + // ).toHaveValue(grandChildArticleData.displayName); + + // // TODO: Commented out due to performance issue with infinite scroll + // // The readArticleInHierarchy function uses infinite scroll which takes 6-7 minutes + // // with large data sets. UI needs to be enhanced to fix this performance issue. + // // await readArticleInHierarchy(page, grandChildArticleData.displayName); + // } + // ); + + // await test.step( + // 'Verify parent and child nodes are expanded in hierarchy', + // async () => { + // // Verify parent node is visible and expanded + // await expect( + // page.locator( + // `[data-testid="page-node-${parentArticleData.displayName}"]` + // ) + // ).toBeVisible(); + + // // Verify child node is visible and expanded (nested under parent) + // await expect( + // page.locator( + // `[data-testid="page-node-${childArticleData.displayName}"]` + // ) + // ).toBeVisible(); + + // // Verify grandchild node is visible (nested under child) + // await expect( + // page.locator( + // `[data-testid="page-node-${grandChildArticleData.displayName}"]` + // ) + // ).toBeVisible(); + + // // Verify the parent node collapse icon is showing as expanded + // await expect( + // page.locator( + // `[data-testid="${parentArticleData.displayName}-collapse-icon"]` + // ) + // ).toBeVisible(); + + // // Verify the child node collapse icon is showing as expanded + // await expect( + // page.locator( + // `[data-testid="${childArticleData.displayName}-collapse-icon"]` + // ) + // ).toBeVisible(); + // } + // ); + + // await test.step('Verify hierarchy structure is correct', async () => { + // // Click on parent to collapse + // await page + // .locator( + // `[data-testid="${parentArticleData.displayName}-collapse-icon"]` + // ) + // .click(); + + // // Wait a bit for collapse animation + // await page.waitForTimeout(500); + + // // Verify child and grandchild are not visible after collapse + // await expect( + // page.locator( + // `[data-testid="page-node-${childArticleData.displayName}"]` + // ) + // ).not.toBeVisible(); + + // await expect( + // page.locator(`[data-testid="page-node-${grandChildArticleFqn}"]`) + // ).not.toBeVisible(); + + // // Expand parent again + // await page + // .locator( + // `[data-testid="${parentArticleData.displayName}-collapse-icon"]` + // ) + // .click(); + + // // Wait for expand animation + // await page.waitForTimeout(500); + + // // Verify child is visible again + // await expect( + // page.locator( + // `[data-testid="page-node-${childArticleData.displayName}"]` + // ) + // ).toBeVisible(); + + // // Verify grandchild is visible again + // await expect( + // page.locator( + // `[data-testid="page-node-${grandChildArticleData.displayName}"]` + // ) + // ).toBeVisible(); + // }); + + // await test.step('Navigate between nested articles', async () => { + // // Click on child article + // await page + // .locator(`[data-testid="page-node-${childArticleData.displayName}"]`) + // .click(); + + // await page.waitForLoadState('networkidle'); + + // // Verify child article is loaded + // await expect( + // page.locator('[data-testid="entity-header-display-name"]') + // ).toHaveValue(childArticleData.displayName); + + // // Click on parent article + // await page + // .locator(`[data-testid="page-node-${parentArticleData.displayName}"]`) + // .click(); + + // await page.waitForLoadState('networkidle'); + + // // Verify parent article is loaded + // await expect( + // page.locator('[data-testid="entity-header-display-name"]') + // ).toHaveValue(parentArticleData.displayName); + // }); + // }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/KnowledgeCenterList.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/KnowledgeCenterList.spec.ts new file mode 100644 index 000000000000..00da20b30e0c --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/KnowledgeCenterList.spec.ts @@ -0,0 +1,328 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { expect, test } from '@playwright/test'; +import { SidebarItem } from '../../constant/sidebar'; +import { KnowledgeCenterClass } from '../../support/entity/KnowledgeCenterClass'; +import { createNewPage, redirectToHomePage } from '../../utils/common'; +import { waitForAllLoadersToDisappear } from '../../utils/entity'; +import { + getKnowledgePageCardByIndex, + getKnowledgePageCardEntityIdentifier, + toggleKnowledgePageBookmark, +} from '../../utils/KnowledgeCenter'; +import { sidebarClick } from '../../utils/sidebar'; + +test.use({ + storageState: 'playwright/.auth/admin.json', +}); + +// 7 cards needed: tests use indices 0-6 +const MIN_CARDS = 7; +const knowledgeCenter = new KnowledgeCenterClass(); + +test.describe('Knowledge Center List', () => { + test.slow(true); + + test.beforeAll(async ({ browser }) => { + const { apiContext, afterAction } = await createNewPage(browser); + await knowledgeCenter.create(apiContext, MIN_CARDS); + await afterAction(); + }); + + test.afterAll(async ({ browser }) => { + const { apiContext, afterAction } = await createNewPage(browser); + await knowledgeCenter.delete(apiContext); + await afterAction(); + }); + + test.beforeEach(async ({ page }) => { + await redirectToHomePage(page); + const listResponse = page.waitForResponse('/api/v1/knowledgeCenter*'); + await sidebarClick(page, SidebarItem.KNOWLEDGE_CENTER); + await listResponse; + await page + .getByTestId('knowledge-page-listing') + .waitFor({ state: 'visible' }); + }); + + test('Knowledge Center List - Verify list visibility and card functionality', async ({ + page, + }) => { + await test.step('Verify knowledge page listing is visible', async () => { + const listing = page.getByTestId('knowledge-page-listing'); + await expect(listing).toBeVisible(); + }); + + await test.step('Verify card has content (title, description, metadata)', async () => { + const card = await getKnowledgePageCardByIndex(page, 0); + await expect(card).toBeVisible(); + + const title = card.getByTestId('entity-header-display-name'); + await expect(title).toBeVisible(); + await expect(title).not.toBeEmpty(); + + const titleDescription = card.getByTestId('knowledge-title-description'); + await expect(titleDescription).toBeVisible(); + + const knowledgePageLink = card.getByTestId('knowledge-page-link'); + await expect(knowledgePageLink).toBeVisible(); + + const dateOwnerCol = card.getByTestId('date-owner-col'); + await expect(dateOwnerCol).toBeVisible(); + + const updatedAt = card.getByTestId('updated-at'); + await expect(updatedAt).toBeVisible(); + await expect(updatedAt).not.toBeEmpty(); + + const metadata = card.getByTestId('knowledge-metadata'); + + await expect(metadata).toBeVisible(); + }); + }); + + test('Knowledge Center List - Test upvote and downvote buttons', async ({ + page, + }) => { + const card = await getKnowledgePageCardByIndex(page, 1); + await expect(card).toBeVisible(); + + // Get initial up-vote count + const initialUpVoteCount = Number.parseInt( + (await card.getByTestId('up-vote-count').textContent()) || '0', + 10 + ); + + const upVoteBtn = card.getByTestId('up-vote-btn'); + const upVoteResponse = page.waitForResponse( + '/api/v1/knowledgeCenter/*/vote' + ); + await upVoteBtn.click(); + await upVoteResponse; + await waitForAllLoadersToDisappear(page); + + const expectedUpVoteCount = initialUpVoteCount + 1; + await expect(card.getByTestId('up-vote-count')).toHaveText( + String(expectedUpVoteCount) + ); + + // Re-read down count after upvote — if the user had an existing downvote, + // the upvote action clears it, making a pre-upvote baseline stale. + const downVoteCountAfterUpvote = Number.parseInt( + (await card.getByTestId('down-vote-count').textContent()) || '0', + 10 + ); + + const downVoteBtn = card.getByTestId('down-vote-btn'); + const downVoteResponse = page.waitForResponse( + '/api/v1/knowledgeCenter/*/vote' + ); + await downVoteBtn.click(); + await downVoteResponse; + await waitForAllLoadersToDisappear(page); + + await expect(card.getByTestId('up-vote-count')).toHaveText( + String(expectedUpVoteCount - 1) + ); + await expect(card.getByTestId('down-vote-count')).toHaveText( + String(downVoteCountAfterUpvote + 1) + ); + }); + + test('Knowledge Center List - Test bookmark functionality', async ({ + page, + }) => { + const card = await getKnowledgePageCardByIndex(page, 2); + await expect(card).toBeVisible(); + + const bookmarkIdentifier = await getKnowledgePageCardEntityIdentifier(card); + + const bookmarkBtn = card.getByTestId('bookmark-btn'); + await expect(bookmarkBtn).toBeVisible(); + + await toggleKnowledgePageBookmark( + page, + bookmarkBtn, + bookmarkIdentifier, + true + ); + + const unbookmarkResponse = page.waitForResponse((response) => { + const url = response.url(); + return ( + url.includes('/api/v1/knowledgeCenter') && url.includes('/followers') + ); + }); + + await bookmarkBtn.click(); + const unbookmarkRes = await unbookmarkResponse; + expect(unbookmarkRes.status()).toBe(200); + await waitForAllLoadersToDisappear(page); + + const rightPanel = page.getByTestId('knowledge-center-right-panel'); + const specificBookmark = rightPanel.getByTestId( + `bookmarked-${bookmarkIdentifier}` + ); + await expect(specificBookmark).not.toBeVisible(); + }); + + test('Knowledge Center List - Verify Recently Viewed widget', async ({ + page, + }) => { + const card = await getKnowledgePageCardByIndex(page, 3); + await expect(card).toBeVisible(); + + const cardIdentifier = await getKnowledgePageCardEntityIdentifier(card); + const cardDisplayText = + ( + await card.getByTestId('entity-header-display-name').textContent() + )?.trim() ?? ''; + + const knowledgePageLink = card.getByTestId('knowledge-page-link'); + + const navigationPromise = page.waitForURL((url) => + url.pathname.includes('/knowledge-center/') + ); + + await knowledgePageLink.click(); + await navigationPromise; + await waitForAllLoadersToDisappear(page); + await page.waitForSelector('.ant-skeleton-active', { + state: 'detached', + }); + + const entityHeader = page.getByTestId('entity-header-display-name'); + await expect(entityHeader).toBeVisible(); + + await waitForAllLoadersToDisappear(page); + + const listResponse = page.waitForResponse('/api/v1/knowledgeCenter*'); + await sidebarClick(page, SidebarItem.KNOWLEDGE_CENTER); + await listResponse; + await page + .getByTestId('knowledge-page-listing') + .waitFor({ state: 'visible' }); + + await waitForAllLoadersToDisappear(page); + + const rightPanel = page.getByTestId('knowledge-center-right-panel'); + await expect(rightPanel).toBeVisible(); + + await expect(rightPanel.getByText('Recently Viewed')).toBeVisible(); + + const recentlyViewedItem = rightPanel.getByTestId( + `recent-viewed-${cardIdentifier}` + ); + + await recentlyViewedItem.scrollIntoViewIfNeeded(); + await expect(recentlyViewedItem).toBeVisible(); + + const recentViewNavigationPromise = page.waitForURL((url) => + url.pathname.includes('/knowledge-center/') + ); + await recentlyViewedItem.click(); + await recentViewNavigationPromise; + + await expect(entityHeader).toBeVisible(); + // "Untitled" is the UI placeholder; the actual textarea value is empty string + const expectedValue = cardDisplayText === 'Untitled' ? '' : cardDisplayText; + await expect(entityHeader).toHaveValue(expectedValue); + }); + + test('Knowledge Center List - Test infinite scroll/pagination', async ({ + page, + }) => { + const listing = page.getByTestId('knowledge-page-listing'); + const cards = listing.locator('.knowledge-card'); + const initialCardCount = await cards.count(); + + const observerElement = page.getByTestId('observer-element'); + const paginationResponse = page.waitForResponse( + (response) => + response.url().includes('/api/v1/knowledgeCenter') && + response.url().includes('after=') + ); + + await observerElement.scrollIntoViewIfNeeded(); + await paginationResponse; + + await waitForAllLoadersToDisappear(page); + + const finalCardCount = await cards.count(); + expect(finalCardCount).toBeGreaterThan(initialCardCount); + }); + + test('Knowledge Center List - Test unbookmark functionality', async ({ + page, + }) => { + const card = await getKnowledgePageCardByIndex(page, 5); + await expect(card).toBeVisible(); + + const bookmarkIdentifier = await getKnowledgePageCardEntityIdentifier(card); + + const bookmarkBtn = card.getByTestId('bookmark-btn'); + await expect(bookmarkBtn).toBeVisible(); + + await toggleKnowledgePageBookmark( + page, + bookmarkBtn, + bookmarkIdentifier, + true + ); + await toggleKnowledgePageBookmark( + page, + bookmarkBtn, + bookmarkIdentifier, + false + ); + }); + + test('Knowledge Center List - Test add article button', async ({ page }) => { + const addButton = page.getByTestId('add-knowledge-page-btn'); + await expect(addButton).toBeVisible(); + + await addButton.click(); + await page.waitForSelector('.ant-dropdown', { state: 'visible' }); + + const articleOption = page.getByRole('menuitem', { name: 'Article' }); + await expect(articleOption).toBeVisible(); + + const quickLinkOption = page.getByRole('menuitem', { name: 'Quick Link' }); + await expect(quickLinkOption).toBeVisible(); + + const createResponse = page.waitForResponse('/api/v1/knowledgeCenter'); + await articleOption.click(); + await createResponse; + + await expect(page.getByTestId('entity-header-display-name')).toBeVisible(); + }); + + test('Knowledge Center List - Test metadata section details', async ({ + page, + }) => { + const card = await getKnowledgePageCardByIndex(page, 6); + await expect(card).toBeVisible(); + + const metadata = card.getByTestId('knowledge-metadata'); + await expect(metadata).toBeVisible(); + + const updatedAtMetadata = metadata.getByTestId('updated-at-metadata'); + await expect(updatedAtMetadata).toBeVisible(); + await expect(updatedAtMetadata).not.toBeEmpty(); + + const dateOwnerCol = card.getByTestId('date-owner-col'); + await expect(dateOwnerCol).toBeVisible(); + + const ownerLink = dateOwnerCol.getByTestId('owner-link'); + await expect(ownerLink).toBeVisible(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/KnowledgeCenterRoleBasedAccess.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/KnowledgeCenterRoleBasedAccess.spec.ts new file mode 100644 index 000000000000..df610a81d2c9 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/KnowledgeCenterRoleBasedAccess.spec.ts @@ -0,0 +1,287 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { expect, Page, test as base } from '@playwright/test'; +import { PolicyClass } from '../../support/access-control/PoliciesClass'; +import { RolesClass } from '../../support/access-control/RolesClass'; +import { UserClass } from '../../support/user/UserClass'; +import { performAdminLogin } from '../../utils/admin'; +import { uuid } from '../../utils/common'; +import { navigateToKnowledgeCenter } from '../../utils/KnowledgeCenter'; +import { test as testWithRolesPages } from '../fixtures/pages'; + +let testUser: UserClass; +const testPolicy = new PolicyClass(); +const testRole = new RolesClass(); + +const test = base.extend<{ + userPage: Page; +}>({ + userPage: async ({ browser }, use) => { + const page = await browser.newPage(); + await testUser.login(page); + await use(page); + await page.close(); + }, +}); + +base.beforeAll(async ({ browser }) => { + const { apiContext, afterAction } = await performAdminLogin(browser); + + testUser = new UserClass(); + await testUser.create(apiContext, false); + + const policyRules = [ + { + name: `KnowledgeCenterPagePolicy-${uuid()}`, + resources: ['page'], + operations: ['ViewAll'], + effect: 'allow', + }, + ]; + + await testPolicy.create(apiContext, policyRules); + await testRole.create(apiContext, [testPolicy.responseData.name]); + + await testUser.patch({ + apiContext, + patchData: [ + { + op: 'add', + path: '/roles/0', + value: { + id: testRole.responseData.id, + type: 'role', + name: testRole.responseData.name, + }, + }, + ], + }); + + await afterAction(); +}); + +test('Knowledge Center ViewAll role based access validations', async ({ + userPage, +}) => { + await navigateToKnowledgeCenter(userPage); + + const addButton = userPage.getByTestId('add-knowledge-page-btn'); + await expect(addButton).not.toBeVisible(); + + const articleResponse = userPage.waitForResponse( + (response) => + response.url().includes('/api/v1/knowledgeCenter/') && + response.request().method() === 'GET' + ); + + await userPage + .getByTestId('knowledge-pages-hierarchy') + .getByRole('link') + .first() + .click(); + + await articleResponse; + + await userPage.waitForSelector('.ant-skeleton-active', { + state: 'detached', + }); + + await expect( + userPage.getByTestId('entity-header-display-name') + ).toHaveAttribute('readOnly', ''); + + // Verify add domain button is not visible + await expect(userPage.getByTestId('add-domain')).not.toBeVisible(); + + // Verify add data product button is not visible + await expect( + userPage + .getByTestId('KnowledgePanel.DataProducts') + .getByTestId('data-products-container') + .getByTestId('add-data-product') + ).not.toBeVisible(); + + // Verify add reviewer button is not visible + await expect(userPage.getByTestId('Add')).not.toBeVisible(); + + // Verify edit owner button is not visible + await expect(userPage.getByTestId('edit-owner')).not.toBeVisible(); + + // Verify add tags button is not visible + await expect( + userPage + .getByTestId('KnowledgePanel.Tags') + .getByTestId('tags-container') + .getByTestId('add-tag') + ).not.toBeVisible(); + + // Verify add glossary terms button is not visible + await expect( + userPage + .getByTestId('KnowledgePanel.GlossaryTerms') + .getByTestId('glossary-container') + .getByTestId('add-tag') + ).not.toBeVisible(); + + // Verify related data assets add button is not visible + await expect(userPage.getByTestId('related-data-assets')).not.toBeVisible(); +}); + +testWithRolesPages( + 'Data Consumer can view and edit content but cannot add article, domain, reviewer, data product, or data assets', + async ({ dataConsumerPage }) => { + await navigateToKnowledgeCenter(dataConsumerPage); + + const addButton = dataConsumerPage.getByTestId('add-knowledge-page-btn'); + await expect(addButton).not.toBeVisible(); + + const articleResponse = dataConsumerPage.waitForResponse( + (response) => + response.url().includes('/api/v1/knowledgeCenter/') && + response.request().method() === 'GET' + ); + + await dataConsumerPage + .getByTestId('knowledge-pages-hierarchy') + .getByRole('link') + .first() + .click(); + + await articleResponse; + + await dataConsumerPage.waitForURL(/\/knowledge-center\/.*/); + + await dataConsumerPage.waitForSelector('.ant-skeleton-active', { + state: 'detached', + }); + + await expect( + dataConsumerPage.getByTestId('entity-header-display-name') + ).toBeVisible(); + + const editor = dataConsumerPage.locator('[contenteditable="true"]').first(); + await expect(editor).toBeVisible(); + await expect(editor).toHaveAttribute('contenteditable', 'true'); + + await expect(dataConsumerPage.getByTestId('add-domain')).not.toBeVisible(); + + await expect( + dataConsumerPage + .getByTestId('KnowledgePanel.DataProducts') + .getByTestId('data-products-container') + .getByTestId('add-data-product') + ).not.toBeVisible(); + + await expect(dataConsumerPage.getByTestId('Add')).not.toBeVisible(); + + await expect(dataConsumerPage.getByTestId('edit-owner')).not.toBeVisible(); + + const ownerLabel = dataConsumerPage.getByTestId('owner-label'); + const hasOwner = await ownerLabel + .getByTestId('owner-link') + .first() + .isVisible(); + + if (hasOwner) { + await expect(dataConsumerPage.getByTestId('add-owner')).not.toBeVisible(); + } else { + await expect(dataConsumerPage.getByTestId('add-owner')).toBeVisible(); + } + + await expect( + dataConsumerPage.getByTestId('add-data-assets-container') + ).not.toBeVisible(); + await expect( + dataConsumerPage.getByTestId('edit-data-assets') + ).not.toBeVisible(); + } +); + +testWithRolesPages( + 'Data Steward can edit content, title, owners, tags, and glossary terms but cannot add article, domain, reviewer, data product, or data assets', + async ({ dataStewardPage }) => { + await navigateToKnowledgeCenter(dataStewardPage); + + const addButton = dataStewardPage.getByTestId('add-knowledge-page-btn'); + await expect(addButton).not.toBeVisible(); + + const articleResponse = dataStewardPage.waitForResponse( + (response) => + response.url().includes('/api/v1/knowledgeCenter/') && + response.request().method() === 'GET' + ); + + await dataStewardPage + .getByTestId('knowledge-pages-hierarchy') + .getByRole('link') + .first() + .click(); + + await articleResponse; + + await dataStewardPage.waitForURL(/\/knowledge-center\/.*/); + + await dataStewardPage.waitForSelector('.ant-skeleton-active', { + state: 'detached', + }); + + await expect( + dataStewardPage.getByTestId('entity-header-display-name') + ).toBeVisible(); + + const titleInput = dataStewardPage.getByTestId( + 'entity-header-display-name' + ); + await expect(titleInput).not.toHaveAttribute('readOnly', ''); + const editor = dataStewardPage.locator('[contenteditable="true"]').first(); + await expect(editor).toBeVisible(); + await expect(editor).toHaveAttribute('contenteditable', 'true'); + + const ownerLabel = dataStewardPage.getByTestId('owner-label'); + const hasOwner = await ownerLabel + .getByTestId('owner-link') + .first() + .isVisible(); + + if (hasOwner) { + await expect(dataStewardPage.getByTestId('edit-owner')).toBeVisible(); + await expect(dataStewardPage.getByTestId('add-owner')).not.toBeVisible(); + } else { + await expect(dataStewardPage.getByTestId('add-owner')).toBeVisible(); + await expect(dataStewardPage.getByTestId('edit-owner')).not.toBeVisible(); + } + + const rightPanel = dataStewardPage.getByTestId('right-panel'); + await rightPanel.evaluate((el) => el.scrollTo(0, el.scrollHeight)); + + await expect( + dataStewardPage.getByTestId('tags-container').getByTestId('add-tag') + ).toBeVisible(); + + await expect(dataStewardPage.getByTestId('add-domain')).not.toBeVisible(); + + await expect( + dataStewardPage + .getByTestId('data-products-container') + .getByTestId('add-data-product') + ).not.toBeVisible(); + await expect(dataStewardPage.getByTestId('Add')).not.toBeVisible(); + + await expect( + dataStewardPage.getByTestId('add-data-assets-container') + ).not.toBeVisible(); + await expect( + dataStewardPage.getByTestId('edit-data-assets') + ).not.toBeVisible(); + } +); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/KnowledgeCenterTextEditor.common.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/KnowledgeCenterTextEditor.common.ts new file mode 100644 index 000000000000..2b7f26571a08 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/KnowledgeCenterTextEditor.common.ts @@ -0,0 +1,403 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { expect, Page, test } from '@playwright/test'; +import { + SHORTCUTS, + SLASH_COMMANDS, +} from '../../constant/KnowledgeCenter.constant'; +import { KnowledgeCenterResponseDataType } from '../../support/entity/KnowledgeCenter.interface'; +import { + applyTextFormatting, + copyContent, + createCallout, + createCodeBlock, + createHeading, + createLink, + createListItems, + createNestedListItems, + createTable, + createTaskListItems, + executeSlashCommand, + getEditor, + moveToNewLine, + moveToNewParagraph, + navigateToArticle, + pasteContent, + redo, + selectAll, + selectAllText, + selectLastWord, + toggleTask, + typeInTableCell, + undo, + verifyCallout, + verifyCodeBlock, + verifyContentPersistence, + verifyNestedList, + verifyTable, + verifyTaskList, + verifyTextFormatting, +} from '../../utils/KnowledgeCenter'; + +export const runSlashCommandsAndBasicBlocksTest = async ( + page: Page, + article: KnowledgeCenterResponseDataType +) => { + await navigateToArticle(page, article.fullyQualifiedName); + const editor = await getEditor(page, true); + await editor.waitFor({ state: 'visible' }); + + await test.step('Test Headings (H1, H2, H3)', async () => { + await editor.click(); + await createHeading(page, 1, 'Heading 1'); + await expect( + page.getByRole('heading', { name: 'Heading 1', level: 1 }) + ).toBeVisible(); + + await page.keyboard.press(SHORTCUTS.enter); + await createHeading(page, 2, 'Heading 2'); + await expect( + page.getByRole('heading', { name: 'Heading 2', level: 2 }) + ).toBeVisible(); + + await page.keyboard.press(SHORTCUTS.enter); + await createHeading(page, 3, 'Heading 3'); + await expect( + page.getByRole('heading', { name: 'Heading 3', level: 3 }) + ).toBeVisible(); + }); + + await test.step('Test Bullet List', async () => { + await moveToNewParagraph(page, editor); + await executeSlashCommand(page, SLASH_COMMANDS.bullet); + await expect(editor.locator('ul')).toBeVisible(); + await createListItems(page, ['Item 1', 'Item 2', 'Item 3']); + + await expect(page.getByText('Item 1')).toBeVisible(); + await expect(page.getByText('Item 2')).toBeVisible(); + await expect(page.getByText('Item 3')).toBeVisible(); + + const bulletList = editor.locator('ul').filter({ hasText: 'Item 1' }); + await expect(bulletList).toBeVisible(); + }); + + await test.step('Test Numbered List', async () => { + await moveToNewParagraph(page, editor); + await executeSlashCommand(page, SLASH_COMMANDS.numbered); + await expect(editor.locator('ol')).toBeVisible(); + await createListItems(page, ['First item', 'Second item', 'Third item']); + + await expect(page.getByText('First item')).toBeVisible(); + await expect(page.getByText('Second item')).toBeVisible(); + await expect(page.getByText('Third item')).toBeVisible(); + const numberedList = editor.locator('ol').filter({ hasText: 'First item' }); + await expect(numberedList).toBeVisible(); + }); + + await test.step('Test Divider', async () => { + await moveToNewLine(page, editor, true); + await executeSlashCommand(page, SLASH_COMMANDS.divider); + const divider = editor.locator('hr'); + await expect(divider).toBeVisible(); + }); + + await test.step('Test Quote/Blockquote', async () => { + await moveToNewLine(page, editor); + await executeSlashCommand(page, SLASH_COMMANDS.quote); + await expect(editor.locator('blockquote')).toBeVisible(); + await page.keyboard.type('This is a quote'); + + await expect(page.getByText('This is a quote')).toBeVisible(); + + // Verify it's in a blockquote element + const blockquote = editor.locator('blockquote'); + const blockquoteWithText = blockquote.filter({ + hasText: 'This is a quote', + }); + await expect(blockquoteWithText.first()).toBeVisible(); + + // Also verify it's NOT in a list + const textInList = editor + .locator('ol, ul') + .filter({ hasText: 'This is a quote' }); + const listCount = await textInList.count(); + expect(listCount).toBe(0); + }); +}; + +export const runTextFormattingTest = async ( + page: Page, + article: KnowledgeCenterResponseDataType +) => { + await navigateToArticle(page, article.fullyQualifiedName); + const editor = await getEditor(page); + await editor.waitFor({ state: 'visible' }); + + await test.step('Apply bold formatting', async () => { + await editor.click(); + await page.keyboard.type('Normal text'); + await page.keyboard.press(SHORTCUTS.enter); + await page.keyboard.type('Bold text'); + + await expect(page.getByText('Bold text')).toBeVisible(); + + await selectAllText(page); + + await applyTextFormatting(page, 'bold'); + + await verifyTextFormatting(editor, 'Bold text', 'bold'); + + await page.keyboard.press(SHORTCUTS.redo); + await expect(page.getByText('Bold text')).toBeVisible(); + }); + + await test.step('Test Italic formatting', async () => { + await page.keyboard.press(SHORTCUTS.enter); + await page.keyboard.type('Normal text '); + await page.keyboard.press(SHORTCUTS.enter); + await page.keyboard.type('Italic text'); + + await expect(page.getByText('Italic text')).toBeVisible(); + await page.getByText('Italic text').selectText(); + await applyTextFormatting(page, 'italic'); + + await expect(page.getByText('Italic text')).toBeVisible(); + await verifyTextFormatting(editor, 'Italic text', 'italic'); + }); + + await test.step('Test Inline Code', async () => { + await page.keyboard.press(SHORTCUTS.enter); + await page.keyboard.type('inline code'); + + await expect(page.getByText('inline code')).toBeVisible(); + await page.getByText('inline code').selectText(); + await applyTextFormatting(page, 'code'); + + await expect(page.getByText('inline code')).toBeVisible(); + await verifyTextFormatting(editor, 'inline code', 'code'); + }); + + await test.step('Test Link', async () => { + await page.keyboard.press(SHORTCUTS.enter); + await createLink( + page, + editor, + 'Visit OpenMetadata', + 'https://open-metadata.org' + ); + + const link = page.getByRole('link', { name: 'Visit OpenMetadata' }); + await expect(link).toBeVisible(); + await expect(link).toHaveAttribute('href', 'https://open-metadata.org'); + }); +}; + +export const runEditorOperationsTest = async ( + page: Page, + article: KnowledgeCenterResponseDataType +) => { + await navigateToArticle(page, article.fullyQualifiedName); + const editor = await getEditor(page); + await editor.waitFor({ state: 'visible' }); + + await test.step('Test Undo/Redo', async () => { + await editor.click(); + + await page.keyboard.press(SHORTCUTS.enter); + await page.keyboard.type('First text'); + await expect(page.getByText('First text')).toBeVisible(); + await undo(page); + await expect(page.getByText('First text')).not.toBeVisible(); + + await editor.click(); + await redo(page); + + await expect(page.getByText('First text')).toBeVisible(); + }); + + await test.step('Test Copy/Paste', async () => { + await page.keyboard.press(SHORTCUTS.enter); + await page.keyboard.type('Text to copy'); + await expect(page.getByText('Text to copy')).toBeVisible(); + await selectAll(page); + + await copyContent(page); + + await page.keyboard.press(SHORTCUTS.enter); + + await pasteContent(page); + const pastedText = page.getByText('Text to copy'); + const count = await pastedText.count(); + expect(count).toBeGreaterThanOrEqual(1); + }); + + await test.step('Test Select All', async () => { + await page.keyboard.press(SHORTCUTS.enter); + await page.keyboard.type('Select all test'); + await expect(page.getByText('Select all test')).toBeVisible(); + + await selectAll(page); + await applyTextFormatting(page, 'bold'); + await verifyTextFormatting(editor, 'Select all test', 'bold'); + }); +}; + +export const runNestedListsTest = async ( + page: Page, + article: KnowledgeCenterResponseDataType +) => { + await navigateToArticle(page, article.fullyQualifiedName); + const editor = await getEditor(page); + await editor.waitFor({ state: 'visible' }); + + await test.step('Test Nested Bullet List', async () => { + await moveToNewParagraph(page, editor); + await executeSlashCommand(page, SLASH_COMMANDS.bullet); + await expect(editor.locator('ul')).toBeVisible(); + + await createNestedListItems( + page, + ['Parent Item 1', 'Parent Item 2'], + ['Nested Item 1', 'Nested Item 2'] + ); + + await expect(page.getByText('Parent Item 1')).toBeVisible(); + await expect(page.getByText('Parent Item 2')).toBeVisible(); + + await verifyNestedList(editor, 'Parent Item 2', 'Nested Item 1'); + await verifyNestedList(editor, 'Parent Item 2', 'Nested Item 2'); + }); + + await test.step('Test Nested Numbered List', async () => { + await moveToNewParagraph(page, editor); + await executeSlashCommand(page, SLASH_COMMANDS.numbered); + await expect(editor.locator('ol')).toBeVisible(); + + await createNestedListItems( + page, + ['First Parent', 'Second Parent'], + ['First Nested', 'Second Nested'] + ); + + await expect(page.getByText('First Parent')).toBeVisible(); + await expect(page.getByText('Second Parent')).toBeVisible(); + + await verifyNestedList(editor, 'Second Parent', 'First Nested'); + await verifyNestedList(editor, 'Second Parent', 'Second Nested'); + }); +}; + +export const runContentPersistenceTest = async ( + page: Page, + article: KnowledgeCenterResponseDataType +) => { + await navigateToArticle(page, article.fullyQualifiedName); + const editor = await getEditor(page); + await editor.waitFor({ state: 'visible' }); + + await test.step('Create content and verify persistence after reload', async () => { + await editor.click(); + await createHeading(page, 1, 'Persistent Heading'); + await page.keyboard.press(SHORTCUTS.enter); + + await page.keyboard.type('Persistent paragraph text'); + await page.keyboard.press(SHORTCUTS.enter); + + await executeSlashCommand(page, SLASH_COMMANDS.bullet); + await expect(editor.locator('ul')).toBeVisible(); + await createListItems(page, ['Persistent Item 1', 'Persistent Item 2']); + await page.keyboard.press(SHORTCUTS.enter); + + await page.keyboard.type('Bold persistent text'); + await expect(page.getByText('Bold persistent text')).toBeVisible(); + await selectLastWord(page, 3, editor); + await applyTextFormatting(page, 'bold'); + + await verifyContentPersistence(page, [ + 'Persistent Heading', + 'Persistent paragraph text', + 'Persistent Item 1', + 'Persistent Item 2', + 'Bold persistent text', + ]); + }); +}; + +export const runAdvancedBlocksTest = async ( + page: Page, + article: KnowledgeCenterResponseDataType +) => { + await navigateToArticle(page, article.fullyQualifiedName); + const editor = await getEditor(page); + await editor.waitFor({ state: 'visible' }); + + await test.step('Test Code Block', async () => { + await editor.click(); + await moveToNewParagraph(page, editor); + + await createCodeBlock( + page, + 'const test = "code block";\nconsole.log(test);' + ); + + await verifyCodeBlock(editor, 'const test = "code block"'); + }); + + await test.step('Test Task List', async () => { + await moveToNewParagraph(page, editor); + await executeSlashCommand(page, SLASH_COMMANDS.task); + await expect(editor.locator('input[type="checkbox"]')).toBeVisible(); + + await createTaskListItems(page, ['Task 1', 'Task 2', 'Task 3']); + + await verifyTaskList(editor, 'Task 1'); + await verifyTaskList(editor, 'Task 2'); + await verifyTaskList(editor, 'Task 3'); + + await toggleTask(page, editor, 'Task 1'); + const taskCheckbox = editor + .locator('li') + .filter({ hasText: 'Task 1' }) + .locator('input[type="checkbox"]'); + await expect(taskCheckbox).toBeChecked(); + }); + + await test.step('Test Callout', async () => { + await moveToNewParagraph(page, editor); + + await createCallout(page, 'This is an important callout message'); + + await verifyCallout(editor, 'This is an important callout message'); + }); + + await test.step('Test Table', async () => { + await moveToNewParagraph(page, editor); + + await createTable(page); + await verifyTable(editor); + + await typeInTableCell(page, 0, 0, 'Header 1'); + await expect( + editor.locator('table td, table th').filter({ hasText: 'Header 1' }) + ).toBeVisible(); + await typeInTableCell(page, 0, 1, 'Header 2'); + await expect( + editor.locator('table td, table th').filter({ hasText: 'Header 2' }) + ).toBeVisible(); + await typeInTableCell(page, 1, 0, 'Row 1 Col 1'); + await expect( + editor.locator('table td, table th').filter({ hasText: 'Row 1 Col 1' }) + ).toBeVisible(); + await typeInTableCell(page, 1, 1, 'Row 1 Col 2'); + }); +}; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/KnowledgeCenterTextEditor.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/KnowledgeCenterTextEditor.spec.ts new file mode 100644 index 000000000000..a2a9dd689338 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/KnowledgeCenterTextEditor.spec.ts @@ -0,0 +1,174 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { KnowledgeCenterClass } from '../../support/entity/KnowledgeCenterClass'; +import { performAdminLogin } from '../../utils/admin'; +import { createNewPage, redirectToHomePage } from '../../utils/common'; +import { test } from '../fixtures/pages'; +import { + runAdvancedBlocksTest, + runContentPersistenceTest, + runEditorOperationsTest, + runNestedListsTest, + runSlashCommandsAndBasicBlocksTest, + runTextFormattingTest, +} from './KnowledgeCenterTextEditor.common'; + +test.describe('Knowledge Center - Text Editor (Admin Role)', () => { + const knowledgeCenter = new KnowledgeCenterClass(); + + test.use({ + storageState: 'playwright/.auth/admin.json', + }); + + test.beforeAll('Setup pre-requests', async ({ browser }) => { + test.slow(true); + const { apiContext, afterAction } = await createNewPage(browser); + await knowledgeCenter.create(apiContext, 6); + await afterAction(); + }); + + test.beforeEach(async ({ page }) => { + await redirectToHomePage(page); + }); + + test('Rich Text Editor - Slash Commands and Basic Blocks', async ({ + page, + }) => { + const article = knowledgeCenter.knowledgePages[0]; + await runSlashCommandsAndBasicBlocksTest(page, article); + }); + + test('Rich Text Editor - Text Formatting', async ({ page }) => { + const article = knowledgeCenter.knowledgePages[1]; + await runTextFormattingTest(page, article); + }); + + test('Rich Text Editor - Editor Operations', async ({ page }) => { + const article = knowledgeCenter.knowledgePages[2]; + await runEditorOperationsTest(page, article); + }); + + test('Rich Text Editor - Nested Lists', async ({ page }) => { + const article = knowledgeCenter.knowledgePages[3]; + await runNestedListsTest(page, article); + }); + + test('Rich Text Editor - Content Persistence', async ({ page }) => { + const article = knowledgeCenter.knowledgePages[4]; + await runContentPersistenceTest(page, article); + }); + + test('Rich Text Editor - Advanced Blocks', async ({ page }) => { + const article = knowledgeCenter.knowledgePages[5]; + await runAdvancedBlocksTest(page, article); + }); +}); + +test.describe('Knowledge Center - Text Editor (Data Consumer Role)', () => { + const knowledgeCenter = new KnowledgeCenterClass(); + + test.beforeAll('Setup pre-requests', async ({ browser }) => { + test.slow(true); + + const { apiContext, afterAction } = await performAdminLogin(browser); + await knowledgeCenter.create(apiContext, 6); + await afterAction(); + }); + + test.beforeEach(async ({ dataConsumerPage }) => { + await redirectToHomePage(dataConsumerPage); + }); + + test('Rich Text Editor - Slash Commands and Basic Blocks', async ({ + dataConsumerPage, + }) => { + const article = knowledgeCenter.knowledgePages[0]; + await runSlashCommandsAndBasicBlocksTest(dataConsumerPage, article); + }); + + test('Rich Text Editor - Text Formatting', async ({ dataConsumerPage }) => { + const article = knowledgeCenter.knowledgePages[1]; + await runTextFormattingTest(dataConsumerPage, article); + }); + + test('Rich Text Editor - Editor Operations', async ({ dataConsumerPage }) => { + const article = knowledgeCenter.knowledgePages[2]; + await runEditorOperationsTest(dataConsumerPage, article); + }); + + test('Rich Text Editor - Nested Lists', async ({ dataConsumerPage }) => { + const article = knowledgeCenter.knowledgePages[3]; + await runNestedListsTest(dataConsumerPage, article); + }); + + test('Rich Text Editor - Content Persistence', async ({ + dataConsumerPage, + }) => { + const article = knowledgeCenter.knowledgePages[4]; + await runContentPersistenceTest(dataConsumerPage, article); + }); + + test('Rich Text Editor - Advanced Blocks', async ({ dataConsumerPage }) => { + const article = knowledgeCenter.knowledgePages[5]; + await runAdvancedBlocksTest(dataConsumerPage, article); + }); +}); + +test.describe('Knowledge Center - Text Editor (Data Steward Role)', () => { + const knowledgeCenter = new KnowledgeCenterClass(); + + test.beforeAll('Setup pre-requests', async ({ browser }) => { + test.slow(true); + const { apiContext, afterAction } = await performAdminLogin(browser); + await knowledgeCenter.create(apiContext, 6); + await afterAction(); + }); + + test.beforeEach(async ({ dataStewardPage }) => { + await redirectToHomePage(dataStewardPage); + }); + + test('Rich Text Editor - Slash Commands and Basic Blocks', async ({ + dataStewardPage, + }) => { + const article = knowledgeCenter.knowledgePages[0]; + await runSlashCommandsAndBasicBlocksTest(dataStewardPage, article); + }); + + test('Rich Text Editor - Text Formatting', async ({ dataStewardPage }) => { + const article = knowledgeCenter.knowledgePages[1]; + await runTextFormattingTest(dataStewardPage, article); + }); + + test('Rich Text Editor - Editor Operations', async ({ dataStewardPage }) => { + const article = knowledgeCenter.knowledgePages[2]; + await runEditorOperationsTest(dataStewardPage, article); + }); + + test('Rich Text Editor - Nested Lists', async ({ dataStewardPage }) => { + const article = knowledgeCenter.knowledgePages[3]; + await runNestedListsTest(dataStewardPage, article); + }); + + test('Rich Text Editor - Content Persistence', async ({ + dataStewardPage, + }) => { + const article = knowledgeCenter.knowledgePages[4]; + await runContentPersistenceTest(dataStewardPage, article); + }); + + test('Rich Text Editor - Advanced Blocks', async ({ dataStewardPage }) => { + const article = knowledgeCenter.knowledgePages[5]; + await runAdvancedBlocksTest(dataStewardPage, article); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/ExplorePageRightPanel_KnowledgeCenter.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/ExplorePageRightPanel_KnowledgeCenter.spec.ts new file mode 100644 index 000000000000..870bc35ce109 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/ExplorePageRightPanel_KnowledgeCenter.spec.ts @@ -0,0 +1,716 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { KnowledgeCenterClass } from '../../support/entity/KnowledgeCenterClass'; +import { expect, test as baseTest } from '../../support/fixtures/userPages'; +import { Glossary } from '../../support/glossary/Glossary'; +import { GlossaryTerm } from '../../support/glossary/GlossaryTerm'; +import { ClassificationClass } from '../../support/tag/ClassificationClass'; +import { TagClass } from '../../support/tag/TagClass'; +import { UserClass } from '../../support/user/UserClass'; +import { performAdminLogin } from '../../utils/admin'; +import { uuid } from '../../utils/common'; +import { + getEntityDisplayName, + waitForAllLoadersToDisappear, +} from '../../utils/entity'; +import { performUserLogin } from '../../utils/user'; +import { OverviewPageObject } from '../PageObject/Explore/OverviewPageObject'; +import { + RightPanelPageObject, + RIGHT_PANEL_TAB, +} from '../PageObject/Explore/RightPanelPageObject'; +import { + addOwnerInKCPanel, + navigateToKCEntity, +} from '../Utils/ExplorePageRightPanelUtils'; + +// Test data setup +export const knowledgeCenter = new KnowledgeCenterClass(); +const user1 = new UserClass(); +export const testClassification = new ClassificationClass(); +export const testTag = new TagClass({ + classification: testClassification.data.name, +}); +export const testGlossary = new Glossary(); +const testGlossaryTerm = new GlossaryTerm(testGlossary); +const testClassification2 = new ClassificationClass(); +const testTag2 = new TagClass({ + classification: testClassification2.data.name, +}); + +const glossaryTermToUpdate = + testGlossaryTerm.responseData?.displayName ?? + testGlossaryTerm.data.displayName; +const tagToUpdate = + testTag.responseData?.displayName ?? testTag.data.displayName; + +// Extend base test with right panel fixtures +export const test = baseTest.extend<{ + rightPanel: RightPanelPageObject; + overview: OverviewPageObject; +}>({ + rightPanel: async ({ adminPage }, use) => { + await use(new RightPanelPageObject(adminPage)); + }, + overview: async ({ rightPanel }, use) => { + await use(new OverviewPageObject(rightPanel)); + }, +}); + +test.describe('Knowledge Center Right Panel Test Suite', () => { + test.beforeAll(async ({ browser }) => { + test.slow(true); + const { apiContext, afterAction } = await performAdminLogin(browser); + + try { + await knowledgeCenter.create(apiContext); + await testClassification.create(apiContext); + await testTag.create(apiContext); + await testGlossary.create(apiContext); + await testGlossaryTerm.create(apiContext); + await testClassification2.create(apiContext); + await testTag2.create(apiContext); + await user1.create(apiContext); + } finally { + await afterAction(); + } + }); + + test.afterAll(async ({ browser }) => { + const { apiContext, afterAction } = await performAdminLogin(browser); + + try { + await knowledgeCenter.delete(apiContext); + await testTag.delete(apiContext); + await testClassification.delete(apiContext); + await testTag2.delete(apiContext); + await testClassification2.delete(apiContext); + await testGlossaryTerm.delete(apiContext); + await testGlossary.delete(apiContext); + await user1.delete(apiContext); + } finally { + await afterAction(); + } + }); + + test.describe('Explore page right panel tests', () => { + test.describe('Overview panel CRUD operations', () => { + test('Should update description for knowledgeCenter', async ({ + adminPage, + rightPanel, + overview, + }) => { + await navigateToKCEntity( + adminPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + await rightPanel.waitForPanelVisible(); + rightPanel.setEntityConfigByType('knowledgeCenter'); + + await overview.navigateToOverviewTab(); + await overview.shouldBeVisible(); + await overview.shouldShowDescriptionSection(); + + const descriptionToUpdate = `knowledgeCenter Test description - ${uuid()}`; + await overview.editDescription(descriptionToUpdate); + await overview.shouldShowDescriptionWithText(descriptionToUpdate); + }); + + test('Should update/edit tags for knowledgeCenter', async ({ + adminPage, + rightPanel, + overview, + }) => { + await navigateToKCEntity( + adminPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + await rightPanel.waitForPanelVisible(); + rightPanel.setEntityConfigByType('knowledgeCenter'); + + await overview.editTags(tagToUpdate); + await overview.shouldShowTagsSection(); + await overview.shouldShowTag(tagToUpdate); + }); + + test('Should update/edit glossary terms for knowledgeCenter', async ({ + adminPage, + rightPanel, + overview, + }) => { + await navigateToKCEntity( + adminPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + await rightPanel.waitForPanelVisible(); + rightPanel.setEntityConfigByType('knowledgeCenter'); + + await overview.editGlossaryTerms(glossaryTermToUpdate); + await overview.shouldShowGlossaryTermsSection(); + }); + + test('Should update owners for knowledgeCenter', async ({ + adminPage, + rightPanel, + overview, + }) => { + await navigateToKCEntity( + adminPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + await rightPanel.waitForPanelLoaded(); + await rightPanel.waitForPanelVisible(); + rightPanel.setEntityConfigByType('knowledgeCenter'); + + await addOwnerInKCPanel(adminPage, user1.getUserDisplayName()); + await overview.shouldShowOwner(user1.getUserDisplayName()); + }); + }); + + test.describe('Right panel validation by asset type', () => { + test('validates visible/hidden tabs and tab content for knowledgeCenter', async ({ + adminPage, + rightPanel, + }) => { + await navigateToKCEntity( + adminPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + await rightPanel.waitForPanelLoaded(); + rightPanel.setEntityConfigByType('knowledgeCenter'); + + // knowledgeCenter supports Overview tab only — Lineage, Data Quality, and Custom Properties are hidden + await expect( + rightPanel.getTabLocator(RIGHT_PANEL_TAB.OVERVIEW) + ).toBeVisible(); + await expect( + rightPanel.getTabLocator(RIGHT_PANEL_TAB.LINEAGE) + ).not.toBeVisible(); + await expect( + rightPanel.getTabLocator(RIGHT_PANEL_TAB.DATA_QUALITY) + ).not.toBeVisible(); + await expect( + rightPanel.getTabLocator(RIGHT_PANEL_TAB.CUSTOM_PROPERTIES) + ).not.toBeVisible(); + }); + }); + + test.describe('Overview panel - Removal operations', () => { + test('Should remove tag for knowledgeCenter', async ({ + adminPage, + rightPanel, + overview, + }) => { + await navigateToKCEntity( + adminPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + await rightPanel.waitForPanelVisible(); + rightPanel.setEntityConfigByType('knowledgeCenter'); + + await overview.editTags(tagToUpdate); + await overview.shouldShowTagsSection(); + await overview.shouldShowTag(tagToUpdate); + + await overview.removeTag([tagToUpdate]); + await waitForAllLoadersToDisappear(adminPage); + + await navigateToKCEntity( + adminPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + const tagElement = adminPage.getByTestId( + `tag-${testClassification.data.name}.${testTag.data.name}` + ); + await expect(tagElement).not.toBeVisible(); + }); + + test('Should remove glossary term for knowledgeCenter', async ({ + adminPage, + rightPanel, + overview, + }) => { + await navigateToKCEntity( + adminPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + await rightPanel.waitForPanelVisible(); + rightPanel.setEntityConfigByType('knowledgeCenter'); + + await overview.editGlossaryTerms(glossaryTermToUpdate); + await overview.shouldShowGlossaryTermsSection(); + + await overview.removeGlossaryTerm([glossaryTermToUpdate]); + await waitForAllLoadersToDisappear(adminPage); + + await navigateToKCEntity( + adminPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + const glossarySection = adminPage.locator('.glossary-terms-section'); + await expect( + glossarySection.getByText(glossaryTermToUpdate) + ).not.toBeVisible(); + }); + + test('Should remove user owner for knowledgeCenter', async ({ + adminPage, + rightPanel, + overview, + }) => { + await navigateToKCEntity( + adminPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + await rightPanel.waitForPanelLoaded(); + rightPanel.setEntityConfigByType('knowledgeCenter'); + + await addOwnerInKCPanel(adminPage, user1.getUserDisplayName()); + await overview.shouldShowOwner(user1.getUserDisplayName()); + + await overview.removeOwner([user1.getUserDisplayName()], 'Users'); + await waitForAllLoadersToDisappear(adminPage); + + await navigateToKCEntity( + adminPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + const ownerElement = adminPage + .locator('.owners-section') + .getByText(user1.getUserDisplayName()); + await expect(ownerElement).not.toBeVisible(); + }); + }); + + test.describe('Overview panel - Deleted entity verification', () => { + test('Should verify deleted user not visible in owner selection for knowledgeCenter', async ({ + adminPage, + rightPanel, + overview, + browser, + }) => { + const deletedUser = new UserClass(); + const { apiContext, afterAction } = await performAdminLogin(browser); + + try { + await deletedUser.create(apiContext); + + await navigateToKCEntity( + adminPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + await rightPanel.waitForPanelLoaded(); + rightPanel.setEntityConfigByType('knowledgeCenter'); + + await addOwnerInKCPanel(adminPage, deletedUser.getUserDisplayName()); + await overview.shouldShowOwner(deletedUser.getUserDisplayName()); + + await deletedUser.delete(apiContext); + await adminPage.reload(); + await rightPanel.waitForPanelLoaded(); + + const deletedOwnerLocator = + await overview.verifyDeletedOwnerNotVisible( + deletedUser.getUserDisplayName(), + 'Users' + ); + await expect(deletedOwnerLocator).not.toBeVisible(); + } finally { + await afterAction(); + } + }); + + test('Should verify deleted tag not visible in tag selection for knowledgeCenter', async ({ + adminPage, + rightPanel, + overview, + browser, + }) => { + const deletedClassification = new ClassificationClass(); + const deletedTag = new TagClass({ + classification: deletedClassification.data.name, + }); + const { apiContext, afterAction } = await performAdminLogin(browser); + + try { + await deletedClassification.create(apiContext); + await deletedTag.create(apiContext); + + const deletedTagDisplayName = + deletedTag.responseData?.displayName ?? deletedTag.data.displayName; + + await navigateToKCEntity( + adminPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + await rightPanel.waitForPanelVisible(); + rightPanel.setEntityConfigByType('knowledgeCenter'); + + await overview.editTags(deletedTagDisplayName); + await overview.shouldShowTag(deletedTagDisplayName); + + await deletedTag.delete(apiContext); + await deletedClassification.delete(apiContext); + await adminPage.reload(); + await rightPanel.waitForPanelLoaded(); + + const deletedTagLocator = await overview.verifyDeletedTagNotVisible( + deletedTagDisplayName + ); + await expect(deletedTagLocator).not.toBeVisible(); + } finally { + await afterAction(); + } + }); + + test('Should verify deleted glossary term not visible in selection for knowledgeCenter', async ({ + adminPage, + rightPanel, + overview, + browser, + }) => { + const deletedGlossary = new Glossary(); + const deletedGlossaryTerm = new GlossaryTerm(deletedGlossary); + const { apiContext, afterAction } = await performAdminLogin(browser); + + try { + await deletedGlossary.create(apiContext); + await deletedGlossaryTerm.create(apiContext); + + const deletedTermDisplayName = + deletedGlossaryTerm.responseData?.displayName ?? + deletedGlossaryTerm.data.displayName; + + await navigateToKCEntity( + adminPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + await rightPanel.waitForPanelVisible(); + rightPanel.setEntityConfigByType('knowledgeCenter'); + + await overview.editGlossaryTerms(deletedTermDisplayName); + await overview.shouldShowGlossaryTermsSection(); + + await deletedGlossaryTerm.delete(apiContext); + await deletedGlossary.delete(apiContext); + await adminPage.reload(); + await rightPanel.waitForPanelLoaded(); + + const deletedTermLocator = + await overview.verifyDeletedGlossaryTermNotVisible( + deletedTermDisplayName + ); + await expect(deletedTermLocator).not.toBeVisible(); + } finally { + await afterAction(); + } + }); + }); + + test.describe('Data Steward User - Permission Verification', () => { + test('Should allow Data Steward to edit description for knowledgeCenter', async ({ + dataStewardPage, + }) => { + await navigateToKCEntity( + dataStewardPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + await dataStewardPage + .locator('[data-testid="entity-summary-panel-container"]') + .waitFor({ state: 'visible' }); + + const rightPanelDS = new RightPanelPageObject(dataStewardPage); + rightPanelDS.setEntityConfigByType('knowledgeCenter'); + rightPanelDS.setRolePermissions('DataSteward'); + + const overviewDS = new OverviewPageObject(rightPanelDS); + await overviewDS.navigateToOverviewTab(); + + const descriptionToUpdate = `DataSteward description - ${uuid()}`; + await overviewDS.editDescription(descriptionToUpdate); + await overviewDS.shouldShowDescriptionWithText(descriptionToUpdate); + }); + + test('Should allow Data Steward to edit owners for knowledgeCenter', async ({ + dataStewardPage, + }) => { + await navigateToKCEntity( + dataStewardPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + + const rightPanelDS = new RightPanelPageObject(dataStewardPage); + await rightPanelDS.waitForPanelLoaded(); + rightPanelDS.setEntityConfigByType('knowledgeCenter'); + rightPanelDS.setRolePermissions('DataSteward'); + + const overviewDS = new OverviewPageObject(rightPanelDS); + await addOwnerInKCPanel(dataStewardPage, user1.getUserDisplayName()); + await overviewDS.shouldShowOwner(user1.getUserDisplayName()); + }); + + test('Should allow Data Steward to edit tags for knowledgeCenter', async ({ + dataStewardPage, + }) => { + await navigateToKCEntity( + dataStewardPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + await dataStewardPage + .locator('[data-testid="entity-summary-panel-container"]') + .waitFor({ state: 'visible' }); + + const rightPanelDS = new RightPanelPageObject(dataStewardPage); + rightPanelDS.setEntityConfigByType('knowledgeCenter'); + rightPanelDS.setRolePermissions('DataSteward'); + + const overviewDS = new OverviewPageObject(rightPanelDS); + await overviewDS.editTags(tagToUpdate); + await overviewDS.shouldShowTag(tagToUpdate); + }); + + test('Should allow Data Steward to edit glossary terms for knowledgeCenter', async ({ + dataStewardPage, + }) => { + await navigateToKCEntity( + dataStewardPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + await dataStewardPage + .locator('[data-testid="entity-summary-panel-container"]') + .waitFor({ state: 'visible' }); + + const rightPanelDS = new RightPanelPageObject(dataStewardPage); + rightPanelDS.setEntityConfigByType('knowledgeCenter'); + rightPanelDS.setRolePermissions('DataSteward'); + + const overviewDS = new OverviewPageObject(rightPanelDS); + await overviewDS.editGlossaryTerms(glossaryTermToUpdate); + await overviewDS.shouldShowGlossaryTermsSection(); + }); + + test('Should NOT show restricted edit buttons for Data Steward for knowledgeCenter', async ({ + dataStewardPage, + }) => { + await navigateToKCEntity( + dataStewardPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + await dataStewardPage + .locator('[data-testid="entity-summary-panel-container"]') + .waitFor({ state: 'visible' }); + + const rightPanelDS = new RightPanelPageObject(dataStewardPage); + rightPanelDS.setEntityConfigByType('knowledgeCenter'); + + const overviewDS = new OverviewPageObject(rightPanelDS); + await overviewDS.navigateToOverviewTab(); + + // DataSteward: canEditDomains=false, canEditDataProducts=false + // Note: edit-tier is not applicable for knowledgeCenter + const summaryPanel = dataStewardPage.locator( + '[data-testid="entity-summary-panel-container"]' + ); + await expect(summaryPanel.getByTestId('add-domain')).not.toBeVisible(); + await expect( + summaryPanel.getByTestId('edit-data-products') + ).not.toBeVisible(); + await expect(summaryPanel.getByTestId('edit-tier')).not.toBeVisible(); + }); + }); + + test.describe('Data Consumer User - Permission Verification', () => { + test('Should allow Data Consumer to edit description for knowledgeCenter', async ({ + dataConsumerPage, + }) => { + await navigateToKCEntity( + dataConsumerPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + await dataConsumerPage + .locator('[data-testid="entity-summary-panel-container"]') + .waitFor({ state: 'visible' }); + + const rightPanelDC = new RightPanelPageObject(dataConsumerPage); + rightPanelDC.setEntityConfigByType('knowledgeCenter'); + rightPanelDC.setRolePermissions('DataConsumer'); + + const overviewDC = new OverviewPageObject(rightPanelDC); + await overviewDC.navigateToOverviewTab(); + + const descriptionToUpdate = `DataConsumer description - ${uuid()}`; + await overviewDC.editDescription(descriptionToUpdate); + await overviewDC.shouldShowDescriptionWithText(descriptionToUpdate); + }); + + test('Should allow Data Consumer to edit tags for knowledgeCenter', async ({ + dataConsumerPage, + }) => { + await navigateToKCEntity( + dataConsumerPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + await dataConsumerPage + .locator('[data-testid="entity-summary-panel-container"]') + .waitFor({ state: 'visible' }); + + const rightPanelDC = new RightPanelPageObject(dataConsumerPage); + rightPanelDC.setEntityConfigByType('knowledgeCenter'); + rightPanelDC.setRolePermissions('DataConsumer'); + + const overviewDC = new OverviewPageObject(rightPanelDC); + await overviewDC.editTags(tagToUpdate); + await overviewDC.shouldShowTag(tagToUpdate); + }); + + test('Should allow Data Consumer to edit glossary terms for knowledgeCenter', async ({ + dataConsumerPage, + }) => { + await navigateToKCEntity( + dataConsumerPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + await dataConsumerPage + .locator('[data-testid="entity-summary-panel-container"]') + .waitFor({ state: 'visible' }); + + const rightPanelDC = new RightPanelPageObject(dataConsumerPage); + rightPanelDC.setEntityConfigByType('knowledgeCenter'); + rightPanelDC.setRolePermissions('DataConsumer'); + + const overviewDC = new OverviewPageObject(rightPanelDC); + await overviewDC.editGlossaryTerms(glossaryTermToUpdate); + await overviewDC.shouldShowGlossaryTermsSection(); + }); + + test('Should follow Data Consumer role policies for ownerless knowledgeCenter', async ({ + browser, + }) => { + const { page: dataConsumerPage, afterAction } = await performUserLogin( + browser, + user1 + ); + + try { + await navigateToKCEntity( + dataConsumerPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + await dataConsumerPage + .locator('[data-testid="entity-summary-panel-container"]') + .waitFor({ state: 'visible' }); + + const rightPanelDC = new RightPanelPageObject(dataConsumerPage); + rightPanelDC.setEntityConfigByType('knowledgeCenter'); + + const overviewDC = new OverviewPageObject(rightPanelDC); + await overviewDC.navigateToOverviewTab(); + + // DataConsumer: canEditDomains=false, canEditDataProducts=false + // Note: edit-tier is not applicable for knowledgeCenter + const summaryPanel = dataConsumerPage.locator( + '[data-testid="entity-summary-panel-container"]' + ); + await expect( + summaryPanel.getByTestId('add-domain') + ).not.toBeVisible(); + await expect( + summaryPanel.getByTestId('edit-data-products') + ).not.toBeVisible(); + } finally { + await afterAction(); + } + }); + }); + + test.describe('Overview panel - Description removal', () => { + test('Should clear description for knowledgeCenter', async ({ + adminPage, + }) => { + const { page: authenticatedPage, afterAction } = + await performAdminLogin(adminPage.context().browser()!); + const rightPanel = new RightPanelPageObject(authenticatedPage); + const localOverview = new OverviewPageObject(rightPanel); + + try { + await navigateToKCEntity( + authenticatedPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + await rightPanel.waitForPanelVisible(); + rightPanel.setEntityConfigByType('knowledgeCenter'); + + const descriptionText = `Description to remove - ${uuid()}`; + await localOverview.editDescription(descriptionText); + await localOverview.shouldShowDescriptionWithText(descriptionText); + + await localOverview.editDescription(''); + + await navigateToKCEntity( + authenticatedPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + await rightPanel.waitForPanelVisible(); + + const descElement = authenticatedPage + .locator('.description-section') + .getByText(descriptionText); + await expect(descElement).not.toBeVisible(); + } finally { + await afterAction(); + } + }); + }); + + test.describe('Overview panel - Multi-tag operations', () => { + test('Should add multiple tags simultaneously for knowledgeCenter', async ({ + adminPage, + }) => { + const { page: authenticatedPage, afterAction } = + await performAdminLogin(adminPage.context().browser()!); + const rightPanel = new RightPanelPageObject(authenticatedPage); + const localOverview = new OverviewPageObject(rightPanel); + + try { + await navigateToKCEntity( + authenticatedPage, + getEntityDisplayName(knowledgeCenter.responseData) + ); + await rightPanel.waitForPanelVisible(); + rightPanel.setEntityConfigByType('knowledgeCenter'); + + // Add first tag + await localOverview.editTags(tagToUpdate); + await localOverview.shouldShowTag(tagToUpdate); + + // Add second tag (via edit) + const secondTag = + testTag2.responseData?.displayName ?? testTag2.data.displayName; + await localOverview.editTags(secondTag); + await localOverview.shouldShowTag(secondTag); + + // Both tags should be visible + await localOverview.shouldShowTag(tagToUpdate); + await localOverview.shouldShowTag(secondTag); + + // Cleanup: remove both tags + await localOverview.removeTag([secondTag]); + await localOverview.removeTag([tagToUpdate]); + } finally { + await afterAction(); + } + }); + }); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Utils/ExplorePageRightPanelUtils.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Utils/ExplorePageRightPanelUtils.ts new file mode 100644 index 000000000000..18c1895c07b2 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Utils/ExplorePageRightPanelUtils.ts @@ -0,0 +1,85 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { expect, Page } from '@playwright/test'; +import { waitForAllLoadersToDisappear } from '../../utils/entity'; +import { navigateToExploreAndSelectEntity } from '../../utils/explore'; + +/** + * Navigate to the explore page and open the right panel for the Knowledge Center entity. + * KC entities are not in ENDPOINT_TO_FILTER_MAP, so no category filter is applied. + */ +export async function navigateToKCEntity(page: Page, entityName: string) { + const navParams = { page, entityName, exploreTab: 'Knowledge Center' }; + const summaryPanel = page.locator( + '[data-testid="entity-summary-panel-container"]' + ); + const entityLink = summaryPanel + .getByTestId('entity-link') + .filter({ hasText: entityName }); + + await navigateToExploreAndSelectEntity(navParams); + await summaryPanel.waitFor({ state: 'visible' }); + + try { + await expect(entityLink).toBeVisible(); + } catch { + await navigateToExploreAndSelectEntity(navParams); + await summaryPanel.waitFor({ state: 'visible' }); + await expect(entityLink).toBeVisible(); + } + + await waitForAllLoadersToDisappear(page); +} + +export const addOwnerInKCPanel = async (page: Page, ownerName: string) => { + const panel = page.locator('[data-testid="entity-summary-panel-container"]'); + await panel.getByTestId('edit-owners').click(); + + const ownerTabs = page.getByTestId('select-owner-tabs'); + await ownerTabs.waitFor({ state: 'visible' }); + + const teamsTab = ownerTabs.locator('[data-node-key="teams"]'); + const usersTab = ownerTabs.locator('[data-node-key="users"]'); + const searchBar = page.getByTestId('owner-select-users-search-bar'); + + await waitForAllLoadersToDisappear(page); + + const isTeamsActive = await teamsTab.evaluate((el) => + el.classList.contains('ant-tabs-tab-active') + ); + + if (isTeamsActive) { + await usersTab.click(); + await waitForAllLoadersToDisappear(page); + } + + await searchBar.waitFor({ state: 'visible', timeout: 30000 }); + await searchBar.scrollIntoViewIfNeeded(); + + const searchResponse = page.waitForResponse( + `/api/v1/search/query?q=*${encodeURIComponent(ownerName)}*` + ); + await searchBar.fill(ownerName); + await searchResponse; + + await waitForAllLoadersToDisappear(page); + + const patchResponse = page.waitForResponse( + (r) => + r.url().includes('/api/v1/knowledgeCenter/') && + r.request().method() === 'PATCH' + ); + await page.getByRole('listitem', { name: ownerName }).click(); + await page.getByTestId('selectable-list-update-btn').click(); + await patchResponse; +}; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/KnowledgeCenter.interface.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/KnowledgeCenter.interface.ts new file mode 100644 index 000000000000..cf5ce07bd813 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/KnowledgeCenter.interface.ts @@ -0,0 +1,57 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { EntityReference } from './Entity.interface'; + +export type KnowledgeCenterResponseDataType = { + id: string; + name: string; + fullyQualifiedName: string; + displayName: string; + description: string; + version: number; + updatedAt: number; + updatedBy: string; + href: string; + pageType: string; + page: { + publicationDate: number; + relatedArticles: unknown[]; + }; + owners?: EntityReference[]; + reviewers?: EntityReference[]; + followers?: EntityReference[]; + tags?: unknown[]; + relatedEntities?: EntityReference[]; + children?: EntityReference[]; + dataProducts?: EntityReference[]; + [key: string]: unknown; +}; + +export type KnowledgeCenterData = { + name: string; + displayName: string; + description: string; + pageType: string; + page: { + publicationDate: Date; + relatedArticles: never[]; + }; + owners?: Array<{ + type: string; + id: string; + }>; + relatedEntities?: Array<{ + type: string; + id: string; + }>; +}; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/KnowledgeCenterClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/KnowledgeCenterClass.ts new file mode 100644 index 000000000000..4077c8ef70d4 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/KnowledgeCenterClass.ts @@ -0,0 +1,215 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { APIRequestContext, expect, Page } from '@playwright/test'; +import cryptoRandomString from 'crypto-random-string-with-promisify-polyfill'; +import { navigateToArticle } from '../../utils/KnowledgeCenter'; +import { + KnowledgeCenterData, + KnowledgeCenterResponseDataType, +} from './KnowledgeCenter.interface'; +import { TableClass } from './TableClass'; + +export class KnowledgeCenterClass { + data: KnowledgeCenterData; + responseData: KnowledgeCenterResponseDataType = + {} as KnowledgeCenterResponseDataType; + knowledgePages: KnowledgeCenterResponseDataType[] = []; + dataAsset: TableClass; + + constructor( + data?: Partial, + owners?: Array<{ type: string; id: string }>, + dataAsset?: TableClass + ) { + const instanceName = `Article_${cryptoRandomString({ + length: 8, + type: 'alphanumeric', + })}`; + + this.dataAsset = dataAsset ?? new TableClass(); + + this.data = { + name: data?.name ?? instanceName, + displayName: data?.displayName ?? instanceName, + description: data?.description ?? instanceName, + pageType: data?.pageType ?? 'Article', + page: data?.page ?? { + publicationDate: new Date(), + relatedArticles: [], + }, + owners: owners ?? data?.owners, + relatedEntities: data?.relatedEntities, + }; + } + + async create(apiContext: APIRequestContext, numberOfPages = 1) { + if (!this.dataAsset.entityResponseData?.id) { + await this.dataAsset.create(apiContext); + } + + for (let i = 0; i < numberOfPages; i++) { + const instanceName = `Article_${cryptoRandomString({ + length: 8, + type: 'alphanumeric', + })}`; + + const apiData = { + name: instanceName, + displayName: instanceName, + description: instanceName, + pageType: this.data.pageType, + page: { + publicationDate: new Date().getTime(), + relatedArticles: [], + }, + ...(this.data.owners && { owners: this.data.owners }), + ...(this.data.relatedEntities && { + relatedEntities: this.data.relatedEntities, + }), + }; + + const response = await apiContext.post('/api/v1/knowledgeCenter', { + data: apiData, + }); + + const pageData = await response.json(); + this.knowledgePages.push(pageData); + + if (i === 0) { + this.responseData = pageData; + } + } + + return this.responseData; + } + + async patch( + apiContext: APIRequestContext, + data: Record[], + pageId?: string + ) { + const id = pageId ?? this.responseData?.id; + if (!id) { + throw new Error('Cannot patch: KnowledgeCenter has not been created'); + } + + const response = await apiContext.patch(`/api/v1/knowledgeCenter/${id}`, { + data, + headers: { + 'Content-Type': 'application/json-patch+json', + }, + }); + + if (!response.ok()) { + const errorText = await response.text(); + throw new Error( + `Failed to patch KnowledgeCenter ${id}: ${response.status()} - ${errorText}` + ); + } + + const updatedData = await response.json(); + + const pageIndex = this.knowledgePages.findIndex((p) => p.id === id); + if (pageIndex !== -1) { + this.knowledgePages[pageIndex] = updatedData; + } + + if (id === this.responseData?.id) { + this.responseData = updatedData; + } + + return updatedData; + } + + async addDataAssetToPage( + apiContext: APIRequestContext, + pageId?: string + ): Promise { + const targetPageId = pageId ?? this.responseData?.id; + const targetPage = this.knowledgePages.find((p) => p.id === targetPageId); + + const dataAssetEntity = { + id: this.dataAsset.entityResponseData.id, + type: 'table', + name: this.dataAsset.entityResponseData.name, + fullyQualifiedName: this.dataAsset.entityResponseData.fullyQualifiedName, + description: this.dataAsset.entityResponseData.description, + displayName: this.dataAsset.entityResponseData.displayName, + deleted: false, + }; + + const currentRelatedEntitiesCount = + targetPage?.relatedEntities?.length ?? 0; + + const updatedData = await this.patch( + apiContext, + [ + { + op: 'add', + path: `/relatedEntities/${currentRelatedEntitiesCount}`, + value: dataAssetEntity, + }, + ], + targetPageId + ); + + expect(updatedData.relatedEntities).toBeDefined(); + const addedEntity = updatedData.relatedEntities?.find( + (entity: { id: string }) => entity.id === dataAssetEntity.id + ); + expect(addedEntity).toBeDefined(); + } + + async visitPage(page: Page, pageIndex = 0) { + const targetPage = this.knowledgePages[pageIndex] || this.responseData; + + await navigateToArticle(page, targetPage.fullyQualifiedName); + + await expect(page.getByTestId('entity-header-display-name')).toHaveValue( + targetPage.displayName + ); + } + + getEntityType(): string { + return 'table'; + } + + get() { + return { + knowledgePages: this.knowledgePages, + dataAsset: this.dataAsset, + responseData: this.responseData, + }; + } + + async delete( + apiContext: APIRequestContext, + deletePages = true, + deleteDataAsset = true + ) { + if (deletePages) { + for (const page of this.knowledgePages) { + if (page.id) { + await apiContext.delete( + `/api/v1/knowledgeCenter/${page.id}?hardDelete=true&recursive=true` + ); + } + } + this.knowledgePages = []; + } + + if (deleteDataAsset && this.dataAsset.entityResponseData?.id) { + await this.dataAsset.delete(apiContext); + } + } +} diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/KnowledgeCenter.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/KnowledgeCenter.ts new file mode 100644 index 000000000000..c9fcbd0ac9aa --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/KnowledgeCenter.ts @@ -0,0 +1,916 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { expect, Locator, Page } from '@playwright/test'; +import { + SHORTCUTS, + SLASH_COMMANDS, +} from '../constant/KnowledgeCenter.constant'; +import { SidebarItem } from '../constant/sidebar'; +import { TopicClass } from '../support/entity/TopicClass'; +import { + descriptionBox, + descriptionBoxReadOnly, + redirectToHomePage, +} from './common'; +import { waitForAllLoadersToDisappear } from './entity'; +import { sidebarClick } from './sidebar'; + +const KNOWLEDGE_PAGE_ROUTE = '/knowledge-center/:fqn'; +const FQN_PLACEHOLDER = ':fqn'; + +export const deletePage = async ( + page: Page, + isQuickLink = false, + entityFqn?: string +) => { + if (!isQuickLink) { + await page.getByTestId('manage-button').click(); + await page.getByTestId('delete-button').click(); + } + + await page.waitForSelector('[role="dialog"].ant-modal'); + + await expect(page.locator('[role="dialog"].ant-modal')).toBeVisible(); + + await page.click('[data-testid="hard-delete-option"]'); + await page.check('[data-testid="hard-delete"]'); + await page.fill('[data-testid="confirmation-text-input"]', 'DELETE'); + + const deleteResponse = page.waitForResponse( + `/api/v1/knowledgeCenter/*?hardDelete=true&recursive=${!isQuickLink}` + ); + + // Register before clicking so we don't miss the response the app fires + // naturally after redirect. Uses the browser session (no 401 risk). + const hierarchyResponse = entityFqn + ? page.waitForResponse( + (response) => + response.url().includes('/api/v1/knowledgeCenter/search/hierarchy') && + response.request().method() === 'GET' + ) + : null; + + await page.getByTestId('confirm-button').click(); + + const deleteRes = await deleteResponse; + expect(deleteRes.status()).toBe(200); + + if (entityFqn && hierarchyResponse) { + const listRes = await hierarchyResponse; + expect(listRes.status()).toBe(200); + const body = await listRes.json(); + expect(JSON.stringify(body)).not.toContain(entityFqn); + } +}; + +export const addTitle = async (page: Page, title: string) => { + const updateTitleResponse = page.waitForResponse( + (response) => + response.url().includes('/api/v1/knowledgeCenter/') && + response.request().method() === 'PATCH' + ); + + await page.getByTestId('entity-header-display-name').fill(title); + + const res = await updateTitleResponse; + expect(res.status()).toBe(200); + + await expect(page.getByTestId('content-change-state')).toHaveText('Saved'); +}; + +export const updateBody = async (page: Page, body: string) => { + await page.fill('.om-block-editor', body); + const updateBodyResponse = page.waitForResponse( + (response) => + response.url().includes('/api/v1/knowledgeCenter/') && + response.request().method() === 'PATCH' + ); + const res = await updateBodyResponse; + expect(res.status()).toBe(200); + + await expect(page.getByTestId('content-change-state')).toHaveText('Saved'); +}; + +export const updateTags = async ( + page: Page, + data: { tag: string; tagFqn: string } +) => { + const updateKnowledgePage = page.waitForResponse( + (response) => + response.url().includes('/api/v1/knowledgeCenter/') && + response.request().method() === 'PATCH' + ); + await page.click('[data-testid="tags-container"] [data-testid="add-tag"]'); + + await page.waitForSelector('[data-testid="tag-selector"] input', { + state: 'visible', + }); + await page.fill('[data-testid="tag-selector"] input', data.tag); + await page.click(`[data-testid='tag-${data.tagFqn}']`); + + await expect( + page.locator( + `[data-testid="tag-selector"] [data-testid="selected-tag-${data.tagFqn}"]` + ) + ).toBeVisible(); + + await page.locator('[data-testid="saveAssociatedTag"]').click(); + const response = await updateKnowledgePage; + expect(response.status()).toBe(200); +}; + +export const updateDataAsset = async ( + page: Page, + dataAsset: TopicClass, + title: string +) => { + const updateKnowledgePage = page.waitForResponse( + (response) => + response.url().includes('/api/v1/knowledgeCenter/') && + response.request().method() === 'PATCH' + ); + await page + .getByTestId('add-data-assets-container') + .locator('span') + .first() + .click(); + + await page.waitForSelector( + '[data-testid="asset-select-list"] > .ant-select-selector input', + { state: 'visible' } + ); + await page.fill( + '[data-testid="asset-select-list"] > .ant-select-selector input', + dataAsset.entity.name + ); + await page + .locator('.ant-select-item-option-content', { + hasText: dataAsset.entity.name, + }) + .click(); + await page.locator('[data-testid="saveDataAssets"]').click(); + + const response = await updateKnowledgePage; + expect(response.status()).toBe(200); + + await page.waitForSelector(`[data-testid="${dataAsset.entity.name}"]`, { + state: 'visible', + }); + await page.click(`[data-testid="${dataAsset.entity.name}"]`); + + await page.getByRole('link', { name: title }).click(); +}; + +export const updateVotes = async (page: Page) => { + await page.click('[data-testid="up-vote-btn"]'); + + await expect(page.locator('[data-testid="up-vote-count"]')).toHaveText('1'); + + await page.click('[data-testid="down-vote-btn"]'); + + await expect(page.locator('[data-testid="up-vote-count"]')).toHaveText('0'); + await expect(page.locator('[data-testid="down-vote-count"]')).toHaveText('1'); +}; + +export const updateFollowers = async (page: Page) => { + await page.click('[data-testid="entity-follow-button"]'); + + await expect( + page.locator('[data-testid="entity-follow-button"] > .ant-typography') + ).toHaveText('1'); +}; + +export const readArticleData = async ( + page: Page, + data: { + title: string; + body: string; + tagFqn: string; + } +) => { + await sidebarClick(page, SidebarItem.KNOWLEDGE_CENTER); + await readArticleInHierarchy(page, data.title); +}; + +export const createQuickLink = async ( + page: Page, + data: { + displayName: string; + url: string; + description: string; + }, + dataAsset: TopicClass +) => { + await page.locator('[data-testid="add-knowledge-page-btn"]').click(); + + await expect( + page.getByRole('menuitem', { name: 'Quick Link' }) + ).toBeVisible(); + + await page.getByRole('menuitem', { name: 'Quick Link' }).click(); + + await expect(page.locator('.ant-modal-title')).toHaveText('Add Quick Link'); + + await page.locator('[data-testid="displayName"]').fill(data.displayName); + await page.locator('[data-testid="url"]').fill(data.url); + await page.locator(descriptionBox).fill(data.description); + + await page + .locator('[data-testid="asset-select-list"] > .ant-select-selector input') + .click(); + await page + .locator('[data-testid="asset-select-list"] > .ant-select-selector input') + .fill(dataAsset.entity.name); + + await expect( + page.locator( + '.ant-select-item-option-content:has-text("' + + dataAsset.entity.name + + '")' + ) + ).toBeVisible(); + + await page.click( + '.ant-select-item-option-content:has-text("' + dataAsset.entity.name + '")' + ); + + await page.click('.ant-modal-footer > #quick-link-form'); +}; + +export const readQuickLink = async ( + page: Page, + quickLink: { + displayName: string; + description: string; + url: string; + } +) => { + await page + .locator(`[data-testid="${quickLink.displayName}"]`) + .scrollIntoViewIfNeeded(); + + await expect( + page.locator( + `[data-testid="${quickLink.displayName}"] ${descriptionBoxReadOnly} > p` + ) + ).toHaveText(quickLink.description); + await expect( + page.locator( + `[data-testid="${quickLink.displayName}"] [data-testid="knowledge-link"]` + ) + ).toHaveAttribute('href', quickLink.url); + await expect( + page.locator( + `[data-testid="${quickLink.displayName}"] [data-testid="knowledge-link"]` + ) + ).toHaveAttribute('target', '_blank'); +}; + +export const updateQuickLink = async ( + page: Page, + knowledgePageQuickLink: { + displayName: string; + updatedDisplayName: string; + updatedUrl: string; + updatedDescription: string; + tag: string; + tagFqn: string; + } +) => { + await page + .locator( + `[data-testid="${knowledgePageQuickLink.displayName}"] [data-testid="edit-quick-link-btn"]` + ) + .click(); + + await expect(page.locator('.ant-modal-title')).toHaveText( + `Edit Quick Link ${knowledgePageQuickLink.displayName}` + ); + + await page + .locator('[data-testid="displayName"]') + .fill(knowledgePageQuickLink.updatedDisplayName); + await page + .locator('[data-testid="url"]') + .fill(knowledgePageQuickLink.updatedUrl); + await page + .locator(descriptionBox) + .fill(knowledgePageQuickLink.updatedDescription); + + await page.locator('[data-testid="tag-selector"] input').first().click(); + await page + .locator('[data-testid="tag-selector"] input') + .first() + .fill(knowledgePageQuickLink.tag); + await page.getByTestId(`tag-${knowledgePageQuickLink.tagFqn}`).click(); + + await page.click('.ant-modal-footer > #quick-link-form'); + + await readQuickLink(page, { + displayName: knowledgePageQuickLink.updatedDisplayName, + description: knowledgePageQuickLink.updatedDescription, + url: knowledgePageQuickLink.updatedUrl, + }); +}; + +export const readArticleInHierarchy = async ( + page: Page, + articleTitle: string +) => { + const article = page.locator(`[data-testid="page-node-${articleTitle}"]`); + + // Reset scroll position to top before starting pagination + const hierarchyElement = page.locator( + '[data-testid="knowledge-pages-hierarchy"]' + ); + await hierarchyElement.hover(); + await page.mouse.wheel(0, -9999); + + await page.waitForTimeout(500); + + // Retry mechanism for pagination + let elementCount = await article.count(); + let retryCount = 0; + const maxRetries = 20; + + while (elementCount === 0 && retryCount < maxRetries) { + await page.locator('[data-testid="knowledge-pages-hierarchy"]').hover(); + await page.mouse.wheel(0, 500); + await page.waitForTimeout(500); + + // Create fresh locator and check if the article is now visible after this retry + const freshArticle = page.getByTestId(`page-node-${articleTitle}`); + const count = await freshArticle.count(); + + // Check if the article is now visible after this retry + elementCount = count; + + // If we found the element, validate it and break out of the loop + if (count > 0) { + await expect(freshArticle).toBeVisible(); + + return; // Exit the function early since we found and validated the article + } + + retryCount++; + } +}; + +export const createMentionInConversation = async ( + page: Page, + userName: string, + message: string +) => { + // Open conversation drawer + await page.locator('[data-testid="conversation"]').click(); + await page.locator('.feed-drawer').waitFor({ state: 'visible' }); + + // Click on Conversations tab + await page.getByRole('tab', { name: 'Conversations' }).click(); + await page.waitForSelector('[data-testid="editor-wrapper"]'); + + // Create message with mention + const messageWithMention = `${message} @${userName}`; + + // Wait for mention suggestions to appear + const userSuggestionsResponse = page.waitForResponse( + `/api/v1/search/query?q=*${userName}***` + ); + + // Type the message with mention + await page + .locator('[data-testid="editor-wrapper"] [contenteditable="true"]') + .click(); + await page + .locator('[data-testid="editor-wrapper"] [contenteditable="true"]') + .fill(messageWithMention); + + await userSuggestionsResponse; + + // Select the mention from suggestions + await page.locator(`[data-value="@${userName}"]`).first().click(); + + // Send the message + const feedResponse = page.waitForResponse('/api/v1/feed'); + await page.locator('[data-testid="send-button"]').click(); + await feedResponse; + + // Verify the message with mention is visible + await expect(page.locator(`text=${messageWithMention}`)).toBeVisible(); + + return messageWithMention; +}; + +export const verifyNotificationAndClick = async ( + page: Page, + expectedUserName: string, + entityName: string +) => { + // Click on notification bell + await page.locator('[data-testid="task-notifications"]').click(); + + // Wait for notification dropdown to appear + await page.waitForSelector('[data-testid="notification-heading"]', { + state: 'visible', + }); + + // Click on Mentions tab + const mentionsTabResponse = page.waitForResponse( + '/api/v1/feed?userId=*&filterType=MENTIONS' + ); + + await page.getByRole('tab', { name: 'Mentions' }).click(); + await mentionsTabResponse; + + // Verify the notification contains the mentioned user and entity type + await expect( + page.getByTestId(`notification-item-${entityName}`).nth(1) + ).toContainText(expectedUserName); + + await expect( + page.getByTestId(`notification-item-${entityName}`).nth(1) + ).toContainText(entityName); + + // Click on the notification to navigate to the entity + await page.getByTestId(`notification-link-${entityName}`).nth(1).click(); +}; + +export const getKnowledgePageCardByIndex = async ( + page: Page, + index: number +) => { + const listing = page.getByTestId('knowledge-page-listing'); + const cards = listing.locator('.knowledge-card'); + await expect(cards.nth(index)).toBeAttached(); + const card = cards.nth(index); + await card.scrollIntoViewIfNeeded(); + await expect(card).toBeVisible(); + return card; +}; +/** + * Derives the entity identifier used by getLink() in KnowledgePageUtils: + * data-testid = `${prefix}-${displayName || fullyQualifiedName}` + * The || (not ??) operator means an empty-string displayName also falls back + * to the FQN. When displayName is empty/null the card shows "Untitled" + * (i18n placeholder), but the widget testid uses the FQN. This helper + * mirrors that logic and returns the correct value for testid lookups. + */ +export const getKnowledgePageCardEntityIdentifier = async ( + card: Locator +): Promise => { + const href = + (await card.getByTestId('knowledge-page-link').getAttribute('href')) ?? ''; + const fqn = href.split('/knowledge-center/').pop() ?? ''; + const displayText = ( + await card.getByTestId('entity-header-display-name').textContent() + )?.trim(); + return displayText && displayText !== 'Untitled' ? displayText : fqn; +}; + +export const toggleKnowledgePageBookmark = async ( + page: Page, + bookmarkBtn: Locator, + bookmarkIdentifier: string, + shouldBeVisible: boolean +) => { + const bookmarkResponse = page.waitForResponse((response) => { + const url = response.url(); + return ( + url.includes('/api/v1/knowledgeCenter') && url.includes('/followers') + ); + }); + const widgetRefreshResponse = page.waitForResponse((response) => { + return ( + response.url().includes('/api/v1/users') && + response.url().includes('fields=follows') + ); + }); + + await bookmarkBtn.click(); + const bookmarkRes = await bookmarkResponse; + const widgetRes = await widgetRefreshResponse; + expect(bookmarkRes.status()).toBe(200); + expect(widgetRes.status()).toBe(200); + await waitForAllLoadersToDisappear(page); + + const rightPanel = page.getByTestId('knowledge-center-right-panel'); + const specificBookmark = rightPanel.getByTestId( + `bookmarked-${bookmarkIdentifier}` + ); + + if (shouldBeVisible) { + await expect(specificBookmark).toBeVisible(); + } else { + await expect(specificBookmark).not.toBeVisible(); + } +}; + +export const createNewKnowledgePageArticle = async ( + page: Page, + articleTitle: string +) => { + const createKnowledgePage = page.waitForResponse('/api/v1/knowledgeCenter'); + + await sidebarClick(page, SidebarItem.KNOWLEDGE_CENTER); + await page + .locator('[data-testid="left-panel"]') + .getByTestId('add-knowledge-page-btn') + .click(); + await page.getByRole('menuitem', { name: 'Article' }).click(); + await createKnowledgePage; + + await addTitle(page, articleTitle); + + return { articleTitle }; +}; + +export const getEditor = async (page: Page, waitForWrapper = false) => { + if (waitForWrapper) { + await waitForAllLoadersToDisappear(page); + await page.waitForSelector('#block-editor-wrapper', { + state: 'visible', + }); + } + + await page.waitForSelector('.ProseMirror[contenteditable="true"]', { + state: 'visible', + }); + return page.locator('.ProseMirror[contenteditable="true"]').first(); +}; + +export const executeSlashCommand = async ( + page: Page, + command: string +): Promise => { + await page.keyboard.type(`/${command}`); + await page.keyboard.press(SHORTCUTS.enter); +}; + +export const moveToNewLine = async ( + page: Page, + editor: Locator, + exitList = false +): Promise => { + await editor.click(); + await page.keyboard.press(SHORTCUTS.end); + await page.keyboard.press(SHORTCUTS.enter); + if (exitList) { + await page.keyboard.press(SHORTCUTS.backspace); + } + await page.keyboard.press(SHORTCUTS.enter); +}; + +export const moveToNewParagraph = async ( + page: Page, + editor: Locator +): Promise => { + await editor.click(); + await page.keyboard.press(SHORTCUTS.end); + await page.keyboard.press(SHORTCUTS.enter); + await page.keyboard.press(SHORTCUTS.enter); + await page.keyboard.press(SHORTCUTS.enter); +}; + +export const selectLastWord = async ( + page: Page, + wordCount = 1, + editor?: Locator +): Promise => { + if (editor) { + await editor.click(); + await expect(editor).toBeFocused(); + } + await page.keyboard.press(SHORTCUTS.end); + if (editor) { + await expect(editor).toBeFocused(); + } + + for (let i = 0; i < wordCount; i++) { + await page.keyboard.press(SHORTCUTS.selectWord); + if (editor && i < wordCount - 1) { + await expect(editor).toBeFocused(); + } + } +}; + +export const applyTextFormatting = async ( + page: Page, + format: 'bold' | 'italic' | 'code' +): Promise => { + await page.keyboard.press(SHORTCUTS[format]); +}; + +export const selectAllText = async (page: Page) => { + await page.keyboard.press(SHORTCUTS.selectAll); +}; + +export const selectEditorLastWord = async (page: Page) => { + await page.keyboard.press(SHORTCUTS.selectWord); +}; + +export const createHeading = async ( + page: Page, + level: 1 | 2 | 3, + text: string +): Promise => { + await page.keyboard.press(SHORTCUTS.enter); + const headingCommand = + level === 1 + ? SLASH_COMMANDS.h1 + : level === 2 + ? SLASH_COMMANDS.h2 + : SLASH_COMMANDS.h3; + await executeSlashCommand(page, headingCommand); + await page.keyboard.type(text); +}; + +export const createListItems = async ( + page: Page, + items: string[] +): Promise => { + for (const item of items) { + await page.keyboard.type(item); + await page.keyboard.press(SHORTCUTS.enter); + } +}; + +export const clearCodeFormatting = async ( + page: Page, + editor: Locator +): Promise => { + await page.keyboard.type('X'); + + const testInCode = editor.locator('code').filter({ hasText: 'X' }); + const isInCode = (await testInCode.count()) > 0; + + if (isInCode) { + await page.keyboard.press(SHORTCUTS.selectWord); + + await page.waitForSelector('.menu-wrapper', { + state: 'visible', + }); + + const codeButton = page.getByRole('button', { name: 'Inline code' }); + await expect(codeButton).toBeVisible(); + await codeButton.click(); + } + + await page.keyboard.press(SHORTCUTS.backspace); +}; + +export const createLink = async ( + page: Page, + editor: Locator, + linkText: string, + url: string +): Promise => { + await clearCodeFormatting(page, editor); + + await page.keyboard.type(linkText); + + await selectLastWord(page, linkText.split(' ').length, editor); + + await page.waitForSelector('.menu-wrapper', { + state: 'visible', + }); + + const linkButton = page.getByRole('button', { name: 'Link' }); + await expect(linkButton).toBeVisible(); + await linkButton.click(); + + const linkDialog = page.getByRole('dialog', { name: 'Add link' }); + await expect(linkDialog).toBeVisible(); + + const linkInput = page.locator('#href'); + await expect(linkInput).toBeVisible(); + await linkInput.fill(url); + + const saveButton = linkDialog.getByRole('button', { name: 'Save' }); + await expect(saveButton).toBeVisible(); + await saveButton.click(); +}; + +export const waitForAutoSave = async (page: Page): Promise => { + await expect(page.getByTestId('content-change-state')).toHaveText('Saved', { + timeout: 15000, + }); +}; + +export const verifyTextFormatting = async ( + editor: Locator, + text: string, + format: 'bold' | 'italic' | 'code' +) => { + const formatTag = { + bold: 'strong', + italic: 'em', + code: 'code', + }[format]; + + await expect(editor.locator(formatTag, { hasText: text })).toBeVisible(); +}; + +export const undo = async (page: Page): Promise => { + await page.keyboard.press(SHORTCUTS.undo); +}; + +export const redo = async (page: Page): Promise => { + await page.keyboard.press(SHORTCUTS.redo); +}; + +export const selectAll = async (page: Page): Promise => { + await page.keyboard.press(SHORTCUTS.selectAll); +}; + +export const copyContent = async (page: Page): Promise => { + await page.keyboard.press(SHORTCUTS.copy); +}; + +export const pasteContent = async (page: Page): Promise => { + await page.keyboard.press(SHORTCUTS.paste); +}; + +export const createNestedListItems = async ( + page: Page, + parentItems: string[], + nestedItems: string[] +): Promise => { + for (let i = 0; i < parentItems.length; i++) { + await page.keyboard.type(parentItems[i]); + await page.keyboard.press(SHORTCUTS.enter); + + if (i === parentItems.length - 1) { + await page.keyboard.press(SHORTCUTS.tab); + + for (const nestedItem of nestedItems) { + await page.keyboard.type(nestedItem); + await page.keyboard.press(SHORTCUTS.enter); + } + } + } +}; + +export const verifyNestedList = async ( + editor: Locator, + parentText: string, + nestedText: string +): Promise => { + await expect( + editor.locator('li').filter({ hasText: parentText }) + ).toBeVisible(); + + const parentLi = editor.locator('li').filter({ hasText: parentText }); + const nestedItem = parentLi.locator('li').filter({ hasText: nestedText }); + await expect(nestedItem).toBeVisible(); +}; + +export const verifyContentPersistence = async ( + page: Page, + expectedTexts: string[] +): Promise => { + await waitForAutoSave(page); + + await page.reload(); + await waitForAllLoadersToDisappear(page); + + const editor = await getEditor(page, true); + await editor.waitFor({ state: 'visible' }); + + for (const text of expectedTexts) { + await expect(page.getByText(text)).toBeVisible(); + } +}; + +export const createCodeBlock = async ( + page: Page, + code: string +): Promise => { + await executeSlashCommand(page, SLASH_COMMANDS.codeBlock); + await page.keyboard.type(code); +}; + +export const verifyCodeBlock = async ( + editor: Locator, + code: string +): Promise => { + const codeBlock = editor.locator('pre code').filter({ hasText: code }); + await expect(codeBlock).toBeVisible(); +}; + +export const createTaskListItems = async ( + page: Page, + tasks: string[] +): Promise => { + for (let i = 0; i < tasks.length; i++) { + await page.keyboard.type(tasks[i]); + if (i < tasks.length - 1) { + await page.keyboard.press(SHORTCUTS.enter); + } + } +}; + +export const verifyTaskList = async ( + editor: Locator, + taskText: string +): Promise => { + const taskItem = editor.locator('li').filter({ hasText: taskText }); + await expect(taskItem).toBeVisible(); + + const checkbox = taskItem.locator('input[type="checkbox"]'); + await expect(checkbox).toBeVisible(); +}; + +export const toggleTask = async ( + page: Page, + editor: Locator, + taskText: string +): Promise => { + const taskItem = editor.locator('li').filter({ hasText: taskText }); + const checkbox = taskItem.locator('input[type="checkbox"]'); + await checkbox.click(); + await page.waitForTimeout(100); +}; + +export const createCallout = async ( + page: Page, + text: string +): Promise => { + await executeSlashCommand(page, SLASH_COMMANDS.callout); + await page.waitForTimeout(200); + await page.keyboard.type(text); +}; + +export const verifyCallout = async ( + editor: Locator, + text: string +): Promise => { + const callout = editor + .locator('[data-type="callout"]') + .filter({ hasText: text }); + await expect(callout).toBeVisible(); +}; + +export const createTable = async (page: Page): Promise => { + await executeSlashCommand(page, SLASH_COMMANDS.table); + await page.waitForTimeout(300); +}; + +export const verifyTable = async (editor: Locator): Promise => { + const table = editor.locator('table'); + await expect(table).toBeVisible(); +}; + +export const typeInTableCell = async ( + page: Page, + row: number, + col: number, + text: string +): Promise => { + const editor = await getEditor(page); + const table = editor.locator('table'); + const rows = table.locator('tr'); + const targetRow = rows.nth(row); + const cells = targetRow.locator('td, th'); + const targetCell = cells.nth(col); + + await targetCell.click(); + await page.keyboard.type(text); +}; + +export const navigateToArticle = async (page: Page, articleFqn: string) => { + // Wait for GET API response when navigating to the article + const getArticleResponse = page.waitForResponse( + (response) => + response.url().includes(`/api/v1/knowledgeCenter/name/${articleFqn}`) && + response.status() === 200 + ); + + const articlePath = KNOWLEDGE_PAGE_ROUTE.replace(FQN_PLACEHOLDER, articleFqn); + await page.goto(articlePath); + await getArticleResponse; + await waitForAllLoadersToDisappear(page); + await getArticleResponse; +}; + +export const navigateToKnowledgeCenter = async (page: Page) => { + await redirectToHomePage(page); + + const knowledgeCenterResponse = page.waitForResponse( + (response) => + response.url().includes('/api/v1/knowledgeCenter') || + response.url().includes('/knowledge-center') + ); + + await sidebarClick(page, SidebarItem.KNOWLEDGE_CENTER); + await knowledgeCenterResponse; +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/img/widgets/knowledge-center-widget.png b/openmetadata-ui/src/main/resources/ui/src/assets/img/widgets/knowledge-center-widget.png new file mode 100644 index 000000000000..08890e618615 Binary files /dev/null and b/openmetadata-ui/src/main/resources/ui/src/assets/img/widgets/knowledge-center-widget.png differ diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-article.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-article.svg new file mode 100644 index 000000000000..1f0b71d0970f --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-article.svg @@ -0,0 +1,3 @@ + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-articles.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-articles.svg new file mode 100644 index 000000000000..9cfbbef5c02b --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-articles.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-bookmark.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-bookmark.svg new file mode 100644 index 000000000000..f4308cf59956 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-bookmark.svg @@ -0,0 +1,3 @@ + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-bookmarked.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-bookmarked.svg new file mode 100644 index 000000000000..d6df115a52c3 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-bookmarked.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-conversation.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-conversation.svg new file mode 100644 index 000000000000..a4cf85d2f618 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-conversation.svg @@ -0,0 +1,3 @@ + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-eye.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-eye.svg new file mode 100644 index 000000000000..cde9ca222902 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-eye.svg @@ -0,0 +1,4 @@ + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-knowledge-center-widget.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-knowledge-center-widget.svg new file mode 100644 index 000000000000..eadd6c0a0772 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-knowledge-center-widget.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-knowledge-center.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-knowledge-center.svg index e68910e5212c..a6488595a321 100644 --- a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-knowledge-center.svg +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-knowledge-center.svg @@ -1,3 +1,11 @@ - - + + + + + + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-knowledge-page.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-knowledge-page.svg new file mode 100644 index 000000000000..00c622e11f77 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-knowledge-page.svg @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-link.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-link.svg new file mode 100644 index 000000000000..a73cd590b818 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-link.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-quick-link.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-quick-link.svg new file mode 100644 index 000000000000..0d44d2f13268 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-quick-link.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-saved.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-saved.svg new file mode 100644 index 000000000000..b1a96eb830ba --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-saved.svg @@ -0,0 +1,3 @@ + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-success.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-success.svg new file mode 100644 index 000000000000..09c70505940f --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-success.svg @@ -0,0 +1,4 @@ + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-unsaved.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-unsaved.svg new file mode 100644 index 000000000000..1aa7bf87ad86 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-unsaved.svg @@ -0,0 +1,3 @@ + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-updated.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-updated.svg new file mode 100644 index 000000000000..1f58b979c4f1 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-updated.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic_add.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic_add.svg index 9d2836d520f7..c4ba0bafe82f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic_add.svg +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic_add.svg @@ -1 +1 @@ - + \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/knowledge-center.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/knowledge-center.svg new file mode 100644 index 000000000000..b055a6bda4ff --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/knowledge-center.svg @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Suggestions.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Suggestions.tsx index ba153c33a4b5..16064dfd6dfc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Suggestions.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Suggestions.tsx @@ -13,6 +13,7 @@ import { Button, Typography } from 'antd'; import { AxiosError } from 'axios'; +import { ContainerSearchSource } from 'interface/search.interface'; import { isEmpty, isString } from 'lodash'; import Qs from 'qs'; import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; @@ -20,9 +21,32 @@ import { useTranslation } from 'react-i18next'; import { ReactComponent as IconSuggestionsBlue } from '../../assets/svg/ic-suggestions-blue.svg'; import { PAGE_SIZE_BASE } from '../../constants/constants'; import { + APICollectionSource, + APIEndpointSource, + ChartSource, + ColumnSource, + DashboardDataModelSource, + DashboardSource, + DatabaseSchemaSource, + DatabaseSource, + DataProductSource, + DirectorySource, + FileSource, + GlossarySource, + KnowledgePageSource, + MetricSource, + MlModelSource, Option, + PipelineSource, + SearchIndexSource, SearchSuggestions, + SpreadsheetSource, + StoredProcedureSource, SuggestionsObject, + TableSource, + TagSource, + TopicSource, + WorksheetSource, } from '../../context/GlobalSearchProvider/GlobalSearchSuggestions/GlobalSearchSuggestions.interface'; import { useTourProvider } from '../../context/TourProvider/TourProvider'; import { SearchIndex } from '../../enums/search.enum'; @@ -83,6 +107,7 @@ const Suggestions = ({ fileSuggestions: [], spreadsheetSuggestions: [], worksheetSuggestions: [], + knowledgeArticleSuggestions: [], }); const { @@ -111,68 +136,102 @@ const Suggestions = ({ const updateSuggestions = (options: Array

Mocked Loader
) +); + +jest.mock('rest/userAPI', () => { + return { + getUserById: jest.fn().mockImplementation(() => { + return Promise.resolve({ + follows: [ + { + type: 'page', + id: 'test-page-id', + displayName: 'test-page-name', + fullyQualifiedName: 'test-page-fqn', + }, + ], + }); + }), + }; +}); + +jest.mock('utils/ToastUtils', () => { + return { + showErrorToast: jest.fn().mockImplementation(() => { + return 'Mocked showErrorToast'; + }), + }; +}); + +jest.mock('../../../utils/KnowledgePageUtils', () => ({ + ...jest.requireActual('../../../utils/KnowledgePageUtils'), + getKnowledgePagePath: jest.fn().mockImplementation(() => { + return '/knowledge-center/test-page-fqn'; + }), +})); + +const mockHandleRefreshBookMarkWidget = jest.fn(); + +const mockProps = { + refresh: false, + handleRefreshBookMarkWidget: mockHandleRefreshBookMarkWidget, +}; + +describe('BookMarkWidget', () => { + it('should render BookMarkWidget', async () => { + render(, { wrapper: MemoryRouter }); + + await waitForElementToBeRemoved(() => screen.getByText('Mocked Loader')); + + const bookmarkedPage = screen.getByTestId('bookmarked-test-page-name'); + + expect(bookmarkedPage).toHaveTextContent('test-page-name'); + expect(bookmarkedPage).toHaveAttribute( + 'href', + '/knowledge-center/test-page-fqn' + ); + + expect(screen.getByText('label.bookmark-plural')).toBeInTheDocument(); + + expect(screen.queryByText('Mocked showErrorToast')).not.toBeInTheDocument(); + + expect(mockHandleRefreshBookMarkWidget).toHaveBeenCalledWith(false); + }); + + it('should not call the getUserById if currentUser is not present', async () => { + (useApplicationStore as unknown as jest.Mock).mockImplementationOnce( + () => ({ + currentUser: null, + }) + ); + + render(, { wrapper: MemoryRouter }); + + expect(screen.queryByText('test-page-name')).not.toBeInTheDocument(); + expect(screen.queryByText('test-page-fqn')).not.toBeInTheDocument(); + expect(screen.queryByText('Mocked showErrorToast')).not.toBeInTheDocument(); + + expect(getUserById).not.toHaveBeenCalled(); + }); + + it('should render the placeholder text if there are no bookmarks', async () => { + (getUserById as jest.Mock).mockImplementation(() => { + return Promise.resolve({ + follows: [], + }); + }); + + render(, { wrapper: MemoryRouter }); + + await waitForElementToBeRemoved(() => screen.getByText('Mocked Loader')); + + expect(screen.queryByText('test-page-name')).not.toBeInTheDocument(); + expect(screen.queryByText('test-page-fqn')).not.toBeInTheDocument(); + expect(screen.queryByText('Mocked showErrorToast')).not.toBeInTheDocument(); + + expect( + screen.getByText('message.not-bookmark-anything') + ).toBeInTheDocument(); + }); + + it("should render the title as 'Untitled' if the displayName is not present", async () => { + (getUserById as jest.Mock).mockImplementation(() => { + return Promise.resolve({ + follows: [ + { + type: 'page', + id: 'test-page-id', + displayName: null, + fullyQualifiedName: 'test-page-fqn', + }, + ], + }); + }); + + render(, { wrapper: MemoryRouter }); + + await waitForElementToBeRemoved(() => screen.getByText('Mocked Loader')); + + // test id should have the fullyQualifiedName + const bookmarkedPage = screen.getByTestId('bookmarked-test-page-fqn'); + + expect(bookmarkedPage).toHaveTextContent('label.untitled'); + expect(bookmarkedPage).toHaveAttribute( + 'href', + '/knowledge-center/test-page-fqn' + ); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/BookMarkWidget/BookMarkWidget.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/BookMarkWidget/BookMarkWidget.tsx new file mode 100644 index 000000000000..cf5df7004e56 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/BookMarkWidget/BookMarkWidget.tsx @@ -0,0 +1,103 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Space, Typography } from 'antd'; +import { AxiosError } from 'axios'; +import { isEmpty, map } from 'lodash'; +import { useEffect, useMemo, useState } from 'react'; +import { ReactComponent as BookMarkIcon } from '../../../assets/svg/ic-bookmark.svg'; +import ExpandableCard from '../../../components/common/ExpandableCard/ExpandableCard'; +import Loader from '../../../components/common/Loader/Loader'; +import { EntityType, TabSpecificField } from '../../../enums/entity.enum'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; +import { KnowledgePage } from '../../../interface/knowledge-center.interface'; +import { getUserById } from '../../../rest/userAPI'; +import { t } from '../../../utils/i18next/LocalUtil'; +import { getLink } from '../../../utils/KnowledgePageUtils'; +import { showErrorToast } from '../../../utils/ToastUtils'; + +const BookMarkWidget = ({ + refresh, + handleRefreshBookMarkWidget, +}: { + refresh: boolean; + handleRefreshBookMarkWidget: (value: boolean) => void; +}) => { + const { currentUser } = useApplicationStore(); + const [isLoading, setIsLoading] = useState(true); + const [data, setData] = useState([]); + + const fetchBookMarks = async () => { + if (!currentUser?.id) { + return; + } + + try { + const userData = await getUserById(currentUser?.id, { + fields: TabSpecificField.FOLLOWS, + }); + const bookmarkData = (userData.follows ?? []).filter( + (reference) => reference.type === EntityType.KNOWLEDGE_PAGE + ); + setData(bookmarkData as unknown as KnowledgePage[]); + } catch (error) { + setData([]); + showErrorToast(error as AxiosError); + } finally { + setIsLoading(false); + handleRefreshBookMarkWidget(false); + } + }; + + const header = useMemo(() => { + return ( +
+ + + {t('label.bookmark-plural')} + +
+ ); + }, [t]); + + useEffect(() => { + fetchBookMarks(); + }, [currentUser]); + + useEffect(() => { + if (refresh) { + fetchBookMarks(); + } + }, [refresh]); + + if (isLoading) { + return ; + } + + return ( + + {isEmpty(data) ? ( + t('message.not-bookmark-anything') + ) : ( + + {map(data, (instance) => getLink(instance, 'bookmarked'))} + + )} + + ); +}; + +export default BookMarkWidget; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCard/KnowledgeCard.mock.ts b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCard/KnowledgeCard.mock.ts new file mode 100644 index 000000000000..772c0f2598c6 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCard/KnowledgeCard.mock.ts @@ -0,0 +1,274 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable max-len */ + +import { + Article, + KnowledgePage, +} from '../../../interface/knowledge-center.interface'; + +export const KNOWLEDGE_PAGE_TAGS = [ + { + tagFQN: 'KnowledgeCenter.HowToGuide', + name: 'HowToGuide', + description: 'How To Guide Quick Link or Article Tag.', + style: { + color: '#25d80e', + }, + source: 'Classification', + labelType: 'Manual', + state: 'Confirmed', + }, + { + tagFQN: 'PersonalData.SpecialCategory', + name: 'SpecialCategory', + description: + 'GDPR special category data is personal information of data subjects that is especially sensitive, the exposure of which could significantly impact the rights and freedoms of data subjects and potentially be used against them for unlawful discrimination.', + source: 'Classification', + labelType: 'Derived', + state: 'Confirmed', + }, + { + tagFQN: 'PII.None', + name: 'None', + description: 'Non PII', + style: {}, + source: 'Classification', + labelType: 'Derived', + state: 'Confirmed', + }, + { + tagFQN: 'testing.testing_term_1', + name: 'testing_term_1', + displayName: 'testing_term_1', + description: 'testing_term_1', + style: {}, + source: 'Glossary', + labelType: 'Manual', + state: 'Confirmed', + }, + { + tagFQN: 'testing.testing_term_4', + name: 'testing_term_4', + displayName: 'testing_term_4', + description: 'testing_term_4', + style: {}, + source: 'Glossary', + labelType: 'Manual', + state: 'Confirmed', + }, +]; + +export const KNOWLEDGE_PAGE_MOCK_DATA = { + id: '8e6427d6-98cc-4334-b2f2-15fb62bde887', + name: 'Article_oRKYYTCu', + fullyQualifiedName: 'Article_oRKYYTCu', + displayName: 'OpenMetadata 1.1.0 Release UI', + description: + 'Less than two months have passed since our exciting OpenMetadata 1.0 Release, and we’re thrilled to announce the completion of Release 1.1 already! The OpenMetadata community thrives on pushing our limits; this latest release is a testament to it. Prepare to be amazed as we unveil a complete UI overhaul, meticulously designed to elevate the user experience across the entire platform. But that’s not all! We’ve also introduced four new connectors, implemented advanced PII masking, and significantly enhanced lineage parsing capabilities, just to name a few of the numerous features we’ve packed into this release. Stay tuned for an exceptional OpenMetadata experience like never before!\n\n_In the upcoming 1.2 Release of OpenMetadata, we are thrilled to introduce exclusive new features specifically tailored for Collate SaaS. You can review Collate’s roadmap here and be as excited as we are 🚀_\n\nCommunity Updates\n-----------------\n\nThanks to the incredible OpenMetadata Community, our growth and activity have skyrocketed. Slack is buzzing with constant engagement, and we truly appreciate your code contributions, feedback, and feature requests. Our webinars are attracting more attendees, and the June community meeting was extra special, thanks to our first Community Spotlight: Gaétan Soulas from Solocal!\n\nWe are excited about our soaring community numbers!\n\nCrossed 2400+ GitHub stars (+200 stars since the previous release)\n\nThe Slack community reached 3200+ members (+500 since the previous release)\n\n168 Open-source GitHub developers (+8 since the previous release)\n\nMerged 526 Commits into the 1.1 Release\n\nOpenMetadata 1.1 Release Highlights\n\nUI Overhaul\n-----------\n\nThis release marks a significant milestone for the OpenMetadata platform, bringing many UI changes that are among the most substantial since the start of the project in 2021.\n\nOur primary focus is to simplify the overall experience for users while building upon our already exceptional UI. We are incredibly excited to share these changes with you as they further enhance the platform’s discovery, collaboration, and data quality experience.\n\nRefined Landing Page\n--------------------', + version: 1.2, + updatedAt: 1695189199255, + updatedBy: 'sachinchaurasiya87', + href: 'http://localhost:8585/api/v1/knowledgeCenter/8e6427d6-98cc-4334-b2f2-15fb62bde887', + changeDescription: { + fieldsAdded: [], + fieldsUpdated: [ + { + name: 'description', + oldValue: + 'Less than two months have passed since our exciting OpenMetadata 1.0 Release, and we’re thrilled to announce the completion of Release 1.1 already! The OpenMetadata community thrives on pushing our limits; this latest release is a testament to it. Prepare to be amazed as we unveil a complete UI overhaul, meticulously designed to elevate the user experience across the entire platform. But that’s not all! We’ve also introduced four new connectors, implemented advanced PII masking, and significantly enhanced lineage parsing capabilities, just to name a few of the numerous features we’ve packed into this release. Stay tuned for an exceptional OpenMetadata experience like never before!\n\nIn the upcoming 1.2 Release of OpenMetadata, we are thrilled to introduce exclusive new features specifically tailored for Collate SaaS. You can review Collate’s roadmap here and be as excited as we are 🚀\n\nCommunity Updates\n-----------------\n\nThanks to the incredible OpenMetadata Community, our growth and activity have skyrocketed. Slack is buzzing with constant engagement, and we truly appreciate your code contributions, feedback, and feature requests. Our webinars are attracting more attendees, and the June community meeting was extra special, thanks to our first Community Spotlight: Gaétan Soulas from Solocal!\n\nWe are excited about our soaring community numbers!\n\nCrossed 2400+ GitHub stars (+200 stars since the previous release)\n\nThe Slack community reached 3200+ members (+500 since the previous release)\n\n168 Open-source GitHub developers (+8 since the previous release)\n\nMerged 526 Commits into the 1.1 Release\n\nOpenMetadata 1.1 Release Highlights\n\nUI Overhaul\n-----------\n\nThis release marks a significant milestone for the OpenMetadata platform, bringing many UI changes that are among the most substantial since the start of the project in 2021.\n\nOur primary focus is to simplify the overall experience for users while building upon our already exceptional UI. We are incredibly excited to share these changes with you as they further enhance the platform’s discovery, collaboration, and data quality experience.\n\nRefined Landing Page\n--------------------', + newValue: + 'Less than two months have passed since our exciting OpenMetadata 1.0 Release, and we’re thrilled to announce the completion of Release 1.1 already! The OpenMetadata community thrives on pushing our limits; this latest release is a testament to it. Prepare to be amazed as we unveil a complete UI overhaul, meticulously designed to elevate the user experience across the entire platform. But that’s not all! We’ve also introduced four new connectors, implemented advanced PII masking, and significantly enhanced lineage parsing capabilities, just to name a few of the numerous features we’ve packed into this release. Stay tuned for an exceptional OpenMetadata experience like never before!\n\n_In the upcoming 1.2 Release of OpenMetadata, we are thrilled to introduce exclusive new features specifically tailored for Collate SaaS. You can review Collate’s roadmap here and be as excited as we are 🚀_\n\nCommunity Updates\n-----------------\n\nThanks to the incredible OpenMetadata Community, our growth and activity have skyrocketed. Slack is buzzing with constant engagement, and we truly appreciate your code contributions, feedback, and feature requests. Our webinars are attracting more attendees, and the June community meeting was extra special, thanks to our first Community Spotlight: Gaétan Soulas from Solocal!\n\nWe are excited about our soaring community numbers!\n\nCrossed 2400+ GitHub stars (+200 stars since the previous release)\n\nThe Slack community reached 3200+ members (+500 since the previous release)\n\n168 Open-source GitHub developers (+8 since the previous release)\n\nMerged 526 Commits into the 1.1 Release\n\nOpenMetadata 1.1 Release Highlights\n\nUI Overhaul\n-----------\n\nThis release marks a significant milestone for the OpenMetadata platform, bringing many UI changes that are among the most substantial since the start of the project in 2021.\n\nOur primary focus is to simplify the overall experience for users while building upon our already exceptional UI. We are incredibly excited to share these changes with you as they further enhance the platform’s discovery, collaboration, and data quality experience.\n\nRefined Landing Page\n--------------------', + }, + ], + fieldsDeleted: [], + previousVersion: 1.1, + }, + owners: [ + { + id: '9304f330-2e9a-4513-883b-c939e29683a8', + type: 'user', + name: 'admin', + fullyQualifiedName: 'admin', + deleted: false, + href: 'http://localhost:8585/api/v1/users/9304f330-2e9a-4513-883b-c939e29683a8', + }, + ], + followers: [ + { + id: '9304f330-2e9a-4513-883b-c939e29683a8', + type: 'user', + name: 'admin', + fullyQualifiedName: 'admin', + deleted: false, + href: 'http://localhost:8585/api/v1/users/9304f330-2e9a-4513-883b-c939e29683a8', + }, + ], + votes: { + upVotes: 1, + downVotes: 0, + upVoters: [ + { + id: '9304f330-2e9a-4513-883b-c939e29683a8', + type: 'user', + name: 'admin', + fullyQualifiedName: 'admin', + deleted: false, + }, + ], + downVoters: [], + }, + pageType: 'Article', + page: { + publicationDate: 1726823190797, + relatedArticles: [], + } as unknown as Article, + deleted: false, + tags: KNOWLEDGE_PAGE_TAGS, +} as KnowledgePage; + +export const KNOWLEDGE_PAGE_PARTIAL_MOCK_DATA = { + id: '8e6427d6-98cc-4334-b2f2-15fb62bde887', + name: 'Article_oRKYYTCu', + fullyQualifiedName: 'Article_oRKYYTCu', + displayName: 'OpenMetadata 1.1.0 Release UI', + description: '', + version: 1.2, + updatedAt: 1695189199255, + updatedBy: 'sachinchaurasiya87', + href: 'http://localhost:8585/api/v1/knowledgeCenter/8e6427d6-98cc-4334-b2f2-15fb62bde887', + changeDescription: { + fieldsAdded: [], + fieldsUpdated: [ + { + name: 'description', + oldValue: + 'Less than two months have passed since our exciting OpenMetadata 1.0 Release, and we’re thrilled to announce the completion of Release 1.1 already! The OpenMetadata community thrives on pushing our limits; this latest release is a testament to it. Prepare to be amazed as we unveil a complete UI overhaul, meticulously designed to elevate the user experience across the entire platform. But that’s not all! We’ve also introduced four new connectors, implemented advanced PII masking, and significantly enhanced lineage parsing capabilities, just to name a few of the numerous features we’ve packed into this release. Stay tuned for an exceptional OpenMetadata experience like never before!\n\nIn the upcoming 1.2 Release of OpenMetadata, we are thrilled to introduce exclusive new features specifically tailored for Collate SaaS. You can review Collate’s roadmap here and be as excited as we are 🚀\n\nCommunity Updates\n-----------------\n\nThanks to the incredible OpenMetadata Community, our growth and activity have skyrocketed. Slack is buzzing with constant engagement, and we truly appreciate your code contributions, feedback, and feature requests. Our webinars are attracting more attendees, and the June community meeting was extra special, thanks to our first Community Spotlight: Gaétan Soulas from Solocal!\n\nWe are excited about our soaring community numbers!\n\nCrossed 2400+ GitHub stars (+200 stars since the previous release)\n\nThe Slack community reached 3200+ members (+500 since the previous release)\n\n168 Open-source GitHub developers (+8 since the previous release)\n\nMerged 526 Commits into the 1.1 Release\n\nOpenMetadata 1.1 Release Highlights\n\nUI Overhaul\n-----------\n\nThis release marks a significant milestone for the OpenMetadata platform, bringing many UI changes that are among the most substantial since the start of the project in 2021.\n\nOur primary focus is to simplify the overall experience for users while building upon our already exceptional UI. We are incredibly excited to share these changes with you as they further enhance the platform’s discovery, collaboration, and data quality experience.\n\nRefined Landing Page\n--------------------', + newValue: + 'Less than two months have passed since our exciting OpenMetadata 1.0 Release, and we’re thrilled to announce the completion of Release 1.1 already! The OpenMetadata community thrives on pushing our limits; this latest release is a testament to it. Prepare to be amazed as we unveil a complete UI overhaul, meticulously designed to elevate the user experience across the entire platform. But that’s not all! We’ve also introduced four new connectors, implemented advanced PII masking, and significantly enhanced lineage parsing capabilities, just to name a few of the numerous features we’ve packed into this release. Stay tuned for an exceptional OpenMetadata experience like never before!\n\n_In the upcoming 1.2 Release of OpenMetadata, we are thrilled to introduce exclusive new features specifically tailored for Collate SaaS. You can review Collate’s roadmap here and be as excited as we are 🚀_\n\nCommunity Updates\n-----------------\n\nThanks to the incredible OpenMetadata Community, our growth and activity have skyrocketed. Slack is buzzing with constant engagement, and we truly appreciate your code contributions, feedback, and feature requests. Our webinars are attracting more attendees, and the June community meeting was extra special, thanks to our first Community Spotlight: Gaétan Soulas from Solocal!\n\nWe are excited about our soaring community numbers!\n\nCrossed 2400+ GitHub stars (+200 stars since the previous release)\n\nThe Slack community reached 3200+ members (+500 since the previous release)\n\n168 Open-source GitHub developers (+8 since the previous release)\n\nMerged 526 Commits into the 1.1 Release\n\nOpenMetadata 1.1 Release Highlights\n\nUI Overhaul\n-----------\n\nThis release marks a significant milestone for the OpenMetadata platform, bringing many UI changes that are among the most substantial since the start of the project in 2021.\n\nOur primary focus is to simplify the overall experience for users while building upon our already exceptional UI. We are incredibly excited to share these changes with you as they further enhance the platform’s discovery, collaboration, and data quality experience.\n\nRefined Landing Page\n--------------------', + }, + ], + fieldsDeleted: [], + previousVersion: 1.1, + }, + pageType: 'Article', + page: { + publicationDate: 1726823190797, + relatedArticles: [], + } as unknown as Article, + deleted: false, +} as KnowledgePage; + +export const QUICK_LINK_MOCK_DATA = { + id: 'fea97e8c-b2ac-4103-b827-29530d1292ad', + name: 'QuickLink_AOJs37ZW', + fullyQualifiedName: 'QuickLink_AOJs37ZW', + displayName: 'OpenMetadata Docs updated', + description: 'Quick Link for OpenMetadata Website updated.', + href: 'http://sandbox-beta.open-metadata.org/api/v1/knowledgeCenter/fea97e8c-b2ac-4103-b827-29530d1292ad', + changeDescription: { + fieldsAdded: [ + { + name: 'tags', + newValue: + '[{"tagFQN":"testing.testing_term_1","name":"testing_term_1","displayName":"testing_term_1","description":"testing_term_1","style":{},"source":"Glossary","labelType":"Manual","state":"Confirmed"},{"tagFQN":"testing.testing_term_4","name":"testing_term_4","displayName":"testing_term_4","description":"testing_term_4","style":{},"source":"Glossary","labelType":"Manual","state":"Confirmed"}]', + }, + ], + fieldsUpdated: [], + fieldsDeleted: [ + { + name: 'tags', + oldValue: + '[{"tagFQN":"testing.testing_term_2","name":"testing_term_2","displayName":"testing_term_2","description":"testing_term_2","style":{},"source":"Glossary","labelType":"Manual","state":"Confirmed"}]', + }, + ], + previousVersion: 0.6, + }, + owners: [ + { + id: 'fcc81c9c-1ca2-4ab6-a44f-722c436c7aa8', + type: 'user', + name: 'rupesh', + fullyQualifiedName: 'rupesh', + description: + 'Amundsen is one of the OSS Data Catalogs that was developed by Lyft and was open-sourced in October 2019. It quickly became popular for solving data discovery and data governance challenges. However, in recent years, Amundsen’s development and growth have slowed down considerably. Without an active community and no clear roadmap to address the emerging needs, the users of Amundsen are looking for alternatives in the OSS space.\n\nOpenMetadata is redefining the modern metadata platform with a bold vision. We have built a centralized metadata repository based on metadata specifications and APIs from the ground up. It is the foundation for innovation with several applications, such as Discovery, Collaboration, Governance, Data Quality, and Data Insights going beyond passive Data Catalogs. Learn more about OpenMetadata’s journey so far here.', + displayName: 'Rupesh Chavan', + deleted: false, + href: 'http://sandbox-beta.open-metadata.org/api/v1/users/fcc81c9c-1ca2-4ab6-a44f-722c436c7aa8', + }, + ], + followers: [], + votes: { + upVotes: 0, + downVotes: 0, + upVoters: [], + downVoters: [], + }, + tags: [ + { + tagFQN: 'KnowledgeCenter.HowToGuide', + name: 'HowToGuide', + description: 'How To Guide Quick Link or Article Tag.', + style: { + color: '#25d80e', + }, + source: 'Classification', + labelType: 'Manual', + state: 'Confirmed', + }, + { + tagFQN: 'PersonalData.SpecialCategory', + name: 'SpecialCategory', + description: + 'GDPR special category data is personal information of data subjects that is especially sensitive, the exposure of which could significantly impact the rights and freedoms of data subjects and potentially be used against them for unlawful discrimination.', + source: 'Classification', + labelType: 'Derived', + state: 'Confirmed', + }, + { + tagFQN: 'PII.None', + name: 'None', + description: 'Non PII', + style: {}, + source: 'Classification', + labelType: 'Derived', + state: 'Confirmed', + }, + { + tagFQN: 'testing.testing_term_1', + name: 'testing_term_1', + displayName: 'testing_term_1', + description: 'testing_term_1', + style: {}, + source: 'Glossary', + labelType: 'Manual', + state: 'Confirmed', + }, + { + tagFQN: 'testing.testing_term_4', + name: 'testing_term_4', + displayName: 'testing_term_4', + description: 'testing_term_4', + style: {}, + source: 'Glossary', + labelType: 'Manual', + state: 'Confirmed', + }, + ], + pageType: 'QuickLink', + page: { + url: 'https://open-metadata.org', + }, + deleted: false, +} as unknown as KnowledgePage; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCard/KnowledgeCard.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCard/KnowledgeCard.test.tsx new file mode 100644 index 000000000000..56bf60c34d50 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCard/KnowledgeCard.test.tsx @@ -0,0 +1,457 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { act, fireEvent, render, screen } from '@testing-library/react'; +import { Settings } from 'luxon'; +import '../../../test/unit/mocks/mui.mock'; + +import { usePermissionProvider } from 'context/PermissionProvider/PermissionProvider'; +import { User } from 'generated/entity/teams/user'; +import { MemoryRouter } from 'react-router-dom'; +import KnowledgeCard, { KnowledgeCardProps } from './KnowledgeCard'; +import { + KNOWLEDGE_PAGE_MOCK_DATA, + KNOWLEDGE_PAGE_PARTIAL_MOCK_DATA, + KNOWLEDGE_PAGE_TAGS, + QUICK_LINK_MOCK_DATA, +} from './KnowledgeCard.mock'; + +const systemLocale = Settings.defaultLocale; +const systemZoneName = Settings.defaultZone; + +const mockOnUpdateVote = jest.fn(); +const mockOnFollow = jest.fn(); +const mockOnUnFollow = jest.fn(); + +const mockProps: KnowledgeCardProps = { + knowledgeItem: KNOWLEDGE_PAGE_MOCK_DATA, + onUpdateVote: mockOnUpdateVote, + onFollow: mockOnFollow, + onUnFollow: mockOnUnFollow, + onDelete: jest.fn(), + onRefreshTagsCategory: jest.fn(), + readonly: false, +}; + +const mockUserData: User = { + name: 'aaron_johnson0', + email: 'testUser1@email.com', + id: '9304f330-2e9a-4513-883b-c939e29683a8', +}; + +jest.mock('hooks/useApplicationStore', () => ({ + useApplicationStore: jest.fn().mockImplementation(() => ({ + currentUser: mockUserData, + userProfilePics: {}, + })), +})); + +jest.mock('components/common/RichTextEditor/RichTextEditorPreviewerV1', () => + jest.fn().mockReturnValue(
Viewer
) +); +jest.mock('components/common/PopOverCard/UserPopOverCard', () => + jest + .fn() + .mockImplementation(({ userName }) => ( +
{userName}
+ )) +); + +jest.mock('../QuickLinkFormModal/QuickLinkFormModal', () => ({ + QuickLinkFormModal: jest + .fn() + .mockReturnValue( +
QuickLinkFormModal
+ ), +})); + +jest.mock('components/common/DeleteWidget/DeleteWidgetModal', () => + jest + .fn() + .mockReturnValue( +
DeleteWidgetModal
+ ) +); + +jest.mock('context/PermissionProvider/PermissionProvider', () => ({ + usePermissionProvider: jest.fn().mockReturnValue({ + getEntityPermissionByFqn: jest.fn().mockImplementation(() => ({ + Create: true, + Delete: true, + ViewAll: true, + EditAll: true, + EditDescription: true, + EditDisplayName: true, + EditTags: true, + })), + }), +})); + +describe('Knowledge Card', () => { + beforeAll(() => { + // Explicitly set locale and time zone to make sure date time manipulations and literal + // results are consistent regardless of where tests are run + Settings.defaultLocale = 'en-US'; + Settings.defaultZone = 'UTC'; + }); + + afterAll(() => { + // Restore locale and time zone + Settings.defaultLocale = systemLocale; + Settings.defaultZone = systemZoneName; + }); + + it('Should render the knowledge card', async () => { + render(, { + wrapper: MemoryRouter, + }); + const dateOwnerElement = screen.getByTestId('date-owner-col'); + const titleDescriptionElement = screen.getByTestId( + 'knowledge-title-description' + ); + + const metadataElement = screen.getByTestId('knowledge-metadata'); + + expect(dateOwnerElement).toBeInTheDocument(); + expect(titleDescriptionElement).toBeInTheDocument(); + expect(metadataElement).toBeInTheDocument(); + + const ownerName = screen.getByTestId('owner-link'); + + expect(ownerName).toHaveTextContent('admin'); + + const lastEditedByName = screen.getByTestId('owner-name'); + + const updatedAt = screen.getByTestId('updated-at'); + + expect(updatedAt).toHaveTextContent('Sep 20, 2023'); + + const title = screen.getByTestId('entity-header-display-name'); + + expect(title).toHaveTextContent('OpenMetadata 1.1.0 Release UI'); + + const description = screen.getByTestId('viewer-container'); + + expect(description).toBeInTheDocument(); + + const upVoteButton = screen.getByTestId('up-vote-btn'); + const upVoteCount = screen.getByTestId('up-vote-count'); + + expect(upVoteButton).toBeInTheDocument(); + expect(upVoteCount).toHaveTextContent('1'); + + const downVoteButton = screen.getByTestId('down-vote-btn'); + const downVoteCount = screen.getByTestId('down-vote-count'); + + expect(downVoteButton).toBeInTheDocument(); + expect(downVoteCount).toHaveTextContent('0'); + + const updatedAtMetadata = screen.getByTestId('updated-at-metadata'); + + expect(lastEditedByName).toHaveTextContent('sachinchaurasiya87'); + expect(updatedAtMetadata).toHaveTextContent('Sep 20, 2023'); + + const bookmarkBtn = screen.getByTestId('bookmark-btn'); + + expect(bookmarkBtn).toBeInTheDocument(); + expect(bookmarkBtn).toHaveAttribute('data-isfollowing', 'true'); + + KNOWLEDGE_PAGE_TAGS.forEach((tag) => { + const tagElement = screen.getByText(tag.name); + + expect(tagElement).toBeInTheDocument(); + }); + }); + + it('Should render the fallback data', async () => { + render( + , + { + wrapper: MemoryRouter, + } + ); + const dateOwnerElement = screen.getByTestId('date-owner-col'); + const titleDescriptionElement = screen.getByTestId( + 'knowledge-title-description' + ); + + const metadataElement = screen.getByTestId('knowledge-metadata'); + + expect(dateOwnerElement).toBeInTheDocument(); + expect(titleDescriptionElement).toBeInTheDocument(); + expect(metadataElement).toBeInTheDocument(); + + const ownerName = screen.getByTestId('owner-link'); + + expect(ownerName).toHaveTextContent('label.no-entity'); + + const updatedAt = screen.getByTestId('updated-at'); + + expect(updatedAt).toHaveTextContent('Sep 20, 2023'); + + const title = screen.getByTestId('entity-header-display-name'); + + expect(title).toHaveTextContent('OpenMetadata 1.1.0 Release UI'); + + const noDescription = screen.getByTestId('no-description'); + + expect(noDescription).toHaveTextContent('label.no-description'); + + const upVoteCount = screen.getByTestId('up-vote-count'); + + expect(upVoteCount).toHaveTextContent('0'); + + const downVoteCount = screen.getByTestId('down-vote-count'); + + expect(downVoteCount).toHaveTextContent('0'); + + const bookmarkBtn = screen.getByTestId('bookmark-btn'); + + expect(bookmarkBtn).toBeInTheDocument(); + expect(bookmarkBtn).toHaveAttribute('data-isfollowing', 'false'); + }); + + it('OnUpdateVote Should work', async () => { + render( + , + { + wrapper: MemoryRouter, + } + ); + + // upVote simulation + const upVoteButton = screen.getByTestId('up-vote-btn'); + + fireEvent.click(upVoteButton); + + expect(mockOnUpdateVote).toHaveBeenCalledWith( + { + updatedVoteType: 'votedUp', + }, + '8e6427d6-98cc-4334-b2f2-15fb62bde887' + ); + + // downVote simulation + const downVoteButton = screen.getByTestId('down-vote-btn'); + + fireEvent.click(downVoteButton); + + expect(mockOnUpdateVote).toHaveBeenCalledWith( + { + updatedVoteType: 'votedDown', + }, + '8e6427d6-98cc-4334-b2f2-15fb62bde887' + ); + }); + + it('onFollow Should work', async () => { + render( + , + { + wrapper: MemoryRouter, + } + ); + + const bookmarkBtn = screen.getByTestId('bookmark-btn'); + + fireEvent.click(bookmarkBtn); + + expect(mockOnFollow).toHaveBeenCalledWith( + '8e6427d6-98cc-4334-b2f2-15fb62bde887' + ); + }); + + it('onUnFollow Should work', async () => { + render(, { + wrapper: MemoryRouter, + }); + + const bookmarkBtn = screen.getByTestId('bookmark-btn'); + + fireEvent.click(bookmarkBtn); + + expect(mockOnUnFollow).toHaveBeenCalledWith( + '8e6427d6-98cc-4334-b2f2-15fb62bde887' + ); + }); + + it('should render the edit and delete button for quick link', async () => { + await act(async () => { + render( + , + { + wrapper: MemoryRouter, + } + ); + }); + + const editButton = screen.getByTestId('edit-quick-link-btn'); + const deleteButton = screen.getByTestId('delete-quick-link-btn'); + + expect(editButton).toBeInTheDocument(); + expect(deleteButton).toBeInTheDocument(); + }); + + it('edit should render the quick link modal', async () => { + await act(async () => { + render( + , + { + wrapper: MemoryRouter, + } + ); + }); + + const editButton = screen.getByTestId('edit-quick-link-btn'); + + fireEvent.click(editButton); + + const quickLinkFormModal = screen.getByTestId('quick-link-form-modal'); + + expect(quickLinkFormModal).toBeInTheDocument(); + }); + + it('delete should render the delete widget modal', async () => { + await act(async () => { + render( + , + { + wrapper: MemoryRouter, + } + ); + }); + + const deleteButton = screen.getByTestId('delete-quick-link-btn'); + + fireEvent.click(deleteButton); + + const deleteWidgetModal = screen.getByTestId('delete-widget-modal'); + + expect(deleteWidgetModal).toBeInTheDocument(); + }); + + it('quick link title should have target as _blank', async () => { + render( + , + { + wrapper: MemoryRouter, + } + ); + + const quickLinkTitle = screen.getByTestId('knowledge-link'); + + expect(quickLinkTitle).toHaveAttribute('target', '_blank'); + }); + + it("should not render the edit and delete button for quick link if user doesn't have permission", async () => { + (usePermissionProvider as jest.Mock).mockImplementationOnce(() => ({ + getEntityPermissionByFqn: jest.fn().mockReturnValue({ + Create: false, + Delete: false, + ViewAll: false, + EditAll: false, + EditDescription: false, + EditDisplayName: false, + EditTags: false, + }), + })); + render( + , + { + wrapper: MemoryRouter, + } + ); + + const editButton = screen.queryByTestId('edit-quick-link-btn'); + const deleteButton = screen.queryByTestId('delete-quick-link-btn'); + + expect(editButton).not.toBeInTheDocument(); + expect(deleteButton).not.toBeInTheDocument(); + }); + + it('should render the edit button for quick link if user have some edit permission', async () => { + (usePermissionProvider as jest.Mock).mockImplementationOnce(() => ({ + getEntityPermissionByFqn: jest.fn().mockReturnValue({ + Create: false, + Delete: false, + ViewAll: false, + EditAll: false, + EditDescription: true, + EditDisplayName: false, + EditTags: true, + }), + })); + await act(async () => { + render( + , + { + wrapper: MemoryRouter, + } + ); + }); + + const editButton = screen.getByTestId('edit-quick-link-btn'); + + expect(editButton).toBeInTheDocument(); + }); + + it('should not render the votes-section and bookmark-section if readonly is true', async () => { + render(, { + wrapper: MemoryRouter, + }); + + const voteSection = screen.queryByTestId('votes-section'); + const bookmarkSection = screen.queryByTestId('bookmark-section'); + + expect(voteSection).not.toBeInTheDocument(); + expect(bookmarkSection).not.toBeInTheDocument(); + }); + + it('should not render the edit and delete button for quick link if readonly is true', async () => { + (usePermissionProvider as jest.Mock).mockImplementationOnce(() => ({ + getEntityPermissionByFqn: jest.fn().mockReturnValue({ + Create: true, + Delete: true, + ViewAll: true, + EditAll: true, + EditDescription: true, + EditDisplayName: true, + EditTags: true, + }), + })); + render( + , + { + wrapper: MemoryRouter, + } + ); + + const editButton = screen.queryByTestId('edit-quick-link-btn'); + const deleteButton = screen.queryByTestId('delete-quick-link-btn'); + + expect(editButton).not.toBeInTheDocument(); + expect(deleteButton).not.toBeInTheDocument(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCard/KnowledgeCard.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCard/KnowledgeCard.tsx new file mode 100644 index 000000000000..4bc90a25697d --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCard/KnowledgeCard.tsx @@ -0,0 +1,492 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Icon from '@ant-design/icons'; +import { Col, Divider, Row, Space, Typography } from 'antd'; +import { AxiosError } from 'axios'; +import { isUndefined } from 'lodash'; +import { ReactComponent as EditIcon } from '../../../assets/svg/edit-new.svg'; +import { ReactComponent as IconDelete } from '../../../assets/svg/ic-delete.svg'; +import { ReactComponent as ThumbsUpFilled } from '../../../assets/svg/thumbs-up-filled.svg'; +import { ReactComponent as ThumbsUpOutline } from '../../../assets/svg/thumbs-up-outline.svg'; +import DeleteWidgetModal from '../../../components/common/DeleteWidget/DeleteWidgetModal'; +import UserPopOverCard from '../../../components/common/PopOverCard/UserPopOverCard'; +import RichTextEditorPreviewerV1 from '../../../components/common/RichTextEditor/RichTextEditorPreviewerV1'; +import { EntityType } from '../../../enums/entity.enum'; + +import TagsViewer from '../../../components/Tag/TagsViewer/TagsViewer'; +import { DisplayType } from '../../../components/Tag/TagsViewer/TagsViewer.interface'; + +import { FC, useCallback, useEffect, useMemo, useState } from 'react'; +import { Link } from 'react-router-dom'; +import { ReactComponent as IconArticle } from '../../../assets/svg/ic-articles.svg'; +import { ReactComponent as BookMarkIcon } from '../../../assets/svg/ic-bookmark.svg'; +import { ReactComponent as BookMarkedIcon } from '../../../assets/svg/ic-bookmarked.svg'; +import { ReactComponent as LinkIcon } from '../../../assets/svg/ic-link.svg'; +import { ReactComponent as UpdatedAtIcon } from '../../../assets/svg/ic-updated.svg'; +import Loader from '../../../components/common/Loader/Loader'; +import { OwnerLabel } from '../../../components/common/OwnerLabel/OwnerLabel.component'; +import { QueryVoteType } from '../../../components/Database/TableQueries/TableQueries.interface'; +import { VotingDataProps } from '../../../components/Entity/Voting/voting.interface'; +import { DE_ACTIVE_COLOR } from '../../../constants/constants'; +import { usePermissionProvider } from '../../../context/PermissionProvider/PermissionProvider'; +import { + OperationPermission, + ResourceEntity, +} from '../../../context/PermissionProvider/PermissionProvider.interface'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; +import { + KnowledgePage, + PageType, + QuickLink, + RecentlyViewedQuickLinks, + RecentViewedKnowledgePage, +} from '../../../interface/knowledge-center.interface'; +import { formatDate } from '../../../utils/date-time/DateTimeUtils'; +import { getFrontEndFormat } from '../../../utils/FeedUtils'; +import { t } from '../../../utils/i18next/LocalUtil'; +import { + addToKnowledgeCenterRecentViewed, + getKnowledgePagePath, + updateKnowledgeCenterRecentViewed, +} from '../../../utils/KnowledgePageUtils'; +import { DEFAULT_ENTITY_PERMISSION } from '../../../utils/PermissionsUtils'; +import { showErrorToast } from '../../../utils/ToastUtils'; +import { + QuickLinkFormModal, + QuickLinkFormModalFormData, +} from '../QuickLinkFormModal/QuickLinkFormModal'; + +import { useCurrentUserPreferences } from '../../../hooks/currentUserStore/useCurrentUserStore'; +import './knowledge-card.less'; + +export interface KnowledgeCardProps { + knowledgeItem: KnowledgePage; + onUpdateVote?: (data: VotingDataProps, id: string) => Promise; + onFollow?: (id: string) => Promise; + onUnFollow?: (id: string) => Promise; + onDelete?: (id: string) => void; + onRefreshTagsCategory?: (value: boolean) => void; + readonly?: boolean; +} + +const KnowledgeCard: FC = ({ + knowledgeItem, + onUpdateVote, + onFollow, + onUnFollow, + onDelete, + onRefreshTagsCategory, + readonly = false, +}) => { + const { getEntityPermissionByFqn } = usePermissionProvider(); + const { currentUser } = useApplicationStore(); + const USERId = currentUser?.id ?? ''; + + const [knowledgePage, setKnowledgePage] = useState(knowledgeItem); + const [permissions, setPermissions] = useState( + DEFAULT_ENTITY_PERMISSION + ); + + const { + name, + displayName = '', + owners = [], + updatedAt, + description = '', + votes, + updatedBy, + followers = [], + } = knowledgePage; + + const [showAddLinkModal, setShowAddLinkModal] = useState(false); + const [isDelete, setIsDelete] = useState(false); + const [votesLoading, setVotesLoading] = useState<{ + votedUp: boolean; + votedDown: boolean; + }>({ + votedUp: false, + votedDown: false, + }); + + const [isBookmaking, setIsBookmaking] = useState(false); + const { + preferences: { recentlyViewedQuickLinks }, + } = useCurrentUserPreferences(); + const recentlyViewed = + recentlyViewedQuickLinks as unknown as RecentlyViewedQuickLinks['data']; + + const fetchPermission = async (fqn: string) => { + try { + const response = await getEntityPermissionByFqn( + ResourceEntity.KNOWLEDGE_PAGE as unknown as ResourceEntity, + fqn + ); + setPermissions(response); + } catch (error) { + showErrorToast(error as AxiosError); + } + }; + + const voteStatus = useMemo(() => { + if (isUndefined(votes)) { + return QueryVoteType.unVoted; + } + + const upVoters = votes.upVoters || []; + const downVoters = votes.downVoters || []; + + if (upVoters.some((user) => user.id === USERId)) { + return QueryVoteType.votedUp; + } else if (downVoters.some((user) => user.id === USERId)) { + return QueryVoteType.votedDown; + } else { + return QueryVoteType.unVoted; + } + }, [votes, USERId]); + + const isVoteUp = voteStatus === QueryVoteType.votedUp; + const isVoteDown = voteStatus === QueryVoteType.votedDown; + const isQuickLink = knowledgePage.pageType === PageType.QUICK_LINK; + const isFollowing = Boolean(followers?.some(({ id }) => id === USERId)); + const path = isQuickLink + ? (knowledgePage.page as QuickLink).url + : getKnowledgePagePath(knowledgePage.fullyQualifiedName); + + const handleVoteChange = async (type: QueryVoteType) => { + let updatedVoteType; + + // current vote is same as selected vote, it means user is removing vote, else up/down voting + if (voteStatus === type) { + updatedVoteType = QueryVoteType.unVoted; + } else { + updatedVoteType = type; + } + + setVotesLoading((prev) => ({ + ...prev, + [type]: true, + })); + + await onUpdateVote?.({ updatedVoteType }, knowledgePage.id); + + setVotesLoading((prev) => ({ + ...prev, + [type]: false, + })); + }; + + const handleBookmarkChange = useCallback( + async (id: string) => { + setIsBookmaking(true); + if (isFollowing) { + await onUnFollow?.(id); + } else { + await onFollow?.(id); + } + setIsBookmaking(false); + }, + [isFollowing, onUnFollow, onFollow] + ); + + const handleQuickLinkUpdate = async ( + formData: QuickLinkFormModalFormData + ) => { + setKnowledgePage((prevKnowledgePage) => ({ + ...prevKnowledgePage, + displayName: formData.displayName, + description: formData.description, + tags: formData.tags, + page: { + url: formData.url, + }, + relatedEntities: formData?.relatedEntities, + })); + onRefreshTagsCategory?.(true); + }; + + const handleToggleDelete = () => { + setKnowledgePage((prev) => { + if (!prev) { + return prev; + } + + return { ...prev, deleted: !prev?.deleted }; + }); + }; + const afterDeleteAction = useCallback( + (isSoftDelete?: boolean) => { + updateKnowledgeCenterRecentViewed( + recentlyViewed.filter((page) => page.id !== knowledgePage?.id) + ); + isSoftDelete ? handleToggleDelete() : onDelete?.(knowledgePage?.id); + onRefreshTagsCategory?.(true); + }, + [knowledgePage, onDelete, handleToggleDelete, onRefreshTagsCategory] + ); + + const quickLinkActions = useMemo(() => { + const editPermission = + permissions?.EditAll || + permissions?.EditDisplayName || + permissions?.EditDescription || + permissions?.EditTags; + + return ( + <> + {editPermission && ( + { + e.stopPropagation(); + e.preventDefault(); + setShowAddLinkModal(true); + }} + /> + )} + {permissions?.Delete && ( + { + e.stopPropagation(); + e.preventDefault(); + setIsDelete(true); + }} + /> + )} + + ); + }, [permissions]); + + const handleQuickLinkRecentView = useCallback(() => { + if (isQuickLink) { + addToKnowledgeCenterRecentViewed( + knowledgePage as RecentViewedKnowledgePage + ); + } + }, [isQuickLink, knowledgePage]); + + useEffect(() => { + setKnowledgePage(knowledgeItem); + if (knowledgeItem.pageType === PageType.QUICK_LINK) { + fetchPermission(knowledgeItem.fullyQualifiedName); + } + }, [knowledgeItem]); + + return ( + + + +
+ +
+

+ + + {formatDate(updatedAt)} + +

+ + + + + + +
+ {isQuickLink ? ( + + ) : ( + + )} + + {knowledgePage?.displayName || t('label.untitled')} + + {isQuickLink && !readonly && quickLinkActions} +
+ + {description.trim() ? ( + + ) : ( + + {t('label.no-description')} + + )} + + + + {(knowledgePage.tags ?? []).length > 0 && ( + + + + )} + + + {!readonly && ( + + +
handleVoteChange(QueryVoteType.votedUp)}> + + {votesLoading.votedUp ? ( + + ) : ( + + )} + + + {votes?.upVotes ?? 0} + + +
+
handleVoteChange(QueryVoteType.votedDown)}> + + {votesLoading.votedDown ? ( + + ) : ( + + )} + + {votes?.downVotes ?? 0} + + +
+ + + + )} + + + {`${t('label.last-edited-by')}:`} + + + + + + + {`${t('label.last-updated')}:`} + + {formatDate(updatedAt)} + + + + {!readonly && ( + + + {isBookmaking ? ( + + ) : ( + handleBookmarkChange(knowledgePage.id)} + /> + )} + + )} + + + {showAddLinkModal && ( + setShowAddLinkModal(false)} + onSave={(data) => { + handleQuickLinkUpdate(data); + setShowAddLinkModal(false); + }} + /> + )} + {isDelete && ( + setIsDelete(false)} + /> + )} + + ); +}; + +export default KnowledgeCard; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCard/knowledge-card.less b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCard/knowledge-card.less new file mode 100644 index 000000000000..11ff6c1d8e8a --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCard/knowledge-card.less @@ -0,0 +1,50 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@import (reference) '../../../styles/variables.less'; + +.knowledge-card { + padding: @padding-mlg; + position: relative; + border-radius: @border-rad-sm; + border-left: 4px solid transparent; + opacity: 0.95; + background-color: @grey-9 !important; + border: 0.5px solid @grey-15; + margin: 0 0 @margin-mlg; + + &.highlight-card { + border-left: 4px solid @blue-12; + background: @primary-button-background !important; + } + + &:last-child { + margin-bottom: 0; + } + + .service-icon { + height: 16px; + } + + .entity-summary-details { + font-size: 12px; + + .no-owner { + color: @text-grey-muted; + } + } + + &:hover { + border-color: @primary-color; + background-color: @primary-button-background !important; + } +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterLayout/KnowledgeCenterLayout.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterLayout/KnowledgeCenterLayout.test.tsx new file mode 100644 index 000000000000..acc8fbd8b19d --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterLayout/KnowledgeCenterLayout.test.tsx @@ -0,0 +1,46 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { render, screen } from '@testing-library/react'; +import KnowledgeCenterLayout from './KnowledgeCenterLayout'; + +jest.mock('components/common/DocumentTitle/DocumentTitle', () => + jest.fn().mockImplementation(({ title }) => { + document.title = title; + + return null; + }) +); + +jest.mock('react-router-dom', () => ({ + useLocation: jest.fn().mockReturnValue({ + pathname: '/', + }), +})); + +describe('KnowledgeCenterLayout', () => { + const mockProps = { + children:
Test Children
, + leftSidebar:
Test Left Sidebar
, + rightSidebar:
Test Right Sidebar
, + pageTitle: 'Test Page Title', + }; + + it('should render correctly', () => { + render(); + + expect(screen.getByText('Test Children')).toBeInTheDocument(); + expect(screen.getByText('Test Left Sidebar')).toBeInTheDocument(); + expect(screen.getByText('Test Right Sidebar')).toBeInTheDocument(); + expect(document.title).toEqual('Test Page Title'); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterLayout/KnowledgeCenterLayout.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterLayout/KnowledgeCenterLayout.tsx new file mode 100644 index 000000000000..39de7c069252 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterLayout/KnowledgeCenterLayout.tsx @@ -0,0 +1,139 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Card, Typography } from 'antd'; +import classNames from 'classnames'; +import React, { FC } from 'react'; +import { useTranslation } from 'react-i18next'; +import { ReflexContainer, ReflexElement, ReflexSplitter } from 'react-reflex'; +import DocumentTitle from '../../../components/common/DocumentTitle/DocumentTitle'; +import '../../../components/common/ResizablePanels/resizable-panels.less'; +import './knowledge-center-layout.less'; + +interface KnowledgeCenterLayoutProps { + children: React.ReactNode; + leftSidebar: React.ReactNode; + rightSidebar: React.ReactNode; + pageTitle: string; + className?: string; + leftSidebarTitle?: React.ReactNode; + rightSidebarTitle?: string; + leftSidebarExtra?: React.ReactNode; // Pass Antd Card extra to the left sidebar card + rightSidebarExtra?: React.ReactNode; // Pass Antd Card extra to the right sidebar card +} + +const KnowledgeCenterLayout: FC = ({ + children, + leftSidebar, + rightSidebar, + pageTitle, + className, + leftSidebarTitle, + rightSidebarTitle, + leftSidebarExtra, + rightSidebarExtra, +}) => { + const { i18n } = useTranslation(); + const isLeftPanelCollapsed = false; + const isRightPanelCollapsed = !rightSidebar; + + return ( +
+ + + {/* left */} + + + {leftSidebarTitle} + + ) + }> + {leftSidebar} + + + + + {!isLeftPanelCollapsed && ( +
+
+
+ )} + + + {/* middle */} + + {children} + + + + {!isRightPanelCollapsed && ( +
+
+
+ )} + + + + + {rightSidebarTitle} + + ) + }> + {rightSidebar} + + + +
+ ); +}; + +export default KnowledgeCenterLayout; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterLayout/SizeAwareElement/SizeAwareElement.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterLayout/SizeAwareElement/SizeAwareElement.test.tsx new file mode 100644 index 000000000000..039cd8869a17 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterLayout/SizeAwareElement/SizeAwareElement.test.tsx @@ -0,0 +1,84 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { render } from '@testing-library/react'; +import { + CENTER_PANEL_DEFAULT_WIDTH, + CENTER_PANEL_PANEL_MARGIN, +} from 'constants/KnowledgeCenter.constant'; +import { SizeAwareElement } from './SizeAwareElement'; + +describe('SizeAwareElement', () => { + it('renders correctly', () => { + const { getByText } = render( + + Test + + ); + + expect(getByText('Test')).toBeInTheDocument(); + }); + + it('applies correct styles when isLeftPanelCollapsed is true', () => { + const { getByText } = render( + + Test + + ); + + expect(getByText('Test')).toHaveStyle( + `max-width: ${CENTER_PANEL_DEFAULT_WIDTH + 20}px` + ); + }); + + it('applies correct styles when isRightPanelCollapsed is true', () => { + const { getByText } = render( + + Test + + ); + + expect(getByText('Test')).toHaveStyle( + `max-width: ${CENTER_PANEL_DEFAULT_WIDTH + 20}px` + ); + }); + + it('applies correct styles when isRightPanelCollapsed and isLeftPanelCollapsed is true', () => { + const { getByText } = render( + + Test + + ); + + expect(getByText('Test')).toHaveStyle( + `max-width: ${CENTER_PANEL_DEFAULT_WIDTH + 100}px` + ); + }); + + it('applies correct styles when dimensions are provided', () => { + const dimensions = { width: 800, height: 600 }; + const { getByText } = render( + + Test + + ); + + expect(getByText('Test')).toHaveStyle( + `max-width: ${dimensions.width - CENTER_PANEL_PANEL_MARGIN}px` + ); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterLayout/SizeAwareElement/SizeAwareElement.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterLayout/SizeAwareElement/SizeAwareElement.tsx new file mode 100644 index 000000000000..8fb9dc37f6ff --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterLayout/SizeAwareElement/SizeAwareElement.tsx @@ -0,0 +1,58 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { CSSProperties, ReactNode, useMemo } from 'react'; +import { + CENTER_PANEL_DEFAULT_WIDTH, + CENTER_PANEL_PADDING_HORIZONTAL, + CENTER_PANEL_PADDING_VERTICAL, + CENTER_PANEL_PANEL_MARGIN, +} from '../../../../constants/KnowledgeCenter.constant'; + +interface SizeAwareElementProps { + isLeftPanelCollapsed: boolean; + isRightPanelCollapsed: boolean; + children: ReactNode; + dimensions?: { width: number; height: number }; +} + +export const SizeAwareElement = ({ + children, + isLeftPanelCollapsed, + isRightPanelCollapsed, + dimensions, +}: SizeAwareElementProps) => { + const maxWidth = useMemo(() => { + let width = CENTER_PANEL_DEFAULT_WIDTH; + + if (isLeftPanelCollapsed && isRightPanelCollapsed) { + width = CENTER_PANEL_DEFAULT_WIDTH + 100; + } else if (isLeftPanelCollapsed || isRightPanelCollapsed) { + width = CENTER_PANEL_DEFAULT_WIDTH + 20; + } else { + width = + (dimensions?.width || CENTER_PANEL_DEFAULT_WIDTH) - + CENTER_PANEL_PANEL_MARGIN; + } + + return width; + }, [dimensions, isLeftPanelCollapsed, isRightPanelCollapsed]); + + const style: CSSProperties = { + maxWidth: `${maxWidth}px`, + margin: '0 auto', + padding: `${CENTER_PANEL_PADDING_VERTICAL} ${CENTER_PANEL_PADDING_HORIZONTAL}`, + height: '100%', + }; + + return
{children}
; +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterLayout/knowledge-center-layout.less b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterLayout/knowledge-center-layout.less new file mode 100644 index 000000000000..0415798b4c0f --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterLayout/knowledge-center-layout.less @@ -0,0 +1,210 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@import (reference) '../../../styles/variables.less'; +@import (reference) '../../../styles/variables.less'; + +@border-color: #0000001a; + +.knowledge-center-layout { + height: @page-height; + + &.knowledge-details-page { + height: @knowledge-center-content-height; + } +} + +.knowledge-center-layout.reflex-container { + justify-content: flex-start; + align-items: stretch; + align-content: stretch; + display: flex; + position: relative; + width: 100%; + + .left-reflex-card { + > .ant-card-body { + padding: @padding-xs 0; + border-radius: @border-rad-sm; + } + } +} + +.knowledge-center-layout.reflex-container.horizontal { + flex-direction: column; + min-height: 1px; +} + +.knowledge-center-layout.reflex-container.vertical { + flex-direction: row; + min-width: 1px; +} + +.knowledge-center-layout.reflex-container.vertical > .reflex-splitter { + cursor: col-resize; + height: 100%; + width: 1px; +} + +.knowledge-center-layout.reflex-container > .reflex-element { + // hide the scrollbar + -ms-overflow-style: none; + scrollbar-width: none; + + &::-webkit-scrollbar { + display: none; + } +} + +// left panel css + +.knowledge-center-layout.reflex-container .left-panel { + position: relative; + overflow-y: scroll; + + &.left-panel-collapsed { + flex: 0 !important; // to override the inline style + border-right: none; + } +} + +.left-panel-collapse-button { + position: absolute; + top: 8px; + right: -18px; + padding: 4px 8px; + z-index: 10; + background: white; + + .collapse-icon { + vertical-align: middle; + width: 16px; + } + + &:hover, + &:focus { + background: white; + } +} + +.left-panel-collapse-button.collapsed { + position: absolute; + top: 8px; + right: auto; + left: -30px; + padding: 4px 16px; + z-index: 10; + background: white; + + .collapse-icon { + margin-left: 4px; + width: 20px; + } + + &:hover, + &:focus { + background: white; + } +} + +// right panel css + +.knowledge-center-layout.reflex-container .right-panel { + overflow-y: scroll; + + &.right-panel-collapsed { + flex: 0 !important; // to override the inline style + border-left: none; + } +} + +.right-panel-collapse-button { + position: absolute; + top: 8px; + left: -18px; + padding: 4px 8px; + z-index: 10; + background: white; + + .collapse-icon { + vertical-align: middle; + width: 16px; + } + + &:hover, + &:focus { + background: white; + } +} + +.right-panel-collapse-button.collapsed { + position: absolute; + top: 8px; + left: auto; + right: -22px; + padding: 4px 12px; + z-index: 10; + background: white; + + .collapse-icon { + margin-left: 4px; + width: 20px; + } + + &:hover, + &:focus { + background: white; + } +} + +// center panel css + +.knowledge-center-layout.reflex-container .center-panel { + overflow-y: scroll; +} + +// rtl css +div[dir='rtl']#knowledge-center-layout-container { + .knowledge-center-layout.reflex-container { + .left-panel-collapse-button { + left: auto; + right: 258px; + } + + .left-panel-collapse-button.collapsed { + left: auto; + right: -16px; + + .collapse-icon { + margin-right: 4px; + } + } + } + + .knowledge-center-layout.reflex-container .left-panel { + border-left: 1px solid @border-color; + position: relative; + } + + .knowledge-center-layout.reflex-container .left-panel.left-panel-collapsed { + border-left: none; + } + + .knowledge-center-layout.reflex-container .right-panel { + border-right: 1px solid @border-color; + position: relative; + } + + .knowledge-center-layout.reflex-container .right-panel.right-panel-collapsed { + border-right: none; + } +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterWidget/KnowledgeCenterWidget.less b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterWidget/KnowledgeCenterWidget.less new file mode 100644 index 000000000000..a4ff36de9405 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterWidget/KnowledgeCenterWidget.less @@ -0,0 +1,97 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import 'styles/variables.less'; + +.knowledge-center-widget { + .ant-card-body { + height: 100%; + } + + .article-header { + color: #0f141b; + } + + .knowledge-icon { + height: 32px; + width: 32px; + padding: 2px; + border-radius: 50%; + background-color: #dcf6ff; + + svg { + fill: none; + stroke: @primary-6; + } + } +} + +.knowledge-center-widget-container { + .ant-card-body { + display: flex; + flex-direction: column; + flex: 1; + padding: 0; + overflow: hidden; + } + + .widget-content { + flex: 1; + display: flex; + flex-direction: column; + min-height: 0; + overflow: hidden; + } + + .cards-scroll-container { + display: grid; + gap: @size-xs; + align-content: start; + } + + .entity-list-body { + flex: 1; + padding: @size-mlg; + margin: 0; + overflow-y: auto; + min-height: 0; + + display: flex; + flex-direction: column; + } + + .widget-footer { + flex-shrink: 0; + width: 100%; + background-color: @white; + border-bottom-right-radius: @size-sm; + border-bottom-left-radius: @size-sm; + } + + .ellipsis-text { + max-width: 230px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + display: inline-block; + vertical-align: bottom; + } + + .article-entry { + padding: @padding-xss @padding-sm; + border-radius: @size-xs; + &:hover { + background-color: @grey-29; + } + } +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterWidget/KnowledgeCenterWidget.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterWidget/KnowledgeCenterWidget.test.tsx new file mode 100644 index 000000000000..fee1a0970343 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterWidget/KnowledgeCenterWidget.test.tsx @@ -0,0 +1,181 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + act, + fireEvent, + render, + screen, + waitFor, +} from '@testing-library/react'; +import { MemoryRouter } from 'react-router-dom'; +import { MOCK_KNOWLEDGE_PAGE_LIST } from '../../../pages/KnowledgeCenterListPage/KnowledgeCenterListPage.mock'; +import { getListKnowledgePages } from '../../../rest/knowledgeCenterAPI'; +import KnowledgeCenterWidget from './KnowledgeCenterWidget'; + +const mockHandleRemoveWidget = jest.fn(); + +jest.mock('../../../rest/knowledgeCenterAPI', () => ({ + getListKnowledgePages: jest.fn().mockImplementation(() => Promise.resolve()), +})); + +jest.mock('hooks/useApplicationStore', () => ({ + useApplicationStore: jest.fn().mockReturnValue({ + currentUser: { + id: '2e424734-761a-443f-bf2a-a5b361823c80', + type: 'user', + name: 'aaron_johnson0', + fullyQualifiedName: 'aaron_johnson0', + displayName: 'Aaron Johnson', + deleted: false, + }, + }), +})); + +jest.mock('components/common/ErrorWithPlaceholder/ErrorPlaceHolder', () => + jest.fn().mockImplementation(() =>
ErrorPlaceHolder
) +); + +jest.mock( + 'components/MyData/Widgets/Common/WidgetEmptyState/WidgetEmptyState', + () => + jest + .fn() + .mockImplementation(() => ( +
WidgetEmptyState
+ )) +); + +jest.mock('utils/EntityUtils', () => ({ + getEntityName: jest.fn(), +})); + +describe('Knowledge center widget', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should render the empty placeholder if no data', async () => { + render(, { + wrapper: MemoryRouter, + }); + + await waitFor(() => { + expect(screen.getByTestId('widget-empty-state')).toBeInTheDocument(); + }); + }); + + it('should render the article list', async () => { + (getListKnowledgePages as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ data: MOCK_KNOWLEDGE_PAGE_LIST, paging: { total: 15 } }) + ); + + render(, { + wrapper: MemoryRouter, + }); + + await waitFor(() => { + expect(screen.getAllByTestId('article-entry')).toHaveLength( + MOCK_KNOWLEDGE_PAGE_LIST.length + ); + }); + }); + + it('should not display edit controls if isEditView is false', async () => { + await act(async () => { + render(, { + wrapper: MemoryRouter, + }); + }); + + expect(screen.queryByTestId('more-options-button')).toBeNull(); + expect(screen.queryByTestId('drag-widget-button')).toBeNull(); + }); + + it('should call the handleRemoveWidget function with the passed widget Key', async () => { + await act(async () => { + render( + , + { + wrapper: MemoryRouter, + } + ); + }); + + // Click the more options button to open the dropdown + fireEvent.click(screen.getByTestId('more-options-button')); + + // Wait for the dropdown menu to appear and click the remove option + await waitFor(() => { + const removeOption = screen.getByText('label.remove'); + fireEvent.click(removeOption); + }); + + expect(mockHandleRemoveWidget).toHaveBeenCalledWith( + 'KnowledgeCenterWidget' + ); + }); + + it('should render link and article icons properly according to the pageType', async () => { + (getListKnowledgePages as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ data: MOCK_KNOWLEDGE_PAGE_LIST, paging: { total: 15 } }) + ); + + await act(async () => { + render(, { + wrapper: MemoryRouter, + }); + }); + + await waitFor(() => { + const linkIcons = screen.getAllByTestId('link-icon'); + const articleIcons = screen.getAllByTestId('article-icon'); + + expect(linkIcons).toHaveLength(7); + expect(articleIcons).toHaveLength(1); + }); + }); + + it("should render url as a link for quick link with target as '_blank'", async () => { + (getListKnowledgePages as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ data: MOCK_KNOWLEDGE_PAGE_LIST, paging: { total: 15 } }) + ); + + await act(async () => { + render(, { + wrapper: MemoryRouter, + }); + }); + + await waitFor(() => { + const quickLink = screen.getAllByTestId('quick-link-link'); + + expect(quickLink[0]).toHaveAttribute('target', '_blank'); + expect(quickLink[0]).toHaveAttribute( + 'href', + 'https://docs.open-metadata.org/v1.1.x/how-to-guides/openmetadata' + ); + + const knowledgePage = screen.getAllByTestId('knowledge-page-link'); + + expect(knowledgePage[0]).toHaveAttribute('target', '_self'); + expect(knowledgePage[0]).toHaveAttribute( + 'href', + '/knowledge-center/Article_oRKYYTCu' + ); + }); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterWidget/KnowledgeCenterWidget.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterWidget/KnowledgeCenterWidget.tsx new file mode 100644 index 000000000000..7206c4a79029 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeCenterWidget/KnowledgeCenterWidget.tsx @@ -0,0 +1,224 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Icon from '@ant-design/icons'; +import { Col, Row, Typography } from 'antd'; +import { AxiosError } from 'axios'; +import classNames from 'classnames'; +import { isEmpty, map } from 'lodash'; +import { useEffect, useMemo, useState } from 'react'; +import { Link, useNavigate } from 'react-router-dom'; +import { ReactComponent as IconArticle } from '../../../assets/svg/ic-article.svg'; +import { ReactComponent as KnowledgeCenterWidgetIcon } from '../../../assets/svg/ic-knowledge-center-widget.svg'; +import { ReactComponent as LinkIcon } from '../../../assets/svg/ic-quick-link.svg'; +import { ReactComponent as KnowledgeCenterNoDataPlaceholder } from '../../../assets/svg/no-folder-data.svg'; +import WidgetEmptyState from '../../../components/MyData/Widgets/Common/WidgetEmptyState/WidgetEmptyState'; +import WidgetFooter from '../../../components/MyData/Widgets/Common/WidgetFooter/WidgetFooter'; +import WidgetHeader from '../../../components/MyData/Widgets/Common/WidgetHeader/WidgetHeader'; +import WidgetWrapper from '../../../components/MyData/Widgets/Common/WidgetWrapper/WidgetWrapper'; +import { PAGE_SIZE_MEDIUM } from '../../../constants/constants'; +import { SIZE } from '../../../enums/common.enum'; +import { EntityType } from '../../../enums/entity.enum'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; +import { + KnowledgePage, + PageType, + QuickLink, +} from '../../../interface/knowledge-center.interface'; +import { WidgetCommonProps } from '../../../pages/CustomizablePage/CustomizablePage.interface'; +import { getListKnowledgePages } from '../../../rest/knowledgeCenterAPI'; +import { getEntityName } from '../../../utils/EntityUtils'; +import { t } from '../../../utils/i18next/LocalUtil'; +import { getKnowledgePagePath } from '../../../utils/KnowledgePageUtils'; +import { showErrorToast } from '../../../utils/ToastUtils'; +import './KnowledgeCenterWidget.less'; + +import { ROUTES } from '../../../constants/constants'; +const KnowledgeCenterWidget = ({ + isEditView = false, + widgetKey, + handleRemoveWidget, + currentLayout, + handleLayoutUpdate, +}: WidgetCommonProps) => { + const navigate = useNavigate(); + const { currentUser } = useApplicationStore(); + const [isLoading, setIsLoading] = useState(true); + const [data, setData] = useState([]); + + const widgetData = useMemo(() => { + return currentLayout?.find((item) => item.i === widgetKey); + }, [currentLayout, widgetKey]); + + const isFullSizeWidget = useMemo(() => { + return currentLayout?.find((layout) => layout.i === widgetKey)?.w === 2; + }, [currentLayout, widgetKey]); + + const fetchUserKnowledgeArticles = async () => { + if (!currentUser?.id) { + return; + } + + setIsLoading(true); + try { + const { data: responseData } = await getListKnowledgePages({ + entityId: currentUser.id, + entityType: EntityType.USER, + limit: PAGE_SIZE_MEDIUM, + }); + setData(responseData); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setIsLoading(false); + } + }; + + const emptyState = useMemo(() => { + return ( + + } + title={t('label.no-knowledge-articles-available')} + /> + ); + }, [t]); + + const getGridTemplateColumns = () => { + if (isFullSizeWidget) { + return 'repeat(3, 1fr)'; + } + + return 'repeat(1, 1fr)'; + }; + + const articlesContent = useMemo(() => { + return ( +
+
+ {map(data, (knowledgePage) => { + const isQuickLink = knowledgePage.pageType === PageType.QUICK_LINK; + const quickLink = knowledgePage.page as QuickLink; + + return ( + + + + + + + + {getEntityName(knowledgePage)} + + + + + ); + })} +
+
+ ); + }, [data, isFullSizeWidget]); + + const showWidgetFooterMoreButton = useMemo( + () => Boolean(!isLoading) && data?.length > 10, + [data, isLoading] + ); + + const footer = useMemo(() => { + return ( + + ); + }, [isLoading]); + + const widgetHeader = useMemo(() => { + return ( + } + isEditView={isEditView} + title={t('label.knowledge-center')} + widgetKey={widgetKey} + onTitleClick={() => navigate(ROUTES.KNOWLEDGE_CENTER)} + /> + ); + }, [ + currentLayout, + handleLayoutUpdate, + handleRemoveWidget, + isEditView, + widgetKey, + widgetData?.w, + ]); + + useEffect(() => { + fetchUserKnowledgeArticles(); + }, [currentUser]); + + return ( + +
+
+ {isEmpty(data) ? emptyState : articlesContent} +
+ {!isEmpty(data) && !isFullSizeWidget && footer} +
+ + ); +}; + +export default KnowledgeCenterWidget; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeDetailPageHeader/KnowledgeDetailPageHeader.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeDetailPageHeader/KnowledgeDetailPageHeader.test.tsx new file mode 100644 index 000000000000..f104dbfe63f2 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeDetailPageHeader/KnowledgeDetailPageHeader.test.tsx @@ -0,0 +1,252 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + fireEvent, + getAllByTestId, + render, + screen, +} from '@testing-library/react'; +import { OperationPermission } from 'context/PermissionProvider/PermissionProvider.interface'; +import { User } from 'generated/entity/teams/user'; +import { + ContentChangeState, + KnowledgePage, +} from 'interface/knowledge-center.interface'; +import { MOCK_KNOWLEDGE_PAGE_DATA } from 'pages/KnowledgePage/KnowledgePage.mock'; +import { MemoryRouter } from 'react-router-dom'; +import KnowledgeDetailPageHeader, { + KnowledgeDetailPageHeaderProps, +} from './KnowledgeDetailPageHeader'; + +const mockOnCopyToClipBoard = jest.fn(); +const mockPush = jest.fn(); + +const mockUserData: User = { + name: 'aaron_johnson0', + email: 'testUser1@email.com', + id: '9304f330-2e9a-4513-883b-c939e29683a8', + isAdmin: true, +}; + +jest.mock('utils/AdvancedSearchClassBase', () => ({ + advancedSearchClassBase: { + autocomplete: jest.fn().mockReturnValue({ + asyncFetch: jest.fn(), + }), + }, +})); + +jest.mock('utils/JSONLogicSearchClassBase', () => ({ + JSONLogicSearchClassBase: jest.fn().mockImplementation(() => ({ + getTree: jest.fn(), + getFilters: jest.fn(), + })), +})); + +jest.mock( + 'components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.component', + () => ({ + useAdvanceSearch: jest.fn().mockReturnValue({}), + }) +); + +jest.mock('hooks/useApplicationStore', () => ({ + useApplicationStore: jest.fn(() => ({ + currentUser: mockUserData, + })), +})); + +jest.mock('hooks/useClipBoard', () => ({ + ...jest.requireActual('hooks/useClipBoard'), + useClipboard: jest + .fn() + .mockImplementation(() => ({ onCopyToClipBoard: mockOnCopyToClipBoard })), +})); + +jest.mock('react-router-dom', () => ({ + ...jest.requireActual('react-router-dom'), + useNavigate: jest.fn().mockImplementation(() => mockPush), +})); + +jest.mock('hooks/useFqn', () => ({ + useFqn: jest.fn().mockReturnValue({ + fqn: '', + }), +})); + +jest.mock('components/common/PopOverCard/UserPopOverCard', () => + jest + .fn() + .mockImplementation(() => ( +
UserPopOverCard
+ )) +); + +jest.mock('components/common/TitleBreadcrumb/TitleBreadcrumb.component', () => { + return jest.fn().mockImplementation(() =>
TitleBreadcrumb
); +}); + +const mockOnSetThreadLink = jest.fn(); +const mockOnVoteChange = jest.fn(); +const mockOnFollowChange = jest.fn(); +const mockOnToggleDelete = jest.fn(); +const mockOnSave = jest.fn(); + +const mockProps: KnowledgeDetailPageHeaderProps = { + isLoading: false, + knowledgePage: MOCK_KNOWLEDGE_PAGE_DATA as unknown as KnowledgePage, + contentChangeState: ContentChangeState.SAVED, + permissions: { + Delete: true, + } as OperationPermission, + onSetThreadLink: mockOnSetThreadLink, + onVoteChange: mockOnVoteChange, + onFollowChange: mockOnFollowChange, + onToggleDelete: mockOnToggleDelete, + fetchKnowledgePageHierarchy: jest.fn(), +}; + +describe('KnowledgeDetailPageHeader', () => { + it('should render KnowledgeDetailPageHeader', () => { + render(, { + wrapper: MemoryRouter, + }); + const updatedByContainer = screen.getByTestId('updated-by-list'); + + const updatedByList = getAllByTestId( + updatedByContainer, + 'user-popover-card' + ); + + expect(updatedByList).toHaveLength(1); + + const conversation = screen.getByTestId('conversation'); + + expect(conversation).toBeInTheDocument(); + + const upVoteButton = screen.getByTestId('up-vote-btn'); + + expect(upVoteButton).toBeInTheDocument(); + + const downVoteButton = screen.getByTestId('down-vote-btn'); + + expect(downVoteButton).toBeInTheDocument(); + + const versionButton = screen.getByTestId('version-button'); + + expect(versionButton).toHaveTextContent('1.2'); + + const followButton = screen.getByTestId('entity-follow-button'); + + expect(followButton).toHaveTextContent('1'); + + const shareButton = screen.getByTestId('share-button'); + + expect(shareButton).toBeInTheDocument(); + + const manageButton = screen.getByTestId('manage-button'); + + expect(manageButton).toBeInTheDocument(); + + expect(screen.getByText('TitleBreadcrumb')).toBeInTheDocument(); + }); + + it('onSetThreadLink should work', async () => { + render(, { + wrapper: MemoryRouter, + }); + + const conversationBtn = screen.getByTestId('conversation'); + + fireEvent.click(conversationBtn); + + expect(mockOnSetThreadLink).toHaveBeenCalled(); + }); + + it('onVoteChange should work', async () => { + render(, { + wrapper: MemoryRouter, + }); + + const upVoteButton = screen.getByTestId('up-vote-btn'); + + fireEvent.click(upVoteButton); + + const downVoteButton = screen.getByTestId('down-vote-btn'); + + fireEvent.click(downVoteButton); + + expect(mockOnVoteChange).toHaveBeenCalledTimes(2); + }); + + it('onFollowChange should work', async () => { + render(, { + wrapper: MemoryRouter, + }); + + const followButton = screen.getByTestId('entity-follow-button'); + + fireEvent.click(followButton); + + expect(mockOnFollowChange).toHaveBeenCalled(); + }); + + it('Version change should work', async () => { + render(, { + wrapper: MemoryRouter, + }); + + const versionButton = screen.getByTestId('version-button'); + + fireEvent.click(versionButton); + + expect(mockPush).toHaveBeenCalled(); + }); + + it('should render Save button for unsaved editable content', () => { + render( + , + { + wrapper: MemoryRouter, + } + ); + + const saveButton = screen.getByTestId('save-button'); + + expect(saveButton).toHaveTextContent('label.save'); + + fireEvent.click(saveButton); + + expect(mockOnSave).toHaveBeenCalled(); + }); + + it('should not render Save button without edit permissions', () => { + render( + , + { + wrapper: MemoryRouter, + } + ); + + expect(screen.queryByTestId('save-button')).not.toBeInTheDocument(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeDetailPageHeader/KnowledgeDetailPageHeader.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeDetailPageHeader/KnowledgeDetailPageHeader.tsx new file mode 100644 index 000000000000..600d14bcde91 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgeDetailPageHeader/KnowledgeDetailPageHeader.tsx @@ -0,0 +1,438 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Icon, { LoadingOutlined } from '@ant-design/icons'; +import { + Button, + Col, + Popover, + Row, + Skeleton, + Space, + Spin, + Tooltip, + Typography, +} from 'antd'; +import ButtonGroup from 'antd/lib/button/button-group'; +import classNames from 'classnames'; +import { isEmpty, isUndefined, map, toString, uniqBy, uniqueId } from 'lodash'; +import { FC, useCallback, useMemo, useState } from 'react'; +import { useNavigate } from 'react-router-dom'; +import { ReactComponent as ConversationIcon } from '../../../assets/svg/ic-conversation.svg'; +import { ReactComponent as IconSaved } from '../../../assets/svg/ic-saved.svg'; +import { ReactComponent as ShareIcon } from '../../../assets/svg/ic-share.svg'; +import { ReactComponent as StarFilledIcon } from '../../../assets/svg/ic-star-filled.svg'; +import { ReactComponent as StarIcon } from '../../../assets/svg/ic-star.svg'; +import { ReactComponent as IconUnSaved } from '../../../assets/svg/ic-unsaved.svg'; +import { ReactComponent as VersionIcon } from '../../../assets/svg/ic-version.svg'; +import { DeleteType } from '../../../components/common/DeleteWidget/DeleteWidget.interface'; +import ManageButton from '../../../components/common/EntityPageInfos/ManageButton/ManageButton'; +import UserPopOverCard from '../../../components/common/PopOverCard/UserPopOverCard'; +import TitleBreadcrumb from '../../../components/common/TitleBreadcrumb/TitleBreadcrumb.component'; +import { QueryVoteType } from '../../../components/Database/TableQueries/TableQueries.interface'; +import { EntityStatusBadge } from '../../../components/Entity/EntityStatusBadge/EntityStatusBadge.component'; +import Voting from '../../../components/Entity/Voting/Voting.component'; +import { VotingDataProps } from '../../../components/Entity/Voting/voting.interface'; +import { ROUTES, TEXT_BODY_COLOR } from '../../../constants/constants'; +import { EntityField } from '../../../constants/Feeds.constants'; +import { OperationPermission } from '../../../context/PermissionProvider/PermissionProvider.interface'; +import { EntityType } from '../../../enums/entity.enum'; +import { EntityStatus } from '../../../generated/entity/data/glossaryTerm'; +import { useCurrentUserPreferences } from '../../../hooks/currentUserStore/useCurrentUserStore'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; +import { useClipboard } from '../../../hooks/useClipBoard'; +import { useFqn } from '../../../hooks/useFqn'; +import { + ContentChangeState, + KnowledgePage, + RecentlyViewedQuickLinks, +} from '../../../interface/knowledge-center.interface'; +import deleteWidgetClassBase from '../../../utils/DeleteWidget/DeleteWidgetClassBase'; +import EntityLink from '../../../utils/EntityLink'; +import { getEntityName } from '../../../utils/EntityUtils'; +import i18n from '../../../utils/i18next/LocalUtil'; +import { + getKnowledgeVersionsPath, + updateKnowledgeCenterRecentViewed, +} from '../../../utils/KnowledgePageUtils'; + +export interface KnowledgeDetailPageHeaderProps { + isLoading: boolean; + contentChangeState: ContentChangeState; + permissions: OperationPermission; + knowledgePage?: KnowledgePage; + onSetThreadLink: (link: string) => void; + onVoteChange: (type: VotingDataProps) => Promise; + onFollowChange: () => Promise; + onToggleDelete: () => void; + onSave?: () => void; + fetchKnowledgePageHierarchy?: (forceRefresh?: boolean) => Promise; +} + +const KnowledgeDetailPageHeader: FC = ({ + knowledgePage, + contentChangeState, + onSetThreadLink, + onVoteChange, + permissions, + onFollowChange, + onToggleDelete, + onSave, + isLoading, + fetchKnowledgePageHierarchy, +}) => { + const navigate = useNavigate(); + const { fqn } = useFqn(); + const { currentUser } = useApplicationStore(); + const USERId = currentUser?.id ?? ''; + const { t } = i18n; + + const [copyTooltip, setCopyTooltip] = useState(); + const { onCopyToClipBoard } = useClipboard(window.location.href); + const [isFollowLoading, setIsFollowLoading] = useState(false); + const { + preferences: { recentlyViewedQuickLinks }, + } = useCurrentUserPreferences(); + const recentlyViewed = + recentlyViewedQuickLinks as unknown as RecentlyViewedQuickLinks['data']; + + const breadcrumbs = useMemo( + () => [ + { + name: t('label.home'), + url: ROUTES.HOME, + }, + { + name: t('label.knowledge-center'), + url: ROUTES.KNOWLEDGE_CENTER, + }, + { + name: (knowledgePage?.displayName ?? '') || t('label.untitled'), + url: '', + activeTitle: false, + }, + ], + [knowledgePage?.displayName] + ); + + const entityStatusBadge = useMemo(() => { + const shouldShowStatus = true; + const entityStatus = knowledgePage?.entityStatus; + + if ( + !shouldShowStatus || + !entityStatus || + entityStatus === EntityStatus.Unprocessed + ) { + return null; + } + + return ; + }, [knowledgePage?.entityStatus]); + + const contentChangeIcon = useMemo(() => { + if (contentChangeState === ContentChangeState.SAVED) { + return ; + } else if (contentChangeState === ContentChangeState.SAVING) { + return ( + + } + /> + ); + } else if (contentChangeState === ContentChangeState.UN_SAVED) { + return ; + } else { + return null; + } + }, [contentChangeState]); + + const editors = useMemo(() => { + const list = uniqBy( + [ + ...(knowledgePage?.editors ?? []), + ...[{ name: knowledgePage?.updatedBy }], + ], + 'name' + ); + + return { upFrontList: list.slice(0, 5), popOverList: list.slice(5) }; + }, [knowledgePage]); + + const editorsPopoverElement = useMemo( + () => + !isEmpty(editors.popOverList) && ( + + {map(editors.popOverList, (user) => ( + + ))} + + } + trigger={['click', 'hover']}> + {`+${editors.popOverList.length}`} + + ), + [editors] + ); + + const voteStatus = useMemo(() => { + if (isUndefined(knowledgePage?.votes)) { + return QueryVoteType.unVoted; + } + + const upVoters = knowledgePage?.votes.upVoters || []; + const downVoters = knowledgePage?.votes.downVoters || []; + + if (upVoters.some((user) => user.id === USERId)) { + return QueryVoteType.votedUp; + } else if (downVoters.some((user) => user.id === USERId)) { + return QueryVoteType.votedDown; + } else { + return QueryVoteType.unVoted; + } + }, [knowledgePage, USERId]); + + const { isFollowing, followers, version } = useMemo(() => { + return { + isFollowing: Boolean( + knowledgePage?.followers?.some(({ id }) => id === USERId) + ), + followers: knowledgePage?.followers?.length ?? 0, + version: knowledgePage?.version ?? '0.1', + }; + }, [knowledgePage, USERId]); + + const { entityName, entityType } = useMemo(() => { + return { + entityName: getEntityName(knowledgePage), + entityType: t('label.article'), + }; + }, [knowledgePage]); + + const deleteOptions = [ + { + title: `${t('label.permanently-delete')} ${entityType} “${entityName}”`, + description: deleteWidgetClassBase.getDeleteMessage( + entityName, + entityType + ), + type: DeleteType.HARD_DELETE, + isAllowed: true, + }, + ]; + + const handleVersionClick = () => { + navigate(getKnowledgeVersionsPath(fqn, toString(version))); + }; + + const handleShareButtonClick = async () => { + await onCopyToClipBoard(); + setCopyTooltip(t('message.copy-to-clipboard')); + setTimeout(() => setCopyTooltip(''), 2000); + }; + + const handleFollowUnFollow = async () => { + setIsFollowLoading(true); + await onFollowChange(); + setIsFollowLoading(false); + }; + + const afterDeleteAction = useCallback( + (isSoftDelete?: boolean) => { + updateKnowledgeCenterRecentViewed( + recentlyViewed.filter((page) => page.id !== knowledgePage?.id) + ); + isSoftDelete ? onToggleDelete() : navigate(ROUTES.KNOWLEDGE_CENTER); + + // fetch knowledge page hierarchy with force refresh to ensure updates are shown + fetchKnowledgePageHierarchy?.(true); + }, + [knowledgePage] + ); + + const showSaveButton = + Boolean(onSave) && + contentChangeState === ContentChangeState.UN_SAVED && + (permissions.EditAll || + permissions.EditDescription || + permissions.EditDisplayName); + + if (isLoading) { + return ( +
+ + {Array(3) + .fill(null) + .map(() => ( + + + + ))} + + + {Array(3) + .fill(null) + .map(() => ( + + + + ))} + +
+ ); + } + + return ( +
+ + + + {entityStatusBadge} + + + + + {contentChangeIcon} + + {contentChangeState} + + + + {map(editors.upFrontList, (user) => ( + + ))} + + {editorsPopoverElement} + + + onSetThreadLink( + EntityLink.getEntityLink( + EntityType.KNOWLEDGE_PAGE, + knowledgePage?.fullyQualifiedName ?? '', + EntityField.DESCRIPTION + ) + ) + } + /> + {showSaveButton && ( + + )} + + + + + + + +
+ ); +}; + +export default KnowledgeDetailPageHeader; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageDetailComponent/KnowledgePageDetailComponent.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageDetailComponent/KnowledgePageDetailComponent.tsx new file mode 100644 index 000000000000..e4befd8c86de --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageDetailComponent/KnowledgePageDetailComponent.tsx @@ -0,0 +1,774 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Tabs } from 'antd'; +import { AxiosError } from 'axios'; +import { compare } from 'fast-json-patch'; +import { cloneDeep, debounce, isEqual, isNil, isUndefined } from 'lodash'; +import { + FC, + KeyboardEvent, + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from 'react'; +import { useLocation, useNavigate } from 'react-router-dom'; +import { useActivityFeedProvider } from '../../../components/ActivityFeed/ActivityFeedProvider/ActivityFeedProvider'; +import { ActivityFeedTab } from '../../../components/ActivityFeed/ActivityFeedTab/ActivityFeedTab.component'; +import { ActivityFeedLayoutType } from '../../../components/ActivityFeed/ActivityFeedTab/ActivityFeedTab.interface'; +import ActivityThreadPanel from '../../../components/ActivityFeed/ActivityThreadPanel/ActivityThreadPanel'; +import BlockEditor from '../../../components/BlockEditor/BlockEditor'; +import { BlockEditorRef } from '../../../components/BlockEditor/BlockEditor.interface'; +import { EntityAttachmentProvider } from '../../../components/common/EntityDescription/EntityAttachmentProvider/EntityAttachmentProvider'; +import ErrorPlaceHolder from '../../../components/common/ErrorWithPlaceholder/ErrorPlaceHolder'; +import TabsLabel from '../../../components/common/TabsLabel/TabsLabel.component'; +import { GenericProvider } from '../../../components/Customization/GenericProvider/GenericProvider'; +import { QueryVoteType } from '../../../components/Database/TableQueries/TableQueries.interface'; +import { VotingDataProps } from '../../../components/Entity/Voting/voting.interface'; +import { + CREATE_PAGE_HASH, + LONG_DELAY, + SHORT_DELAY, +} from '../../../constants/constants'; +import { CustomizeEntityType } from '../../../constants/Customize.constants'; +import { FEED_COUNT_INITIAL_DATA } from '../../../constants/entity.constants'; +import { + getKnowledgePageFields, + KNOWLEDGE_PAGE_FIELDS, + KNOWLEDGE_PAGE_UN_SAVED_CHANGE_STATE, +} from '../../../constants/KnowledgeCenter.constant'; +import { usePermissionProvider } from '../../../context/PermissionProvider/PermissionProvider'; +import { + OperationPermission, + ResourceEntity, +} from '../../../context/PermissionProvider/PermissionProvider.interface'; +import { ERROR_PLACEHOLDER_TYPE } from '../../../enums/common.enum'; +import { EntityTabs, EntityType } from '../../../enums/entity.enum'; +import { + CreateThread, + ThreadType, +} from '../../../generated/api/feed/createThread'; +import { TagLabel } from '../../../generated/type/tagLabel'; +import { useCurrentUserPreferences } from '../../../hooks/currentUserStore/useCurrentUserStore'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; +import useCustomLocation from '../../../hooks/useCustomLocation/useCustomLocation'; +import { FeedCounts } from '../../../interface/feed.interface'; +import { + ContentChangeState, + KnowledgeCenterPageProps, + KnowledgePage, + RecentlyViewedQuickLinks, +} from '../../../interface/knowledge-center.interface'; +import { EntityTags } from '../../../Models'; +import { postThread } from '../../../rest/feedsAPI'; +import { + followKnowledgePage, + getKnowledgePageByFqn, + patchKnowledgePage, + unFollowKnowledgePage, + updateKnowledgePageVote, +} from '../../../rest/knowledgeCenterAPI'; +import { getFeedCounts } from '../../../utils/CommonUtils'; +import i18n from '../../../utils/i18next/LocalUtil'; +import { + addToKnowledgeCenterRecentViewed, + getKnowledgePagePath, + updateKnowledgeCenterRecentViewed, +} from '../../../utils/KnowledgePageUtils'; +import { DEFAULT_ENTITY_PERMISSION } from '../../../utils/PermissionsUtils'; +import { getTagsWithoutTier } from '../../../utils/TableUtils'; +import { createTagObject } from '../../../utils/TagsUtils'; +import { showErrorToast } from '../../../utils/ToastUtils'; +import { useRequiredParams } from '../../../utils/useRequiredParams'; +import KnowledgeDetailPageHeader from '../KnowledgeDetailPageHeader/KnowledgeDetailPageHeader'; +import KnowledgePageDetailRightPanel from '../KnowledgePageDetailRightPanel/KnowledgePageDetailRightPanel'; +import { TitleComponent } from '../TitleComponent/TitleComponent'; +import KnowledgePageDetailSkeleton from './KnowledgePageDetailSkeleton'; + +interface KnowledgePageDetailComponentProps { + onPageChange: (page: Partial) => void; + fetchKnowledgePageHierarchy?: (forceRefresh?: boolean) => Promise; +} + +const KnowledgePageDetailComponent: FC = ({ + onPageChange, + fetchKnowledgePageHierarchy, +}) => { + const { t } = i18n; + const { hash } = useCustomLocation(); + const { currentUser } = useApplicationStore(); + const editorRef = useRef({} as BlockEditorRef); + const titleRef = useRef(null); + const { getEntityPermissionByFqn } = usePermissionProvider(); + const location = useLocation(); + const navigate = useNavigate(); + + const { postFeed, deleteFeed, updateFeed } = useActivityFeedProvider(); + const USERId = currentUser?.id ?? ''; + + const { fqn, tab } = useRequiredParams<{ fqn: string; tab?: string }>(); + const [isLoading, setIsLoading] = useState(true); + const [knowledgePage, setKnowledgePage] = useState(); + const [activeTab, setActiveTab] = useState( + tab ?? EntityTabs.OVERVIEW + ); + const [feedCount, setFeedCount] = useState( + FEED_COUNT_INITIAL_DATA + ); + + const [threadLink, setThreadLink] = useState(''); + const [permissions, setPermissions] = useState( + DEFAULT_ENTITY_PERMISSION + ); + const [contentChangeState, setContentChangeState] = + useState(ContentChangeState.SAVED); + + const { + preferences: { recentlyViewedQuickLinks }, + } = useCurrentUserPreferences(); + const recentlyViewed = + recentlyViewedQuickLinks as unknown as RecentlyViewedQuickLinks['data']; + + const fetchPermission = async () => { + setIsLoading(true); + try { + const response = await getEntityPermissionByFqn( + ResourceEntity.KNOWLEDGE_PAGE as unknown as ResourceEntity, + fqn + ); + setPermissions(response); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setIsLoading(false); + } + }; + + const fetchKnowledgePage = async (fqn: string) => { + setIsLoading(true); + try { + const response = await getKnowledgePageByFqn(fqn, { + fields: getKnowledgePageFields([ + KNOWLEDGE_PAGE_FIELDS.RELATED_ARTICLES, + KNOWLEDGE_PAGE_FIELDS.EDITORS, + KNOWLEDGE_PAGE_FIELDS.PARENT, + ]), + }); + setKnowledgePage(response); + addToKnowledgeCenterRecentViewed({ ...response, timestamp: 0 }); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setIsLoading(false); + } + }; + + const updateVoteHandler = async (data: VotingDataProps, id: string) => { + try { + const { entity } = await updateKnowledgePageVote(id, data); + + setKnowledgePage((prev) => { + const currentKnowledgePage = prev as KnowledgePage; + + return { ...currentKnowledgePage, votes: entity.votes }; + }); + } catch (error) { + showErrorToast(error as AxiosError); + } + }; + + const voteStatus = useMemo(() => { + if (isUndefined(knowledgePage?.votes)) { + return QueryVoteType.unVoted; + } + + const upVoters = knowledgePage?.votes.upVoters || []; + const downVoters = knowledgePage?.votes.downVoters || []; + + if (upVoters.some((user) => user.id === USERId)) { + return QueryVoteType.votedUp; + } else if (downVoters.some((user) => user.id === USERId)) { + return QueryVoteType.votedDown; + } else { + return QueryVoteType.unVoted; + } + }, [knowledgePage, USERId]); + + const updateDelay = useMemo( + () => (hash.slice(1) === CREATE_PAGE_HASH ? LONG_DELAY : SHORT_DELAY), + [hash] + ); + + const handleVoteChange = async (type: VotingDataProps) => { + let updatedVoteType; + + // current vote is same as selected vote, it means user is removing vote, else up/down voting + if (voteStatus === type.updatedVoteType) { + updatedVoteType = QueryVoteType.unVoted; + } else { + updatedVoteType = type.updatedVoteType; + } + + knowledgePage && + (await updateVoteHandler({ updatedVoteType }, knowledgePage.id)); + }; + + const { isFollowing, tags, displayName } = useMemo( + () => ({ + isFollowing: Boolean( + knowledgePage?.followers?.some(({ id }) => id === USERId) + ), + + displayName: knowledgePage?.displayName ?? '', + + tags: getTagsWithoutTier(knowledgePage?.tags ?? []), + }), + [knowledgePage, USERId] + ); + + const handleToggleDelete = () => { + setKnowledgePage((prev) => { + if (!prev) { + return prev; + } + + return { ...prev, deleted: !prev?.deleted }; + }); + }; + + const followKnowledgePageHandler = async (knowledgePageId: string) => { + try { + const res = await followKnowledgePage(knowledgePageId, USERId); + const { newValue } = res.changeDescription.fieldsAdded[0]; + setKnowledgePage((prev) => { + if (!prev) { + return prev; + } + + return { + ...prev, + followers: [...(prev?.followers ?? []), ...newValue], + }; + }); + } catch (error) { + showErrorToast(error as AxiosError); + } + }; + + const unFollowKnowledgePageHandler = async (knowledgePageId: string) => { + try { + const res = await unFollowKnowledgePage(knowledgePageId, USERId); + const { oldValue } = res.changeDescription.fieldsDeleted[0]; + + setKnowledgePage((prev) => { + if (!prev) { + return prev; + } + + return { + ...prev, + followers: (prev?.followers ?? []).filter( + (follower) => follower.id !== oldValue[0].id + ), + }; + }); + } catch (error) { + showErrorToast(error as AxiosError); + } + }; + + const handleFollowChange = async () => { + const knowledgePageId = knowledgePage?.id ?? ''; + + if (isFollowing) { + await unFollowKnowledgePageHandler(knowledgePageId); + } else { + await followKnowledgePageHandler(knowledgePageId); + } + }; + + const createThread = async (data: CreateThread) => { + try { + await postThread(data); + } catch (error) { + showErrorToast(error as AxiosError); + } + }; + + const updatedPageContent = useCallback( + async (updatedContent: string) => { + const hasContentEditPermission = + permissions.EditAll || permissions.EditDescription; + + if (isUndefined(knowledgePage) || !hasContentEditPermission) { + return; + } + + const currentKnowledgePage = cloneDeep(knowledgePage); + const existingContent = currentKnowledgePage.description; + + if (existingContent === updatedContent) { + return; + } + + try { + setContentChangeState(ContentChangeState.SAVING); + const updatedKnowledgePage: KnowledgePage = { + ...currentKnowledgePage, + description: updatedContent, + }; + + const patch = compare(currentKnowledgePage, updatedKnowledgePage); + + const response = await patchKnowledgePage( + currentKnowledgePage.id, + patch + ); + + setKnowledgePage({ + ...currentKnowledgePage, + version: response.version, + }); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setContentChangeState(ContentChangeState.SAVED); + } + }, + [knowledgePage, setKnowledgePage, permissions] + ); + + const handleContentSave = useCallback( + debounce(updatedPageContent, updateDelay), + [updatedPageContent, updateDelay, permissions] + ); + + const handleContentOnChange = useCallback( + (content: string) => { + const isChanged = !isEqual(knowledgePage?.description ?? '', content); + setContentChangeState( + isChanged ? ContentChangeState.UN_SAVED : ContentChangeState.SAVED + ); + handleContentSave(content); + }, + [knowledgePage, handleContentSave] + ); + + const updatePage = async (updatedKnowledgePage: KnowledgePage) => { + if (isUndefined(knowledgePage)) { + return; + } + const currentKnowledgePage = cloneDeep(knowledgePage); + try { + const patch = compare(currentKnowledgePage, updatedKnowledgePage); + const response = await patchKnowledgePage(currentKnowledgePage.id, patch); + + setKnowledgePage({ + ...currentKnowledgePage, + tags: response.tags, + owners: response.owners, + reviewers: response.reviewers, + domains: response.domains, + dataProducts: response.dataProducts, + version: response.version, + }); + } catch (error) { + showErrorToast(error as AxiosError); + } + }; + + const updatePageTag = async (selectedTags: EntityTags[]) => { + if (isUndefined(knowledgePage) || isUndefined(selectedTags)) { + return; + } + + const updatedTags: TagLabel[] = createTagObject(selectedTags); + const currentKnowledgePage = cloneDeep(knowledgePage); + try { + const updatedKnowledgePage: KnowledgePage = { + ...currentKnowledgePage, + tags: updatedTags, + }; + const patch = compare(currentKnowledgePage, updatedKnowledgePage); + + const response = await patchKnowledgePage(currentKnowledgePage.id, patch); + + setKnowledgePage({ + ...currentKnowledgePage, + tags: response.tags, + version: response.version, + }); + } catch (error) { + showErrorToast(error as AxiosError); + } + }; + + const handleDisplayNameUpdate = useCallback( + async (updatedDisplayName: string) => { + const hasDisplayNameEditPermission = + permissions.EditAll || permissions.EditDisplayName; + + if (!knowledgePage || !hasDisplayNameEditPermission) { + return; + } + const currentKnowledgePage = cloneDeep(knowledgePage); + const updatedKnowledgePage = { + ...knowledgePage, + displayName: updatedDisplayName.trim(), + }; + try { + setContentChangeState(ContentChangeState.SAVING); + const patch = compare(currentKnowledgePage, updatedKnowledgePage); + + const response = await patchKnowledgePage( + currentKnowledgePage.id, + patch + ); + updateKnowledgeCenterRecentViewed( + recentlyViewed.map((recentView) => { + if (recentView.id === response.id) { + return { ...recentView, displayName: response.displayName }; + } + + return recentView; + }) + ); + + setKnowledgePage({ + ...currentKnowledgePage, + displayName: response.displayName, + version: response.version, + }); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setContentChangeState(ContentChangeState.SAVED); + } + }, + [knowledgePage, setKnowledgePage, permissions] + ); + + const handleDisplayNameSave = useCallback( + debounce(handleDisplayNameUpdate, updateDelay), + [handleDisplayNameUpdate, updateDelay, permissions] + ); + + const handleSave = useCallback(() => { + handleDisplayNameSave.flush(); + handleContentSave.flush(); + }, [handleDisplayNameSave, handleContentSave]); + + const handleDisplayNameChange = useCallback( + (updatedDisplayName: string) => { + const isChanged = !isEqual( + knowledgePage?.displayName ?? '', + updatedDisplayName + ); + setContentChangeState( + isChanged ? ContentChangeState.UN_SAVED : ContentChangeState.SAVED + ); + handleDisplayNameSave(updatedDisplayName); + }, + [knowledgePage, handleDisplayNameSave] + ); + + const handleRelatedEntitiesUpdate = async ( + updatedRelatedEntities: KnowledgePage['relatedEntities'] + ) => { + if (isUndefined(knowledgePage)) { + return; + } + + const currentKnowledgePage = cloneDeep(knowledgePage); + try { + const updatedKnowledgePage: KnowledgePage = { + ...currentKnowledgePage, + relatedEntities: updatedRelatedEntities, + }; + const patch = compare(currentKnowledgePage, updatedKnowledgePage); + + const response = await patchKnowledgePage(currentKnowledgePage.id, patch); + + setKnowledgePage((previousPage) => ({ + ...((previousPage ?? {}) as KnowledgePage), + relatedEntities: response['relatedEntities'], + version: response.version, + })); + } catch (error) { + showErrorToast(error as AxiosError); + } + }; + + const handleFeedCount = useCallback((data: FeedCounts) => { + setFeedCount(data); + }, []); + + const getEntityFeedCount = () => { + if (knowledgePage?.fullyQualifiedName) { + getFeedCounts( + EntityType.KNOWLEDGE_PAGE, + knowledgePage.fullyQualifiedName, + handleFeedCount + ); + } + }; + + const handleTabChange = (activeKey: string) => { + if (activeKey !== activeTab) { + navigate(getKnowledgePagePath(fqn, activeKey)); + setActiveTab(activeKey); + } + }; + + const handleTitleKeyDown = (e: KeyboardEvent) => { + if (e.shiftKey) { + return; + } + if (e.key === 'Enter' || e.key === 'ArrowDown') { + e.preventDefault(); + if (!isNil(editorRef.current.editor)) { + editorRef.current.editor.commands.focus('start'); + } + } + }; + + const tabs = useMemo(() => { + const items = [ + { + label:
{String(t('label.content'))}
, + key: EntityTabs.OVERVIEW, + children: ( + <> + + + + + + ), + }, + { + label: ( + + ), + key: EntityTabs.ACTIVITY_FEED, + children: ( + fetchKnowledgePage(fqn)} + /> + ), + }, + ]; + + return items; + }, [knowledgePage, feedCount, activeTab, permissions, displayName, fqn]); + + const hasViewPermission = useMemo( + () => permissions.ViewAll || permissions.ViewBasic, + [permissions] + ); + + const isContentUnsaved = useMemo( + () => KNOWLEDGE_PAGE_UN_SAVED_CHANGE_STATE.includes(contentChangeState), + [contentChangeState] + ); + + const getHeaderElement = useCallback( + () => ( + + ), + [ + contentChangeState, + isLoading, + knowledgePage, + permissions, + setThreadLink, + handleToggleDelete, + handleVoteChange, + handleSave, + ] + ); + + useEffect(() => { + fetchPermission(); + }, []); + + useEffect(() => { + if (tab) { + setActiveTab(tab); + } + }, [tab]); + + useEffect(() => { + if (hasViewPermission) { + fetchKnowledgePage(fqn); + } + }, [fqn, hasViewPermission]); + + useEffect(() => { + if (knowledgePage?.fullyQualifiedName) { + getEntityFeedCount(); + } + }, [knowledgePage?.fullyQualifiedName]); + + useEffect(() => { + const handleBeforeUnload = (event: BeforeUnloadEvent) => { + if (isContentUnsaved) { + event.preventDefault(); + event.returnValue = ''; + } + }; + + const handleBeforeNavigate = (event: PopStateEvent) => { + if (isContentUnsaved) { + const confirm = window.confirm(t('message.unsaved-change-in-page')); + if (!confirm) { + event.preventDefault(); + window.history.pushState(null, '', location.pathname); + } + } + }; + + window.addEventListener('beforeunload', handleBeforeUnload); + window.addEventListener('popstate', handleBeforeNavigate); + + return () => { + window.removeEventListener('beforeunload', handleBeforeUnload); + window.removeEventListener('popstate', handleBeforeNavigate); + }; + }, [isContentUnsaved, location.pathname]); + + const pageConfig = useMemo(() => { + let rightPanel = null; + if (activeTab !== EntityTabs.ACTIVITY_FEED && knowledgePage) { + rightPanel = ( + + + + ); + } + + return { + data: knowledgePage, + header:
{getHeaderElement()}
, + rightPanel, + title: (knowledgePage?.displayName ?? '') || t('label.untitled'), + activeTab, + }; + }, [ + knowledgePage, + isLoading, + permissions, + contentChangeState, + activeTab, + tags, + getHeaderElement, + ]); + + useEffect(() => { + onPageChange(pageConfig); + }, [pageConfig, onPageChange]); + + if (isLoading) { + return ; + } + + if (!hasViewPermission) { + return ( + + ); + } + + if (!knowledgePage) { + return ; + } + + return ( + <> + + {threadLink ? ( + setThreadLink('')} + /> + ) : null} + + ); +}; + +export default KnowledgePageDetailComponent; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageDetailComponent/KnowledgePageDetailSkeleton.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageDetailComponent/KnowledgePageDetailSkeleton.tsx new file mode 100644 index 000000000000..b07148c9c160 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageDetailComponent/KnowledgePageDetailSkeleton.tsx @@ -0,0 +1,37 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Col, Row, Skeleton } from 'antd'; + +const KnowledgePageDetailSkeleton = () => { + return ( + + + + +
+ + +
+ + + + + ); +}; + +export default KnowledgePageDetailSkeleton; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageDetailRightPanel/KnowledgePageDetailRightPanel.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageDetailRightPanel/KnowledgePageDetailRightPanel.tsx new file mode 100644 index 000000000000..83c4fe7a1217 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageDetailRightPanel/KnowledgePageDetailRightPanel.tsx @@ -0,0 +1,166 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Col, Row } from 'antd'; +import { AxiosError } from 'axios'; +import { FC, useCallback, useMemo } from 'react'; +import { useGenericContext } from '../../../components/Customization/GenericProvider/GenericProvider'; +import { DomainLabelV2 } from '../../../components/DataAssets/DomainLabelV2/DomainLabelV2'; +import { OwnerLabelV2 } from '../../../components/DataAssets/OwnerLabelV2/OwnerLabelV2'; +import { ReviewerLabelV2 } from '../../../components/DataAssets/ReviewerLabelV2/ReviewerLabelV2'; +import DataProductsContainer from '../../../components/DataProducts/DataProductsContainer/DataProductsContainer.component'; +import TagsContainerV2 from '../../../components/Tag/TagsContainerV2/TagsContainerV2'; +import { DisplayType } from '../../../components/Tag/TagsViewer/TagsViewer.interface'; +import { OperationPermission } from '../../../context/PermissionProvider/PermissionProvider.interface'; +import { EntityType } from '../../../enums/entity.enum'; +import { DataProduct } from '../../../generated/entity/domains/dataProduct'; +import { EntityReference } from '../../../generated/entity/type'; +import { TagSource } from '../../../generated/type/tagLabel'; +import { KnowledgePage } from '../../../interface/knowledge-center.interface'; +import { EntityTags } from '../../../Models'; +import { showErrorToast } from '../../../utils/ToastUtils'; +import RelatedDataAssets from '../RelatedDataAssets/RelatedDataAssets'; +import './knowledge-page.less'; +import KnowledgePageDetailRightPanelSkeleton from './KnowledgePageDetailRightPanelSkeleton'; +interface KnowledgePageDetailRightPanelProps { + isLoading: boolean; + permissions: OperationPermission; + tags: Array; + knowledgePage?: KnowledgePage; + updatePageTag: (tags: Array) => Promise; + handleRelatedEntitiesUpdate: ( + relatedEntities?: Array + ) => Promise; +} + +const KnowledgePageDetailRightPanel: FC = ({ + isLoading, + knowledgePage, + permissions, + tags, + updatePageTag, + handleRelatedEntitiesUpdate, +}) => { + const { + entityRules, + data, + onUpdate, + permissions: genericPermissions, + } = useGenericContext(); + + const handleDataProductsSave = useCallback( + async (selectedDataProducts: DataProduct[]) => { + try { + const updatedEntity = { ...data }; + updatedEntity.dataProducts = selectedDataProducts.map((dp) => ({ + id: dp.id, + fullyQualifiedName: dp.fullyQualifiedName, + name: dp.name, + displayName: dp.displayName, + type: EntityType.DATA_PRODUCT, + })); + + await onUpdate(updatedEntity); + } catch (err) { + showErrorToast(err as AxiosError); + } + }, + [data, onUpdate] + ); + + const handleDomainSave = useCallback( + async (selectedDomain: EntityReference | EntityReference[]) => { + try { + const updatedEntity = { ...data }; + updatedEntity.domains = Array.isArray(selectedDomain) + ? selectedDomain + : [selectedDomain]; + + await onUpdate(updatedEntity); + } catch (err) { + showErrorToast(err as AxiosError); + } + }, + [data, onUpdate] + ); + + const hasDataProductsPermission = useMemo(() => { + return genericPermissions?.EditAll && !data?.deleted; + }, [genericPermissions?.EditAll, data?.deleted]); + + if (isLoading) { + return ; + } + + return ( +
+ + + + + +
+ +
+ + + + + + + + + + + + + + + + + +
+ ); +}; + +export default KnowledgePageDetailRightPanel; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageDetailRightPanel/KnowledgePageDetailRightPanelSkeleton.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageDetailRightPanel/KnowledgePageDetailRightPanelSkeleton.tsx new file mode 100644 index 000000000000..b179b9a642ca --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageDetailRightPanel/KnowledgePageDetailRightPanelSkeleton.tsx @@ -0,0 +1,45 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Col, Row, Skeleton, Space } from 'antd'; +import { uniqueId } from 'lodash'; + +const KnowledgePageDetailRightPanelSkeleton = () => { + return ( +
+ + + + {Array.from({ length: 3 }).map(() => ( + + + + + + + ))} + +
+ ); +}; + +export default KnowledgePageDetailRightPanelSkeleton; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageDetailRightPanel/knowledge-page.less b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageDetailRightPanel/knowledge-page.less new file mode 100644 index 000000000000..fe71e62a38f4 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageDetailRightPanel/knowledge-page.less @@ -0,0 +1,17 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +.knowledge-page-right-panel { + .new-header-border-card { + height: initial; + } +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageListComponent/KnowledgePageListComponent.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageListComponent/KnowledgePageListComponent.tsx new file mode 100644 index 000000000000..cf37d43ad331 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageListComponent/KnowledgePageListComponent.tsx @@ -0,0 +1,530 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { PlusOutlined } from '@ant-design/icons'; +import { + Button, + Col, + Dropdown, + MenuProps, + Row, + Skeleton, + Space, + Typography, +} from 'antd'; +import { AxiosError } from 'axios'; +import cryptoRandomString from 'crypto-random-string-with-promisify-polyfill'; +import { isEmpty, map, uniqBy, uniqueId } from 'lodash'; +import { + forwardRef, + RefObject, + useCallback, + useEffect, + useImperativeHandle, + useMemo, + useState, +} from 'react'; +import { useTranslation } from 'react-i18next'; +import { useNavigate } from 'react-router-dom'; +import { ReactComponent as AddPlaceHolderIcon } from '../../../assets/svg/add-placeholder.svg'; +import ErrorPlaceHolder from '../../../components/common/ErrorWithPlaceholder/ErrorPlaceHolder'; +import Loader from '../../../components/common/Loader/Loader'; +import { VotingDataProps } from '../../../components/Entity/Voting/voting.interface'; +import { + CREATE_PAGE_HASH, + PAGE_SIZE_MEDIUM, +} from '../../../constants/constants'; +import { KNOWLEDGE_CENTER_DOC_LINK } from '../../../constants/docs.constant'; +import { getKnowledgePageFields } from '../../../constants/KnowledgeCenter.constant'; +import { useLimitStore } from '../../../context/LimitsProvider/useLimitsStore'; +import { OperationPermission } from '../../../context/PermissionProvider/PermissionProvider.interface'; +import { ERROR_PLACEHOLDER_TYPE, SIZE } from '../../../enums/common.enum'; +import { Paging } from '../../../generated/type/paging'; +import LimitWrapper from '../../../hoc/LimitWrapper'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; +import { useElementInView } from '../../../hooks/useElementInView'; +import { + CreateKnowledgePage, + KnowledgeCenterPageProps, + KnowledgeCenterPageRef, + KnowledgePage, + PageType, +} from '../../../interface/knowledge-center.interface'; +import { + followKnowledgePage, + getListKnowledgePages, + postKnowledgePage, + unFollowKnowledgePage, + updateKnowledgePageVote, +} from '../../../rest/knowledgeCenterAPI'; +import { Transi18next } from '../../../utils/i18next/LocalUtil'; +import { getKnowledgePagePath } from '../../../utils/KnowledgePageUtils'; +import { showErrorToast } from '../../../utils/ToastUtils'; +import KnowledgeCard from '../KnowledgeCard/KnowledgeCard'; +import KnowledgePageListRightPanel from '../KnowledgePageListRightPanel/KnowledgePageListRightPanel'; +import { + QuickLinkFormModal, + QuickLinkFormModalFormData, +} from '../QuickLinkFormModal/QuickLinkFormModal'; +import './knowledge-page-list.less'; + +interface KnowledgePageListComponentProps { + onPageChange: (page: Partial) => void; + permissions: OperationPermission; +} + +const KnowledgePageListComponent = forwardRef< + KnowledgeCenterPageRef, + KnowledgePageListComponentProps +>(({ onPageChange, permissions }, ref) => { + const { currentUser, theme } = useApplicationStore(); + const { t } = useTranslation(); + const navigate = useNavigate(); + const USERId = currentUser?.id ?? ''; + const [elementRef, isInView] = useElementInView({}); + const [isLoading, setIsLoading] = useState(true); + const [isLoadingMore, setIsLoadingMore] = useState(false); + const [knowledgePages, setKnowledgePages] = useState([]); + const [paging, setPaging] = useState({ total: 0 }); + const [isCreatingNewPage, setIsCreatingNewPage] = useState(false); + const [showAddLinkModal, setShowAddLinkModal] = useState(false); + const { getResourceLimit } = useLimitStore(); + + const [refreshBookMarkWidget, setRefreshBookMarkWidget] = + useState(false); + const [refreshTagsCategory, setRefreshTagsCategory] = + useState(false); + + const handleRefreshBookMarkWidget = (value: boolean) => + setRefreshBookMarkWidget(value); + + const handleRefreshTagsCategory = (value: boolean) => + setRefreshTagsCategory(value); + + const fetchKnowledgePages = async (after?: string) => { + if (after) { + setIsLoadingMore(true); + } else { + setIsLoading(true); + } + try { + const { data, paging: pagingObj } = await getListKnowledgePages({ + fields: getKnowledgePageFields(), + after, + limit: PAGE_SIZE_MEDIUM, + }); + setKnowledgePages((prev) => + uniqBy(after ? [...prev, ...data] : data, 'id') + ); + setPaging(pagingObj); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setIsLoading(false); + setIsLoadingMore(false); + } + }; + + const addArticleKnowledgePage = async () => { + try { + setIsCreatingNewPage(true); + const instanceName = `${PageType.ARTICLE}_${cryptoRandomString({ + length: 8, + type: 'alphanumeric', + })}`; + + const data: CreateKnowledgePage = { + name: instanceName, + displayName: '', + description: '', + pageType: PageType.ARTICLE, + page: { + publicationDate: new Date(), + relatedArticles: [], + }, + owners: [ + { + type: 'user', + id: USERId, + }, + ], + }; + const response = await postKnowledgePage(data); + getResourceLimit('knowledgeCenter', true, true); + navigate({ + pathname: getKnowledgePagePath(response.fullyQualifiedName), + hash: CREATE_PAGE_HASH, + }); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setIsCreatingNewPage(false); + } + }; + const addQuickLinkKnowledgePage = async ( + formData: QuickLinkFormModalFormData + ) => { + try { + const tags = [ + ...(formData.tags ?? []), + ...(formData.glossaryTerms ?? []), + ]; + + const data: CreateKnowledgePage = { + name: `${PageType.QUICK_LINK}_${cryptoRandomString({ + length: 8, + type: 'alphanumeric', + })}`, + displayName: formData.displayName ?? '', + description: formData.description, + pageType: PageType.QUICK_LINK, + page: { + url: formData.url, + }, + owners: [ + { + type: 'user', + id: USERId, + }, + ], + tags, + relatedEntities: formData?.relatedEntities, + }; + const response = await postKnowledgePage(data); + setKnowledgePages((prevPages) => [response, ...prevPages]); + setRefreshTagsCategory(true); + } catch (error) { + showErrorToast(error as AxiosError); + } + }; + + const updateVoteHandler = async (data: VotingDataProps, id: string) => { + try { + const { entity } = await updateKnowledgePageVote(id, data); + + setKnowledgePages((prevPages) => + map(prevPages, (page) => { + if (page.id === entity.id) { + return { ...page, votes: entity.votes }; + } + + return page; + }) + ); + } catch (error) { + showErrorToast(error as AxiosError); + } + }; + + const followKnowledgePageHandler = async (knowledgePageId: string) => { + try { + const res = await followKnowledgePage(knowledgePageId, USERId); + const { newValue } = res.changeDescription.fieldsAdded[0]; + setKnowledgePages((prevPages) => + map(prevPages, (page) => { + if (page.id === knowledgePageId) { + return { + ...page, + followers: [...(page?.followers ?? []), ...newValue], + }; + } + + return page; + }) + ); + setRefreshBookMarkWidget(true); + } catch (error) { + showErrorToast(error as AxiosError); + } + }; + + const unFollowKnowledgePageHandler = async (knowledgePageId: string) => { + try { + const res = await unFollowKnowledgePage(knowledgePageId, USERId); + const { oldValue } = res.changeDescription.fieldsDeleted[0]; + + setKnowledgePages((prevPages) => + map(prevPages, (page) => { + if (page.id === knowledgePageId) { + return { + ...page, + followers: (page?.followers ?? []).filter( + (follower) => follower.id !== oldValue[0].id + ), + }; + } + + return page; + }) + ); + setRefreshBookMarkWidget(true); + } catch (error) { + showErrorToast(error as AxiosError); + } + }; + + const handleDelete = (id: string | string[]) => { + setKnowledgePages((prevPages) => + prevPages.filter((page) => { + if (Array.isArray(id)) { + return !id.includes(page.id); + } + + return page.id !== id; + }) + ); + }; + + const hasViewPermission = useMemo( + () => permissions.ViewAll || permissions.ViewBasic, + [permissions] + ); + + useEffect(() => { + if (hasViewPermission) { + fetchKnowledgePages(); + } else { + setIsLoading(false); + } + }, [hasViewPermission]); + + /** + * Handle infinite scrolling + */ + useEffect(() => { + const after = paging.after; + if (isInView && after && !isLoadingMore && hasViewPermission) { + fetchKnowledgePages(after); + } + }, [isInView, paging, isLoadingMore, hasViewPermission]); + + const items: MenuProps['items'] = [ + { + label: t('label.article'), + key: PageType.ARTICLE, + onClick: addArticleKnowledgePage, + }, + { + label: t('label.quick-link'), + key: PageType.QUICK_LINK, + onClick: () => setShowAddLinkModal(true), + }, + ]; + + const getRightPanelElement = useCallback( + () => ( + setShowAddLinkModal(true)} + onRefreshBookMarkWidget={handleRefreshBookMarkWidget} + onRefreshTagsCategory={handleRefreshTagsCategory} + /> + ), + [permissions, refreshBookMarkWidget, refreshTagsCategory] + ); + + const addQuickLinkModalElement = useMemo( + () => + showAddLinkModal && ( + setShowAddLinkModal(false)} + onSave={(data) => { + addQuickLinkKnowledgePage(data); + setShowAddLinkModal(false); + }} + /> + ), + [showAddLinkModal] + ); + + useEffect(() => { + onPageChange({ + title: t('label.knowledge-center'), + rightPanel: getRightPanelElement(), + data: undefined, + header: null, + }); + }, [permissions, refreshBookMarkWidget, refreshTagsCategory]); + + useImperativeHandle(ref, () => ({ + onPageDelete: handleDelete, + addKnowledgePage: (knowledgePage: KnowledgePage) => + setKnowledgePages((prevPages) => [knowledgePage, ...prevPages]), + })); + + if (isLoading || isCreatingNewPage) { + return ( + + {Array.from({ length: 4 }).map(() => ( + + + + + + + + + + + + + + + + + + + + + + ))} + + ); + } + + if (!hasViewPermission) { + return ( + + ); + } + + if (!isLoading && isEmpty(knowledgePages)) { + return ( + + } + type={ERROR_PLACEHOLDER_TYPE.CUSTOM}> +
+ +
+ + {t('message.adding-new-entity-is-easy-just-give-it-a-spin', { + entity: t('label.knowledge-page'), + })} + + + + + } + values={{ + doc: t('label.doc-plural-lowercase'), + }} + /> + + + {permissions.Create && ( + + + + + + )} +
+ +
+ {addQuickLinkModalElement} + + ); + } + + return ( + <> + + {map(knowledgePages, (knowledgePage) => ( + + + + ))} + + {isLoadingMore ? : null} +
} + style={{ height: '2px' }} + /> + {addQuickLinkModalElement} + + ); +}); + +export default KnowledgePageListComponent; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageListComponent/knowledge-page-list.less b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageListComponent/knowledge-page-list.less new file mode 100644 index 000000000000..1b6b40e8c9f0 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageListComponent/knowledge-page-list.less @@ -0,0 +1,17 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +.knowledge-center-list-right-panel { + .ant-card-head { + padding: 0 20px; + } +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageListRightPanel/KnowledgePageListRightPanel.mock.ts b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageListRightPanel/KnowledgePageListRightPanel.mock.ts new file mode 100644 index 000000000000..00079848b11e --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageListRightPanel/KnowledgePageListRightPanel.mock.ts @@ -0,0 +1,77 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export const QUICK_LINK_MOCK_DATA = { + id: '1e62e2f6-7441-4c1b-bd15-15a23af23181', + name: 'QuickLink_38ZC2KXX', + fullyQualifiedName: 'QuickLink_38ZC2KXX', + displayName: 'The Six Pillars of OpenMetadata', + description: + 'OpenMetadata is an all-in-one platform for data discovery, lineage, data quality, observability, governance, and team collaboration. Powered by a centralized metadata store based on Open Metadata Standards/APIs, supporting connectors to a wide range of data services, OpenMetadata enables end-to-end metadata management, giving you the freedom to unlock the value of your data assets.\n\nOpenMetadata is a complete package for data teams to break down team silos, share data assets from multiple sources securely, collaborate around data, and build a documentation-first data culture in the organization.\n\nLet us learn more about the six pillars of OpenMetadata that helps maintain its ground as the best in effective metadata management:\n\n1. Data Discovery,\n \n2. Data Collaboration,\n \n3. Data Quality and Profiler,\n \n4. Data Lineage,\n \n5. Data insights, and\n \n6. [**Data Governance**](https://docs.open-metadata.org/v1.1.x/how-to-guides/openmetadata/data-governance).', + version: 0.2, + updatedAt: 1695188836184, + updatedBy: 'admin', + href: 'http://localhost:8585/api/v1/knowledgeCenter/1e62e2f6-7441-4c1b-bd15-15a23af23181', + changeDescription: { + fieldsAdded: [], + fieldsUpdated: [ + { + name: 'description', + oldValue: 'The Six Pillars of OpenMetadata', + newValue: + 'OpenMetadata is an all-in-one platform for data discovery, lineage, data quality, observability, governance, and team collaboration. Powered by a centralized metadata store based on Open Metadata Standards/APIs, supporting connectors to a wide range of data services, OpenMetadata enables end-to-end metadata management, giving you the freedom to unlock the value of your data assets.\n\nOpenMetadata is a complete package for data teams to break down team silos, share data assets from multiple sources securely, collaborate around data, and build a documentation-first data culture in the organization.\n\nLet us learn more about the six pillars of OpenMetadata that helps maintain its ground as the best in effective metadata management:\n\n1. Data Discovery,\n \n2. Data Collaboration,\n \n3. Data Quality and Profiler,\n \n4. Data Lineage,\n \n5. Data insights, and\n \n6. [**Data Governance**](https://docs.open-metadata.org/v1.1.x/how-to-guides/openmetadata/data-governance).', + }, + ], + fieldsDeleted: [], + previousVersion: 0.1, + }, + owner: { + id: '9304f330-2e9a-4513-883b-c939e29683a8', + type: 'user', + name: 'admin', + fullyQualifiedName: 'admin', + deleted: false, + href: 'http://localhost:8585/api/v1/users/9304f330-2e9a-4513-883b-c939e29683a8', + }, + tags: [], + pageType: 'QuickLink', + page: { + url: 'https://docs.open-metadata.org/v1.1.x/how-to-guides/openmetadata', + }, + deleted: false, +}; + +export const MOCK_KNOWLEDGE_CENTER_TAG = { + id: 'ea9dd24d-96de-490e-a62a-54a6ab61b1ae', + name: 'application-customization', + displayName: 'Application Customisation', + fullyQualifiedName: 'KnowledgeCenter.application-customization', + description: 'Application Customisation', + classification: { + id: '569009a1-b478-4142-b06a-b174c197e24a', + type: 'classification', + name: 'KnowledgeCenter', + fullyQualifiedName: 'KnowledgeCenter', + description: + 'Category describing the knowledge center articles or quickLinks. E.g., How-To-Guide, Quick-Link etc.', + deleted: false, + href: 'http://localhost:8585/api/v1/classifications/569009a1-b478-4142-b06a-b174c197e24a', + }, + version: 0.1, + updatedAt: 1695622161800, + updatedBy: 'admin', + href: 'http://localhost:8585/api/v1/tags/ea9dd24d-96de-490e-a62a-54a6ab61b1ae', + deprecated: false, + deleted: false, + provider: 'user', + mutuallyExclusive: false, +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageListRightPanel/KnowledgePageListRightPanel.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageListRightPanel/KnowledgePageListRightPanel.test.tsx new file mode 100644 index 000000000000..3264f34f958c --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageListRightPanel/KnowledgePageListRightPanel.test.tsx @@ -0,0 +1,137 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + act, + fireEvent, + render, + screen, + waitForElementToBeRemoved, +} from '@testing-library/react'; + +import { OperationPermission } from 'context/PermissionProvider/PermissionProvider.interface'; +import { MemoryRouter } from 'react-router-dom'; +import { getTags } from 'rest/tagAPI'; +import { getListKnowledgePages } from '../../../rest/knowledgeCenterAPI'; +import KnowledgePageListRightPanel, { + KnowledgePageListRightPanelProps, +} from './KnowledgePageListRightPanel'; +import { + MOCK_KNOWLEDGE_CENTER_TAG, + QUICK_LINK_MOCK_DATA, +} from './KnowledgePageListRightPanel.mock'; + +const mockAdd = jest.fn(); + +const mockPermissions = { + Create: true, + Delete: true, + ViewAll: true, + EditAll: true, +} as OperationPermission; + +jest.mock('../../../rest/knowledgeCenterAPI', () => ({ + getListKnowledgePages: jest + .fn() + .mockImplementation(() => Promise.resolve({ data: [] })), +})); +jest.mock('rest/tagAPI', () => ({ + getTags: jest + .fn() + .mockImplementation(() => + Promise.resolve({ data: [MOCK_KNOWLEDGE_CENTER_TAG] }) + ), +})); + +jest.mock('../BookMarkWidget/BookMarkWidget', () => + jest.fn().mockReturnValue(
BookMark Widget
) +); + +jest.mock('../../../utils/KnowledgePageUtils', () => ({ + ...jest.requireActual('../../../utils/KnowledgePageUtils'), + getKnowledgeCenterRecentViewed: jest.fn().mockImplementation(() => []), +})); + +const mockProps: KnowledgePageListRightPanelProps = { + onAdd: mockAdd, + permissions: mockPermissions, + refreshBookMarkWidget: false, + onRefreshBookMarkWidget: jest.fn(), + refreshTagsCategory: false, + onRefreshTagsCategory: jest.fn(), +}; + +describe('KnowledgePageListRightPanel', () => { + it('Should render the error placeholder if no data', async () => { + (getTags as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ data: [] }) + ); + + render(, { + wrapper: MemoryRouter, + }); + + await waitForElementToBeRemoved(() => screen.getByTestId('loader')); + + expect( + screen.getByTestId('create-error-placeholder-label.quick-link-plural') + ).toBeInTheDocument(); + + const addQuickLinkBtn = screen.getByTestId('add-quick-link'); + + expect(addQuickLinkBtn).toBeInTheDocument(); + + // add button should call the onAdd callback + + fireEvent.click(addQuickLinkBtn); + + expect(mockAdd).toHaveBeenCalled(); + }); + + it('Should render the no recent view placeholder if no recently viewed data', async () => { + (getListKnowledgePages as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ data: [QUICK_LINK_MOCK_DATA] }) + ); + + render(, { + wrapper: MemoryRouter, + }); + + await waitForElementToBeRemoved(() => screen.getByTestId('loader')); + + expect( + screen.getByText('message.no-recently-viewed-date') + ).toBeInTheDocument(); + }); + + it('Should render the recently viewed data', async () => { + (getListKnowledgePages as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ + data: [ + { ...QUICK_LINK_MOCK_DATA, fullyQualifiedName: 'QuickLink_Testing' }, + ], + }) + ); + + await act(async () => { + render(, { + wrapper: MemoryRouter, + }); + }); + + expect( + screen.getByTestId( + `tag-category-KnowledgeCenter.application-customization-${QUICK_LINK_MOCK_DATA.displayName}` + ) + ).toBeInTheDocument(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageListRightPanel/KnowledgePageListRightPanel.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageListRightPanel/KnowledgePageListRightPanel.tsx new file mode 100644 index 000000000000..e5d0178880b7 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageListRightPanel/KnowledgePageListRightPanel.tsx @@ -0,0 +1,255 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Skeleton, Space, Typography } from 'antd'; +import { AxiosError } from 'axios'; +import { groupBy, isEmpty, map, startCase, uniqueId } from 'lodash'; +import { FC, useEffect, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { ReactComponent as IconArticle } from '../../../assets/svg/ic-articles.svg'; +import { ReactComponent as EyeIcon } from '../../../assets/svg/ic-eye.svg'; +import ErrorPlaceHolder from '../../../components/common/ErrorWithPlaceholder/ErrorPlaceHolder'; +import ExpandableCard from '../../../components/common/ExpandableCard/ExpandableCard'; +import Loader from '../../../components/common/Loader/Loader'; +import { FQN_SEPARATOR_CHAR } from '../../../constants/char.constants'; +import { + KNOWLEDGE_CENTER_CLASSIFICATION, + PAGE_SIZE_MEDIUM, +} from '../../../constants/constants'; +import { OperationPermission } from '../../../context/PermissionProvider/PermissionProvider.interface'; +import { ERROR_PLACEHOLDER_TYPE, SIZE } from '../../../enums/common.enum'; +import { TabSpecificField } from '../../../enums/entity.enum'; +import { Tag } from '../../../generated/entity/classification/tag'; +import { useCurrentUserPreferences } from '../../../hooks/currentUserStore/useCurrentUserStore'; +import { + KnowledgePage, + RecentlyViewedQuickLinks, +} from '../../../interface/knowledge-center.interface'; +import { getListKnowledgePages } from '../../../rest/knowledgeCenterAPI'; +import { getTags } from '../../../rest/tagAPI'; +import { getLink } from '../../../utils/KnowledgePageUtils'; +import { showErrorToast } from '../../../utils/ToastUtils'; +import BookMarkWidget from '../BookMarkWidget/BookMarkWidget'; + +export interface KnowledgePageListRightPanelProps { + onAdd: () => void; + permissions: OperationPermission; + refreshBookMarkWidget: boolean; + refreshTagsCategory: boolean; + onRefreshBookMarkWidget: (value: boolean) => void; + onRefreshTagsCategory: (value: boolean) => void; +} + +type QuickLinkTuple = [string, KnowledgePage[]]; + +type QuickLinkByTag = Array; + +const KnowledgePageListRightPanel: FC = ({ + onAdd, + permissions, + refreshBookMarkWidget, + refreshTagsCategory, + onRefreshTagsCategory, + onRefreshBookMarkWidget, +}) => { + const { t } = useTranslation(); + const [quickLinksByTag, setQuickLinksByTag] = useState([]); + const [knowledgeCenterTags, setKnowledgeCenterTags] = useState([]); + const [isLoading, setIsLoading] = useState(true); + const { + preferences: { recentlyViewedQuickLinks }, + } = useCurrentUserPreferences(); + const recentlyViewed = + recentlyViewedQuickLinks as unknown as RecentlyViewedQuickLinks['data']; + + const fetchQuickLinkByTag = async (tagFqn: string) => { + try { + const { data } = await getListKnowledgePages({ + fields: `${TabSpecificField.OWNERS},${TabSpecificField.TAGS}`, + tagFQN: tagFqn, + }); + + return data; + } catch { + return []; + } + }; + + const fetchKnowledgeCenterTags = async () => { + setIsLoading(true); + try { + const { data } = await getTags({ + parent: KNOWLEDGE_CENTER_CLASSIFICATION, + limit: PAGE_SIZE_MEDIUM, + }); + + setKnowledgeCenterTags(data); + + const tagsObj = groupBy(data, 'fullyQualifiedName'); + + // Fetch all quick links concurrently and set state only once + const quickLinkPromises = Object.keys(tagsObj).map(async (tag) => { + const quickLinks = await fetchQuickLinkByTag(tag); + + return [tag, quickLinks] as QuickLinkTuple; + }); + + const allQuickLinks = await Promise.all(quickLinkPromises); + setQuickLinksByTag(allQuickLinks); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setIsLoading(false); + } + }; + + const handleRefreshTagsCategory = async () => { + // reset quick links + setQuickLinksByTag([]); + try { + const tagsObj = groupBy(knowledgeCenterTags, 'fullyQualifiedName'); + + // Fetch all quick links concurrently and set state only once + const quickLinkPromises = Object.keys(tagsObj).map(async (tag) => { + const quickLinks = await fetchQuickLinkByTag(tag); + + return [tag, quickLinks] as QuickLinkTuple; + }); + + const allQuickLinks = await Promise.all(quickLinkPromises); + setQuickLinksByTag(allQuickLinks); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + onRefreshTagsCategory(false); + } + }; + + useEffect(() => { + fetchKnowledgeCenterTags(); + }, []); + + useEffect(() => { + if (refreshTagsCategory) { + handleRefreshTagsCategory(); + } + }, [refreshTagsCategory, knowledgeCenterTags]); + + if (isLoading) { + return ( +
+ {Array.from({ length: 3 }).map(() => ( +
+ + + + +
+ ))} +
+ ); + } + + if (!isLoading && isEmpty(quickLinksByTag) && !refreshTagsCategory) { + return ( + + ); + } + + const recentViewsElement = map(recentlyViewed, (page) => + getLink(page, 'recent-viewed') + ); + + return ( +
+ + + + + + {t('label.recently-viewed')} + +
+ ), + }}> + {isEmpty(recentlyViewed) ? ( + t('message.no-recently-viewed-date') + ) : ( + + {recentViewsElement} + + )} + + + {refreshTagsCategory ? ( + + ) : ( + <> + {map(quickLinksByTag, ([tagFqn, uniqueLinks]) => { + if (isEmpty(uniqueLinks)) { + return null; + } + + return ( + + + + {startCase(tagFqn.split(FQN_SEPARATOR_CHAR)[1])} + +
+ ), + }}> + + {map(uniqueLinks, (matchedQuickLink) => + getLink(matchedQuickLink, `tag-category-${tagFqn}`) + )} + + + ); + })} + + )} +
+ ); +}; + +export default KnowledgePageListRightPanel; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageOwners/KnowledgePageOwners.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageOwners/KnowledgePageOwners.tsx new file mode 100644 index 000000000000..0499fb8544fa --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageOwners/KnowledgePageOwners.tsx @@ -0,0 +1,100 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Button, Space, Tooltip, Typography } from 'antd'; +import classNames from 'classnames'; +import { FC } from 'react'; +import { useTranslation } from 'react-i18next'; +import { ReactComponent as EditIcon } from '../../../assets/svg/edit-new.svg'; +import { ReactComponent as PlusIcon } from '../../../assets/svg/plus-primary.svg'; +import TagButton from '../../../components/common/TagButton/TagButton.component'; +import { UserTeamSelectableList } from '../../../components/common/UserTeamSelectableList/UserTeamSelectableList.component'; +import { DE_ACTIVE_COLOR } from '../../../constants/constants'; +import { OperationPermission } from '../../../context/PermissionProvider/PermissionProvider.interface'; +import { TabSpecificField } from '../../../enums/entity.enum'; +import { Glossary } from '../../../generated/entity/data/glossary'; +import { EntityReference } from '../../../generated/entity/data/page'; +import { KnowledgePage } from '../../../interface/knowledge-center.interface'; +import { getOwnerVersionLabel } from '../../../utils/EntityVersionUtils'; + +interface KnowledgePageOwnersProps { + permissions: OperationPermission; + knowledgePage?: KnowledgePage; + onOwnerUpdate: (updatedOwners?: EntityReference[]) => Promise; +} + +const KnowledgePageOwners: FC = ({ + knowledgePage, + permissions, + onOwnerUpdate, +}) => { + const { t } = useTranslation(); + const hasOwners = knowledgePage?.owners && knowledgePage?.owners.length > 0; + const canEditOwners = permissions.EditOwners || permissions.EditAll; + const owners = knowledgePage?.owners ?? []; + + return ( + +
+ + {t('label.owner-plural')} + + {canEditOwners && ( + + {hasOwners ? ( + +
+ {hasOwners && ( + + {getOwnerVersionLabel( + knowledgePage as unknown as Glossary, + false, + TabSpecificField.OWNERS, + canEditOwners + )} + + )} + + ); +}; + +export default KnowledgePageOwners; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageSummary/KnowledgePageSummary.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageSummary/KnowledgePageSummary.test.tsx new file mode 100644 index 000000000000..aa0ebd851481 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageSummary/KnowledgePageSummary.test.tsx @@ -0,0 +1,85 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { render, screen } from '@testing-library/react'; +import { KnowledgePage, PageType } from 'interface/knowledge-center.interface'; +import { MOCK_KNOWLEDGE_PAGE_DATA } from 'pages/KnowledgePage/KnowledgePage.mock'; +import { MemoryRouter } from 'react-router-dom'; +import KnowledgePageSummary from './KnowledgePageSummary'; + +jest.mock('components/common/OwnerLabel/OwnerLabel.component', () => ({ + OwnerLabel: jest.fn().mockImplementation(() => { + return
OwnerLabel
; + }), +})); +jest.mock( + 'components/common/SummaryTagsDescription/SummaryTagsDescription.component', + () => + jest.fn().mockImplementation(() => { + return
SummaryTagsDescription
; + }) +); +jest.mock( + 'components/Explore/EntitySummaryPanel/CommonEntitySummaryInfo/CommonEntitySummaryInfo', + () => + jest.fn().mockImplementation(() => { + return
CommonEntitySummaryInfo
; + }) +); +jest.mock( + 'components/common/Skeleton/SummaryPanelSkeleton/SummaryPanelSkeleton.component', + () => + jest.fn().mockImplementation(({ children }) => { + return
{children}
; + }) +); + +jest.mock('utils/EntityUtils', () => ({ + DRAWER_NAVIGATION_OPTIONS: { + explore: 'Explore', + lineage: 'Lineage', + }, +})); + +const mockData = { ...MOCK_KNOWLEDGE_PAGE_DATA } as unknown as KnowledgePage; + +describe('KnowledgePageSummary', () => { + it('should render correctly', async () => { + render(, { + wrapper: MemoryRouter, + }); + + expect(screen.getByText('CommonEntitySummaryInfo')).toBeInTheDocument(); + expect(screen.getByText('SummaryTagsDescription')).toBeInTheDocument(); + }); + + it('should render correctly with quick link', async () => { + render( + , + { + wrapper: MemoryRouter, + } + ); + + expect(screen.getByText('CommonEntitySummaryInfo')).toBeInTheDocument(); + expect(screen.getByText('SummaryTagsDescription')).toBeInTheDocument(); + expect(screen.getByTestId('quick-link-data')).toBeInTheDocument(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageSummary/KnowledgePageSummary.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageSummary/KnowledgePageSummary.tsx new file mode 100644 index 000000000000..15a8d247e12e --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageSummary/KnowledgePageSummary.tsx @@ -0,0 +1,110 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Col, Divider, Row, Typography } from 'antd'; +import { isEmpty } from 'lodash'; +import { OwnerLabel } from '../../../components/common/OwnerLabel/OwnerLabel.component'; +import SummaryTagsDescription from '../../../components/common/SummaryTagsDescription/SummaryTagsDescription.component'; +import CommonEntitySummaryInfo from '../../../components/Explore/EntitySummaryPanel/CommonEntitySummaryInfo/CommonEntitySummaryInfo'; +import { EntityUnion } from '../../../components/Explore/ExplorePage.interface'; +import { + KnowledgePage, + PageType, + QuickLink, +} from '../../../interface/knowledge-center.interface'; + +import { useMemo } from 'react'; +import { Link } from 'react-router-dom'; +import SummaryPanelSkeleton from '../../../components/common/Skeleton/SummaryPanelSkeleton/SummaryPanelSkeleton.component'; +import { DRAWER_NAVIGATION_OPTIONS } from '../../../utils/EntityUtils'; +import i18n, { t } from '../../../utils/i18next/LocalUtil'; +import RelatedDataAssets from '../RelatedDataAssets/RelatedDataAssets'; + +const KnowledgePageSummary = ({ + entityDetails, +}: { + entityDetails: KnowledgePage; +}) => { + const entityInfo = useMemo(() => { + const owners = entityDetails?.owners ?? []; + + return [ + { + name: i18n.t('label.owner-plural'), + value: , + }, + ]; + }, [entityDetails]); + + const isQuickLink = entityDetails?.pageType === PageType.QUICK_LINK; + + const quickLinkData = isQuickLink + ? (entityDetails.page as QuickLink) + : undefined; + + return ( + + <> + + + + + + {quickLinkData?.url && ( + <> + + + + {t('label.link')} + + + + + {quickLinkData.url} + + + + + + )} + + + + {/* read only data assets */} + + + + + + + + ); +}; + +export default KnowledgePageSummary; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageVersion/KnowledgePageVersion.mock.ts b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageVersion/KnowledgePageVersion.mock.ts new file mode 100644 index 000000000000..0b0beec50329 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageVersion/KnowledgePageVersion.mock.ts @@ -0,0 +1,95 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + Article, + KnowledgePage, +} from '../../../interface/knowledge-center.interface'; + +/* eslint-disable max-len */ +export const MOCK_KNOWLEDGE_PAGE_VERSION_DATA = { + id: '8e6427d6-98cc-4334-b2f2-15fb62bde887', + name: 'Article_oRKYYTCu', + fullyQualifiedName: 'Article_oRKYYTCu', + href: '', + displayName: 'OpenMetadata 1.1.0 Release UI', + description: + 'Less than two months have passed since our exciting OpenMetadata 1.0 Release, and we’re thrilled to announce the completion of Release 1.1 already! The OpenMetadata community thrives on pushing our limits; this latest release is a testament to it. Prepare to be amazed as we unveil a complete UI overhaul, meticulously designed to elevate the user experience across the entire platform. But that’s not all! We’ve also introduced four new connectors, implemented advanced PII masking, and significantly enhanced lineage parsing capabilities, just to name a few of the numerous features we’ve packed into this release. Stay tuned for an exceptional OpenMetadata experience like never before!\n\nIn the upcoming 1.2 Release of OpenMetadata, we are thrilled to introduce exclusive new features specifically tailored for Collate SaaS. You can review Collate’s roadmap here and be as excited as we are 🚀\n\nCommunity Updates\n-----------------\n\nThanks to the incredible OpenMetadata Community, our growth and activity have skyrocketed. Slack is buzzing with constant engagement, and we truly appreciate your code contributions, feedback, and feature requests. Our webinars are attracting more attendees, and the June community meeting was extra special, thanks to our first Community Spotlight: Gaétan Soulas from Solocal!\n\nWe are excited about our soaring community numbers!\n\nCrossed 2400+ GitHub stars (+200 stars since the previous release)\n\nThe Slack community reached 3200+ members (+500 since the previous release)\n\n168 Open-source GitHub developers (+8 since the previous release)\n\nMerged 526 Commits into the 1.1 Release\n\nOpenMetadata 1.1 Release Highlights\n\nUI Overhaul\n-----------\n\nThis release marks a significant milestone for the OpenMetadata platform, bringing many UI changes that are among the most substantial since the start of the project in 2021.\n\nOur primary focus is to simplify the overall experience for users while building upon our already exceptional UI. We are incredibly excited to share these changes with you as they further enhance the platform’s discovery, collaboration, and data quality experience.\n\nRefined Landing Page\n--------------------', + version: 1.1, + updatedAt: 1695189186624, + updatedBy: 'admin', + changeDescription: { + fieldsAdded: [], + fieldsUpdated: [ + { + name: 'description', + oldValue: + 'Less than two months have passed since our exciting OpenMetadata 1.0 Release, and we’re thrilled to announce the completion of Release 1.1 already! The OpenMetadata community thrives on pushing our limits; this latest release is a testament to it. Prepare to be amazed as we unveil a complete UI overhaul, meticulously designed to elevate the user experience across the entire platform. But that’s not all! We’ve also introduced four new connectors, implemented advanced PII masking, and significantly enhanced lineage parsing capabilities, just to name a few of the numerous features we’ve packed into this release. Stay tuned for an exceptional OpenMetadata experience like never before!\n\nIn the upcoming 1.2 Release of OpenMetadata, we are thrilled to introduce exclusive new features specifically tailored for Collate SaaS. You can review Collate’s roadmap here and be as excited as we are 🚀\n\nCommunity Updates\n-----------------\n\nThanks to the incredible OpenMetadata Community, our growth and activity have skyrocketed. Slack is buzzing with constant engagement, and we truly appreciate your code contributions, feedback, and feature requests. Our webinars are attracting more attendees, and the June community meeting was extra special, thanks to our first Community Spotlight: Gaétan Soulas from Solocal!\n\nWe are excited about our soaring community numbers!\n\nCrossed 2400+ GitHub stars (+200 stars since the previous release)\n\nThe Slack community reached 3200+ members (+500 since the previous release)\n\n168 Open-source GitHub developers (+8 since the previous release)\n\nMerged 526 Commits into the 1.1 Release\n\nOpenMetadata 1.1 Release Highlights\n\nUI Overhaul\n-----------\n\nThis release marks a significant milestone for the OpenMetadata platform, bringing many UI changes that are among the most substantial since the start of the project in 2021.\n\nOur primary focus is to simplify the overall experience for users while building upon our already exceptional UI. We are incredibly excited to share these changes with you as they further enhance the platform’s discovery, collaboration, and data quality experience.\n\nRefined Landing Page', + newValue: + 'Less than two months have passed since our exciting OpenMetadata 1.0 Release, and we’re thrilled to announce the completion of Release 1.1 already! The OpenMetadata community thrives on pushing our limits; this latest release is a testament to it. Prepare to be amazed as we unveil a complete UI overhaul, meticulously designed to elevate the user experience across the entire platform. But that’s not all! We’ve also introduced four new connectors, implemented advanced PII masking, and significantly enhanced lineage parsing capabilities, just to name a few of the numerous features we’ve packed into this release. Stay tuned for an exceptional OpenMetadata experience like never before!\n\nIn the upcoming 1.2 Release of OpenMetadata, we are thrilled to introduce exclusive new features specifically tailored for Collate SaaS. You can review Collate’s roadmap here and be as excited as we are 🚀\n\nCommunity Updates\n-----------------\n\nThanks to the incredible OpenMetadata Community, our growth and activity have skyrocketed. Slack is buzzing with constant engagement, and we truly appreciate your code contributions, feedback, and feature requests. Our webinars are attracting more attendees, and the June community meeting was extra special, thanks to our first Community Spotlight: Gaétan Soulas from Solocal!\n\nWe are excited about our soaring community numbers!\n\nCrossed 2400+ GitHub stars (+200 stars since the previous release)\n\nThe Slack community reached 3200+ members (+500 since the previous release)\n\n168 Open-source GitHub developers (+8 since the previous release)\n\nMerged 526 Commits into the 1.1 Release\n\nOpenMetadata 1.1 Release Highlights\n\nUI Overhaul\n-----------\n\nThis release marks a significant milestone for the OpenMetadata platform, bringing many UI changes that are among the most substantial since the start of the project in 2021.\n\nOur primary focus is to simplify the overall experience for users while building upon our already exceptional UI. We are incredibly excited to share these changes with you as they further enhance the platform’s discovery, collaboration, and data quality experience.\n\nRefined Landing Page\n--------------------', + }, + ], + fieldsDeleted: [], + previousVersion: 1, + }, + owners: [ + { + id: '9304f330-2e9a-4513-883b-c939e29683a8', + type: 'user', + name: 'admin', + fullyQualifiedName: 'admin', + deleted: false, + }, + ], + followers: [ + { + id: '9304f330-2e9a-4513-883b-c939e29683a8', + type: 'user', + name: 'admin', + fullyQualifiedName: 'admin', + deleted: false, + }, + ], + votes: { + upVotes: 1, + downVotes: 0, + upVoters: [ + { + id: '9304f330-2e9a-4513-883b-c939e29683a8', + type: 'user', + name: 'admin', + fullyQualifiedName: 'admin', + deleted: false, + }, + ], + downVoters: [], + }, + tags: [], + pageType: 'Article', + page: { + publicationDate: 1726823190797, + relatedArticles: [], + } as unknown as Article, + relatedEntities: [ + { + id: '8d5ccfe5-b9d7-4f4f-8927-8e775bf77eb3', + type: 'team', + name: 'Organization', + fullyQualifiedName: 'Organization', + description: + 'Organization under which all the other team hierarchy is created', + displayName: 'Organization', + deleted: false, + }, + ], + deleted: false, +} as KnowledgePage; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageVersion/KnowledgePageVersion.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageVersion/KnowledgePageVersion.test.tsx new file mode 100644 index 000000000000..7cb542cfe089 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageVersion/KnowledgePageVersion.test.tsx @@ -0,0 +1,127 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { Settings } from 'luxon'; +import KnowledgePageVersion from './KnowledgePageVersion'; +import { MOCK_KNOWLEDGE_PAGE_VERSION_DATA } from './KnowledgePageVersion.mock'; + +const systemLocale = Settings.defaultLocale; +const systemZoneName = Settings.defaultZone; + +const mockPush = jest.fn(); + +jest.mock('utils/EntityUtils', () => ({ + getEntityName: jest.fn(), +})); + +jest.mock('components/common/OwnerLabel/OwnerLabel.component', () => ({ + OwnerLabel: jest.fn().mockImplementation(() => { + return
OwnerLabel
; + }), +})); + +jest.mock('components/Tag/TagsContainerV2/TagsContainerV2', () => + jest + .fn() + .mockImplementation(() => ( +
TagsContainerV2
+ )) +); + +jest.mock('components/common/Loader/Loader', () => + jest.fn().mockImplementation(() =>
Loader
) +); + +jest.mock('react-router-dom', () => ({ + DataNode: jest.fn(), + useNavigate: jest.fn().mockImplementation(() => mockPush), +})); + +jest.mock('components/BlockEditor/BlockEditor', () => { + return jest + .fn() + .mockReturnValue(
Block Editor
); +}); + +jest.mock('components/common/ProfilePicture/ProfilePicture', () => { + return jest.fn().mockReturnValue(
Avatar
); +}); + +const mockProps = { + knowledgePage: MOCK_KNOWLEDGE_PAGE_VERSION_DATA, + loading: false, +}; + +describe('Knowledge page version', () => { + beforeAll(() => { + // Explicitly set locale and time zone to make sure date time manipulations and literal + // results are consistent regardless of where tests are run + Settings.defaultLocale = 'en-US'; + Settings.defaultZone = 'UTC'; + }); + + afterAll(() => { + // Restore locale and time zone + Settings.defaultLocale = systemLocale; + Settings.defaultZone = systemZoneName; + }); + + it('Should render the components', async () => { + render(); + + const header = screen.getByTestId('entity-header-display-name'); + + const ownerName = screen.getByTestId('owner-label'); + + const updateAt = screen.getByTestId('updated-at'); + + const versionButton = screen.getByTestId('version-button'); + + const tagsContainer = screen.getAllByTestId('tags-container'); + + const blockEditor = screen.getByTestId('block-editor'); + + expect(header).toBeInTheDocument(); + expect(ownerName).toBeInTheDocument(); + expect(updateAt).toBeInTheDocument(); + expect(versionButton).toBeInTheDocument(); + expect(tagsContainer).toHaveLength(2); + + expect(blockEditor).toBeInTheDocument(); + + expect(updateAt).toHaveTextContent('Sep 20, 2023'); + expect(versionButton).toHaveTextContent('1.1'); + }); + + it('Should show the loader if loading is true', async () => { + render(); + + expect(screen.getByTestId('loader')).toBeInTheDocument(); + }); + + it('Version button should redirect to entity page', async () => { + render(); + + const versionButton = screen.getByTestId('version-button'); + + expect(versionButton).toHaveTextContent('1.1'); + + fireEvent.click(versionButton); + + await waitFor(() => { + expect(mockPush).toHaveBeenCalledWith( + '/knowledge-center/Article_oRKYYTCu' + ); + }); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageVersion/KnowledgePageVersion.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageVersion/KnowledgePageVersion.tsx new file mode 100644 index 000000000000..b51e462af015 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePageVersion/KnowledgePageVersion.tsx @@ -0,0 +1,222 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Icon from '@ant-design/icons'; +import { Button, Col, Row, Space, Typography } from 'antd'; +import classNames from 'classnames'; +import { diffWordsWithSpace } from 'diff'; +import { isEmpty, map, toString } from 'lodash'; +import { FC, useMemo } from 'react'; +import { useNavigate } from 'react-router-dom'; +import { ReactComponent as VersionIcon } from '../../../assets/svg/ic-version.svg'; +import BlockEditor from '../../../components/BlockEditor/BlockEditor'; +import Loader from '../../../components/common/Loader/Loader'; +import { OwnerLabel } from '../../../components/common/OwnerLabel/OwnerLabel.component'; +import TagsContainerV2 from '../../../components/Tag/TagsContainerV2/TagsContainerV2'; +import { LayoutType } from '../../../components/Tag/TagsViewer/TagsViewer.interface'; +import { EntityField } from '../../../constants/Feeds.constants'; +import { TagSource } from '../../../generated/type/tagLabel'; +import { KnowledgePage } from '../../../interface/knowledge-center.interface'; +import { formatDate } from '../../../utils/date-time/DateTimeUtils'; +import { getEntityName } from '../../../utils/EntityUtils'; +import { + getChangedEntityNewValue, + getChangedEntityOldValue, + getCommonExtraInfoForVersionDetails, + getDiffByFieldName, + getEntityVersionByField, + getEntityVersionTags, +} from '../../../utils/EntityVersionUtils'; +import { VersionEntityTypes } from '../../../utils/EntityVersionUtils.interface'; +import { getFrontEndFormat } from '../../../utils/FeedUtils'; +import i18n from '../../../utils/i18next/LocalUtil'; +import { getKnowledgePagePath } from '../../../utils/KnowledgePageUtils'; +import { stringToHTML } from '../../../utils/StringsUtils'; + +interface KnowledgePageVersionProps { + knowledgePage: KnowledgePage; + loading: boolean; +} + +const KnowledgePageVersion: FC = ({ + knowledgePage, + loading, +}) => { + const { t } = i18n; + const navigate = useNavigate(); + const { version } = useMemo( + () => ({ + entityName: getEntityName(knowledgePage), + version: knowledgePage.version, + }), + [knowledgePage] + ); + + const descriptionDiff = useMemo(() => { + const changeDescription = knowledgePage.changeDescription ?? {}; + const currentDescription = knowledgePage.description; + + const fieldDiff = getDiffByFieldName( + EntityField.DESCRIPTION, + changeDescription + ); + + const oldField = getFrontEndFormat( + toString(getChangedEntityOldValue(fieldDiff)) + ); + const newField = getFrontEndFormat( + toString(getChangedEntityNewValue(fieldDiff)) + ); + + if (isEmpty(newField) && isEmpty(oldField)) { + return currentDescription; + } + + const diffArr = diffWordsWithSpace(oldField, newField); + + const result = map(diffArr, (diff) => { + const value = diff.value.trim().replaceAll('\n', '
'); + + if (diff.added && value) { + return `${value}`; + } + if (diff.removed && value) { + return `${value}`; + } + + if (value) { + return `${value}`; + } + + return ''; + }); + + return result.join(''); + }, [knowledgePage]); + + const tags = useMemo(() => { + return getEntityVersionTags( + knowledgePage as VersionEntityTypes, + knowledgePage.changeDescription ?? {} + ); + }, [knowledgePage]); + + const displayName = useMemo(() => { + return getEntityVersionByField( + knowledgePage.changeDescription ?? {}, + EntityField.DISPLAYNAME, + knowledgePage.displayName + ); + }, [knowledgePage]); + + const { ownerDisplayName, ownerRef } = useMemo( + () => + getCommonExtraInfoForVersionDetails( + knowledgePage.changeDescription ?? {}, + knowledgePage.owners + ), + [knowledgePage] + ); + + const handleVersionClick = () => { + navigate(getKnowledgePagePath(knowledgePage.fullyQualifiedName)); + }; + + if (loading) { + return ; + } + + return ( + + + + + {stringToHTML(displayName || knowledgePage.name)} + + + + + + + + {formatDate(knowledgePage.updatedAt)} + + + + + + + + + + + + + + + + {`${t('label.tag-plural')}:`} + + + + + + + + {`${t('label.glossary-term-plural')}:`} + + + + + + + + + + + ); +}; + +export default KnowledgePageVersion; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePages/KnowledgePages.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePages/KnowledgePages.test.tsx new file mode 100644 index 000000000000..5e1e6d06198d --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePages/KnowledgePages.test.tsx @@ -0,0 +1,204 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { render, screen } from '@testing-library/react'; +import { MOCK_KNOWLEDGE_PAGES } from 'pages/KnowledgePage/KnowledgePage.mock'; +import React from 'react'; +import { MemoryRouter } from 'react-router-dom'; +import { act } from 'react-test-renderer'; +import { getListKnowledgePages } from 'rest/knowledgeCenterAPI'; +import KnowledgePages from './KnowledgePages'; + +const mockProps = { + entityId: '6a7b8c9d0e1f2g3h4i5j6k7l8m9n0o1p2q3r4s5t6u7v8w9x0y1z2', + entityType: 'table', +}; + +jest.mock('rest/knowledgeCenterAPI'); + +jest.mock('components/Customization/GenericProvider/GenericProvider', () => ({ + GenericProvider: ({ children }: { children: React.ReactNode }) => ( + <>{children} + ), + useGenericContext: jest.fn().mockImplementation(() => ({ + data: { id: mockProps.entityId }, + type: mockProps.entityType, + filterWidgets: jest.fn(), + })), +})); + +jest.mock('utils/EntityUtils', () => ({ + getEntityName: jest + .fn() + .mockImplementation( + ({ displayName }: { displayName: string }) => displayName + ), +})); + +describe('KnowledgePages', () => { + it('should render correctly', async () => { + (getListKnowledgePages as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ + data: [...MOCK_KNOWLEDGE_PAGES], + paging: { + total: 10, + }, + }) + ); + await act(async () => { + render(, { wrapper: MemoryRouter }); + }); + + expect(screen.getByTestId('knowledge-pages')).toBeInTheDocument(); + expect(screen.getByText('label.knowledge-center')).toBeInTheDocument(); + + // article page + expect(screen.getByText('Data Collaboration')).toBeInTheDocument(); + expect(screen.getByTestId('article-icon')).toBeInTheDocument(); + + // quick link page + expect(screen.getByText('Blog')).toBeInTheDocument(); + expect(screen.getByTestId('link-icon')).toBeInTheDocument(); + }); + + it('should render the correct page link for quick link', async () => { + (getListKnowledgePages as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ + data: [MOCK_KNOWLEDGE_PAGES[1]], + paging: { + total: 10, + }, + }) + ); + await act(async () => { + render(, { wrapper: MemoryRouter }); + }); + + // quick link page + expect(screen.getByText('Blog')).toBeInTheDocument(); + expect(screen.getByTestId('link-icon')).toBeInTheDocument(); + + const pageLink = screen.getByTestId('page-link'); + + expect(pageLink).toBeInTheDocument(); + expect(pageLink).toHaveAttribute( + 'href', + 'https://blog.open-metadata.org/openmetadata-release-1-2-531f0e3c6d9a' + ); + expect(pageLink).toHaveAttribute('target', '_blank'); + }); + + it('should not render when data is empty', async () => { + (getListKnowledgePages as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ + data: [], + paging: { + total: 0, + }, + }) + ); + await act(async () => { + render(, { wrapper: MemoryRouter }); + }); + + expect(screen.queryByTestId('knowledge-pages')).not.toBeInTheDocument(); + }); + + it('should not render when api fails', async () => { + (getListKnowledgePages as jest.Mock).mockImplementationOnce(() => + Promise.reject({ + data: undefined, + paging: { + total: 0, + }, + }) + ); + await act(async () => { + render(, { wrapper: MemoryRouter }); + }); + + expect(screen.queryByTestId('knowledge-pages')).not.toBeInTheDocument(); + }); + + it('should render view all link if total length is greater than 10', async () => { + (getListKnowledgePages as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ + data: [...MOCK_KNOWLEDGE_PAGES], + paging: { + total: 22, + }, + }) + ); + await act(async () => { + render(, { wrapper: MemoryRouter }); + }); + + expect( + screen.getByTestId('view-all-data-asset-related-articles') + ).toBeInTheDocument(); + }); + + it('view all should have correct link', async () => { + (getListKnowledgePages as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ + data: [...MOCK_KNOWLEDGE_PAGES], + paging: { + total: 22, + }, + }) + ); + await act(async () => { + render(, { wrapper: MemoryRouter }); + }); + + const viewAllLink = screen.getByTestId( + 'view-all-data-asset-related-articles' + ); + + expect(viewAllLink).toHaveAttribute( + 'href', + `/knowledge-center-filter?entityId=${mockProps.entityId}&entityType=${mockProps.entityType}` + ); + }); + + it('should not render view all link if total length is less than or equal to 10', async () => { + (getListKnowledgePages as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ + data: [...MOCK_KNOWLEDGE_PAGES], + paging: { + total: 10, + }, + }) + ); + await act(async () => { + render(, { wrapper: MemoryRouter }); + }); + + expect( + screen.queryByTestId('view-all-data-asset-related-articles') + ).not.toBeInTheDocument(); + }); + + it('should not render and call the api when entityId and entityType are empty', async () => { + mockProps.entityId = ''; + mockProps.entityType = ''; + await act(async () => { + render(, { + wrapper: MemoryRouter, + }); + }); + + expect(getListKnowledgePages).not.toHaveBeenCalled(); + + expect(screen.queryByTestId('knowledge-pages')).not.toBeInTheDocument(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePages/KnowledgePages.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePages/KnowledgePages.tsx new file mode 100644 index 000000000000..d628708b16a8 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePages/KnowledgePages.tsx @@ -0,0 +1,174 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Col, Row, Typography } from 'antd'; +import classNames from 'classnames'; +import { isEmpty, map } from 'lodash'; +import { FC, useEffect, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { Link } from 'react-router-dom'; +import { ReactComponent as IconArticle } from '../../../assets/svg/ic-articles.svg'; +import { ReactComponent as LinkIcon } from '../../../assets/svg/ic-link.svg'; +import ExpandableCard from '../../../components/common/ExpandableCard/ExpandableCard'; +import Loader from '../../../components/common/Loader/Loader'; +import { useGenericContext } from '../../../components/Customization/GenericProvider/GenericProvider'; +import { PAGE_SIZE, ROUTES } from '../../../constants/constants'; +import { DetailPageWidgetKeys } from '../../../enums/CustomizeDetailPage.enum'; +import { Paging } from '../../../generated/type/paging'; +import { + KnowledgePage, + PageType, + QuickLink, +} from '../../../interface/knowledge-center.interface'; +import { getListKnowledgePages } from '../../../rest/knowledgeCenterAPI'; +import { getEntityName } from '../../../utils/EntityUtils'; +import { getKnowledgePagePath } from '../../../utils/KnowledgePageUtils'; + +const KnowledgePages: FC = () => { + const { t } = useTranslation(); + const [isLoading, setIsLoading] = useState(true); + const [knowledgePages, setKnowledgePages] = useState([]); + const [paging, setPaging] = useState({ total: 0 }); + const { + data: { id: entityId = '' } = {}, + type: entityType, + filterWidgets, + } = useGenericContext(); + + const fetchKnowledgePages = async () => { + setIsLoading(true); + try { + const { data, paging } = await getListKnowledgePages({ + entityId, + entityType, + }); + setKnowledgePages(data); + setPaging(paging); + } catch { + // we will not throw error toast here + } finally { + setIsLoading(false); + } + }; + + useEffect(() => { + if (entityId && entityType) { + fetchKnowledgePages(); + } else { + setIsLoading(false); + } + }, [entityId, entityType]); + + const header = ( +
+ + {t('label.knowledge-center')} + + {/* only show view all if length is greater than PAGE_SIZE i.e 10 */} + {paging?.total > PAGE_SIZE && ( + + {t('label.view-all')} + + )} +
+ ); + + const content = ( +
+ {map(knowledgePages, (knowledgePage, index) => { + const isQuickLink = knowledgePage.pageType === PageType.QUICK_LINK; + const quickLink = knowledgePage.page as QuickLink; + + return ( + + + + {isQuickLink ? ( + + ) : ( + + )} + + + + {getEntityName(knowledgePage)} + + + + + ); + })} +
+ ); + + useEffect(() => { + if (!isLoading && isEmpty(knowledgePages)) { + filterWidgets?.([DetailPageWidgetKeys.KNOWLEDGE_ARTICLE]); + } + }, [isLoading, knowledgePages]); + + if (isLoading) { + return ; + } + + if (!isLoading && isEmpty(knowledgePages)) { + return null; + } + + return ( + + {content} + + ); +}; + +export default KnowledgePages; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePagesHierarchy/KnowledgePagesHierarchy.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePagesHierarchy/KnowledgePagesHierarchy.test.tsx new file mode 100644 index 000000000000..4aaa4a4d4a95 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePagesHierarchy/KnowledgePagesHierarchy.test.tsx @@ -0,0 +1,641 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + act, + fireEvent, + render, + screen, + waitFor, +} from '@testing-library/react'; +import { User } from 'generated/entity/teams/user'; +import { MemoryRouter } from 'react-router-dom'; +import { DEFAULT_ENTITY_PERMISSION } from 'utils/PermissionsUtils'; +import KnowledgePagesHierarchy from './KnowledgePagesHierarchy'; + +const PageHierarchy = [ + { + id: '62bec763-522d-4b70-ad85-f487b2f6102f', + pageType: 'Article', + name: 'Article_XJIGIKX2', + description: 'description', + fullyQualifiedName: 'Article_XJIGIKX2', + displayName: 'How to Discover Assets of Interest', + children: [ + { + id: 'ae65ca82-a284-4d3e-9554-dd4c94086613', + pageType: 'Article', + name: 'Article_2p7Z8MAN', + description: '', + fullyQualifiedName: 'Article_2p7Z8MAN', + displayName: 'How to Discover Assets of Interest Child 1', + children: [ + { + id: '27c39402-9691-4776-becd-23a69d06db75', + pageType: 'Article', + name: 'Article_UqfRMCZw', + description: '', + fullyQualifiedName: 'Article_UqfRMCZw', + displayName: 'How to Discover Assets of Interest Child 11', + children: [ + { + id: '838c8ce7-b949-4f58-9a6c-1ef268fc920d', + pageType: 'Article', + name: 'Article_LtyX9wX3', + description: '', + fullyQualifiedName: 'Article_LtyX9wX3', + displayName: 'How to Discover Assets of Interest Child 111', + children: [ + { + id: 'a31ca2ba-e841-4673-bbc2-478f0dea4692', + pageType: 'Article', + name: 'Article_atU2ADuH', + description: '', + fullyQualifiedName: 'Article_atU2ADuH', + displayName: + 'How to Discover Assets of Interest Child 1111', + children: [], + }, + ], + }, + ], + }, + ], + }, + ], + }, + { + id: '45d4f5dd-5946-40d5-abcf-8ef9ff1fa64e', + pageType: 'Article', + name: 'Article_YjCzUcBl', + description: '', + fullyQualifiedName: 'Article_YjCzUcBl', + displayName: 'This is Updated', + children: [ + { + id: '163a3ff2-f853-4040-a180-6e23717b9cd3', + pageType: 'Article', + name: 'Article_mWtepYKg', + description: '', + fullyQualifiedName: 'Article_mWtepYKg', + displayName: '', + children: [], + }, + ], + }, + { + id: '7f774865-a111-4cfa-ad9c-a9b1b34bd6fb', + pageType: 'Article', + name: 'Knowledge Article with children', + description: 'description', + fullyQualifiedName: 'Knowledge Article with children', + displayName: 'Knowledge Article with children', + children: [ + { + id: '16d75850-0fd3-475d-965b-fc2d3ef38900', + pageType: 'Article', + name: 'Article_5K3xBSov', + description: 'description', + fullyQualifiedName: 'Article_5K3xBSov', + displayName: 'Overview of Data Discovery data', + children: [], + }, + { + id: 'c21abbc6-5c72-4998-aacd-8c98c37be772', + pageType: 'Article', + name: 'Article_iSUbmc2V', + description: + '

This is the simple test now I will select and show you the bubble menu

', + fullyQualifiedName: 'Article_iSUbmc2V', + displayName: 'Notion like editor', + children: [ + { + id: 'b09e88ab-b2cf-4b21-9650-0a20a51ba6a8', + pageType: 'Article', + name: 'Article_bfPSYGdU', + description: '', + fullyQualifiedName: 'Article_bfPSYGdU', + displayName: '', + children: [ + { + id: '93f5f97e-7c92-40e4-a215-124bc1c475ee', + pageType: 'Article', + name: 'Article_eJAFUCiA', + description: '', + fullyQualifiedName: 'Article_eJAFUCiA', + displayName: 'I updated va;', + children: [ + { + id: '2097349d-d128-496d-b8f8-95474bcb3689', + pageType: 'Article', + name: 'Article_2er2H4E4', + description: '

', + fullyQualifiedName: 'Article_2er2H4E4', + displayName: 'Updated title', + children: [], + }, + ], + }, + ], + }, + ], + }, + { + id: '7d76837c-058e-4ac5-84e6-f7adb342aa79', + pageType: 'Article', + name: 'Article_qgqrKSse', + description: '', + fullyQualifiedName: 'Article_qgqrKSse', + displayName: '', + children: [], + }, + { + id: '08481f32-fa7e-44bf-9cd1-5a130adb4cf8', + pageType: 'Article', + name: 'Article_v8dwycta', + description: '', + fullyQualifiedName: 'Article_v8dwycta', + displayName: '', + children: [], + }, + ], + }, +]; + +jest.mock('rest/knowledgeCenterAPI', () => ({ + getPageHierarchyFromES: jest.fn().mockImplementation(() => + Promise.resolve({ + data: PageHierarchy, + paging: { limit: 100, offset: 0, total: PageHierarchy.length }, + }) + ), + postKnowledgePage: jest.fn().mockImplementation(() => + Promise.resolve({ + id: 'new-page-id', + name: 'newPage', + fullyQualifiedName: 'newPage', + displayName: '', + description: '', + pageType: 'Article', + }) + ), +})); + +const mockPush = jest.fn(); +const fqn = 'Article_XJIGIKX2'; + +jest.mock('react-router-dom', () => ({ + ...jest.requireActual('react-router-dom'), + useHistory: jest.fn().mockImplementation(() => ({ + push: mockPush, + })), + useParams: jest.fn().mockImplementation(() => ({ fqn })), + useNavigate: jest.fn().mockImplementation(() => mockPush), +})); + +jest.mock('utils/useRequiredParams', () => ({ + useRequiredParams: jest.fn().mockImplementation(() => ({ fqn })), +})); + +jest.mock('crypto-random-string-with-promisify-polyfill', () => + jest.fn().mockReturnValue('randomString') +); + +const mockUserData: User = { + name: 'aaron_johnson0', + email: 'testUser1@email.com', + id: '9304f330-2e9a-4513-883b-c939e29683a8', +}; + +jest.mock('hooks/useApplicationStore', () => ({ + useApplicationStore: jest.fn().mockImplementation(() => ({ + currentUser: mockUserData, + })), +})); + +jest.mock('context/LimitsProvider/useLimitsStore', () => ({ + useLimitStore: jest + .fn() + .mockImplementation(() => ({ getResourceLimit: jest.fn() })), +})); + +jest.mock('components/common/DeleteWidget/DeleteWidgetModal', () => + jest + .fn() + .mockReturnValue(
DeleteWidgetModal
) +); + +describe('KnowledgePagesHierarchy', () => { + it('should render KnowledgePagesHierarchy', async () => { + await act(async () => { + render( + , + { wrapper: MemoryRouter } + ); + }); + + expect(screen.getByTestId('knowledge-pages-hierarchy')).toBeInTheDocument(); + + // should render the tree first level nodes + expect( + screen.getByText('How to Discover Assets of Interest') + ).toBeInTheDocument(); + expect(screen.getByText('This is Updated')).toBeInTheDocument(); + expect( + screen.getByText('Knowledge Article with children') + ).toBeInTheDocument(); + + // should render the collapse button + expect( + screen.getByTestId('How to Discover Assets of Interest-collapse-icon') + ).toBeInTheDocument(); + expect( + screen.getByTestId('This is Updated-collapse-icon') + ).toBeInTheDocument(); + expect( + screen.getByTestId('Knowledge Article with children-collapse-icon') + ).toBeInTheDocument(); + + // should render the page icon + expect(screen.getAllByTestId('page-icon')).toHaveLength(3); + }); + + it('should render the active node', async () => { + await act(async () => { + render( + , + { + wrapper: MemoryRouter, + } + ); + }); + + expect( + screen.getByTestId('page-node-How to Discover Assets of Interest') + ).toBeInTheDocument(); + expect( + screen.getByTestId('page-node-How to Discover Assets of Interest') + ).toHaveAttribute('data-isactive', 'true'); + }); + + it('should render the children if node is expanded', async () => { + await act(async () => { + render( + , + { + wrapper: MemoryRouter, + } + ); + }); + + const collapseButton = screen.getByTestId( + `How to Discover Assets of Interest-collapse-icon` + ); + + fireEvent.click(collapseButton); + + expect( + screen.getByText('How to Discover Assets of Interest Child 1') + ).toBeInTheDocument(); + }); + + it('delete flow should work', async () => { + await act(async () => { + render( + , + { + wrapper: MemoryRouter, + } + ); + }); + + const deleteButton = screen.getByTestId( + `How to Discover Assets of Interest-delete-page-btn` + ); + + fireEvent.click(deleteButton); + + expect(screen.getByTestId('delete-widget')).toBeInTheDocument(); + }); + + it('add page flow should work', async () => { + await act(async () => { + render( + , + { + wrapper: MemoryRouter, + } + ); + }); + + const addButton = screen.getByTestId( + `How to Discover Assets of Interest-add-page-btn` + ); + + fireEvent.click(addButton); + + await waitFor(() => { + expect(mockPush).toHaveBeenCalledWith({ + pathname: '/knowledge-center/newPage', + }); + }); + }); + + describe('Scroll Pagination', () => { + const mockGetPageHierarchyFromES = jest.requireMock( + 'rest/knowledgeCenterAPI' + ).getPageHierarchyFromES; + + const mockScrollFn = jest.fn(); + + beforeEach(() => { + jest.clearAllMocks(); + // Mock window.innerHeight + Object.defineProperty(window, 'innerHeight', { + writable: true, + configurable: true, + value: 1000, + }); + }); + + it('should trigger pagination when scroll reaches bottom (exact match)', async () => { + await act(async () => { + render( + , + { wrapper: MemoryRouter } + ); + }); + + const treeElement = screen.getByTestId('knowledge-pages-hierarchy'); + const scrollableElement = treeElement.getElementsByClassName( + 'ant-tree-list-holder' + )[0]; + + // Set the scroll properties on the element + Object.defineProperty(scrollableElement, 'scrollHeight', { + value: 3200, + writable: true, + configurable: true, + }); + + // Try a different approach for scrollTop + Object.defineProperty(scrollableElement, 'scrollTop', { + get: () => 2390, + set: mockScrollFn, + configurable: true, + }); + + // Simulate scroll event with scrollHeight exactly at windowHeight - 190 + const scrollEvent = { + currentTarget: { + ...scrollableElement, + scrollHeight: 3200, + scrollTop: 2390, + }, + }; + + await act(async () => { + fireEvent.scroll(scrollableElement, scrollEvent); + }); + + await waitFor(() => { + expect(mockGetPageHierarchyFromES).toHaveBeenCalledWith( + undefined, + undefined, + 100, // offset should be incremented by 100 + 100, + fqn + ); + }); + }); + + it('should trigger pagination when scrollHeight is within range (windowHeight - 191)', async () => { + await act(async () => { + render( + , + { wrapper: MemoryRouter } + ); + }); + + const treeElement = screen.getByTestId('knowledge-pages-hierarchy'); + const scrollableElement = treeElement.getElementsByClassName( + 'ant-tree-list-holder' + )[0]; + + // Set the scroll properties on the element + Object.defineProperty(scrollableElement, 'scrollHeight', { + value: 3200, + writable: true, + configurable: true, + }); + + Object.defineProperty(scrollableElement, 'scrollTop', { + get: () => 2391, + set: mockScrollFn, + configurable: true, + }); + + // Simulate scroll event with scrollHeight at windowHeight - 191 (within -1 range) + const scrollEvent = { + currentTarget: { + ...scrollableElement, + scrollHeight: 3200, + scrollTop: 2391, + }, + }; + fireEvent.scroll(scrollableElement, scrollEvent); + + await waitFor(() => { + expect(mockGetPageHierarchyFromES).toHaveBeenCalledWith( + undefined, + undefined, + 100, + 100, + fqn + ); + }); + }); + + it('should trigger pagination when scrollHeight is within range (windowHeight - 189)', async () => { + await act(async () => { + render( + , + { wrapper: MemoryRouter } + ); + }); + + const treeElement = screen.getByTestId('knowledge-pages-hierarchy'); + const scrollableElement = treeElement.getElementsByClassName( + 'ant-tree-list-holder' + )[0]; + // Set the scroll properties on the element + Object.defineProperty(scrollableElement, 'scrollHeight', { + value: 3200, + writable: true, + configurable: true, + }); + Object.defineProperty(scrollableElement, 'scrollTop', { + get: () => 2389, + set: mockScrollFn, + configurable: true, + }); + + // Simulate scroll event with scrollHeight at windowHeight - 189 (within +1 range) + const scrollEvent = { + currentTarget: { + ...scrollableElement, + scrollHeight: 3200, + scrollTop: 2389, + }, + }; + + fireEvent.scroll(scrollableElement, scrollEvent); + + await waitFor(() => { + expect(mockGetPageHierarchyFromES).toHaveBeenCalledWith( + undefined, + undefined, + 100, + 100, + fqn + ); + }); + }); + + it('should NOT trigger pagination when scrollHeight is outside range (too high)', async () => { + await act(async () => { + render( + , + { wrapper: MemoryRouter } + ); + }); + + const treeElement = screen.getByTestId('knowledge-pages-hierarchy'); + const scrollableElement = treeElement.getElementsByClassName( + 'ant-tree-list-holder' + )[0]; + + // Set the scroll properties on the element + Object.defineProperty(scrollableElement, 'scrollHeight', { + value: 1000, + writable: true, + configurable: true, + }); + + Object.defineProperty(scrollableElement, 'scrollTop', { + get: () => 800, + set: mockScrollFn, + configurable: true, + }); + + // Simulate scroll event with scrollHeight at windowHeight - 191 (within -1 range) + const scrollEvent = { + currentTarget: { + ...scrollableElement, + scrollHeight: 1000, + scrollTop: 800, + }, + }; + + fireEvent.scroll(scrollableElement, scrollEvent); + + await waitFor(() => { + expect(mockGetPageHierarchyFromES).not.toHaveBeenCalledWith( + undefined, + undefined, + 100, + 100, + fqn + ); + }); + }); + + it('should NOT trigger pagination when scrollHeight is outside range (too low)', async () => { + await act(async () => { + render( + , + { wrapper: MemoryRouter } + ); + }); + + const treeElement = screen.getByTestId('knowledge-pages-hierarchy'); + const scrollableElement = treeElement.getElementsByClassName( + 'ant-tree-list-holder' + )[0]; + + // Set the scroll properties on the element + Object.defineProperty(scrollableElement, 'scrollHeight', { + value: 1000, + writable: true, + configurable: true, + }); + + Object.defineProperty(scrollableElement, 'scrollTop', { + get: () => 820, + set: mockScrollFn, + configurable: true, + }); + + // Simulate scroll event with scrollHeight at windowHeight - 191 (within -1 range) + const scrollEvent = { + currentTarget: { + ...scrollableElement, + scrollHeight: 1000, + scrollTop: 820, + }, + }; + + fireEvent.scroll(scrollableElement, scrollEvent); + + await waitFor(() => { + expect(mockGetPageHierarchyFromES).not.toHaveBeenCalledWith( + undefined, + undefined, + 100, + 100, + fqn + ); + }); + }); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePagesHierarchy/KnowledgePagesHierarchy.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePagesHierarchy/KnowledgePagesHierarchy.tsx new file mode 100644 index 000000000000..fcacd0db52e8 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePagesHierarchy/KnowledgePagesHierarchy.tsx @@ -0,0 +1,826 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Button, Modal, Skeleton, Tree, Typography } from 'antd'; +import { DataNode } from 'antd/es/tree'; +import { AntTreeNodeProps, DirectoryTreeProps, TreeProps } from 'antd/lib/tree'; +import { AxiosError } from 'axios'; +import { ReactComponent as KnowledgeCenterIcon } from '../../../assets/svg/ic-knowledge-page.svg'; +import { CREATE_PAGE_HASH, ROUTES } from '../../../constants/constants'; +import { + CreateKnowledgePage, + KnowledgePage, + KnowledgePagesHierarchyRef, + MovedEntity, + PageHierarchy, + PageType, + RecentlyViewedQuickLinks, +} from '../../../interface/knowledge-center.interface'; + +import { + forwardRef, + Key, + ReactNode, + UIEventHandler, + useCallback, + useEffect, + useImperativeHandle, + useMemo, + useReducer, + useRef, + useState, +} from 'react'; +import { Link, useNavigate } from 'react-router-dom'; +import { + getPageHierarchyFromES, + patchKnowledgePage, + postKnowledgePage, +} from '../../../rest/knowledgeCenterAPI'; +import { showErrorToast } from '../../../utils/ToastUtils'; + +import { PlusOutlined } from '@ant-design/icons'; +import classNames from 'classnames'; +import cryptoRandomString from 'crypto-random-string-with-promisify-polyfill'; +import { compare } from 'fast-json-patch'; +import { isUndefined, uniq } from 'lodash'; +import { useTranslation } from 'react-i18next'; +import { ReactComponent as DragIcon } from '../../../assets/svg/drag.svg'; +import { ReactComponent as IconDown } from '../../../assets/svg/ic-arrow-down.svg'; +import { ReactComponent as IconRight } from '../../../assets/svg/ic-arrow-right.svg'; +import { ReactComponent as DeleteIcon } from '../../../assets/svg/ic-delete.svg'; +import DeleteWidgetModal from '../../../components/common/DeleteWidget/DeleteWidgetModal'; +import CreateErrorPlaceHolder from '../../../components/common/ErrorWithPlaceholder/CreateErrorPlaceHolder'; +import Loader from '../../../components/common/Loader/Loader'; +import { DE_ACTIVE_COLOR } from '../../../constants/constants'; +import { + KNOWLEDGE_CENTER_INSTANCE_NAME_LENGTH, + KNOWLEDGE_CENTER_PAGINATION_LIMIT, + KNOWLEDGE_CENTER_PAGINATION_OFFSET_INCREMENT, + KNOWLEDGE_CENTER_TREE_HEIGHT_OFFSET, + KNOWLEDGE_CENTER_TREE_HEIGHT_OFFSET_CHILD_ARTICLE, +} from '../../../constants/KnowledgeCenter.constant'; +import { useLimitStore } from '../../../context/LimitsProvider/useLimitsStore'; +import { OperationPermission } from '../../../context/PermissionProvider/PermissionProvider.interface'; +import { SIZE } from '../../../enums/common.enum'; +import { EntityType } from '../../../enums/entity.enum'; +import { useCurrentUserPreferences } from '../../../hooks/currentUserStore/useCurrentUserStore'; +import { useApplicationStore } from '../../../hooks/useApplicationStore'; +import useCustomLocation from '../../../hooks/useCustomLocation/useCustomLocation'; +import { getEntityName } from '../../../utils/EntityUtils'; +import Fqn from '../../../utils/Fqn'; +import { Transi18next } from '../../../utils/i18next/LocalUtil'; +import { + convertToTreeData, + extractKnowledgePageParentFQN, + findPageAndParentInTreeData, + findPageInTreeData, + getExpandedNodeKeys, + getKnowledgePagePath, + getPageAllChildren, + getUpdatePageHierarchy, + getUpdatePageHierarchyForDelete, + hierarchyPaginationInitialState, + hierarchyPaginationReducer, + integrateNodesIntoHierarchy, + updateKnowledgeCenterRecentViewed, + updateTreeData, +} from '../../../utils/KnowledgePageUtils'; +import { useRequiredParams } from '../../../utils/useRequiredParams'; +import './knowledge-pages-hierarchy.less'; +const { DirectoryTree } = Tree; + +interface KnowledgePagesHierarchyProps { + permissions: OperationPermission; + isPageHeaderAvailable: boolean; + activeKey?: DirectoryTreeProps['activeKey']; + activePage?: KnowledgePage; + onPageDelete?: (id: string | string[]) => void; + onLoading?: (isLoading: boolean) => void; +} + +const KnowledgePagesHierarchy = forwardRef< + KnowledgePagesHierarchyRef, + KnowledgePagesHierarchyProps +>( + ( + { + activeKey, + activePage, + onPageDelete, + onLoading, + permissions, + isPageHeaderAvailable, + }, + ref + ) => { + const { fqn } = useRequiredParams<{ fqn: string }>(); + const navigate = useNavigate(); + const { hash } = useCustomLocation(); + const { currentUser } = useApplicationStore(); + const { t } = useTranslation(); + const [knowledgePageHierarchy, setKnowledgePageHierarchy] = useState< + PageHierarchy[] + >([]); + const { getResourceLimit } = useLimitStore(); + + // Cache to track if initial hierarchy has been loaded + const [isHierarchyInitialized, setIsHierarchyInitialized] = + useState(false); + // Track the last fqn for which hierarchy was fetched + const lastFetchedFqnRef = useRef(null); + + const [isLoading, setIsLoading] = useState(false); + const [expandedKeys, setExpandedKeys] = useState([]); + const [deletePage, setDeletePage] = useState(); + + const [movedPage, setMovedPage] = useState(); + const [isMovingPage, setIsMovingPage] = useState(false); + const { + preferences: { recentlyViewedQuickLinks: recentlyViewed }, + } = useCurrentUserPreferences(); + + const [paginationState, setPaginationState] = useReducer( + hierarchyPaginationReducer, + hierarchyPaginationInitialState + ); + + const TREE_HEIGHT = useMemo( + () => + window.innerHeight - + (isPageHeaderAvailable + ? KNOWLEDGE_CENTER_TREE_HEIGHT_OFFSET_CHILD_ARTICLE + : KNOWLEDGE_CENTER_TREE_HEIGHT_OFFSET), + [isPageHeaderAvailable] + ); + + const treeData: DataNode[] = useMemo(() => { + return convertToTreeData(activePage, knowledgePageHierarchy); + }, [knowledgePageHierarchy, activePage]); + + const fetchKnowledgePageHierarchy = async ( + setLoading = true, + isPaginationLoading = false, + offset = 0, + limit = KNOWLEDGE_CENTER_PAGINATION_LIMIT, + forceRefresh = false + ) => { + const isCreateHash = Boolean(hash && hash.slice(1) === CREATE_PAGE_HASH); + + // Skip fetching if hierarchy is already initialized and not forcing refresh + // and not doing pagination loading and the fqn hasn't changed + if ( + !forceRefresh && + !isPaginationLoading && + isHierarchyInitialized && + knowledgePageHierarchy.length > 0 && + lastFetchedFqnRef.current === fqn && + !isCreateHash + ) { + return; + } + + if (setLoading && !isCreateHash) { + setIsLoading(true); + } + + if (isPaginationLoading) { + setPaginationState({ + type: 'SET_PAGINATION_LOADING', + value: true, + }); + } + try { + const { data, paging } = await getPageHierarchyFromES( + undefined, + undefined, + offset, + limit, + fqn + ); + + // Update the last fetched fqn + lastFetchedFqnRef.current = fqn; + + // set the pagination state + setPaginationState({ + type: 'SET_PAGING_VALUE', + value: paging, + }); + + // if the data is empty or the total count is equal to the current hierarchy + if ( + data.length === 0 || + knowledgePageHierarchy.length === paging.total + ) { + setPaginationState({ + type: 'SET_IS_PAGINATION_END', + value: true, + }); + } + + if (isCreateHash) { + setKnowledgePageHierarchy(data); + } else { + // Check if we have an activeFqn that represents a nested child node + const fqnParts = fqn ? Fqn.split(fqn) : []; + const isNestedNode = fqnParts.length > 1; + + // If it's a nested node, we need to ensure all parent nodes exist in the hierarchy + if (isNestedNode && data.length > 0) { + // Extract all parent FQNs from the activeFqn + const parentFQN = extractKnowledgePageParentFQN(fqn); + setKnowledgePageHierarchy((prevHierarchy) => { + return integrateNodesIntoHierarchy(prevHierarchy, data); + }); + + // Ensure all parent nodes are expanded + setExpandedKeys((prevKeys) => uniq([...prevKeys, ...parentFQN])); + } else { + // Standard merging logic for root-level items + setKnowledgePageHierarchy((prevHierarchy) => { + const mergedArray = prevHierarchy.concat(data); + const updatedHierarchy = Array.from( + new Map(mergedArray.map((item) => [item.id, item])).values() + ); + + return updatedHierarchy; + }); + } + } + setIsHierarchyInitialized(true); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setIsLoading(false); + setPaginationState({ + type: 'SET_PAGINATION_LOADING', + value: false, + }); + } + }; + + const onLoadData = useCallback( + async (node: DataNode) => { + try { + if (node.children) { + return; + } + + const { data: children } = await getPageHierarchyFromES( + node.key as string + ); + + setKnowledgePageHierarchy( + updateTreeData( + knowledgePageHierarchy, + children, + node.key.toString() + ) + ); + } catch { + // do nothing + } + }, + [knowledgePageHierarchy] + ); + + const handleDeletePage = useCallback( + (pageNode: DataNode) => { + // find the page in the tree data + const page = findPageInTreeData( + knowledgePageHierarchy, + pageNode.key as string + ); + // if page is not found, return + if (!page) { + return; + } + // set the page to be deleted + setDeletePage(page); + }, + [knowledgePageHierarchy] + ); + + const handleAfterDeletePage = useCallback( + async (deletedPageData: PageHierarchy) => { + const deletedPageHierarchy = findPageInTreeData( + knowledgePageHierarchy, + deletedPageData?.fullyQualifiedName ?? '' + ); + + const isActivePageParent = findPageInTreeData( + [...(deletedPageHierarchy?.children ?? [])], + activePage?.fullyQualifiedName ?? '' + ); + + const deletedPages = [ + deletedPageData.id, + ...getPageAllChildren(deletedPageHierarchy?.children ?? []).map( + (children) => children.id + ), + ]; + + // call the callback if provided + onPageDelete?.(deletedPages); + + // Update current count when Create / Delete operation performed + await getResourceLimit('knowledgeCenter', true, true); + + // update the recent views + updateKnowledgeCenterRecentViewed( + recentlyViewed.filter( + (page) => !deletedPages.includes(page.id) + ) as unknown as RecentlyViewedQuickLinks['data'] + ); + + // refresh the hierarchy + deletedPageData && + setKnowledgePageHierarchy((prevHierarchy) => + getUpdatePageHierarchyForDelete( + deletedPageData.fullyQualifiedName, + prevHierarchy + ) + ); + + // if the deleted page is the active page or parent of active page, navigate to knowledge center + if ( + activeKey === deletedPageData.fullyQualifiedName || + isActivePageParent + ) { + navigate(ROUTES.KNOWLEDGE_CENTER); + } + }, + [knowledgePageHierarchy, onPageDelete, activeKey, activePage] + ); + + const handleAddPage = useCallback( + async (pageNode: DataNode) => { + // find the page in the tree data + const page = findPageInTreeData( + knowledgePageHierarchy, + pageNode.key as string + ); + + // if page is not found, return + if (!page) { + return; + } + + try { + onLoading?.(true); + const instanceName = `${PageType.ARTICLE}_${cryptoRandomString({ + length: KNOWLEDGE_CENTER_INSTANCE_NAME_LENGTH, + type: 'alphanumeric', + })}`; + + // create a new page + const data: CreateKnowledgePage = { + name: instanceName, + displayName: '', + description: '', + pageType: PageType.ARTICLE, + page: { + publicationDate: new Date(), + relatedArticles: [], + }, + owners: [ + { + type: 'user', + id: currentUser?.id ?? '', + }, + ], + parent: { id: page.id, type: 'page' }, + }; + const response = await postKnowledgePage(data); + + // Convert the created page response to PageHierarchy format + const newPageHierarchy: PageHierarchy = { + id: response.id, + name: response.name, + fullyQualifiedName: response.fullyQualifiedName, + displayName: response.displayName, + description: response.description, + pageType: response.pageType, + childrenCount: 0, + }; + + // Add the newly created page to the hierarchy tree immediately + setKnowledgePageHierarchy((prevHierarchy) => + updateTreeData( + prevHierarchy, + [newPageHierarchy], + page.fullyQualifiedName + ) + ); + + // Ensure parent node is expanded to show the new child + setExpandedKeys((prevKeys) => + uniq([...prevKeys, page.fullyQualifiedName]) + ); + + // Update resource limit count + await getResourceLimit('knowledgeCenter', true, true); + + // push to the newly created page + navigate({ + pathname: getKnowledgePagePath(response.fullyQualifiedName), + }); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + onLoading?.(false); + } + }, + [currentUser, knowledgePageHierarchy, getResourceLimit] + ); + + const titleRender = useCallback( + (node: DataNode) => { + const nodeKey = node.key as string; + + return ( +
+ +
+ + + + + {node.title as ReactNode} + +
+ +
+ + +
+
+ ); + }, + [handleDeletePage, handleAddPage, activeKey] + ); + + const handleMovePage = async (movedPageData: MovedEntity) => { + try { + setIsMovingPage(true); + const { sourceNode, sourceNodeParent, targetNode } = movedPageData; + + const newExpandedKeys = []; + + // step1: update the source node parent + const updatedSourceNodeForPatch = { + ...sourceNode, + parent: targetNode + ? { + id: targetNode.id, + type: 'page', + fullyQualifiedName: targetNode.fullyQualifiedName, + name: targetNode.name, + displayName: targetNode.displayName, + } + : undefined, + }; + + const sourceNodePatch = compare(sourceNode, updatedSourceNodeForPatch); + + await patchKnowledgePage(sourceNode.id, sourceNodePatch); + + if (!isUndefined(targetNode)) { + // step2: fetch updated children for the target node + const targetNodeChildren = await getPageHierarchyFromES( + targetNode.fullyQualifiedName + ); + + setKnowledgePageHierarchy((prevHierarchy) => { + return getUpdatePageHierarchy( + prevHierarchy, + { + ...targetNode, + children: targetNodeChildren.data, + }, + true + ); + }); + + newExpandedKeys.push(targetNode.fullyQualifiedName); + + // step3: fetch updated children for the source node parent + if (sourceNodeParent) { + const sourceNodeParentChildren = await getPageHierarchyFromES( + sourceNodeParent.fullyQualifiedName + ); + + setKnowledgePageHierarchy((prevHierarchy) => { + return getUpdatePageHierarchy( + prevHierarchy, + { + ...sourceNodeParent, + children: sourceNodeParentChildren.data, + }, + true + ); + }); + + newExpandedKeys.push(sourceNodeParent.fullyQualifiedName); + } else { + // if the source node parent is not found, remove the source node from the hierarchy + setKnowledgePageHierarchy((prevHierarchy) => { + return prevHierarchy.filter((page) => page.id !== sourceNode.id); + }); + } + + // step4: update expanded keys + setExpandedKeys(newExpandedKeys); + } else { + fetchKnowledgePageHierarchy( + true, + false, + 0, + KNOWLEDGE_CENTER_PAGINATION_LIMIT, + true + ); + setExpandedKeys([]); + } + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setMovedPage(undefined); + setIsMovingPage(false); + } + }; + + const handleDragAndDrop: TreeProps['onDrop'] = async (info) => { + const isDropPositionParentLevel = info.dropPosition === -1; + const sources = info.dragNode; + const target = info.node; + + // if the source and target are same, return + if (sources.key === target.key) { + return; + } + + const targetNode = findPageInTreeData( + knowledgePageHierarchy, + target.key.toString() + ); + + if (!targetNode) { + return; + } + + const { page: sourceNode, parent: sourceNodeParent } = + findPageAndParentInTreeData( + knowledgePageHierarchy, + sources.key.toString() + ); + + if (!sourceNode) { + return; + } + + // if the source node is already a direct child of the target node, return + const isChild = (targetNode.children ?? []).find( + (child) => child.id === sourceNode.id + ); + + if (isChild && !isDropPositionParentLevel) { + return; + } + + const movedPageData = { + sourceNode, + sourceNodeParent, + targetNode: isDropPositionParentLevel ? undefined : targetNode, + }; + + setMovedPage(movedPageData); + }; + + const handleScroll: UIEventHandler = useCallback( + (e) => { + const scrollHeight = + e.currentTarget.scrollHeight - e.currentTarget.scrollTop; + const windowHeight = + window.innerHeight - KNOWLEDGE_CENTER_TREE_HEIGHT_OFFSET; + + // if the scroll height is within the range of window height, fetch the next page, + // since on bigger screen there can be a chance the height is not exactly window height + + const finalScrollHeight = + scrollHeight + (isPageHeaderAvailable ? 70 : 0); // to maintain the height of panel after header added + if ( + finalScrollHeight >= windowHeight - 1 && + finalScrollHeight <= windowHeight + 1 && + !paginationState.isPaginationEnd && + !paginationState.paginationLoading + ) { + fetchKnowledgePageHierarchy( + false, + true, + paginationState.paging.offset + + KNOWLEDGE_CENTER_PAGINATION_OFFSET_INCREMENT + ); + } + }, + [isPageHeaderAvailable, paginationState] + ); + + useImperativeHandle(ref, () => ({ + fetchKnowledgePageHierarchy: (forceRefresh = false) => + fetchKnowledgePageHierarchy( + true, + false, + 0, + KNOWLEDGE_CENTER_PAGINATION_LIMIT, + forceRefresh + ), + })); + + useEffect(() => { + // Only fetch on initial mount or when hash changes to CREATE_PAGE_HASH + const isCreateHash = Boolean(hash && hash.slice(1) === CREATE_PAGE_HASH); + + if (!isHierarchyInitialized || isCreateHash) { + fetchKnowledgePageHierarchy(); + } else if (fqn !== lastFetchedFqnRef.current) { + // FQN changed but we already have hierarchy data, just update the ref + // The tree selection will be handled by activeKey prop + lastFetchedFqnRef.current = fqn; + } + }, [hash, fqn]); + + useEffect(() => { + if (activeKey) { + setExpandedKeys((prevKeys) => + uniq([ + ...prevKeys, + ...getExpandedNodeKeys(knowledgePageHierarchy, activeKey as string), + ]) + ); + } + }, [activeKey, knowledgePageHierarchy]); + + useEffect(() => { + if (activePage) { + setKnowledgePageHierarchy((prevHierarchy) => { + const updatedHierarchy = getUpdatePageHierarchy( + prevHierarchy, + activePage + ); + + return updatedHierarchy; + }); + } + }, [activePage]); + + if (isLoading) { + return ( +
+ +
+ ); + } + + if (!isLoading && knowledgePageHierarchy.length === 0) { + return ( + + ); + } + + return ( +
+ , + nodeDraggable: () => true, + }} + expandAction={false} + expandedKeys={expandedKeys} + height={TREE_HEIGHT} + icon={null} + loadData={onLoadData} + loadedKeys={expandedKeys} + selectedKeys={activeKey ? [activeKey] : []} + switcherIcon={(props: AntTreeNodeProps) => { + return props.expanded ? ( + + ) : ( + + ); + }} + titleRender={titleRender} + treeData={treeData} + onDrop={handleDragAndDrop} + onExpand={(keys) => setExpandedKeys(keys)} + onScroll={handleScroll} + /> + + {paginationState.paginationLoading && } + + {deletePage && ( + handleAfterDeletePage(deletePage)} + allowSoftDelete={false} + entityId={deletePage.id} + entityName={deletePage.displayName || t('label.untitled')} + entityType={EntityType.KNOWLEDGE_CENTER} + prepareType={false} + successMessage={t('server.entity-deleted-successfully', { + entity: t('label.article'), + })} + visible={!isUndefined(deletePage)} + onCancel={() => setDeletePage(undefined)} + /> + )} + + {movedPage && ( + setMovedPage(undefined)} + onOk={() => handleMovePage(movedPage)}> + } + values={{ + from: getEntityName(movedPage.sourceNode), + to: movedPage.targetNode + ? getEntityName(movedPage.targetNode) + : t('label.base-knowledge'), + entity: t('label.page-lowercase'), + }} + /> + + )} +
+ ); + } +); + +export default KnowledgePagesHierarchy; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePagesHierarchy/knowledge-pages-hierarchy.less b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePagesHierarchy/knowledge-pages-hierarchy.less new file mode 100644 index 000000000000..151ce0dd24e8 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/KnowledgePagesHierarchy/knowledge-pages-hierarchy.less @@ -0,0 +1,134 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@import (reference) '../../../styles/variables.less'; + +@add-first-page-btn-bg-color: #00000006; +@active-bg-color: #f5f5f5; + +.knowledge-pages-hierarchy-wrapper { + .ant-tree-treenode { + position: relative; + } + + .ant-tree.ant-tree-directory + .ant-tree-treenode.ant-tree-treenode-selected::before { + background: @radio-button-checked-bg; + border-left: 2px solid @radio-button-checked-bg; + } + + .knowledge-hierarchy-action-btn { + background-color: @active-bg-color; + display: none; + } + + .ant-tree .ant-tree-treenode-draggable .ant-tree-draggable-icon { + cursor: grab; + visibility: hidden; + flex-shrink: 0; + + & svg { + vertical-align: middle; + width: 12px; + height: 8px; + } + } + + .ant-tree.ant-tree-directory .ant-tree-treenode:hover { + .knowledge-hierarchy-action-btn { + display: flex; + position: absolute; + right: 2px; + top: 0; + } + + .knowledge-hierarchy-action-btn-item { + padding: 0px 8px; + height: 24px; + display: flex; + align-items: center; + justify-content: center; + } + + .ant-tree-draggable-icon { + visibility: visible; + opacity: 1; + } + } + + .knowledge-hierarchy-page-title-wrapper { + &.leaf-node-title { + padding-left: 18px; + } + + display: flex; + align-items: center; + gap: 4px; + } + + .ant-tree .ant-tree-node-content-wrapper { + padding: 0px; + display: block; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + + .node-page-icon { + > svg { + vertical-align: middle; + } + + display: flex; + justify-content: center; + align-items: center; + position: relative; + flex: none; + align-self: stretch; + margin: 0; + width: 18px; + line-height: 24px; + text-align: center; + } + + .ant-tree-switcher-noop { + display: none; + } + + .ant-tree-switcher { + justify-content: normal; + width: 18px; + + > .ant-tree-switcher-icon { + width: 12px; + height: 12px; + color: @grey-4; + } + } + + .anchor-no-underline { + &:hover, + &:focus { + text-decoration: none; + } + } + + .ant-tree.ant-tree-directory .ant-tree-treenode.ant-tree-treenode-selected { + font-weight: 700; + } +} + +.add-first-page-btn { + width: 100%; + margin-top: 48px; + background: @add-first-page-btn-bg-color; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/QuickLinkFormModal/QuickLinkFormModal.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/QuickLinkFormModal/QuickLinkFormModal.test.tsx new file mode 100644 index 000000000000..f8ded3d5da26 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/QuickLinkFormModal/QuickLinkFormModal.test.tsx @@ -0,0 +1,113 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { act, fireEvent, render, screen } from '@testing-library/react'; +import { OperationPermission } from 'context/PermissionProvider/PermissionProvider.interface'; +import { + QuickLinkFormModal, + QuickLinkFormModalProps, +} from './QuickLinkFormModal'; + +jest.mock( + 'components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList', + () => jest.fn(() =>
) +); + +const mockSave = jest.fn(); + +const mockCancel = jest.fn(); + +jest.mock('utils/EntityUtils', () => ({ + getEntityName: jest.fn().mockImplementation((entity) => entity.displayName), +})); +jest.mock('utils/TableUtils', () => ({ + getTagsWithoutTier: jest.fn(), +})); + +jest.mock('utils/TableTags/TableTags.utils', () => ({ + getFilterTags: jest.fn(), +})); + +jest.mock('utils/ToastUtils', () => ({ + showErrorToast: jest.fn(), +})); + +jest.mock('pages/TasksPage/shared/DescriptionTask'); +jest.mock('pages/TasksPage/shared/DescriptionTaskNew'); + +const mockProps: QuickLinkFormModalProps = { + isOpen: true, + onSave: mockSave, + onCancel: mockCancel, + permissions: { + EditAll: true, + EditDisplayName: true, + EditDescription: true, + EditTags: true, + } as OperationPermission, +}; + +describe('QuickLinkFormModal', () => { + it('Should render the form inputs', async () => { + render(); + + const displayNameInput = screen.getByTestId('displayName'); + const urlInput = screen.getByTestId('url'); + const descriptionEditor = screen.getByTestId('editor'); + const tagSelectors = screen.getAllByTestId('tag-selector'); + + expect(displayNameInput).toBeInTheDocument(); + expect(urlInput).toBeInTheDocument(); + expect(descriptionEditor).toBeInTheDocument(); + expect(tagSelectors).toHaveLength(2); + expect( + screen.getByTestId('data-asset-async-select-list') + ).toBeInTheDocument(); + }); + + it('onSave should work', async () => { + render(); + + const displayNameInput = screen.getByTestId('displayName'); + const urlInput = screen.getByTestId('url'); + + fireEvent.change(displayNameInput, { target: { value: 'displayName' } }); + fireEvent.change(urlInput, { target: { value: 'https://example.coms' } }); + + const submitBtn = screen.getByText('label.save'); + + await act(async () => { + fireEvent.click(submitBtn); + }); + + expect(mockSave).toHaveBeenCalledWith({ + description: '', + displayName: 'displayName', + glossaryTerms: undefined, + relatedEntities: [], + tags: undefined, + url: 'https://example.coms', + }); + }); + + it('onCancel should work', async () => { + render(); + + const cancelBtn = screen.getByText('label.back'); + + await act(async () => { + fireEvent.click(cancelBtn); + }); + + expect(mockCancel).toHaveBeenCalled(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/QuickLinkFormModal/QuickLinkFormModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/QuickLinkFormModal/QuickLinkFormModal.tsx new file mode 100644 index 000000000000..4f0785b951e7 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/QuickLinkFormModal/QuickLinkFormModal.tsx @@ -0,0 +1,380 @@ +/* + * Copyright 2023 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Form, FormProps, Modal } from 'antd'; +import { AxiosError } from 'axios'; +import { compare } from 'fast-json-patch'; +import { cloneDeep, isEqual, isNil, isUndefined } from 'lodash'; + +import { FC, useMemo, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import DataAssetAsyncSelectList from '../../../components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList'; +import { DataAssetOption } from '../../../components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList.interface'; +import { getKnowledgePageFields } from '../../../constants/KnowledgeCenter.constant'; +import { OperationPermission } from '../../../context/PermissionProvider/PermissionProvider.interface'; +import { EntityReference } from '../../../generated/entity/type'; +import { TagLabel, TagSource } from '../../../generated/type/tagLabel'; +import { FieldProp, FieldTypes } from '../../../interface/FormUtils.interface'; +import { + CreateKnowledgePage, + KnowledgePage, + QuickLink, +} from '../../../interface/knowledge-center.interface'; +import { + getKnowledgePageByFqn, + patchKnowledgePage, +} from '../../../rest/knowledgeCenterAPI'; +import { getEntityName } from '../../../utils/EntityUtils'; +import { generateFormFields } from '../../../utils/formUtils'; +import i18n from '../../../utils/i18next/LocalUtil'; +import { getFilterTags } from '../../../utils/TableTags/TableTags.utils'; +import { getTagsWithoutTier } from '../../../utils/TableUtils'; +import { showErrorToast } from '../../../utils/ToastUtils'; + +export interface QuickLinkFormModalFormData + extends Pick { + url: string; + tags?: TagLabel[]; + glossaryTerms?: TagLabel[]; + relatedEntities?: EntityReference[]; +} + +export interface QuickLinkFormModalProps { + isOpen: boolean; + quickLink?: KnowledgePage; + permissions: OperationPermission; + onSave: (data: QuickLinkFormModalFormData) => void; + onCancel: () => void; +} + +export const QuickLinkFormModal: FC = ({ + isOpen, + quickLink, + permissions, + onCancel, + onSave, +}) => { + const [form] = Form.useForm(); + const { t } = useTranslation('translation', { i18n }); + + const [isUpdating, setIsUpdating] = useState(false); + + const { classification, glossaries, initialValues } = useMemo(() => { + if (isUndefined(quickLink)) { + return { initialValues: {}, classification: [], glossaries: [] }; + } + + const tagsWithoutTier = getTagsWithoutTier(quickLink.tags ?? []); + + const { Classification: classification, Glossary: glossaries } = + getFilterTags(tagsWithoutTier); + + return { + initialValues: { + displayName: quickLink?.displayName, + url: (quickLink.page as QuickLink)?.url, + description: quickLink?.description, + tags: classification, + glossaryTerms: glossaries, + }, + classification, + glossaries, + }; + }, [quickLink]); + + const { + defaultDataAssetsValues, + initialDataAssetsOptions, + restRelatedDataAssets, + } = useMemo(() => { + if (isUndefined(quickLink)) { + return { + initialDataAssetsOptions: [], + defaultDataAssetsValues: [], + filteredRelatedDataAssets: [], + restRelatedDataAssets: [], + }; + } + + const relatedDataAssets = quickLink.relatedEntities ?? []; + + const { filteredRelatedDataAssets, restRelatedDataAssets } = + relatedDataAssets.reduce( + (acc, item) => { + // filter out team and user as they are not data assets + if (!['team', 'user'].includes(item.type)) { + acc.filteredRelatedDataAssets.push(item); + } else { + acc.restRelatedDataAssets.push(item); + } + + return acc; + }, + { + filteredRelatedDataAssets: [] as EntityReference[], + restRelatedDataAssets: [] as EntityReference[], + } + ); + + const initialDataAssetsOptions: DataAssetOption[] = + filteredRelatedDataAssets.map((item) => { + return { + displayName: getEntityName(item), + reference: item, + label: getEntityName(item), + value: item.id, + }; + }); + + const defaultDataAssetsValues = filteredRelatedDataAssets.map( + (item) => item.id + ); + + return { + initialDataAssetsOptions, + defaultDataAssetsValues, + filteredRelatedDataAssets, + restRelatedDataAssets, + }; + }, [quickLink]); + + const handleQuickLinkUpdate = async ( + knowledgePage: KnowledgePage, + formData: QuickLinkFormModalFormData + ) => { + const currentKnowledgePage = cloneDeep(knowledgePage); + + const tags = [...(formData.tags ?? []), ...(formData.glossaryTerms ?? [])]; + + let existingTags = currentKnowledgePage.tags ?? []; + + // derive the new tags + const newTags = tags.filter( + (tag) => !existingTags.find((t) => t.tagFQN === tag.tagFQN) + ); + + // update the existing tags with the new tags + existingTags = existingTags.filter((tag) => + tags.find((t) => t.tagFQN === tag.tagFQN) + ); + + const updatedKnowledgePage: KnowledgePage = { + ...currentKnowledgePage, + displayName: formData.displayName, + description: formData.description, + tags: [...existingTags, ...newTags], + page: { + ...currentKnowledgePage.page, + url: formData.url, + }, + relatedEntities: formData?.relatedEntities, + }; + + if (isEqual(currentKnowledgePage, updatedKnowledgePage)) { + onCancel(); + + return; + } + + try { + setIsUpdating(true); + const patch = compare(currentKnowledgePage, updatedKnowledgePage); + + await patchKnowledgePage(currentKnowledgePage.id, patch); + const response = await getKnowledgePageByFqn( + currentKnowledgePage.fullyQualifiedName, + { + fields: getKnowledgePageFields(), + } + ); + + const updatedData = { + displayName: response.displayName, + description: response.description, + tags: response.tags, + url: (response.page as QuickLink)?.url, + relatedEntities: response?.relatedEntities, + }; + onSave(updatedData); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setIsUpdating(false); + } + }; + + const handleSubmit: FormProps< + Omit & { + relatedEntities?: DataAssetOption[]; + } + >['onFinish'] = (values) => { + // filter out empty values + const relatedEntitiesData = values['relatedEntities']?.filter( + (entity) => !isNil(entity) + ); + + const mappedRelatedDataAssets = relatedEntitiesData?.reduce((acc, item) => { + let reference; + + if (typeof item === 'string') { + const foundOption = initialDataAssetsOptions.find( + (option) => option.reference.id === item + ); + reference = foundOption?.reference; + } else { + reference = item.reference; + } + + if (!isNil(reference)) { + acc.push(reference); + } + + return acc; + }, [] as EntityReference[]); + + const relatedEntities = [ + ...restRelatedDataAssets, + ...(mappedRelatedDataAssets ?? []), + ]; + + const updatedValues = { ...values, relatedEntities }; + + if (!isUndefined(quickLink)) { + handleQuickLinkUpdate(quickLink, updatedValues); + } else { + onSave(updatedValues); + } + }; + + const formFields: FieldProp[] = [ + { + name: 'displayName', + id: 'root/displayName', + required: false, + label: t('label.display-name'), + type: FieldTypes.TEXT, + props: { + 'data-testid': 'displayName', + disabled: !(permissions.EditAll || permissions.EditDisplayName), + }, + placeholder: t('label.display-name'), + }, + { + name: 'url', + id: 'root/url', + required: true, + label: t('label.url-uppercase'), + type: FieldTypes.TEXT, + props: { + 'data-testid': 'url', + type: 'url', + disabled: !permissions.EditAll, + }, + placeholder: t('label.url-uppercase'), + }, + { + name: 'description', + required: false, + label: t('label.description'), + id: 'root/description', + type: FieldTypes.DESCRIPTION, + props: { + 'data-testid': 'description', + initialValue: '', + readonly: !(permissions.EditAll || permissions.EditDescription), + }, + }, + { + name: 'tags', + required: false, + label: t('label.tag-plural'), + id: 'root/tags', + type: FieldTypes.TAG_SUGGESTION, + props: { + 'data-testid': 'tags-container', + initialOptions: classification.map((tag) => ({ + label: tag.tagFQN, + value: tag.tagFQN, + data: tag, + })), + disabled: !(permissions.EditAll || permissions.EditTags), + }, + }, + { + name: 'glossaryTerms', + required: false, + label: t('label.glossary-term'), + id: 'root/glossaryTerms', + type: FieldTypes.TAG_SUGGESTION, + props: { + 'data-testid': 'glossaryTerms-container', + open: false, + hasNoActionButtons: true, + isTreeSelect: true, + tagType: TagSource.Glossary, + placeholder: t('label.select-field', { + field: t('label.glossary-term'), + }), + initialOptions: glossaries.map((glossary) => ({ + label: glossary.tagFQN, + value: glossary.tagFQN, + data: glossary, + })), + disabled: !(permissions.EditAll || permissions.EditTags), + }, + }, + ]; + + return ( + + + + ); +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/RelatedDataAssets/RelatedDataAssets.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/RelatedDataAssets/RelatedDataAssets.test.tsx new file mode 100644 index 000000000000..2741aaaff791 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/RelatedDataAssets/RelatedDataAssets.test.tsx @@ -0,0 +1,269 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { act, fireEvent, render, screen } from '@testing-library/react'; +import { MemoryRouter } from 'react-router-dom'; +import RelatedDataAssets from './RelatedDataAssets'; + +jest.mock('./RelatedDataAssetsForm', () => ({ + RelatedDataAssetsForm: () =>
, +})); + +const mockRelatedDataAssets = [ + { id: 'team-123', type: 'team', name: 'team1', displayName: 'team1' }, + { id: 'user-123', type: 'user', name: 'user1', displayName: 'user1' }, + { id: 'table-123', type: 'table', name: 'table1', displayName: 'table1' }, + { id: 'topic-123', type: 'topic', name: 'topic1', displayName: 'topic1' }, +]; +const mockOnRelatedDataAssetsUpdate = jest.fn(); + +jest.mock('utils/EntityUtils', () => ({ + getEntityName: jest + .fn() + .mockImplementation((entity) => entity.displayName || entity.name), +})); +jest.mock('utils/TableUtils', () => ({ + getEntityIcon: jest.fn(), +})); + +jest.mock('pages/TasksPage/shared/DescriptionTaskNew'); +jest.mock('pages/TasksPage/shared/DescriptionTask'); + +describe('RelatedDataAssets', () => { + it('should render', () => { + render( + , + { wrapper: MemoryRouter } + ); + + expect(screen.getByTestId('header-label')).toBeInTheDocument(); + expect(screen.getByText('label.data-asset-plural')).toBeInTheDocument(); + + expect(screen.getByTestId('edit-data-assets')).toBeInTheDocument(); + + expect(screen.getByTestId('data-assets-list-body')).toBeInTheDocument(); + + // should render the assets + expect(screen.getByTestId('table1')).toBeInTheDocument(); + expect(screen.getByTestId('topic1')).toBeInTheDocument(); + + // should not render the team type as it is not a data asset + expect(screen.queryByTestId('team1')).not.toBeInTheDocument(); + + // should not render the user type as it is not a data asset + expect(screen.queryByTestId('user1')).not.toBeInTheDocument(); + + // should not render the add data assets button + expect( + screen.queryByTestId('add-data-assets-button') + ).not.toBeInTheDocument(); + + // should not render the show more button + expect(screen.queryByTestId('show-more-button')).not.toBeInTheDocument(); + + // should not render the edit form + expect(screen.queryByTestId('dataAssetsForm')).not.toBeInTheDocument(); + }); + + it('should render the add data assets button if no relatedDataAssets', async () => { + render( + , + { wrapper: MemoryRouter } + ); + + expect( + await screen.findByTestId('add-data-assets-container') + ).toBeInTheDocument(); + + // should not render the edit button + expect(screen.queryByTestId('edit-data-assets')).not.toBeInTheDocument(); + }); + + it('should render the show more button if relatedDataAssets.length > 5', () => { + render( + , + { wrapper: MemoryRouter } + ); + + expect(screen.getByTestId('show-more')).toBeInTheDocument(); + }); + + it('should render the edit form when edit button is clicked', async () => { + render( + , + { wrapper: MemoryRouter } + ); + + expect(screen.queryByTestId('dataAssetsForm')).not.toBeInTheDocument(); + + // click on edit button + await act(async () => { + fireEvent.click(screen.getByTestId('edit-data-assets')); + }); + + expect(screen.getByTestId('dataAssetsForm')).toBeInTheDocument(); + }); + + it('should render the show less button when show more button is clicked', async () => { + render( + , + { wrapper: MemoryRouter } + ); + + // should not render the hidden assets + expect(screen.queryByTestId('table59')).not.toBeInTheDocument(); + expect(screen.queryByTestId('table60')).not.toBeInTheDocument(); + + // click on show more button + await act(async () => { + fireEvent.click(screen.getByTestId('show-more')); + }); + + expect(screen.getByTestId('show-less')).toBeInTheDocument(); + + // should render the hidden assets + expect(screen.getByTestId('table59')).toBeInTheDocument(); + expect(screen.getByTestId('table60')).toBeInTheDocument(); + }); + + it('should render the show more button when show less button is clicked', async () => { + render( + , + { wrapper: MemoryRouter } + ); + + // click on show more button + await act(async () => { + fireEvent.click(screen.getByTestId('show-more')); + }); + + // should render the hidden assets + expect(screen.getByTestId('table59')).toBeInTheDocument(); + expect(screen.getByTestId('table60')).toBeInTheDocument(); + + // click on show less button + await act(async () => { + fireEvent.click(screen.getByTestId('show-less')); + }); + + expect(screen.getByTestId('show-more')).toBeInTheDocument(); + + // should not render the hidden assets + expect(screen.queryByTestId('table59')).not.toBeInTheDocument(); + expect(screen.queryByTestId('table60')).not.toBeInTheDocument(); + }); + + it("should render nothing if user doesn't have permission and no related data assets", () => { + render( + , + { wrapper: MemoryRouter } + ); + + expect(screen.queryByTestId('header-label')).not.toBeInTheDocument(); + expect( + screen.queryByText('label.data-asset-plural') + ).not.toBeInTheDocument(); + + expect(screen.queryByTestId('edit-data-assets')).not.toBeInTheDocument(); + + expect( + screen.queryByTestId('data-assets-list-body') + ).not.toBeInTheDocument(); + + // should not render the add data assets button + expect( + screen.queryByTestId('add-data-assets-button') + ).not.toBeInTheDocument(); + + // should not render the show more button + expect(screen.queryByTestId('show-more-button')).not.toBeInTheDocument(); + + // should not render the edit form + expect(screen.queryByTestId('dataAssetsForm')).not.toBeInTheDocument(); + }); + + it("should not render the add data assets button if no relatedDataAssets and user doesn't have permission", () => { + render( + , + { wrapper: MemoryRouter } + ); + + expect( + screen.queryByTestId('add-data-assets-button') + ).not.toBeInTheDocument(); + }); + + it("should not render the edit data assets button if user doesn't have permission", () => { + render( + , + { wrapper: MemoryRouter } + ); + + expect(screen.queryByTestId('edit-data-assets')).not.toBeInTheDocument(); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/RelatedDataAssets/RelatedDataAssets.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/RelatedDataAssets/RelatedDataAssets.tsx new file mode 100644 index 000000000000..140724410baa --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/RelatedDataAssets/RelatedDataAssets.tsx @@ -0,0 +1,248 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Button, Space, Typography } from 'antd'; +import { AxiosError } from 'axios'; +import { isEmpty } from 'lodash'; +import { KnowledgePage } from '../../../interface/knowledge-center.interface'; + +import { FC, useCallback, useMemo, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { Link } from 'react-router-dom'; +import ExpandableCard from '../../../components/common/ExpandableCard/ExpandableCard'; +import { + EditIconButton, + PlusIconButton, +} from '../../../components/common/IconButtons/EditIconButton'; +import { DataAssetOption } from '../../../components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList.interface'; +import { EntityReference } from '../../../generated/entity/type'; +import entityUtilClassBase from '../../../utils/EntityUtilClassBase'; +import { getEntityName } from '../../../utils/EntityUtils'; +import { getEntityIcon } from '../../../utils/TableUtils'; +import { showErrorToast } from '../../../utils/ToastUtils'; +import { RelatedDataAssetsForm } from './RelatedDataAssetsForm'; + +interface RelatedDataAssetsProps { + hasPermission: boolean; + relatedDataAssets: KnowledgePage['relatedEntities']; + onRelatedDataAssetsUpdate?: ( + data: KnowledgePage['relatedEntities'] + ) => Promise; +} + +const RelatedDataAssets: FC = ({ + relatedDataAssets = [], + onRelatedDataAssetsUpdate, + hasPermission, +}) => { + const { t } = useTranslation(); + const [isEdit, setIsEdit] = useState(false); + const [isShowMore, setIsShowMore] = useState(false); + + const { + filteredRelatedDataAssets, + defaultValue, + initialOptions, + restRelatedDataAssets, + } = useMemo(() => { + const { filteredRelatedDataAssets, restRelatedDataAssets } = + relatedDataAssets.reduce( + (acc, item) => { + // filter out team and user as they are not data assets + if (!['team', 'user'].includes(item.type)) { + acc.filteredRelatedDataAssets.push(item); + } else { + acc.restRelatedDataAssets.push(item); + } + + return acc; + }, + { + filteredRelatedDataAssets: [] as EntityReference[], + restRelatedDataAssets: [] as EntityReference[], + } + ); + + const initialOptions: DataAssetOption[] = filteredRelatedDataAssets.map( + (item) => { + return { + displayName: getEntityName(item), + reference: item, + label: getEntityName(item), + value: item.id, + }; + } + ); + + const defaultValue = filteredRelatedDataAssets.map((item) => item.id); + + return { + initialOptions, + defaultValue, + filteredRelatedDataAssets, + restRelatedDataAssets, + }; + }, [relatedDataAssets]); + + const { visibleDataAssets, hiddenDataAssets } = useMemo(() => { + const visibleDataAssets = filteredRelatedDataAssets.slice(0, 5); + const hiddenDataAssets = filteredRelatedDataAssets.slice(5); + + return { visibleDataAssets, hiddenDataAssets }; + }, [filteredRelatedDataAssets]); + + const showMoreLessElement = useMemo(() => { + return ( + setIsShowMore(!isShowMore)}> + {isShowMore ? t('label.show-less') : t('label.show-more')} + + ); + }, [isShowMore, hiddenDataAssets]); + + const getDataAssetListing = useCallback((dataAssets: EntityReference[]) => { + return dataAssets.map((item) => { + return ( +
+
+ +
+ } + title={getEntityName(item)} + type="text"> + + {getEntityName(item)} + + + +
+
+ ); + }); + }, []); + + const handleAssetsUpdate = useCallback( + async (updatedAssets: DataAssetOption[]) => { + try { + const updatedRelatedDataAssets = updatedAssets.map( + (item) => item.reference + ); + await onRelatedDataAssetsUpdate?.([ + ...restRelatedDataAssets, + ...updatedRelatedDataAssets, + ]); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + setIsEdit(false); + } + }, + [onRelatedDataAssetsUpdate, restRelatedDataAssets] + ); + + const header = useMemo(() => { + return ( + + + {t('label.data-asset-plural')} + + {!isEdit && + hasPermission && + (isEmpty(filteredRelatedDataAssets) ? ( + setIsEdit(true)} + /> + ) : ( + setIsEdit(true)} + /> + ))} + + ); + }, [isEdit, hasPermission, filteredRelatedDataAssets]); + + const content = useMemo(() => { + if (isEdit) { + return ( + setIsEdit(false)} + onSubmit={handleAssetsUpdate} + /> + ); + } + + return isEmpty(filteredRelatedDataAssets) ? null : ( +
+ {getDataAssetListing(visibleDataAssets)} + {isShowMore && getDataAssetListing(hiddenDataAssets)} + {!isEmpty(hiddenDataAssets) && showMoreLessElement} +
+ ); + }, [ + isEdit, + hasPermission, + filteredRelatedDataAssets, + isShowMore, + visibleDataAssets, + hiddenDataAssets, + ]); + + if (isEmpty(filteredRelatedDataAssets) && !hasPermission) { + return null; + } + + return ( + + {content} + + ); +}; + +export default RelatedDataAssets; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/RelatedDataAssets/RelatedDataAssetsForm.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/RelatedDataAssets/RelatedDataAssetsForm.test.tsx new file mode 100644 index 000000000000..795d20efb73f --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/RelatedDataAssets/RelatedDataAssetsForm.test.tsx @@ -0,0 +1,72 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; + +import { DataAssetOption } from 'components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList.interface'; +import { RelatedDataAssetsForm } from './RelatedDataAssetsForm'; +jest.mock( + 'components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList', + () => jest.fn(() =>
) +); + +const mockCancel = jest.fn(); +const mockSubmit = jest.fn(); +const mockDefaultValues: string[] = []; +const mockInitialOptions: DataAssetOption[] = []; + +describe('RelatedDataAssetsForm', () => { + it('should render', () => { + render( + + ); + + expect(screen.getByTestId('DataAssetAsyncSelectList')).toBeInTheDocument(); + expect(screen.getByTestId('cancelDataAssets')).toBeInTheDocument(); + expect(screen.getByTestId('saveDataAssets')).toBeInTheDocument(); + }); + + it('should call onCancel when cancel button is clicked', () => { + render( + + ); + fireEvent.click(screen.getByTestId('cancelDataAssets')); + + expect(mockCancel).toHaveBeenCalled(); + }); + + it('should call onSubmit when save button is clicked', async () => { + render( + + ); + fireEvent.click(screen.getByTestId('saveDataAssets')); + + await waitFor(() => { + expect(mockSubmit).toHaveBeenCalled(); + }); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/RelatedDataAssets/RelatedDataAssetsForm.tsx b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/RelatedDataAssets/RelatedDataAssetsForm.tsx new file mode 100644 index 000000000000..ef44f3fca8ec --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/KnowledgeCenter/RelatedDataAssets/RelatedDataAssetsForm.tsx @@ -0,0 +1,96 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { CheckOutlined, CloseOutlined } from '@ant-design/icons'; +import { Button, Col, Form, Row, Space } from 'antd'; +import DataAssetAsyncSelectList from '../../../components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList'; +import { DataAssetOption } from '../../../components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList.interface'; + +import { FC, useState } from 'react'; +import i18n from '../../../utils/i18next/LocalUtil'; + +interface RelatedDataAssetsFormProps { + defaultValue?: string[]; + initialOptions?: DataAssetOption[]; + onSubmit: (option: DataAssetOption[]) => Promise; + onCancel: () => void; +} + +const knowledgeCenterQueryFilter = { + query: { + bool: { + must_not: [ + { term: { entityType: 'dataProduct' } }, + { term: { entityType: 'domain' } }, + { match: { isBot: true } }, + ], + }, + }, +}; + +export const RelatedDataAssetsForm: FC = ({ + defaultValue, + initialOptions, + onCancel, + onSubmit, +}) => { + const { t } = i18n; + const [form] = Form.useForm(); + const [isSubmitLoading, setIsSubmitLoading] = useState(false); + + return ( +
{ + setIsSubmitLoading(true); + onSubmit(data['dataAssets']); + }}> + + + +