diff --git a/.vscode/mcp.json b/.vscode/mcp.json new file mode 100644 index 000000000..1e68ac5d6 --- /dev/null +++ b/.vscode/mcp.json @@ -0,0 +1,9 @@ +{ + "servers": { + "figma": { + "url": "http://127.0.0.1:3845/mcp", + "type": "http" + } + }, + "inputs": [] +} \ No newline at end of file diff --git a/web/.env b/web/.env index 6b193d147..631859d4f 100644 --- a/web/.env +++ b/web/.env @@ -893,3 +893,4 @@ OIDC_DISABLE_DPOP=false # Can be "auto", "iframe" or "full page redirect" OIDC_SESSION_RESTORATION_METHOD=auto + diff --git a/web/package.json b/web/package.json index ffcfd0e77..df5df482c 100644 --- a/web/package.json +++ b/web/package.json @@ -54,7 +54,7 @@ "async-mutex": "^0.5.0", "axios": "^1.9.0", "bytes": "^3.1.2", - "clean-architecture": "^6.0.3", + "clean-architecture": "^6.1.0", "codemirror": "6.0.1", "codemirror-json-schema": "0.7.9", "compare-versions": "^6.1.1", @@ -70,7 +70,7 @@ "minimal-polyfills": "^2.2.3", "mui-icons-material-lazy": "^1.0.4", "oidc-spa": "^10.0.4", - "onyxia-ui": "^6.7.8", + "onyxia-ui": "^6.10.0", "pathe": "^2.0.3", "powerhooks": "^2.0.1", "react": "^18.3.1", @@ -78,7 +78,7 @@ "react-dom": "^18.3.1", "run-exclusive": "^2.2.19", "screen-scaler": "^2.0.0", - "tsafe": "^1.8.5", + "tsafe": "^1.8.12", "tss-react": "^4.9.18", "type-route": "1.1.0", "xterm": "^5.3.0", diff --git a/web/spec.md b/web/spec.md new file mode 100644 index 000000000..6624f2c1b --- /dev/null +++ b/web/spec.md @@ -0,0 +1,57 @@ +Before: + +```js +{ + workingDirectory: { + bucketMode: "multi", + bucketNamePrefix: "", + bucketNamePrefixGroup: "projet-" + }, + bookmarkedDirectories: [ + { + fullPath: "donnees-insee/diffusion/", + title: { + fr: "Données de diffusion", + en: "Dissemination Data" + }, + description: { + fr: "Bucket public destiné à la diffusion de données", + en: "Public bucket intended for data dissemination" + } + } + ] +} +``` + +After: + +```js +{ + bookmarkedDirectories: [ + { + fullPath: "$1/", + title: "Personal", + description: "Personal Bucket", + claimName: "preferred_username" + }, + { + fullPath: "projet-$1/", + title: "Group $1", + description: "Shared bucket among members of project $1", + claimName: "groups", + excludedClaimPattern: "^USER_ONYXIA$" + }, + { + fullPath: "donnees-insee/diffusion/", + title: { + fr: "Données de diffusion", + en: "Dissemination Data" + }, + description: { + fr: "Bucket public destiné à la diffusion de données", + en: "Public bucket intended for data dissemination" + } + } + ]; +} +``` diff --git a/web/src/core/adapters/onyxiaApi/ApiTypes.ts b/web/src/core/adapters/onyxiaApi/ApiTypes.ts index 1e1464a2d..b6b982ab0 100644 --- a/web/src/core/adapters/onyxiaApi/ApiTypes.ts +++ b/web/src/core/adapters/onyxiaApi/ApiTypes.ts @@ -83,6 +83,7 @@ export type ApiTypes = { }; data?: { S3?: ArrayOrNot<{ + profileName?: string; URL: string; pathStyleAccess?: true; @@ -90,16 +91,25 @@ export type ApiTypes = { sts?: { URL?: string; durationSeconds?: number; - role: - | { - roleARN: string; - roleSessionName: string; - } - | undefined; + role?: ArrayOrNot< + { + roleARN: string; + roleSessionName: string; + profileName: string; + } & ( + | { claimName?: undefined } + | { + claimName: string; + includedClaimPattern?: string; + excludedClaimPattern?: string; + } + ) + >; oidcConfiguration?: Partial; }; /** Ok to be undefined only if sts is undefined */ + // NOTE: Remove in next major workingDirectory?: | { bucketMode: "shared"; @@ -115,14 +125,15 @@ export type ApiTypes = { bookmarkedDirectories?: ({ fullPath: string; title: LocalizedString; - description: LocalizedString | undefined; - tags: LocalizedString[] | undefined; + description?: LocalizedString; + tags?: LocalizedString[]; + forProfileName?: string | string[]; } & ( - | { claimName: undefined } + | { claimName?: undefined } | { claimName: string; - includedClaimPattern: string; - excludedClaimPattern: string; + includedClaimPattern?: string; + excludedClaimPattern?: string; } ))[]; }>; diff --git a/web/src/core/adapters/onyxiaApi/onyxiaApi.ts b/web/src/core/adapters/onyxiaApi/onyxiaApi.ts index 02c9892c2..96094b159 100644 --- a/web/src/core/adapters/onyxiaApi/onyxiaApi.ts +++ b/web/src/core/adapters/onyxiaApi/onyxiaApi.ts @@ -232,7 +232,141 @@ export function createOnyxiaApi(params: { return [value]; })(); - const s3ConfigCreationFormDefaults: DeploymentRegion["s3ConfigCreationFormDefaults"] = + const s3Profiles: DeploymentRegion.S3Profile[] = + s3Configs_api + .filter( + s3Configs_api => + s3Configs_api.sts !== undefined + ) + .map( + ( + s3Config_api + ): DeploymentRegion.S3Profile => { + return { + profileName: s3Config_api.profileName, + url: s3Config_api.URL, + pathStyleAccess: + s3Config_api.pathStyleAccess ?? + true, + region: s3Config_api.region, + sts: (() => { + const sts_api = s3Config_api.sts; + + assert(sts_api !== undefined); + + return { + url: sts_api.URL, + durationSeconds: + sts_api.durationSeconds, + roles: (() => { + if ( + sts_api.role === + undefined + ) { + return []; + } + + const rolesArray = + sts_api.role instanceof + Array + ? sts_api.role + : [sts_api.role]; + + return rolesArray.map( + ( + role_api + ): DeploymentRegion.S3Profile.StsRole => ({ + roleARN: + role_api.roleARN, + roleSessionName: + role_api.roleSessionName, + profileName: + role_api.profileName, + ...(role_api.claimName === + undefined + ? { + claimName: + undefined + } + : { + claimName: + role_api.claimName, + includedClaimPattern: + role_api.includedClaimPattern, + excludedClaimPattern: + role_api.excludedClaimPattern + }) + }) + ); + })(), + oidcParams: + apiTypesOidcConfigurationToOidcParams_Partial( + sts_api.oidcConfiguration + ) + } as any; + })(), + bookmarks: ( + s3Config_api.bookmarkedDirectories ?? + [] + ).map( + ( + bookmarkedDirectory_api + ): DeploymentRegion.S3Profile.Bookmark => { + return id( + { + s3UriStr_templated: `s3://${bookmarkedDirectory_api.fullPath}`, + title: bookmarkedDirectory_api.title, + description: + bookmarkedDirectory_api.description, + tags: + bookmarkedDirectory_api.tags ?? + [], + forProfileNames: + (() => { + const v = + bookmarkedDirectory_api.forProfileName; + + if ( + v === + undefined + ) { + return []; + } + + if ( + typeof v === + "string" + ) { + return [ + v + ]; + } + + return v; + })(), + ...(bookmarkedDirectory_api.claimName === + undefined + ? { + claimName: + undefined + } + : { + claimName: + bookmarkedDirectory_api.claimName, + includedClaimPattern: + bookmarkedDirectory_api.includedClaimPattern, + excludedClaimPattern: + bookmarkedDirectory_api.excludedClaimPattern + }) + } + ); + } + ) + }; + } + ); + + const s3Profiles_defaultValuesOfCreationForm: DeploymentRegion["s3Profiles_defaultValuesOfCreationForm"] = (() => { const s3Config_api = (() => { config_without_sts: { @@ -265,54 +399,13 @@ export function createOnyxiaApi(params: { url: s3Config_api.URL, pathStyleAccess: s3Config_api.pathStyleAccess ?? true, - region: s3Config_api.region, - workingDirectory: - s3Config_api.workingDirectory + region: s3Config_api.region }; })(); - const s3Configs: DeploymentRegion["s3Configs"] = - s3Configs_api - .map(({ sts, workingDirectory, ...rest }) => { - if (sts === undefined) { - return undefined; - } - assert( - workingDirectory !== undefined, - "If region.data.S3.sts is not undefined workingDirectory must be specified" - ); - - return { - sts, - workingDirectory, - ...rest - }; - }) - .filter(exclude(undefined)) - .map(s3Config_api => ({ - url: s3Config_api.URL, - pathStyleAccess: - s3Config_api.pathStyleAccess ?? true, - region: s3Config_api.region, - sts: { - url: s3Config_api.sts.URL, - durationSeconds: - s3Config_api.sts.durationSeconds, - role: s3Config_api.sts.role, - oidcParams: - apiTypesOidcConfigurationToOidcParams_Partial( - s3Config_api.sts.oidcConfiguration - ) - }, - workingDirectory: - s3Config_api.workingDirectory, - bookmarkedDirectories: - s3Config_api.bookmarkedDirectories ?? [] - })); - return { - s3Configs, - s3ConfigCreationFormDefaults + s3Profiles, + s3Profiles_defaultValuesOfCreationForm }; })(), allowedURIPatternForUserDefinedInitScript: diff --git a/web/src/core/adapters/s3Client/s3Client.ts b/web/src/core/adapters/s3Client/s3Client.ts index c97b5dd0e..420916414 100644 --- a/web/src/core/adapters/s3Client/s3Client.ts +++ b/web/src/core/adapters/s3Client/s3Client.ts @@ -1,22 +1,16 @@ -import type { S3BucketPolicy, S3Client, S3Object } from "core/ports/S3Client"; +import type { S3Client } from "core/ports/S3Client"; import { getNewlyRequestedOrCachedTokenFactory, createSessionStorageTokenPersistence } from "core/tools/getNewlyRequestedOrCachedToken"; -import { assert, is } from "tsafe/assert"; +import { assert, is, type Equals } from "tsafe/assert"; import type { Oidc } from "core/ports/Oidc"; -import { bucketNameAndObjectNameFromS3Path } from "./utils/bucketNameAndObjectNameFromS3Path"; -import { exclude } from "tsafe/exclude"; +import { getS3UriKey, parseS3Uri } from "core/tools/S3Uri"; +import { exclude, id } from "tsafe"; import { fnv1aHashToHex } from "core/tools/fnv1aHashToHex"; -import { getPolicyAttributes } from "core/tools/getPolicyAttributes"; -import { zS3BucketPolicy } from "./utils/policySchema"; -import { - addObjectNameToListBucketCondition, - addResourceArnInGetObjectStatement, - removeObjectNameFromListBucketCondition, - removeResourceArnInGetObjectStatement -} from "./utils/bucketPolicy"; import type { OidcParams_Partial } from "core/ports/OnyxiaApi"; +import { Evt } from "evt"; +import * as runExclusive from "run-exclusive"; export type ParamsOfCreateS3Client = | ParamsOfCreateS3Client.NoSts @@ -51,7 +45,6 @@ export namespace ParamsOfCreateS3Client { roleSessionName: string; } | undefined; - nameOfBucketToCreateIfNotExist: string | undefined; }; } @@ -222,51 +215,6 @@ export function createS3Client( return { getAwsS3Client }; })(); - create_bucket: { - if (!params.isStsEnabled) { - break create_bucket; - } - - const { nameOfBucketToCreateIfNotExist } = params; - - if (nameOfBucketToCreateIfNotExist === undefined) { - break create_bucket; - } - - const { awsS3Client } = await getAwsS3Client(); - - const { CreateBucketCommand, BucketAlreadyExists, BucketAlreadyOwnedByYou } = - await import("@aws-sdk/client-s3"); - - try { - await awsS3Client.send( - new CreateBucketCommand({ - Bucket: nameOfBucketToCreateIfNotExist - }) - ); - } catch (error) { - assert(is(error)); - - if ( - !(error instanceof BucketAlreadyExists) && - !(error instanceof BucketAlreadyOwnedByYou) - ) { - console.log( - "An unexpected error occurred while creating the bucket, we ignore it:", - error - ); - break create_bucket; - } - - console.log( - [ - `The above network error is expected we tried creating the `, - `bucket ${nameOfBucketToCreateIfNotExist} in case it didn't exist but it did.` - ].join(" ") - ); - } - } - return { getNewlyRequestedOrCachedToken, clearCachedToken, getAwsS3Client }; })(); @@ -280,325 +228,212 @@ export function createS3Client( return getNewlyRequestedOrCachedToken(); }, - listObjects: async ({ path }) => { - const { bucketName, prefix } = (() => { - const { bucketName, objectName } = - bucketNameAndObjectNameFromS3Path(path); - - const prefix = - objectName === "" - ? "" - : objectName.endsWith("/") - ? objectName - : `${objectName}/`; - - return { - bucketName, - prefix - }; - })(); - + listObjects: async ({ s3Uri }) => { const { getAwsS3Client } = await prApi; const { awsS3Client } = await getAwsS3Client(); - const { isBucketPolicyAvailable, allowedPrefix, bucketPolicy } = - await (async () => { - const { GetBucketPolicyCommand, S3ServiceException } = await import( - "@aws-sdk/client-s3" - ); - - let sendResp: import("@aws-sdk/client-s3").GetBucketPolicyCommandOutput; - try { - sendResp = await awsS3Client.send( - new GetBucketPolicyCommand({ Bucket: bucketName }) - ); - } catch (error) { - if (!(error instanceof S3ServiceException)) { - console.error( - "An unknown error occurred when fetching bucket policy", - error - ); - return { - isBucketPolicyAvailable: false, - bucketPolicy: undefined, - allowedPrefix: [] - }; - } + const Bucket = s3Uri.bucket; + const Delimiter = s3Uri.delimiter; - switch (error.$metadata?.httpStatusCode) { - case 404: - console.info( - "Bucket policy does not exist (404), it's ok." - ); - return { - isBucketPolicyAvailable: true, - bucketPolicy: undefined, - allowedPrefix: [] - }; - case 403: - console.info("Access denied to bucket policy (403)."); - break; - default: - console.error("An S3 error occurred:", error.message); - break; - } - return { - isBucketPolicyAvailable: false, - bucketPolicy: undefined, - allowedPrefix: [] - }; - } - - if (!sendResp.Policy) { - return { - isBucketPolicyAvailable: true, - bucketPolicy: undefined, - allowedPrefix: [] - }; - } - - const s3BucketPolicy = (() => { - const s3BucketPolicy = JSON.parse(sendResp.Policy); - - try { - // Validate and parse the policy - zS3BucketPolicy.parse(s3BucketPolicy); - } catch (error) { - console.error( - "Bucket policy isn't of the expected shape", - error - ); - return undefined; - } + const listObjectsV2Command = new ( + await import("@aws-sdk/client-s3") + ).ListObjectsV2Command({ + Bucket, + Prefix: getS3UriKey(s3Uri), + Delimiter, + MaxKeys: 1_000 + }); - assert(is(s3BucketPolicy)); + let resp: import("@aws-sdk/client-s3").ListObjectsV2CommandOutput; - return s3BucketPolicy; - })(); + try { + resp = await awsS3Client.send(listObjectsV2Command); + } catch (error) { + const { NoSuchBucket, S3ServiceException } = await import( + "@aws-sdk/client-s3" + ); - if (s3BucketPolicy === undefined) { - return { - isBucketPolicyAvailable: false, - bucketPolicy: undefined, - allowedPrefix: [] - }; - } + if (error instanceof NoSuchBucket) { + return id({ + isSuccess: false, + errorCase: "no such bucket" + }); + } - // Extract allowed prefixes based on the policy statements - const allowedPrefix = (s3BucketPolicy.Statement ?? []) - .filter( - statement => - statement.Effect === "Allow" && - (statement.Action.includes("s3:GetObject") || - statement.Action.includes("s3:*")) - ) - .flatMap(statement => - Array.isArray(statement.Resource) - ? statement.Resource - : [statement.Resource] - ) - .map(resource => - resource.replace(`arn:aws:s3:::${bucketName}/`, "") - ); + if ( + error instanceof S3ServiceException && + error.$metadata?.httpStatusCode === 403 + ) { + return id({ + isSuccess: false, + errorCase: "access denied" + }); + } - return { - isBucketPolicyAvailable: true, - bucketPolicy: s3BucketPolicy, - allowedPrefix - }; - })(); + throw error; + } - const Contents: import("@aws-sdk/client-s3")._Object[] = []; - const CommonPrefixes: import("@aws-sdk/client-s3").CommonPrefix[] = []; + return id({ + isSuccess: true, + objects: (resp.Contents ?? []) + .map(({ Key, LastModified, Size }) => + Key === undefined + ? undefined + : { + key: Key, + LastModified, + Size + } + ) + .filter(exclude(undefined)) + .map(({ key, LastModified, Size }) => { + assert(LastModified !== undefined); + assert(Size !== undefined); + const s3Uri = parseS3Uri({ + delimiter: Delimiter, + value: `s3://${Bucket}/${key}` + }); + assert(!s3Uri.isDelimiterTerminated); - { - let continuationToken: string | undefined; + return id({ + s3Uri, + lastModified: LastModified.getTime(), + size: Size + }); + }), - do { - const resp = await awsS3Client.send( - new (await import("@aws-sdk/client-s3")).ListObjectsV2Command({ - Bucket: bucketName, - Prefix: prefix, - Delimiter: "/", - ContinuationToken: continuationToken - }) - ); + prefixes: (resp.CommonPrefixes ?? []) + .map(({ Prefix }) => Prefix) + .filter(prefix => prefix !== undefined) + .map(prefix => { + const s3Uri = parseS3Uri({ + delimiter: Delimiter, + value: `s3://${Bucket}/${prefix}` + }); - Contents.push(...(resp.Contents ?? [])); + assert(s3Uri.isDelimiterTerminated); - CommonPrefixes.push(...(resp.CommonPrefixes ?? [])); + return s3Uri; + }) + }); + }, + putObject: (() => { + const putObject_actual: S3Client["putObject"] = runExclusive.build( + async ({ s3Uri, blob, onUploadProgress, evtCancel }) => { + const { getAwsS3Client } = await prApi; - continuationToken = resp.NextContinuationToken; - } while (continuationToken !== undefined); - } + const [{ awsS3Client }, Upload] = await Promise.all([ + getAwsS3Client(), + import("@aws-sdk/lib-storage").then(({ Upload }) => Upload) + ]); - const policyAttributes = (path: string) => { - return getPolicyAttributes(allowedPrefix, path); - }; + if (evtCancel.postCount !== 0) { + return { + status: "canceled" + }; + } - const directories = CommonPrefixes.filter(exclude(undefined)) - .map(({ Prefix }) => Prefix) - .filter(exclude(undefined)) - .map(directoryPath => { - const split = directoryPath.split("/"); - return { - kind: "directory", - basename: split[split.length - 2], - ...policyAttributes(directoryPath) - } satisfies S3Object; - }); + const upload = new Upload({ + client: awsS3Client, + params: { + Bucket: s3Uri.bucket, + Key: getS3UriKey(s3Uri), + Body: blob, + ContentType: blob.type + } + }); - const files = Contents.filter(({ Key }) => Key !== undefined).map( - ({ Key, LastModified, Size }) => { - assert(Key !== undefined); - const split = Key.split("/"); - return { - kind: "file", - basename: split[split.length - 1], - size: Size, - lastModified: LastModified, - ...policyAttributes(Key) - } satisfies S3Object; - } - ); + const onHttpUploadProgress = (params: { + total?: number; + loaded?: number; + }) => { + const { total, loaded } = params; - return { - objects: [...directories, ...files], - bucketPolicy, - isBucketPolicyAvailable - }; - }, - setPathAccessPolicy: async ({ currentBucketPolicy, policy, path }) => { - const { getAwsS3Client } = await prApi; - const { awsS3Client } = await getAwsS3Client(); + if (total === undefined || loaded === undefined) { + return; + } - const { bucketName, objectName } = bucketNameAndObjectNameFromS3Path(path); - - const resourceArn = `arn:aws:s3:::${bucketName}/${objectName}*`; - const bucketArn = `arn:aws:s3:::${bucketName}`; - - const updatedStatements = (() => { - switch (policy) { - case "public": - return addResourceArnInGetObjectStatement( - addObjectNameToListBucketCondition( - currentBucketPolicy.Statement, - bucketArn, - objectName - ), - resourceArn - ); - case "private": - return removeResourceArnInGetObjectStatement( - removeObjectNameFromListBucketCondition( - currentBucketPolicy.Statement, - bucketArn, - objectName - ), - resourceArn - ); - } - })(); + if (total === 0) { + onUploadProgress?.({ uploadPercent: 99 }); + return; + } - const newBucketPolicy = { - ...currentBucketPolicy, - Statement: updatedStatements - } satisfies S3BucketPolicy; + const uploadPercent = Math.floor((loaded / total) * 100); - const command = new ( - await import("@aws-sdk/client-s3") - ).PutBucketPolicyCommand({ - Bucket: bucketName, - Policy: JSON.stringify(newBucketPolicy) - }); + if (uploadPercent !== 100) { + onUploadProgress?.({ uploadPercent }); + } + }; - await awsS3Client.send(command); + upload.on("httpUploadProgress", onHttpUploadProgress); - return newBucketPolicy; - }, - uploadFile: async ({ blob, path, onUploadProgress }) => { - const { getAwsS3Client } = await prApi; + evtCancel.attachOnce(() => { + upload.off("httpUploadProgress", onHttpUploadProgress); + upload.abort(); + }); - const [{ awsS3Client }, Upload] = await Promise.all([ - getAwsS3Client(), - import("@aws-sdk/lib-storage").then(({ Upload }) => Upload) - ]); - - const { bucketName, objectName } = bucketNameAndObjectNameFromS3Path(path); - - const upload = new Upload({ - client: awsS3Client, - params: { - Bucket: bucketName, - Key: objectName, - Body: blob, - ContentType: blob.type - }, - partSize: 5 * 1024 * 1024, // optional size of each part - leavePartsOnError: false // optional manually handle dropped parts - }); - upload.on("httpUploadProgress", ({ total, loaded }) => { - if (total === undefined || loaded === undefined) { - return; + const completionStatus = await Promise.race([ + (async () => { + try { + await upload.done(); + } catch (error) { + assert(error instanceof Error); + return { + case: "failed" as const, + error + }; + } + return { case: "success" as const }; + })(), + evtCancel.waitFor().then(() => ({ case: "canceled" as const })) + ]); + + switch (completionStatus.case) { + case "canceled": + return { status: "canceled" }; + case "failed": + return { status: "failed", error: completionStatus.error }; + case "success": + onUploadProgress?.({ uploadPercent: 100 }); + return { status: "success" }; + default: + assert>(false); + } } + ); - if (total === 0) { - onUploadProgress?.({ uploadPercent: 100 }); - return; - } + return async params => { + const ctx = Evt.newCtx(); - const uploadPercent = Math.floor((loaded / total) * 100); + const evtCancel = params.evtCancel.pipe(ctx); - onUploadProgress?.({ uploadPercent }); - }); + const putObjectResult = await Promise.race([ + putObject_actual({ + ...params, + evtCancel + }), + evtCancel.waitFor().then(() => ({ status: "canceled" as const })) + ]); - await upload.done(); - }, - deleteFile: async ({ path }) => { - const { bucketName, objectName } = bucketNameAndObjectNameFromS3Path(path); + ctx.done(); + return putObjectResult; + }; + })(), + deleteObject: async ({ s3Uri }) => { const { getAwsS3Client } = await prApi; const { awsS3Client } = await getAwsS3Client(); await awsS3Client.send( new (await import("@aws-sdk/client-s3")).DeleteObjectCommand({ - Bucket: bucketName, - Key: objectName + Bucket: s3Uri.bucket, + Key: getS3UriKey(s3Uri) }) ); }, - deleteFiles: async ({ paths }) => { - //bucketName is the same for all paths - const { bucketName } = bucketNameAndObjectNameFromS3Path(paths[0]); - - const { getAwsS3Client } = await prApi; - - const { awsS3Client } = await getAwsS3Client(); - - const { DeleteObjectsCommand } = await import("@aws-sdk/client-s3"); - - const objects = paths.map(path => { - const { objectName } = bucketNameAndObjectNameFromS3Path(path); - return { Key: objectName }; - }); - - try { - await awsS3Client.send( - new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { Objects: objects } - }) - ); - } catch (err) { - console.warn("Bulk delete failed, falling back to single deletes:", err); - await Promise.all(paths.map(path => s3Client.deleteFile({ path }))); - } - }, - getFileDownloadUrl: async ({ path, validityDurationSecond }) => { - const { bucketName, objectName } = bucketNameAndObjectNameFromS3Path(path); - + generateSignedDownloadUrl: async ({ s3Uri, validityDurationSecond }) => { const { getAwsS3Client } = await prApi; const { awsS3Client } = await getAwsS3Client(); @@ -608,8 +443,8 @@ export function createS3Client( ).getSignedUrl( awsS3Client, new (await import("@aws-sdk/client-s3")).GetObjectCommand({ - Bucket: bucketName, - Key: objectName + Bucket: s3Uri.bucket, + Key: getS3UriKey(s3Uri) }), { expiresIn: validityDurationSecond @@ -619,9 +454,7 @@ export function createS3Client( return downloadUrl; }, - getFileContent: async ({ path, range }) => { - const { bucketName, objectName } = bucketNameAndObjectNameFromS3Path(path); - + getObjectContent: async ({ s3Uri, range }) => { const { getAwsS3Client } = await prApi; const { awsS3Client } = await getAwsS3Client(); @@ -629,8 +462,8 @@ export function createS3Client( const response = await awsS3Client.send( new GetObjectCommand({ - Bucket: bucketName, - Key: objectName, + Bucket: s3Uri.bucket, + Key: getS3UriKey(s3Uri), ...(range !== undefined ? { Range: range } : {}) }) ); @@ -639,27 +472,70 @@ export function createS3Client( return { stream: response.Body, - lastModified: response.LastModified, size: response.ContentLength, contentType: response.ContentType }; }, - getFileContentType: async ({ path }) => { - const { bucketName, objectName } = bucketNameAndObjectNameFromS3Path(path); - + getObjectContentType: async ({ s3Uri }) => { const { getAwsS3Client } = await prApi; const { awsS3Client } = await getAwsS3Client(); const head = await awsS3Client.send( new (await import("@aws-sdk/client-s3")).HeadObjectCommand({ - Bucket: bucketName, - Key: objectName + Bucket: s3Uri.bucket, + Key: getS3UriKey(s3Uri) }) ); return head.ContentType; + }, + createBucket: async ({ bucket }) => { + const { getAwsS3Client } = await prApi; + + const { awsS3Client } = await getAwsS3Client(); + + const { + CreateBucketCommand, + BucketAlreadyExists, + BucketAlreadyOwnedByYou, + S3ServiceException + } = await import("@aws-sdk/client-s3"); + + try { + await awsS3Client.send( + new CreateBucketCommand({ + Bucket: bucket + }) + ); + } catch (error) { + assert(is(error)); + + if ( + error instanceof S3ServiceException && + error.$metadata?.httpStatusCode === 403 + ) { + return { + isSuccess: false, + errorCase: "access denied", + errorMessage: error.message + }; + } + + if ( + !(error instanceof BucketAlreadyExists) && + !(error instanceof BucketAlreadyOwnedByYou) + ) { + return { + isSuccess: false, + errorCase: "already exist", + errorMessage: error.message + }; + } + } + + return { isSuccess: true }; } }; diff --git a/web/src/core/adapters/s3Client/utils/bucketNameAndObjectNameFromS3Path.ts b/web/src/core/adapters/s3Client/utils/bucketNameAndObjectNameFromS3Path.ts deleted file mode 100644 index 1603254d1..000000000 --- a/web/src/core/adapters/s3Client/utils/bucketNameAndObjectNameFromS3Path.ts +++ /dev/null @@ -1,16 +0,0 @@ -/** - * "/bucket-name/object/name" => { bucketName: "bucket-name", objectName: "object/name" } - * "bucket-name/object/name" => { bucketName: "bucket-name", objectName: "object/name" } - * "bucket-name/object/name/" => { bucketName: "bucket-name", objectName: "object/name/" } - * "bucket-name/" => { bucketName: "bucket-name", objectName: "" } - * "bucket-name" => { bucketName: "bucket-name", objectName: "" } - * "s3://bucket-name/object/name" => { bucketName: "bucket-name", objectName: "object/name" } - */ -export function bucketNameAndObjectNameFromS3Path(path: string) { - const [bucketName, ...rest] = path.replace(/^(s3:)?\/+/, "").split("/"); - - return { - bucketName, - objectName: rest.join("/") - }; -} diff --git a/web/src/core/adapters/s3Client/utils/bucketPolicy.test.ts b/web/src/core/adapters/s3Client/utils/bucketPolicy.test.ts deleted file mode 100644 index 98a6d0587..000000000 --- a/web/src/core/adapters/s3Client/utils/bucketPolicy.test.ts +++ /dev/null @@ -1,548 +0,0 @@ -import { it, expect, describe } from "vitest"; -import { symToStr } from "tsafe/symToStr"; -import { - addObjectNameToListBucketCondition, - removeObjectNameFromListBucketCondition -} from "./bucketPolicy"; -import type { S3BucketPolicy } from "core/ports/S3Client"; - -describe(symToStr({ removeObjectNameFromListBucketCondition }), () => { - const bucketArn = "bucketArn"; - - describe("File", () => { - it("removes single file prefix, leaving no statement", () => { - const objectName = "objectName"; - const statements: S3BucketPolicy["Statement"] = [ - { - Action: ["s3:ListBucket"], - Condition: { - StringLike: { - "s3:prefix": [objectName] - } - }, - Effect: "Allow", - Principal: { AWS: ["*"] }, - Resource: [bucketArn] - } - ]; - - expect( - removeObjectNameFromListBucketCondition(statements, bucketArn, objectName) - ).toStrictEqual([]); - }); - - it("removes file from multiple, keeps other files", () => { - const objectName = "objectName"; - const statements: S3BucketPolicy["Statement"] = [ - { - Action: ["s3:ListBucket"], - Condition: { - StringLike: { - "s3:prefix": [objectName, "objectName2"] - } - }, - Effect: "Allow", - Principal: { AWS: ["*"] }, - Resource: [bucketArn] - } - ]; - - const expected = [ - { - ...statements[0], - Condition: { - StringLike: { - "s3:prefix": ["objectName2"] - } - } - } - ]; - - expect( - removeObjectNameFromListBucketCondition(statements, bucketArn, objectName) - ).toStrictEqual(expected); - }); - - it("keeps statement when file prefix is not found", () => { - const statements: S3BucketPolicy["Statement"] = [ - { - Action: ["s3:ListBucket"], - Condition: { - StringLike: { - "s3:prefix": ["objectName1", "objectName2"] - } - }, - Effect: "Allow", - Principal: { AWS: ["*"] }, - Resource: [bucketArn] - } - ]; - - expect( - removeObjectNameFromListBucketCondition( - statements, - bucketArn, - "nonexistent" - ) - ).toStrictEqual(statements); - }); - - it("keeps other statements unaffected", () => { - const objectName = "objectName"; - const statements: S3BucketPolicy["Statement"] = [ - { - Action: ["s3:ListBucket"], - Condition: { - StringLike: { - "s3:prefix": [objectName, "objectName2"] - } - }, - Effect: "Allow", - Principal: { AWS: ["*"] }, - Resource: [bucketArn] - }, - { - Action: ["s3:ListBucket"], - Condition: { - StringLike: { - "s3:prefix": ["otherDir"] - } - }, - Effect: "Allow", - Principal: { AWS: ["*"] }, - Resource: [bucketArn] - } - ]; - - const expected = [ - { - ...statements[0], - Condition: { - StringLike: { - "s3:prefix": ["objectName2"] - } - } - }, - statements[1] - ]; - - expect( - removeObjectNameFromListBucketCondition(statements, bucketArn, objectName) - ).toStrictEqual(expected); - }); - - it("removes file and leaves unrelated conditions intact", () => { - const objectName = "objectName"; - const statements: S3BucketPolicy["Statement"] = [ - { - Action: ["s3:ListBucket"], - Condition: { - StringLike: { - "s3:prefix": [objectName] - }, - NumericEquals: { - "aws:MultiFactorAuthAge": 300 - } - }, - Effect: "Allow", - Principal: { AWS: ["*"] }, - Resource: [bucketArn] - } - ]; - - const expected = [ - { - Action: ["s3:ListBucket"], - Condition: { - NumericEquals: { - "aws:MultiFactorAuthAge": 300 - } - }, - Effect: "Allow", - Principal: { AWS: ["*"] }, - Resource: [bucketArn] - } - ]; - - expect( - removeObjectNameFromListBucketCondition(statements, bucketArn, objectName) - ).toStrictEqual(expected); - }); - }); - - describe("Directory (StringLike)", () => { - it("removes single directory prefix, leaving no statement", () => { - const objectName = "directoryName/"; - const statements: S3BucketPolicy["Statement"] = [ - { - Action: ["s3:ListBucket"], - Condition: { - StringLike: { - "s3:prefix": ["directoryName/*"] - } - }, - Effect: "Allow", - Principal: { AWS: ["*"] }, - Resource: [bucketArn] - } - ]; - - expect( - removeObjectNameFromListBucketCondition(statements, bucketArn, objectName) - ).toStrictEqual([]); - }); - - it("removes one directory prefix and leaves others", () => { - const objectName = "dir1/"; - const statements: S3BucketPolicy["Statement"] = [ - { - Action: ["s3:ListBucket"], - Condition: { - StringLike: { - "s3:prefix": ["dir1/*", "dir2/*"] - } - }, - Effect: "Allow", - Principal: { AWS: ["*"] }, - Resource: [bucketArn] - } - ]; - - const expected = [ - { - ...statements[0], - Condition: { - StringLike: { - "s3:prefix": ["dir2/*"] - } - } - } - ]; - - expect( - removeObjectNameFromListBucketCondition(statements, bucketArn, objectName) - ).toStrictEqual(expected); - }); - - it("removes directory but preserves unrelated file prefix", () => { - const objectName = "folder/"; - const statements: S3BucketPolicy["Statement"] = [ - { - Action: ["s3:ListBucket"], - Condition: { - StringLike: { - "s3:prefix": ["folder/*", "file.txt"] - } - }, - Effect: "Allow", - Principal: { AWS: ["*"] }, - Resource: [bucketArn] - } - ]; - - const expected = [ - { - ...statements[0], - Condition: { - StringLike: { - "s3:prefix": ["file.txt"] - } - } - } - ]; - - expect( - removeObjectNameFromListBucketCondition(statements, bucketArn, objectName) - ).toStrictEqual(expected); - }); - }); - - describe("Edge cases", () => { - it("returns empty when input is empty", () => { - expect( - removeObjectNameFromListBucketCondition([], bucketArn, "objectName") - ).toStrictEqual([]); - }); - - it("returns unchanged when bucket ARN does not match", () => { - const statements: S3BucketPolicy["Statement"] = [ - { - Action: ["s3:ListBucket"], - Condition: { - StringLike: { - "s3:prefix": ["objectName"] - } - }, - Effect: "Allow", - Principal: { AWS: ["*"] }, - Resource: ["otherBucketArn"] - } - ]; - - expect( - removeObjectNameFromListBucketCondition( - statements, - bucketArn, - "objectName" - ) - ).toStrictEqual(statements); - }); - - it("keeps statement if unrelated condition type used", () => { - const objectName = "objectName"; - const statements: S3BucketPolicy["Statement"] = [ - { - Action: ["s3:ListBucket"], - Condition: { - StringNotEquals: { - "s3:prefix": [objectName] - } - }, - Effect: "Allow", - Principal: { AWS: ["*"] }, - Resource: [bucketArn] - } - ]; - - expect( - removeObjectNameFromListBucketCondition(statements, bucketArn, objectName) - ).toStrictEqual(statements); - }); - }); -}); - -describe(symToStr({ addObjectNameToListBucketCondition }), () => { - const bucketArn = "arn:aws:s3:::my-bucket"; - - it("adds a StringLike condition when none exist", () => { - const result = addObjectNameToListBucketCondition(null, bucketArn, "file.txt"); - - const expected: S3BucketPolicy["Statement"] = [ - { - Effect: "Allow", - Principal: { AWS: ["*"] }, - Action: ["s3:ListBucket"], - Resource: [bucketArn], - Condition: { - StringLike: { - "s3:prefix": ["file.txt"] - } - } - } - ]; - - expect(result).toStrictEqual(expected); - }); - - it("adds a StringLike condition when objectName ends with '/'", () => { - const result = addObjectNameToListBucketCondition(null, bucketArn, "folder/"); - - const expected: S3BucketPolicy["Statement"] = [ - { - Effect: "Allow", - Principal: { AWS: ["*"] }, - Action: ["s3:ListBucket"], - Resource: [bucketArn], - Condition: { - StringLike: { - "s3:prefix": ["folder/*"] - } - } - } - ]; - - expect(result).toStrictEqual(expected); - }); - - it("adds new prefix to existing StringLike array", () => { - const statements: S3BucketPolicy["Statement"] = [ - { - Effect: "Allow", - Principal: { AWS: ["*"] }, - Action: ["s3:ListBucket"], - Resource: [bucketArn], - Condition: { - StringLike: { - "s3:prefix": ["file1.txt"] - } - } - } - ]; - - const result = addObjectNameToListBucketCondition( - statements, - bucketArn, - "file2.txt" - ); - - const expected: S3BucketPolicy["Statement"] = [ - { - Effect: "Allow", - Principal: { AWS: ["*"] }, - Action: ["s3:ListBucket"], - Resource: [bucketArn], - Condition: { - StringLike: { - "s3:prefix": ["file1.txt", "file2.txt"] - } - } - } - ]; - - expect(result).toStrictEqual(expected); - }); - - it("adds new prefix to existing StringLike string", () => { - const statements: S3BucketPolicy["Statement"] = [ - { - Effect: "Allow", - Principal: { AWS: ["*"] }, - Action: ["s3:ListBucket"], - Resource: [bucketArn], - Condition: { - StringLike: { - "s3:prefix": "file1.txt" - } - } - } - ]; - - const result = addObjectNameToListBucketCondition( - statements, - bucketArn, - "file2.txt" - ); - - const expected: S3BucketPolicy["Statement"] = [ - { - Effect: "Allow", - Principal: { AWS: ["*"] }, - Action: ["s3:ListBucket"], - Resource: [bucketArn], - Condition: { - StringLike: { - "s3:prefix": ["file1.txt", "file2.txt"] - } - } - } - ]; - - expect(result).toStrictEqual(expected); - }); - - it("updates StringLike without affecting existing StringLike", () => { - const statements: S3BucketPolicy["Statement"] = [ - { - Effect: "Allow", - Principal: { AWS: ["*"] }, - Action: ["s3:ListBucket"], - Resource: [bucketArn], - Condition: { - StringLike: { - "s3:prefix": ["file.txt"] - } - } - } - ]; - - const result = addObjectNameToListBucketCondition(statements, bucketArn, "dir/"); - - const expected: S3BucketPolicy["Statement"] = [ - { - Effect: "Allow", - Principal: { AWS: ["*"] }, - Action: ["s3:ListBucket"], - Resource: [bucketArn], - Condition: { - StringLike: { - "s3:prefix": ["file.txt", "dir/*"] - } - } - } - ]; - - expect(result).toStrictEqual(expected); - }); - - it("does not duplicate existing prefix", () => { - const statements: S3BucketPolicy["Statement"] = [ - { - Effect: "Allow", - Principal: { AWS: ["*"] }, - Action: ["s3:ListBucket"], - Resource: [bucketArn], - Condition: { - StringLike: { - "s3:prefix": ["file.txt"] - } - } - } - ]; - - const result = addObjectNameToListBucketCondition( - statements, - bucketArn, - "file.txt" - ); - - const expected: S3BucketPolicy["Statement"] = [...statements]; - - expect(result).toStrictEqual(expected); - }); - - it("does not duplicate existing prefix for directory", () => { - const statements: S3BucketPolicy["Statement"] = [ - { - Effect: "Allow", - Principal: { AWS: ["*"] }, - Action: ["s3:ListBucket"], - Resource: [bucketArn], - Condition: { - StringLike: { - "s3:prefix": ["file/*"] - } - } - } - ]; - - const result = addObjectNameToListBucketCondition(statements, bucketArn, "file/"); - - const expected: S3BucketPolicy["Statement"] = [...statements]; - - expect(result).toStrictEqual(expected); - }); - - it("adds new statement if no ListBucket action is found", () => { - const statements: S3BucketPolicy["Statement"] = [ - { - Effect: "Allow", - Principal: { AWS: ["*"] }, - Action: ["s3:GetObject"], - Resource: ["arn:aws:s3:::my-bucket/file.txt"] - } - ]; - - const result = addObjectNameToListBucketCondition( - statements, - bucketArn, - "file.txt" - ); - - const expected: S3BucketPolicy["Statement"] = [ - ...statements, - { - Effect: "Allow", - Principal: { AWS: ["*"] }, - Action: ["s3:ListBucket"], - Resource: [bucketArn], - Condition: { - StringLike: { - "s3:prefix": ["file.txt"] - } - } - } - ]; - - expect(result).toStrictEqual(expected); - }); -}); diff --git a/web/src/core/adapters/s3Client/utils/bucketPolicy.ts b/web/src/core/adapters/s3Client/utils/bucketPolicy.ts deleted file mode 100644 index 32ae826fb..000000000 --- a/web/src/core/adapters/s3Client/utils/bucketPolicy.ts +++ /dev/null @@ -1,251 +0,0 @@ -import type { S3BucketPolicy } from "core/ports/S3Client"; - -// Adds `objectName` to the `s3:prefix` condition in the `s3:ListBucket` statement -export const addObjectNameToListBucketCondition = ( - statements: S3BucketPolicy["Statement"], - bucketArn: string, - objectName: string -): S3BucketPolicy["Statement"] => { - const { conditionKey, objectPrefix } = getConditionKeyAndPrefix(objectName); - - if (statements === null) { - return [ - { - Effect: "Allow", - Principal: { AWS: ["*"] }, - Action: ["s3:ListBucket"], - Resource: [bucketArn], - Condition: { - [conditionKey]: { - "s3:prefix": [objectPrefix] - } - } - } - ]; - } - - const listBucketStatementIndex = statements.findIndex( - statement => - statement.Action.includes("s3:ListBucket") && - statement.Resource.includes(bucketArn) - ); - - // If no statements exist or s3:ListBucket is not found, add a new statement - if (listBucketStatementIndex === -1) { - return [ - ...statements, - { - Effect: "Allow", - Principal: { AWS: ["*"] }, - Action: ["s3:ListBucket"], - Resource: [bucketArn], - Condition: { - [conditionKey]: { - "s3:prefix": [objectPrefix] - } - } - } - ]; - } - - const statement = statements[listBucketStatementIndex]; - const updatedStatement = { ...statement }; - - // Get existing "s3:prefix" array (or string) for the correct condition key - const existingCondition = updatedStatement.Condition?.[conditionKey]; - const existingPrefixesRaw = existingCondition?.["s3:prefix"]; - - const existingPrefixes: string[] = existingPrefixesRaw - ? Array.isArray(existingPrefixesRaw) - ? existingPrefixesRaw - : [existingPrefixesRaw] - : []; - - const newPrefixes = Array.from(new Set([...existingPrefixes, objectPrefix])); - - return statements.map((s, i) => - i === listBucketStatementIndex - ? { - ...s, - Condition: { - ...s.Condition, - [conditionKey]: { - ...s.Condition?.[conditionKey], - "s3:prefix": newPrefixes - } - } - } - : s - ); -}; - -// Removes `objectName` from the `s3:prefix` condition in the `s3:ListBucket` statement -export const removeObjectNameFromListBucketCondition = ( - statements: S3BucketPolicy["Statement"], - bucketArn: string, - objectName: string -) => { - if (statements === null) { - return null; - } - - const { conditionKey, objectPrefix } = getConditionKeyAndPrefix(objectName); - - return statements - .map(statement => { - if ( - statement.Action.includes("s3:ListBucket") && - statement.Resource.includes(bucketArn) - ) { - const updatedPrefixCondition: string[] = - statement.Condition?.[conditionKey]?.["s3:prefix"]?.filter( - (prefix: string) => prefix !== objectPrefix - ) ?? []; - - if (updatedPrefixCondition.length > 0) { - return { - ...statement, - Condition: { - ...statement.Condition, - [conditionKey]: { - ...statement.Condition?.[conditionKey], - "s3:prefix": updatedPrefixCondition - } - } - }; - } - - const updatedStringEquals = removeKey( - statement.Condition?.[conditionKey], - "s3:prefix" - ); - - // If "conditionKey" is still valid, return updated statement - if (updatedStringEquals) { - return { - ...statement, - Condition: { - ...statement.Condition, - [conditionKey]: updatedStringEquals - } - }; - } - - // Remove "conditionKey" from Condition - const updatedCondition = removeKey(statement.Condition, conditionKey); - - // If Condition is still valid, return updated statement - return updatedCondition - ? { ...statement, Condition: updatedCondition } - : undefined; - } - return statement; - }) - .filter(statement => statement !== undefined); -}; - -// Adds a new `s3:GetObject` statement for `resourceArn` -export const addResourceArnInGetObjectStatement = ( - statements: S3BucketPolicy["Statement"], - resourceArn: string -): S3BucketPolicy["Statement"] => { - if (statements === null) { - return [ - { - Effect: "Allow", - Principal: { AWS: ["*"] }, - Action: ["s3:GetObject"], - Resource: [resourceArn] - } - ]; - } - - const existingStatementIndex = statements.findIndex(statement => - Array.isArray(statement.Action) - ? statement.Action.includes("s3:GetObject") - : statement.Action === "s3:GetObject" - ); - - if (existingStatementIndex === -1) { - return [ - ...statements, - { - Effect: "Allow", - Principal: { AWS: ["*"] }, - Action: ["s3:GetObject"], - Resource: [resourceArn] - } - ]; - } - - return statements.map((statement, index) => - index === existingStatementIndex - ? { - ...statement, - Resource: Array.from(new Set([...statement.Resource, resourceArn])) // Avoid duplicate ARNs - } - : statement - ); -}; - -export const removeResourceArnInGetObjectStatement = ( - statements: S3BucketPolicy["Statement"], - resourceArn: string -): S3BucketPolicy["Statement"] => { - if (statements === null) { - return null; - } - const existingStatementIndex = statements.findIndex( - statement => - (Array.isArray(statement.Action) - ? statement.Action.includes("s3:GetObject") - : statement.Action === "s3:GetObject") && - statement.Resource.includes(resourceArn) - ); - - if (existingStatementIndex === -1) { - return statements; - } - - const existingStatement = statements[existingStatementIndex]; - const updatedResources = existingStatement.Resource.filter( - arn => arn !== resourceArn - ); - - return updatedResources.length > 0 - ? statements.map((statement, index) => - index === existingStatementIndex - ? { ...statement, Resource: updatedResources } - : statement - ) - : statements.filter((_, index) => index !== existingStatementIndex); -}; - -const removeKey = , K extends keyof T>( - obj: T | undefined, - key: K -): Omit | undefined => { - if (!obj || !(key in obj)) { - return obj; - } - - const { [key]: _, ...rest } = obj; - return Object.keys(rest).length > 0 ? rest : undefined; -}; - -function getConditionKeyAndPrefix(objectName: string): { - conditionKey: "StringLike"; - objectPrefix: string; -} { - if (objectName.endsWith("/")) { - return { - conditionKey: "StringLike", - objectPrefix: `${objectName}*` - }; - } - - return { - conditionKey: "StringLike", - objectPrefix: objectName - }; -} diff --git a/web/src/core/adapters/s3Client/utils/policySchema.ts b/web/src/core/adapters/s3Client/utils/policySchema.ts deleted file mode 100644 index d48999517..000000000 --- a/web/src/core/adapters/s3Client/utils/policySchema.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { S3BucketPolicy } from "core/ports/S3Client"; -import { assert, type Equals } from "tsafe"; -import { z } from "zod"; -import { id } from "tsafe/id"; - -const zS3Action = z.custom<`s3:${string}`>( - val => typeof val === "string" && val.startsWith("s3:") -); - -type S3PolicyStatement = NonNullable[number]; - -const zS3PolicyStatement = (() => { - type TargetType = S3PolicyStatement; - - const zTargetType = z.object({ - Effect: z.enum(["Allow", "Deny"]), - Principal: z.union([z.string(), z.object({ AWS: z.string().array() })]), - Action: z.union([zS3Action, zS3Action.array()]), - Resource: z.string().array(), - Condition: z.record(z.any()).optional() - }); - - type InferredType = z.infer; - - assert>; - - return id>(zTargetType); -})(); - -export const zS3BucketPolicy = (() => { - type TargetType = S3BucketPolicy; - - const zTargetType = z.object({ - Version: z.literal("2012-10-17"), - Statement: z.array(zS3PolicyStatement).nullable() - }); - - type InferredType = z.infer; - - assert>; - - return id>(zTargetType); -})(); diff --git a/web/src/core/adapters/sqlOlap/sqlOlap.ts b/web/src/core/adapters/sqlOlap/sqlOlap.ts index 9893da3c6..6febf3797 100644 --- a/web/src/core/adapters/sqlOlap/sqlOlap.ts +++ b/web/src/core/adapters/sqlOlap/sqlOlap.ts @@ -18,6 +18,7 @@ import type { S3Client } from "core/ports/S3Client"; import { Deferred } from "evt/tools/Deferred"; import { streamToArrayBuffer } from "core/tools/streamToArrayBuffer"; import { getHttpUrlWithoutRedirect } from "core/tools/getHttpUrlWithoutRedirect"; +import { parseS3Uri } from "core/tools/S3Uri"; export const createDuckDbSqlOlap = (params: { getS3Client: () => Promise< @@ -96,9 +97,15 @@ export const createDuckDbSqlOlap = (params: { dOut.resolve({ errorCause: errorCause_getS3Client }); return new Promise(() => {}); } + const s3Uri = parseS3Uri({ + value: sourceUrl, + delimiter: "/" + }); + + assert(!s3Uri.isDelimiterTerminated); - const result = await s3Client.getFileContent({ - path: sourceUrl.replace(/^s3:\/\//, ""), + const result = await s3Client.getObjectContent({ + s3Uri, range: "bytes=0-15" }); diff --git a/web/src/core/bootstrap.ts b/web/src/core/bootstrap.ts index 7dd134067..c07bf863f 100644 --- a/web/src/core/bootstrap.ts +++ b/web/src/core/bootstrap.ts @@ -157,7 +157,7 @@ export async function bootstrapCore( } const result = await dispatch( - usecases.s3ConfigManagement.protectedThunks.getS3ConfigAndClientForExplorer() + usecases.s3ProfilesManagement.protectedThunks.getAmbientS3ProfileAndClient() ); if (result === undefined) { @@ -166,15 +166,15 @@ export async function bootstrapCore( }; } - const { s3Config, s3Client } = result; + const { s3Profile, s3Client } = result; return { s3Client, - s3_endpoint: s3Config.paramsOfCreateS3Client.url, - s3_url_style: s3Config.paramsOfCreateS3Client.pathStyleAccess + s3_endpoint: s3Profile.paramsOfCreateS3Client.url, + s3_url_style: s3Profile.paramsOfCreateS3Client.pathStyleAccess ? "path" : "vhost", - s3_region: s3Config.region + s3_region: s3Profile.paramsOfCreateS3Client.region }; } }) @@ -272,7 +272,7 @@ export async function bootstrapCore( } if (oidc.isUserLoggedIn) { - await dispatch(usecases.s3ConfigManagement.protectedThunks.initialize()); + await dispatch(usecases.s3ProfilesManagement.protectedThunks.initialize()); } pluginSystemInitCore({ core, context }); diff --git a/web/src/core/ports/OnyxiaApi/DeploymentRegion.ts b/web/src/core/ports/OnyxiaApi/DeploymentRegion.ts index 8d8453a3b..88fc990cd 100644 --- a/web/src/core/ports/OnyxiaApi/DeploymentRegion.ts +++ b/web/src/core/ports/OnyxiaApi/DeploymentRegion.ts @@ -19,12 +19,11 @@ export type DeploymentRegion = { } | undefined; initScriptUrl: string; - s3Configs: DeploymentRegion.S3Config[]; - s3ConfigCreationFormDefaults: - | (Pick & { - workingDirectory: DeploymentRegion.S3Config["workingDirectory"] | undefined; - }) + s3Profiles: DeploymentRegion.S3Profile[]; + s3Profiles_defaultValuesOfCreationForm: + | Pick | undefined; + allowedURIPatternForUserDefinedInitScript: string; kafka: | { @@ -105,59 +104,55 @@ export type DeploymentRegion = { | undefined; }; export namespace DeploymentRegion { - /** https://github.com/InseeFrLab/onyxia-api/blob/main/docs/region-configuration.md#s3 */ - export type S3Config = { + export type S3Profile = { + profileName: string | undefined; url: string; pathStyleAccess: boolean; region: string | undefined; sts: { url: string | undefined; durationSeconds: number | undefined; - role: - | { - roleARN: string; - roleSessionName: string; - } - | undefined; + roles: S3Profile.StsRole[]; oidcParams: OidcParams_Partial; }; - workingDirectory: + bookmarks: S3Profile.Bookmark[]; + }; + + export namespace S3Profile { + export type StsRole = { + roleARN: string; + roleSessionName: string; + profileName: string; + } & ( | { - bucketMode: "shared"; - bucketName: string; - prefix: string; - prefixGroup: string; + claimName: undefined; + includedClaimPattern?: never; + excludedClaimPattern?: never; } | { - bucketMode: "multi"; - bucketNamePrefix: string; - bucketNamePrefixGroup: string; - }; - bookmarkedDirectories: S3Config.BookmarkedDirectory[]; - }; - - export namespace S3Config { - export type BookmarkedDirectory = - | BookmarkedDirectory.Static - | BookmarkedDirectory.Dynamic; - - export namespace BookmarkedDirectory { - export type Common = { - fullPath: string; - title: LocalizedString; - description: LocalizedString | undefined; - tags: LocalizedString[] | undefined; - }; - - export type Static = Common & { - claimName: undefined; - }; + claimName: string; + includedClaimPattern: string | undefined; + excludedClaimPattern: string | undefined; + } + ); - export type Dynamic = Common & { - claimName: string; - includedClaimPattern: string | undefined; - excludedClaimPattern: string | undefined; - }; - } + export type Bookmark = { + s3UriStr_templated: string; + title: LocalizedString; + description: LocalizedString | undefined; + tags: LocalizedString[]; + forProfileNames: string[]; + } & ( + | { + claimName: undefined; + includedClaimPattern?: never; + excludedClaimPattern?: never; + } + | { + claimName: string; + includedClaimPattern: string | undefined; + excludedClaimPattern: string | undefined; + } + ); } } diff --git a/web/src/core/ports/OnyxiaApi/XOnyxia.ts b/web/src/core/ports/OnyxiaApi/XOnyxia.ts index 275a85090..ed76d4d3a 100644 --- a/web/src/core/ports/OnyxiaApi/XOnyxia.ts +++ b/web/src/core/ports/OnyxiaApi/XOnyxia.ts @@ -115,19 +115,8 @@ export type XOnyxiaContext = { AWS_SESSION_TOKEN: string | undefined; AWS_DEFAULT_REGION: string; AWS_S3_ENDPOINT: string; - AWS_BUCKET_NAME: string; port: number; pathStyleAccess: boolean; - /** - * The user is assumed to have read/write access on every - * object starting with this prefix on the bucket - **/ - objectNamePrefix: string; - /** - * Only for making it easier for charts editors. - * / - * */ - workingDirectoryPath: string; /** * If true the bucket's (directory) should be accessible without any credentials. * In this case s3.AWS_ACCESS_KEY_ID, s3.AWS_SECRET_ACCESS_KEY and s3.AWS_SESSION_TOKEN diff --git a/web/src/core/ports/S3Client.ts b/web/src/core/ports/S3Client.ts index d560c19a8..6621b335d 100644 --- a/web/src/core/ports/S3Client.ts +++ b/web/src/core/ports/S3Client.ts @@ -1,24 +1,6 @@ -/** All path are supposed to start with / */ +import type { S3Uri } from "core/tools/S3Uri"; +import type { NonPostableEvt } from "evt"; -export type S3Object = S3Object.File | S3Object.Directory; - -export namespace S3Object { - export type Base = { - basename: string; - policy: "public" | "private"; - canChangePolicy: boolean; - }; - - export type File = Base & { - kind: "file"; - size: number | undefined; - lastModified: Date | undefined; - }; - - export type Directory = Base & { - kind: "directory"; - }; -} export type S3Client = { getToken: (params: { doForceRenew: boolean }) => Promise< | { @@ -34,60 +16,74 @@ export type S3Client = { /** * In charge of creating bucket if doesn't exist. */ - listObjects: (params: { path: string }) => Promise<{ - objects: S3Object[]; - bucketPolicy: S3BucketPolicy | undefined; - isBucketPolicyAvailable: boolean; - }>; - - setPathAccessPolicy: (params: { - path: string; - policy: "public" | "private"; - currentBucketPolicy: S3BucketPolicy; - }) => Promise; + listObjects: (params: { s3Uri: S3Uri }) => Promise; - /** Completed when 100% uploaded */ - uploadFile: (params: { + putObject: (params: { + s3Uri: S3Uri.NonTerminatedByDelimiter; blob: Blob; - path: string; onUploadProgress: (params: { uploadPercent: number }) => void; - }) => Promise; - - deleteFile: (params: { path: string }) => Promise; + evtCancel: NonPostableEvt; + }) => Promise< + | { status: "success" } + | { status: "canceled" } + | { status: "failed"; error: Error } + >; - deleteFiles: (params: { paths: string[] }) => Promise; + deleteObject: (params: { s3Uri: S3Uri.NonTerminatedByDelimiter }) => Promise; - getFileDownloadUrl: (params: { - path: string; + generateSignedDownloadUrl: (params: { + s3Uri: S3Uri.NonTerminatedByDelimiter; validityDurationSecond: number; }) => Promise; - getFileContent: (params: { path: string; range?: string }) => Promise<{ + getObjectContent: (params: { + s3Uri: S3Uri.NonTerminatedByDelimiter; + range: `bytes=0-${number}` | undefined; + }) => Promise<{ stream: ReadableStream; - lastModified: Date | undefined; size: number | undefined; contentType: string | undefined; }>; - getFileContentType: (params: { path: string }) => Promise; + getObjectContentType: (params: { + s3Uri: S3Uri.NonTerminatedByDelimiter; + }) => Promise; - // getPresignedUploadUrl: (params: { - // path: string; - // validityDurationSecond: number; - // }) => Promise; + createBucket: (params: { bucket: string }) => Promise< + | { isSuccess: true } + | { + isSuccess: false; + errorCase: "already exist" | "access denied" | "unknown"; + errorMessage: string; + } + >; }; -type s3Action = `s3:${string}`; +export namespace S3Client { + export type ListObjectsReturn = ListObjectsReturn.Error | ListObjectsReturn.Success; -export type S3BucketPolicy = { - Version: "2012-10-17"; - Statement: - | { - Effect: "Allow" | "Deny"; - Principal: string | { AWS: string[] }; - Action: s3Action | s3Action[]; - Resource: string[]; - Condition?: Record; - }[] - | null; -}; + export namespace ListObjectsReturn { + export type Success = { + isSuccess: true; + objects: { + s3Uri: S3Uri.NonTerminatedByDelimiter; + lastModified: number; + size: number; + }[]; + prefixes: S3Uri.TerminatedByDelimiter[]; + }; + + export namespace Success { + export type Object = { + s3Uri: S3Uri.NonTerminatedByDelimiter; + lastModified: number; + size: number; + }; + } + + export type Error = { + isSuccess: false; + errorCase: "access denied" | "no such bucket"; + }; + } +} diff --git a/web/src/core/tools/S3Uri.test.ts b/web/src/core/tools/S3Uri.test.ts new file mode 100644 index 000000000..ed4305c34 --- /dev/null +++ b/web/src/core/tools/S3Uri.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "vitest"; +import { symToStr } from "tsafe/symToStr"; +import { getIsInside, parseS3Uri } from "./S3Uri"; + +describe(symToStr({ getIsInside }), () => { + it("returns false when prefix and uri are identical", () => { + const s3UriPrefix = parseS3Uri({ + value: "s3://mybucket/aa/bb/", + delimiter: "/" + }); + + const s3Uri = parseS3Uri({ + value: "s3://mybucket/aa/bb/", + delimiter: "/" + }); + + const got = getIsInside({ s3UriPrefix, s3Uri }); + + expect(got).toStrictEqual({ isInside: false }); + }); + + it("detects a top-level child under a terminated prefix", () => { + const s3UriPrefix = parseS3Uri({ + value: "s3://mybucket/aa/bb/", + delimiter: "/" + }); + + const s3Uri = parseS3Uri({ + value: "s3://mybucket/aa/bb/cc", + delimiter: "/" + }); + + const got = getIsInside({ s3UriPrefix, s3Uri }); + + expect(got).toStrictEqual({ isInside: true, isTopLevel: true }); + }); + + it("detects a top-level sibling when prefix targets a non-terminated object", () => { + const s3UriPrefix = parseS3Uri({ + value: "s3://mybucket/aa/bb/cc", + delimiter: "/" + }); + + const s3Uri = parseS3Uri({ + value: "s3://mybucket/aa/bb/ccc", + delimiter: "/" + }); + + const got = getIsInside({ s3UriPrefix, s3Uri }); + + expect(got).toStrictEqual({ isInside: true, isTopLevel: true }); + }); + + it("returns false when uri is not inside the prefix", () => { + const s3UriPrefix = parseS3Uri({ + value: "s3://mybucket/aa/bb/ccc", + delimiter: "/" + }); + + const s3Uri = parseS3Uri({ + value: "s3://mybucket/aa/bb/cc", + delimiter: "/" + }); + + const got = getIsInside({ s3UriPrefix, s3Uri }); + + expect(got).toStrictEqual({ isInside: false }); + }); +}); diff --git a/web/src/core/tools/S3Uri.ts b/web/src/core/tools/S3Uri.ts new file mode 100644 index 000000000..24b2f4ca0 --- /dev/null +++ b/web/src/core/tools/S3Uri.ts @@ -0,0 +1,151 @@ +import { assert, type Equals, id } from "tsafe"; +import { same } from "evt/tools/inDepth/same"; +import { z } from "zod"; + +export type S3Uri = S3Uri.TerminatedByDelimiter | S3Uri.NonTerminatedByDelimiter; + +export namespace S3Uri { + type Common = { + bucket: string; + delimiter: string; + keySegments: string[]; + }; + + export type TerminatedByDelimiter = Common & { + isDelimiterTerminated: true; + }; + + export type NonTerminatedByDelimiter = Common & { + isDelimiterTerminated: false; + }; +} + +export function stringifyS3Uri(s3Uri: S3Uri): string { + return [ + "s3://", + `${s3Uri.bucket}/`, + s3Uri.keySegments + .map( + (keySegment, i) => + `${keySegment}${s3Uri.isDelimiterTerminated ? s3Uri.delimiter : i === s3Uri.keySegments.length - 1 ? "" : s3Uri.delimiter}` + ) + .join("") + ].join(""); +} + +export function getS3UriKey(s3Uri: S3Uri): string { + return stringifyS3Uri(s3Uri).slice(`s3://${s3Uri.bucket}/`.length); +} + +export function parseS3Uri(params: { value: string; delimiter: string }): S3Uri { + const { value, delimiter } = params; + + const match = value.match(/^s3:\/\/([^/]+)(\/?.*)$/); + + if (match === null) { + throw new Error(`Malformed S3 URI: ${value}`); + } + + const bucket = match[1]; + + const group2 = match[2]; + + if (group2 === "" || group2 === "/") { + return id({ + bucket, + delimiter, + keySegments: [], + isDelimiterTerminated: true + }); + } + + const key = group2.slice(1); + + const split = key.split(delimiter); + + const isDelimiterTerminated = split.at(-1) === ""; + + if (isDelimiterTerminated) { + split.pop(); + } + + return { + bucket, + delimiter, + keySegments: split, + isDelimiterTerminated + }; +} + +export function getIsInside(params: { + s3UriPrefix: S3Uri; + s3Uri: S3Uri; +}): { isInside: false; isTopLevel?: never } | { isInside: true; isTopLevel: boolean } { + const { s3UriPrefix, s3Uri } = params; + + if ( + !stringifyS3Uri(s3Uri).startsWith(stringifyS3Uri(s3UriPrefix)) || + same(s3Uri, s3UriPrefix) + ) { + return { isInside: false }; + } + + return { + isInside: true, + isTopLevel: same( + s3UriPrefix.isDelimiterTerminated + ? s3UriPrefix.keySegments + : s3UriPrefix.keySegments.slice(0, -1), + s3Uri.keySegments.slice(0, -1) + ) + }; +} + +export const zS3Uri_NonTerminatedByDelimiter = (() => { + type TargetType = S3Uri.NonTerminatedByDelimiter; + + const zTargetType = z.object({ + bucket: z.string(), + delimiter: z.string(), + keySegments: z.array(z.string()), + isDelimiterTerminated: z.literal(false) + }); + + type InferredType = z.infer; + + assert>; + + return id>(zTargetType); +})(); + +export const zS3Uri_TerminatedByDelimiter = (() => { + type TargetType = S3Uri.TerminatedByDelimiter; + + const zTargetType = z.object({ + bucket: z.string(), + delimiter: z.string(), + keySegments: z.array(z.string()), + isDelimiterTerminated: z.literal(true) + }); + + type InferredType = z.infer; + + assert>; + + return id>(zTargetType); +})(); + +export const zS3Uri = (() => { + type TargetType = S3Uri; + + const zTargetType = z.union([ + zS3Uri_NonTerminatedByDelimiter, + zS3Uri_TerminatedByDelimiter + ]); + + type InferredType = z.infer; + + assert>; + + return id>(zTargetType); +})(); diff --git a/web/src/core/usecases/fileExplorer/selectors.ts b/web/src/core/usecases/fileExplorer/selectors.ts deleted file mode 100644 index b5d5e160a..000000000 --- a/web/src/core/usecases/fileExplorer/selectors.ts +++ /dev/null @@ -1,376 +0,0 @@ -import type { State as RootState } from "core/bootstrap"; -import { type State, name } from "./state"; -import { createSelector } from "clean-architecture"; -import * as userConfigs from "core/usecases/userConfigs"; -import * as s3ConfigManagement from "core/usecases/s3ConfigManagement"; -import * as deploymentRegionManagement from "core/usecases/deploymentRegionManagement"; -import { assert } from "tsafe/assert"; -import * as userAuthentication from "core/usecases/userAuthentication"; -import { id } from "tsafe/id"; -import type { S3Object } from "core/ports/S3Client"; -import { join as pathJoin, relative as pathRelative } from "pathe"; -import { getUploadProgress } from "./decoupledLogic/uploadProgress"; - -const state = (rootState: RootState): State => rootState[name]; - -const isDownloadPreparing = createSelector( - createSelector(state, state => state.ongoingOperations), - (ongoingOperations): boolean => - ongoingOperations.some(operation => operation.operation === "downloading") -); - -const uploadProgress = createSelector( - createSelector(state, state => state.s3FilesBeingUploaded), - s3FilesBeingUploaded => getUploadProgress(s3FilesBeingUploaded) -); - -const commandLogsEntries = createSelector( - state, - userConfigs.selectors.userConfigs, - (state, userConfigs) => - !userConfigs.isCommandBarEnabled ? undefined : state.commandLogsEntries -); - -export type CurrentWorkingDirectoryView = { - directoryPath: string; - items: CurrentWorkingDirectoryView.Item[]; - isBucketPolicyFeatureEnabled: boolean; -}; - -export namespace CurrentWorkingDirectoryView { - export type Item = Item.File | Item.Directory; - export namespace Item { - export type Common = { - basename: string; - policy: "public" | "private"; - canChangePolicy: boolean; - isBeingDeleted: boolean; - isPolicyChanging: boolean; - } & ( - | { - isBeingCreated: false; - } - | { - isBeingCreated: true; - uploadPercent: number; - } - ); - - export type File = Common & { - kind: "file"; - size: number | undefined; - lastModified: Date | undefined; - }; - - export type Directory = Common & { - kind: "directory"; - }; - } -} - -const currentWorkingDirectoryView = createSelector( - createSelector(state, state => state.directoryPath), - createSelector(state, state => state.objects), - createSelector(state, state => state.ongoingOperations), - createSelector(state, state => state.s3FilesBeingUploaded), - createSelector(state, state => state.isBucketPolicyAvailable), - ( - directoryPath, - objects, - ongoingOperations, - s3FilesBeingUploaded, - isBucketPolicyAvailable - ): CurrentWorkingDirectoryView | null => { - if (directoryPath === undefined) { - return null; - } - const items = [ - ...objects, - ...ongoingOperations - .filter( - ongoingOperation => - ongoingOperation.operation === "create" && - pathRelative(directoryPath, ongoingOperation.directoryPath) == "" - ) - .map(ongoingOperation => ongoingOperation.objects) - .flat() - .filter( - object => - objects.find( - object_i => - object_i.kind === object.kind && - object_i.basename === object.basename - ) === undefined - ) - ] - .map((object): CurrentWorkingDirectoryView.Item => { - const { isBeingDeleted, isPolicyChanging, isBeingCreated } = (() => { - const operation = ongoingOperations.find( - op => - pathRelative(op.directoryPath, directoryPath) === "" && - op.objects.find( - ongoingObject => - ongoingObject.basename === object.basename - ) !== undefined - )?.operation; - - return { - isBeingDeleted: operation === "delete", - isPolicyChanging: operation === "modifyPolicy", - isBeingCreated: operation === "create" - }; - })(); - - const common: CurrentWorkingDirectoryView.Item.Common = { - basename: object.basename, - policy: object.policy, - canChangePolicy: object.canChangePolicy, - isBeingDeleted, - isPolicyChanging, - ...(!isBeingCreated - ? { - isBeingCreated: false - } - : { - isBeingCreated: true, - uploadPercent: (() => { - const fileOrDirectoryPath = pathJoin( - directoryPath, - object.basename - ); - - const s3FilesBeingUploaded_relevant = - s3FilesBeingUploaded.filter(o => { - const filePath_i = pathJoin( - o.directoryPath, - o.basename - ); - - if ( - pathRelative( - fileOrDirectoryPath, - filePath_i - ).startsWith("..") - ) { - return false; - } - - return true; - }); - - if (s3FilesBeingUploaded_relevant.length === 0) { - return 0; - } - - return getUploadProgress(s3FilesBeingUploaded_relevant) - .overallProgress.uploadPercent; - })() - }) - }; - - switch (object.kind) { - case "file": { - const { size, lastModified } = object; - - return id({ - kind: "file", - ...common, - size, - lastModified - }); - } - case "directory": - return id({ - kind: "directory", - ...common - }); - } - }) - .sort((a, b) => { - // Sort directories first - if (a.kind === "directory" && b.kind !== "directory") return -1; - if (a.kind !== "directory" && b.kind === "directory") return 1; - - // Sort alphabetically by basename - return a.basename.localeCompare(b.basename); - }); - - return { - directoryPath, - items, - isBucketPolicyFeatureEnabled: isBucketPolicyAvailable - }; - } -); - -export type ShareView = ShareView.PublicFile | ShareView.PrivateFile; - -export namespace ShareView { - export type Common = { - file: S3Object.File; - }; - - export type PublicFile = Common & { - isPublic: true; - url: string; - }; - - export type PrivateFile = Common & { - isPublic: false; - validityDurationSecond: number; - validityDurationSecondOptions: number[]; - url: string | undefined; - isSignedUrlBeingRequested: boolean; - }; -} - -const shareView = createSelector( - createSelector(state, state => state.directoryPath), - createSelector(state, state => state.objects), - createSelector(state, state => state.share), - (directoryPath, objects, share): ShareView | undefined | null => { - if (directoryPath === undefined) { - return null; - } - - if (share === undefined) { - return undefined; - } - - const common: ShareView.Common = { - file: (() => { - const file = objects.find( - obj => obj.basename === share.fileBasename && obj.kind === "file" - ); - - assert(file !== undefined); - assert(file.kind === "file"); - - return file; - })() - }; - - const isPublic = share.isSignedUrlBeingRequested === undefined; - - if (isPublic) { - assert(share.url !== undefined); - - return id({ - ...common, - isPublic: true, - url: share.url - }); - } - - const { - url, - isSignedUrlBeingRequested, - validityDurationSecond, - validityDurationSecondOptions - } = share; - - assert(isSignedUrlBeingRequested !== undefined); - assert(validityDurationSecond !== undefined); - assert(validityDurationSecondOptions !== undefined); - - return id({ - ...common, - isPublic: false, - isSignedUrlBeingRequested, - url, - validityDurationSecond, - validityDurationSecondOptions - }); - } -); - -const isNavigationOngoing = createSelector(state, state => state.isNavigationOngoing); - -const workingDirectoryPath = createSelector( - s3ConfigManagement.selectors.s3Configs, - s3Configs => { - const s3Config = s3Configs.find(s3Config => s3Config.isExplorerConfig); - assert(s3Config !== undefined); - return s3Config.workingDirectoryPath; - } -); - -const pathMinDepth = createSelector(workingDirectoryPath, workingDirectoryPath => { - // "jgarrone/" -> 0 - // "jgarrone/foo/" -> 1 - // "jgarrone/foo/bar/" -> 2 - return workingDirectoryPath.split("/").length - 2; -}); - -const main = createSelector( - createSelector(state, state => state.directoryPath), - uploadProgress, - commandLogsEntries, - currentWorkingDirectoryView, - isNavigationOngoing, - pathMinDepth, - createSelector(state, state => state.viewMode), - shareView, - isDownloadPreparing, - ( - directoryPath, - uploadProgress, - commandLogsEntries, - currentWorkingDirectoryView, - isNavigationOngoing, - pathMinDepth, - viewMode, - shareView, - isDownloadPreparing - ) => { - if (directoryPath === undefined) { - return { - isCurrentWorkingDirectoryLoaded: false as const, - isNavigationOngoing, - uploadProgress, - commandLogsEntries, - pathMinDepth, - viewMode, - isDownloadPreparing - }; - } - - assert(currentWorkingDirectoryView !== null); - assert(shareView !== null); - - return { - isCurrentWorkingDirectoryLoaded: true as const, - isNavigationOngoing, - uploadProgress, - commandLogsEntries, - pathMinDepth, - currentWorkingDirectoryView, - viewMode, - shareView, - isDownloadPreparing - }; - } -); - -const isFileExplorerEnabled = (rootState: RootState) => { - const { isUserLoggedIn } = userAuthentication.selectors.main(rootState); - - if (!isUserLoggedIn) { - const { s3Configs } = - deploymentRegionManagement.selectors.currentDeploymentRegion(rootState); - - return s3Configs.length !== 0; - } else { - return ( - s3ConfigManagement.selectors - .s3Configs(rootState) - .find(s3Config => s3Config.isExplorerConfig) !== undefined - ); - } -}; - -const directoryPath = createSelector(state, state => state.directoryPath); - -export const protectedSelectors = { workingDirectoryPath, directoryPath, shareView }; - -export const selectors = { main, isFileExplorerEnabled }; diff --git a/web/src/core/usecases/fileExplorer/state.ts b/web/src/core/usecases/fileExplorer/state.ts deleted file mode 100644 index 255b4e1a7..000000000 --- a/web/src/core/usecases/fileExplorer/state.ts +++ /dev/null @@ -1,398 +0,0 @@ -import { id } from "tsafe/id"; -import { assert, type Equals } from "tsafe/assert"; -import { createUsecaseActions } from "clean-architecture"; -import type { S3BucketPolicy, S3Object } from "core/ports/S3Client"; -import { relative as pathRelative } from "pathe"; -import type { S3FilesBeingUploaded } from "./decoupledLogic/uploadProgress"; - -//All explorer paths are expected to be absolute (start with /) - -export type State = { - directoryPath: string | undefined; - viewMode: "list" | "block"; - objects: S3Object[]; - isNavigationOngoing: boolean; - ongoingOperations: { - operationId: string; - operation: "create" | "delete" | "modifyPolicy" | "downloading"; - directoryPath: string; - objects: S3Object[]; - }[]; - s3FilesBeingUploaded: S3FilesBeingUploaded; - commandLogsEntries: { - cmdId: number; - cmd: string; - resp: string | undefined; - }[]; - bucketPolicy: S3BucketPolicy; - isBucketPolicyAvailable: boolean; - share: - | { - fileBasename: string; - url: string | undefined; - validityDurationSecond: number | undefined; - validityDurationSecondOptions: number[] | undefined; - isSignedUrlBeingRequested: boolean | undefined; - } - | undefined; -}; - -export const name = "fileExplorer"; - -export const { reducer, actions } = createUsecaseActions({ - name, - initialState: id({ - directoryPath: undefined, - objects: [], - viewMode: "list", - isNavigationOngoing: false, - ongoingOperations: [], - s3FilesBeingUploaded: [], - commandLogsEntries: [], - bucketPolicy: { - Version: "2012-10-17", - Statement: [] - }, - isBucketPolicyAvailable: true, - share: undefined - }), - reducers: { - fileUploadStarted: ( - state, - { - payload - }: { - payload: { - directoryPath: string; - basename: string; - size: number; - }; - } - ) => { - const { directoryPath, basename, size } = payload; - - state.s3FilesBeingUploaded.push({ - directoryPath, - basename, - size, - uploadPercent: 0 - }); - }, - uploadProgressUpdated: ( - state, - { - payload - }: { - payload: { - directoryPath: string; - basename: string; - uploadPercent: number; - }; - } - ) => { - const { basename, directoryPath, uploadPercent } = payload; - const { s3FilesBeingUploaded } = state; - - const s3FileBeingUploaded = s3FilesBeingUploaded.find( - s3FileBeingUploaded => - s3FileBeingUploaded.directoryPath === directoryPath && - s3FileBeingUploaded.basename === basename - ); - assert(s3FileBeingUploaded !== undefined); - s3FileBeingUploaded.uploadPercent = uploadPercent; - - if ( - s3FilesBeingUploaded.find( - ({ uploadPercent }) => uploadPercent !== 100 - ) !== undefined - ) { - return; - } - - state.s3FilesBeingUploaded = []; - }, - navigationStarted: state => { - assert(state.share === undefined); - state.isNavigationOngoing = true; - }, - navigationCompleted: ( - state, - { - payload - }: { - payload: { - directoryPath: string; - objects: S3Object[]; - bucketPolicy: S3BucketPolicy | undefined; - isBucketPolicyAvailable: boolean; - }; - } - ) => { - const { directoryPath, objects, bucketPolicy, isBucketPolicyAvailable } = - payload; - - state.directoryPath = directoryPath; - state.objects = objects; - state.isNavigationOngoing = false; - if (bucketPolicy) { - state.bucketPolicy = bucketPolicy; - } - state.isBucketPolicyAvailable = isBucketPolicyAvailable; - }, - operationStarted: ( - state, - { - payload - }: { - payload: { - operationId: string; - objects: S3Object[]; - operation: "create" | "delete" | "modifyPolicy" | "downloading"; - }; - } - ) => { - const { objects, operation, operationId } = payload; - - assert(state.directoryPath !== undefined); - - const { directoryPath } = state; - - state.ongoingOperations.push({ - operationId, - operation, - directoryPath, - objects - }); - }, - operationCompleted: ( - state, - { - payload - }: { - payload: { - operationId: string; - }; - } - ) => { - const { operationId } = payload; - - assert(state.directoryPath !== undefined); - - const { ongoingOperations } = state; - - const ongoingOperation = ongoingOperations.find( - o => o.operationId === operationId - ); - - assert(ongoingOperation !== undefined); - - ongoingOperations.splice(ongoingOperations.indexOf(ongoingOperation), 1); - - assert( - pathRelative(ongoingOperation.directoryPath, state.directoryPath) === "" - ); - - switch (ongoingOperation.operation) { - case "create": - state.objects.push( - ...ongoingOperation.objects.filter( - object_created_i => - state.objects.find( - object_i => - object_i.kind === object_created_i.kind && - object_i.basename === object_created_i.basename - ) === undefined - ) - ); - break; - case "delete": - state.objects = state.objects.filter( - object_i => - ongoingOperation.objects.find( - object_deleted_i => - object_deleted_i.kind === object_i.kind && - object_deleted_i.basename === object_i.basename - ) === undefined - ); - break; - case "downloading": - break; - case "modifyPolicy": - break; - default: - assert>; - } - }, - commandLogIssued: ( - state, - { - payload - }: { - payload: { - cmdId: number; - cmd: string; - }; - } - ) => { - const { cmdId, cmd } = payload; - - state.commandLogsEntries.push({ - cmdId, - cmd, - resp: undefined - }); - }, - commandLogCancelled: ( - state, - { - payload - }: { - payload: { - cmdId: number; - }; - } - ) => { - const { cmdId } = payload; - - const index = state.commandLogsEntries.findIndex( - entry => entry.cmdId === cmdId - ); - - assert(index >= 0); - - state.commandLogsEntries.splice(index, 1); - }, - commandLogResponseReceived: ( - state, - { - payload - }: { - payload: { - cmdId: number; - resp: string; - }; - } - ) => { - const { cmdId, resp } = payload; - - const entry = state.commandLogsEntries.find(entry => entry.cmdId === cmdId); - - assert(entry !== undefined); - - entry.resp = resp; - }, - workingDirectoryChanged: state => { - state.directoryPath = undefined; - state.objects = []; - state.isNavigationOngoing = false; - }, - viewModeChanged: ( - state, - { payload }: { payload: { viewMode: "list" | "block" } } - ) => { - const { viewMode } = payload; - state.viewMode = viewMode; - }, - bucketPolicyModified: ( - state, - { - payload - }: { - payload: { - kind: "file" | "directory"; - basename: string; - policy: "public" | "private"; - bucketPolicy: S3BucketPolicy; - }; - } - ) => { - const { bucketPolicy, policy, basename, kind } = payload; - - { - const object = state.objects.find( - object => object.kind === kind && object.basename === basename - ); - assert(object !== undefined); - object.policy = policy; - } - - state.bucketPolicy = bucketPolicy; - }, - shareOpened: ( - state, - { - payload - }: { - payload: { - fileBasename: string; - url: string | undefined; - validityDurationSecondOptions: number[] | undefined; - }; - } - ) => { - const { fileBasename, url, validityDurationSecondOptions } = payload; - - if (url !== undefined) { - state.share = { - fileBasename, - url, - isSignedUrlBeingRequested: undefined, - validityDurationSecondOptions: undefined, - validityDurationSecond: undefined - }; - } else { - assert(validityDurationSecondOptions !== undefined); - - state.share = { - fileBasename, - url, - isSignedUrlBeingRequested: false, - validityDurationSecondOptions, - validityDurationSecond: validityDurationSecondOptions[0] - }; - } - }, - shareClosed: state => { - state.share = undefined; - }, - shareSelectedValidityDurationChanged: ( - state, - { - payload - }: { - payload: { - validityDurationSecond: number; - }; - } - ) => { - const { validityDurationSecond } = payload; - - assert(state.share !== undefined); - assert(state.share.validityDurationSecondOptions !== undefined); - assert( - state.share.validityDurationSecondOptions.includes(validityDurationSecond) - ); - state.share.validityDurationSecond = validityDurationSecond; - }, - requestSignedUrlStarted: state => { - assert(state.share !== undefined); - state.share.isSignedUrlBeingRequested = true; - }, - requestSignedUrlCompleted: ( - state, - { - payload - }: { - payload: { - url: string; - }; - } - ) => { - const { url } = payload; - - assert(state.share !== undefined); - state.share.isSignedUrlBeingRequested = false; - state.share.url = url; - } - } -}); diff --git a/web/src/core/usecases/fileExplorer/thunks.ts b/web/src/core/usecases/fileExplorer/thunks.ts deleted file mode 100644 index 2a9d7b46a..000000000 --- a/web/src/core/usecases/fileExplorer/thunks.ts +++ /dev/null @@ -1,1046 +0,0 @@ -import { assert } from "tsafe/assert"; -import { Evt } from "evt"; -import { Zip, ZipPassThrough } from "fflate/browser"; -import type { Thunks } from "core/bootstrap"; -import { name, actions } from "./state"; -import { protectedSelectors } from "./selectors"; -import { join as pathJoin, basename as pathBasename } from "pathe"; -import { crawlFactory } from "core/tools/crawl"; -import * as s3ConfigManagement from "core/usecases/s3ConfigManagement"; -import type { S3Object } from "core/ports/S3Client"; -import { formatDuration } from "core/tools/timeFormat/formatDuration"; -import { relative as pathRelative } from "pathe"; -import { id } from "tsafe/id"; -import { isAmong } from "tsafe/isAmong"; -import { removeDuplicates } from "evt/tools/reducers/removeDuplicates"; - -const privateThunks = { - startOperationWhenAllConflictingOperationHaveCompleted: - (params: { - operation: "create" | "delete" | "modifyPolicy" | "downloading"; - objects: S3Object[]; - }) => - async (...args) => { - const [dispatch, getState] = args; - - const { directoryPath } = getState()[name]; - - assert(directoryPath !== undefined); - - const { operation, objects } = params; - - const operationId = `${operation}-${Date.now()}`; - - dispatch(actions.operationStarted({ operationId, objects, operation })); - - await dispatch( - privateThunks.waitForNoOngoingOperation({ - directoryPath, - objects_ref: objects, - ignoreOperationId: operationId - }) - ); - return operationId; - }, - waitForNoOngoingOperation: - (params: { - directoryPath: string; - objects_ref: { kind: "file" | "directory"; basename: string }[]; - ignoreOperationId?: string; - }) => - async (...args) => { - const [, getState, { evtAction }] = args; - - const { directoryPath, objects_ref, ignoreOperationId } = params; - - const { ongoingOperations } = getState()[name]; - - const relevantOperationIds = ongoingOperations - .filter( - ongoingOperation => - pathRelative(directoryPath, ongoingOperation.directoryPath) === "" - ) - .filter( - ignoreOperationId === undefined - ? () => true - : ongoingOperation => - ongoingOperation.operationId !== ignoreOperationId - ) - .filter(({ objects }) => { - for (const object_ref of objects_ref) { - const object_match = objects.find( - object => - object.kind === object_ref.kind && - object.basename === object_ref.basename - ); - - if (object_match === undefined) { - continue; - } - - return true; - } - - return false; - }) - .map(ongoingOperation => ongoingOperation.operationId); - - if (relevantOperationIds.length === 0) { - return; - } - - await Promise.all( - relevantOperationIds.map(operationId => - evtAction.waitFor( - event => - event.usecaseName === "fileExplorer" && - event.actionName === "operationCompleted" && - event.payload.operationId === operationId - ) - ) - ); - }, - /** - * NOTE: It IS possible to navigate to a directory currently being renamed or created. - */ - navigate: - (params: { directoryPath: string; doListAgainIfSamePath: boolean }) => - async (...args) => { - const { doListAgainIfSamePath } = params; - - // Ensure trailing slash for consistency. - const directoryPath = params.directoryPath.replace(/\/+$/, "") + "/"; - - const [dispatch, getState, { evtAction }] = args; - - if ( - !doListAgainIfSamePath && - getState()[name].directoryPath === directoryPath - ) { - return; - } - - dispatch(actions.navigationStarted()); - - const ctx = Evt.newCtx(); - - evtAction.attachOnce( - event => - event.usecaseName === name && - event.actionName === "navigationStarted", - ctx, - () => ctx.abort(new Error("Other navigation started")) - ); - - await dispatch( - privateThunks.waitForNoOngoingOperation({ - directoryPath: pathJoin(directoryPath, "..") + "/", - objects_ref: [ - { - kind: "directory", - basename: pathBasename(directoryPath) - } - ] - }) - ); - - const cmdId = Date.now(); - - dispatch( - actions.commandLogIssued({ - cmdId, - cmd: `mc ls ${pathJoin("s3", directoryPath)}` - }) - ); - - const s3Client = await dispatch( - s3ConfigManagement.protectedThunks.getS3ConfigAndClientForExplorer() - ).then(r => { - assert(r !== undefined); - return r.s3Client; - }); - - const { objects, bucketPolicy, isBucketPolicyAvailable } = - await s3Client.listObjects({ - path: directoryPath - }); - - if (ctx.completionStatus !== undefined) { - dispatch(actions.commandLogCancelled({ cmdId })); - return; - } - - ctx.done(); - - dispatch( - actions.commandLogResponseReceived({ - cmdId, - resp: objects - .map(({ kind, basename }) => - kind === "directory" ? `${basename}/` : basename - ) - .join("\n") - }) - ); - - dispatch( - actions.navigationCompleted({ - directoryPath, - objects, - bucketPolicy, - isBucketPolicyAvailable - }) - ); - }, - downloadObject: - (params: { s3Object: S3Object }) => - async (...args) => { - const [dispatch, getState] = args; - - const { directoryPath } = getState()[name]; - assert(directoryPath !== undefined); - - const { s3Object } = params; - - const s3Client = await dispatch( - s3ConfigManagement.protectedThunks.getS3ConfigAndClientForExplorer() - ).then(r => { - assert(r !== undefined); - return r.s3Client; - }); - - const basename = s3Object.basename; - const path = pathJoin(directoryPath, basename); - - const cmdId = Date.now(); - - dispatch( - actions.commandLogIssued({ - cmdId, - cmd: `mc cp ${pathJoin("s3", path)} .` - }) - ); - - const { stream, size } = await s3Client.getFileContent({ - path - }); - - dispatch( - actions.commandLogResponseReceived({ - cmdId, - resp: `...${path}: 100% of ${size} Bytes uploaded` - }) - ); - - return { stream }; - }, - downloadObjectsAsZip: - (params: { s3Objects: S3Object[] }) => - async (...args) => { - const [dispatch, getState] = args; - - const { directoryPath } = getState()[name]; - assert(directoryPath !== undefined); - - const { s3Objects } = params; - - const s3Client = await dispatch( - s3ConfigManagement.protectedThunks.getS3ConfigAndClientForExplorer() - ).then(r => { - assert(r !== undefined); - return r.s3Client; - }); - - const { readable, writable } = new TransformStream(); - const writer = writable.getWriter(); - - const cmdId = Date.now(); - - dispatch( - actions.commandLogIssued({ - cmdId, - cmd: `mc cp --recursive ${s3Objects - .map(({ basename }) => - pathJoin("s3", pathJoin(directoryPath, basename)) - ) - .join(" ")} .` - }) - ); - - let totalSize: number = 0; - - { - const zip = new Zip((err, chunk, final) => { - if (err) { - writer.abort(err); - throw err; - } - - writer.write(chunk); - - if (final) { - writer.close(); - } - }); - - const { crawl } = crawlFactory({ - list: async ({ directoryPath }) => { - const { objects } = await s3Client.listObjects({ - path: directoryPath - }); - - return objects.reduce<{ - fileBasenames: string[]; - directoryBasenames: string[]; - }>( - (acc, { kind, basename }) => { - switch (kind) { - case "directory": - acc.directoryBasenames.push(basename); - break; - case "file": - if (basename !== ".keep") { - acc.fileBasenames.push(basename); - } - break; - } - return acc; - }, - { - fileBasenames: [], - directoryBasenames: [] - } - ); - } - }); - - const createZipEntryFromStream = async ({ - zipPath, - stream, - modifiedDate - }: { - zipPath: string; - stream: ReadableStream; - modifiedDate?: string | number | Date; - }) => { - const entry = new ZipPassThrough(zipPath); - if (modifiedDate) entry.mtime = modifiedDate; - - zip.add(entry); - - const reader = stream.getReader(); - while (true) { - const { done, value } = await reader.read(); - if (done) break; - entry.push(value); - } - - entry.push(new Uint8Array(0), true); - }; - - const downloadTasks: Promise[] = []; - - for (const object of s3Objects) { - const basePath = pathJoin(directoryPath, object.basename); - - switch (object.kind) { - case "directory": { - const { filePaths, directoryPaths } = await crawl({ - directoryPath: basePath - }); - - directoryPaths.forEach(path => { - const zipEntry = new ZipPassThrough( - `${pathJoin(object.basename, path)}/` - ); - zip.add(zipEntry); - zipEntry.push(new Uint8Array(0), true); - }); - - for (const relativeFilePath of filePaths) { - const absolutePath = pathJoin(basePath, relativeFilePath); - const zipEntryPath = pathJoin( - object.basename, - relativeFilePath - ); - - const { stream, size, lastModified } = - await s3Client.getFileContent({ - path: absolutePath - }); - - totalSize += size ?? 0; - downloadTasks.push( - createZipEntryFromStream({ - zipPath: zipEntryPath, - stream, - modifiedDate: lastModified - }) - ); - } - break; - } - - case "file": { - const { stream, size } = await s3Client.getFileContent({ - path: basePath - }); - - totalSize += size ?? 0; - - downloadTasks.push( - createZipEntryFromStream({ - zipPath: object.basename, - stream, - modifiedDate: object.lastModified - }) - ); - break; - } - } - } - - await Promise.all(downloadTasks); - - zip.end(); - } - - dispatch( - actions.commandLogResponseReceived({ - cmdId, - resp: `...${pathJoin(directoryPath, s3Objects.at(-1)?.basename ?? "")}: 100% of ${totalSize} Bytes uploaded` - }) - ); - - return { - stream: readable, - zipFileName: - s3Objects.length === 1 - ? `${s3Objects[0].basename}.zip` - : `onyxia-download-${new Date().toISOString()}.zip` - }; - }, - uploadFileAndLogCommand: - (params: { - path: string; - blob: Blob; - onUploadProgress: (params: { uploadPercent: number }) => void; - }) => - async (...args) => { - const [dispatch] = args; - - const { path, blob, onUploadProgress } = params; - - const cmdId = Date.now(); - - dispatch( - actions.commandLogIssued({ - cmdId, - cmd: `mc cp ${pathJoin(".", pathBasename(path))} ${pathJoin( - "s3", - path - )}` - }) - ); - - const s3Client = await dispatch( - s3ConfigManagement.protectedThunks.getS3ConfigAndClientForExplorer() - ).then(r => { - assert(r !== undefined); - return r.s3Client; - }); - - await s3Client.uploadFile({ - path, - blob, - onUploadProgress: ({ uploadPercent }) => { - onUploadProgress({ uploadPercent }); - - dispatch( - actions.commandLogResponseReceived({ - cmdId, - resp: `... ${uploadPercent}% of ${blob.size} Bytes uploaded` - }) - ); - } - }); - } -} satisfies Thunks; - -export const thunks = { - initialize: - (params: { directoryPath: string; viewMode: "list" | "block" }) => - async (...args) => { - const { directoryPath, viewMode } = params; - - const [dispatch] = args; - - dispatch(actions.viewModeChanged({ viewMode })); - - await dispatch( - privateThunks.navigate({ - directoryPath: directoryPath, - doListAgainIfSamePath: false - }) - ); - }, - - changeCurrentDirectory: - (params: { directoryPath: string }) => - async (...args) => { - const { directoryPath } = params; - - const [dispatch] = args; - - await dispatch( - privateThunks.navigate({ - directoryPath: directoryPath, - doListAgainIfSamePath: false - }) - ); - }, - changeViewMode: - (params: { viewMode: "list" | "block" }) => - async (...args) => { - const { viewMode } = params; - - const [dispatch] = args; - - dispatch(actions.viewModeChanged({ viewMode })); - }, - changePolicy: - (params: { - basename: string; - policy: S3Object["policy"]; - kind: S3Object["kind"]; - }) => - async (...args) => { - const { policy, basename, kind } = params; - - const [dispatch, getState] = args; - - const state = getState()[name]; - - const { directoryPath, objects, isBucketPolicyAvailable } = state; - - if (!isBucketPolicyAvailable) { - console.info("Bucket policy is not available"); - return; - } - - const object = objects.find(o => o.basename === basename && o.kind === kind); - - assert(object !== undefined); - assert(directoryPath !== undefined); - - const operationId = await dispatch( - privateThunks.startOperationWhenAllConflictingOperationHaveCompleted({ - operation: "modifyPolicy", - objects: [{ ...object, policy }] - }) - ); - const s3Client = await dispatch( - s3ConfigManagement.protectedThunks.getS3ConfigAndClientForExplorer() - ).then(r => { - assert(r !== undefined); - return r.s3Client; - }); - - const itemPath = - pathJoin(directoryPath, basename) + (kind === "directory" ? "/" : ""); - const s3Prefix = pathJoin("s3", itemPath); - - const cmdId = Date.now(); - - dispatch( - actions.commandLogIssued({ - cmdId, - cmd: `mc anonymous set ${(() => { - switch (policy) { - case "public": - return "download"; - case "private": - return "none"; - } - })()} ${s3Prefix}` - }) - ); - - const modifiedBucketPolicy = await s3Client.setPathAccessPolicy({ - path: itemPath, - policy, - currentBucketPolicy: getState()[name].bucketPolicy - }); - - dispatch( - actions.operationCompleted({ - operationId - }) - ); - - dispatch( - actions.commandLogResponseReceived({ - cmdId, - resp: `Access permission for \`${s3Prefix}\` is set to \`${(() => { - switch (policy) { - case "public": - return "download"; - case "private": - return "custom"; - } - })()}\`` - }) - ); - - dispatch( - actions.bucketPolicyModified({ - bucketPolicy: modifiedBucketPolicy, - kind, - basename, - policy - }) - ); - }, - refreshCurrentDirectory: - () => - async (...args) => { - const [dispatch, getState] = args; - - const { directoryPath } = getState()[name]; - - assert(directoryPath !== undefined); - - await dispatch( - privateThunks.navigate({ directoryPath, doListAgainIfSamePath: true }) - ); - }, - - uploadFiles: - (params: { - files: { - directoryRelativePath: string; - basename: string; - blob: Blob; - }[]; - }) => - async (...args) => { - const { files } = params; - - const [dispatch, getState] = args; - - const state = getState()[name]; - - const { directoryPath } = state; - - assert(directoryPath !== undefined); - - const operationId = await dispatch( - privateThunks.startOperationWhenAllConflictingOperationHaveCompleted({ - operation: "create", - objects: files - .map(file => - isAmong([".", ""], file.directoryRelativePath) - ? id({ - kind: "file", - basename: file.basename, - policy: "private", - size: file.blob.size, - lastModified: new Date(), - canChangePolicy: false - }) - : id({ - kind: "directory", - basename: file.directoryRelativePath - .replace(/^\.\//, "") - .split("/")[0], - policy: "private", - canChangePolicy: false - }) - ) - .reduce( - ...removeDuplicates( - (object1, object2) => - object1.kind === "directory" && - object2.kind === "directory" && - object1.basename === object2.basename - ) - ) - }) - ); - - await Promise.all( - files.map(async file => { - //TODO policy can be public if uploaded inside public directory - const directoryPath_uploadedFile = pathJoin( - directoryPath, - file.directoryRelativePath - ); - - dispatch( - actions.fileUploadStarted({ - basename: file.basename, - directoryPath: directoryPath_uploadedFile, - size: file.blob.size - }) - ); - - await dispatch( - privateThunks.uploadFileAndLogCommand({ - path: pathJoin(directoryPath_uploadedFile, file.basename), - blob: file.blob, - onUploadProgress: ({ uploadPercent }) => - dispatch( - actions.uploadProgressUpdated({ - basename: file.basename, - directoryPath: directoryPath_uploadedFile, - uploadPercent - }) - ) - }) - ); - }) - ); - - dispatch( - actions.operationCompleted({ - operationId - }) - ); - }, - - createNewEmptyDirectory: - (params: { basename: string }) => - async (...args) => { - const { basename } = params; - - const [dispatch, getState] = args; - - const state = getState()[name]; - - const { directoryPath } = state; - - assert(directoryPath !== undefined); - - const operationId = await dispatch( - privateThunks.startOperationWhenAllConflictingOperationHaveCompleted({ - operation: "create", - objects: [ - id({ - kind: "directory", - basename: basename, - policy: "private", - canChangePolicy: false - }) - ] - }) - ); - - await dispatch( - privateThunks.uploadFileAndLogCommand({ - path: pathJoin(directoryPath, params.basename, ".keep"), - blob: new Blob(["This file tells that a directory exists"], { - type: "text/plain" - }), - onUploadProgress: () => {} - }) - ); - - dispatch( - actions.operationCompleted({ - operationId - }) - ); - }, - - bulkDelete: - (params: { s3Objects: S3Object[] }) => - async (...args): Promise => { - const { s3Objects } = params; - - const [dispatch, getState] = args; - - const state = getState()[name]; - - const { directoryPath } = state; - - assert(directoryPath !== undefined); - - const operationId = await dispatch( - privateThunks.startOperationWhenAllConflictingOperationHaveCompleted({ - operation: "delete", - objects: s3Objects - }) - ); - - const s3Client = await dispatch( - s3ConfigManagement.protectedThunks.getS3ConfigAndClientForExplorer() - ).then(r => { - assert(r !== undefined); - return r.s3Client; - }); - - const getFilesPathsToDelete = async (s3Object: S3Object) => { - if (s3Object.kind === "file") { - return [pathJoin(directoryPath, s3Object.basename)]; - } - - const { crawl } = crawlFactory({ - list: async ({ directoryPath }) => { - const { objects } = await s3Client.listObjects({ - path: directoryPath - }); - return { - fileBasenames: objects - .filter(obj => obj.kind === "file") - .map(obj => obj.basename), - directoryBasenames: objects - .filter(obj => obj.kind === "directory") - .map(obj => obj.basename) - }; - } - }); - - const directoryToDeletePath = pathJoin(directoryPath, s3Object.basename); - const { filePaths } = await crawl({ - directoryPath: directoryToDeletePath - }); - - return filePaths.map(filePathRelative => - pathJoin(directoryToDeletePath, filePathRelative) - ); - }; - - const objectNamesToDelete = ( - await Promise.all(s3Objects.map(getFilesPathsToDelete)) - ).flat(); - - const cmdId = Date.now(); - - dispatch( - actions.commandLogIssued({ - cmdId, - cmd: `mc rm -r --force \\\n ${s3Objects.map(({ basename }) => pathJoin("s3", directoryPath, basename)).join(" \\\n ")}` - }) - ); - - await s3Client.deleteFiles({ paths: objectNamesToDelete }); - - dispatch( - actions.commandLogResponseReceived({ - cmdId, - resp: objectNamesToDelete - .map(filePath => `Removed \`${pathJoin("s3", filePath)}\``) - .join("\n") - }) - ); - - dispatch( - actions.operationCompleted({ - operationId - }) - ); - }, - getFileDownloadUrl: - (params: { basename: string; validityDurationSecond?: number }) => - async (...args): Promise => { - const { basename, validityDurationSecond = 3_600 } = params; - - const [dispatch, getState] = args; - - const state = getState()[name]; - - const { directoryPath } = state; - - assert(directoryPath !== undefined); - - const path = pathJoin(directoryPath, basename); - - const cmdId = Date.now(); - - const prettyDurationValue = formatDuration({ - durationSeconds: validityDurationSecond, - t: undefined - }); - dispatch( - actions.commandLogIssued({ - cmdId, - cmd: `mc share download --expire ${prettyDurationValue} ${pathJoin("s3", path)}` - }) - ); - - const s3Client = await dispatch( - s3ConfigManagement.protectedThunks.getS3ConfigAndClientForExplorer() - ).then(r => { - assert(r !== undefined); - return r.s3Client; - }); - - const downloadUrl = await s3Client.getFileDownloadUrl({ - path, - validityDurationSecond - }); - - dispatch( - actions.commandLogResponseReceived({ - cmdId, - resp: [ - `URL: ${downloadUrl.split("?")[0]}`, - `Expire: ${prettyDurationValue}`, - `Share: ${downloadUrl}` - ].join("\n") - }) - ); - - return downloadUrl; - }, - openShare: - (params: { fileBasename: string }) => - async (...args) => { - const { fileBasename } = params; - - const [dispatch, getState] = args; - - const { directoryPath, objects } = getState()[name]; - - assert(directoryPath !== undefined); - - const { s3Client, s3Config } = await dispatch( - s3ConfigManagement.protectedThunks.getS3ConfigAndClientForExplorer() - ).then(r => { - assert(r !== undefined); - return r; - }); - - const currentObj = objects.find( - o => o.basename === fileBasename && o.kind === "file" - ); - - assert(currentObj !== undefined); - - if (currentObj.policy === "public") { - dispatch( - actions.shareOpened({ - fileBasename, - url: `${s3Config.paramsOfCreateS3Client.url}/${pathJoin(directoryPath, fileBasename)}`, - validityDurationSecondOptions: undefined - }) - ); - return; - } - - const tokens = await s3Client.getToken({ doForceRenew: false }); - - assert(tokens !== undefined); - - const { expirationTime = Infinity } = tokens; - - const validityDurationSecondOptions = [ - 3_600, - 12 * 3_600, - 24 * 3_600, - 48 * 3_600, - 7 * 24 * 3_600 - ].filter(validityDuration => validityDuration < expirationTime - Date.now()); - - dispatch( - actions.shareOpened({ - fileBasename, - url: undefined, - validityDurationSecondOptions - }) - ); - }, - closeShare: - () => - (...args) => { - const [dispatch, getState] = args; - - if (getState()[name].share === undefined) { - return; - } - - dispatch(actions.shareClosed()); - }, - changeShareSelectedValidityDuration: - (params: { validityDurationSecond: number }) => - (...args) => { - const { validityDurationSecond } = params; - - const [dispatch] = args; - - dispatch( - actions.shareSelectedValidityDurationChanged({ validityDurationSecond }) - ); - }, - requestShareSignedUrl: - () => - async (...args) => { - const [dispatch, getState] = args; - - const shareView = protectedSelectors.shareView(getState()); - - assert(shareView !== null); - assert(shareView !== undefined); - assert(!shareView.isPublic); - - dispatch(actions.requestSignedUrlStarted()); - - const url = await dispatch( - thunks.getFileDownloadUrl({ - basename: shareView.file.basename, - validityDurationSecond: shareView.validityDurationSecond - }) - ); - - dispatch(actions.requestSignedUrlCompleted({ url })); - }, - getBlobUrl: - (params: { s3Objects: S3Object[] }) => - async (...args): Promise<{ url: string; filename: string }> => { - const { s3Objects } = params; - - const [dispatch, getState] = args; - - const { directoryPath } = getState()[name]; - assert(directoryPath !== undefined); - - const operationId = await dispatch( - privateThunks.startOperationWhenAllConflictingOperationHaveCompleted({ - operation: "downloading", - objects: s3Objects - }) - ); - - const { stream, filename } = - s3Objects.length === 1 && s3Objects[0].kind === "file" - ? await (async () => { - const { stream } = await dispatch( - privateThunks.downloadObject({ s3Object: s3Objects[0] }) - ); - - return { - stream, - filename: s3Objects[0].basename - }; - })() - : await (async () => { - const { stream, zipFileName } = await dispatch( - privateThunks.downloadObjectsAsZip({ s3Objects }) - ); - return { - stream, - filename: zipFileName - }; - })(); - - const blobUrl = URL.createObjectURL(await new Response(stream).blob()); - - dispatch( - actions.operationCompleted({ - operationId - }) - ); - - return { url: blobUrl, filename }; - } -} satisfies Thunks; diff --git a/web/src/core/usecases/index.ts b/web/src/core/usecases/index.ts index bd292355c..6903bcc17 100644 --- a/web/src/core/usecases/index.ts +++ b/web/src/core/usecases/index.ts @@ -2,14 +2,10 @@ import * as autoLogoutCountdown from "./autoLogoutCountdown"; import * as catalog from "./catalog"; import * as clusterEventsMonitor from "./clusterEventsMonitor"; import * as deploymentRegionManagement from "./deploymentRegionManagement"; -import * as fileExplorer from "./fileExplorer"; import * as secretExplorer from "./secretExplorer"; import * as launcher from "./launcher"; import * as podLogs from "./podLogs"; import * as restorableConfigManagement from "./restorableConfigManagement"; -import * as s3ConfigConnectionTest from "./s3ConfigConnectionTest"; -import * as s3ConfigCreation from "./s3ConfigCreation"; -import * as s3ConfigManagement from "./s3ConfigManagement"; import * as serviceDetails from "./serviceDetails"; import * as serviceManagement from "./serviceManagement"; import * as userAuthentication from "./userAuthentication"; @@ -25,19 +21,19 @@ import * as projectManagement from "./projectManagement"; import * as viewQuotas from "./viewQuotas"; import * as dataCollection from "./dataCollection"; +import * as s3ProfilesManagement from "./s3ProfilesManagement"; +import * as s3ProfilesCreationUiController from "./s3ProfilesCreationUiController"; +import * as s3ExplorerUiController from "./s3ExplorerUiController"; + export const usecases = { autoLogoutCountdown, catalog, clusterEventsMonitor, deploymentRegionManagement, - fileExplorer, secretExplorer, launcher, podLogs, restorableConfigManagement, - s3ConfigConnectionTest, - s3ConfigCreation, - s3ConfigManagement, serviceDetails, serviceManagement, userAuthentication, @@ -51,5 +47,8 @@ export const usecases = { dataExplorer, projectManagement, viewQuotas, - dataCollection + dataCollection, + s3ProfilesManagement, + s3ProfilesCreationUiController, + s3ExplorerUiController }; diff --git a/web/src/core/usecases/launcher/selectors.ts b/web/src/core/usecases/launcher/selectors.ts index 546d4a0ac..cc68a0956 100644 --- a/web/src/core/usecases/launcher/selectors.ts +++ b/web/src/core/usecases/launcher/selectors.ts @@ -7,7 +7,7 @@ import * as projectManagement from "core/usecases/projectManagement"; import * as userConfigs from "core/usecases/userConfigs"; import { exclude } from "tsafe/exclude"; import { createSelector } from "clean-architecture"; -import * as s3ConfigManagement from "core/usecases/s3ConfigManagement"; +import * as s3ConfigManagement from "core/usecases/s3ProfilesManagement"; import { id } from "tsafe/id"; import { computeRootForm } from "./decoupledLogic"; import { computeDiff } from "core/tools/Stringifyable"; @@ -154,8 +154,8 @@ const chartVersion = createSelector(readyState, state => { return state.chartVersion; }); -const s3ConfigSelect = createSelector( - s3ConfigManagement.selectors.s3Configs, +const s3ProfileSelect = createSelector( + s3ConfigManagement.selectors.s3Profiles, isReady, projectManagement.selectors.canInjectPersonalInfos, createSelector(readyState, state => { @@ -177,7 +177,7 @@ const s3ConfigSelect = createSelector( } const availableConfigs = s3Configs.filter( - config => canInjectPersonalInfos || config.origin !== "deploymentRegion" + config => canInjectPersonalInfos || config.origin !== "defined in region" ); // We don't display the s3 config selector if there is no config or only one @@ -186,15 +186,8 @@ const s3ConfigSelect = createSelector( } return { - options: availableConfigs.map(s3Config => ({ - optionValue: s3Config.id, - label: { - dataSource: s3Config.dataSource, - friendlyName: - s3Config.origin === "project" ? s3Config.friendlyName : undefined - } - })), - selectedOptionValue: s3Config.s3ConfigId + availableProfileNames: availableConfigs.map(s3Config => s3Config.profileName), + selectedProfileName: s3Config.s3ProfileName }; } ); @@ -218,7 +211,7 @@ const restorableConfig = createSelector( if (state === null) { return null; } - return state.s3Config.isChartUsingS3 ? state.s3Config.s3ConfigId : undefined; + return state.s3Config.isChartUsingS3 ? state.s3Config.s3ProfileName : undefined; }), helmValues, createSelector(readyState, state => { @@ -235,7 +228,7 @@ const restorableConfig = createSelector( catalogId, chartName, chartVersion, - s3ConfigId, + s3ProfileName, helmValues, helmValues_default ): projectManagement.ProjectConfigs.RestorableServiceConfig | null => { @@ -248,7 +241,7 @@ const restorableConfig = createSelector( assert(isShared !== null); assert(chartName !== null); assert(chartVersion !== null); - assert(s3ConfigId !== null); + assert(s3ProfileName !== null); assert(helmValues !== null); assert(helmValues_default !== null); @@ -263,7 +256,7 @@ const restorableConfig = createSelector( friendlyName, isShared, chartVersion, - s3ConfigId, + s3ProfileName, helmValuesPatch: diffPatch }; } @@ -316,7 +309,7 @@ const isDefaultConfiguration = createSelector( return null; } const { s3Config } = state; - return s3Config.isChartUsingS3 ? s3Config.s3ConfigId_default : undefined; + return s3Config.isChartUsingS3 ? s3Config.s3ProfileName_default : undefined; }), restorableConfig, ( @@ -324,7 +317,7 @@ const isDefaultConfiguration = createSelector( friendlyName_default, chartVersion_default, isShared_default, - s3ConfigId_default, + s3ProfileName_default, restorableConfig ) => { if (!isReady) { @@ -333,14 +326,15 @@ const isDefaultConfiguration = createSelector( assert(friendlyName_default !== null); assert(chartVersion_default !== null); assert(isShared_default !== null); - assert(s3ConfigId_default !== null); + assert(s3ProfileName_default !== null); assert(restorableConfig !== null); return ( restorableConfig.chartVersion === chartVersion_default && restorableConfig.isShared === isShared_default && restorableConfig.friendlyName === friendlyName_default && - restorableConfig.helmValuesPatch.length === 0 + restorableConfig.helmValuesPatch.length === 0 && + restorableConfig.s3ProfileName === s3ProfileName_default ); } ); @@ -583,7 +577,7 @@ const groupProjectName = createSelector( currentProject.group === undefined ? undefined : currentProject.name ); -const main = createSelector( +const mainView = createSelector( isReady, friendlyName, isShared, @@ -600,7 +594,7 @@ const main = createSelector( launchScript, commandLogsEntries, groupProjectName, - s3ConfigSelect, + s3ProfileSelect, labeledHelmChartSourceUrls, helmValues, createSelector(readyState, state => { @@ -633,7 +627,7 @@ const main = createSelector( launchScript, commandLogsEntries, groupProjectName, - s3ConfigSelect, + s3ProfileSelect, labeledHelmChartSourceUrls, helmValues, helmValuesSchema_forDataTextEditor, @@ -660,7 +654,7 @@ const main = createSelector( assert(launchScript !== null); assert(commandLogsEntries !== null); assert(groupProjectName !== null); - assert(s3ConfigSelect !== null); + assert(s3ProfileSelect !== null); assert(labeledHelmChartSourceUrls !== null); assert(helmValues !== null); assert(helmValuesSchema_forDataTextEditor !== null); @@ -683,7 +677,7 @@ const main = createSelector( launchScript, commandLogsEntries, groupProjectName, - s3ConfigSelect, + s3ProfileSelect, labeledHelmChartSourceUrls, helmValues, helmValuesSchema_forDataTextEditor, @@ -692,7 +686,7 @@ const main = createSelector( } ); -export const selectors = { main }; +export const selectors = { mainView }; export const privateSelectors = { helmReleaseName, diff --git a/web/src/core/usecases/launcher/state.ts b/web/src/core/usecases/launcher/state.ts index e9f3a5e4c..955150d5c 100644 --- a/web/src/core/usecases/launcher/state.ts +++ b/web/src/core/usecases/launcher/state.ts @@ -47,8 +47,8 @@ export declare namespace State { | { isChartUsingS3: false } | { isChartUsingS3: true; - s3ConfigId: string | undefined; - s3ConfigId_default: string | undefined; + s3ProfileName: string | undefined; + s3ProfileName_default: string | undefined; }; helmDependencies: { diff --git a/web/src/core/usecases/launcher/thunks.ts b/web/src/core/usecases/launcher/thunks.ts index bc5da3e6f..6f4b06399 100644 --- a/web/src/core/usecases/launcher/thunks.ts +++ b/web/src/core/usecases/launcher/thunks.ts @@ -3,10 +3,9 @@ import { assert, type Equals, is } from "tsafe/assert"; import * as userAuthentication from "../userAuthentication"; import * as deploymentRegionManagement from "core/usecases/deploymentRegionManagement"; import * as projectManagement from "core/usecases/projectManagement"; -import * as s3ConfigManagement from "core/usecases/s3ConfigManagement"; +import * as s3ProfilesManagement from "core/usecases/s3ProfilesManagement"; import * as userConfigsUsecase from "core/usecases/userConfigs"; import * as userProfileForm from "core/usecases/userProfileForm"; -import { bucketNameAndObjectNameFromS3Path } from "core/adapters/s3Client/utils/bucketNameAndObjectNameFromS3Path"; import { parseUrl } from "core/tools/parseUrl"; import * as secretExplorer from "../secretExplorer"; import { actions } from "./state"; @@ -28,7 +27,7 @@ type RestorableServiceConfigLike = { chartVersion: string | undefined; friendlyName: string | undefined; isShared: boolean | undefined; - s3ConfigId: string | undefined; + s3ProfileName: string | undefined; helmValuesPatch: | { path: (string | number)[]; @@ -47,7 +46,7 @@ type RestorableServiceConfigLike = { : RestorableServiceConfig[Key] | undefined; }; - assert>(); + assert>; } export const thunks = { @@ -122,7 +121,7 @@ export const thunks = { chartVersion: chartVersion_pinned, friendlyName, isShared, - s3ConfigId: s3ConfigId_pinned, + s3ProfileName: s3ProfileName_pinned, helmValuesPatch }, autoLaunch @@ -172,47 +171,50 @@ export const thunks = { const doInjectPersonalInfos = projectManagement.selectors.canInjectPersonalInfos(getState()); - const { s3ConfigId, s3ConfigId_default } = (() => { - const s3Configs = s3ConfigManagement.selectors - .s3Configs(getState()) + const { s3ProfileName, s3ProfileName_default } = (() => { + const s3Profiles = s3ProfilesManagement.selectors + .s3Profiles(getState()) .filter(s3Config => - doInjectPersonalInfos ? true : s3Config.origin === "project" + doInjectPersonalInfos + ? true + : s3Config.origin === + "created by user (or group project member)" ); - const s3ConfigId_default = (() => { - const s3Config = s3Configs.find( - s3Config => s3Config.isXOnyxiaDefault - ); - if (s3Config === undefined) { - return undefined; - } - - return s3Config.id; - })(); - - const s3ConfigId = (() => { - use_pinned_s3_config: { - if (s3ConfigId_pinned === undefined) { - break use_pinned_s3_config; + const s3ProfileName_default = ( + s3Profiles.find( + s3Profile => s3Profile.profileName === "default" + ) ?? + s3Profiles.find( + s3Profile => s3Profile.origin === "defined in region" + ) ?? + s3Profiles.find(() => true) + )?.profileName; + + const s3ProfileName = (() => { + use_pinned_s3_profile: { + if (s3ProfileName_pinned === undefined) { + break use_pinned_s3_profile; } - const s3Config = s3Configs.find( - s3Config => s3Config.id === s3ConfigId_pinned + const s3Config = s3Profiles.find( + s3Profile => + s3Profile.profileName === s3ProfileName_pinned ); if (s3Config === undefined) { - break use_pinned_s3_config; + break use_pinned_s3_profile; } - return s3Config.id; + return s3Config.profileName; } - return s3ConfigId_default; + return s3ProfileName_default; })(); - return { s3ConfigId, s3ConfigId_default }; + return { s3ProfileName, s3ProfileName_default }; })(); const xOnyxiaContext = await dispatch( protectedThunks.getXOnyxiaContext({ - s3ConfigId, + s3ProfileName, doInjectPersonalInfos }) ); @@ -285,8 +287,8 @@ export const thunks = { ? { isChartUsingS3: false } : { isChartUsingS3: true, - s3ConfigId, - s3ConfigId_default + s3ProfileName, + s3ProfileName_default }, helmDependencies, @@ -362,7 +364,7 @@ export const thunks = { friendlyName: undefined, helmValuesPatch: undefined, isShared: undefined, - s3ConfigId: undefined + s3ProfileName: undefined }, autoLaunch: false }) @@ -394,18 +396,18 @@ export const thunks = { }) ); }, - changeS3Config: - (params: { s3ConfigId: string }) => + changeS3Profile: + (params: { s3ProfileName: string }) => async (...args) => { const [dispatch, getState] = args; - const { s3ConfigId } = params; + const { s3ProfileName } = params; const restorableConfig = privateSelectors.restorableConfig(getState()); assert(restorableConfig !== null); - if (restorableConfig.s3ConfigId === s3ConfigId) { + if (restorableConfig.s3ProfileName === s3ProfileName) { // NOTE: No changes, skip. return; } @@ -414,7 +416,7 @@ export const thunks = { thunks.initialize({ restorableConfig: { ...restorableConfig, - s3ConfigId + s3ProfileName }, autoLaunch: false }) @@ -574,9 +576,9 @@ const { getContext, setContext, getIsContextSet } = createUsecaseContextApi<{ export const protectedThunks = { getXOnyxiaContext: - (params: { s3ConfigId: string | undefined; doInjectPersonalInfos: boolean }) => + (params: { s3ProfileName: string | undefined; doInjectPersonalInfos: boolean }) => async (...args): Promise => { - const { s3ConfigId, doInjectPersonalInfos } = params; + const { s3ProfileName, doInjectPersonalInfos } = params; const [ dispatch, @@ -675,16 +677,16 @@ export const protectedThunks = { }; })(), s3: await (async () => { - const s3Config = (() => { - if (s3ConfigId === undefined) { + const s3Profile = (() => { + if (s3ProfileName === undefined) { return undefined; } - const s3Configs = - s3ConfigManagement.selectors.s3Configs(getState()); + const s3Profiles = + s3ProfilesManagement.selectors.s3Profiles(getState()); - const s3Config = s3Configs.find( - s3Config => s3Config.id === s3ConfigId + const s3Config = s3Profiles.find( + s3Profile => s3Profile.profileName === s3ProfileName ); assert(s3Config !== undefined); @@ -692,40 +694,33 @@ export const protectedThunks = { return s3Config; })(); - if (s3Config === undefined) { + if (s3Profile === undefined) { return undefined; } const { host = "", port = 443 } = - s3Config.paramsOfCreateS3Client.url !== "" - ? parseUrl(s3Config.paramsOfCreateS3Client.url) + s3Profile.paramsOfCreateS3Client.url !== "" + ? parseUrl(s3Profile.paramsOfCreateS3Client.url) : {}; - const { bucketName, objectName: objectNamePrefix } = - bucketNameAndObjectNameFromS3Path(s3Config.workingDirectoryPath); - const s3: XOnyxiaContext["s3"] = { isEnabled: true, AWS_ACCESS_KEY_ID: undefined, AWS_SECRET_ACCESS_KEY: undefined, AWS_SESSION_TOKEN: undefined, - AWS_BUCKET_NAME: bucketName, - AWS_DEFAULT_REGION: s3Config.region ?? "us-east-1", + AWS_DEFAULT_REGION: + s3Profile.paramsOfCreateS3Client.region ?? "us-east-1", AWS_S3_ENDPOINT: host, port, - pathStyleAccess: s3Config.paramsOfCreateS3Client.pathStyleAccess, - objectNamePrefix, - workingDirectoryPath: s3Config.workingDirectoryPath, + pathStyleAccess: s3Profile.paramsOfCreateS3Client.pathStyleAccess, isAnonymous: false }; - if (s3Config.paramsOfCreateS3Client.isStsEnabled) { + if (s3Profile.paramsOfCreateS3Client.isStsEnabled) { const s3Client = await dispatch( - s3ConfigManagement.protectedThunks.getS3ClientForSpecificConfig( - { - s3ConfigId: s3Config.id - } - ) + s3ProfilesManagement.protectedThunks.getS3Client({ + profileName: s3Profile.profileName + }) ); const tokens = await s3Client.getToken({ doForceRenew: false }); @@ -736,14 +731,14 @@ export const protectedThunks = { s3.AWS_SECRET_ACCESS_KEY = tokens.secretAccessKey; s3.AWS_SESSION_TOKEN = tokens.sessionToken; } else if ( - s3Config.paramsOfCreateS3Client.credentials !== undefined + s3Profile.paramsOfCreateS3Client.credentials !== undefined ) { s3.AWS_ACCESS_KEY_ID = - s3Config.paramsOfCreateS3Client.credentials.accessKeyId; + s3Profile.paramsOfCreateS3Client.credentials.accessKeyId; s3.AWS_SECRET_ACCESS_KEY = - s3Config.paramsOfCreateS3Client.credentials.secretAccessKey; + s3Profile.paramsOfCreateS3Client.credentials.secretAccessKey; s3.AWS_SESSION_TOKEN = - s3Config.paramsOfCreateS3Client.credentials.sessionToken; + s3Profile.paramsOfCreateS3Client.credentials.sessionToken; } return s3; diff --git a/web/src/core/usecases/projectManagement/decoupledLogic/ProjectConfigs.ts b/web/src/core/usecases/projectManagement/decoupledLogic/ProjectConfigs.ts index 7e90cc574..4940045b3 100644 --- a/web/src/core/usecases/projectManagement/decoupledLogic/ProjectConfigs.ts +++ b/web/src/core/usecases/projectManagement/decoupledLogic/ProjectConfigs.ts @@ -5,26 +5,22 @@ import { z } from "zod"; import { id } from "tsafe/id"; import type { OptionalIfCanBeUndefined } from "core/tools/OptionalIfCanBeUndefined"; import { zStringifyableAtomic } from "core/tools/Stringifyable"; +import { type S3Uri, zS3Uri } from "core/tools/S3Uri"; export type ProjectConfigs = { - __modelVersion: 1; + __modelVersion: 2; servicePassword: string; - restorableConfigs: ProjectConfigs.RestorableServiceConfig[]; - s3: { - s3Configs: ProjectConfigs.S3Config[]; - s3ConfigId_defaultXOnyxia: string | undefined; - s3ConfigId_explorer: string | undefined; - }; + restorableServiceConfigs: ProjectConfigs.RestorableServiceConfig[]; + s3Profiles: ProjectConfigs.S3Profile[]; clusterNotificationCheckoutTime: number; }; export namespace ProjectConfigs { - export type S3Config = { + export type S3Profile = { + profileName: string; creationTime: number; - friendlyName: string; url: string; region: string | undefined; - workingDirectoryPath: string; pathStyleAccess: boolean; credentials: | { @@ -33,15 +29,23 @@ export namespace ProjectConfigs { sessionToken: string | undefined; } | undefined; + bookmarks: S3Profile.Bookmark[] | undefined; }; + export namespace S3Profile { + export type Bookmark = { + displayName: string | undefined; + s3Uri: S3Uri; + }; + } + export type RestorableServiceConfig = { friendlyName: string; isShared: boolean | undefined; catalogId: string; chartName: string; chartVersion: string; - s3ConfigId: string | undefined; + s3ProfileName: string | undefined; helmValuesPatch: { path: (string | number)[]; value: StringifyableAtomic | undefined; @@ -75,7 +79,7 @@ const zRestorableServiceConfig = (() => { catalogId: z.string(), chartName: z.string(), chartVersion: z.string(), - s3ConfigId: z.union([z.string(), z.undefined()]), + s3ProfileName: z.union([z.string(), z.undefined()]), helmValuesPatch: z.array(zHelmValuesPatch) }); @@ -86,7 +90,7 @@ const zRestorableServiceConfig = (() => { })(); const zS3Credentials = (() => { - type TargetType = Exclude; + type TargetType = Exclude; const zTargetType = z.object({ accessKeyId: z.string(), @@ -100,17 +104,12 @@ const zS3Credentials = (() => { return id>(zTargetType); })(); -const zS3Config = (() => { - type TargetType = ProjectConfigs.S3Config; +const zS3ConfigBookmark = (() => { + type TargetType = ProjectConfigs.S3Profile.Bookmark; const zTargetType = z.object({ - creationTime: z.number(), - friendlyName: z.string(), - url: z.string(), - region: z.union([z.string(), z.undefined()]), - workingDirectoryPath: z.string(), - pathStyleAccess: z.boolean(), - credentials: z.union([zS3Credentials, z.undefined()]) + displayName: z.union([z.string(), z.undefined()]), + s3Uri: zS3Uri }); assert, OptionalIfCanBeUndefined>>(); @@ -119,16 +118,20 @@ const zS3Config = (() => { return id>(zTargetType); })(); -const zS3 = (() => { - type TargetType = ProjectConfigs["s3"]; +const zS3Profile = (() => { + type TargetType = ProjectConfigs.S3Profile; const zTargetType = z.object({ - s3Configs: z.array(zS3Config), - s3ConfigId_defaultXOnyxia: z.union([z.string(), z.undefined()]), - s3ConfigId_explorer: z.union([z.string(), z.undefined()]) + creationTime: z.number(), + profileName: z.string(), + url: z.string(), + region: z.union([z.string(), z.undefined()]), + pathStyleAccess: z.boolean(), + credentials: z.union([zS3Credentials, z.undefined()]), + bookmarks: z.union([z.array(zS3ConfigBookmark), z.undefined()]) }); - assert, OptionalIfCanBeUndefined>>(); + assert, OptionalIfCanBeUndefined>>; // @ts-expect-error return id>(zTargetType); @@ -138,10 +141,10 @@ export const zProjectConfigs = (() => { type TargetType = ProjectConfigs; const zTargetType = z.object({ - __modelVersion: z.literal(1), + __modelVersion: z.literal(2), servicePassword: z.string(), - restorableConfigs: z.array(zRestorableServiceConfig), - s3: zS3, + restorableServiceConfigs: z.array(zRestorableServiceConfig), + s3Profiles: z.array(zS3Profile), clusterNotificationCheckoutTime: z.number() }); diff --git a/web/src/core/usecases/projectManagement/decoupledLogic/clearProjectConfigs.ts b/web/src/core/usecases/projectManagement/decoupledLogic/clearProjectConfigs.ts index 0c57fbf99..44978c58c 100644 --- a/web/src/core/usecases/projectManagement/decoupledLogic/clearProjectConfigs.ts +++ b/web/src/core/usecases/projectManagement/decoupledLogic/clearProjectConfigs.ts @@ -27,8 +27,8 @@ export async function clearProjectConfigs(params: { [ "__modelVersion", "servicePassword", - "restorableConfigs", - "s3", + "restorableServiceConfigs", + "s3Profiles", "clusterNotificationCheckoutTime" ] as const ).map(async key => { diff --git a/web/src/core/usecases/projectManagement/decoupledLogic/projectConfigsMigration/index.ts b/web/src/core/usecases/projectManagement/decoupledLogic/projectConfigsMigration/index.ts deleted file mode 100644 index 8ab4e9ee4..000000000 --- a/web/src/core/usecases/projectManagement/decoupledLogic/projectConfigsMigration/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./projectConfigsMigration"; diff --git a/web/src/core/usecases/projectManagement/decoupledLogic/projectConfigsMigration/projectConfigsMigration.ts b/web/src/core/usecases/projectManagement/decoupledLogic/projectConfigsMigration/projectConfigsMigration.ts deleted file mode 100644 index 81f0f398b..000000000 --- a/web/src/core/usecases/projectManagement/decoupledLogic/projectConfigsMigration/projectConfigsMigration.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { v0ToV1 } from "./v0ToV1"; -import type { SecretsManager } from "core/ports/SecretsManager"; -import type { ProjectConfigs } from "../ProjectConfigs"; -import { assert, type Equals } from "tsafe/assert"; -import { clearProjectConfigs } from "../clearProjectConfigs"; -import { join as pathJoin } from "pathe"; -import { secretToValue } from "../secretParsing"; - -export async function projectConfigsMigration(params: { - projectVaultTopDirPath_reserved: string; - secretsManager: SecretsManager; -}) { - const { projectVaultTopDirPath_reserved, secretsManager } = params; - - const modelVersion = await (async () => { - const key = "__modelVersion"; - - assert>>(); - - const modelVersion = await secretsManager - .get({ - path: pathJoin(projectVaultTopDirPath_reserved, key) - }) - .then( - ({ secret }) => secretToValue(secret) as number, - () => { - console.log("The above error is ok"); - return undefined; - } - ); - - return modelVersion ?? 0; - })(); - - try { - if (modelVersion < 1) { - await v0ToV1({ - projectVaultTopDirPath_reserved, - secretsManager - }); - } - } catch { - console.warn("Migration of the ProjectConfigs failed, clearing everything"); - - await clearProjectConfigs({ - projectVaultTopDirPath_reserved, - secretsManager - }); - } -} diff --git a/web/src/core/usecases/projectManagement/decoupledLogic/projectConfigsMigration/v0ToV1.ts b/web/src/core/usecases/projectManagement/decoupledLogic/projectConfigsMigration/v0ToV1.ts deleted file mode 100644 index affa7943a..000000000 --- a/web/src/core/usecases/projectManagement/decoupledLogic/projectConfigsMigration/v0ToV1.ts +++ /dev/null @@ -1,313 +0,0 @@ -import { assert, type Equals } from "tsafe/assert"; -import type { StringifyableAtomic } from "core/tools/Stringifyable"; -import type { SecretsManager } from "core/ports/SecretsManager"; -import { join as pathJoin } from "pathe"; -import { secretToValue, valueToSecret } from "../secretParsing"; -import YAML from "yaml"; -import { getS3Configs } from "core/usecases/s3ConfigManagement/decoupledLogic/getS3Configs"; - -namespace v0 { - export type ProjectConfigs = { - servicePassword: string; - restorableConfigs: ProjectConfigs.RestorableServiceConfig[]; - s3: { - customConfigs: ProjectConfigs.CustomS3Config[]; - indexForXOnyxia: number | undefined; - indexForExplorer: number | undefined; - }; - clusterNotificationCheckoutTime: number; - }; - - namespace ProjectConfigs { - export type CustomS3Config = { - url: string; - region: string; - workingDirectoryPath: string; - pathStyleAccess: boolean; - accountFriendlyName: string; - credentials: - | { - accessKeyId: string; - secretAccessKey: string; - sessionToken: string | undefined; - } - | undefined; - }; - - export type RestorableServiceConfig = { - friendlyName: string; - isShared: boolean | undefined; - catalogId: string; - chartName: string; - chartVersion: string; - formFieldsValueDifferentFromDefault: FormFieldValue[]; - }; - } - - type FormFieldValue = { - path: string[]; - value: FormFieldValue.Value; - }; - - namespace FormFieldValue { - export type Value = string | boolean | number | Value.Yaml; - - export namespace Value { - export type Yaml = { - type: "yaml"; - yamlStr: string; - }; - } - } -} - -export namespace v1 { - export type ProjectConfigs = { - __modelVersion: 1; - servicePassword: string; - restorableConfigs: ProjectConfigs.RestorableServiceConfig[]; - s3: { - s3Configs: ProjectConfigs.S3Config[]; - s3ConfigId_defaultXOnyxia: string | undefined; - s3ConfigId_explorer: string | undefined; - }; - clusterNotificationCheckoutTime: number; - }; - - export namespace ProjectConfigs { - export type S3Config = { - creationTime: number; - friendlyName: string; - url: string; - region: string | undefined; - workingDirectoryPath: string; - pathStyleAccess: boolean; - credentials: - | { - accessKeyId: string; - secretAccessKey: string; - sessionToken: string | undefined; - } - | undefined; - }; - - export type RestorableServiceConfig = { - friendlyName: string; - isShared: boolean | undefined; - catalogId: string; - chartName: string; - chartVersion: string; - s3ConfigId: string | undefined; - helmValuesPatch: { - path: (string | number)[]; - value: StringifyableAtomic | undefined; - }[]; - }; - } -} - -export async function v0ToV1(params: { - projectVaultTopDirPath_reserved: string; - secretsManager: SecretsManager; -}) { - const { projectVaultTopDirPath_reserved, secretsManager } = params; - - console.log("Performing v0 to v1 migration"); - - for (const key of [ - "servicePassword", - "s3", - "restorableConfigs", - "clusterNotificationCheckoutTime" - ] as const) { - assert>(); - - switch (key) { - case "servicePassword": - assert< - Equals - >(); - break; - case "restorableConfigs": - { - const path = pathJoin(projectVaultTopDirPath_reserved, key); - - assert(); - - const legacyValue = await secretsManager - .get({ path }) - .then( - ({ secret }) => - secretToValue(secret) as v0.ProjectConfigs[typeof key] - ); - - const newValue: v1.ProjectConfigs[typeof key] = []; - - legacyValue.forEach(restorableServiceConfig_legacy => { - newValue.push({ - friendlyName: restorableServiceConfig_legacy.friendlyName, - isShared: restorableServiceConfig_legacy.isShared, - catalogId: restorableServiceConfig_legacy.catalogId, - chartName: restorableServiceConfig_legacy.chartName, - chartVersion: restorableServiceConfig_legacy.chartVersion, - s3ConfigId: undefined, - helmValuesPatch: (() => { - const helmValuesPatch: { - path: (string | number)[]; - value: StringifyableAtomic | undefined; - }[] = []; - - restorableServiceConfig_legacy.formFieldsValueDifferentFromDefault.forEach( - formFieldValue => { - if (typeof formFieldValue.value === "object") { - assert(formFieldValue.value.type === "yaml"); - - let parsed: unknown; - - try { - parsed = YAML.parse( - formFieldValue.value.yamlStr - ); - } catch { - return undefined; - } - - if ( - typeof parsed !== "object" || - parsed === null - ) { - return; - } - - (function callee( - path: (string | number)[], - o: object - ) { - Object.entries(o).forEach( - ([segment, value]) => { - const newPath = [ - ...path, - segment - ]; - - if ( - typeof value === "object" && - value !== null - ) { - callee(newPath, value); - return; - } - - helmValuesPatch.push({ - path: newPath, - value - }); - } - ); - })(formFieldValue.path, parsed); - - return; - } - - helmValuesPatch.push({ - path: formFieldValue.path, - value: formFieldValue.value - }); - } - ); - - return helmValuesPatch; - })() - }); - }); - - await secretsManager.put({ - path, - secret: valueToSecret(newValue) - }); - } - break; - case "s3": - { - const path = pathJoin(projectVaultTopDirPath_reserved, key); - - assert(); - - const legacyValue = await secretsManager - .get({ path }) - .then( - ({ secret }) => - secretToValue(secret) as v0.ProjectConfigs[typeof key] - ); - - const newValue: v1.ProjectConfigs[typeof key] = { - s3Configs: [], - s3ConfigId_defaultXOnyxia: undefined, - s3ConfigId_explorer: undefined - }; - - legacyValue.customConfigs.forEach((customS3Config_legacy, i) => { - newValue.s3Configs.push({ - creationTime: Date.now() + i, - friendlyName: customS3Config_legacy.accountFriendlyName, - url: customS3Config_legacy.url, - region: customS3Config_legacy.region, - workingDirectoryPath: - customS3Config_legacy.workingDirectoryPath, - pathStyleAccess: customS3Config_legacy.pathStyleAccess, - credentials: customS3Config_legacy.credentials - }); - }); - - { - const s3Configs = getS3Configs({ - projectConfigsS3: newValue, - s3RegionConfigs: [], - configTestResults: [], - ongoingConfigTests: [], - resolvedAdminBookmarks: [], - username: "johndoe", - projectGroup: undefined, - groupProjects: [] - }); - - for (const [propertyName_legacy, propertyName] of [ - ["indexForXOnyxia", "s3ConfigId_defaultXOnyxia"], - ["indexForExplorer", "s3ConfigId_explorer"] - ] as const) { - if (legacyValue[propertyName_legacy] !== undefined) { - const entry = - newValue.s3Configs[legacyValue[propertyName_legacy]]; - - assert(entry !== undefined); - - const s3Config = s3Configs.find( - s3Config => - s3Config.origin === "project" && - s3Config.creationTime === entry.creationTime - ); - - assert(s3Config !== undefined); - - newValue[propertyName] = s3Config.id; - } else { - newValue[propertyName] = - "a-config-id-that-does-not-exist"; - } - } - } - - await secretsManager.put({ - path, - secret: valueToSecret(newValue) - }); - } - break; - case "clusterNotificationCheckoutTime": - assert< - Equals - >(); - break; - } - } -} diff --git a/web/src/core/usecases/projectManagement/thunks.ts b/web/src/core/usecases/projectManagement/thunks.ts index 92c317979..e89583078 100644 --- a/web/src/core/usecases/projectManagement/thunks.ts +++ b/web/src/core/usecases/projectManagement/thunks.ts @@ -7,12 +7,8 @@ import { protectedSelectors } from "./selectors"; import * as userConfigs from "core/usecases/userConfigs"; import { same } from "evt/tools/inDepth"; import { id } from "tsafe/id"; -import { updateDefaultS3ConfigsAfterPotentialDeletion } from "core/usecases/s3ConfigManagement/decoupledLogic/updateDefaultS3ConfigsAfterPotentialDeletion"; -import * as deploymentRegionManagement from "core/usecases/deploymentRegionManagement"; import { getProjectVaultTopDirPath_reserved } from "./decoupledLogic/projectVaultTopDirPath_reserved"; import { secretToValue, valueToSecret } from "./decoupledLogic/secretParsing"; -import { projectConfigsMigration } from "./decoupledLogic/projectConfigsMigration"; -import { symToStr } from "tsafe/symToStr"; import { type ProjectConfigs, zProjectConfigs } from "./decoupledLogic/ProjectConfigs"; import { clearProjectConfigs } from "./decoupledLogic/clearProjectConfigs"; import { Mutex } from "async-mutex"; @@ -22,11 +18,8 @@ export const thunks = { changeProject: (params: { projectId: string }) => async (...args) => { - const [ - dispatch, - getState, - { onyxiaApi, secretsManager, paramsOfBootstrapCore } - ] = args; + const [dispatch, , { onyxiaApi, secretsManager, paramsOfBootstrapCore }] = + args; const { projectId } = params; @@ -69,11 +62,6 @@ export const thunks = { projectVaultTopDirPath }); - await projectConfigsMigration({ - secretsManager, - projectVaultTopDirPath_reserved - }); - const { projectConfigs } = await (async function getProjectConfig(): Promise<{ projectConfigs: ProjectConfigs; }> { @@ -131,46 +119,6 @@ export const thunks = { await prOnboarding; - maybe_update_pinned_default_s3_configs: { - const actions = updateDefaultS3ConfigsAfterPotentialDeletion({ - projectConfigsS3: projectConfigs.s3, - s3RegionConfigs: - deploymentRegionManagement.selectors.currentDeploymentRegion( - getState() - ).s3Configs - }); - - let needUpdate = false; - - for (const propertyName of [ - "s3ConfigId_defaultXOnyxia", - "s3ConfigId_explorer" - ] as const) { - const action = actions[propertyName]; - - if (!action.isUpdateNeeded) { - continue; - } - - needUpdate = true; - - projectConfigs.s3[propertyName] = action.s3ConfigId; - } - - if (!needUpdate) { - break maybe_update_pinned_default_s3_configs; - } - - { - const { s3 } = projectConfigs; - - await secretsManager.put({ - path: pathJoin(projectVaultTopDirPath_reserved, symToStr({ s3 })), - secret: valueToSecret(s3) - }); - } - } - const projectWithInjectedPersonalInfos = projects.map(project => ({ ...project, doInjectPersonalInfos: @@ -209,8 +157,8 @@ export const thunks = { const keys = [ "__modelVersion", "servicePassword", - "restorableConfigs", - "s3", + "restorableServiceConfigs", + "s3Profiles", "clusterNotificationCheckoutTime" ] as const; @@ -220,7 +168,7 @@ function getDefaultConfig(key_: K): ProjectConfi const key = key_ as keyof ProjectConfigs; switch (key) { case "__modelVersion": { - const out: ProjectConfigs[typeof key] = 1; + const out: ProjectConfigs[typeof key] = 2; // @ts-expect-error return out; } @@ -229,18 +177,13 @@ function getDefaultConfig(key_: K): ProjectConfi // @ts-expect-error return out; } - case "restorableConfigs": { + case "restorableServiceConfigs": { const out: ProjectConfigs[typeof key] = []; // @ts-expect-error return out; } - case "s3": { - const out: ProjectConfigs[typeof key] = { - s3Configs: [], - // NOTE: We will set to the correct default at initialization - s3ConfigId_defaultXOnyxia: "a-config-id-that-does-not-exist", - s3ConfigId_explorer: "a-config-id-that-does-not-exist" - }; + case "s3Profiles": { + const out: ProjectConfigs[typeof key] = []; // @ts-expect-error return out; } diff --git a/web/src/core/usecases/restorableConfigManagement/decoupledLogic/getAreSameRestorableConfig.ts b/web/src/core/usecases/restorableConfigManagement/decoupledLogic/getAreSameRestorableConfig.ts index dd3f9910d..ccfd76d18 100644 --- a/web/src/core/usecases/restorableConfigManagement/decoupledLogic/getAreSameRestorableConfig.ts +++ b/web/src/core/usecases/restorableConfigManagement/decoupledLogic/getAreSameRestorableConfig.ts @@ -12,7 +12,7 @@ export function getAreSameRestorableConfig( "catalogId", "chartName", "chartVersion", - "s3ConfigId", + "s3ProfileName", "helmValuesPatch" ] as const) { assert< @@ -23,7 +23,7 @@ export function getAreSameRestorableConfig( "creationTime" > > - >(); + >; if (key === "helmValuesPatch") { if ( diff --git a/web/src/core/usecases/restorableConfigManagement/selectors.ts b/web/src/core/usecases/restorableConfigManagement/selectors.ts index 73d849cc4..0f66bd813 100644 --- a/web/src/core/usecases/restorableConfigManagement/selectors.ts +++ b/web/src/core/usecases/restorableConfigManagement/selectors.ts @@ -3,15 +3,13 @@ import type { State as RootState } from "core/bootstrap"; import { name } from "./state"; import * as projectManagement from "core/usecases/projectManagement"; -function state(rootState: RootState) { - return rootState[name]; -} +const state = (rootState: RootState) => rootState[name]; const restorableConfigs = createSelector( projectManagement.protectedSelectors.projectConfig, createSelector(state, state => state.indexedChartsIcons), - ({ restorableConfigs }, indexedChartsIcons) => - restorableConfigs.map(restorableConfig => ({ + ({ restorableServiceConfigs }, indexedChartsIcons) => + restorableServiceConfigs.map(restorableConfig => ({ ...restorableConfig, chartIconUrl: indexedChartsIcons[restorableConfig.catalogId]?.[ diff --git a/web/src/core/usecases/restorableConfigManagement/thunks.ts b/web/src/core/usecases/restorableConfigManagement/thunks.ts index 30ac40e1c..345d0a663 100644 --- a/web/src/core/usecases/restorableConfigManagement/thunks.ts +++ b/web/src/core/usecases/restorableConfigManagement/thunks.ts @@ -46,11 +46,11 @@ export const thunks = { const { restorableConfig } = params; - const { restorableConfigs } = + const { restorableServiceConfigs } = projectManagement.protectedSelectors.projectConfig(getState()); const restorableConfig_withSameRef = (() => { - const results = restorableConfigs.filter(restorableConfig_i => + const results = restorableServiceConfigs.filter(restorableConfig_i => getAreSameRestorableConfigRef(restorableConfig_i, restorableConfig) ); @@ -71,12 +71,12 @@ export const thunks = { return; } - const restorableConfigs_new = [...restorableConfigs]; + const restorableConfigs_new = [...restorableServiceConfigs]; if (restorableConfig_withSameRef === undefined) { restorableConfigs_new.unshift(restorableConfig); } else { - const i = restorableConfigs.indexOf(restorableConfig_withSameRef); + const i = restorableServiceConfigs.indexOf(restorableConfig_withSameRef); assert(i !== -1); @@ -85,7 +85,7 @@ export const thunks = { await dispatch( projectManagement.protectedThunks.updateConfigValue({ - key: "restorableConfigs", + key: "restorableServiceConfigs", value: restorableConfigs_new }) ); @@ -97,10 +97,10 @@ export const thunks = { const { restorableConfigRef: ref } = params; - const { restorableConfigs } = + const { restorableServiceConfigs } = projectManagement.protectedSelectors.projectConfig(getState()); - const index_toDelete = restorableConfigs.findIndex(c => + const index_toDelete = restorableServiceConfigs.findIndex(c => getAreSameRestorableConfigRef(c, ref) ); @@ -109,13 +109,13 @@ export const thunks = { return; } - const restorableConfigs_new = restorableConfigs.filter( + const restorableConfigs_new = restorableServiceConfigs.filter( (_, index) => index !== index_toDelete ); await dispatch( projectManagement.protectedThunks.updateConfigValue({ - key: "restorableConfigs", + key: "restorableServiceConfigs", value: restorableConfigs_new }) ); @@ -130,10 +130,10 @@ export const thunks = { const { restorableConfigRef: ref, targetIndex } = params; - const { restorableConfigs } = + const { restorableServiceConfigs } = projectManagement.protectedSelectors.projectConfig(getState()); - const index_current = restorableConfigs.findIndex(c => + const index_current = restorableServiceConfigs.findIndex(c => getAreSameRestorableConfigRef(c, ref) ); @@ -143,16 +143,16 @@ export const thunks = { return; } - const restorableConfigs_new = [...restorableConfigs]; + const restorableConfigs_new = [...restorableServiceConfigs]; - const restorableConfig = restorableConfigs[index_current]; + const restorableConfig = restorableServiceConfigs[index_current]; restorableConfigs_new.splice(index_current, 1); restorableConfigs_new.splice(targetIndex, 0, restorableConfig); await dispatch( projectManagement.protectedThunks.updateConfigValue({ - key: "restorableConfigs", + key: "restorableServiceConfigs", value: restorableConfigs_new }) ); @@ -167,10 +167,10 @@ export const thunks = { const { restorableConfigRef: ref, newFriendlyName } = params; - const { restorableConfigs } = + const { restorableServiceConfigs } = projectManagement.protectedSelectors.projectConfig(getState()); - const restorableConfig_current = restorableConfigs.find(c => + const restorableConfig_current = restorableServiceConfigs.find(c => getAreSameRestorableConfigRef(c, ref) ); @@ -180,11 +180,11 @@ export const thunks = { return; } - const index = restorableConfigs.indexOf(restorableConfig_current); + const index = restorableServiceConfigs.indexOf(restorableConfig_current); assert(index !== -1); - const restorableConfigs_new = [...restorableConfigs]; + const restorableConfigs_new = [...restorableServiceConfigs]; const restorableConfig_new = structuredClone(restorableConfig_current); @@ -194,7 +194,7 @@ export const thunks = { await dispatch( projectManagement.protectedThunks.updateConfigValue({ - key: "restorableConfigs", + key: "restorableServiceConfigs", value: restorableConfigs_new }) ); diff --git a/web/src/core/usecases/s3CodeSnippets/selectors.ts b/web/src/core/usecases/s3CodeSnippets/selectors.ts index 976550963..072689fe4 100644 --- a/web/src/core/usecases/s3CodeSnippets/selectors.ts +++ b/web/src/core/usecases/s3CodeSnippets/selectors.ts @@ -2,7 +2,7 @@ import type { State as RootState } from "core/bootstrap"; import { name } from "./state"; import { createSelector } from "clean-architecture"; import { assert } from "tsafe/assert"; -import * as s3ConfigManagement from "core/usecases/s3ConfigManagement"; +import * as s3ProfilesManagement from "core/usecases/s3ProfilesManagement"; const state = (rootState: RootState) => rootState[name]; @@ -321,15 +321,25 @@ const main = createSelector( export const selectors = { main }; -const s3Config = createSelector(s3ConfigManagement.selectors.s3Configs, s3Configs => - s3Configs.find( - s3Config => - s3Config.origin === "deploymentRegion" && - s3Config.paramsOfCreateS3Client.isStsEnabled - ) +const s3Profile = createSelector( + s3ProfilesManagement.selectors.ambientS3Profile, + s3ProfilesManagement.selectors.s3Profiles, + (ambientS3Profile, s3Profiles) => { + if ( + ambientS3Profile !== undefined && + ambientS3Profile.origin === "defined in region" + ) { + return ambientS3Profile; + } + + return ( + s3Profiles.find(s3Profile => s3Profile.profileName === "default") ?? + s3Profiles.find(s3Profile => s3Profile.origin === "defined in region") + ); + } ); export const privateSelectors = { - s3Config, + s3Profile, isRefreshing }; diff --git a/web/src/core/usecases/s3CodeSnippets/thunks.ts b/web/src/core/usecases/s3CodeSnippets/thunks.ts index 5b5a13eff..ececf3128 100644 --- a/web/src/core/usecases/s3CodeSnippets/thunks.ts +++ b/web/src/core/usecases/s3CodeSnippets/thunks.ts @@ -1,7 +1,7 @@ import type { Thunks } from "core/bootstrap"; import { actions } from "./state"; import { assert } from "tsafe/assert"; -import * as s3ConfigManagement from "core/usecases/s3ConfigManagement"; +import * as s3ProfilesManagement from "core/usecases/s3ProfilesManagement"; import type { Technology } from "./state"; import { parseUrl } from "core/tools/parseUrl"; import { privateSelectors } from "./selectors"; @@ -14,7 +14,7 @@ export const thunks = { () => (...args): boolean => { const [, getState] = args; - return privateSelectors.s3Config(getState()) !== undefined; + return privateSelectors.s3Profile(getState()) !== undefined; }, /** Refresh is expected to be called whenever the component that use this slice mounts */ refresh: @@ -30,24 +30,24 @@ export const thunks = { dispatch(actions.refreshStarted()); - const s3Config = privateSelectors.s3Config(getState()); + const s3Profile = privateSelectors.s3Profile(getState()); - assert(s3Config !== undefined); + assert(s3Profile !== undefined); const { region, host, port } = (() => { const { host, port = 443 } = parseUrl( - s3Config.paramsOfCreateS3Client.url + s3Profile.paramsOfCreateS3Client.url ); - const region = s3Config.paramsOfCreateS3Client.region; + const region = s3Profile.paramsOfCreateS3Client.region; return { region, host, port }; })(); const { tokens } = await (async () => { const s3Client = await dispatch( - s3ConfigManagement.protectedThunks.getS3ClientForSpecificConfig({ - s3ConfigId: s3Config.id + s3ProfilesManagement.protectedThunks.getS3Client({ + profileName: s3Profile.profileName }) ); diff --git a/web/src/core/usecases/s3ConfigConnectionTest/selectors.ts b/web/src/core/usecases/s3ConfigConnectionTest/selectors.ts deleted file mode 100644 index 286f13ffd..000000000 --- a/web/src/core/usecases/s3ConfigConnectionTest/selectors.ts +++ /dev/null @@ -1,13 +0,0 @@ -import type { State as RootState } from "core/bootstrap"; -import { createSelector } from "clean-architecture"; -import { name } from "./state"; - -const state = (rootState: RootState) => rootState[name]; - -const configTestResults = createSelector(state, state => state.configTestResults); -const ongoingConfigTests = createSelector(state, state => state.ongoingConfigTests); - -export const protectedSelectors = { - configTestResults, - ongoingConfigTests -}; diff --git a/web/src/core/usecases/s3ConfigConnectionTest/state.ts b/web/src/core/usecases/s3ConfigConnectionTest/state.ts deleted file mode 100644 index 3fb6eecd7..000000000 --- a/web/src/core/usecases/s3ConfigConnectionTest/state.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { createUsecaseActions } from "clean-architecture"; -import { id } from "tsafe/id"; -import type { ParamsOfCreateS3Client } from "core/adapters/s3Client"; -import { same } from "evt/tools/inDepth/same"; - -type State = { - configTestResults: ConfigTestResult[]; - ongoingConfigTests: OngoingConfigTest[]; -}; - -export type OngoingConfigTest = { - paramsOfCreateS3Client: ParamsOfCreateS3Client; - workingDirectoryPath: string; -}; - -export type ConfigTestResult = { - paramsOfCreateS3Client: ParamsOfCreateS3Client; - workingDirectoryPath: string; - result: - | { - isSuccess: true; - } - | { - isSuccess: false; - errorMessage: string; - }; -}; - -export const name = "s3ConfigConnectionTest"; - -export const { actions, reducer } = createUsecaseActions({ - name, - initialState: id({ - configTestResults: [], - ongoingConfigTests: [] - }), - reducers: { - testStarted: ( - state, - { - payload - }: { - payload: State["ongoingConfigTests"][number]; - } - ) => { - const { paramsOfCreateS3Client, workingDirectoryPath } = payload; - - if ( - state.ongoingConfigTests.find(e => - same(e, { paramsOfCreateS3Client, workingDirectoryPath }) - ) !== undefined - ) { - return; - } - - state.ongoingConfigTests.push({ - paramsOfCreateS3Client, - workingDirectoryPath - }); - }, - testCompleted: ( - state, - { - payload - }: { - payload: State["configTestResults"][number]; - } - ) => { - const { paramsOfCreateS3Client, workingDirectoryPath, result } = payload; - - remove_from_ongoing: { - const entry = state.ongoingConfigTests.find(e => - same(e, { paramsOfCreateS3Client, workingDirectoryPath }) - ); - - if (entry === undefined) { - break remove_from_ongoing; - } - - state.ongoingConfigTests.splice( - state.ongoingConfigTests.indexOf(entry), - 1 - ); - } - - remove_existing_result: { - const entry = state.configTestResults.find( - e => - same(e.paramsOfCreateS3Client, paramsOfCreateS3Client) && - e.workingDirectoryPath === workingDirectoryPath - ); - - if (entry === undefined) { - break remove_existing_result; - } - - state.configTestResults.splice(state.configTestResults.indexOf(entry), 1); - } - - state.configTestResults.push({ - paramsOfCreateS3Client, - workingDirectoryPath, - result - }); - } - } -}); diff --git a/web/src/core/usecases/s3ConfigConnectionTest/thunks.ts b/web/src/core/usecases/s3ConfigConnectionTest/thunks.ts deleted file mode 100644 index c7deed032..000000000 --- a/web/src/core/usecases/s3ConfigConnectionTest/thunks.ts +++ /dev/null @@ -1,55 +0,0 @@ -import type { Thunks } from "core/bootstrap"; -import { actions } from "./state"; -import { assert } from "tsafe/assert"; - -import type { ParamsOfCreateS3Client } from "core/adapters/s3Client"; - -export const thunks = {} satisfies Thunks; - -export const protectedThunks = { - testS3Connection: - (params: { - paramsOfCreateS3Client: ParamsOfCreateS3Client.NoSts; - workingDirectoryPath: string; - }) => - async (...args) => { - const { paramsOfCreateS3Client, workingDirectoryPath } = params; - - const [dispatch] = args; - - dispatch( - actions.testStarted({ paramsOfCreateS3Client, workingDirectoryPath }) - ); - - const result = await (async () => { - const { createS3Client } = await import("core/adapters/s3Client"); - - const getOidc = () => { - assert(false); - }; - - const s3Client = createS3Client(paramsOfCreateS3Client, getOidc); - - try { - await s3Client.listObjects({ - path: workingDirectoryPath - }); - } catch (error) { - return { - isSuccess: false as const, - errorMessage: String(error) - }; - } - - return { isSuccess: true as const }; - })(); - - dispatch( - actions.testCompleted({ - paramsOfCreateS3Client, - workingDirectoryPath, - result - }) - ); - } -} satisfies Thunks; diff --git a/web/src/core/usecases/s3ConfigCreation/selectors.ts b/web/src/core/usecases/s3ConfigCreation/selectors.ts deleted file mode 100644 index b9811873e..000000000 --- a/web/src/core/usecases/s3ConfigCreation/selectors.ts +++ /dev/null @@ -1,409 +0,0 @@ -import type { State as RootState } from "core/bootstrap"; -import { createSelector } from "clean-architecture"; -import { name } from "./state"; -import { objectKeys } from "tsafe/objectKeys"; -import { assert, type Equals } from "tsafe/assert"; -import { bucketNameAndObjectNameFromS3Path } from "core/adapters/s3Client/utils/bucketNameAndObjectNameFromS3Path"; -import { id } from "tsafe/id"; -import type { ProjectConfigs } from "core/usecases/projectManagement"; -import type { ParamsOfCreateS3Client } from "core/adapters/s3Client"; -import * as s3ConfigConnectionTest from "core/usecases/s3ConfigConnectionTest"; -import { same } from "evt/tools/inDepth/same"; -import { parseProjectS3ConfigId } from "core/usecases/s3ConfigManagement/decoupledLogic/projectS3ConfigId"; - -const readyState = (rootState: RootState) => { - const state = rootState[name]; - - if (state.stateDescription !== "ready") { - return null; - } - - return state; -}; - -const isReady = createSelector(readyState, state => state !== null); - -const formValues = createSelector(readyState, state => { - if (state === null) { - return null; - } - - return state.formValues; -}); - -const formValuesErrors = createSelector(formValues, formValues => { - if (formValues === null) { - return null; - } - - const out: Record< - keyof typeof formValues, - "must be an url" | "is required" | "not a valid access key id" | undefined - > = {} as any; - - for (const key of objectKeys(formValues)) { - out[key] = (() => { - required_fields: { - if ( - !( - key === "url" || - key === "workingDirectoryPath" || - key === "friendlyName" || - (!formValues.isAnonymous && - (key === "accessKeyId" || key === "secretAccessKey")) - ) - ) { - break required_fields; - } - - const value = formValues[key]; - - if ((value ?? "").trim() !== "") { - break required_fields; - } - - return "is required"; - } - - if (key === "url") { - const value = formValues[key]; - - try { - new URL(value.startsWith("http") ? value : `https://${value}`); - } catch { - return "must be an url"; - } - } - - return undefined; - })(); - } - - return out; -}); - -const isFormSubmittable = createSelector( - isReady, - formValuesErrors, - (isReady, formValuesErrors) => { - if (!isReady) { - return null; - } - - assert(formValuesErrors !== null); - - return objectKeys(formValuesErrors).every( - key => formValuesErrors[key] === undefined - ); - } -); - -const formattedFormValuesUrl = createSelector( - isReady, - formValues, - formValuesErrors, - (isReady, formValues, formValuesErrors) => { - if (!isReady) { - return null; - } - assert(formValues !== null); - assert(formValuesErrors !== null); - - if (formValuesErrors.url !== undefined) { - return undefined; - } - - const trimmedValue = formValues.url.trim(); - - return trimmedValue.startsWith("http") ? trimmedValue : `https://${trimmedValue}`; - } -); - -const formattedFormValuesWorkingDirectoryPath = createSelector( - isReady, - formValues, - formValuesErrors, - (isReady, formValues, formValuesErrors) => { - if (!isReady) { - return null; - } - assert(formValues !== null); - assert(formValuesErrors !== null); - - if (formValuesErrors.workingDirectoryPath !== undefined) { - return undefined; - } - - return ( - formValues.workingDirectoryPath - .trim() - .replace(/\/\//g, "/") // Remove double slashes if any - .replace(/^\//g, "") // Ensure no leading slash - .replace(/\/*$/g, "") + "/" - ); // Enforce trailing slash - } -); - -const action = createSelector(readyState, state => { - if (state === null) { - return null; - } - - return state.action; -}); - -const submittableFormValuesAsProjectS3Config = createSelector( - isReady, - formValues, - formattedFormValuesUrl, - formattedFormValuesWorkingDirectoryPath, - isFormSubmittable, - action, - ( - isReady, - formValues, - formattedFormValuesUrl, - formattedFormValuesWorkingDirectoryPath, - isFormSubmittable, - action - ) => { - if (!isReady) { - return null; - } - assert(formValues !== null); - assert(formattedFormValuesUrl !== null); - assert(formattedFormValuesWorkingDirectoryPath !== null); - assert(formattedFormValuesUrl !== null); - assert(formattedFormValuesWorkingDirectoryPath !== null); - assert(isFormSubmittable !== null); - assert(action !== null); - - if (!isFormSubmittable) { - return undefined; - } - - assert(formattedFormValuesUrl !== undefined); - assert(formattedFormValuesWorkingDirectoryPath !== undefined); - - return id({ - creationTime: (() => { - switch (action.type) { - case "create new config": - return action.creationTime; - case "update existing config": - return parseProjectS3ConfigId({ s3ConfigId: action.s3ConfigId }) - .creationTime; - } - assert>(false); - })(), - friendlyName: formValues.friendlyName.trim(), - url: formattedFormValuesUrl, - region: formValues.region?.trim(), - workingDirectoryPath: formattedFormValuesWorkingDirectoryPath, - pathStyleAccess: formValues.pathStyleAccess, - credentials: (() => { - if (formValues.isAnonymous) { - return undefined; - } - - assert(formValues.accessKeyId !== undefined); - assert(formValues.secretAccessKey !== undefined); - - return { - accessKeyId: formValues.accessKeyId, - secretAccessKey: formValues.secretAccessKey, - sessionToken: formValues.sessionToken - }; - })() - }); - } -); - -const paramsOfCreateS3Client = createSelector( - isReady, - submittableFormValuesAsProjectS3Config, - (isReady, submittableFormValuesAsProjectS3Config) => { - if (!isReady) { - return null; - } - - assert(submittableFormValuesAsProjectS3Config !== null); - - if (submittableFormValuesAsProjectS3Config === undefined) { - return undefined; - } - - return id({ - url: submittableFormValuesAsProjectS3Config.url, - pathStyleAccess: submittableFormValuesAsProjectS3Config.pathStyleAccess, - isStsEnabled: false, - region: submittableFormValuesAsProjectS3Config.region, - credentials: submittableFormValuesAsProjectS3Config.credentials - }); - } -); - -type ConnectionTestStatus = - | { status: "test ongoing" } - | { status: "test succeeded" } - | { status: "test failed"; errorMessage: string } - | { status: "not tested" }; - -const connectionTestStatus = createSelector( - isReady, - isFormSubmittable, - paramsOfCreateS3Client, - formattedFormValuesWorkingDirectoryPath, - s3ConfigConnectionTest.protectedSelectors.configTestResults, - s3ConfigConnectionTest.protectedSelectors.ongoingConfigTests, - ( - isReady, - isFormSubmittable, - paramsOfCreateS3Client, - workingDirectoryPath, - configTestResults, - ongoingConfigTests - ): ConnectionTestStatus | null => { - if (!isReady) { - return null; - } - - assert(isFormSubmittable !== null); - assert(paramsOfCreateS3Client !== null); - assert(workingDirectoryPath !== null); - - if (!isFormSubmittable) { - return { status: "not tested" }; - } - - assert(paramsOfCreateS3Client !== undefined); - assert(workingDirectoryPath !== undefined); - - if ( - ongoingConfigTests.find( - e => - same(e.paramsOfCreateS3Client, paramsOfCreateS3Client) && - e.workingDirectoryPath === workingDirectoryPath - ) !== undefined - ) { - return { status: "test ongoing" }; - } - - has_result: { - const { result } = - configTestResults.find( - e => - same(e.paramsOfCreateS3Client, paramsOfCreateS3Client) && - e.workingDirectoryPath === workingDirectoryPath - ) ?? {}; - - if (result === undefined) { - break has_result; - } - - return result.isSuccess - ? { status: "test succeeded" } - : { status: "test failed", errorMessage: result.errorMessage }; - } - - return { status: "not tested" } as ConnectionTestStatus; - } -); - -const urlStylesExamples = createSelector( - isReady, - formattedFormValuesUrl, - formattedFormValuesWorkingDirectoryPath, - (isReady, formattedFormValuesUrl, formattedFormValuesWorkingDirectoryPath) => { - if (!isReady) { - return null; - } - - assert(formattedFormValuesUrl !== null); - assert(formattedFormValuesWorkingDirectoryPath !== null); - - if ( - formattedFormValuesUrl === undefined || - formattedFormValuesWorkingDirectoryPath === undefined - ) { - return undefined; - } - - const urlObject = new URL(formattedFormValuesUrl); - - const { bucketName, objectName: objectNamePrefix } = - bucketNameAndObjectNameFromS3Path(formattedFormValuesWorkingDirectoryPath); - - const domain = formattedFormValuesUrl - .split(urlObject.protocol)[1] - .split("//")[1] - .replace(/\/$/, ""); - - return { - pathStyle: `${domain}/${bucketName}/${objectNamePrefix}`, - virtualHostedStyle: `${bucketName}.${domain}/${objectNamePrefix}` - }; - } -); - -const isEditionOfAnExistingConfig = createSelector(isReady, action, (isReady, action) => { - if (!isReady) { - return null; - } - - assert(action !== null); - - return action.type === "update existing config"; -}); - -const main = createSelector( - isReady, - formValues, - formValuesErrors, - isFormSubmittable, - urlStylesExamples, - isEditionOfAnExistingConfig, - connectionTestStatus, - ( - isReady, - formValues, - formValuesErrors, - isFormSubmittable, - urlStylesExamples, - isEditionOfAnExistingConfig, - connectionTestStatus - ) => { - if (!isReady) { - return { - isReady: false as const - }; - } - - assert(formValues !== null); - assert(formValuesErrors !== null); - assert(isFormSubmittable !== null); - assert(urlStylesExamples !== null); - assert(isEditionOfAnExistingConfig !== null); - assert(connectionTestStatus !== null); - - return { - isReady: true, - formValues, - formValuesErrors, - isFormSubmittable, - urlStylesExamples, - isEditionOfAnExistingConfig, - connectionTestStatus - }; - } -); - -export const privateSelectors = { - formattedFormValuesUrl, - submittableFormValuesAsProjectS3Config, - formValuesErrors, - paramsOfCreateS3Client, - formattedFormValuesWorkingDirectoryPath -}; - -export const selectors = { main }; diff --git a/web/src/core/usecases/s3ConfigCreation/thunks.ts b/web/src/core/usecases/s3ConfigCreation/thunks.ts deleted file mode 100644 index c55586c13..000000000 --- a/web/src/core/usecases/s3ConfigCreation/thunks.ts +++ /dev/null @@ -1,237 +0,0 @@ -import type { Thunks } from "core/bootstrap"; -import { actions, type State, type ChangeValueParams } from "./state"; -import { assert } from "tsafe/assert"; -import { privateSelectors } from "./selectors"; -import * as s3ConfigManagement from "core/usecases/s3ConfigManagement"; -import * as deploymentRegionManagement from "core/usecases/deploymentRegionManagement"; -import { getWorkingDirectoryPath } from "core/usecases/s3ConfigManagement/decoupledLogic/getWorkingDirectoryPath"; -import * as projectManagement from "core/usecases/projectManagement"; -import * as userAuthentication from "core/usecases/userAuthentication"; -import * as s3ConfigConnectionTest from "core/usecases/s3ConfigConnectionTest"; - -export const thunks = { - initialize: - (params: { s3ConfigIdToEdit: string | undefined }) => - async (...args) => { - const { s3ConfigIdToEdit } = params; - - const [dispatch, getState] = args; - - const s3Configs = s3ConfigManagement.selectors.s3Configs(getState()); - - update_existing_config: { - if (s3ConfigIdToEdit === undefined) { - break update_existing_config; - } - - const s3Config = s3Configs.find( - s3Config => s3Config.id === s3ConfigIdToEdit - ); - - assert(s3Config !== undefined); - assert(s3Config.origin === "project"); - - dispatch( - actions.initialized({ - s3ConfigIdToEdit, - initialFormValues: { - friendlyName: s3Config.friendlyName, - url: s3Config.paramsOfCreateS3Client.url, - region: s3Config.region, - workingDirectoryPath: s3Config.workingDirectoryPath, - pathStyleAccess: - s3Config.paramsOfCreateS3Client.pathStyleAccess, - ...(() => { - if ( - s3Config.paramsOfCreateS3Client.credentials === - undefined - ) { - return { - isAnonymous: true, - accessKeyId: undefined, - secretAccessKey: undefined, - sessionToken: undefined - }; - } - - return { - isAnonymous: false, - accessKeyId: - s3Config.paramsOfCreateS3Client.credentials - .accessKeyId, - secretAccessKey: - s3Config.paramsOfCreateS3Client.credentials - .secretAccessKey, - sessionToken: - s3Config.paramsOfCreateS3Client.credentials - .sessionToken - }; - })() - } - }) - ); - - return; - } - - const { s3ConfigCreationFormDefaults } = - deploymentRegionManagement.selectors.currentDeploymentRegion(getState()); - - if (s3ConfigCreationFormDefaults === undefined) { - dispatch( - actions.initialized({ - s3ConfigIdToEdit: undefined, - initialFormValues: { - friendlyName: "", - url: "", - region: undefined, - workingDirectoryPath: "", - pathStyleAccess: false, - isAnonymous: true, - accessKeyId: undefined, - secretAccessKey: undefined, - sessionToken: undefined - } - }) - ); - return; - } - - const workingDirectoryPath = - s3ConfigCreationFormDefaults.workingDirectory === undefined - ? undefined - : getWorkingDirectoryPath({ - context: (() => { - const project = - projectManagement.protectedSelectors.currentProject( - getState() - ); - const { isUserLoggedIn, user } = - userAuthentication.selectors.main(getState()); - - assert(isUserLoggedIn); - - return project.group === undefined - ? { - type: "personalProject" as const, - username: user.username - } - : { - type: "groupProject" as const, - projectGroup: project.group - }; - })(), - workingDirectory: s3ConfigCreationFormDefaults.workingDirectory - }); - - dispatch( - actions.initialized({ - s3ConfigIdToEdit: undefined, - initialFormValues: { - friendlyName: "", - url: s3ConfigCreationFormDefaults.url, - region: s3ConfigCreationFormDefaults.region, - workingDirectoryPath: workingDirectoryPath ?? "", - pathStyleAccess: - s3ConfigCreationFormDefaults.pathStyleAccess ?? false, - isAnonymous: false, - accessKeyId: undefined, - secretAccessKey: undefined, - sessionToken: undefined - } - }) - ); - }, - reset: - () => - (...args) => { - const [dispatch] = args; - - dispatch(actions.stateResetToNotInitialized()); - }, - submit: - () => - async (...args) => { - const [dispatch, getState] = args; - - const projectS3Config = - privateSelectors.submittableFormValuesAsProjectS3Config(getState()); - - assert(projectS3Config !== null); - assert(projectS3Config !== undefined); - - await dispatch( - s3ConfigManagement.protectedThunks.createS3Config({ - projectS3Config - }) - ); - - dispatch(actions.stateResetToNotInitialized()); - }, - changeValue: - (params: ChangeValueParams) => - async (...args) => { - const { key, value } = params; - - const [dispatch, getState] = args; - dispatch(actions.formValueChanged({ key, value })); - - preset_pathStyleAccess: { - if (key !== "url") { - break preset_pathStyleAccess; - } - - const url = privateSelectors.formattedFormValuesUrl(getState()); - - assert(url !== null); - - if (url === undefined) { - break preset_pathStyleAccess; - } - - if (url.toLowerCase().includes("amazonaws.com")) { - dispatch( - actions.formValueChanged({ - key: "pathStyleAccess", - value: false - }) - ); - break preset_pathStyleAccess; - } - - if (url.toLocaleLowerCase().includes("minio")) { - dispatch( - actions.formValueChanged({ - key: "pathStyleAccess", - value: true - }) - ); - break preset_pathStyleAccess; - } - } - }, - testConnection: - () => - async (...args) => { - const [dispatch, getState] = args; - - const projectS3Config = - privateSelectors.submittableFormValuesAsProjectS3Config(getState()); - - assert(projectS3Config !== null); - assert(projectS3Config !== undefined); - - await dispatch( - s3ConfigConnectionTest.protectedThunks.testS3Connection({ - paramsOfCreateS3Client: { - isStsEnabled: false, - url: projectS3Config.url, - pathStyleAccess: projectS3Config.pathStyleAccess, - region: projectS3Config.region, - credentials: projectS3Config.credentials - }, - workingDirectoryPath: projectS3Config.workingDirectoryPath - }) - ); - } -} satisfies Thunks; diff --git a/web/src/core/usecases/s3ConfigManagement/decoupledLogic/getS3Configs.ts b/web/src/core/usecases/s3ConfigManagement/decoupledLogic/getS3Configs.ts deleted file mode 100644 index e737e9b3b..000000000 --- a/web/src/core/usecases/s3ConfigManagement/decoupledLogic/getS3Configs.ts +++ /dev/null @@ -1,340 +0,0 @@ -import * as projectManagement from "core/usecases/projectManagement"; -import type { DeploymentRegion } from "core/ports/OnyxiaApi/DeploymentRegion"; -import { bucketNameAndObjectNameFromS3Path } from "core/adapters/s3Client/utils/bucketNameAndObjectNameFromS3Path"; -import type { ParamsOfCreateS3Client } from "core/adapters/s3Client"; -import { same } from "evt/tools/inDepth/same"; -import { getWorkingDirectoryPath } from "./getWorkingDirectoryPath"; -import { getWorkingDirectoryBucketToCreate } from "./getWorkingDirectoryBucket"; -import { fnv1aHashToHex } from "core/tools/fnv1aHashToHex"; -import { assert, type Equals } from "tsafe/assert"; -import { getProjectS3ConfigId } from "./projectS3ConfigId"; -import type * as s3ConfigConnectionTest from "core/usecases/s3ConfigConnectionTest"; -import type { LocalizedString } from "core/ports/OnyxiaApi"; -import type { ResolvedAdminBookmark } from "./resolveS3AdminBookmarks"; - -export type S3Config = S3Config.FromDeploymentRegion | S3Config.FromProject; - -export namespace S3Config { - type Common = { - id: string; - dataSource: string; - region: string | undefined; - workingDirectoryPath: string; - isXOnyxiaDefault: boolean; - isExplorerConfig: boolean; - }; - - export type FromDeploymentRegion = Common & { - origin: "deploymentRegion"; - paramsOfCreateS3Client: ParamsOfCreateS3Client; - locations: FromDeploymentRegion.Location[]; - }; - - export namespace FromDeploymentRegion { - export type Location = - | Location.Personal - | Location.Project - | Location.AdminBookmark; - - export namespace Location { - type Common = { directoryPath: string }; - - export type Personal = Common & { - type: "personal"; - }; - - export type Project = Common & { - type: "project"; - projectName: string; - }; - export type AdminBookmark = Common & { - type: "bookmark"; - title: LocalizedString; - description?: LocalizedString; - tags: LocalizedString[] | undefined; - }; - } - } - - export type FromProject = Common & { - origin: "project"; - paramsOfCreateS3Client: ParamsOfCreateS3Client.NoSts; - creationTime: number; - friendlyName: string; - connectionTestStatus: - | { status: "not tested" } - | { status: "test ongoing" } - | { status: "test failed"; errorMessage: string } - | { status: "test succeeded" }; - }; -} - -export function getS3Configs(params: { - projectConfigsS3: projectManagement.ProjectConfigs["s3"]; - s3RegionConfigs: DeploymentRegion.S3Config[]; - resolvedAdminBookmarks: ResolvedAdminBookmark[]; - configTestResults: s3ConfigConnectionTest.ConfigTestResult[]; - ongoingConfigTests: s3ConfigConnectionTest.OngoingConfigTest[]; - username: string; - projectGroup: string | undefined; - groupProjects: { - name: string; - group: string; - }[]; -}): S3Config[] { - const { - projectConfigsS3: { - s3Configs: s3ProjectConfigs, - s3ConfigId_defaultXOnyxia, - s3ConfigId_explorer - }, - s3RegionConfigs, - resolvedAdminBookmarks, - configTestResults, - ongoingConfigTests, - username, - projectGroup, - groupProjects - } = params; - - const getDataSource = (params: { - url: string; - pathStyleAccess: boolean; - workingDirectoryPath: string; - }): string => { - const { url, pathStyleAccess, workingDirectoryPath } = params; - - let out = url; - - out = out.replace(/^https?:\/\//, "").replace(/\/$/, ""); - - const { bucketName, objectName } = - bucketNameAndObjectNameFromS3Path(workingDirectoryPath); - - out = pathStyleAccess - ? `${out}/${bucketName}/${objectName}` - : `${bucketName}.${out}/${objectName}`; - - return out; - }; - - const getConnectionTestStatus = (params: { - workingDirectoryPath: string; - paramsOfCreateS3Client: ParamsOfCreateS3Client; - }): S3Config.FromProject["connectionTestStatus"] => { - const { workingDirectoryPath, paramsOfCreateS3Client } = params; - - if ( - ongoingConfigTests.find( - e => - same(e.paramsOfCreateS3Client, paramsOfCreateS3Client) && - e.workingDirectoryPath === workingDirectoryPath - ) !== undefined - ) { - return { status: "test ongoing" }; - } - - has_result: { - const { result } = - configTestResults.find( - e => - same(e.paramsOfCreateS3Client, paramsOfCreateS3Client) && - e.workingDirectoryPath === workingDirectoryPath - ) ?? {}; - - if (result === undefined) { - break has_result; - } - - return result.isSuccess - ? { status: "test succeeded" } - : { status: "test failed", errorMessage: result.errorMessage }; - } - - return { status: "not tested" }; - }; - - const s3Configs: S3Config[] = [ - ...s3ProjectConfigs - .map((c): S3Config.FromProject => { - const id = getProjectS3ConfigId({ - creationTime: c.creationTime - }); - - const workingDirectoryPath = c.workingDirectoryPath; - const url = c.url; - const pathStyleAccess = c.pathStyleAccess; - const region = c.region; - - const paramsOfCreateS3Client: ParamsOfCreateS3Client.NoSts = { - url, - pathStyleAccess, - isStsEnabled: false, - region, - credentials: c.credentials - }; - - return { - origin: "project", - creationTime: c.creationTime, - friendlyName: c.friendlyName, - id, - dataSource: getDataSource({ - url, - pathStyleAccess, - workingDirectoryPath - }), - region, - workingDirectoryPath, - paramsOfCreateS3Client, - isXOnyxiaDefault: false, - isExplorerConfig: false, - connectionTestStatus: getConnectionTestStatus({ - paramsOfCreateS3Client, - workingDirectoryPath - }) - }; - }) - .sort((a, b) => b.creationTime - a.creationTime), - ...s3RegionConfigs.map((c, i): S3Config.FromDeploymentRegion => { - const id = `region-${fnv1aHashToHex( - JSON.stringify( - Object.fromEntries( - Object.entries(c).sort(([key1], [key2]) => - key1.localeCompare(key2) - ) - ) - ) - )}`; - - const workingDirectoryContext = - projectGroup === undefined - ? { - type: "personalProject" as const, - username - } - : { - type: "groupProject" as const, - projectGroup - }; - - const workingDirectoryPath = getWorkingDirectoryPath({ - workingDirectory: c.workingDirectory, - context: workingDirectoryContext - }); - - const personalWorkingDirectoryPath = getWorkingDirectoryPath({ - workingDirectory: c.workingDirectory, - context: { - type: "personalProject" as const, - username - } - }); - - const url = c.url; - const pathStyleAccess = c.pathStyleAccess; - const region = c.region; - - const paramsOfCreateS3Client: ParamsOfCreateS3Client.Sts = { - url, - pathStyleAccess, - isStsEnabled: true, - stsUrl: c.sts.url, - region, - oidcParams: c.sts.oidcParams, - durationSeconds: c.sts.durationSeconds, - role: c.sts.role, - nameOfBucketToCreateIfNotExist: getWorkingDirectoryBucketToCreate({ - workingDirectory: c.workingDirectory, - context: workingDirectoryContext - }) - }; - - const adminBookmarks: S3Config.FromDeploymentRegion.Location.AdminBookmark[] = - (() => { - const entry = resolvedAdminBookmarks.find( - ({ s3ConfigIndex }) => s3ConfigIndex === i - ); - - if (entry === undefined) { - return []; - } - - return entry.bookmarkedDirectories.map( - ({ title, description, fullPath, tags }) => ({ - title, - description, - type: "bookmark", - directoryPath: fullPath, - tags - }) - ); - })(); - - const projectsLocations: S3Config.FromDeploymentRegion.Location.Project[] = - groupProjects.map(({ group }) => { - const directoryPath = getWorkingDirectoryPath({ - workingDirectory: c.workingDirectory, - context: { - type: "groupProject", - projectGroup: group - } - }); - return { type: "project", directoryPath, projectName: group }; - }); - - const dataSource = getDataSource({ - url, - pathStyleAccess, - workingDirectoryPath - }); - - return { - origin: "deploymentRegion", - id, - dataSource, - region, - workingDirectoryPath, - locations: [ - { type: "personal", directoryPath: personalWorkingDirectoryPath }, - ...projectsLocations, - ...adminBookmarks - ], - paramsOfCreateS3Client, - isXOnyxiaDefault: false, - isExplorerConfig: false - }; - }) - ]; - - ( - [ - ["defaultXOnyxia", s3ConfigId_defaultXOnyxia], - ["explorer", s3ConfigId_explorer] - ] as const - ).forEach(([prop, s3ConfigId]) => { - if (s3ConfigId === undefined) { - return; - } - - const s3Config = - s3Configs.find(({ id }) => id === s3ConfigId) ?? - s3Configs.find(s3Config => s3Config.origin === "deploymentRegion"); - - if (s3Config === undefined) { - return; - } - - switch (prop) { - case "defaultXOnyxia": - s3Config.isXOnyxiaDefault = true; - return; - case "explorer": - s3Config.isExplorerConfig = true; - return; - } - assert>(false); - }); - - return s3Configs; -} diff --git a/web/src/core/usecases/s3ConfigManagement/decoupledLogic/getWorkingDirectoryBucket.ts b/web/src/core/usecases/s3ConfigManagement/decoupledLogic/getWorkingDirectoryBucket.ts deleted file mode 100644 index b776713dc..000000000 --- a/web/src/core/usecases/s3ConfigManagement/decoupledLogic/getWorkingDirectoryBucket.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { assert, type Equals } from "tsafe/assert"; -import type { DeploymentRegion } from "core/ports/OnyxiaApi"; - -export function getWorkingDirectoryBucketToCreate(params: { - workingDirectory: DeploymentRegion.S3Config["workingDirectory"]; - context: - | { - type: "personalProject"; - username: string; - } - | { - type: "groupProject"; - projectGroup: string; - }; -}): string | undefined { - const { workingDirectory, context } = params; - - switch (workingDirectory.bucketMode) { - case "shared": - return undefined; - case "multi": - return (() => { - switch (context.type) { - case "personalProject": - return `${workingDirectory.bucketNamePrefix}${context.username}`; - case "groupProject": - return `${workingDirectory.bucketNamePrefixGroup}${context.projectGroup}`; - } - assert>(false); - })(); - } - assert>(false); -} diff --git a/web/src/core/usecases/s3ConfigManagement/decoupledLogic/getWorkingDirectoryPath.ts b/web/src/core/usecases/s3ConfigManagement/decoupledLogic/getWorkingDirectoryPath.ts deleted file mode 100644 index 8c310703c..000000000 --- a/web/src/core/usecases/s3ConfigManagement/decoupledLogic/getWorkingDirectoryPath.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { assert, type Equals } from "tsafe/assert"; -import type { DeploymentRegion } from "core/ports/OnyxiaApi"; -import { getWorkingDirectoryBucketToCreate } from "./getWorkingDirectoryBucket"; - -export function getWorkingDirectoryPath(params: { - workingDirectory: DeploymentRegion.S3Config["workingDirectory"]; - context: - | { - type: "personalProject"; - username: string; - } - | { - type: "groupProject"; - projectGroup: string; - }; -}): string { - const { workingDirectory, context } = params; - - return ( - (() => { - switch (workingDirectory.bucketMode) { - case "multi": { - const bucketName = getWorkingDirectoryBucketToCreate({ - workingDirectory, - context - }); - assert(bucketName !== undefined); - return bucketName; - } - case "shared": - return [ - workingDirectory.bucketName, - (() => { - switch (context.type) { - case "personalProject": - return `${workingDirectory.prefix}${context.username}`; - case "groupProject": - return `${workingDirectory.prefixGroup}${context.projectGroup}`; - } - assert>(true); - })() - ].join("/"); - } - assert>(false); - })() - .trim() - .replace(/\/\//g, "/") // Remove double slashes if any - .replace(/^\//g, "") // Ensure no leading slash - .replace(/\/+$/g, "") + "/" // Enforce trailing slash - ); -} diff --git a/web/src/core/usecases/s3ConfigManagement/decoupledLogic/projectS3ConfigId.ts b/web/src/core/usecases/s3ConfigManagement/decoupledLogic/projectS3ConfigId.ts deleted file mode 100644 index 4ed797345..000000000 --- a/web/src/core/usecases/s3ConfigManagement/decoupledLogic/projectS3ConfigId.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { assert } from "tsafe/assert"; - -const prefix = "project-"; - -export function getProjectS3ConfigId(params: { creationTime: number }): string { - const { creationTime } = params; - - return `${prefix}${creationTime}`; -} - -export function parseProjectS3ConfigId(params: { s3ConfigId: string }): { - creationTime: number; -} { - const { s3ConfigId } = params; - - const creationTimeStr = s3ConfigId.replace(prefix, ""); - - const creationTime = parseInt(creationTimeStr); - - assert(!isNaN(creationTime), "Not a valid s3 project config id"); - - return { creationTime }; -} diff --git a/web/src/core/usecases/s3ConfigManagement/decoupledLogic/resolveS3AdminBookmarks.ts b/web/src/core/usecases/s3ConfigManagement/decoupledLogic/resolveS3AdminBookmarks.ts deleted file mode 100644 index 0de91ffa6..000000000 --- a/web/src/core/usecases/s3ConfigManagement/decoupledLogic/resolveS3AdminBookmarks.ts +++ /dev/null @@ -1,190 +0,0 @@ -import type { - DeploymentRegion, - LocalizedString, - OidcParams_Partial -} from "core/ports/OnyxiaApi"; -import { assert } from "tsafe/assert"; -import { id } from "tsafe/id"; -import memoizee from "memoizee"; - -export type DeploymentRegion_S3ConfigLike = { - sts: { - oidcParams: OidcParams_Partial; - }; - bookmarkedDirectories: DeploymentRegion.S3Config.BookmarkedDirectory[]; -}; - -assert; - -export type ResolvedAdminBookmark = { - s3ConfigIndex: number; - bookmarkedDirectories: DeploymentRegion.S3Config.BookmarkedDirectory.Common[]; -}; - -export async function resolveS3AdminBookmarks(params: { - deploymentRegion_s3Configs: DeploymentRegion_S3ConfigLike[]; - getDecodedIdToken: (params: { - oidcParams_partial: OidcParams_Partial; - }) => Promise>; -}): Promise<{ - resolvedAdminBookmarks: ResolvedAdminBookmark[]; -}> { - const { deploymentRegion_s3Configs, getDecodedIdToken } = params; - - const resolvedAdminBookmarks = await Promise.all( - deploymentRegion_s3Configs.map(async (s3Config, i) => { - const getDecodedIdToken_memo = memoizee( - () => - getDecodedIdToken({ - oidcParams_partial: s3Config.sts.oidcParams - }), - { promise: true } - ); - - return id({ - s3ConfigIndex: i, - bookmarkedDirectories: ( - await Promise.all( - s3Config.bookmarkedDirectories.map(async entry => { - if (entry.claimName === undefined) { - return [ - id( - { - fullPath: entry.fullPath, - description: entry.description, - tags: entry.tags, - title: entry.title - } - ) - ]; - } - - const { - claimName, - excludedClaimPattern, - includedClaimPattern - } = entry; - - const decodedIdToken = await getDecodedIdToken_memo(); - - const claimValue_arr: string[] = (() => { - const value = decodedIdToken[claimName]; - - if (!value) return []; - - if (typeof value === "string") return [value]; - if (Array.isArray(value)) return value.map(e => `${e}`); - - assert( - false, - () => - `${claimName} not in expected format! ${JSON.stringify(decodedIdToken)}` - ); - })(); - - const includedRegex = includedClaimPattern - ? new RegExp(includedClaimPattern) - : undefined; - const excludedRegex = excludedClaimPattern - ? new RegExp(excludedClaimPattern) - : undefined; - - return claimValue_arr - .map(value => { - if ( - excludedRegex !== undefined && - excludedRegex.test(value) - ) - return []; - - if (includedRegex === undefined) { - return []; - } - - const match = includedRegex.exec(value); - - if (!match) { - return []; - } - - return [ - id( - { - fullPath: substituteTemplateString({ - template: entry.fullPath, - match - }), - title: substituteLocalizedString({ - localizedString: entry.title, - match - }), - description: substituteLocalizedString({ - localizedString: entry.description, - match - }), - tags: substituteLocalizedStringArray({ - array: entry.tags, - match - }) - } - ) - ]; - }) - .flat(); - }) - ) - ).flat() - }); - }) - ); - - return { resolvedAdminBookmarks }; -} - -function substituteTemplateString(params: { - template: string; - match: RegExpExecArray; -}): string { - const { template, match } = params; - return template.replace(/\$(\d+)/g, (_, i) => match[parseInt(i)] ?? ""); -} - -const substituteLocalizedStringArray = (params: { - array: LocalizedString[] | undefined; - match: RegExpExecArray; -}): LocalizedString[] | undefined => { - const { array, match } = params; - - if (array === undefined) return undefined; - - return array.map(str => - substituteLocalizedString({ - localizedString: str, - match - }) - ); -}; - -function substituteLocalizedString(params: { - localizedString: T; - match: RegExpExecArray; -}): T { - const { localizedString: input, match } = params; - - if (input === undefined) return undefined as T; - - if (typeof input === "string") { - return substituteTemplateString({ template: input, match }) as T; - } - - const result = Object.fromEntries( - Object.entries(input).map(([lang, value]) => [ - lang, - typeof value === "string" - ? substituteTemplateString({ template: value, match }) - : value - ]) - ); - - return result as T; -} diff --git a/web/src/core/usecases/s3ConfigManagement/decoupledLogic/updateDefaultS3ConfigsAfterPotentialDeletion.ts b/web/src/core/usecases/s3ConfigManagement/decoupledLogic/updateDefaultS3ConfigsAfterPotentialDeletion.ts deleted file mode 100644 index 7113da304..000000000 --- a/web/src/core/usecases/s3ConfigManagement/decoupledLogic/updateDefaultS3ConfigsAfterPotentialDeletion.ts +++ /dev/null @@ -1,71 +0,0 @@ -import * as projectManagement from "core/usecases/projectManagement"; -import type { DeploymentRegion } from "core/ports/OnyxiaApi/DeploymentRegion"; -import { getS3Configs } from "./getS3Configs"; - -type R = Record< - "s3ConfigId_defaultXOnyxia" | "s3ConfigId_explorer", - | { - isUpdateNeeded: false; - } - | { - isUpdateNeeded: true; - s3ConfigId: string | undefined; - } ->; - -export function updateDefaultS3ConfigsAfterPotentialDeletion(params: { - projectConfigsS3: { - s3Configs: projectManagement.ProjectConfigs.S3Config[]; - s3ConfigId_defaultXOnyxia: string | undefined; - s3ConfigId_explorer: string | undefined; - }; - s3RegionConfigs: DeploymentRegion.S3Config[]; -}): R { - const { projectConfigsS3, s3RegionConfigs } = params; - - const s3Configs = getS3Configs({ - projectConfigsS3, - s3RegionConfigs, - configTestResults: [], - resolvedAdminBookmarks: [], - ongoingConfigTests: [], - username: "johndoe", - projectGroup: undefined, - groupProjects: [] - }); - - const actions: R = { - s3ConfigId_defaultXOnyxia: { - isUpdateNeeded: false - }, - s3ConfigId_explorer: { - isUpdateNeeded: false - } - }; - - for (const propertyName of [ - "s3ConfigId_defaultXOnyxia", - "s3ConfigId_explorer" - ] as const) { - const s3ConfigId_default = projectConfigsS3[propertyName]; - - if (s3ConfigId_default === undefined) { - continue; - } - - if (s3Configs.find(({ id }) => id === s3ConfigId_default) !== undefined) { - continue; - } - - const s3ConfigId_toUseAsDefault = s3Configs.find( - ({ origin }) => origin === "deploymentRegion" - )?.id; - - actions[propertyName] = { - isUpdateNeeded: true, - s3ConfigId: s3ConfigId_toUseAsDefault - }; - } - - return actions; -} diff --git a/web/src/core/usecases/s3ConfigManagement/index.ts b/web/src/core/usecases/s3ConfigManagement/index.ts deleted file mode 100644 index 479cc3f02..000000000 --- a/web/src/core/usecases/s3ConfigManagement/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from "./state"; -export * from "./selectors"; -export * from "./thunks"; -export type { S3Config } from "./decoupledLogic/getS3Configs"; diff --git a/web/src/core/usecases/s3ConfigManagement/selectors.ts b/web/src/core/usecases/s3ConfigManagement/selectors.ts deleted file mode 100644 index 4586886a7..000000000 --- a/web/src/core/usecases/s3ConfigManagement/selectors.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { createSelector } from "clean-architecture"; -import type { LocalizedString } from "core/ports/OnyxiaApi"; -import * as deploymentRegionManagement from "core/usecases/deploymentRegionManagement"; -import * as projectManagement from "core/usecases/projectManagement"; -import * as s3ConfigConnectionTest from "core/usecases/s3ConfigConnectionTest"; -import * as userAuthentication from "core/usecases/userAuthentication"; -import { assert } from "tsafe/assert"; -import { exclude } from "tsafe/exclude"; -import { getS3Configs, type S3Config } from "./decoupledLogic/getS3Configs"; -import { name } from "./state"; -import type { State as RootState } from "core/bootstrap"; - -const resolvedAdminBookmarks = createSelector( - (state: RootState) => state[name], - state => state.resolvedAdminBookmarks -); - -const s3Configs = createSelector( - createSelector( - projectManagement.protectedSelectors.projectConfig, - projectConfig => projectConfig.s3 - ), - createSelector( - deploymentRegionManagement.selectors.currentDeploymentRegion, - deploymentRegion => deploymentRegion.s3Configs - ), - resolvedAdminBookmarks, - s3ConfigConnectionTest.protectedSelectors.configTestResults, - s3ConfigConnectionTest.protectedSelectors.ongoingConfigTests, - createSelector(userAuthentication.selectors.main, ({ isUserLoggedIn, user }) => { - assert(isUserLoggedIn); - return user.username; - }), - createSelector( - projectManagement.protectedSelectors.currentProject, - project => project.group - ), - createSelector(projectManagement.protectedSelectors.projects, projects => { - return projects - .map(({ name, group }) => (group === undefined ? undefined : { name, group })) - .filter(exclude(undefined)); - }), - ( - projectConfigsS3, - s3RegionConfigs, - resolvedAdminBookmarks, - configTestResults, - ongoingConfigTests, - username, - projectGroup, - groupProjects - ): S3Config[] => - getS3Configs({ - projectConfigsS3, - s3RegionConfigs, - resolvedAdminBookmarks, - configTestResults, - ongoingConfigTests, - username, - projectGroup, - groupProjects - }) -); - -type IndexedS3Locations = - | IndexedS3Locations.AdminCreatedS3Config - | IndexedS3Locations.UserCreatedS3Config; - -namespace IndexedS3Locations { - export namespace AdminCreatedS3Config { - type Common = { directoryPath: string }; - - export type PersonalLocation = Common & { - type: "personal"; - }; - - export type ProjectLocation = Common & { - type: "project"; - projectName: string; - }; - export type AdminBookmarkLocation = Common & { - type: "bookmark"; - title: LocalizedString; - description?: LocalizedString; - tags: LocalizedString[] | undefined; - }; - - export type Location = PersonalLocation | ProjectLocation | AdminBookmarkLocation; - } - - export type AdminCreatedS3Config = { - type: "admin created s3 config"; - locations: AdminCreatedS3Config.Location[]; - }; - - export type UserCreatedS3Config = { - type: "user created s3 config"; - directoryPath: string; - dataSource: string; - }; -} - -const indexedS3Locations = createSelector(s3Configs, (s3Configs): IndexedS3Locations => { - const s3Config = s3Configs.find(({ isExplorerConfig }) => isExplorerConfig); - - assert(s3Config !== undefined); - - switch (s3Config.origin) { - case "deploymentRegion": - return { - type: "admin created s3 config", - locations: s3Config.locations - }; - case "project": - return { - type: "user created s3 config", - directoryPath: s3Config.workingDirectoryPath, - dataSource: s3Config.dataSource - }; - } -}); - -export const selectors = { s3Configs, indexedS3Locations }; - -export const privateSelectors = { - resolvedAdminBookmarks -}; diff --git a/web/src/core/usecases/s3ConfigManagement/state.ts b/web/src/core/usecases/s3ConfigManagement/state.ts deleted file mode 100644 index e97d2c9d1..000000000 --- a/web/src/core/usecases/s3ConfigManagement/state.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { ResolvedAdminBookmark } from "./decoupledLogic/resolveS3AdminBookmarks"; -import { - createUsecaseActions, - createObjectThatThrowsIfAccessed -} from "clean-architecture"; - -type State = { - resolvedAdminBookmarks: ResolvedAdminBookmark[]; -}; - -export const name = "s3ConfigManagement"; - -export const { reducer, actions } = createUsecaseActions({ - name, - initialState: createObjectThatThrowsIfAccessed(), - reducers: { - initialized: ( - _, - { - payload - }: { - payload: { - resolvedAdminBookmarks: ResolvedAdminBookmark[]; - }; - } - ) => { - const { resolvedAdminBookmarks } = payload; - - const state: State = { - resolvedAdminBookmarks - }; - - return state; - } - } -}); diff --git a/web/src/core/usecases/s3ConfigManagement/thunks.ts b/web/src/core/usecases/s3ConfigManagement/thunks.ts deleted file mode 100644 index 88cf38457..000000000 --- a/web/src/core/usecases/s3ConfigManagement/thunks.ts +++ /dev/null @@ -1,356 +0,0 @@ -import type { Thunks } from "core/bootstrap"; -import { selectors, privateSelectors } from "./selectors"; -import type { S3Config } from "./decoupledLogic/getS3Configs"; -import * as projectManagement from "core/usecases/projectManagement"; -import type { ProjectConfigs } from "core/usecases/projectManagement"; -import { assert } from "tsafe/assert"; -import type { S3Client } from "core/ports/S3Client"; -import { createUsecaseContextApi } from "clean-architecture"; -import { getProjectS3ConfigId } from "./decoupledLogic/projectS3ConfigId"; -import * as s3ConfigConnectionTest from "core/usecases/s3ConfigConnectionTest"; -import { updateDefaultS3ConfigsAfterPotentialDeletion } from "./decoupledLogic/updateDefaultS3ConfigsAfterPotentialDeletion"; -import structuredClone from "@ungap/structured-clone"; -import * as deploymentRegionManagement from "core/usecases/deploymentRegionManagement"; -import { fnv1aHashToHex } from "core/tools/fnv1aHashToHex"; -import { resolveS3AdminBookmarks } from "./decoupledLogic/resolveS3AdminBookmarks"; -import { actions } from "./state"; - -export const thunks = { - testS3Connection: - (params: { projectS3ConfigId: string }) => - async (...args) => { - const { projectS3ConfigId } = params; - const [dispatch, getState] = args; - - const s3Configs = selectors.s3Configs(getState()); - - const s3Config = s3Configs.find( - s3Config => s3Config.id === projectS3ConfigId - ); - - assert(s3Config !== undefined); - assert(s3Config.origin === "project"); - - await dispatch( - s3ConfigConnectionTest.protectedThunks.testS3Connection({ - paramsOfCreateS3Client: s3Config.paramsOfCreateS3Client, - workingDirectoryPath: s3Config.workingDirectoryPath - }) - ); - }, - deleteS3Config: - (params: { projectS3ConfigId: string }) => - async (...args) => { - const { projectS3ConfigId } = params; - - const [dispatch, getState] = args; - - const projectConfigsS3 = structuredClone( - projectManagement.protectedSelectors.projectConfig(getState()).s3 - ); - - const i = projectConfigsS3.s3Configs.findIndex( - projectS3Config_i => - getProjectS3ConfigId({ - creationTime: projectS3Config_i.creationTime - }) === projectS3ConfigId - ); - - assert(i !== -1); - - projectConfigsS3.s3Configs.splice(i, 1); - - { - const actions = updateDefaultS3ConfigsAfterPotentialDeletion({ - projectConfigsS3, - s3RegionConfigs: - deploymentRegionManagement.selectors.currentDeploymentRegion( - getState() - ).s3Configs - }); - - await Promise.all( - (["s3ConfigId_defaultXOnyxia", "s3ConfigId_explorer"] as const).map( - async propertyName => { - const action = actions[propertyName]; - - if (!action.isUpdateNeeded) { - return; - } - - projectConfigsS3[propertyName] = action.s3ConfigId; - } - ) - ); - } - - await dispatch( - projectManagement.protectedThunks.updateConfigValue({ - key: "s3", - value: projectConfigsS3 - }) - ); - }, - changeIsDefault: - (params: { - s3ConfigId: string; - usecase: "defaultXOnyxia" | "explorer"; - value: boolean; - }) => - async (...args) => { - const { s3ConfigId, usecase, value } = params; - - const [dispatch, getState] = args; - - const projectConfigsS3 = structuredClone( - projectManagement.protectedSelectors.projectConfig(getState()).s3 - ); - - const propertyName = (() => { - switch (usecase) { - case "defaultXOnyxia": - return "s3ConfigId_defaultXOnyxia"; - case "explorer": - return "s3ConfigId_explorer"; - } - })(); - - { - const currentDefault = projectConfigsS3[propertyName]; - - if (value) { - if (currentDefault === s3ConfigId) { - return; - } - } else { - if (currentDefault !== s3ConfigId) { - return; - } - } - } - - projectConfigsS3[propertyName] = value ? s3ConfigId : undefined; - - await dispatch( - projectManagement.protectedThunks.updateConfigValue({ - key: "s3", - value: projectConfigsS3 - }) - ); - } -} satisfies Thunks; - -export const protectedThunks = { - getS3ClientForSpecificConfig: - (params: { s3ConfigId: string | undefined }) => - async (...args): Promise => { - const { s3ConfigId } = params; - const [, getState, rootContext] = args; - - const { prS3ClientByConfigId } = getContext(rootContext); - - const s3Config = (() => { - const s3Configs = selectors.s3Configs(getState()); - - const s3Config = s3Configs.find(s3Config => s3Config.id === s3ConfigId); - assert(s3Config !== undefined); - - return s3Config; - })(); - - use_cached_s3Client: { - const prS3Client = prS3ClientByConfigId.get(s3Config.id); - - if (prS3Client === undefined) { - break use_cached_s3Client; - } - - return prS3Client; - } - - const prS3Client = (async () => { - const { createS3Client } = await import("core/adapters/s3Client"); - const { createOidc, mergeOidcParams } = await import( - "core/adapters/oidc" - ); - const { paramsOfBootstrapCore, onyxiaApi } = rootContext; - - return createS3Client( - s3Config.paramsOfCreateS3Client, - async oidcParams_partial => { - const { oidcParams } = - await onyxiaApi.getAvailableRegionsAndOidcParams(); - - assert(oidcParams !== undefined); - - const oidc_s3 = await createOidc({ - ...mergeOidcParams({ - oidcParams, - oidcParams_partial - }), - autoLogin: true, - transformBeforeRedirectForKeycloakTheme: - paramsOfBootstrapCore.transformBeforeRedirectForKeycloakTheme, - getCurrentLang: paramsOfBootstrapCore.getCurrentLang, - enableDebugLogs: paramsOfBootstrapCore.enableOidcDebugLogs - }); - - const doClearCachedS3Token_groupClaimValue: boolean = - await (async () => { - const { projects } = await onyxiaApi.getUserAndProjects(); - - const KEY = "onyxia:s3:projects-hash"; - - const hash = fnv1aHashToHex(JSON.stringify(projects)); - - if ( - !oidc_s3.isNewBrowserSession && - sessionStorage.getItem(KEY) === hash - ) { - return false; - } - - sessionStorage.setItem(KEY, hash); - return true; - })(); - - const doClearCachedS3Token_s3BookmarkClaimValue: boolean = - (() => { - const resolvedAdminBookmarks = - privateSelectors.resolvedAdminBookmarks(getState()); - - const KEY = "onyxia:s3:resolvedAdminBookmarks-hash"; - - const hash = fnv1aHashToHex( - JSON.stringify(resolvedAdminBookmarks) - ); - - if ( - !oidc_s3.isNewBrowserSession && - sessionStorage.getItem(KEY) === hash - ) { - return false; - } - - sessionStorage.setItem(KEY, hash); - return true; - })(); - - return { - oidc: oidc_s3, - doClearCachedS3Token: - doClearCachedS3Token_groupClaimValue || - doClearCachedS3Token_s3BookmarkClaimValue - }; - } - ); - })(); - - prS3ClientByConfigId.set(s3Config.id, prS3Client); - - return prS3Client; - }, - getS3ConfigAndClientForExplorer: - () => - async ( - ...args - ): Promise => { - const [dispatch, getState] = args; - - const s3Config = selectors - .s3Configs(getState()) - .find(s3Config => s3Config.isExplorerConfig); - - if (s3Config === undefined) { - return undefined; - } - - const s3Client = await dispatch( - protectedThunks.getS3ClientForSpecificConfig({ - s3ConfigId: s3Config.id - }) - ); - - return { s3Client, s3Config }; - }, - createS3Config: - (params: { projectS3Config: ProjectConfigs.S3Config }) => - async (...args) => { - const { projectS3Config } = params; - - const [dispatch, getState] = args; - - const projectConfigsS3 = structuredClone( - projectManagement.protectedSelectors.projectConfig(getState()).s3 - ); - - const i = projectConfigsS3.s3Configs.findIndex( - projectS3Config_i => - getProjectS3ConfigId({ - creationTime: projectS3Config_i.creationTime - }) === - getProjectS3ConfigId({ - creationTime: projectS3Config.creationTime - }) - ); - - if (i < 0) { - projectConfigsS3.s3Configs.push(projectS3Config); - } else { - projectConfigsS3.s3Configs[i] = projectS3Config; - } - - await dispatch( - projectManagement.protectedThunks.updateConfigValue({ - key: "s3", - value: projectConfigsS3 - }) - ); - }, - - initialize: - () => - async (...args) => { - const [dispatch, getState, { onyxiaApi, paramsOfBootstrapCore }] = args; - - const { oidcParams } = await onyxiaApi.getAvailableRegionsAndOidcParams(); - - if (oidcParams === undefined) { - dispatch(actions.initialized({ resolvedAdminBookmarks: [] })); - return; - } - const deploymentRegion = - deploymentRegionManagement.selectors.currentDeploymentRegion(getState()); - - const { resolvedAdminBookmarks } = await resolveS3AdminBookmarks({ - deploymentRegion_s3Configs: deploymentRegion.s3Configs, - getDecodedIdToken: async ({ oidcParams_partial }) => { - const { createOidc, mergeOidcParams } = await import( - "core/adapters/oidc" - ); - - const oidc = await createOidc({ - ...mergeOidcParams({ - oidcParams, - oidcParams_partial - }), - autoLogin: true, - transformBeforeRedirectForKeycloakTheme: - paramsOfBootstrapCore.transformBeforeRedirectForKeycloakTheme, - getCurrentLang: paramsOfBootstrapCore.getCurrentLang, - enableDebugLogs: paramsOfBootstrapCore.enableOidcDebugLogs - }); - - const { decodedIdToken } = await oidc.getTokens(); - - return decodedIdToken; - } - }); - - dispatch(actions.initialized({ resolvedAdminBookmarks })); - } -} satisfies Thunks; - -const { getContext } = createUsecaseContextApi(() => ({ - prS3ClientByConfigId: new Map>() -})); diff --git a/web/src/core/usecases/s3ExplorerUiController/decoupledLogic/computeUploadStatusAtPrefix.ts b/web/src/core/usecases/s3ExplorerUiController/decoupledLogic/computeUploadStatusAtPrefix.ts new file mode 100644 index 000000000..0c1f42416 --- /dev/null +++ b/web/src/core/usecases/s3ExplorerUiController/decoupledLogic/computeUploadStatusAtPrefix.ts @@ -0,0 +1,99 @@ +import { assert } from "tsafe/assert"; +import { type S3Uri, getIsInside } from "core/tools/S3Uri"; +import type { MainView } from "../selectors"; + +export function computeUploadStatusAtPrefix(params: { + s3Uri: S3Uri; + uploads: MainView["uploads"]; +}): MainView.Item[] { + const { s3Uri, uploads } = params; + + const items: MainView.Item[] = []; + + const progressesByDisplayName: Record< + string, + { size: number; completionPercent: number }[] + > = {}; + + for (const upload of uploads) { + if (upload.stoppedStatus !== undefined) { + continue; + } + + const { isInside, isTopLevel } = getIsInside({ + s3UriPrefix: s3Uri, + s3Uri: upload.s3Uri + }); + + if (!isInside) { + continue; + } + + if (isTopLevel) { + items.push({ + type: "object", + displayName: (() => { + const lastSegment = upload.s3Uri.keySegments.at(-1); + + assert(lastSegment !== undefined); + + return lastSegment; + })(), + s3Uri: upload.s3Uri, + uploadProgressPercent: upload.completionPercent, + isDeleting: false, + size: upload.size, + lastModified: upload.uploadStartTime + }); + continue; + } + + const s3Uri_newItem: S3Uri.TerminatedByDelimiter = { + bucket: upload.s3Uri.bucket, + delimiter: upload.s3Uri.delimiter, + keySegments: upload.s3Uri.keySegments.slice(0, s3Uri.keySegments.length + 1), + isDelimiterTerminated: true + }; + + const displayName = s3Uri_newItem.keySegments.at(-1); + + assert(displayName !== undefined); + + if ( + items.find( + item => item.type === "prefix segment" && item.displayName === displayName + ) === undefined + ) { + items.push({ + type: "prefix segment", + displayName, + s3Uri: s3Uri_newItem, + isDeleting: false, + uploadProgressPercent: NaN + }); + } + + (progressesByDisplayName[displayName] ??= []).push({ + completionPercent: upload.completionPercent, + size: upload.size + }); + } + + for (const item of items) { + if (item.type !== "prefix segment") { + continue; + } + + const progresses = progressesByDisplayName[item.displayName]; + + assert(progresses !== undefined); + + item.uploadProgressPercent = + progresses.reduce( + (acc, { size, completionPercent }) => acc + size * completionPercent, + 0 + ) / progresses.reduce((acc, { size }) => acc + size, 0); + } + + return items; +} diff --git a/web/src/core/usecases/fileExplorer/decoupledLogic/uploadProgress.ts b/web/src/core/usecases/s3ExplorerUiController/decoupledLogic/uploadProgress.ts similarity index 100% rename from web/src/core/usecases/fileExplorer/decoupledLogic/uploadProgress.ts rename to web/src/core/usecases/s3ExplorerUiController/decoupledLogic/uploadProgress.ts diff --git a/web/src/core/usecases/s3ExplorerUiController/evt.ts b/web/src/core/usecases/s3ExplorerUiController/evt.ts new file mode 100644 index 000000000..8edafe465 --- /dev/null +++ b/web/src/core/usecases/s3ExplorerUiController/evt.ts @@ -0,0 +1,247 @@ +import type { CreateEvt } from "core/bootstrap"; +import { Evt } from "evt"; +import { onlyIfChanged } from "evt/operators/onlyIfChanged"; +import { privateSelectors, type RouteParams } from "./selectors"; +import { Reflect, id } from "tsafe"; +import { name } from "./state"; +import { AccessError } from "clean-architecture"; +import * as s3ProfilesManagement from "core/usecases/s3ProfilesManagement"; +import { assert } from "tsafe"; +import { actions } from "./state"; +import { thunks, evtAskOverwriteConfirmation } from "./thunks"; +import { getIsInside } from "core/tools/S3Uri"; +import * as dataExplorer from "core/usecases/dataExplorer"; +import { stringifyS3Uri } from "core/tools/S3Uri"; +import type { S3Uri } from "core/tools/S3Uri"; + +export const createEvt = (({ evtAction, dispatch, getState }) => { + const evt = Evt.create< + | { + action: "updateRoute"; + method: "replace" | "push"; + routeParams: RouteParams; + } + | { + action: "ask confirmation for bucket creation attempt"; + bucket: string; + createBucket: () => Promise<{ isSuccess: boolean }>; + } + | { + action: "ask overwrite confirmation"; + s3Uri: S3Uri.NonTerminatedByDelimiter; + resolveResponse: (params: { doOverwrite: boolean }) => void; + } + >(); + + evtAskOverwriteConfirmation.attach(({ s3Uri, resolveResponse }) => + evt.post({ + action: "ask overwrite confirmation", + s3Uri, + resolveResponse + }) + ); + + evtAction + .pipe(action => action.usecaseName === name) + .pipe(() => [privateSelectors.doesListedPrefixHaveFinishedUpload(getState())]) + .pipe(onlyIfChanged()) + .attach( + doesListedPrefixHaveFinishedUpload => doesListedPrefixHaveFinishedUpload, + () => + dispatch( + thunks.listPrefix({ + s3Uri: privateSelectors.s3Uri(getState()), + debounce: false + }) + ) + ); + + evtAction + .pipe(action => action.usecaseName === name) + .pipe(() => [privateSelectors.isFullyQualifiedDataFileUri(getState())]) + .pipe(onlyIfChanged()) + .attach( + isFullyQualifiedDataFileUri => isFullyQualifiedDataFileUri, + () => { + const s3Uri = privateSelectors.s3Uri(getState()); + + assert(s3Uri !== undefined); + + dispatch( + dataExplorer.thunks.load({ + routeParams: { + source: stringifyS3Uri(s3Uri) + } + }) + ); + } + ); + + evtAction + .pipe(action => action.usecaseName === name) + .attach( + action => { + if (action.actionName !== "deletionCompleted") { + return false; + } + + const profileName = privateSelectors.profileName(getState()); + + if (profileName !== action.payload.profileName) { + return false; + } + + const s3Uri = privateSelectors.s3Uri(getState()); + + if (s3Uri === undefined) { + return false; + } + + const { isTopLevel } = getIsInside({ + s3UriPrefix: s3Uri, + s3Uri: action.payload.s3Uri + }); + + if (!isTopLevel) { + return false; + } + + return true; + }, + () => { + dispatch( + thunks.listPrefix({ + s3Uri: privateSelectors.s3Uri(getState()), + debounce: false + }) + ); + } + ); + + evtAction.$attach( + action => { + if (action.usecaseName !== name) { + return null; + } + + if (action.actionName !== "listingFailed") { + return null; + } + + if (action.payload.errorCase !== "no such bucket") { + return null; + } + + const { profileName, s3Uri } = action.payload; + + const s3Profile = s3ProfilesManagement.selectors + .s3Profiles(getState()) + .find(s3Profile => s3Profile.profileName === profileName); + + assert(s3Profile !== undefined); + + if ( + s3Profile.bookmarks.find( + bookmark => + bookmark.isReadonly && bookmark.s3Uri.bucket === s3Uri.bucket + ) === undefined + ) { + return null; + } + + return [{ profileName, s3Uri }]; + }, + async ({ profileName, s3Uri }) => { + evt.post({ + action: "ask confirmation for bucket creation attempt", + bucket: s3Uri.bucket, + createBucket: async () => { + const s3Client = await dispatch( + s3ProfilesManagement.protectedThunks.getS3Client({ + profileName + }) + ); + + const { bucket } = s3Uri; + + const cmdId = Date.now(); + + dispatch( + actions.commandLogIssued({ + cmdId, + cmd: `mc mb s3/${bucket}` + }) + ); + + const result = await s3Client.createBucket({ bucket }); + + dispatch( + actions.commandLogResponseReceived({ + cmdId, + resp: result.isSuccess + ? `Bucket \`s3/${bucket}\` created` + : (() => { + switch (result.errorCase) { + case "already exist": + return `Bucket \`s3/${bucket}\` already exists`; + case "access denied": + return `Access denied while creating \`s3/${bucket}\`: ${result.errorMessage}`; + case "unknown": + return `Failed to create \`s3/${bucket}\`: ${result.errorMessage}`; + } + })() + }) + ); + + if (result.isSuccess) { + dispatch(thunks.listPrefix({ s3Uri, debounce: false })); + } + + return { isSuccess: result.isSuccess }; + } + }); + } + ); + + evtAction + .pipe(() => { + try { + return [privateSelectors.routeParams(getState())]; + } catch (error) { + // NOTE: If it's too early to get the route params, we skip. + if (!(error instanceof AccessError)) { + throw error; + } + + return null; + } + }) + .pipe(onlyIfChanged()) + .pipe([ + (routeParams, { routeParams: routeParams_prev }) => [ + { + routeParams, + method: + routeParams.s3UriWithoutScheme === + routeParams_prev.s3UriWithoutScheme + ? "replace" + : "push" + } as const + ], + { + routeParams: id({ + s3UriWithoutScheme: "" + }), + method: Reflect<"push" | "replace">() + } + ]) + .attach(({ method, routeParams }) => { + evt.post({ + action: "updateRoute", + method, + routeParams + }); + }); + + return evt; +}) satisfies CreateEvt; diff --git a/web/src/core/usecases/fileExplorer/index.ts b/web/src/core/usecases/s3ExplorerUiController/index.ts similarity index 77% rename from web/src/core/usecases/fileExplorer/index.ts rename to web/src/core/usecases/s3ExplorerUiController/index.ts index 6e655c5cd..8cede8377 100644 --- a/web/src/core/usecases/fileExplorer/index.ts +++ b/web/src/core/usecases/s3ExplorerUiController/index.ts @@ -1,3 +1,4 @@ -export * from "./state"; export * from "./thunks"; export * from "./selectors"; +export * from "./state"; +export * from "./evt"; diff --git a/web/src/core/usecases/s3ExplorerUiController/selectors.ts b/web/src/core/usecases/s3ExplorerUiController/selectors.ts new file mode 100644 index 000000000..db11d3b80 --- /dev/null +++ b/web/src/core/usecases/s3ExplorerUiController/selectors.ts @@ -0,0 +1,699 @@ +import { createSelector } from "clean-architecture"; +import * as s3ProfilesManagement from "core/usecases/s3ProfilesManagement"; +import type { LocalizedString } from "core/ports/OnyxiaApi"; +import { type S3Uri, stringifyS3Uri, getIsInside } from "core/tools/S3Uri"; +import type { State as RootState } from "core/bootstrap"; +import { assert, type Equals } from "tsafe"; +import { id } from "tsafe/id"; +import { same } from "evt/tools/inDepth/same"; +import { computeUploadStatusAtPrefix } from "./decoupledLogic/computeUploadStatusAtPrefix"; +import { name, type State } from "./state"; + +export type RouteParams = { + profile?: string; + s3UriWithoutScheme: string; +}; + +export type MainView = { + // NOTE: Undefined is when no profile (user should be prompted to create one) + profileSelect: + | { + availableProfileNames: string[]; + selectedProfile: { + /** Assert match one of the availableProfiles */ + name: string; + url: string; + isReadonly: boolean; + }; + } + | undefined; + + bookmarks: { + items: { + displayName: LocalizedString | undefined; + s3Uri: S3Uri; + isReadonly: boolean; + }[]; + activeItemS3Uri: S3Uri | undefined; + }; + + uploads: State.Upload[]; + + uriBar: { + s3Uri: S3Uri | undefined; + hints: { + type: "object" | "key-segment" | "bookmark"; + text: string; + s3Uri: S3Uri; + }[]; + bookmarkStatus: + | { + isBookmarked: false; + } + | { + isBookmarked: true; + isReadonly: boolean; + }; + }; + + isBackButtonDisabled: boolean; + + directoryCreationButton: + | { + isDisabled: true; + } + | { + isDisabled: false; + exclude: string[]; + }; + + isUploadButtonDisabled: boolean; + + isListing: boolean; + + fullyQualifiedUri: + | { + isFullyQualifiedUri: false; + } + | { + isFullyQualifiedUri: true; + isDataObject: boolean; + }; + + listedPrefix: + | { + isErrored: true; + errorCase: State.ListedPrefix.ErrorCase; + } + | { + isErrored: false; + items: MainView.Item[]; + } + | undefined; + + commandLogsEntries: State.CommandLogsEntry[]; +}; + +export namespace MainView { + export type Item = Item.PrefixSegment | Item.Object; + + export namespace Item { + type Common = { + uploadProgressPercent: number | undefined; + isDeleting: boolean; + displayName: string; + }; + + export type PrefixSegment = Common & { + type: "prefix segment"; + s3Uri: S3Uri.TerminatedByDelimiter; + }; + + export type Object = Common & { + type: "object"; + s3Uri: S3Uri.NonTerminatedByDelimiter; + size: number; + lastModified: number; + }; + } +} + +const state = (rootState: RootState): State => rootState[name]; + +const profileName = createSelector( + s3ProfilesManagement.selectors.ambientS3Profile, + ambientS3Profile => { + if (ambientS3Profile === undefined) { + return undefined; + } + return ambientS3Profile.profileName; + } +); + +const s3Uri = createSelector( + createSelector(state, state => state.listedPrefixByProfile), + profileName, + (listedPrefixByProfile, profileName): S3Uri | undefined => { + if (profileName === undefined) { + return undefined; + } + + const listedPrefix = listedPrefixByProfile[profileName]; + + if (listedPrefix === undefined) { + return undefined; + } + + if (listedPrefix.next !== undefined) { + return listedPrefix.next.s3Uri; + } + + if (listedPrefix.current === undefined) { + return undefined; + } + + return listedPrefix.current.s3Uri; + } +); + +const routeParams = createSelector( + profileName, + s3Uri, + (profileName, s3Uri): RouteParams => ({ + profile: profileName, + s3UriWithoutScheme: + s3Uri === undefined ? "" : stringifyS3Uri(s3Uri).slice("s3://".length) + }) +); + +const profileSelect = createSelector( + s3ProfilesManagement.selectors.ambientS3Profile, + s3ProfilesManagement.selectors.s3Profiles, + (ambientS3Profile, s3Profiles): MainView["profileSelect"] => { + if (ambientS3Profile === undefined) { + return undefined; + } + + return { + selectedProfile: { + name: ambientS3Profile.profileName, + url: ambientS3Profile.paramsOfCreateS3Client.url, + isReadonly: ambientS3Profile.origin === "defined in region" + }, + availableProfileNames: s3Profiles.map(s3Profile => s3Profile.profileName) + }; + } +); + +const bookmarks = createSelector( + s3ProfilesManagement.selectors.ambientS3Profile, + s3Uri, + (ambientS3Profile, s3Uri): MainView["bookmarks"] => { + if (ambientS3Profile === undefined) { + return { + items: [], + activeItemS3Uri: undefined + }; + } + + const items = ambientS3Profile.bookmarks.map(bookmark => ({ + displayName: bookmark.displayName, + s3Uri: bookmark.s3Uri, + isReadonly: bookmark.isReadonly + })); + + return { + items, + activeItemS3Uri: items.find(item => same(item.s3Uri, s3Uri))?.s3Uri + }; + } +); + +const listedPrefix_state = createSelector( + state, + profileName, + (state, profileName): State.ListedPrefix | undefined => { + if (profileName === undefined) { + return undefined; + } + + return state.listedPrefixByProfile[profileName]; + } +); + +const isListing = createSelector( + listedPrefix_state, + (listedPrefix_state): MainView["isListing"] => { + if (listedPrefix_state === undefined) { + return false; + } + + if (listedPrefix_state.next === undefined) { + return false; + } + + if (listedPrefix_state.next.errorCase !== undefined) { + return false; + } + + return true; + } +); + +const uploads_profile = createSelector( + createSelector(state, state => state.uploads), + profileName, + (uploads, profileName): State.Upload[] => { + if (profileName === undefined) { + return []; + } + + return uploads.filter(upload => upload.profileName === profileName); + } +); + +const deletions_profile = createSelector( + createSelector(state, state => state.deletions), + profileName, + (deletions, profileName): State.Deletion[] => { + if (profileName === undefined) { + return []; + } + + return deletions.filter(deletion => deletion.profileName === profileName); + } +); + +const items = createSelector( + listedPrefix_state, + uploads_profile, + deletions_profile, + ( + listedPrefix_state, + uploads_profile, + deletions_profile + ): MainView.Item[] | undefined => { + if (listedPrefix_state === undefined) { + return undefined; + } + + if (listedPrefix_state.current === undefined) { + return undefined; + } + + const items_upload: MainView.Item[] = computeUploadStatusAtPrefix({ + s3Uri: listedPrefix_state.current.s3Uri, + uploads: uploads_profile + }); + + const items_actual: MainView.Item[] = listedPrefix_state.current.items.map( + item => { + switch (item.type) { + case "object": + return id({ + type: "object", + displayName: (() => { + const keyBasename = item.s3Uri.keySegments.at(-1); + + assert(keyBasename !== undefined); + + return keyBasename; + })(), + s3Uri: item.s3Uri, + uploadProgressPercent: undefined, + isDeleting: false, + lastModified: item.lastModified, + size: item.size + }); + case "prefix": + return id({ + type: "prefix segment", + displayName: (() => { + const lastSegment = item.s3Uri.keySegments.at(-1); + + assert(lastSegment !== undefined); + + return lastSegment; + })(), + s3Uri: item.s3Uri, + uploadProgressPercent: undefined, + isDeleting: false + }); + } + } + ); + + const items: MainView.Item[] = []; + + for (const item_upload of items_upload) { + if ( + item_upload.uploadProgressPercent === 100 && + items_actual.find( + item_actual => item_actual.displayName === item_upload.displayName + ) !== undefined + ) { + continue; + } + + items.push(item_upload); + } + + for (const item_actual of items_actual) { + if ( + items.find(item => item.displayName === item_actual.displayName) !== + undefined + ) { + continue; + } + + if (item_actual.displayName === ".keep") { + continue; + } + + items.push(item_actual); + } + + for (const item of items) { + if ( + deletions_profile.find(({ s3Uri }) => same(item.s3Uri, s3Uri)) !== + undefined + ) { + item.isDeleting = true; + } + } + + items.sort((a, b) => a.displayName.localeCompare(b.displayName)); + + return items; + } +); + +const uploads = createSelector(uploads_profile, (uploads_profile): MainView["uploads"] => + uploads_profile.filter(upload => upload.s3Uri.keySegments.at(-1) !== ".keep") +); + +const listedPrefix = createSelector( + listedPrefix_state, + s3ProfilesManagement.selectors.ambientS3Profile, + items, + (listedPrefix_state, ambientS3Profile, items): MainView["listedPrefix"] => { + if (listedPrefix_state === undefined) { + return undefined; + } + + if ( + listedPrefix_state.next !== undefined && + listedPrefix_state.next.errorCase !== undefined + ) { + return { + isErrored: true, + errorCase: listedPrefix_state.next.errorCase + }; + } + + if (listedPrefix_state.current === undefined) { + return undefined; + } + + assert(ambientS3Profile !== undefined); + assert(items !== undefined); + + return { + isErrored: false, + items + }; + } +); + +const fullyQualifiedUri = createSelector( + listedPrefix_state, + (listedPrefix_state): MainView["fullyQualifiedUri"] => { + if (listedPrefix_state === undefined) { + return { isFullyQualifiedUri: false }; + } + + const { current } = listedPrefix_state; + + if (current === undefined) { + return { isFullyQualifiedUri: false }; + } + + const [item, ...rest] = current.items; + + if (item === undefined || rest.length !== 0) { + return { isFullyQualifiedUri: false }; + } + + if (!same(current.s3Uri, item.s3Uri)) { + return { isFullyQualifiedUri: false }; + } + + const isFullyQualifiedUri = true as const; + + const s3Uri_str = stringifyS3Uri(item.s3Uri); + + return { + isFullyQualifiedUri, + isDataObject: + s3Uri_str.endsWith(".parquet") || + s3Uri_str.endsWith(".csv") || + s3Uri_str.endsWith(".json") + }; + } +); + +const isBackButtonDisabled = createSelector( + s3Uri, + (s3Uri): MainView["isBackButtonDisabled"] => + s3Uri === undefined || s3Uri.keySegments.length === 0 +); + +const directoryCreationButton = createSelector( + s3Uri, + listedPrefix, + (s3Uri, listedPrefix): MainView["directoryCreationButton"] => { + const isUploadButtonDisabled = + s3Uri === undefined || + !s3Uri.isDelimiterTerminated || + listedPrefix === undefined || + listedPrefix.isErrored; + + if (isUploadButtonDisabled) { + return { + isDisabled: true + }; + } + + return { + isDisabled: false, + exclude: listedPrefix.items.map(item => item.displayName) + }; + } +); + +const isUploadButtonDisabled = createSelector( + directoryCreationButton, + (directoryCreationButton): MainView["isUploadButtonDisabled"] => + directoryCreationButton.isDisabled +); + +const uriBar = createSelector( + s3Uri, + bookmarks, + listedPrefix, + isListing, + (s3Uri, bookmarks, listedPrefix, isListing): MainView["uriBar"] => { + if (s3Uri === undefined) { + return { + s3Uri: undefined, + hints: bookmarks.items.map(bookmark => ({ + type: "bookmark", + text: stringifyS3Uri(bookmark.s3Uri), + s3Uri: bookmark.s3Uri + })), + bookmarkStatus: { + isBookmarked: false + } + }; + } + + const bookmarkStatus: MainView["uriBar"]["bookmarkStatus"] = (() => { + if (bookmarks.activeItemS3Uri === undefined) { + return { isBookmarked: false }; + } + + const bookmark = bookmarks.items.find(bookmark => + same(bookmark.s3Uri, bookmarks.activeItemS3Uri) + ); + + assert(bookmark !== undefined); + + return { + isBookmarked: true, + isReadonly: bookmark.isReadonly + }; + })(); + + const hints: MainView["uriBar"]["hints"] = bookmarks.items + .filter( + bookmark => + getIsInside({ s3UriPrefix: s3Uri, s3Uri: bookmark.s3Uri }).isInside + ) + .map(bookmark => ({ + type: "bookmark" as const, + text: (() => { + const n = s3Uri.isDelimiterTerminated + ? s3Uri.keySegments.length + : s3Uri.keySegments.length - 1; + + let text = bookmark.s3Uri.keySegments + .slice(n) + .join(bookmark.s3Uri.delimiter); + + if (bookmark.s3Uri.isDelimiterTerminated) { + text += bookmark.s3Uri.delimiter; + } + + return text; + })(), + s3Uri: bookmark.s3Uri + })); + + if (listedPrefix === undefined || listedPrefix.isErrored || isListing) { + return { + s3Uri, + hints, + bookmarkStatus + }; + } + + listedPrefix.items.forEach(item => { + if ( + bookmarks.items.find(bookmark => same(bookmark.s3Uri, item.s3Uri)) !== + undefined + ) { + return; + } + + const keyBasename = item.s3Uri.keySegments.at(-1); + assert(keyBasename !== undefined); + + switch (item.type) { + case "object": + if (same(item.s3Uri, s3Uri)) { + return; + } + hints.push({ + type: "object", + text: keyBasename, + s3Uri: item.s3Uri + }); + + break; + + case "prefix segment": + { + hints.push({ + type: "key-segment", + text: keyBasename, + s3Uri: item.s3Uri + }); + } + break; + default: + assert>(false); + } + }); + + hints.sort((a, b) => { + const getRank = (hint: typeof a) => { + switch (hint.type) { + case "bookmark": + return 1; + case "key-segment": + return 2; + case "object": + return 3; + default: + assert>(false); + } + }; + return getRank(a) - getRank(b); + }); + + return { + s3Uri, + hints, + bookmarkStatus + }; + } +); + +const commandLogsEntries = createSelector( + state, + (state): MainView["commandLogsEntries"] => state.commandLogsEntries +); + +const mainView = createSelector( + profileSelect, + bookmarks, + uploads, + uriBar, + isBackButtonDisabled, + directoryCreationButton, + isUploadButtonDisabled, + fullyQualifiedUri, + isListing, + listedPrefix, + commandLogsEntries, + ( + profileSelect, + bookmarks, + uploads, + uriBar, + isBackButtonDisabled, + directoryCreationButton, + isUploadButtonDisabled, + fullyQualifiedUri, + isListing, + listedPrefix, + commandLogsEntries + ): MainView => ({ + profileSelect, + bookmarks, + uploads, + uriBar, + isBackButtonDisabled, + directoryCreationButton, + isUploadButtonDisabled, + fullyQualifiedUri, + isListing, + listedPrefix, + commandLogsEntries + }) +); + +export const selectors = { + mainView +}; + +const s3Uri_currentlyListing = createSelector(listedPrefix_state, listedPrefix_state => { + if (listedPrefix_state === undefined) { + return undefined; + } + + if (listedPrefix_state.next === undefined) { + return undefined; + } + + if (listedPrefix_state.next.errorCase !== undefined) { + return undefined; + } + + return listedPrefix_state.next.s3Uri; +}); + +const doesListedPrefixHaveFinishedUpload = createSelector( + listedPrefix, + listedPrefix => + listedPrefix !== undefined && + !listedPrefix.isErrored && + listedPrefix.items.find(item => item.uploadProgressPercent === 100) !== undefined +); + +const isFullyQualifiedDataFileUri = createSelector( + fullyQualifiedUri, + fullyQualifiedUri => + fullyQualifiedUri.isFullyQualifiedUri && fullyQualifiedUri.isDataObject +); + +export const privateSelectors = { + routeParams, + s3Uri, + profileName, + s3Uri_currentlyListing, + doesListedPrefixHaveFinishedUpload, + listedPrefix_state, + isFullyQualifiedDataFileUri, + uploads: createSelector(state, state => state.uploads) +}; diff --git a/web/src/core/usecases/s3ExplorerUiController/state.ts b/web/src/core/usecases/s3ExplorerUiController/state.ts new file mode 100644 index 000000000..949550216 --- /dev/null +++ b/web/src/core/usecases/s3ExplorerUiController/state.ts @@ -0,0 +1,442 @@ +import { assert, id } from "tsafe"; +import { createUsecaseActions } from "clean-architecture"; +import { type S3Uri, stringifyS3Uri, getIsInside } from "core/tools/S3Uri"; +import { same } from "evt/tools/inDepth/same"; + +//All explorer paths are expected to be absolute (start with /) + +export type State = { + commandLogsEntries: State.CommandLogsEntry[]; + uploads: State.Upload[]; + deletions: State.Deletion[]; + listedPrefixByProfile: Record; +}; + +export namespace State { + export type CommandLogsEntry = { + cmdId: number; + cmd: string; + resp: string | undefined; + }; + + export type Upload = { + profileName: string; + s3Uri: S3Uri.NonTerminatedByDelimiter; + size: number; + completionPercent: number; + uploadStartTime: number; + stoppedStatus: + | { case: "canceled" } + | { case: "errored"; errorMessage: string } + | undefined; + }; + + export type Deletion = { + profileName: string; + s3Uri: S3Uri; + }; + + export type ListedPrefix = { + next: + | { + s3Uri: S3Uri; + errorCase: ListedPrefix.ErrorCase | undefined; + } + | undefined; + current: + | { + s3Uri: S3Uri; + items: ListedPrefix.Item[]; + } + | undefined; + }; + + export namespace ListedPrefix { + export type ErrorCase = "access denied" | "no such bucket"; + + export type Item = Item.Prefix | Item.Object; + + export namespace Item { + export type Prefix = { + type: "prefix"; + s3Uri: S3Uri.TerminatedByDelimiter; + }; + + export type Object = { + type: "object"; + s3Uri: S3Uri.NonTerminatedByDelimiter; + size: number; + lastModified: number; + }; + } + } +} + +export const name = "s3ExplorerUiController"; + +export const { reducer, actions } = createUsecaseActions({ + name, + initialState: id({ + commandLogsEntries: [], + uploads: [], + deletions: [], + listedPrefixByProfile: {} + }), + reducers: { + putObjectStarted: ( + state, + { + payload + }: { + payload: { + profileName: string; + s3Uri: S3Uri.NonTerminatedByDelimiter; + size: number; + }; + } + ) => { + const { profileName, s3Uri, size } = payload; + + retry_case: { + const upload = state.uploads.find( + upload => + upload.profileName === profileName && same(upload.s3Uri, s3Uri) + ); + + if (upload === undefined) { + break retry_case; + } + + assert( + upload.stoppedStatus !== undefined && + upload.stoppedStatus.case === "errored" + ); + + upload.completionPercent = 0; + upload.stoppedStatus = undefined; + upload.uploadStartTime = Date.now(); + + return; + } + + state.uploads.push({ + profileName, + s3Uri, + size, + completionPercent: 0, + uploadStartTime: Date.now(), + stoppedStatus: undefined + }); + }, + uploadFlushed: state => { + state.uploads = []; + }, + uploadCanceled: ( + state, + { + payload + }: { payload: { profileName: string; s3Uri: S3Uri.NonTerminatedByDelimiter } } + ) => { + const { profileName, s3Uri } = payload; + + const upload_toDelete = state.uploads.find( + upload => upload.profileName === profileName && same(upload.s3Uri, s3Uri) + ); + + assert(upload_toDelete !== undefined); + assert(upload_toDelete.stoppedStatus === undefined); + + const i = state.uploads.indexOf(upload_toDelete); + + assert(i !== -1); + + state.uploads.splice(i, 1); + }, + putObjectProgressReported: ( + state, + { + payload + }: { + payload: { + profileName: string; + s3Uri: S3Uri.NonTerminatedByDelimiter; + completionPercent: number; + }; + } + ) => { + const { profileName, s3Uri, completionPercent } = payload; + + const upload = state.uploads.find( + upload => upload.profileName === profileName && same(upload.s3Uri, s3Uri) + ); + + assert(upload !== undefined); + assert(upload.stoppedStatus === undefined); + + upload.stoppedStatus = undefined; + upload.completionPercent = completionPercent; + }, + putObjectStopped: ( + state, + { + payload + }: { + payload: { + profileName: string; + s3Uri: S3Uri.NonTerminatedByDelimiter; + stoppedStatus: + | { case: "canceled" } + | { case: "errored"; errorMessage: string } + | undefined; + }; + } + ) => { + const { profileName, s3Uri, stoppedStatus } = payload; + + const upload = state.uploads.find( + upload => upload.profileName === profileName && same(upload.s3Uri, s3Uri) + ); + + assert(upload !== undefined); + assert(upload.stoppedStatus === undefined); + assert(upload.completionPercent !== 100); + + upload.stoppedStatus = stoppedStatus; + }, + listingCleared: (state, { payload }: { payload: { profileName: string } }) => { + const { profileName } = payload; + + state.listedPrefixByProfile[profileName] = { + current: undefined, + next: undefined + }; + }, + listingStarted: ( + state, + { + payload + }: { + payload: { + profileName: string; + s3Uri: S3Uri; + }; + } + ) => { + const { profileName, s3Uri } = payload; + + const listedPrefix = (state.listedPrefixByProfile[profileName] ??= { + next: undefined, + current: undefined + }); + + listedPrefix.next = { + s3Uri, + errorCase: undefined + }; + }, + listingCompletedSuccessfully: ( + state, + { + payload + }: { + payload: { + profileName: string; + items: State.ListedPrefix.Item[]; + }; + } + ) => { + const { profileName, items } = payload; + + const listedPrefix = state.listedPrefixByProfile[profileName]; + + assert(listedPrefix !== undefined); + + assert( + listedPrefix.next !== undefined && + listedPrefix.next.errorCase === undefined + ); + + const { s3Uri } = listedPrefix.next; + + listedPrefix.next = undefined; + + listedPrefix.current = { + s3Uri, + items + }; + }, + listingCompletedSuccessfully_inferFromCurrentState: ( + state, + { + payload + }: { + payload: { + profileName: string; + s3Uri: S3Uri; + }; + } + ) => { + const { profileName, s3Uri } = payload; + + const listedPrefix = state.listedPrefixByProfile[profileName]; + + assert(listedPrefix !== undefined); + + listedPrefix.next = undefined; + + assert(listedPrefix.current !== undefined); + + const { items } = listedPrefix.current; + + listedPrefix.current = { + s3Uri, + items: items.filter(item => + stringifyS3Uri(item.s3Uri).startsWith(stringifyS3Uri(s3Uri)) + ) + }; + }, + + listingFailed: ( + state, + { + payload + }: { + payload: { + profileName: string; + s3Uri: S3Uri; + errorCase: State.ListedPrefix.ErrorCase; + }; + } + ) => { + const { profileName, s3Uri, errorCase } = payload; + + const listedPrefix = state.listedPrefixByProfile[profileName]; + + assert(listedPrefix !== undefined); + + assert( + listedPrefix.next !== undefined && + listedPrefix.next.errorCase === undefined && + same(listedPrefix.next.s3Uri, s3Uri) + ); + + listedPrefix.next.errorCase = errorCase; + }, + commandLogIssued: ( + state, + { + payload + }: { + payload: { + cmdId: number; + cmd: string; + }; + } + ) => { + const { cmdId, cmd } = payload; + + state.commandLogsEntries.push({ + cmdId, + cmd, + resp: undefined + }); + }, + commandLogCancelled: ( + state, + { + payload + }: { + payload: { + cmdId: number; + }; + } + ) => { + const { cmdId } = payload; + + const index = state.commandLogsEntries.findIndex( + entry => entry.cmdId === cmdId + ); + + assert(index >= 0); + + state.commandLogsEntries.splice(index, 1); + }, + commandLogResponseReceived: ( + state, + { + payload + }: { + payload: { + cmdId: number; + resp: string; + }; + } + ) => { + const { cmdId, resp } = payload; + + const entry = state.commandLogsEntries.find(entry => entry.cmdId === cmdId); + + assert(entry !== undefined); + + entry.resp = resp; + }, + deletionStarted: ( + state, + { + payload + }: { + payload: { + profileName: string; + s3Uri: S3Uri; + }; + } + ) => { + const { profileName, s3Uri } = payload; + + state.deletions.push({ + profileName, + s3Uri + }); + + { + state.uploads + .filter( + upload => + upload.profileName === profileName && + (same(s3Uri, upload.s3Uri) || + getIsInside({ s3UriPrefix: s3Uri, s3Uri: upload.s3Uri }) + .isInside) + ) + .forEach(upload => { + const index = state.uploads.indexOf(upload); + + assert(index !== -1); + + state.uploads.splice(index, 1); + }); + } + }, + deletionCompleted: ( + state, + { + payload + }: { + payload: { + profileName: string; + s3Uri: S3Uri; + }; + } + ) => { + const { profileName, s3Uri } = payload; + + const i = state.deletions.findIndex( + deletion => + deletion.profileName === profileName && same(deletion.s3Uri, s3Uri) + ); + + assert(i !== -1); + + state.deletions.splice(i, 1); + } + } +}); diff --git a/web/src/core/usecases/s3ExplorerUiController/thunks.ts b/web/src/core/usecases/s3ExplorerUiController/thunks.ts new file mode 100644 index 000000000..4f163f3a8 --- /dev/null +++ b/web/src/core/usecases/s3ExplorerUiController/thunks.ts @@ -0,0 +1,869 @@ +import type { Thunks } from "core/bootstrap"; +import { privateSelectors } from "./selectors"; +import * as s3ProfilesManagement from "core/usecases/s3ProfilesManagement"; +import type { RouteParams } from "./selectors"; +import { assert, type Equals } from "tsafe/assert"; +import { actions } from "./state"; +import * as s3ProfileManagement from "core/usecases/s3ProfilesManagement"; +import { formatDuration } from "core/tools/timeFormat/formatDuration"; +import { id } from "tsafe/id"; +import { type S3Uri, parseS3Uri, stringifyS3Uri, getIsInside } from "core/tools/S3Uri"; +import { same } from "evt/tools/inDepth/same"; +import { createWaitForDebounce } from "core/tools/waitForDebounce"; +import type { State } from "./state"; +import { Evt } from "evt"; +import { onlyIfChanged } from "evt/operators"; +import { Deferred } from "evt/tools/Deferred"; + +const { waitForDebounce: waitForDebounce_notifyRouteParamsExternallyUpdated } = + createWaitForDebounce({ + delay: 10 + }); + +const { waitForDebounce: waitForDebounce_listPrefix } = createWaitForDebounce({ + delay: 300 +}); + +const erroredUploadBlobs: { + profileName: string; + s3Uri: S3Uri.NonTerminatedByDelimiter; + blob: Blob; +}[] = []; + +export const evtAskOverwriteConfirmation = Evt.create<{ + s3Uri: S3Uri.NonTerminatedByDelimiter; + resolveResponse: (params: { doOverwrite: boolean }) => void; +}>(); + +export const thunks = { + load: + (params: { routeParams: RouteParams }) => + (...args): { routeParams_toSet: RouteParams } => { + const [dispatch, getState] = args; + + const { routeParams } = params; + + if (routeParams.profile !== undefined) { + const { doesProfileExist } = dispatch( + s3ProfilesManagement.protectedThunks.changeAmbientProfile({ + profileName: routeParams.profile + }) + ); + + if (!doesProfileExist) { + return dispatch( + thunks.load({ routeParams: { s3UriWithoutScheme: "" } }) + ); + } + + dispatch( + thunks.listPrefix({ + s3Uri: + routeParams.s3UriWithoutScheme === "" + ? undefined + : parseS3Uri({ + value: `s3://${routeParams.s3UriWithoutScheme}`, + delimiter: "/" + }), + debounce: false + }) + ); + } + + return { + routeParams_toSet: privateSelectors.routeParams(getState()) + }; + }, + notifyRouteParamsExternallyUpdated: + (params: { routeParams: RouteParams }) => + async (...args) => { + const { routeParams } = params; + const [dispatch, getState] = args; + + // NOTE: We need a debounce here to avoid cycles since the ambient s3 profile + // and the s3 prefix location are not on the same slice and cannot be dispatched + // in a single action. + await waitForDebounce_notifyRouteParamsExternallyUpdated(); + + update_profile: { + const profileName = routeParams.profile; + + if (profileName === undefined) { + break update_profile; + } + + const profileName_current = + s3ProfileManagement.selectors.ambientS3Profile( + getState() + )?.profileName; + + if (profileName_current === profileName) { + break update_profile; + } + + const { doesProfileExist } = dispatch( + s3ProfilesManagement.protectedThunks.changeAmbientProfile({ + profileName + }) + ); + + assert(doesProfileExist); + } + + update_location: { + const s3Uri_current = privateSelectors.s3Uri(getState()); + + const s3Uri = + routeParams.s3UriWithoutScheme === "" + ? undefined + : parseS3Uri({ + value: `s3://${routeParams.s3UriWithoutScheme}`, + delimiter: "/" + }); + + if (same(s3Uri_current, s3Uri)) { + break update_location; + } + + dispatch( + thunks.listPrefix({ + s3Uri, + debounce: false + }) + ); + } + }, + updateSelectedS3Profile: + (params: { profileName: string }) => + async (...args) => { + const [dispatch] = args; + + const { profileName } = params; + + const { doesProfileExist } = dispatch( + s3ProfilesManagement.protectedThunks.changeAmbientProfile({ + profileName + }) + ); + + assert(doesProfileExist); + + dispatch( + thunks.listPrefix({ + s3Uri: undefined, + debounce: false + }) + ); + }, + deleteBookmark: (() => { + let isRunning = false; + + return (params: { s3Uri: S3Uri }) => + async (...args) => { + if (isRunning) { + return; + } + + isRunning = true; + + const { s3Uri } = params; + + const [dispatch, getState] = args; + + const s3Profile = + s3ProfileManagement.selectors.ambientS3Profile(getState()); + + assert(s3Profile !== undefined); + + await dispatch( + s3ProfilesManagement.protectedThunks.createDeleteOrUpdateBookmark({ + profileName: s3Profile.profileName, + s3Uri, + action: { + type: "delete" + } + }) + ); + + isRunning = false; + }; + })(), + updateBookmarkDisplayName: + (params: { s3Uri: S3Uri; displayName: string }) => + async (...args) => { + const { s3Uri, displayName } = params; + + const [dispatch, getState] = args; + + const s3Profile = s3ProfileManagement.selectors.ambientS3Profile(getState()); + + assert(s3Profile !== undefined); + + await dispatch( + s3ProfilesManagement.protectedThunks.createDeleteOrUpdateBookmark({ + profileName: s3Profile.profileName, + s3Uri, + action: { + type: "create or update", + displayName + } + }) + ); + }, + toggleIsS3UriBookmarked: (() => { + let isRunning = false; + + return (params: { + getDisplayName: (params: { + s3Uri: S3Uri; + }) => Promise< + { doProceed: true; displayName: string } | { doProceed: false } + >; + }) => + async (...args) => { + if (isRunning) { + return; + } + + isRunning = true; + + const { getDisplayName } = params; + + const [dispatch, getState] = args; + + const s3Uri = privateSelectors.s3Uri(getState()); + const s3Profile = + s3ProfileManagement.selectors.ambientS3Profile(getState()); + + assert(s3Profile !== undefined); + assert(s3Uri !== undefined); + + const isBookmarked = s3Profile.bookmarks.find(bookmark => + same(bookmark.s3Uri, s3Uri) + ); + + let action: + | { type: "create or update"; displayName: string | undefined } + | { type: "delete" }; + + if (isBookmarked) { + action = { type: "delete" }; + } else { + const resultOfGetDisplayName = await getDisplayName({ s3Uri }); + + if (!resultOfGetDisplayName.doProceed) { + isRunning = false; + return; + } + + action = { + type: "create or update", + displayName: resultOfGetDisplayName.displayName + }; + } + + await dispatch( + s3ProfilesManagement.protectedThunks.createDeleteOrUpdateBookmark({ + profileName: s3Profile.profileName, + s3Uri, + action + }) + ); + + isRunning = false; + }; + })(), + listPrefix: + (params: { s3Uri: S3Uri | undefined; debounce: boolean }) => + async (...args) => { + const [dispatch, getState] = args; + + const { s3Uri, debounce } = params; + + const profileName = privateSelectors.profileName(getState()); + + assert(profileName !== undefined); + + if (s3Uri === undefined) { + dispatch(actions.listingCleared({ profileName })); + return; + } + + { + const s3Uri_currentlyListing = + privateSelectors.s3Uri_currentlyListing(getState()); + + if ( + s3Uri_currentlyListing !== undefined && + same(s3Uri_currentlyListing, s3Uri) + ) { + return; + } + } + + infer_from_current_state: { + if (s3Uri.isDelimiterTerminated) { + break infer_from_current_state; + } + + const listedPrefix = privateSelectors.listedPrefix_state(getState()); + + if (listedPrefix === undefined) { + break infer_from_current_state; + } + + if (listedPrefix.current === undefined) { + break infer_from_current_state; + } + + { + const { isInside, isTopLevel } = getIsInside({ + s3UriPrefix: listedPrefix.current.s3Uri, + s3Uri + }); + + if (!isInside || !isTopLevel) { + break infer_from_current_state; + } + } + + dispatch( + actions.listingCompletedSuccessfully_inferFromCurrentState({ + profileName, + s3Uri + }) + ); + + return; + } + + dispatch(actions.listingStarted({ profileName, s3Uri })); + + const maybeCancel = async (): Promise => { + const s3Uri_currentlyListing = + privateSelectors.s3Uri_currentlyListing(getState()); + + if ( + s3Uri_currentlyListing === undefined || + !same(s3Uri_currentlyListing, s3Uri) + ) { + dispatch(actions.commandLogCancelled({ cmdId })); + await new Promise(() => {}); + } + }; + + { + const prDebounce = waitForDebounce_listPrefix(); + + if (debounce) { + await prDebounce; + } + } + + await maybeCancel(); + + const cmdId = Date.now(); + + dispatch( + actions.commandLogIssued({ + cmdId, + cmd: `aws s3 ls ${stringifyS3Uri(s3Uri)}` + }) + ); + + const s3Client = await dispatch( + s3ProfilesManagement.protectedThunks.getS3Client({ profileName }) + ); + + await maybeCancel(); + + const listObjectResult = await s3Client.listObjects({ s3Uri }); + + await maybeCancel(); + + dispatch( + actions.commandLogResponseReceived({ + cmdId, + resp: (() => { + if (listObjectResult.isSuccess) { + return [ + ...listObjectResult.prefixes.map( + s3Uri => + `PRE ${s3Uri.keySegments.at(-1)}${s3Uri.delimiter}` + ), + ...listObjectResult.objects.map( + ({ s3Uri }) => `OBJ ${s3Uri.keySegments.at(-1)}` + ) + ].join("\n"); + } + + switch (listObjectResult.errorCase) { + case "access denied": + return "Access Denied"; + case "no such bucket": + return "No Such Bucket"; + default: + assert>( + false + ); + } + })() + }) + ); + + if (!listObjectResult.isSuccess) { + dispatch( + actions.listingFailed({ + profileName, + s3Uri, + errorCase: listObjectResult.errorCase + }) + ); + return; + } + + dispatch( + actions.listingCompletedSuccessfully({ + profileName, + items: [ + ...listObjectResult.prefixes.map(s3Uri => + id({ + type: "prefix", + s3Uri + }) + ), + ...listObjectResult.objects.map(({ s3Uri, lastModified, size }) => + id({ + type: "object", + s3Uri, + lastModified, + size + }) + ) + ] + }) + ); + }, + retryPutObject: + (params: { profileName: string; s3Uri: S3Uri.NonTerminatedByDelimiter }) => + async (...args) => { + const { profileName, s3Uri } = params; + + const [dispatch] = args; + + const blobWrap = erroredUploadBlobs.find( + o => o.profileName === profileName && same(o.s3Uri, s3Uri) + ); + + assert(blobWrap !== undefined); + + await dispatch( + privateThunks.putObject({ + profileName, + s3Uri, + blob: blobWrap.blob + }) + ); + }, + flushUploads: + () => + (...args) => { + const [dispatch] = args; + + dispatch(actions.uploadFlushed()); + }, + + cancelUpload: + (params: { profileName: string; s3Uri: S3Uri.NonTerminatedByDelimiter }) => + (...args) => { + const { profileName, s3Uri } = params; + const [dispatch] = args; + + dispatch(actions.uploadCanceled({ profileName, s3Uri })); + }, + navigateBack: + () => + async (...args) => { + const [dispatch, getState] = args; + + const s3Uri = privateSelectors.s3Uri(getState()); + + assert(s3Uri !== undefined); + assert(s3Uri.keySegments.length !== 0); + + await dispatch( + thunks.listPrefix({ + s3Uri: { + ...s3Uri, + keySegments: s3Uri.keySegments.slice(0, -1), + isDelimiterTerminated: true + }, + debounce: false + }) + ); + }, + putObjects: + (params: { + files: { + relativePathSegments: string[]; + fileBasename: string; + blob: Blob; + }[]; + }) => + async (...args) => { + const { files } = params; + + const [dispatch, getState] = args; + + const profileName = privateSelectors.profileName(getState()); + + assert(profileName !== undefined); + + const s3Uri = privateSelectors.s3Uri(getState()); + + assert(s3Uri !== undefined); + + await Promise.all( + files + .map(file => ({ + file, + s3Uri_object: id({ + delimiter: s3Uri.delimiter, + bucket: s3Uri.bucket, + keySegments: [ + ...s3Uri.keySegments, + ...file.relativePathSegments, + file.fileBasename + ], + isDelimiterTerminated: false + }) + })) + .sort((a, b) => + stringifyS3Uri(a.s3Uri_object).localeCompare( + stringifyS3Uri(b.s3Uri_object) + ) + ) + .map(({ file, s3Uri_object }) => + dispatch( + privateThunks.putObject({ + profileName, + s3Uri: s3Uri_object, + blob: file.blob + }) + ) + ) + ); + }, + + createDirectory: + (params: { prefixSegment: string }) => + async (...args) => { + const { prefixSegment } = params; + + const [dispatch] = args; + + await dispatch( + thunks.putObjects({ + files: [ + { + relativePathSegments: [prefixSegment], + fileBasename: ".keep", + blob: new Blob(["This file tells that a directory exists"], { + type: "text/plain" + }) + } + ] + }) + ); + }, + + delete: + (params: { s3Uris: S3Uri[] }) => + async (...args): Promise => { + const { s3Uris } = params; + + const [dispatch, getState] = args; + + const profileName = privateSelectors.profileName(getState()); + + assert(profileName !== undefined); + + const s3Client = await dispatch( + s3ProfileManagement.protectedThunks.getS3Client({ profileName }) + ); + + const crawl = async (params: { + s3UriPrefix: S3Uri.TerminatedByDelimiter; + }): Promise => { + const { s3UriPrefix } = params; + + const result = await s3Client.listObjects({ s3Uri: s3UriPrefix }); + + assert(result.isSuccess); + + return [ + ...result.objects.map(({ s3Uri }) => s3Uri), + ...( + await Promise.all( + result.prefixes.map(s3Uri => crawl({ s3UriPrefix: s3Uri })) + ) + ).flat() + ]; + }; + + const deleteObject = async (params: { + s3Uri: S3Uri.NonTerminatedByDelimiter; + }) => { + const { s3Uri } = params; + + const cmdId = Math.random(); + + dispatch( + actions.commandLogIssued({ + cmdId, + cmd: `aws s3 rm ${stringifyS3Uri(s3Uri)}` + }) + ); + + await s3Client.deleteObject({ s3Uri }); + + dispatch( + actions.commandLogResponseReceived({ + cmdId, + resp: `Removed ${stringifyS3Uri(s3Uri)}` + }) + ); + }; + + await Promise.all( + s3Uris.map(async s3Uri => { + dispatch(actions.deletionStarted({ profileName, s3Uri })); + + const s3Uris = s3Uri.isDelimiterTerminated + ? await crawl({ s3UriPrefix: s3Uri }) + : [s3Uri]; + + await Promise.all(s3Uris.map(s3Uri => deleteObject({ s3Uri }))); + + dispatch(actions.deletionCompleted({ profileName, s3Uri })); + }) + ); + }, + getPreSignedUrl: + (params: { + s3Uri: S3Uri.NonTerminatedByDelimiter; + validityDurationSecond?: number; + }) => + async (...args): Promise => { + const { s3Uri, validityDurationSecond = 3_600 } = params; + + const [dispatch, getState] = args; + + const profileName = privateSelectors.profileName(getState()); + + assert(profileName !== undefined); + + const s3Client = await dispatch( + s3ProfileManagement.protectedThunks.getS3Client({ profileName }) + ); + + const cmdId = Date.now(); + dispatch( + actions.commandLogIssued({ + cmdId, + cmd: `aws s3 presign ${stringifyS3Uri(s3Uri)} --expires-in ${validityDurationSecond}` + }) + ); + + const downloadUrl = await s3Client.generateSignedDownloadUrl({ + s3Uri, + validityDurationSecond + }); + + dispatch( + actions.commandLogResponseReceived({ + cmdId, + resp: [ + `URL: ${downloadUrl.split("?")[0]}`, + `Expire: ${formatDuration({ + durationSeconds: validityDurationSecond, + t: undefined + })}`, + `Share: ${downloadUrl}` + ].join("\n") + }) + ); + + return downloadUrl; + } +} satisfies Thunks; + +export const privateThunks = { + putObject: + (params: { + profileName: string; + s3Uri: S3Uri.NonTerminatedByDelimiter; + blob: Blob; + }) => + async (...args) => { + const { profileName, s3Uri, blob } = params; + + const [dispatch, getState, { evtAction }] = args; + + { + const doesExist = await (async () => { + const s3Client = await dispatch( + s3ProfileManagement.protectedThunks.getS3Client({ profileName }) + ); + + const resultOfListObject = await s3Client.listObjects({ s3Uri }); + + if (!resultOfListObject.isSuccess) { + return false; + } + + return ( + resultOfListObject.objects.find(object => + same(object.s3Uri, s3Uri) + ) !== undefined + ); + })(); + + if (doesExist) { + const dDoOverwrite = new Deferred(); + + evtAskOverwriteConfirmation.post({ + s3Uri, + resolveResponse: ({ doOverwrite }) => { + dDoOverwrite.resolve(doOverwrite); + } + }); + + const doOverwrite = await dDoOverwrite.pr; + + if (!doOverwrite) { + return; + } + + await dispatch(thunks.delete({ s3Uris: [s3Uri] })); + } + } + + { + const i = erroredUploadBlobs.findIndex( + o => o.profileName === profileName && same(o.s3Uri, s3Uri) + ); + + if (i !== -1) { + erroredUploadBlobs.splice(i, 1); + } + } + + const cmdId = Date.now(); + + dispatch( + actions.commandLogIssued({ + cmdId, + cmd: `mc cp ./${s3Uri.keySegments.at(-1)} ${stringifyS3Uri(s3Uri)}` + }) + ); + + dispatch( + actions.putObjectStarted({ + profileName, + s3Uri: s3Uri, + size: blob.size + }) + ); + + const evtCancel = Evt.create(); + + const ctx = Evt.newCtx(); + + evtAction + .pipe(ctx, () => [privateSelectors.uploads(getState())]) + .pipe(onlyIfChanged()) + .attach(uploads => { + const upload = uploads.find( + upload => + upload.profileName === profileName && + same(upload.s3Uri, s3Uri) + ); + + if (upload === undefined) { + evtCancel.post(); + } + }); + + evtCancel.attachOnce(() => { + actions.commandLogCancelled({ + cmdId + }); + }); + + const s3Client = await dispatch( + s3ProfileManagement.protectedThunks.getS3Client({ profileName }) + ); + + const resultOfPutObject = await s3Client.putObject({ + s3Uri: s3Uri, + blob, + onUploadProgress: ({ uploadPercent }) => { + dispatch( + actions.putObjectProgressReported({ + profileName, + s3Uri: s3Uri, + completionPercent: uploadPercent + }) + ); + + dispatch( + actions.commandLogResponseReceived({ + cmdId, + resp: `... ${uploadPercent}% of ${blob.size} Bytes uploaded` + }) + ); + }, + evtCancel + }); + + ctx.done(); + + switch (resultOfPutObject.status) { + case "success": + break; + case "canceled": + dispatch( + actions.putObjectStopped({ + profileName, + s3Uri: s3Uri, + stoppedStatus: { case: "canceled" } + }) + ); + break; + case "failed": + console.error( + `Error uploading ${stringifyS3Uri(s3Uri)}: `, + resultOfPutObject.error + ); + + erroredUploadBlobs.push({ + profileName, + s3Uri: s3Uri, + blob + }); + + dispatch( + actions.putObjectStopped({ + profileName, + s3Uri: s3Uri, + stoppedStatus: { + case: "errored", + errorMessage: resultOfPutObject.error.message + } + }) + ); + break; + } + } +} satisfies Thunks; diff --git a/web/src/core/usecases/s3ConfigConnectionTest/index.ts b/web/src/core/usecases/s3ProfilesCreationUiController/index.ts similarity index 100% rename from web/src/core/usecases/s3ConfigConnectionTest/index.ts rename to web/src/core/usecases/s3ProfilesCreationUiController/index.ts diff --git a/web/src/core/usecases/s3ProfilesCreationUiController/selectors.ts b/web/src/core/usecases/s3ProfilesCreationUiController/selectors.ts new file mode 100644 index 000000000..e5fae77c9 --- /dev/null +++ b/web/src/core/usecases/s3ProfilesCreationUiController/selectors.ts @@ -0,0 +1,337 @@ +import type { State as RootState } from "core/bootstrap"; +import { createSelector } from "clean-architecture"; +import { name } from "./state"; +import { objectKeys } from "tsafe/objectKeys"; +import { assert } from "tsafe/assert"; +import { id } from "tsafe/id"; +import type { ProjectConfigs } from "core/usecases/projectManagement"; +import * as s3ProfilesManagement from "core/usecases/s3ProfilesManagement"; +import * as projectManagement from "core/usecases/projectManagement"; + +const readyState = (rootState: RootState) => { + const state = rootState[name]; + + if (state.stateDescription !== "ready") { + return null; + } + + return state; +}; + +const isReady = createSelector(readyState, state => state !== null); + +const formValues = createSelector(readyState, state => { + if (state === null) { + return null; + } + + return state.formValues; +}); + +const existingProfileNames = createSelector( + isReady, + createSelector(readyState, state => { + if (state === null) { + return null; + } + return state.creationTimeOfProfileToEdit; + }), + s3ProfilesManagement.selectors.s3Profiles, + (isReady, creationTimeOfProfileToEdit, s3Profiles) => { + if (!isReady) { + return null; + } + + assert(creationTimeOfProfileToEdit !== null); + + return s3Profiles + .filter(s3Profile => { + if (creationTimeOfProfileToEdit === undefined) { + return true; + } + + if (s3Profile.origin !== "created by user (or group project member)") { + return true; + } + + if (s3Profile.creationTime === creationTimeOfProfileToEdit) { + return false; + } + + return true; + }) + .map(s3Profile => s3Profile.profileName); + } +); + +const formValuesErrors = createSelector( + isReady, + formValues, + existingProfileNames, + (isReady, formValues, existingProfileNames) => { + if (!isReady) { + return null; + } + + assert(formValues !== null); + assert(existingProfileNames !== null); + + const out: Record< + keyof typeof formValues, + | "must be an url" + | "is required" + | "not a valid access key id" + | "profile name already used" + | undefined + > = {} as any; + + for (const key of objectKeys(formValues)) { + out[key] = (() => { + required_fields: { + if ( + !( + key === "url" || + key === "profileName" || + (!formValues.isAnonymous && + (key === "accessKeyId" || key === "secretAccessKey")) + ) + ) { + break required_fields; + } + + const value = formValues[key]; + + if ((value ?? "").trim() !== "") { + break required_fields; + } + + return "is required"; + } + + if (key === "url") { + const value = formValues[key]; + + try { + new URL(value.startsWith("http") ? value : `https://${value}`); + } catch { + return "must be an url"; + } + } + + if (key === "profileName") { + const value = formValues[key]; + + if (existingProfileNames.includes(value)) { + return "profile name already used"; + } + } + + return undefined; + })(); + } + + return out; + } +); + +const isFormSubmittable = createSelector( + isReady, + formValuesErrors, + (isReady, formValuesErrors) => { + if (!isReady) { + return null; + } + + assert(formValuesErrors !== null); + + return objectKeys(formValuesErrors).every( + key => formValuesErrors[key] === undefined + ); + } +); + +const formattedFormValuesUrl = createSelector( + isReady, + formValues, + formValuesErrors, + (isReady, formValues, formValuesErrors) => { + if (!isReady) { + return null; + } + assert(formValues !== null); + assert(formValuesErrors !== null); + + if (formValuesErrors.url !== undefined) { + return undefined; + } + + const trimmedValue = formValues.url.trim(); + + return trimmedValue.startsWith("http") ? trimmedValue : `https://${trimmedValue}`; + } +); + +const submittableFormValuesAsS3Profile_vault = createSelector( + isReady, + formValues, + formattedFormValuesUrl, + isFormSubmittable, + createSelector(readyState, state => { + if (state === null) { + return null; + } + return state.creationTimeOfProfileToEdit; + }), + projectManagement.protectedSelectors.projectConfig, + ( + isReady, + formValues, + formattedFormValuesUrl, + isFormSubmittable, + creationTimeOfProfileToEdit, + projectConfig + ) => { + if (!isReady) { + return null; + } + assert(formValues !== null); + assert(formattedFormValuesUrl !== null); + assert(isFormSubmittable !== null); + assert(creationTimeOfProfileToEdit !== null); + + if (!isFormSubmittable) { + return undefined; + } + + assert(formattedFormValuesUrl !== undefined); + + const s3Profile_vault_current = (() => { + if (creationTimeOfProfileToEdit === undefined) { + return undefined; + } + + const s3Profile_vault_current = projectConfig.s3Profiles.find( + s3Config => s3Config.creationTime === creationTimeOfProfileToEdit + ); + + assert(s3Profile_vault_current !== undefined); + + return s3Profile_vault_current; + })(); + + return id< + Omit & { + creationTime: number | undefined; + } + >({ + creationTime: + s3Profile_vault_current === undefined + ? undefined + : s3Profile_vault_current.creationTime, + profileName: formValues.profileName.trim(), + url: formattedFormValuesUrl, + region: formValues.region?.trim(), + pathStyleAccess: formValues.pathStyleAccess, + credentials: (() => { + if (formValues.isAnonymous) { + return undefined; + } + + assert(formValues.accessKeyId !== undefined); + assert(formValues.secretAccessKey !== undefined); + + return { + accessKeyId: formValues.accessKeyId, + secretAccessKey: formValues.secretAccessKey, + sessionToken: formValues.sessionToken + }; + })(), + bookmarks: + s3Profile_vault_current === undefined + ? [] + : s3Profile_vault_current.bookmarks + }); + } +); + +const urlStylesExamples = createSelector( + isReady, + formattedFormValuesUrl, + (isReady, formattedFormValuesUrl) => { + if (!isReady) { + return null; + } + + assert(formattedFormValuesUrl !== null); + + if (formattedFormValuesUrl === undefined) { + return undefined; + } + + const urlObject = new URL(formattedFormValuesUrl); + + const bucketName = "mybucket"; + const objectNamePrefix = "my/object/name/prefix/"; + + const domain = formattedFormValuesUrl + .split(urlObject.protocol)[1] + .split("//")[1] + .replace(/\/$/, ""); + + return { + pathStyle: `${domain}/${bucketName}/${objectNamePrefix}`, + virtualHostedStyle: `${bucketName}.${domain}/${objectNamePrefix}` + }; + } +); + +const main = createSelector( + isReady, + formValues, + formValuesErrors, + isFormSubmittable, + urlStylesExamples, + createSelector(readyState, state => { + if (state === null) { + return null; + } + return state.creationTimeOfProfileToEdit !== undefined; + }), + ( + isReady, + formValues, + formValuesErrors, + isFormSubmittable, + urlStylesExamples, + isEditionOfAnExistingConfig + ) => { + if (!isReady) { + return { + isReady: false as const + }; + } + + assert(formValues !== null); + assert(formValuesErrors !== null); + assert(isFormSubmittable !== null); + assert(urlStylesExamples !== null); + assert(isEditionOfAnExistingConfig !== null); + + return { + isReady: true, + formValues, + formValuesErrors, + isFormSubmittable, + urlStylesExamples, + isEditionOfAnExistingConfig + }; + } +); + +export const privateSelectors = { + formattedFormValuesUrl, + submittableFormValuesAsS3Profile_vault, + formValuesErrors +}; + +export const selectors = { main }; diff --git a/web/src/core/usecases/s3ConfigCreation/state.ts b/web/src/core/usecases/s3ProfilesCreationUiController/state.ts similarity index 72% rename from web/src/core/usecases/s3ConfigCreation/state.ts rename to web/src/core/usecases/s3ProfilesCreationUiController/state.ts index 170dd70a6..98aaed094 100644 --- a/web/src/core/usecases/s3ConfigCreation/state.ts +++ b/web/src/core/usecases/s3ProfilesCreationUiController/state.ts @@ -12,17 +12,14 @@ export namespace State { export type Ready = { stateDescription: "ready"; formValues: Ready.FormValues; - action: - | { type: "update existing config"; s3ConfigId: string } - | { type: "create new config"; creationTime: number }; + creationTimeOfProfileToEdit: number | undefined; }; export namespace Ready { export type FormValues = { - friendlyName: string; + profileName: string; url: string; region: string | undefined; - workingDirectoryPath: string; pathStyleAccess: boolean; isAnonymous: boolean; accessKeyId: string | undefined; @@ -39,7 +36,7 @@ export type ChangeValueParams< value: State.Ready.FormValues[K]; }; -export const name = "s3ConfigCreation"; +export const name = "s3ProfilesCreationUiController"; export const { reducer, actions } = createUsecaseActions({ name, @@ -55,26 +52,17 @@ export const { reducer, actions } = createUsecaseActions({ payload }: { payload: { - s3ConfigIdToEdit: string | undefined; + creationTimeOfProfileToEdit: number | undefined; initialFormValues: State.Ready["formValues"]; }; } ) => { - const { s3ConfigIdToEdit, initialFormValues } = payload; + const { creationTimeOfProfileToEdit, initialFormValues } = payload; return id({ stateDescription: "ready", formValues: initialFormValues, - action: - s3ConfigIdToEdit === undefined - ? { - type: "create new config", - creationTime: Date.now() - } - : { - type: "update existing config", - s3ConfigId: s3ConfigIdToEdit - } + creationTimeOfProfileToEdit }); }, formValueChanged: ( diff --git a/web/src/core/usecases/s3ProfilesCreationUiController/thunks.ts b/web/src/core/usecases/s3ProfilesCreationUiController/thunks.ts new file mode 100644 index 000000000..97b84d2c5 --- /dev/null +++ b/web/src/core/usecases/s3ProfilesCreationUiController/thunks.ts @@ -0,0 +1,186 @@ +import type { Thunks } from "core/bootstrap"; +import { actions, type State, type ChangeValueParams } from "./state"; +import { assert } from "tsafe/assert"; +import { privateSelectors } from "./selectors"; +import * as s3ProfilesManagement from "core/usecases/s3ProfilesManagement"; +import * as deploymentRegionManagement from "core/usecases/deploymentRegionManagement"; + +export const thunks = { + initialize: + (params: { + // NOTE: Undefined for creation + profileName_toUpdate: string | undefined; + }) => + async (...args) => { + const { profileName_toUpdate } = params; + + const [dispatch, getState] = args; + + const s3Profiles = s3ProfilesManagement.selectors.s3Profiles(getState()); + + update_existing_config: { + if (profileName_toUpdate === undefined) { + break update_existing_config; + } + + const s3Profile = s3Profiles.find( + s3Profile => s3Profile.profileName === profileName_toUpdate + ); + + assert(s3Profile !== undefined); + assert(s3Profile.origin === "created by user (or group project member)"); + + dispatch( + actions.initialized({ + creationTimeOfProfileToEdit: s3Profile.creationTime, + initialFormValues: { + profileName: s3Profile.profileName, + url: s3Profile.paramsOfCreateS3Client.url, + region: s3Profile.paramsOfCreateS3Client.region, + pathStyleAccess: + s3Profile.paramsOfCreateS3Client.pathStyleAccess, + ...(() => { + if ( + s3Profile.paramsOfCreateS3Client.credentials === + undefined + ) { + return { + isAnonymous: true, + accessKeyId: undefined, + secretAccessKey: undefined, + sessionToken: undefined + }; + } + + return { + isAnonymous: false, + accessKeyId: + s3Profile.paramsOfCreateS3Client.credentials + .accessKeyId, + secretAccessKey: + s3Profile.paramsOfCreateS3Client.credentials + .secretAccessKey, + sessionToken: + s3Profile.paramsOfCreateS3Client.credentials + .sessionToken + }; + })() + } + }) + ); + + return; + } + + const { s3Profiles_defaultValuesOfCreationForm } = + deploymentRegionManagement.selectors.currentDeploymentRegion(getState()); + + if (s3Profiles_defaultValuesOfCreationForm === undefined) { + dispatch( + actions.initialized({ + creationTimeOfProfileToEdit: undefined, + initialFormValues: { + profileName: "", + url: "", + region: undefined, + pathStyleAccess: false, + isAnonymous: true, + accessKeyId: undefined, + secretAccessKey: undefined, + sessionToken: undefined + } + }) + ); + return; + } + + dispatch( + actions.initialized({ + creationTimeOfProfileToEdit: undefined, + initialFormValues: { + profileName: "", + url: s3Profiles_defaultValuesOfCreationForm.url, + region: s3Profiles_defaultValuesOfCreationForm.region, + pathStyleAccess: + s3Profiles_defaultValuesOfCreationForm.pathStyleAccess ?? + false, + isAnonymous: false, + accessKeyId: undefined, + secretAccessKey: undefined, + sessionToken: undefined + } + }) + ); + }, + reset: + () => + (...args) => { + const [dispatch] = args; + + dispatch(actions.stateResetToNotInitialized()); + }, + submit: + () => + async (...args) => { + const [dispatch, getState] = args; + + const s3Profile_vault = + privateSelectors.submittableFormValuesAsS3Profile_vault(getState()); + + assert(s3Profile_vault !== null); + assert(s3Profile_vault !== undefined); + + await dispatch( + s3ProfilesManagement.protectedThunks.createOrUpdateS3Profile({ + s3Profile_vault: { + ...s3Profile_vault, + creationTime: s3Profile_vault.creationTime ?? Date.now() + } + }) + ); + + dispatch(actions.stateResetToNotInitialized()); + }, + changeValue: + (params: ChangeValueParams) => + async (...args) => { + const { key, value } = params; + + const [dispatch, getState] = args; + dispatch(actions.formValueChanged({ key, value })); + + preset_pathStyleAccess: { + if (key !== "url") { + break preset_pathStyleAccess; + } + + const url = privateSelectors.formattedFormValuesUrl(getState()); + + assert(url !== null); + + if (url === undefined) { + break preset_pathStyleAccess; + } + + if (url.toLowerCase().includes("amazonaws.com")) { + dispatch( + actions.formValueChanged({ + key: "pathStyleAccess", + value: false + }) + ); + break preset_pathStyleAccess; + } + + if (url.toLocaleLowerCase().includes("minio")) { + dispatch( + actions.formValueChanged({ + key: "pathStyleAccess", + value: true + }) + ); + break preset_pathStyleAccess; + } + } + } +} satisfies Thunks; diff --git a/web/src/core/usecases/s3ProfilesManagement/decoupledLogic/resolveTemplatedBookmark.ts b/web/src/core/usecases/s3ProfilesManagement/decoupledLogic/resolveTemplatedBookmark.ts new file mode 100644 index 000000000..74487420a --- /dev/null +++ b/web/src/core/usecases/s3ProfilesManagement/decoupledLogic/resolveTemplatedBookmark.ts @@ -0,0 +1,139 @@ +import type { DeploymentRegion } from "core/ports/OnyxiaApi"; +import { id } from "tsafe/id"; +import type { LocalizedString } from "ui/i18n"; +import { z } from "zod"; +import { getValueAtPath } from "core/tools/Stringifyable"; +import { type S3Uri, parseS3Uri } from "core/tools/S3Uri"; + +export type ResolvedTemplateBookmark = { + title: LocalizedString; + description: LocalizedString | undefined; + tags: LocalizedString[]; + s3Uri: S3Uri; + forProfileNames: string[]; +}; + +export async function resolveTemplatedBookmark(params: { + bookmark_region: DeploymentRegion.S3Profile.Bookmark; + getDecodedIdToken: () => Promise>; +}): Promise { + const { bookmark_region, getDecodedIdToken } = params; + + if (bookmark_region.claimName === undefined) { + return [ + id({ + s3Uri: parseS3Uri({ + value: bookmark_region.s3UriStr_templated, + delimiter: "/" + }), + title: bookmark_region.title, + description: bookmark_region.description, + tags: bookmark_region.tags, + forProfileNames: bookmark_region.forProfileNames + }) + ]; + } + + const { claimName, excludedClaimPattern, includedClaimPattern } = bookmark_region; + + const decodedIdToken = await getDecodedIdToken(); + + const claimValue_arr: string[] = (() => { + let claimValue_untrusted: unknown = (() => { + const candidate = decodedIdToken[claimName]; + + if (candidate !== undefined) { + return candidate; + } + + const claimPath = claimName.split("."); + + if (claimPath.length === 1) { + return undefined; + } + + return getValueAtPath({ + // @ts-expect-error: We know decodedIdToken is Stringifyable + stringifyableObjectOrArray: decodedIdToken, + doDeleteFromSource: false, + doFailOnUnresolved: false, + path: claimPath + }); + })(); + + if (!claimValue_untrusted) { + return []; + } + + let claimValue: string | string[]; + + try { + claimValue = z + .union([z.string(), z.array(z.string())]) + .parse(claimValue_untrusted); + } catch (error) { + throw new Error( + [ + `decodedIdToken -> ${claimName} is supposed to be`, + `string or array of string`, + `The decoded id token is:`, + JSON.stringify(decodedIdToken, null, 2) + ].join(" "), + { cause: error } + ); + } + + return claimValue instanceof Array ? claimValue : [claimValue]; + })(); + + const includedRegex = + includedClaimPattern !== undefined ? new RegExp(includedClaimPattern) : /^(.+)$/; + const excludedRegex = + excludedClaimPattern !== undefined ? new RegExp(excludedClaimPattern) : undefined; + + return claimValue_arr + .map(value => { + if (excludedRegex !== undefined && excludedRegex.test(value)) { + return undefined; + } + + const match = includedRegex.exec(value); + + if (match === null) { + return undefined; + } + + const substituteTemplateString = (str: string) => + str.replace(/\$(\d+)/g, (_, i) => match[parseInt(i)] ?? ""); + + const substituteLocalizedString = ( + locStr: LocalizedString + ): LocalizedString => { + if (typeof locStr === "string") { + return substituteTemplateString(locStr); + } + return Object.fromEntries( + Object.entries(locStr) + .filter(([, value]) => value !== undefined) + .map(([lang, value]) => [lang, substituteTemplateString(value)]) + ); + }; + + return id({ + s3Uri: parseS3Uri({ + value: substituteTemplateString(bookmark_region.s3UriStr_templated), + delimiter: "/" + }), + title: substituteLocalizedString(bookmark_region.title), + description: + bookmark_region.description === undefined + ? undefined + : substituteLocalizedString(bookmark_region.description), + tags: bookmark_region.tags.map(tag => substituteLocalizedString(tag)), + forProfileNames: bookmark_region.forProfileNames.map(profileName => + substituteTemplateString(profileName) + ) + }); + }) + .filter(x => x !== undefined); +} diff --git a/web/src/core/usecases/s3ProfilesManagement/decoupledLogic/resolveTemplatedStsRole.ts b/web/src/core/usecases/s3ProfilesManagement/decoupledLogic/resolveTemplatedStsRole.ts new file mode 100644 index 000000000..99ef632c4 --- /dev/null +++ b/web/src/core/usecases/s3ProfilesManagement/decoupledLogic/resolveTemplatedStsRole.ts @@ -0,0 +1,107 @@ +import type { DeploymentRegion } from "core/ports/OnyxiaApi"; +import { id } from "tsafe/id"; +import { z } from "zod"; +import { getValueAtPath } from "core/tools/Stringifyable"; + +export type ResolvedTemplateStsRole = { + roleARN: string; + roleSessionName: string; + profileName: string; +}; + +export async function resolveTemplatedStsRole(params: { + stsRole_region: DeploymentRegion.S3Profile.StsRole; + getDecodedIdToken: () => Promise>; +}): Promise { + const { stsRole_region, getDecodedIdToken } = params; + + if (stsRole_region.claimName === undefined) { + return [ + id({ + roleARN: stsRole_region.roleARN, + roleSessionName: stsRole_region.roleSessionName, + profileName: stsRole_region.profileName + }) + ]; + } + + const { claimName, excludedClaimPattern, includedClaimPattern } = stsRole_region; + + const decodedIdToken = await getDecodedIdToken(); + + const claimValue_arr: string[] = (() => { + let claimValue_untrusted: unknown = (() => { + const candidate = decodedIdToken[claimName]; + + if (candidate !== undefined) { + return candidate; + } + + const claimPath = claimName.split("."); + + if (claimPath.length === 1) { + return undefined; + } + + return getValueAtPath({ + // @ts-expect-error: We know decodedIdToken is Stringifyable + stringifyableObjectOrArray: decodedIdToken, + doDeleteFromSource: false, + doFailOnUnresolved: false, + path: claimPath + }); + })(); + + if (!claimValue_untrusted) { + return []; + } + + let claimValue: string | string[]; + + try { + claimValue = z + .union([z.string(), z.array(z.string())]) + .parse(claimValue_untrusted); + } catch (error) { + throw new Error( + [ + `decodedIdToken -> ${claimName} is supposed to be`, + `string or array of string`, + `The decoded id token is:`, + JSON.stringify(decodedIdToken, null, 2) + ].join(" "), + { cause: error } + ); + } + + return claimValue instanceof Array ? claimValue : [claimValue]; + })(); + + const includedRegex = + includedClaimPattern !== undefined ? new RegExp(includedClaimPattern) : /^(.+)$/; + const excludedRegex = + excludedClaimPattern !== undefined ? new RegExp(excludedClaimPattern) : undefined; + + return claimValue_arr + .map(value => { + if (excludedRegex !== undefined && excludedRegex.test(value)) { + return undefined; + } + + const match = includedRegex.exec(value); + + if (match === null) { + return undefined; + } + + const substituteTemplateString = (str: string) => + str.replace(/\$(\d+)/g, (_, i) => match[parseInt(i)] ?? ""); + + return id({ + roleARN: substituteTemplateString(stsRole_region.roleARN), + roleSessionName: substituteTemplateString(stsRole_region.roleSessionName), + profileName: substituteTemplateString(stsRole_region.profileName) + }); + }) + .filter(x => x !== undefined); +} diff --git a/web/src/core/usecases/s3ProfilesManagement/decoupledLogic/s3Profiles.ts b/web/src/core/usecases/s3ProfilesManagement/decoupledLogic/s3Profiles.ts new file mode 100644 index 000000000..596ff53f0 --- /dev/null +++ b/web/src/core/usecases/s3ProfilesManagement/decoupledLogic/s3Profiles.ts @@ -0,0 +1,245 @@ +import * as projectManagement from "core/usecases/projectManagement"; +import type { DeploymentRegion } from "core/ports/OnyxiaApi/DeploymentRegion"; +import type { ParamsOfCreateS3Client } from "core/adapters/s3Client"; +import { assert } from "tsafe"; +import type { LocalizedString } from "core/ports/OnyxiaApi"; +import type { ResolvedTemplateBookmark } from "./resolveTemplatedBookmark"; +import type { ResolvedTemplateStsRole } from "./resolveTemplatedStsRole"; +import type { S3Uri } from "core/tools/S3Uri"; +import { parseUserConfigsS3BookmarksStr } from "./userConfigsS3Bookmarks"; + +export type S3Profile = S3Profile.DefinedInRegion | S3Profile.CreatedByUser; + +export namespace S3Profile { + type Common = { + profileName: string; + bookmarks: Bookmark[]; + }; + + export type DefinedInRegion = Common & { + origin: "defined in region"; + paramsOfCreateS3Client: ParamsOfCreateS3Client.Sts; + }; + + export type CreatedByUser = Common & { + origin: "created by user (or group project member)"; + creationTime: number; + paramsOfCreateS3Client: ParamsOfCreateS3Client.NoSts; + }; + + export type Bookmark = { + isReadonly: boolean; + displayName: LocalizedString | undefined; + s3Uri: S3Uri; + }; +} + +export function aggregateS3ProfilesFromVaultAndRegionIntoAnUnifiedSet(params: { + fromVault: { + s3Profiles: projectManagement.ProjectConfigs.S3Profile[]; + userConfigs_s3BookmarksStr: string | null; + }; + fromRegion: { + s3Profiles: DeploymentRegion.S3Profile[]; + // NOTE: The resolvedXXX can be undefined only when the function is used to + // the stablish the default profiles (for explorer and services) + resolvedTemplatedBookmarks: + | { + correspondingS3ConfigIndexInRegion: number; + bookmarks: ResolvedTemplateBookmark[]; + }[] + | undefined; + resolvedTemplatedStsRoles: + | { + correspondingS3ConfigIndexInRegion: number; + stsRoles: ResolvedTemplateStsRole[]; + }[] + | undefined; + }; +}): S3Profile[] { + const { fromVault, fromRegion } = params; + + const s3Profiles: S3Profile[] = [ + ...fromVault.s3Profiles + .map((c): S3Profile.CreatedByUser => { + const url = c.url; + const pathStyleAccess = c.pathStyleAccess; + const region = c.region; + + const paramsOfCreateS3Client: ParamsOfCreateS3Client.NoSts = { + url, + pathStyleAccess, + isStsEnabled: false, + region, + credentials: c.credentials + }; + + return { + origin: "created by user (or group project member)", + profileName: c.profileName, + creationTime: c.creationTime, + paramsOfCreateS3Client, + bookmarks: (c.bookmarks ?? []).map(({ displayName, s3Uri }) => ({ + displayName, + s3Uri, + isReadonly: false + })) + }; + }) + .sort((a, b) => b.creationTime - a.creationTime), + ...fromRegion.s3Profiles + .map((c, index): S3Profile.DefinedInRegion[] => { + const resolvedTemplatedBookmarks_forThisProfile = (() => { + if (fromRegion.resolvedTemplatedBookmarks === undefined) { + return []; + } + + const entry = fromRegion.resolvedTemplatedBookmarks.find( + e => e.correspondingS3ConfigIndexInRegion === index + ); + + assert(entry !== undefined); + + return entry.bookmarks; + })(); + + const buildFromRole = (params: { + resolvedTemplatedStsRole: ResolvedTemplateStsRole | undefined; + }): S3Profile.DefinedInRegion => { + const { resolvedTemplatedStsRole } = params; + + const paramsOfCreateS3Client: ParamsOfCreateS3Client.Sts = { + url: c.url, + pathStyleAccess: c.pathStyleAccess, + isStsEnabled: true, + stsUrl: c.sts.url, + region: c.region, + oidcParams: c.sts.oidcParams, + durationSeconds: c.sts.durationSeconds, + role: resolvedTemplatedStsRole + }; + + const profileName = (() => { + if (resolvedTemplatedStsRole === undefined) { + assert(c.profileName !== undefined); + return c.profileName; + } + + return resolvedTemplatedStsRole.profileName; + })(); + + return { + origin: "defined in region", + profileName, + bookmarks: [ + ...resolvedTemplatedBookmarks_forThisProfile + .filter(({ forProfileNames }) => { + if (forProfileNames.length === 0) { + return true; + } + + if (resolvedTemplatedStsRole === undefined) { + return false; + } + + const getDoMatch = (params: { + stringWithWildcards: string; + candidate: string; + }): boolean => { + const { stringWithWildcards, candidate } = params; + + if (!stringWithWildcards.includes("*")) { + return stringWithWildcards === candidate; + } + + const escapedRegex = stringWithWildcards + .replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + .replace(/\\\*/g, ".*"); + + return new RegExp(`^${escapedRegex}$`).test( + candidate + ); + }; + + return forProfileNames.some(profileName => + getDoMatch({ + stringWithWildcards: profileName, + candidate: + resolvedTemplatedStsRole.profileName + }) + ); + }) + .map(({ title, s3Uri }) => ({ + isReadonly: true, + displayName: title, + s3Uri + })), + ...parseUserConfigsS3BookmarksStr({ + userConfigs_s3BookmarksStr: + fromVault.userConfigs_s3BookmarksStr + }) + .filter(entry => entry.profileName === profileName) + .map(entry => ({ + isReadonly: false, + displayName: entry.displayName ?? undefined, + s3Uri: entry.s3Uri + })) + ], + paramsOfCreateS3Client + }; + }; + + const resolvedTemplatedStsRoles_forThisProfile = (() => { + if (fromRegion.resolvedTemplatedStsRoles === undefined) { + return []; + } + + const entry = fromRegion.resolvedTemplatedStsRoles.find( + e => e.correspondingS3ConfigIndexInRegion === index + ); + + assert(entry !== undefined); + + return entry.stsRoles; + })(); + + if (resolvedTemplatedStsRoles_forThisProfile.length === 0) { + return [buildFromRole({ resolvedTemplatedStsRole: undefined })]; + } + + return resolvedTemplatedStsRoles_forThisProfile.map( + resolvedTemplatedStsRole => + buildFromRole({ resolvedTemplatedStsRole }) + ); + }) + .flat() + ]; + + for (const s3Profile of [...s3Profiles].sort((a, b) => { + if (a.origin === b.origin) { + return 0; + } + + return a.origin === "defined in region" ? -1 : 1; + })) { + const s3Profiles_conflicting = s3Profiles.filter( + s3Profile_i => + s3Profile_i !== s3Profile && + s3Profile_i.profileName === s3Profile.profileName + ); + + if (s3Profiles_conflicting.length === 0) { + continue; + } + + console.warn(`The is more than one s3Profile named: ${s3Profile.profileName}`); + + for (const s3Profile_conflicting of s3Profiles_conflicting) { + const i = s3Profiles.indexOf(s3Profile_conflicting); + + s3Profiles.splice(i, 1); + } + } + + return s3Profiles; +} diff --git a/web/src/core/usecases/s3ProfilesManagement/decoupledLogic/userConfigsS3Bookmarks.ts b/web/src/core/usecases/s3ProfilesManagement/decoupledLogic/userConfigsS3Bookmarks.ts new file mode 100644 index 000000000..a7677cbfc --- /dev/null +++ b/web/src/core/usecases/s3ProfilesManagement/decoupledLogic/userConfigsS3Bookmarks.ts @@ -0,0 +1,57 @@ +import { type S3Uri, zS3Uri } from "core/tools/S3Uri"; +import { z } from "zod"; +import { assert, type Equals, id, is } from "tsafe"; +import type { OptionalIfCanBeUndefined } from "core/tools/OptionalIfCanBeUndefined"; + +export type UserConfigs_S3Bookmark = { + profileName: string; + displayName: string | undefined; + s3Uri: S3Uri; +}; + +const zUserProfileS3Bookmark = (() => { + type TargetType = UserConfigs_S3Bookmark; + + const zTargetType = z.object({ + profileName: z.string(), + displayName: z.union([z.string(), z.undefined()]), + s3Uri: zS3Uri + }); + + type InferredType = z.infer; + + assert>>; + + // @ts-expect-error + return id>(zTargetType); +})(); + +export function parseUserConfigsS3BookmarksStr(params: { + userConfigs_s3BookmarksStr: string | null; +}): UserConfigs_S3Bookmark[] { + const { userConfigs_s3BookmarksStr } = params; + + if (userConfigs_s3BookmarksStr === null) { + return []; + } + + const userProfileS3Bookmarks: unknown = JSON.parse(userConfigs_s3BookmarksStr); + + try { + z.array(zUserProfileS3Bookmark).parse(userProfileS3Bookmarks); + } catch { + return []; + } + + assert(is(userProfileS3Bookmarks)); + + return userProfileS3Bookmarks; +} + +export function serializeUserConfigsS3Bookmarks(params: { + userConfigs_s3Bookmarks: UserConfigs_S3Bookmark[]; +}) { + const { userConfigs_s3Bookmarks } = params; + + return JSON.stringify(userConfigs_s3Bookmarks); +} diff --git a/web/src/core/usecases/s3ConfigCreation/index.ts b/web/src/core/usecases/s3ProfilesManagement/index.ts similarity index 56% rename from web/src/core/usecases/s3ConfigCreation/index.ts rename to web/src/core/usecases/s3ProfilesManagement/index.ts index 3f3843384..84fe07fe2 100644 --- a/web/src/core/usecases/s3ConfigCreation/index.ts +++ b/web/src/core/usecases/s3ProfilesManagement/index.ts @@ -1,3 +1,4 @@ export * from "./state"; export * from "./selectors"; export * from "./thunks"; +export type { S3Profile } from "./decoupledLogic/s3Profiles"; diff --git a/web/src/core/usecases/s3ProfilesManagement/selectors.ts b/web/src/core/usecases/s3ProfilesManagement/selectors.ts new file mode 100644 index 000000000..89c57e16c --- /dev/null +++ b/web/src/core/usecases/s3ProfilesManagement/selectors.ts @@ -0,0 +1,101 @@ +import { createSelector } from "clean-architecture"; +import * as deploymentRegionManagement from "core/usecases/deploymentRegionManagement"; +import * as projectManagement from "core/usecases/projectManagement"; +import * as userConfigs from "core/usecases/userConfigs"; +import { + type S3Profile, + aggregateS3ProfilesFromVaultAndRegionIntoAnUnifiedSet +} from "./decoupledLogic/s3Profiles"; +import { name } from "./state"; +import type { State as RootState } from "core/bootstrap"; +import * as userAuthentication from "core/usecases/userAuthentication"; + +const state = (rootState: RootState) => rootState[name]; + +const resolvedTemplatedBookmarks = createSelector( + state, + state => state.resolvedTemplatedBookmarks +); + +const resolvedTemplatedStsRoles = createSelector( + state, + state => state.resolvedTemplatedStsRoles +); + +const userConfigs_s3BookmarksStr = createSelector( + userConfigs.selectors.userConfigs, + userConfigs => userConfigs.s3BookmarksStr +); + +const s3Profiles = createSelector( + createSelector( + projectManagement.protectedSelectors.projectConfig, + projectConfig => projectConfig.s3Profiles + ), + createSelector( + deploymentRegionManagement.selectors.currentDeploymentRegion, + deploymentRegion => deploymentRegion.s3Profiles + ), + resolvedTemplatedBookmarks, + resolvedTemplatedStsRoles, + userConfigs_s3BookmarksStr, + ( + s3Profiles_vault, + s3Profiles_region, + resolvedTemplatedBookmarks, + resolvedTemplatedStsRoles, + userConfigs_s3BookmarksStr + ): S3Profile[] => + aggregateS3ProfilesFromVaultAndRegionIntoAnUnifiedSet({ + fromVault: { + s3Profiles: s3Profiles_vault, + userConfigs_s3BookmarksStr + }, + fromRegion: { + s3Profiles: s3Profiles_region, + resolvedTemplatedBookmarks, + resolvedTemplatedStsRoles + } + }) +); + +/** Can be used even when not authenticated */ +const isS3ExplorerEnabled = (rootState: RootState) => { + const { isUserLoggedIn } = userAuthentication.selectors.main(rootState); + + if (!isUserLoggedIn) { + return ( + deploymentRegionManagement.selectors.currentDeploymentRegion(rootState) + .s3Profiles.length !== 0 + ); + } + + return s3Profiles(rootState).length !== 0; +}; + +const ambientS3Profile = createSelector( + s3Profiles, + createSelector(state, state => state.ambientProfileName), + (s3Profiles, ambientProfileName) => { + return ( + s3Profiles.find( + ambientProfileName === undefined + ? () => false + : s3Profiles => s3Profiles.profileName === ambientProfileName + ) ?? + s3Profiles.find(s3Profile => s3Profile.profileName === "default") ?? + s3Profiles.find(s3Profile => s3Profile.origin === "defined in region") ?? + s3Profiles.find(() => true) + ); + } +); + +export const protectedSelectors = { + resolvedTemplatedBookmarks +}; + +export const selectors = { + isS3ExplorerEnabled, + s3Profiles, + ambientS3Profile +}; diff --git a/web/src/core/usecases/s3ProfilesManagement/state.ts b/web/src/core/usecases/s3ProfilesManagement/state.ts new file mode 100644 index 000000000..26999f9f0 --- /dev/null +++ b/web/src/core/usecases/s3ProfilesManagement/state.ts @@ -0,0 +1,56 @@ +import { + createUsecaseActions, + createObjectThatThrowsIfAccessed +} from "clean-architecture"; +import type { ResolvedTemplateBookmark } from "./decoupledLogic/resolveTemplatedBookmark"; +import type { ResolvedTemplateStsRole } from "./decoupledLogic/resolveTemplatedStsRole"; + +type State = { + ambientProfileName: string | undefined; + resolvedTemplatedBookmarks: { + correspondingS3ConfigIndexInRegion: number; + bookmarks: ResolvedTemplateBookmark[]; + }[]; + resolvedTemplatedStsRoles: { + correspondingS3ConfigIndexInRegion: number; + stsRoles: ResolvedTemplateStsRole[]; + }[]; +}; + +export const name = "s3ProfilesManagement"; + +export const { reducer, actions } = createUsecaseActions({ + name, + initialState: createObjectThatThrowsIfAccessed(), + reducers: { + initialized: ( + _, + { + payload + }: { + payload: { + resolvedTemplatedBookmarks: State["resolvedTemplatedBookmarks"]; + resolvedTemplatedStsRoles: State["resolvedTemplatedStsRoles"]; + }; + } + ) => { + const { resolvedTemplatedBookmarks, resolvedTemplatedStsRoles } = payload; + + const state: State = { + ambientProfileName: undefined, + resolvedTemplatedBookmarks, + resolvedTemplatedStsRoles + }; + + return state; + }, + ambientProfileChanged: ( + state, + { payload }: { payload: { profileName: string } } + ) => { + const { profileName } = payload; + + state.ambientProfileName = profileName; + } + } +}); diff --git a/web/src/core/usecases/s3ProfilesManagement/thunks.ts b/web/src/core/usecases/s3ProfilesManagement/thunks.ts new file mode 100644 index 000000000..5772b32bc --- /dev/null +++ b/web/src/core/usecases/s3ProfilesManagement/thunks.ts @@ -0,0 +1,468 @@ +import type { Thunks } from "core/bootstrap"; +import { selectors, protectedSelectors } from "./selectors"; +import * as projectManagement from "core/usecases/projectManagement"; +import { assert } from "tsafe/assert"; +import type { S3Client } from "core/ports/S3Client"; +import structuredClone from "@ungap/structured-clone"; +import * as deploymentRegionManagement from "core/usecases/deploymentRegionManagement"; +import { fnv1aHashToHex } from "core/tools/fnv1aHashToHex"; +import { resolveTemplatedBookmark } from "./decoupledLogic/resolveTemplatedBookmark"; +import { resolveTemplatedStsRole } from "./decoupledLogic/resolveTemplatedStsRole"; +import { actions } from "./state"; +import type { S3Profile } from "./decoupledLogic/s3Profiles"; +import type { OidcParams_Partial } from "core/ports/OnyxiaApi/OidcParams"; +import type { S3Uri } from "core/tools/S3Uri"; +import { same } from "evt/tools/inDepth/same"; +import { + parseUserConfigsS3BookmarksStr, + serializeUserConfigsS3Bookmarks +} from "./decoupledLogic/userConfigsS3Bookmarks"; +import * as userConfigs from "core/usecases/userConfigs"; +import { removeDuplicates } from "evt/tools/reducers/removeDuplicates"; + +const globalContext = { + prS3ClientByProfileName: new Map>() +}; + +export const protectedThunks = { + getS3Client: + (params: { profileName: string }) => + async (...args): Promise => { + const { profileName } = params; + const [, getState, rootContext] = args; + + const s3Profile = (() => { + const s3Profiles = selectors.s3Profiles(getState()); + + const s3Config = s3Profiles.find( + s3Profile => s3Profile.profileName === profileName + ); + assert(s3Config !== undefined); + + return s3Config; + })(); + + use_cached_s3Client: { + const prS3Client = globalContext.prS3ClientByProfileName.get( + s3Profile.profileName + ); + + if (prS3Client === undefined) { + break use_cached_s3Client; + } + + return prS3Client; + } + + const prS3Client = (async () => { + const { createS3Client } = await import("core/adapters/s3Client"); + const { createOidc, mergeOidcParams } = await import( + "core/adapters/oidc" + ); + const { paramsOfBootstrapCore, onyxiaApi } = rootContext; + + return createS3Client( + s3Profile.paramsOfCreateS3Client, + async oidcParams_partial => { + const { oidcParams } = + await onyxiaApi.getAvailableRegionsAndOidcParams(); + + assert(oidcParams !== undefined); + + const oidc_s3 = await createOidc({ + ...mergeOidcParams({ + oidcParams, + oidcParams_partial + }), + autoLogin: true, + transformBeforeRedirectForKeycloakTheme: + paramsOfBootstrapCore.transformBeforeRedirectForKeycloakTheme, + getCurrentLang: paramsOfBootstrapCore.getCurrentLang, + enableDebugLogs: paramsOfBootstrapCore.enableOidcDebugLogs + }); + + const doClearCachedS3Token_groupClaimValue: boolean = + await (async () => { + const { projects } = await onyxiaApi.getUserAndProjects(); + + const KEY = "onyxia:s3:projects-hash"; + + const hash = fnv1aHashToHex(JSON.stringify(projects)); + + if ( + !oidc_s3.isNewBrowserSession && + sessionStorage.getItem(KEY) === hash + ) { + return false; + } + + sessionStorage.setItem(KEY, hash); + return true; + })(); + + const doClearCachedS3Token_s3BookmarkClaimValue: boolean = + (() => { + const resolvedTemplatedBookmarks = + protectedSelectors.resolvedTemplatedBookmarks( + getState() + ); + + const KEY = "onyxia:s3:resolvedAdminBookmarks-hash"; + + const hash = fnv1aHashToHex( + JSON.stringify(resolvedTemplatedBookmarks) + ); + + if ( + !oidc_s3.isNewBrowserSession && + sessionStorage.getItem(KEY) === hash + ) { + return false; + } + + sessionStorage.setItem(KEY, hash); + return true; + })(); + + return { + oidc: oidc_s3, + doClearCachedS3Token: + doClearCachedS3Token_groupClaimValue || + doClearCachedS3Token_s3BookmarkClaimValue + }; + } + ); + })(); + + globalContext.prS3ClientByProfileName.set(s3Profile.profileName, prS3Client); + + return prS3Client; + }, + getAmbientS3ProfileAndClient: + () => + async ( + ...args + ): Promise => { + const [dispatch, getState] = args; + + const s3Profile = selectors.ambientS3Profile(getState()); + + if (s3Profile === undefined) { + return undefined; + } + + const s3Client = await dispatch( + protectedThunks.getS3Client({ + profileName: s3Profile.profileName + }) + ); + + return { s3Client, s3Profile }; + }, + createOrUpdateS3Profile: + (params: { s3Profile_vault: projectManagement.ProjectConfigs.S3Profile }) => + async (...args) => { + const { s3Profile_vault } = params; + + const [dispatch, getState] = args; + + const s3Profiles_vault = structuredClone( + projectManagement.protectedSelectors.projectConfig(getState()).s3Profiles + ); + + const i = s3Profiles_vault.findIndex( + s3Profile_vault_i => + s3Profile_vault_i.creationTime === s3Profile_vault.creationTime + ); + + if (i === -1) { + s3Profiles_vault.push(s3Profile_vault); + } else { + s3Profiles_vault[i] = s3Profile_vault; + } + + assert( + s3Profiles_vault + .map(s3Profile => s3Profile.profileName) + .reduce(...removeDuplicates()).length === + s3Profiles_vault.length + ); + + await dispatch( + projectManagement.protectedThunks.updateConfigValue({ + key: "s3Profiles", + value: s3Profiles_vault + }) + ); + }, + deleteS3Profile: + (params: { profileName: string }) => + async (...args) => { + const { profileName } = params; + + const [dispatch, getState] = args; + + const s3Profiles_vault = structuredClone( + projectManagement.protectedSelectors.projectConfig(getState()).s3Profiles + ); + + const i = s3Profiles_vault.findIndex( + s3Profile => s3Profile.profileName === profileName + ); + + assert(i !== -1); + + s3Profiles_vault.splice(i, 1); + + await dispatch( + projectManagement.protectedThunks.updateConfigValue({ + key: "s3Profiles", + value: s3Profiles_vault + }) + ); + }, + createDeleteOrUpdateBookmark: + (params: { + profileName: string; + s3Uri: S3Uri; + action: + | { + type: "create or update"; + displayName: string | undefined; + } + | { + type: "delete"; + }; + }) => + async (...args) => { + const { profileName, s3Uri, action } = params; + + const [dispatch, getState] = args; + + const s3Profiles = selectors.s3Profiles(getState()); + + const s3Profile = s3Profiles.find( + s3Profile => s3Profile.profileName === profileName + ); + + assert(s3Profile !== undefined); + + switch (s3Profile.origin) { + case "created by user (or group project member)": + { + const s3Profiles_vault = structuredClone( + projectManagement.protectedSelectors.projectConfig(getState()) + .s3Profiles + ); + + const s3Profile_vault = s3Profiles_vault.find( + s3Profile => s3Profile.creationTime === s3Profile.creationTime + ); + + assert(s3Profile_vault !== undefined); + + s3Profile_vault.bookmarks ??= []; + + const index = s3Profile_vault.bookmarks.findIndex(bookmark => + same(bookmark.s3Uri, s3Uri) + ); + + switch (action.type) { + case "create or update": + { + const bookmark_new = { + displayName: action.displayName, + s3Uri + }; + + if (index === -1) { + s3Profile_vault.bookmarks.push(bookmark_new); + } else { + s3Profile_vault.bookmarks[index] = bookmark_new; + } + } + break; + case "delete": + { + assert(index !== -1); + + s3Profile_vault.bookmarks.splice(index, 1); + } + break; + } + + await dispatch( + projectManagement.protectedThunks.updateConfigValue({ + key: "s3Profiles", + value: s3Profiles_vault + }) + ); + } + break; + case "defined in region": + { + const { s3BookmarksStr } = + userConfigs.selectors.userConfigs(getState()); + + const userConfigs_s3Bookmarks = parseUserConfigsS3BookmarksStr({ + userConfigs_s3BookmarksStr: s3BookmarksStr + }); + + const index = userConfigs_s3Bookmarks.findIndex( + entry => + entry.profileName === s3Profile.profileName && + same(entry.s3Uri, s3Uri) + ); + + switch (action.type) { + case "create or update": + { + const bookmark_new = { + profileName: s3Profile.profileName, + displayName: action.displayName, + s3Uri + }; + + if (index === -1) { + userConfigs_s3Bookmarks.push(bookmark_new); + } else { + userConfigs_s3Bookmarks[index] = bookmark_new; + } + } + break; + case "delete": + { + assert(index !== -1); + + userConfigs_s3Bookmarks.splice(index, 1); + } + break; + } + + await dispatch( + userConfigs.thunks.changeValue({ + key: "s3BookmarksStr", + value: serializeUserConfigsS3Bookmarks({ + userConfigs_s3Bookmarks + }) + }) + ); + } + break; + } + }, + changeAmbientProfile: + (params: { profileName: string }) => + (...args) => { + const { profileName } = params; + + const [dispatch, getState] = args; + + const s3Profiles = selectors.s3Profiles(getState()); + + const doesProfileExist = + s3Profiles.find(s3Profile => s3Profile.profileName === profileName) !== + undefined; + + if (!doesProfileExist) { + return { doesProfileExist }; + } + + dispatch(actions.ambientProfileChanged({ profileName })); + + return { doesProfileExist }; + }, + initialize: + () => + async (...args) => { + const [dispatch, getState, { onyxiaApi, paramsOfBootstrapCore }] = args; + + const deploymentRegion = + deploymentRegionManagement.selectors.currentDeploymentRegion(getState()); + + const getDecodedIdToken = async (params: { + oidcParams_partial: OidcParams_Partial; + }) => { + const { oidcParams_partial } = params; + + const { createOidc, mergeOidcParams } = await import( + "core/adapters/oidc" + ); + + const { oidcParams } = await onyxiaApi.getAvailableRegionsAndOidcParams(); + + assert(oidcParams !== undefined); + + const oidc = await createOidc({ + ...mergeOidcParams({ + oidcParams, + oidcParams_partial + }), + autoLogin: true, + transformBeforeRedirectForKeycloakTheme: + paramsOfBootstrapCore.transformBeforeRedirectForKeycloakTheme, + getCurrentLang: paramsOfBootstrapCore.getCurrentLang, + enableDebugLogs: paramsOfBootstrapCore.enableOidcDebugLogs + }); + + const { decodedIdToken } = await oidc.getTokens(); + + return decodedIdToken; + }; + + const resolvedTemplatedBookmarks = await Promise.all( + deploymentRegion.s3Profiles.map(async (s3Config, s3ConfigIndex) => { + const { bookmarks: bookmarks_region, sts } = s3Config; + + return { + correspondingS3ConfigIndexInRegion: s3ConfigIndex, + bookmarks: ( + await Promise.all( + bookmarks_region.map(bookmark => + resolveTemplatedBookmark({ + bookmark_region: bookmark, + getDecodedIdToken: () => + getDecodedIdToken({ + oidcParams_partial: sts.oidcParams + }) + }) + ) + ) + ).flat() + }; + }) + ); + + const resolvedTemplatedStsRoles = await Promise.all( + deploymentRegion.s3Profiles.map(async (s3Config, s3ConfigIndex) => { + const { sts } = s3Config; + + return { + correspondingS3ConfigIndexInRegion: s3ConfigIndex, + stsRoles: ( + await Promise.all( + sts.roles.map(stsRole_region => + resolveTemplatedStsRole({ + stsRole_region, + getDecodedIdToken: () => + getDecodedIdToken({ + oidcParams_partial: sts.oidcParams + }) + }) + ) + ) + ).flat() + }; + }) + ); + + dispatch( + actions.initialized({ + resolvedTemplatedBookmarks, + resolvedTemplatedStsRoles + }) + ); + } +} satisfies Thunks; + +export const thunks = {} satisfies Thunks; diff --git a/web/src/core/usecases/userConfigs.ts b/web/src/core/usecases/userConfigs.ts index 5667c6bef..b3a3f7463 100644 --- a/web/src/core/usecases/userConfigs.ts +++ b/web/src/core/usecases/userConfigs.ts @@ -32,6 +32,7 @@ export type UserConfigs = Id< selectedProjectId: string | null; isCommandBarEnabled: boolean; userProfileStr: string | null; + s3BookmarksStr: string | null; } >; @@ -153,7 +154,8 @@ export const protectedThunks = { doDisplayAcknowledgeConfigVolatilityDialogIfNoVault: true, selectedProjectId: null, isCommandBarEnabled: paramsOfBootstrapCore.isCommandBarEnabledByDefault, - userProfileStr: null + userProfileStr: null, + s3BookmarksStr: null }; const dirPath = await dispatch(privateThunks.getDirPath()); diff --git a/web/src/core/usecases/userProfileForm/thunks.ts b/web/src/core/usecases/userProfileForm/thunks.ts index 357f79790..f9ead1aa4 100644 --- a/web/src/core/usecases/userProfileForm/thunks.ts +++ b/web/src/core/usecases/userProfileForm/thunks.ts @@ -123,7 +123,7 @@ export const protectedThunks = { xOnyxiaContext: await dispatch( launcher.protectedThunks.getXOnyxiaContext({ doInjectPersonalInfos: true, - s3ConfigId: undefined + s3ProfileName: undefined }) ), helmValuesYaml: "{}", diff --git a/web/src/ui/App/App.tsx b/web/src/ui/App/App.tsx index eb28fd714..5c0e6c7df 100644 --- a/web/src/ui/App/App.tsx +++ b/web/src/ui/App/App.tsx @@ -15,6 +15,7 @@ import { useDomRect } from "powerhooks/useDomRect"; import { evtIsScreenScalerOutOfBound } from "screen-scaler"; import { useRerenderOnStateChange } from "evt/hooks/useRerenderOnStateChange"; import { evtTheme } from "ui/theme"; +import { Uploads } from "ui/pages/s3Explorer/Uploads"; triggerCoreBootstrap({ apiUrl: env.ONYXIA_API_URL, @@ -73,6 +74,7 @@ export function App() {