diff --git a/frontend/apps/artcraft/app/src/components/signaled/TopBar/TopBar.tsx b/frontend/apps/artcraft/app/src/components/signaled/TopBar/TopBar.tsx index 7dcd38eeaa..562375625d 100644 --- a/frontend/apps/artcraft/app/src/components/signaled/TopBar/TopBar.tsx +++ b/frontend/apps/artcraft/app/src/components/signaled/TopBar/TopBar.tsx @@ -467,6 +467,8 @@ export const TopBar = ({ pageName }: Props) => { return "Image to 3D Object"; case "IMAGE_TO_3D_WORLD": return "Image to 3D World"; + case "MOTION_CONTROL": + return "Motion Control"; case "APPS": return "ArtCraft Apps"; default: diff --git a/frontend/apps/artcraft/app/src/config/appMenu.tsx b/frontend/apps/artcraft/app/src/config/appMenu.tsx index 39add01b60..f221c05daf 100644 --- a/frontend/apps/artcraft/app/src/config/appMenu.tsx +++ b/frontend/apps/artcraft/app/src/config/appMenu.tsx @@ -10,6 +10,7 @@ import { faWandMagicSparkles, faPenNib, faCrosshairs, + faPersonRunning, } from "@fortawesome/pro-solid-svg-icons"; import { useTabStore, TabId } from "~/pages/Stores/TabState"; import { set3DPageMounted } from "~/pages/PageEnigma/Editor/editor"; @@ -26,7 +27,8 @@ export type AppId = | "IMAGE_TO_3D_OBJECT" | "IMAGE_TO_3D_WORLD" | "REMOVE_BACKGROUND" - | "ANGLES"; + | "ANGLES" + | "MOTION_CONTROL"; export interface AppDescriptor { id: AppId; @@ -48,6 +50,11 @@ export const APP_DESCRIPTORS: AppDescriptor[] = [ label: "Generate Video", icon: faFilm, }, + { + id: "MOTION_CONTROL", + label: "Motion Control", + icon: faPersonRunning, + }, { id: "2D", label: "Image Editor", @@ -96,6 +103,16 @@ export const ALL_APPS: FullAppItem[] = [ action: "VIDEO", color: "bg-amber-500/40", }, + { + id: "motion-control", + label: "Motion Control", + description: "Transfer movements from video to character", + icon: faPersonRunning, + category: "generate", + action: "MOTION_CONTROL", + color: "bg-orange-500/40", + badge: "NEW", + }, { id: "image-to-3d-object", label: "Image to 3D Object", @@ -223,6 +240,7 @@ export const goToApp = (action?: string) => { "IMAGE_TO_3D_WORLD", "REMOVE_BACKGROUND", "ANGLES", + "MOTION_CONTROL", ].includes(action) ) { if (action === "3D") { diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/PageEditor.tsx b/frontend/apps/artcraft/app/src/pages/PageEnigma/PageEditor.tsx index 83c96caae0..20bcd8cd7a 100644 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/PageEditor.tsx +++ b/frontend/apps/artcraft/app/src/pages/PageEnigma/PageEditor.tsx @@ -18,6 +18,7 @@ import { ImageTo3DObject } from "../PageImageTo3DObject"; import { ImageTo3DWorld } from "../PageImageTo3DWorld"; import { RemoveBackground } from "../PageRemoveBackground"; import { Angles } from "../PageAngles"; +import MotionControl from "../PageMotionControl/MotionControl"; import { timelineHeight, @@ -643,6 +644,11 @@ export const PageEditor = () => { )} + {tabStore.activeTabId == "MOTION_CONTROL" && ( +
+ +
+ )} ); }; diff --git a/frontend/apps/artcraft/app/src/pages/PageMotionControl/MotionControl.tsx b/frontend/apps/artcraft/app/src/pages/PageMotionControl/MotionControl.tsx new file mode 100644 index 0000000000..d3a7ff7de8 --- /dev/null +++ b/frontend/apps/artcraft/app/src/pages/PageMotionControl/MotionControl.tsx @@ -0,0 +1,548 @@ +import { useCallback, useRef, useState } from "react"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { + faFilm, + faImages, + faPlus, + faSpinnerThird, + faXmark, + faArrowRight, + faSliders, +} from "@fortawesome/pro-solid-svg-icons"; +import { faImage as faImageRegular } from "@fortawesome/pro-regular-svg-icons"; +import { twMerge } from "tailwind-merge"; +import { UploadImageMedia, UploadVideoMedia } from "@storyteller/api"; +import { UploaderState, UploaderStates } from "@storyteller/common"; +import { + ClassyModelSelector, + MOTION_CONTROL_PAGE_MODEL_LIST, + ModelPage, + useSelectedVideoModel, + useSelectedProviderForModel, +} from "@storyteller/ui-model-selector"; +import { CostCalculatorButton } from "@storyteller/ui-pricing-modal"; +import { HelpMenuButton } from "@storyteller/ui-help-menu"; +import { Button, ToggleButton, GenerateButton } from "@storyteller/ui-button"; +import { Tooltip } from "@storyteller/ui-tooltip"; +import { PopoverMenu, PopoverItem } from "@storyteller/ui-popover"; +import { useMotionControlStore } from "./MotionControlStore"; +import { + useMotionControlCompleteEvent, + MotionControlCompleteEvent, +} from "@storyteller/tauri-events"; +import BackgroundGallery from "../PageVideo/BackgroundGallery"; +import { TabSelector } from "@storyteller/ui-tab-selector"; +import { GalleryModal, GalleryItem } from "@storyteller/ui-gallery-modal"; + +type UploadMediaFn = (args: { + title: string; + assetFile: File; + progressCallback: (newState: UploaderState) => void; +}) => Promise; + +const PAGE_ID: ModelPage = ModelPage.MotionControl; + +const ORIENTATION_TABS = [ + { id: "video", label: "Video" }, + { id: "image", label: "Image" }, +]; + +type CharacterOrientation = "video" | "image"; +type Resolution = "720p" | "1080p"; + +interface UploadedMedia { + url: string; + mediaToken: string; + file: File; +} + +const MotionControl = () => { + const completeBatch = useMotionControlStore((s) => s.completeBatch); + const startBatch = useMotionControlStore((s) => s.startBatch); + + const selectedVideoModel = useSelectedVideoModel(PAGE_ID); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const _selectedProvider = useSelectedProviderForModel( + PAGE_ID, + selectedVideoModel?.id, + ); + + // Upload state + const [motionVideo, setMotionVideo] = useState(); + const [characterImage, setCharacterImage] = useState< + UploadedMedia | undefined + >(); + const [uploadingVideo, setUploadingVideo] = useState(false); + const [uploadingImage, setUploadingImage] = useState(false); + + // Settings state + const [showSettings, setShowSettings] = useState(false); + const [orientation, setOrientation] = useState("video"); + const [prompt, setPrompt] = useState(""); + const [resolution, setResolution] = useState("720p"); + + const resolutionOptions: PopoverItem[] = ( + ["720p", "1080p"] as Resolution[] + ).map((r) => ({ + label: r, + selected: r === resolution, + })); + + const handleResolutionSelect = useCallback( + (item: PopoverItem) => setResolution(item.label as Resolution), + [], + ); + + // File input refs + const videoInputRef = useRef(null); + const imageInputRef = useRef(null); + + // Listen for generation complete events + useMotionControlCompleteEvent(async (event: MotionControlCompleteEvent) => { + if (!event.generated_video) return; + completeBatch( + { + cdn_url: event.generated_video.cdn_url, + media_token: event.generated_video.media_token, + }, + event.maybe_frontend_subscriber_id, + ); + }); + + const handleUpload = useCallback( + ( + file: File, + uploadFn: UploadMediaFn, + setUploading: (v: boolean) => void, + setMedia: (m: UploadedMedia | undefined) => void, + prefix: string, + ) => { + setUploading(true); + const reader = new FileReader(); + reader.onloadend = async () => { + await uploadFn({ + title: `${prefix}-${Math.random().toString(36).substring(2, 15)}`, + assetFile: file, + progressCallback: (newState) => { + if (newState.status === UploaderStates.success && newState.data) { + setMedia({ + url: reader.result as string, + mediaToken: newState.data, + file, + }); + setUploading(false); + } else if ( + newState.status === UploaderStates.assetError || + newState.status === UploaderStates.imageCreateError + ) { + setUploading(false); + } + }, + }); + }; + reader.readAsDataURL(file); + }, + [], + ); + + const handleVideoUpload = useCallback( + (e: React.ChangeEvent) => { + const file = e.target.files?.[0]; + if (!file) return; + handleUpload( + file, + UploadVideoMedia, + setUploadingVideo, + setMotionVideo, + "motion-ref", + ); + if (videoInputRef.current) videoInputRef.current.value = ""; + }, + [handleUpload], + ); + + const handleImageUpload = useCallback( + (e: React.ChangeEvent) => { + const file = e.target.files?.[0]; + if (!file) return; + handleUpload( + file, + UploadImageMedia, + setUploadingImage, + setCharacterImage, + "motion-char", + ); + if (imageInputRef.current) imageInputRef.current.value = ""; + }, + [handleUpload], + ); + + // Library picker state + type PickerTarget = "video" | "image"; + const [pickerOpen, setPickerOpen] = useState(false); + const [pickerTarget, setPickerTarget] = useState("video"); + const [pickerSelectedIds, setPickerSelectedIds] = useState([]); + + const openPicker = useCallback((target: PickerTarget) => { + setPickerTarget(target); + setPickerSelectedIds([]); + setPickerOpen(true); + }, []); + + const handlePickerSelect = useCallback((id: string) => { + setPickerSelectedIds((prev) => + prev.includes(id) ? prev.filter((x) => x !== id) : [id], + ); + }, []); + + const handlePickerUse = useCallback( + (items: GalleryItem[]) => { + const item = items[0]; + if (!item) return; + const url = item.fullImage || item.thumbnail || ""; + const media: UploadedMedia = { + url, + mediaToken: item.id, + file: new File([], "library-pick"), + }; + if (pickerTarget === "video") { + setMotionVideo(media); + } else { + setCharacterImage(media); + } + setPickerOpen(false); + setPickerSelectedIds([]); + }, + [pickerTarget], + ); + + const canGenerate = !!motionVideo && !!characterImage; + + const handleGenerate = useCallback(() => { + if (!canGenerate) return; + const subscriberId = crypto.randomUUID + ? crypto.randomUUID() + : Math.random().toString(36).slice(2); + const modelLabel = selectedVideoModel?.fullName ?? "Motion Control"; + startBatch(prompt, modelLabel, subscriberId); + + // TODO: Send to backend when API is ready + // The payload would be: + // { + // prompt, + // image_url / image_media_token: characterImage.mediaToken, + // video_url / video_media_token: motionVideo.mediaToken, + // character_orientation: orientation, + // resolution, + // model, provider, subscriberId, etc. + // } + }, [ + canGenerate, + prompt, + orientation, + resolution, + selectedVideoModel, + startBatch, + motionVideo, + characterImage, + ]); + + return ( +
+
+
+ {/* Title */} +
+

Motion Control

+ + Transfer movements from a reference video to any character + +
+ + {/* Prompt box area */} +
+ {/* Upload row */} +
+ setMotionVideo(undefined)} + onPickFromLibrary={() => openPicker("video")} + /> + +
+ +
+ + setCharacterImage(undefined)} + onPickFromLibrary={() => openPicker("image")} + /> +
+ + {/* Bottom controls */} +
+
+ + + + + setShowSettings((v) => !v)} + /> + +
+ + + Generate + +
+ + {/* Advanced settings panel (below controls) */} + {showSettings && ( + setShowSettings(false)} + /> + )} +
+ + + + {/* Library picker modal */} + setPickerOpen(false)} + /> + + {/* Bottom-left model selector */} +
+ +
+ + {/* Bottom-right controls */} +
+ + +
+
+
+
+ ); +}; + +export default MotionControl; + +// ── Sub-components ─────────────────────────────────────────────────────── + +interface UploadSlotProps { + label: string; + subtitle: string; + icon: any; + accept: string; + media?: UploadedMedia; + uploading: boolean; + inputRef: React.RefObject; + onFileChange: (e: React.ChangeEvent) => void; + onClear: () => void; + onPickFromLibrary?: () => void; +} + +const UploadSlot = ({ + label, + subtitle, + icon, + accept, + media, + uploading, + inputRef, + onFileChange, + onClear, + onPickFromLibrary, +}: UploadSlotProps) => ( +
+ + {media ? ( +
+ {accept.startsWith("video") ? ( +
+ ) : uploading ? ( +
+ +
+ ) : ( + + + {onPickFromLibrary && ( + + )} +
+ } + > + + + )} + +); + +interface AdvancedSettingsProps { + orientation: CharacterOrientation; + setOrientation: (v: CharacterOrientation) => void; + prompt: string; + setPrompt: (v: string) => void; + onClose: () => void; +} + +const AdvancedSettings = ({ + orientation, + setOrientation, + prompt, + setPrompt, + onClose, +}: AdvancedSettingsProps) => ( +
+ {/* Character orientation */} +
+ + Character orientation + + setOrientation(id as CharacterOrientation)} + /> +
+ + {/* Prompt */} +
+
+ Prompt (optional) +
+