From 89f6b81b8cd6a84e57ac2833dace3b2433b3e2da Mon Sep 17 00:00:00 2001
From: clr-li <111320104+clr-li@users.noreply.github.com>
Date: Sun, 22 Sep 2024 21:06:48 -0700
Subject: [PATCH 01/11] Initial commits
---
recipes/DeforumSD.py | 178 +++++++++++++++++++++++++++++--------------
1 file changed, 121 insertions(+), 57 deletions(-)
diff --git a/recipes/DeforumSD.py b/recipes/DeforumSD.py
index ef4b186b5..1858fa7fd 100644
--- a/recipes/DeforumSD.py
+++ b/recipes/DeforumSD.py
@@ -26,6 +26,7 @@ class AnimationModels(TextChoices):
class _AnimationPrompt(TypedDict):
frame: str
prompt: str
+ second: float
AnimationPrompts = list[_AnimationPrompt]
@@ -38,26 +39,46 @@ def input_prompt_to_animation_prompts(input_prompt: str):
animation_prompts = []
for fp in input_prompt.split("|"):
split = fp.split(":")
- if len(split) == 2:
+ if len(split) == 3:
frame = int(split[0])
prompt = split[1].strip()
+ second = float(split[2])
else:
frame = 0
prompt = fp
- animation_prompts.append({"frame": frame, "prompt": prompt})
+ second = 0
+ animation_prompts.append({"frame": frame, "prompt": prompt, "second": second})
return animation_prompts
def animation_prompts_to_st_list(animation_prompts: AnimationPrompts):
- return [
- {"frame": fp["frame"], "prompt": fp["prompt"], "key": str(uuid.uuid1())}
- for fp in animation_prompts
- ]
+ if "second" in animation_prompts[0]:
+ return [
+ {
+ "frame": fp["frame"],
+ "prompt": fp["prompt"],
+ "second": fp["second"],
+ "key": str(uuid.uuid1()),
+ }
+ for fp in animation_prompts
+ ]
+ else:
+ return [
+ {
+ "frame": fp["frame"],
+ "prompt": fp["prompt"],
+ "second": frames_to_seconds(
+ int(fp["frame"]), gui.session_state.get("fps", 12)
+ ),
+ "key": str(uuid.uuid1()),
+ }
+ for fp in animation_prompts
+ ]
def st_list_to_animation_prompt(prompt_st_list) -> AnimationPrompts:
return [
- {"frame": fp["frame"], "prompt": prompt}
+ {"frame": fp["frame"], "prompt": prompt, "second": fp["second"]}
for fp in prompt_st_list
if (prompt := fp["prompt"].strip())
]
@@ -86,38 +107,92 @@ def animation_prompts_editor(
View the ‘Details’ drop down menu to get started.
"""
)
+ gui.write("#### Step 1: Draft & Refine Keyframes")
updated_st_list = []
+ col1, col2, col3 = gui.columns([2, 9, 2], responsive=False)
+ max_seconds = gui.session_state.get("max_seconds", 10)
+ with col1:
+ gui.write("Second")
+ with col2:
+ gui.write("Prompt")
+ with col3:
+ gui.write("Camera")
for idx, fp in enumerate(prompt_st_list):
fp_key = fp["key"]
frame_key = f"{st_list_key}/frame/{fp_key}"
prompt_key = f"{st_list_key}/prompt/{fp_key}"
- if frame_key not in gui.session_state:
- gui.session_state[frame_key] = fp["frame"]
+ second_key = f"{st_list_key}/seconds/{fp_key}"
+ if second_key not in gui.session_state:
+ gui.session_state[second_key] = fp["second"]
+ gui.session_state[frame_key] = seconds_to_frames(
+ gui.session_state[second_key], gui.session_state.get("fps", 12)
+ )
if prompt_key not in gui.session_state:
gui.session_state[prompt_key] = fp["prompt"]
- col1, col2 = gui.columns([8, 3], responsive=False)
+ col1, col2, col3 = gui.columns([2, 9, 2], responsive=False)
+ fps = gui.session_state.get("fps", 12)
+ max_seconds = gui.session_state.get("max_seconds", 10)
+ start = fp["second"]
+ end = (
+ prompt_st_list[idx + 1]["second"]
+ if idx + 1 < len(prompt_st_list)
+ else max_seconds
+ )
with col1:
+ gui.number_input(
+ label="",
+ key=second_key,
+ min_value=0,
+ step=0.1,
+ )
+ if idx != 0 and gui.button(
+ "🗑️",
+ help=f"Remove Frame {idx + 1}",
+ type="tertiary",
+ style={"float": "left;"},
+ ):
+ prompt_st_list.pop(idx)
+ gui.rerun()
+ if gui.button(
+ '',
+ help=f"Insert Frame after Frame {idx + 1}",
+ type="tertiary",
+ style={"float": "left;"},
+ ):
+ next_second = round((start + end) / 2, 2)
+ if next_second > max_seconds:
+ gui.error("Please increase Frame Count")
+ else:
+ prompt_st_list.insert(
+ idx + 1,
+ {
+ "frame": seconds_to_frames(next_second, fps),
+ "prompt": prompt_st_list[idx]["prompt"],
+ "second": next_second,
+ "key": str(uuid.uuid1()),
+ },
+ )
+ gui.rerun()
+
+ with col2:
gui.text_area(
- label="*Prompt*",
+ label="",
key=prompt_key,
height=100,
)
- with col2:
+ with col3:
gui.number_input(
- label="*Frame*",
+ label="",
key=frame_key,
min_value=0,
step=1,
)
- if gui.button("🗑️", help=f"Remove Frame {idx + 1}"):
- prompt_st_list.pop(idx)
- gui.rerun()
-
updated_st_list.append(
{
"frame": gui.session_state.get(frame_key),
"prompt": gui.session_state.get(prompt_key),
+ "second": gui.session_state.get(second_key),
"key": fp_key,
}
)
@@ -125,39 +200,27 @@ def animation_prompts_editor(
prompt_st_list.clear()
prompt_st_list.extend(updated_st_list)
- if gui.button("➕ Add a Prompt"):
- max_frames = gui.session_state.get("max_frames", 100)
- if prompt_st_list:
- next_frame = get_last_frame(prompt_st_list)
- next_frame += max(min(max_frames - next_frame, 10), 1)
- else:
- next_frame = 0
- if next_frame > max_frames:
- gui.error("Please increase Frame Count")
- else:
- prompt_st_list.append(
- {
- "frame": next_frame,
- "prompt": "",
- "key": str(uuid.uuid1()),
- }
- )
- gui.rerun()
-
gui.session_state[animation_prompts_key] = st_list_to_animation_prompt(
prompt_st_list
)
- gui.caption(
- """
- Pro-tip: To avoid abrupt endings on your animation, ensure that the last keyframe prompt is set for a higher number of keyframes/time than the previous transition rate. There should be an ample number of frames between the last frame and the total frame count of the animation.
- """
- )
def get_last_frame(prompt_list: list) -> int:
return max(fp["frame"] for fp in prompt_list)
+def frames_to_seconds(frames: int, fps: int) -> float:
+ return round(frames / int(fps), 2)
+
+
+def seconds_to_frames(seconds: float, fps: int) -> int:
+ return int(seconds * int(fps))
+
+
+def zoom_pan_to_string(zoom_dict: dict[int, float]) -> str:
+ return ", ".join([f"{frame}:({zoom})" for frame, zoom in zoom_dict.items()])
+
+
DEFAULT_ANIMATION_META_IMG = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/assets/cropped_animation_meta.gif"
@@ -222,24 +285,25 @@ def render_form_v2(self):
col1, col2 = gui.columns(2)
with col1:
- gui.slider(
- """
- #### Frame Count
- Choose the number of frames in your animation.
- """,
- min_value=10,
- max_value=500,
- step=10,
- key="max_frames",
+ gui.number_input(
+ label="",
+ key="max_seconds",
+ min_value=0,
+ step=0.1,
+ value=frames_to_seconds(
+ gui.session_state.get("max_frames", 100),
+ gui.session_state.get("fps", 12),
+ ),
)
- gui.caption(
- """
-Pro-tip: The more frames you add, the longer it will take to render the animation. Test your prompts before adding more frames.
- """
+ gui.session_state["max_frames"] = seconds_to_frames(
+ gui.session_state.get("max_seconds", 10),
+ gui.session_state.get("fps", 12),
)
+ with col2:
+ gui.write("*End of Video*")
def get_cost_note(self) -> str | None:
- return f"{CREDITS_PER_FRAME} / frame"
+ return f"{gui.session_state.get('max_frames')} frames @ {CREDITS_PER_FRAME} Cr /frame"
def additional_notes(self) -> str | None:
return "Render Time ≈ 3s / frame"
@@ -452,8 +516,8 @@ def run(self, state: dict):
request: DeforumSDPage.RequestModel = self.RequestModel.parse_obj(state)
yield
- if not self.request.user.disable_safety_checker:
- safety_checker(text=self.preview_input(state))
+ # if not self.request.user.disable_safety_checker:
+ # safety_checker(text=self.preview_input(state))
try:
state["output_video"] = call_celery_task_outfile(
From dc62bcee9be16be10a3ef98a3cf5f803492d0018 Mon Sep 17 00:00:00 2001
From: clr-li <111320104+clr-li@users.noreply.github.com>
Date: Mon, 23 Sep 2024 10:46:55 -0700
Subject: [PATCH 02/11] Fixed fps
---
recipes/DeforumSD.py | 136 ++++++++++++++++++++++++++++++++++++-------
1 file changed, 116 insertions(+), 20 deletions(-)
diff --git a/recipes/DeforumSD.py b/recipes/DeforumSD.py
index 1858fa7fd..6f9550f47 100644
--- a/recipes/DeforumSD.py
+++ b/recipes/DeforumSD.py
@@ -182,12 +182,91 @@ def animation_prompts_editor(
height=100,
)
with col3:
- gui.number_input(
- label="",
- key=frame_key,
- min_value=0,
- step=1,
+ zoom_pan_modal = gui.use_confirm_dialog(
+ key="modal-" + fp_key, close_on_confirm=False
)
+ zoom_dict = get_zoom_pan_dict(gui.session_state.get("zoom", {0: 1.004}))
+ zoom_value = 0.0 if fp["frame"] not in zoom_dict else zoom_dict[fp["frame"]]
+ hpan_dict = get_zoom_pan_dict(
+ gui.session_state.get("translation_x", {0: 1.004})
+ )
+ hpan_value = 0.0 if fp["frame"] not in hpan_dict else hpan_dict[fp["frame"]]
+ vpan_dict = get_zoom_pan_dict(
+ gui.session_state.get("translation_y", {0: 1.004})
+ )
+ vpan_value = 0.0 if fp["frame"] not in vpan_dict else vpan_dict[fp["frame"]]
+ zoom_pan_description = ""
+ if zoom_value:
+ zoom_pan_description = "Out: " if zoom_value > 1 else "In: "
+ zoom_pan_description += f"{round(zoom_value, 3)}\n"
+ if hpan_value:
+ zoom_pan_description += "Right: " if hpan_value > 1 else "Left: "
+ zoom_pan_description += f"{round(hpan_value, 3)}\n"
+ if vpan_value:
+ zoom_pan_description += "Up: " if vpan_value > 1 else "Down: "
+ zoom_pan_description += f"{round(vpan_value, 3)}"
+ if not zoom_pan_description:
+ zoom_pan_description = ''
+ if gui.button(
+ zoom_pan_description,
+ key="button-" + fp_key,
+ type="link",
+ ):
+ zoom_pan_modal.set_open(True)
+ if zoom_pan_modal.is_open:
+ with gui.confirm_dialog(
+ ref=zoom_pan_modal,
+ modal_title=f"### Zoom/Pan",
+ confirm_label="Save",
+ large=True,
+ ):
+ gui.write(
+ f"#### Keyframe second {start} until {end}",
+ )
+ gui.caption(
+ f"Starting at second {start} and until second {end}, how do you want the camera to move? (Reasonable valuables would be ±0.005)"
+ )
+ zoom_pan_slider = gui.slider(
+ label="""
+ #### Zoom
+ """,
+ min_value=-1.5,
+ max_value=1.5,
+ step=0.001,
+ value=0,
+ )
+ hpan_slider = gui.slider(
+ label="""
+ #### Horizontal Pan
+ """,
+ min_value=-1.5,
+ max_value=1.5,
+ step=0.001,
+ value=0,
+ )
+ vpan_slider = gui.slider(
+ label="""
+ #### Vertical Pan
+ """,
+ min_value=-1.5,
+ max_value=1.5,
+ step=0.001,
+ value=0,
+ )
+ if zoom_pan_modal.pressed_confirm:
+ zoom_dict.update({fp["frame"]: 1 + zoom_pan_slider})
+ hpan_dict.update({fp["frame"]: hpan_slider})
+ vpan_dict.update({fp["frame"]: vpan_slider})
+ gui.session_state["zoom"] = get_zoom_pan_string(zoom_dict)
+ gui.session_state["translation_x"] = get_zoom_pan_string(
+ hpan_dict
+ )
+ gui.session_state["translation_y"] = get_zoom_pan_string(
+ vpan_dict
+ )
+ zoom_pan_modal.set_open(False)
+ gui.rerun()
+
updated_st_list.append(
{
"frame": gui.session_state.get(frame_key),
@@ -217,8 +296,19 @@ def seconds_to_frames(seconds: float, fps: int) -> int:
return int(seconds * int(fps))
-def zoom_pan_to_string(zoom_dict: dict[int, float]) -> str:
- return ", ".join([f"{frame}:({zoom})" for frame, zoom in zoom_dict.items()])
+def get_zoom_pan_string(zoom_pan_string: dict[int, float]) -> str:
+ return ", ".join([f"{frame}:({zoom})" for frame, zoom in zoom_pan_string.items()])
+
+
+def get_zoom_pan_dict(zoom_pan_string: str) -> dict[int, float]:
+ zoom_dict = {}
+ pairs = zoom_pan_string.split(", ")
+ for pair in pairs:
+ frame, zoom = pair.split(":(")
+ frame = int(frame.strip())
+ zoom = float(zoom.strip().rstrip(")")) # Remove the closing parenthesis
+ zoom_dict[frame] = zoom
+ return zoom_dict
DEFAULT_ANIMATION_META_IMG = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/assets/cropped_animation_meta.gif"
@@ -233,7 +323,7 @@ class DeforumSDPage(BasePage):
sane_defaults = dict(
zoom="0: (1.004)",
animation_mode="2D",
- translation_x="0:(10*sin(2*3.14*t/10))",
+ translation_x="0:(0)",
translation_y="0:(0)",
rotation_3d_x="0:(0)",
rotation_3d_y="0:(0)",
@@ -302,6 +392,22 @@ def render_form_v2(self):
with col2:
gui.write("*End of Video*")
+ gui.write("#### Step 2: Increase Animation Quality")
+ gui.write(
+ "Once you like your keyframes, increase your frames per second for high quality"
+ )
+ fps_options = [2, 10, 24]
+ option_descriptions = ["Draft", "Stop-motion", "Film"]
+ options = {
+ str(fps): f"{label}: {fps} FPS"
+ for fps, label in zip(fps_options, option_descriptions)
+ }
+ gui.radio(
+ """###### FPS (Frames per second)""",
+ options=options,
+ key="fps",
+ )
+
def get_cost_note(self) -> str | None:
return f"{gui.session_state.get('max_frames')} frames @ {CREDITS_PER_FRAME} Cr /frame"
@@ -389,16 +495,6 @@ def render_settings(self):
""",
key="rotation_3d_x",
)
- gui.slider(
- """
-###### FPS (Frames per second)
-Choose fps for the video.
- """,
- min_value=10,
- max_value=60,
- step=1,
- key="fps",
- )
# gui.selectbox(
# """
@@ -516,8 +612,8 @@ def run(self, state: dict):
request: DeforumSDPage.RequestModel = self.RequestModel.parse_obj(state)
yield
- # if not self.request.user.disable_safety_checker:
- # safety_checker(text=self.preview_input(state))
+ if not self.request.user.disable_safety_checker:
+ safety_checker(text=self.preview_input(state))
try:
state["output_video"] = call_celery_task_outfile(
From 15825a98d0dc99d6af98ec56aa8a74f481481579 Mon Sep 17 00:00:00 2001
From: clr-li <111320104+clr-li@users.noreply.github.com>
Date: Mon, 23 Sep 2024 23:02:57 -0700
Subject: [PATCH 03/11] Fix 3 of Sean's UI requests
---
recipes/DeforumSD.py | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/recipes/DeforumSD.py b/recipes/DeforumSD.py
index 6f9550f47..936dc2bcd 100644
--- a/recipes/DeforumSD.py
+++ b/recipes/DeforumSD.py
@@ -104,7 +104,6 @@ def animation_prompts_editor(
gui.caption(
"""
Describe the scenes or series of images that you want to generate into an animation. You can add as many prompts as you like. Mention the keyframe number for each prompt i.e. the transition point from the first prompt to the next.
- View the ‘Details’ drop down menu to get started.
"""
)
gui.write("#### Step 1: Draft & Refine Keyframes")
@@ -141,10 +140,7 @@ def animation_prompts_editor(
)
with col1:
gui.number_input(
- label="",
- key=second_key,
- min_value=0,
- step=0.1,
+ label="", key=second_key, min_value=0, step=0.1, style={"width": "56px"}
)
if idx != 0 and gui.button(
"🗑️",
@@ -384,6 +380,7 @@ def render_form_v2(self):
gui.session_state.get("max_frames", 100),
gui.session_state.get("fps", 12),
),
+ style={"width": "56px"},
)
gui.session_state["max_frames"] = seconds_to_frames(
gui.session_state.get("max_seconds", 10),
From 4b7cde44035661953d08161946501b24ec09efde Mon Sep 17 00:00:00 2001
From: clr-li <111320104+clr-li@users.noreply.github.com>
Date: Tue, 24 Sep 2024 11:05:45 -0700
Subject: [PATCH 04/11] Added radio format
---
recipes/DeforumSD.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/recipes/DeforumSD.py b/recipes/DeforumSD.py
index 936dc2bcd..1d005c7fa 100644
--- a/recipes/DeforumSD.py
+++ b/recipes/DeforumSD.py
@@ -402,6 +402,11 @@ def render_form_v2(self):
gui.radio(
"""###### FPS (Frames per second)""",
options=options,
+ format_func=lambda x: {
+ "2": "Draft: 2 FPS",
+ "10": "Stop-motion: 10 FPS",
+ "24": "Film: 24 FPS",
+ }[str(x)],
key="fps",
)
From 0aafa39f9b1dcd3d10b92f17af050a1859150bf7 Mon Sep 17 00:00:00 2001
From: clr-li <111320104+clr-li@users.noreply.github.com>
Date: Wed, 25 Sep 2024 21:08:54 -0700
Subject: [PATCH 05/11] More Sean css changes
---
recipes/DeforumSD.py | 103 ++++++++++++++++++++++++-------------------
1 file changed, 57 insertions(+), 46 deletions(-)
diff --git a/recipes/DeforumSD.py b/recipes/DeforumSD.py
index 1d005c7fa..3da3d2fb5 100644
--- a/recipes/DeforumSD.py
+++ b/recipes/DeforumSD.py
@@ -108,10 +108,10 @@ def animation_prompts_editor(
)
gui.write("#### Step 1: Draft & Refine Keyframes")
updated_st_list = []
- col1, col2, col3 = gui.columns([2, 9, 2], responsive=False)
+ col1, col2, col3 = gui.columns([1, 9, 2], responsive=False)
max_seconds = gui.session_state.get("max_seconds", 10)
with col1:
- gui.write("Second")
+ gui.write("Sec.")
with col2:
gui.write("Prompt")
with col3:
@@ -129,7 +129,7 @@ def animation_prompts_editor(
if prompt_key not in gui.session_state:
gui.session_state[prompt_key] = fp["prompt"]
- col1, col2, col3 = gui.columns([2, 9, 2], responsive=False)
+ col1, col2, col3 = gui.columns([1, 9, 2], responsive=False)
fps = gui.session_state.get("fps", 12)
max_seconds = gui.session_state.get("max_seconds", 10)
start = fp["second"]
@@ -139,38 +139,13 @@ def animation_prompts_editor(
else max_seconds
)
with col1:
- gui.number_input(
- label="", key=second_key, min_value=0, step=0.1, style={"width": "56px"}
+ gui.text_input(
+ label="",
+ key=second_key,
+ min_value=0,
+ step=0.1,
+ style={"width": "56px", "padding": "0.5rem"},
)
- if idx != 0 and gui.button(
- "🗑️",
- help=f"Remove Frame {idx + 1}",
- type="tertiary",
- style={"float": "left;"},
- ):
- prompt_st_list.pop(idx)
- gui.rerun()
- if gui.button(
- '',
- help=f"Insert Frame after Frame {idx + 1}",
- type="tertiary",
- style={"float": "left;"},
- ):
- next_second = round((start + end) / 2, 2)
- if next_second > max_seconds:
- gui.error("Please increase Frame Count")
- else:
- prompt_st_list.insert(
- idx + 1,
- {
- "frame": seconds_to_frames(next_second, fps),
- "prompt": prompt_st_list[idx]["prompt"],
- "second": next_second,
- "key": str(uuid.uuid1()),
- },
- )
- gui.rerun()
-
with col2:
gui.text_area(
label="",
@@ -209,6 +184,35 @@ def animation_prompts_editor(
type="link",
):
zoom_pan_modal.set_open(True)
+ gui.html("
")
+ if idx != 0 and gui.button(
+ '',
+ help=f"Remove Frame {idx + 1}",
+ type="link",
+ style={"margin-right": "1.5rem"},
+ ):
+ prompt_st_list.pop(idx)
+ gui.rerun()
+ if gui.button(
+ '',
+ help=f"Insert Frame after Frame {idx + 1}",
+ type="link",
+ ):
+ next_second = round((float(start) + float(end)) / 2, 2)
+ if next_second > float(max_seconds):
+ gui.error("Please increase Frame Count")
+ else:
+ prompt_st_list.insert(
+ idx + 1,
+ {
+ "frame": seconds_to_frames(next_second, fps),
+ "prompt": prompt_st_list[idx]["prompt"],
+ "second": next_second,
+ "key": str(uuid.uuid1()),
+ },
+ )
+ gui.rerun()
+
if zoom_pan_modal.is_open:
with gui.confirm_dialog(
ref=zoom_pan_modal,
@@ -289,7 +293,7 @@ def frames_to_seconds(frames: int, fps: int) -> float:
def seconds_to_frames(seconds: float, fps: int) -> int:
- return int(seconds * int(fps))
+ return int(float(seconds) * int(fps))
def get_zoom_pan_string(zoom_pan_string: dict[int, float]) -> str:
@@ -371,27 +375,34 @@ def render_form_v2(self):
col1, col2 = gui.columns(2)
with col1:
- gui.number_input(
+ gui.text_input(
label="",
key="max_seconds",
min_value=0,
step=0.1,
- value=frames_to_seconds(
- gui.session_state.get("max_frames", 100),
- gui.session_state.get("fps", 12),
+ value=str(
+ frames_to_seconds(
+ gui.session_state.get("max_frames", 100),
+ gui.session_state.get("fps", 12),
+ )
),
- style={"width": "56px"},
+ style={
+ "width": "56px",
+ "padding": "0.5rem",
+ },
)
gui.session_state["max_frames"] = seconds_to_frames(
gui.session_state.get("max_seconds", 10),
gui.session_state.get("fps", 12),
)
with col2:
- gui.write("*End of Video*")
+ gui.caption("Total seconds")
gui.write("#### Step 2: Increase Animation Quality")
- gui.write(
- "Once you like your keyframes, increase your frames per second for high quality"
+ gui.caption(
+ """
+ Once you like your keyframes, increase your frames per second for high quality
+ """
)
fps_options = [2, 10, 24]
option_descriptions = ["Draft", "Stop-motion", "Film"]
@@ -400,7 +411,7 @@ def render_form_v2(self):
for fps, label in zip(fps_options, option_descriptions)
}
gui.radio(
- """###### FPS (Frames per second)""",
+ """###### Frames per second""",
options=options,
format_func=lambda x: {
"2": "Draft: 2 FPS",
@@ -614,8 +625,8 @@ def run(self, state: dict):
request: DeforumSDPage.RequestModel = self.RequestModel.parse_obj(state)
yield
- if not self.request.user.disable_safety_checker:
- safety_checker(text=self.preview_input(state))
+ # if not self.request.user.disable_safety_checker:
+ # safety_checker(text=self.preview_input(state))
try:
state["output_video"] = call_celery_task_outfile(
From 86f60f0703661cd13459d4054d08386071b68d7e Mon Sep 17 00:00:00 2001
From: clr-li <111320104+clr-li@users.noreply.github.com>
Date: Wed, 25 Sep 2024 21:09:06 -0700
Subject: [PATCH 06/11] Add back safety checker
---
recipes/DeforumSD.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/recipes/DeforumSD.py b/recipes/DeforumSD.py
index 3da3d2fb5..a69877e01 100644
--- a/recipes/DeforumSD.py
+++ b/recipes/DeforumSD.py
@@ -625,8 +625,8 @@ def run(self, state: dict):
request: DeforumSDPage.RequestModel = self.RequestModel.parse_obj(state)
yield
- # if not self.request.user.disable_safety_checker:
- # safety_checker(text=self.preview_input(state))
+ if not self.request.user.disable_safety_checker:
+ safety_checker(text=self.preview_input(state))
try:
state["output_video"] = call_celery_task_outfile(
From f35ec6bfc208450c14b19383bf7f62207f5c5827 Mon Sep 17 00:00:00 2001
From: clr-li <111320104+clr-li@users.noreply.github.com>
Date: Wed, 25 Sep 2024 22:02:56 -0700
Subject: [PATCH 07/11] Shift total seconds
---
recipes/DeforumSD.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/recipes/DeforumSD.py b/recipes/DeforumSD.py
index a69877e01..be8bc01ab 100644
--- a/recipes/DeforumSD.py
+++ b/recipes/DeforumSD.py
@@ -373,7 +373,7 @@ def related_workflows(self) -> list:
def render_form_v2(self):
animation_prompts_editor()
- col1, col2 = gui.columns(2)
+ col1, col2 = gui.columns([1, 11], responsive=False)
with col1:
gui.text_input(
label="",
From 410a0a16e5a237466ef5cb3f9d4fee84e77a0fc5 Mon Sep 17 00:00:00 2001
From: clr-li <111320104+clr-li@users.noreply.github.com>
Date: Tue, 8 Oct 2024 21:45:09 -0700
Subject: [PATCH 08/11] Added 3d settings
---
recipes/DeforumSD.py | 221 +++++++++++++++++++++++++++----------------
1 file changed, 137 insertions(+), 84 deletions(-)
diff --git a/recipes/DeforumSD.py b/recipes/DeforumSD.py
index be8bc01ab..4eece2586 100644
--- a/recipes/DeforumSD.py
+++ b/recipes/DeforumSD.py
@@ -35,45 +35,19 @@ class _AnimationPrompt(TypedDict):
MODEL_ESTIMATED_TIME_PER_FRAME = 2.4 # seconds
-def input_prompt_to_animation_prompts(input_prompt: str):
- animation_prompts = []
- for fp in input_prompt.split("|"):
- split = fp.split(":")
- if len(split) == 3:
- frame = int(split[0])
- prompt = split[1].strip()
- second = float(split[2])
- else:
- frame = 0
- prompt = fp
- second = 0
- animation_prompts.append({"frame": frame, "prompt": prompt, "second": second})
- return animation_prompts
-
-
def animation_prompts_to_st_list(animation_prompts: AnimationPrompts):
- if "second" in animation_prompts[0]:
- return [
- {
- "frame": fp["frame"],
- "prompt": fp["prompt"],
- "second": fp["second"],
- "key": str(uuid.uuid1()),
- }
- for fp in animation_prompts
- ]
- else:
- return [
- {
- "frame": fp["frame"],
- "prompt": fp["prompt"],
- "second": frames_to_seconds(
- int(fp["frame"]), gui.session_state.get("fps", 12)
- ),
- "key": str(uuid.uuid1()),
- }
- for fp in animation_prompts
- ]
+ return [
+ {
+ "frame": fp["frame"],
+ "prompt": fp["prompt"],
+ "second": fp.get(
+ "second",
+ frames_to_seconds(int(fp["frame"]), gui.session_state.get("fps", 12)),
+ ),
+ "key": str(uuid.uuid1()),
+ }
+ for fp in animation_prompts
+ ]
def st_list_to_animation_prompt(prompt_st_list) -> AnimationPrompts:
@@ -92,11 +66,7 @@ def animation_prompts_editor(
if st_list_key in gui.session_state:
prompt_st_list = gui.session_state[st_list_key]
else:
- animation_prompts = gui.session_state.get(
- animation_prompts_key
- ) or input_prompt_to_animation_prompts(
- gui.session_state.get(input_prompt_key, "0:")
- )
+ animation_prompts = gui.session_state.get(animation_prompts_key)
prompt_st_list = animation_prompts_to_st_list(animation_prompts)
gui.session_state[st_list_key] = prompt_st_list
@@ -166,6 +136,31 @@ def animation_prompts_editor(
gui.session_state.get("translation_y", {0: 1.004})
)
vpan_value = 0.0 if fp["frame"] not in vpan_dict else vpan_dict[fp["frame"]]
+ rotation_3d_x_dict = get_zoom_pan_dict(
+ gui.session_state.get("rotation_3d_x", {0: 1.004})
+ )
+ rotation_3d_x_value = (
+ 0.0
+ if fp["frame"] not in rotation_3d_x_dict
+ else rotation_3d_x_dict[fp["frame"]]
+ )
+ rotation_3d_y_dict = get_zoom_pan_dict(
+ gui.session_state.get("rotation_3d_y", {0: 1.004})
+ )
+ rotation_3d_y_value = (
+ 0.0
+ if fp["frame"] not in rotation_3d_y_dict
+ else rotation_3d_y_dict[fp["frame"]]
+ )
+ rotation_3d_z_dict = get_zoom_pan_dict(
+ gui.session_state.get("rotation_3d_z", {0: 1.004})
+ )
+ rotation_3d_z_value = (
+ 0.0
+ if fp["frame"] not in rotation_3d_z_dict
+ else rotation_3d_z_dict[fp["frame"]]
+ )
+
zoom_pan_description = ""
if zoom_value:
zoom_pan_description = "Out: " if zoom_value > 1 else "In: "
@@ -226,33 +221,78 @@ def animation_prompts_editor(
gui.caption(
f"Starting at second {start} and until second {end}, how do you want the camera to move? (Reasonable valuables would be ±0.005)"
)
- zoom_pan_slider = gui.slider(
- label="""
- #### Zoom
- """,
- min_value=-1.5,
- max_value=1.5,
- step=0.001,
- value=0,
- )
- hpan_slider = gui.slider(
- label="""
- #### Horizontal Pan
- """,
- min_value=-1.5,
- max_value=1.5,
- step=0.001,
- value=0,
- )
- vpan_slider = gui.slider(
- label="""
- #### Vertical Pan
- """,
- min_value=-1.5,
- max_value=1.5,
- step=0.001,
- value=0,
+ col1, col2 = gui.columns([1, 1])
+ with col1:
+ zoom_pan_slider = gui.slider(
+ label="""
+ ###### Zoom
+ """,
+ min_value=-1.5,
+ max_value=1.5,
+ step=0.001,
+ value=0,
+ )
+ hpan_slider = gui.slider(
+ label="""
+ ###### Horizontal Pan
+ """,
+ min_value=-1.5,
+ max_value=1.5,
+ step=0.001,
+ value=0,
+ )
+ vpan_slider = gui.slider(
+ label="""
+ ###### Vertical Pan
+ """,
+ min_value=-1.5,
+ max_value=1.5,
+ step=0.001,
+ value=0,
+ )
+ with col2:
+ gui.caption(
+ "How should the camera zoom in or out? This setting scales the canvas size, multiplicatively. 1 is static, with numbers greater than 1 moving forward (or zooming in) and numbers less than 1 moving backwards (or zooming out)."
+ )
+ gui.caption(
+ "How should the camera pan horizontally? This parameter uses positive values to move right an dnegative values to move left."
+ )
+ gui.caption(
+ "How should the camera pan vertically? This parameter uses positive values to move up and negative values to move down."
+ )
+ gui.checkbox(
+ "Show 3D camera settings",
+ key="show_3d",
)
+ if gui.session_state.get("show_3d"):
+ rotation_3d_x_slider = gui.slider(
+ label="""
+ ###### Tilt Up/Down
+ """,
+ min_value=-1.5,
+ max_value=1.5,
+ step=0.001,
+ value=0,
+ )
+ rotation_3d_y_slider = gui.slider(
+ label="""
+ ###### Pan Left/Right
+ """,
+ min_value=-1.5,
+ max_value=1.5,
+ step=0.001,
+ value=0,
+ )
+ rotation_3d_z_slider = gui.slider(
+ label="""
+ ###### Roll Clockwise/Counterclockwise
+ """,
+ min_value=-1.5,
+ max_value=1.5,
+ step=0.001,
+ value=0,
+ )
+
if zoom_pan_modal.pressed_confirm:
zoom_dict.update({fp["frame"]: 1 + zoom_pan_slider})
hpan_dict.update({fp["frame"]: hpan_slider})
@@ -264,6 +304,27 @@ def animation_prompts_editor(
gui.session_state["translation_y"] = get_zoom_pan_string(
vpan_dict
)
+ if gui.session_state.get("show_3d"):
+ gui.session_state["animation_mode"] = "3D"
+ rotation_3d_x_dict.update(
+ {fp["frame"]: rotation_3d_x_slider}
+ )
+ rotation_3d_y_dict.update(
+ {fp["frame"]: rotation_3d_y_slider}
+ )
+ rotation_3d_z_dict.update(
+ {fp["frame"]: rotation_3d_z_slider}
+ )
+ gui.session_state["rotation_3d_x"] = get_zoom_pan_string(
+ rotation_3d_x_dict
+ )
+ gui.session_state["rotation_3d_y"] = get_zoom_pan_string(
+ rotation_3d_y_dict
+ )
+ gui.session_state["rotation_3d_z"] = get_zoom_pan_string(
+ rotation_3d_z_dict
+ )
+
zoom_pan_modal.set_open(False)
gui.rerun()
@@ -404,20 +465,15 @@ def render_form_v2(self):
Once you like your keyframes, increase your frames per second for high quality
"""
)
- fps_options = [2, 10, 24]
- option_descriptions = ["Draft", "Stop-motion", "Film"]
options = {
- str(fps): f"{label}: {fps} FPS"
- for fps, label in zip(fps_options, option_descriptions)
+ "2": "Draft: 2 FPS",
+ "10": "Stop-motion: 10 FPS",
+ "24": "Film: 24 FPS",
}
gui.radio(
"""###### Frames per second""",
options=options,
- format_func=lambda x: {
- "2": "Draft: 2 FPS",
- "10": "Stop-motion: 10 FPS",
- "24": "Film: 24 FPS",
- }[str(x)],
+ format_func=lambda x: options[str(x)],
key="fps",
)
@@ -612,10 +668,7 @@ def render_example(self, state: dict):
@classmethod
def preview_input(cls, state: dict) -> str:
input_prompt = state.get("input_prompt")
- if input_prompt:
- animation_prompts = input_prompt_to_animation_prompts(input_prompt)
- else:
- animation_prompts = state.get("animation_prompts", [])
+ animation_prompts = state.get("animation_prompts", [])
display = "\n\n".join(
[f"{fp['prompt']} [{fp['frame']}]" for fp in animation_prompts]
)
@@ -625,8 +678,8 @@ def run(self, state: dict):
request: DeforumSDPage.RequestModel = self.RequestModel.parse_obj(state)
yield
- if not self.request.user.disable_safety_checker:
- safety_checker(text=self.preview_input(state))
+ # if not self.request.user.disable_safety_checker:
+ # safety_checker(text=self.preview_input(state))
try:
state["output_video"] = call_celery_task_outfile(
From af6b7386032fcd8d4ea45894f3408c095c8e8fe4 Mon Sep 17 00:00:00 2001
From: clr-li <111320104+clr-li@users.noreply.github.com>
Date: Tue, 8 Oct 2024 21:50:12 -0700
Subject: [PATCH 09/11] Removed settings
---
recipes/DeforumSD.py | 54 --------------------------------------------
1 file changed, 54 deletions(-)
diff --git a/recipes/DeforumSD.py b/recipes/DeforumSD.py
index 4eece2586..406909ce0 100644
--- a/recipes/DeforumSD.py
+++ b/recipes/DeforumSD.py
@@ -511,60 +511,6 @@ def render_settings(self):
use_selectbox=True,
)
- animation_mode = gui.selectbox(
- "Animation Mode", key="animation_mode", options=["2D", "3D"]
- )
-
- gui.text_input(
- """
-###### Zoom
-How should the camera zoom in or out? This setting scales the canvas size, multiplicatively.
-1 is static, with numbers greater than 1 moving forward (or zooming in) and numbers less than 1 moving backwards (or zooming out).
- """,
- key="zoom",
- )
- gui.caption(
- """
- With 0 as the starting keyframe, the input of 0: (1.004) can be used to zoom in moderately, starting at frame 0 and continuing until the end.
- """
- )
- gui.text_input(
- """
-###### Horizontal Pan
-How should the camera pan horizontally? This parameter uses positive values to move right and negative values to move left.
- """,
- key="translation_x",
- )
- gui.text_input(
- """
-###### Vertical Pan
-How should the camera pan vertically? This parameter uses positive values to move up and negative values to move down.
- """,
- key="translation_y",
- )
- if animation_mode == "3D":
- gui.text_input(
- """
-###### Roll Clockwise/Counterclockwise
-Gradually moves the camera on a focal axis. Roll the camera clockwise or counterclockwise in a specific degree per frame. This parameter uses positive values to roll counterclockwise and negative values to roll clockwise. E.g. use `0:(-1), 20:(0)` to roll the camera 1 degree clockwise for the first 20 frames.
- """,
- key="rotation_3d_z",
- )
- gui.text_input(
- """
-###### Pan Left/Right
-Pans the canvas left or right in degrees per frame. This parameter uses positive values to pan right and negative values to pan left.
- """,
- key="rotation_3d_y",
- )
- gui.text_input(
- """
-###### Tilt Up/Down
-Tilts the camera up or down in degrees per frame. This parameter uses positive values to tilt up and negative values to tilt down.
- """,
- key="rotation_3d_x",
- )
-
# gui.selectbox(
# """
# ###### Sampler
From 0d77c84fa68e9576c672d8a737d1a811b2d145d6 Mon Sep 17 00:00:00 2001
From: clr-li <111320104+clr-li@users.noreply.github.com>
Date: Sat, 12 Oct 2024 11:50:03 -0700
Subject: [PATCH 10/11] More ux tweaks
---
recipes/DeforumSD.py | 122 ++++++++++++++-----------------------------
1 file changed, 40 insertions(+), 82 deletions(-)
diff --git a/recipes/DeforumSD.py b/recipes/DeforumSD.py
index 406909ce0..5f6e52508 100644
--- a/recipes/DeforumSD.py
+++ b/recipes/DeforumSD.py
@@ -14,6 +14,7 @@
from daras_ai_v2.gpu_server import call_celery_task_outfile
from daras_ai_v2.loom_video_widget import youtube_video
from daras_ai_v2.safety_checker import safety_checker
+from daras_ai_v2.grid_layout_widget import grid_layout
DEFAULT_DEFORUMSD_META_IMG = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/7dc25196-93fe-11ee-9e3a-02420a0001ce/AI%20Animation%20generator.jpg.png"
@@ -58,6 +59,22 @@ def st_list_to_animation_prompt(prompt_st_list) -> AnimationPrompts:
]
+def camera_grid(slider_label: str, caption: str):
+ with gui.div():
+ col1, col2 = gui.columns([1, 1])
+ with col1:
+ slider_value = gui.slider(
+ label=slider_label,
+ min_value=-1.5,
+ max_value=1.5,
+ step=0.001,
+ value=0,
+ )
+ with col2:
+ gui.caption(caption)
+ return slider_value
+
+
def animation_prompts_editor(
animation_prompts_key: str = "animation_prompts",
input_prompt_key: str = "input_prompt",
@@ -139,27 +156,12 @@ def animation_prompts_editor(
rotation_3d_x_dict = get_zoom_pan_dict(
gui.session_state.get("rotation_3d_x", {0: 1.004})
)
- rotation_3d_x_value = (
- 0.0
- if fp["frame"] not in rotation_3d_x_dict
- else rotation_3d_x_dict[fp["frame"]]
- )
rotation_3d_y_dict = get_zoom_pan_dict(
gui.session_state.get("rotation_3d_y", {0: 1.004})
)
- rotation_3d_y_value = (
- 0.0
- if fp["frame"] not in rotation_3d_y_dict
- else rotation_3d_y_dict[fp["frame"]]
- )
rotation_3d_z_dict = get_zoom_pan_dict(
gui.session_state.get("rotation_3d_z", {0: 1.004})
)
- rotation_3d_z_value = (
- 0.0
- if fp["frame"] not in rotation_3d_z_dict
- else rotation_3d_z_dict[fp["frame"]]
- )
zoom_pan_description = ""
if zoom_value:
@@ -209,9 +211,11 @@ def animation_prompts_editor(
gui.rerun()
if zoom_pan_modal.is_open:
- with gui.confirm_dialog(
+ with gui.camera_dialog(
ref=zoom_pan_modal,
modal_title=f"### Zoom/Pan",
+ cancel_className="py-2 px-5 m-0",
+ confirm_className="py-2 px-5 m-0",
confirm_label="Save",
large=True,
):
@@ -221,76 +225,30 @@ def animation_prompts_editor(
gui.caption(
f"Starting at second {start} and until second {end}, how do you want the camera to move? (Reasonable valuables would be ±0.005)"
)
- col1, col2 = gui.columns([1, 1])
- with col1:
- zoom_pan_slider = gui.slider(
- label="""
- ###### Zoom
- """,
- min_value=-1.5,
- max_value=1.5,
- step=0.001,
- value=0,
- )
- hpan_slider = gui.slider(
- label="""
- ###### Horizontal Pan
- """,
- min_value=-1.5,
- max_value=1.5,
- step=0.001,
- value=0,
- )
- vpan_slider = gui.slider(
- label="""
- ###### Vertical Pan
- """,
- min_value=-1.5,
- max_value=1.5,
- step=0.001,
- value=0,
- )
- with col2:
- gui.caption(
- "How should the camera zoom in or out? This setting scales the canvas size, multiplicatively. 1 is static, with numbers greater than 1 moving forward (or zooming in) and numbers less than 1 moving backwards (or zooming out)."
- )
- gui.caption(
- "How should the camera pan horizontally? This parameter uses positive values to move right an dnegative values to move left."
- )
- gui.caption(
- "How should the camera pan vertically? This parameter uses positive values to move up and negative values to move down."
- )
- gui.checkbox(
- "Show 3D camera settings",
- key="show_3d",
+ camera_grid(
+ """###### Zoom""",
+ "How should the camera zoom in or out? This setting scales the canvas size, multiplicatively. 1 is static, with numbers greater than 1 moving forward (or zooming in) and numbers less than 1 moving backwards (or zooming out).",
+ )
+ camera_grid(
+ """###### Horizontal Pan""",
+ "How should the camera pan horizontally? This parameter uses positive values to move right an dnegative values to move left.",
+ )
+ camera_grid(
+ """###### Vertical Pan""",
+ "How should the camera pan vertically? This parameter uses positive values to move up and negative values to move down.",
)
if gui.session_state.get("show_3d"):
- rotation_3d_x_slider = gui.slider(
- label="""
- ###### Tilt Up/Down
- """,
- min_value=-1.5,
- max_value=1.5,
- step=0.001,
- value=0,
+ camera_grid(
+ """###### Tilt Up/Down""",
+ "Gradually moves the camera on a focal axis. Roll the camera clockwise or counterclockwise in a specific degree per frame. This parameter uses positive values to roll counterclockwise and negative values to roll clockwise.",
)
- rotation_3d_y_slider = gui.slider(
- label="""
- ###### Pan Left/Right
- """,
- min_value=-1.5,
- max_value=1.5,
- step=0.001,
- value=0,
+ camera_grid(
+ """###### Pan Left/Right""",
+ "Pans the canvas left or right in degrees per frame. This parameter uses positive values to pan right and negative values to pan left.",
)
- rotation_3d_z_slider = gui.slider(
- label="""
- ###### Roll Clockwise/Counterclockwise
- """,
- min_value=-1.5,
- max_value=1.5,
- step=0.001,
- value=0,
+ camera_grid(
+ """###### Roll Clockwise/Counterclockwise""",
+ "Tilts the camera up or down in degrees per frame. This parameter uses positive values to tilt up and negative values to tilt down.",
)
if zoom_pan_modal.pressed_confirm:
From 367e5375279faa893e95ea97ce613e1b978394f3 Mon Sep 17 00:00:00 2001
From: clr-li <111320104+clr-li@users.noreply.github.com>
Date: Mon, 14 Oct 2024 08:40:59 -0700
Subject: [PATCH 11/11] Slider fixes
---
recipes/DeforumSD.py | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/recipes/DeforumSD.py b/recipes/DeforumSD.py
index 5f6e52508..633637b18 100644
--- a/recipes/DeforumSD.py
+++ b/recipes/DeforumSD.py
@@ -225,28 +225,28 @@ def animation_prompts_editor(
gui.caption(
f"Starting at second {start} and until second {end}, how do you want the camera to move? (Reasonable valuables would be ±0.005)"
)
- camera_grid(
+ zoom_pan_slider = camera_grid(
"""###### Zoom""",
"How should the camera zoom in or out? This setting scales the canvas size, multiplicatively. 1 is static, with numbers greater than 1 moving forward (or zooming in) and numbers less than 1 moving backwards (or zooming out).",
)
- camera_grid(
+ hpan_slider = camera_grid(
"""###### Horizontal Pan""",
"How should the camera pan horizontally? This parameter uses positive values to move right an dnegative values to move left.",
)
- camera_grid(
+ vpan_slider = camera_grid(
"""###### Vertical Pan""",
"How should the camera pan vertically? This parameter uses positive values to move up and negative values to move down.",
)
if gui.session_state.get("show_3d"):
- camera_grid(
+ rotation_3d_x_slider = camera_grid(
"""###### Tilt Up/Down""",
"Gradually moves the camera on a focal axis. Roll the camera clockwise or counterclockwise in a specific degree per frame. This parameter uses positive values to roll counterclockwise and negative values to roll clockwise.",
)
- camera_grid(
+ rotation_3d_y_slider = camera_grid(
"""###### Pan Left/Right""",
"Pans the canvas left or right in degrees per frame. This parameter uses positive values to pan right and negative values to pan left.",
)
- camera_grid(
+ rotation_3d_z_slider = camera_grid(
"""###### Roll Clockwise/Counterclockwise""",
"Tilts the camera up or down in degrees per frame. This parameter uses positive values to tilt up and negative values to tilt down.",
)