Scene-to-Image Prompt Layering System

data/scn2img_examples/cat_at_beach.scn2img.md

@@ -0,0 +1,40 @@
+// blend it together and finish it with details
+prompt: cute happy orange cat sitting at beach, beach in background, trending on artstation:1 cute happy cat:1
+sampler_name:k_euler_a
+ddim_steps: 35
+denoising_strength: 0.55
+variation: 3
+initial_seed: 1
+
+# put foreground onto background 
+size: 512, 512
+color: 0,0,0
+
+## create foreground
+size:512,512
+color:0,0,0,0
+resize: 300, 300
+pos: 256, 350
+
+// select mask by probing some pixels from the image 
+mask_by_color_at: 15, 15,   15, 256,   85, 465,  100, 480
+mask_by_color_threshold:80
+mask_by_color_space: HLS
+
+// some pixels inside the cat may be selected, remove them with mask_open
+mask_open: 15
+
+// there is still some background pixels left at the edge between cat and background
+// grow the mask to get them as well
+mask_grow: 15
+
+// we want to remove whatever is masked:
+mask_invert: True
+
+####
+prompt: cute happy orange cat, white background
+ddim_steps: 25
+variation: 1
+
+## create background
+prompt:beach landscape, beach with ocean in background, photographic, beautiful:1 red:-0.4

data/scn2img_examples/corgi_3d_transformation.scn2img.md

@@ -0,0 +1,50 @@
+initial_seed: 2
+
+// select background and img2img over it
+mask_by_color_at: 64, 64
+mask_invert: True
+
+prompt: corgi
+ddim_steps: 50
+seed: 242886303
+
+mask_mode: 0
+denoising_strength: 0.8
+//cfg_scale: 15
+mask_restore: True
+image_editor_mode:Mask
+
+# estimate depth and transform the corgi in 3d
+transform3d: True
+transform3d_depth_near: 0.5
+transform3d_depth_scale: 10
+transform3d_from_hfov: 45
+transform3d_to_hfov: 45
+transform3d_from_pose: 0,0,0,  0,0,0
+transform3d_to_pose: 0.5,0,0,  0,-5,0
+transform3d_min_mask: 0
+transform3d_max_mask: 255
+transform3d_inpaint_radius: 1
+transform3d_inpaint_method: 0
+
+## put foreground onto background 
+size: 512, 512
+
+
+### create foreground
+size: 512, 512
+
+mask_depth: True
+mask_depth_model: 1
+mask_depth_min: -0.05
+mask_depth_max: 0.5
+mask_depth_invert:False
+
+####
+prompt: corgi
+ddim_steps: 25
+seed: 242886303
+
+### background
+size: 512,512
+color: #9F978D

data/scn2img_examples/corgi_at_beach.scn2img.md

@@ -0,0 +1,25 @@
+// blend it together and finish it with some details
+prompt: cute corgi at beach, trending on artstation
+ddim_steps: 50
+denoising_strength: 0.5
+initial_seed: 2
+
+# put foreground onto background 
+size: 512, 512
+
+## create foreground
+size: 512, 512
+
+// estimate depth from image and select mask by depth
+// https://huggingface.co/spaces/atsantiago/Monocular_Depth_Filter
+mask_depth: True
+mask_depth_min: -0.05
+mask_depth_max: 0.4
+mask_depth_invert:False
+
+###
+prompt: corgi
+ddim_steps: 25
+
+## create background
+prompt:beach landscape, beach with ocean in background, photographic, beautiful:1 red:-0.4

data/scn2img_examples/corgi_at_beach_2.scn2img.md

@@ -0,0 +1,34 @@
+// give it some polish and details
+size: 512, 512
+prompt: cute corgi at beach, intricate details, photorealistic, trending on artstation
+variation: 0
+seed: 1360051694
+initial_seed: 5
+
+# blend it together
+prompt: beautiful corgi:1.5 cute corgi at beach, trending on artstation:1 photorealistic:1.5
+ddim_steps: 50
+denoising_strength: 0.5
+variation: 0
+
+## put foreground in front of background
+size: 512, 512
+
+### select foreground
+size: 512, 512
+
+// estimate depth from image and select mask by depth
+// https://huggingface.co/spaces/atsantiago/Monocular_Depth_Filter
+mask_depth: True
+mask_depth_min: -0.05
+mask_depth_max: 0.4
+mask_depth_invert:False
+
+#### create foreground
+prompt: corgi
+ddim_steps: 25
+seed: 242886303
+
+### create background
+prompt:beach landscape, beach with ocean in background, photographic, beautiful:1 red:-0.4
+variation: 3

data/scn2img_examples/landscape_3d.scn2img.md

@@ -0,0 +1,37 @@
+size: 512,512
+mask_blur: 6
+
+prompt: fantasy landscape with castle and forest and waterfalls, trending on artstation
+denoising_strength: 0.6
+seed: 1
+image_editor_mode: Mask
+mask_mode: 0
+mask_restore: True
+
+# mask the left which contains artifacts
+color: 255,255,255,0
+blend:multiply
+size: 100,512
+pos: 50,256
+
+# mask the top-left which contains lots of artifacts
+color: 255,255,255,0
+blend:multiply
+size: 280,128
+pos: 128,64
+
+# go forward and turn head left to look at the left waterfalls
+transform3d: True
+transform3d_depth_scale: 10000
+transform3d_from_hfov: 60
+transform3d_to_hfov: 60
+transform3d_from_pose: 0,0,0,  0,0,0
+transform3d_to_pose: 4000,0,2000,  0,-50,0
+transform3d_min_mask: 0
+transform3d_max_mask: 255
+transform3d_inpaint_radius: 5
+transform3d_inpaint_method: 1
+
+##
+prompt: fantasy landscape with castle and forest and waterfalls, trending on artstation
+seed: 1

environment.yaml

@@ -40,7 +40,7 @@ dependencies:
     - albumentations==0.4.3
     - basicsr>=1.3.4.0
     - diffusers==0.3.0
-    - einops==0.3.0
+    - einops==0.3.1
     - facexlib>=0.2.3
     - ftfy==6.1.1
     - fairscale==0.4.4
@@ -72,9 +72,10 @@ dependencies:
     - streamlit-tensorboard==0.0.2
     - test-tube>=0.7.5
     - tensorboard==2.10.1
-    - timm==0.4.12
+    - timm==0.6.7
     - torch-fidelity==0.3.0
     - torchmetrics==0.6.0
     - transformers==4.19.2
+    - tensorflow==2.10.0
     - tqdm==4.64.0
-    
+