HarmonyView update

.idea/workspace.xml

@@ -6,6 +6,7 @@
   <component name="ChangeListManager">
     <list default="true" id="a993d736-6297-4164-9c29-6b2ab1055a96" name="변경" comment="HarmonyView update">
       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/app.py" beforeDir="false" afterPath="$PROJECT_DIR$/app.py" afterDir="false" />
     </list>
     <option name="SHOW_DIALOG" value="false" />
     <option name="HIGHLIGHT_CONFLICTS" value="true" />
@@ -32,7 +33,7 @@
     "RunOnceActivity.OpenProjectViewOnStart": "true",
     "RunOnceActivity.ShowReadmeOnStart": "true",
     "git-widget-placeholder": "main",
-    "last_opened_file_path": "/home/byeongjun/PycharmProjects/HarmonyView"
+    "last_opened_file_path": "/home/byeongjun/HarmonyView"
   }
 }]]></component>
   <component name="RecentsManager">
@@ -57,6 +58,9 @@
       <updated>1703058146297</updated>
       <workItem from="1703058147338" duration="769000" />
       <workItem from="1703224127800" duration="3023000" />
+      <workItem from="1703644898438" duration="448000" />
+      <workItem from="1703677779702" duration="730000" />
+      <workItem from="1703731914758" duration="8515000" />
     </task>
     <task id="LOCAL-00001" summary="error resolve">
       <option name="closed" value="true" />
@@ -138,7 +142,31 @@
       <option name="project" value="LOCAL" />
       <updated>1703132120481</updated>
     </task>
-    <option name="localTasksCounter" value="11" />
+    <task id="LOCAL-00011" summary="HarmonyView update">
+      <option name="closed" value="true" />
+      <created>1703645193143</created>
+      <option name="number" value="00011" />
+      <option name="presentableId" value="LOCAL-00011" />
+      <option name="project" value="LOCAL" />
+      <updated>1703645193143</updated>
+    </task>
+    <task id="LOCAL-00012" summary="HarmonyView update">
+      <option name="closed" value="true" />
+      <created>1703645337792</created>
+      <option name="number" value="00012" />
+      <option name="presentableId" value="LOCAL-00012" />
+      <option name="project" value="LOCAL" />
+      <updated>1703645337792</updated>
+    </task>
+    <task id="LOCAL-00013" summary="HarmonyView update">
+      <option name="closed" value="true" />
+      <created>1703677831151</created>
+      <option name="number" value="00013" />
+      <option name="presentableId" value="LOCAL-00013" />
+      <option name="project" value="LOCAL" />
+      <updated>1703677831151</updated>
+    </task>
+    <option name="localTasksCounter" value="14" />
     <servers />
   </component>
   <component name="Vcs.Log.Tabs.Properties">

app.py

@@ -24,14 +24,14 @@ Given a single-view image, HarmonyView is able to generate multiview-consistent
 
 Procedure: </br>
 **Step 1**. Upload an image or select an example.  ==> The foreground is masked out by SAM and we crop it as inputs. </br>
-**Step 2**. Select "Elevation angle "and click "Run generation". ==> Generate multiview images. The **Elevation angle** is the elevation of the input image. (This costs about 30s.) </br>
+**Step 2**. Select "Elevation angle "and click "Run generation". ==> Generate multiview images. The **Elevation angle** is the elevation of the input image. (This costs about 45s.) </br>
 You may adjust the **Crop size** and **Elevation angle** to get a better result! <br>
 To reconstruct a NeRF or a 3D mesh from the generated images, please refer to our [github repository](https://github.com/byeongjun-park/HarmonyView). <br>
 We have heavily borrowed codes from [Syncdreamer](https://huggingface.co/spaces/liuyuan-pal/SyncDreamer), which is an our strong baseline. 
 '''
 _USER_GUIDE0 = "Step1: Please upload an image in the block above (or choose an example shown in the left)."
 # _USER_GUIDE1 = "Step1: Please select a **Crop size** and click **Crop it**."
-_USER_GUIDE2 = "Step2: Please choose a **Elevation angle** and click **Run Generate**. The **Elevation angle** is the elevation of the input image. This costs about 30s."
+_USER_GUIDE2 = "Step2: Please choose a **Elevation angle** and click **Run Generate**. The **Elevation angle** is the elevation of the input image. This costs about 45s."
 _USER_GUIDE3 = "Generated multiview images are shown below! (You may adjust the **Crop size** and **Elevation angle** to get a better result!)"
 
 others = '''**Step 1**. Select "Crop size" and click "Crop it". ==> The foreground object is centered and resized. </br>'''
@@ -65,7 +65,8 @@ class BackgroundRemoval:
         image = self.interface([image])[0]
         return image
 
-def resize_inputs(image_input, crop_size):
+def resize_inputs(original_image, sam_image, crop_size, background_removal):
+    image_input = original_image if background_removal == "Original Image" else sam_image
     if image_input is None: return None
     alpha_np = np.asarray(image_input)[:, :, 3]
     coords = np.stack(np.nonzero(alpha_np), 1)[:, (1, 0)]
@@ -194,8 +195,7 @@ def run_demo():
         with gr.Row():
             with gr.Column(scale=1):
                 gr.Markdown('# ' + _TITLE)
-            # with gr.Column(scale=0):
-            #     gr.DuplicateButton(value='Duplicate Space for private use', elem_id='duplicate-button')
+
         gr.Markdown(_DESCRIPTION)
 
         with gr.Row(variant='panel'):
@@ -211,50 +211,39 @@ def run_demo():
 
             with gr.Column(scale=0.8):
                 image_block.render()
-                guide_text = gr.Markdown(_USER_GUIDE0, visible=True)
+                # guide_text = gr.Markdown(_USER_GUIDE0, visible=True)
+                crop_size.render()
                 fig0 = gr.Image(value=Image.open('assets/crop_size.jpg'), type='pil', image_mode='RGB', height=256, show_label=False, tool=None, interactive=False)
 
 
             with gr.Column(scale=0.8):
                 sam_block = gr.Image(type='pil', image_mode='RGBA', label="SAM output", height=256, interactive=False)
-                crop_size.render()
                 # crop_btn = gr.Button('Crop it', variant='primary', interactive=True)
+                elevation.render()
                 fig1 = gr.Image(value=Image.open('assets/elevation.jpg'), type='pil', image_mode='RGB', height=256, show_label=False, tool=None, interactive=False)
 
             with gr.Column(scale=0.8):
-                input_block = gr.Image(type='pil', image_mode='RGBA', label="Input to SyncDreamer", height=256, interactive=False)
-                elevation.render()
-                with gr.Accordion('Advanced options', open=False):
-                    cfg_scale_1 = gr.Slider(1.0, 5.0, 2.0, step=0.1, label='Classifier free guidance 1', interactive=True)
-                    cfg_scale_2 = gr.Slider(0.5, 1.5, 1.0, step=0.1, label='Classifier free guidance 2', interactive=True)
-                    sample_num = gr.Slider(1, 2, 1, step=1, label='Sample num', interactive=False, info='How many instance (16 images per instance)')
-                    sample_steps = gr.Slider(10, 300, 50, step=10, label='Sample steps', interactive=False)
-                    batch_view_num = gr.Slider(1, 16, 16, step=1, label='Batch num', interactive=True)
+                input_block = gr.Image(type='pil', image_mode='RGBA', label="Input to HarmonyView", height=256, interactive=False)
+                with gr.Accordion('Advanced options', open=True):
+                    background_removal = gr.CheckboxGroup(["Original Image", "SAM Output"], label="Input to HarmonyView", info="Which image do you want for the input?")
+                    cfg_scale_1 = gr.Slider(1.0, 5.0, 2.0, step=0.1, label='Classifier free guidance 1', info='How consistent to be with the input image', interactive=True)
+                    cfg_scale_2 = gr.Slider(0.5, 1.5, 1.0, step=0.1, label='Classifier free guidance 2', info='How diverse a novel view to create', interactive=True)
                     seed = gr.Number(6033, label='Random seed', interactive=True)
                 run_btn = gr.Button('Run generation', variant='primary', interactive=True)
 
+        output_block = gr.Image(type='pil', image_mode='RGB', label="Outputs of HarmonyView", height=256, interactive=False)
 
-        output_block = gr.Image(type='pil', image_mode='RGB', label="Outputs of SyncDreamer", height=256, interactive=False)
-
-        def update_guide2(text, im):
-            if im is None:
-                return _USER_GUIDE0
-            else:
-                return text
-        update_guide = lambda GUIDE_TEXT: gr.update(value=GUIDE_TEXT)
 
-        image_block.clear(fn=partial(update_guide, _USER_GUIDE0), outputs=[guide_text], queue=False)
         image_block.change(fn=partial(sam_predict, mask_predictor, removal), inputs=[image_block], outputs=[sam_block], queue=True) \
-                   .success(fn=resize_inputs, inputs=[sam_block, crop_size], outputs=[input_block], queue=True)\
-                   .success(fn=partial(update_guide2, _USER_GUIDE2), inputs=[image_block], outputs=[guide_text], queue=False)\
+                   .success(fn=resize_inputs, inputs=[image_block, sam_block, crop_size, background_removal], outputs=[input_block], queue=True)
 
-        crop_size.change(fn=resize_inputs, inputs=[sam_block, crop_size], outputs=[input_block], queue=True)\
-                 .success(fn=partial(update_guide, _USER_GUIDE2), outputs=[guide_text], queue=False)
-        # crop_btn.click(fn=resize_inputs, inputs=[sam_block, crop_size], outputs=[input_block], queue=False)\
-        #                .success(fn=partial(update_guide, _USER_GUIDE2), outputs=[guide_text], queue=False)
+        background_removal.change(fn=resize_inputs, inputs=[image_block, sam_block, crop_size, background_removal], outputs=[input_block], queue=True)
+        crop_size.change(fn=resize_inputs, inputs=[image_block, sam_block, crop_size, background_removal], outputs=[input_block], queue=True)
 
-        run_btn.click(partial(generate, model), inputs=[sample_steps, batch_view_num, sample_num, cfg_scale_1, cfg_scale_2, seed, input_block, elevation], outputs=[output_block], queue=True)\
-               .success(fn=partial(update_guide, _USER_GUIDE3), outputs=[guide_text], queue=False)
+        sample_num = 1
+        sample_steps = 50
+        batch_view_num = 16
+        run_btn.click(partial(generate, model), inputs=[sample_steps, batch_view_num, sample_num, cfg_scale_1, cfg_scale_2, seed, input_block, elevation], outputs=[output_block], queue=True)
 
     demo.queue().launch(share=False, max_threads=80)  # auth=("admin", os.environ['PASSWD'])