diff --git a/applications/gradio_autolable.py b/applications/gradio_autolable.py new file mode 100644 index 000000000..c2b765e90 --- /dev/null +++ b/applications/gradio_autolable.py @@ -0,0 +1,187 @@ +from paddlemix.appflow import Appflow +from ppdiffusers.utils import load_image +import paddle +import cv2 + +import os +import json +from zipfile import ZipFile +import zipfile +import numpy as np +from PIL import Image, ImageDraw +import gradio as gr +import traceback +import math +import tempfile + + +task = Appflow(app="auto_label", + models=["paddlemix/blip2-caption-opt2.7b","GroundingDino/groundingdino-swint-ogc","Sam/SamVitH-1024"]) + + +def auto_label(img, prompt): + result = task(image=img,blip2_prompt = prompt) + return result + + +def result2json(result, filename): + label_data = {'version': '0.0.0', + 'flags': {} , + 'shapes': [], + 'imagePath': filename, + 'imageHeight': result['image'].size[1], + 'imageWidth': result['image'].size[0]} + + for i in range(len(result['labels'])): + # label去掉末尾的置信度 + label = result['labels'][i] + spl_idx = -1 + for j in range(len(label)): + if label[j] == '(': + spl_idx = j + if spl_idx == -1: + label = label + else: + label = label[:spl_idx] + + # 增加bbox + rect = result['boxes'][i].tolist() + xmin, ymin, xmax, ymax = rect + label_data['shapes'].append( + {'label': label, + 'points': [[xmin, ymin],[xmax, ymax]], + 'group_id': None, + 'shape_type': 'rectangle', + 'flags': {} + } + ) + + # 记录polygen + seg_mask = result['seg_masks'][i].numpy()[0] + mask_img = seg_mask.astype('uint8')*255 + contours, _ = cv2.findContours(mask_img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + points = [] + for contour in contours: + for point in contour: + points.append(point[0].tolist()) + + # 增加polygen + rect = result['boxes'][i] + xmin, ymin, xmax, ymax = rect + label_data['shapes'].append( + {'label': label, + 'points': points, + 'group_id': None, + 'shape_type': 'polygon', + 'flags': {} + } + ) + + return label_data + + +def generate_mask(img, result_masks): + divide_part = int(255/(math.ceil(len(result_masks)/3)+1)) + np_img = np.array(img) + for i in range(len(result_masks)): + color = [0,0,0] + c = i%3 + p = i//3+1 + color[c] = divide_part*p + mask = result_masks[i] + M = mask.numpy()[0] + np_img[M] = color + print(color) + img = Image.fromarray(np_img) + return img + + +def al_fun(img, prompt): + img = Image.fromarray(img.astype('uint8')).convert('RGB') + result = auto_label(img, prompt) + label_data = result2json(result, "tmpimg") + # Draw BBox + draw = ImageDraw.Draw(img) + for i in range(len(result['boxes'])): + rect = result['boxes'][i].tolist() + draw.rectangle(rect, width=10) + # Draw Mask + mask_img = generate_mask(result['image'], result['seg_masks']) + # Write File + labeled_file = os.path.join(tmpdir,'labeled_date.json') + with open(labeled_file,'w') as f: + json.dump(label_data, f, indent=4) + return img, mask_img, labeled_file + + +def al_file_fun(file_in, prompt): + out_zip_file = os.path.join(tmpdir, "labeled.zip") + with ZipFile(out_zip_file, "w") as zipObj: + for _, imgname in enumerate(file_in): + image_pil = Image.open(imgname.name) + result = auto_label(image_pil, prompt) + label_data = result2json(result, imgname.name.split("/")[-1]) + labeled_file = os.path.join(tmpdir,imgname.name.split("/")[-1]+'.josn') + with open(labeled_file,'w') as f: + json.dump(label_data, f, indent=4) + zipObj.write(labeled_file) + return out_zip_file + + +def al_zip_fun(zip_in, prompt): + for _, zipname in enumerate(zip_in): + with open('test.txt', 'a') as f: + f.write(zipname.name+'\n') + f.write(zipname.name+'\n') + zipfile.ZipFile(zipname.name).extractall(tmpdir) + with open('test.txt', 'a') as f: + f.write('\n after extract \n') + out_zip_file = os.path.join(tmpdir, "labeled.zip") + with ZipFile(out_zip_file, "w") as zipObj: + for root, _, files in os.walk(tmpdir, topdown=False): + for name in files: + if name.split('.')[-1] in ['jpg', 'png', 'jpeg', 'JPG', 'PNG', 'JPEG']: + img_path = os.path.join(root, name) + json_path = os.path.join(root, name+'.json') + + image_pil = Image.open(img_path) + result = auto_label(image_pil, prompt) + label_data = result2json(result, img_path) + with open(json_path,'w') as f: + json.dump(label_data, f, indent=4) + zipObj.write(json_path) + os.remove(img_path) + return out_zip_file + + +with gr.Blocks() as demo: + gr.Markdown("# 自动标注(AutoLabel)") + with gr.Tab("单张图片标注"): + with gr.Row(): + al_image_in = gr.Image(label = "输入图片") + al_image_out1 = gr.Image(label = "BBox标注图片") + al_image_out2 = gr.Image(label = "Mask标注图片") + al_text_in = gr.Text(label = "Prompt", value="describe the image") + al_file_out_ = gr.File(label = "标注文件") + al_button = gr.Button() + al_button.click(fn=al_fun, inputs = [al_image_in, al_text_in], outputs = [al_image_out1, al_image_out2, al_file_out_]) + with gr.Tab("上传多张图片批量标注"): + with gr.Row(): + al_file_in = gr.Files(label = "上传多张图片", file_types=['.jpg', '.png', '.jpeg', '.JPG', '.PNG', '.JPEG']) + al_file_out = gr.File(label = "标注结果") + al_file_text_in = gr.Text(label = "Prompt", value="describe the image") + al_file_button = gr.Button() + al_file_button.click(fn=al_file_fun, inputs = [al_file_in, al_file_text_in], outputs = [al_file_out]) + with gr.Tab("上传压缩包批量标注"): + with gr.Row(): + al_zip_in = gr.Files(label = "上传压缩包", file_types=['.zip']) + al_zip_out = gr.File(label = "标注结果") + al_zip_text_in = gr.Text(label = "Prompt", value="describe the image") + al_zip_button = gr.Button() + al_zip_button.click(fn=al_zip_fun, inputs = [al_zip_in, al_zip_text_in], outputs = [al_zip_out]) + + +# for download file, use the tempfile +global tmpdir +with tempfile.TemporaryDirectory(dir='.') as tmpdir: + demo.launch() diff --git a/applications/gradio_text2image.py b/applications/gradio_text2image.py new file mode 100644 index 000000000..1b9ee6f14 --- /dev/null +++ b/applications/gradio_text2image.py @@ -0,0 +1,54 @@ +from paddlemix.appflow import Appflow +from ppdiffusers.utils import load_image +import paddle +import imageio + +from PIL import Image +import gradio as gr +import traceback + +# upscaling +def ups_fun(low_res_img, prompt): + low_res_img = Image.fromarray(low_res_img.astype('uint8')).convert('RGB') + app = Appflow(app='image2image_text_guided_upscaling',models=['stabilityai/stable-diffusion-x4-upscaler']) + image = app(prompt=prompt,image=low_res_img)['result'] + return image + +# text_guided_generation +def tge_fun(image, prompt_pos, prompt_neg): + image = Image.fromarray(image.astype('uint8')).convert('RGB') + app = Appflow(app='image2image_text_guided_generation',models=['Linaqruf/anything-v3.0']) + image = app(prompt=prompt_pos,negative_prompt=prompt_neg,image=image)['result'] + return image + +# video_generation +def vge_fun(prompt): + app = Appflow(app='text_to_video_generation',models=['damo-vilab/text-to-video-ms-1.7b']) + video_frames = app(prompt=prompt,num_inference_steps=25)['result'] + imageio.mimsave("gen_video.gif", video_frames, duration=8) + return "gen_video.gif" + +with gr.Blocks() as demo: + gr.Markdown("# Appflow应用:text2image") + with gr.Tab("文本引导的图像放大"): + with gr.Row(): + ups_image_in = gr.Image(label = "输入图片") + ups_image_out = gr.Image(label = "输出图片") + ups_text_in = gr.Text(label = "Prompt") + ups_button = gr.Button() + ups_button.click(fn=ups_fun, inputs = [ups_image_in, ups_text_in], outputs = [ups_image_out]) + with gr.Tab("文本引导的图像变换"): + with gr.Row(): + tge_image_in = gr.Image(label = "输入图片") + tge_image_out = gr.Image(label = "输出图片") + tge_text_pos_in = gr.Text(label = "Positive Prompt") + tge_text_neg_in = gr.Text(label = "Negative Prompt") + tge_button = gr.Button() + tge_button.click(fn=tge_fun, inputs = [tge_image_in, tge_text_pos_in, tge_text_neg_in], outputs = [tge_image_out]) + with gr.Tab("文本条件的视频生成"): + vge_text_in = gr.Text(label = "Prompt") + vge_video_out = gr.Video(label = "输出视频") + vge_button = gr.Button() + vge_button.click(fn=vge_fun, inputs = [vge_text_in], outputs = [vge_video_out]) + +demo.launch()