Spaces:
Runtime error
Runtime error
| import subprocess | |
| subprocess.run(['wget', '-N', '-q', 'https://matematica.unipv.it/gualandi/solvers/ipopt-linux64.zip'], stdout=subprocess.DEVNULL) | |
| subprocess.run(['unzip', '-o', '-q', 'ipopt-linux64'], stdout=subprocess.DEVNULL) | |
| from pyomo.environ import * | |
| import pickle | |
| import pickle | |
| dict_files = { | |
| 'banana republic': 'annotations/banana_annotations.pickle', | |
| 'h&m': 'annotations/hnm_annotations.pickle', | |
| 'uniqlo': 'annotations/uniqlo_annotations.pickle', | |
| 'street': 'annotations/street_annotations.pickle', | |
| 'abercrombie women': 'annotations/abercrombie_women.pickle', | |
| 'aritzia': 'annotations/aritzia.pickle', | |
| 'dynamite': 'annotations/dynamite.pickle', | |
| } | |
| style_gender = { | |
| 'banana republic': 'male', | |
| 'h&m': 'male', | |
| 'uniqlo': 'male', | |
| 'street': 'male', | |
| 'abercrombie women': 'female', | |
| 'aritzia': 'female', | |
| 'dynamite': 'female', | |
| } | |
| gender_article_keys = { | |
| 'male': ['shirt', 'pants_or_shorts', 'hoodie_or_jacket', 'hat', 'shoes'], | |
| 'female': ['top', 'bottom', 'dress', 'hoodie_or_jacket', 'hat', 'shoes'] | |
| } | |
| style_annotations_dict = {} | |
| def get_annotations(dict_file): | |
| with open(dict_file, 'rb') as f: | |
| annotations = pickle.load(f) | |
| for a in annotations: | |
| if not( ( a.get('shirt', '') and a.get('pants_or_shorts', '') ) or | |
| ( a.get('top', '') and a.get('bottom', '') ) or | |
| a.get('dress', '') | |
| ): | |
| print( dict_file, '\n', a ) | |
| filtered_annotations = [ a for a in annotations if ( (a.get('shirt', '') and a.get('pants_or_shorts', '')) or ( | |
| a.get('gender', '') == 'female' ) ) ] | |
| return filtered_annotations | |
| style_annotations_dict = { | |
| k: get_annotations(file_path) for k, file_path in dict_files.items() | |
| } | |
| import collections | |
| from collections import defaultdict | |
| optional = [False, False, True, True, False] | |
| article_keys = ['shirt', 'pants_or_shorts', 'hoodie_or_jacket', 'hat', 'shoes'] | |
| def annotation_to_tuple(annotation): | |
| t = tuple( annotation[key] for key in article_keys ) | |
| return t | |
| def remove_owned_articles(annotations_tuple, clothes_owned): | |
| return tuple( article if article not in clothes_owned else '' | |
| for article in annotations_tuple ) | |
| def annotations_value(annotation_tuple_count, clothes_owned): | |
| new_scores = {} | |
| for annotation_tuple, count in annotation_tuple_count.items(): | |
| num_missing = 0 | |
| for clothing in annotation_tuple: | |
| if not (clothing == '' or clothing in clothes_owned): | |
| num_missing += 1 | |
| if num_missing > 0: | |
| new_scores[annotation_tuple] = count/num_missing | |
| return new_scores | |
| def is_outfit_subset(smaller_outfit, larger_outfit): | |
| if smaller_outfit == larger_outfit: | |
| return False | |
| for i in range(len(larger_outfit)): # assume both r tuples of fixed size | |
| if smaller_outfit[i] != '' and smaller_outfit[i] != larger_outfit[i]: | |
| return False | |
| return True | |
| def most_outfits_helper(annotations, capacity=5, clothes_owned=[]): | |
| annotations = [ annotation_to_tuple(e) for e in annotations ] | |
| outfit_count = dict(collections.Counter(annotations)) | |
| outfits = list(outfit_count.keys()) | |
| for small_outfit in outfit_count: | |
| for larger_outfit in outfit_count: | |
| if is_outfit_subset(small_outfit, larger_outfit): | |
| outfit_count[small_outfit] += outfit_count[larger_outfit] | |
| clothes_owned += [''] | |
| best_outfits = most_outfits(outfit_count, capacity, set(clothes_owned)) | |
| best_outfits.remove('') | |
| return best_outfits | |
| # consider every image in the dataset | |
| # each image has an outfit and casts a vote for that outfit | |
| # a outfit is a tuple of pants_or_short + shirt + jacket_or_hoodie? | |
| # images with shoes or hats have an additional vote for the shoes and hats | |
| # greedily pick the clothes with the highest votes | |
| # then remove those clothes from all outfits in that dataset and recount | |
| # can be solved as an integer programming problem | |
| def most_outfits(annotation_tuple_count, capacity, clothes_owned): | |
| if capacity == 0 : | |
| return clothes_owned | |
| # merge counts based on new keys | |
| updated_annotations = defaultdict(int) | |
| for tup, count in annotation_tuple_count.items(): | |
| updated_annotations[ remove_owned_articles(tup, clothes_owned) ] += count | |
| annotation_tuple_count = updated_annotations | |
| outfits_scores = annotations_value(annotation_tuple_count, clothes_owned) | |
| outfits_scores = sorted(outfits_scores.items(), key=lambda x:-x[1]) | |
| num_new = 1 | |
| for outfit, score in outfits_scores: | |
| clothes_proposed = clothes_owned.union(set(outfit)) | |
| num_new = len(clothes_proposed) - len(clothes_owned) | |
| if num_new <= capacity: | |
| clothes_owned = clothes_proposed | |
| break | |
| return most_outfits( annotation_tuple_count, capacity-num_new, clothes_owned ) | |
| from pyomo.environ import * | |
| from functools import reduce | |
| def cover_style_ip(annotations, capacity=10, clothes_owned=[], mask=None, solver='ipopt'): | |
| """Use integer program to find the set of clothes that makes as many outfits as possible. | |
| annotations: List[ Dict ], contains maps from clothing categories to string types | |
| capacity: int, number of new clothes to find | |
| clothes_owned: List[ Str ], iterable of strings of clothing types | |
| mask: Optional( List[ Str ] ), optional iterable of categories of clothes to consider. | |
| Uses all clothing types by default. | |
| solver: Str, the nonlinear optimization solver to use for max bool sat | |
| """ | |
| unique_clothes = set() | |
| interested_clothing_types = article_keys if mask is None else mask | |
| for a in annotations: | |
| for key in interested_clothing_types: | |
| unique_clothes.add( a[key] ) | |
| if '' in unique_clothes: | |
| unique_clothes.remove('') | |
| model = ConcreteModel() | |
| # 1. Create the variables we want to optimize | |
| clothing_literals_dict = {clothing: Var(within=Binary) for clothing in unique_clothes} | |
| # set literals to true for clothes already owned | |
| for clothing in clothes_owned: | |
| clothing_literals_dict[clothing] = 1 | |
| # capacity changes from number of new clothes to total clothes | |
| capacity += len(clothes_owned) | |
| capacity_constraint = Constraint(expr=sum(list(clothing_literals_dict.values())) <= capacity) | |
| outfit_clauses = [] | |
| for a in annotations: | |
| # get the clothing literal if it exists, otherwise it's masked and say it's absent | |
| outfit_literals = [ clothing_literals_dict.get(a[key], 0) for key in article_keys if a[key] != '' ] | |
| outfit_clauses += [ reduce(lambda x,y: x*y, outfit_literals) ] | |
| # 3. Maximize the objective function | |
| objective = Objective( expr=sum( outfit_clauses ), sense=maximize) | |
| for name, literal in clothing_literals_dict.items(): | |
| setattr(model, name, literal) | |
| model.capacity_constraint = capacity_constraint | |
| model.objective = objective | |
| SolverFactory(solver).solve(model) | |
| basis_clothes = [ name for name, literal in clothing_literals_dict.items() if not isinstance(literal, int) and round(literal()) ] | |
| return basis_clothes | |
| def annotation_str(annotation): | |
| output = "" | |
| output += annotation['hat'] | |
| output += ' + ' + annotation['shirt'] if output and annotation['shirt'] else annotation['shirt'] | |
| output += ' + ' + annotation['hoodie_or_jacket'] if output and annotation['hoodie_or_jacket'] else annotation['hoodie_or_jacket'] | |
| output += ' + ' + annotation['pants_or_shorts'] if output and annotation['pants_or_shorts'] else annotation['pants_or_shorts'] | |
| output += ' + ' + annotation['shoes'] if output and annotation['shoes'] else annotation['shoes'] | |
| # output += ' ' + annotation['url'] if annotation.get('url') else '' | |
| return output | |
| def annotation_str(annotation): | |
| gender = annotation.get('gender', 'male') | |
| output = "" | |
| for k in gender_article_keys[gender]: | |
| output += ' + ' + annotation[k] if output and annotation.get(k) else annotation.get(k, '') | |
| return output | |
| def annotation_to_key(annotation): | |
| gender = annotation.get('gender', 'male') | |
| output = "" | |
| if gender == 'male': | |
| output += ' + ' + annotation['shirt'] if output and annotation['shirt'] else annotation['shirt'] | |
| output += ' + ' + annotation['hoodie_or_jacket'] if output and annotation['hoodie_or_jacket'] else annotation['hoodie_or_jacket'] | |
| output += ' + ' + annotation['pants_or_shorts'] if output and annotation['pants_or_shorts'] else annotation['pants_or_shorts'] | |
| else: | |
| useful_keys = ['dress', 'top', 'bottom', 'hoodie_or_jacket'] | |
| for k in useful_keys: | |
| output += ' + ' + annotation[k] if output and annotation.get(k) else annotation.get(k, '') | |
| return output | |
| def get_num_outfits(annotations, articles): | |
| outfits = set() | |
| for e in annotations: | |
| if (sum([(e[key] == '' or e[key] in articles ) for key in article_keys]) == len(article_keys) | |
| # and e['shirt'] and e['pants_or_shorts'] | |
| ): | |
| str_form = annotation_to_key(e) # ignore +- hat, shoes otherwise use annotation_str | |
| outfits.add(str_form) | |
| return sorted(list(outfits)) | |
| def get_outfit_urls(annotations, outfits): | |
| outfit_urls = defaultdict(list) | |
| for e in annotations: | |
| str_form = annotation_to_key(e) | |
| if e.get('url') and str_form in outfits: | |
| outfit_urls[str_form] += [ e['url'] ] | |
| return dict(outfit_urls) | |
| def cover_style(annotations, capacity=10, clothes_owned=[]): | |
| clothes = list(cover_style_ip(annotations, capacity=capacity, clothes_owned=clothes_owned)) | |
| reachable_outfits = get_num_outfits(annotations, set(clothes+clothes_owned) ) | |
| if len(clothes_owned) == 0: # return basis outfits from scratch | |
| return clothes, get_outfit_urls(annotations, reachable_outfits) | |
| elif capacity == 0: # return reach of current clothes | |
| return clothes, get_outfit_urls(annotations, reachable_outfits) | |
| else: # capacity > 0 and len(clothes_owned) > 0, show new clothes and outfits | |
| new_clothes = [ c for c in clothes if c not in clothes_owned ] | |
| reachable_outfits_base = get_num_outfits(annotations, clothes_owned) | |
| new_outfits = [ o for o in reachable_outfits if o not in reachable_outfits_base ] | |
| return new_clothes, get_outfit_urls(annotations, new_outfits) | |
| def str_to_list(input): | |
| tokens = [ token.strip() for token in input.split(',') ] | |
| return [ t for t in tokens if t != '' ] | |
| CLOTHES_HEADER = f'## Most Effective Clothes' | |
| OUTFITS_HEADER = f'## Possible 👕😎 Outfits' | |
| def cover_style_helper(styles, capacity=10, clothes_owned=''): | |
| if len(styles) == 0: | |
| return f'{CLOTHES_HEADER}\nPlease pick at least one style from the left.' , OUTFITS_HEADER | |
| clothes_owned = str_to_list(clothes_owned) | |
| annotations = [] | |
| for style in styles: | |
| annotations += style_annotations_dict[style][:500] | |
| if len(styles) == 1: # hack for h&m having wayyyyy more examples than other brands | |
| annotations = style_annotations_dict[style] | |
| clothes, outfit_urls = cover_style(annotations, capacity, clothes_owned) | |
| clothes_str = f'{CLOTHES_HEADER}\n' | |
| for c in clothes: | |
| clothes_str += f'* {c}\n' | |
| outfits_str = f'{OUTFITS_HEADER} ({len(outfit_urls)})\n' | |
| for outfit, urls in outfit_urls.items(): | |
| outfits_str += f'1. {str(outfit)}: ' | |
| for i, u in enumerate(urls): | |
| outfits_str += f'<a href="{u}" target="_blank">[{i+1}]</a>' # f'[\[{i}\]]({u})' | |
| outfits_str += '\n' | |
| return clothes_str, outfits_str | |
| # article_keys = gender_article_keys['male'] | |
| # print(cover_style_helper( ['banana republic'] )) | |
| def cover_style_helper_wrapper(markdown, styles, capacity=10, clothes_owned=''): | |
| if len(styles) > 0: | |
| global article_keys | |
| gender = style_gender[styles[0]] | |
| article_keys = gender_article_keys[gender] | |
| return cover_style_helper(styles, capacity, clothes_owned) | |
| def filter_test(annotation, filter_set): | |
| for f in filter_set: | |
| num_occur = sum([1 for key in article_keys+['caption'] if f in annotation[key]]) | |
| if num_occur == 0: | |
| return False | |
| return True | |
| import gradio as gr | |
| def change_gallery(style_choice, start_from=0, text_filter=''): | |
| global article_keys | |
| gender = style_gender[style_choice] | |
| article_keys = gender_article_keys[gender] | |
| chosen_annotations = style_annotations_dict[style_choice] | |
| if text_filter: | |
| text_filter = set([t.strip() for t in text_filter.split(',')]) | |
| chosen_annotations = [ a for a in chosen_annotations if filter_test(a, text_filter) ] | |
| start_from = int(start_from/100*len(chosen_annotations)) | |
| # print(len(chosen_annotations), [a['url'] for a in chosen_annotations[start_from:start_from+20]]) | |
| return [a['url'] for a in chosen_annotations[start_from:start_from+20]] | |
| def update_styles(gender): | |
| global article_keys | |
| article_keys = gender_article_keys[gender] | |
| default_values = ["banana republic"] if gender == "male" else ["aritzia"] | |
| return gr.CheckboxGroup.update(choices=[k for k in style_annotations_dict.keys() if style_gender[k] == gender], value=default_values, label='Styles') | |
| article_keys = gender_article_keys['male'] | |
| INTRO_MARKDOWN = """**Good clothes are the ones that you can use to make many outfits.** | |
| Finding stuff that works with your style and wardrobe is hard. | |
| This program helps you find the clothes that make as many fashionable outfits as possible, | |
| given your current wardrobe, style, and budget. | |
| """ | |
| with gr.Blocks() as demo: | |
| with gr.Tabs(selected=0): | |
| with gr.TabItem('Find the Literal Optimal Clothes', id=0): | |
| with gr.Box(): | |
| default_clothes = """black dress pants, | |
| blue shirt, | |
| white sweater, | |
| black sweater, | |
| white t shirt, | |
| black t shirt, | |
| black shorts, | |
| white shirt, | |
| grey hoodie, | |
| black t shirt, | |
| gray sweater, | |
| black jacket, | |
| black sneakers, | |
| white sneakers, | |
| blue jeans, | |
| black baseball hat, | |
| black jeans""" | |
| with gr.Row(): | |
| with gr.Column(): | |
| gr.Markdown(INTRO_MARKDOWN) | |
| with gr.Row(): | |
| with gr.Group(): | |
| gender = gr.Radio(["male", "female"], value="male", label='gender') | |
| styles = gr.CheckboxGroup([k for k in style_annotations_dict.keys() if style_gender[k] == gender.value], value=["banana republic"], label='Styles') | |
| gender.change(update_styles, inputs=[gender], outputs=[styles]) | |
| with gr.Group(): | |
| capacity = gr.Number(5, precision=0, label='Number of clothes to recommend') | |
| clothes_owned = gr.Textbox( | |
| label="Clothes Owned", | |
| lines=3, | |
| value=default_clothes, #json.dumps(default_clothes), | |
| ) | |
| with gr.Row(): | |
| btn = gr.Button("Recommend Clothes") | |
| with gr.Row(): | |
| clothes_markdown, outfits_markdown = cover_style_helper(styles.value, capacity.value, clothes_owned.value) | |
| clothes_recommended = gr.Markdown(clothes_markdown) | |
| possible_outfits = gr.Markdown(outfits_markdown) | |
| btn.click(cover_style_helper, inputs=[styles, capacity, clothes_owned], outputs=[clothes_recommended, possible_outfits]) | |
| gr.Markdown("## 3 Different Example Uses") | |
| example_label = gr.Textbox('', label='Explanation', visible=False) | |
| gr.Examples(examples=[ ['Find the central clothes that make a style.', ['street'], 10, ''], | |
| ['Find new outfits hidden in your wardrobe.', ['h&m'], 0, 'white t shirt, grey t shirt, black pants, black shorts, black t shirt, blue jeans'], | |
| ['Find the clothes that best fit your existing wardrobe.', ['banana republic', 'street'], 5, 'white t shirt, grey t shirt, black pants, black shorts, black t shirt, blue jeans']], | |
| inputs=[example_label, styles, capacity, clothes_owned], | |
| outputs=[clothes_recommended, possible_outfits], | |
| fn=cover_style_helper_wrapper, | |
| cache_examples=False) | |
| EXPLORE_MARKDOWN = """# Explore Styles | |
| Hint: Click an image to enable arrow key browsing. | |
| You put in clothes you own to find outfit ideas! | |
| """ | |
| with gr.TabItem('Explore Styles', id=1): | |
| with gr.Box(): | |
| gr.Markdown(EXPLORE_MARKDOWN) | |
| with gr.Row(): | |
| with gr.Column(): | |
| bad_urls = ['dynamite', 'abercrombie women'] | |
| styles = gr.Radio([ k for k in style_annotations_dict.keys() if k not in bad_urls ], value="uniqlo", label='Styles') | |
| start_from = gr.Slider(0, 100, label='Start from', value=0, step=10) | |
| with gr.Group(): | |
| text_filter = gr.Textbox(value='white t shirt, jeans', label='Outfit includes') | |
| filter_button = gr.Button('Apply Text Filter') | |
| with gr.Column(): | |
| outfits_gallery = gr.Gallery(label='Outfit Examples', value=change_gallery(styles.value, start_from.value, text_filter.value)) | |
| styles.change(fn=change_gallery, inputs=[styles, start_from, text_filter], outputs=outfits_gallery) | |
| start_from.change(fn=change_gallery, inputs=[styles, start_from, text_filter], outputs=outfits_gallery) | |
| filter_button.click(fn=change_gallery, inputs=[styles, start_from, text_filter], outputs=outfits_gallery) | |
| with gr.TabItem('How it works', id=2): | |
| ABOUT_MARKDOWN = """ | |
| # Why | |
| I don't really enjoy shopping, it takes a while a look at all the clothes, | |
| and I'm not sure if I should buy what I picked out. | |
| This program can tell me which clothes are the best for my wardrobe and style, | |
| so I don't have to worry about buying the wrong stuff. | |
| # How | |
| This program poses the problem as a nonlinear integer programming problem. | |
| The problem is to maximize the number of outfits, while contraining the number of clothes to be <= "this max you want to buy". | |
| Let a, b, c etc be binary literals that represent the use of clothing type a, b, c. | |
| These are clothing types like white dress shirt, or black hat. | |
| We maximize the nonlinear expression abc + ade + bec + ... similar to a disjuctive normal form, | |
| where each clause represents an outfit and is the product of clothing literals. | |
| Each outfit clause has value 1 iff every article of clothing is used in the solution. | |
| Pyomo is used to solve this optimization problem. | |
| This objective takes the form of something like, | |
|  | |
| Hart, William E., Carl Laird, Jean-Paul Watson, David L. Woodruff, Gabriel A. Hackebeil, Bethany L. Nicholson, and John D. Siirola. Pyomo – Optimization Modeling in Python. Springer, 2017. | |
| # Caveats | |
| This model understands fashion from a macro level. | |
| It understands what a white shirt is, but is entirely blind to micro features like fit, brand, or shape. | |
| It also bins different kinds of clothing and colors together. | |
| So the model could mix up light grey sweatpants and dark grey chinos because they're both grey pants. | |
| There is also error introduced from bad image annotations, approximate solvers, and style dataset size. | |
| Hence it's important to look at the outfits images under possible outfits for a human touch! | |
| """ | |
| with gr.Box(): | |
| gr.Markdown(ABOUT_MARKDOWN) | |
| article_keys = gender_article_keys['male'] | |
| demo.launch(debug=False) | |