add glossary function

app.py

@@ -216,7 +216,7 @@ def get_language(struct_lang, copy_lang):
 
 def get_model(model_name):
     if model_name.startswith("gpt"):
-        chat = ChatOpenAI(model=model_name, max_tokens=8192)
+        chat = ChatOpenAI(model=model_name, max_tokens=4096)
     elif model_name.startswith("claude"):
         chat = ChatAnthropic(model_name=model_name, anthropic_api_key=os.environ["ANTHROPIC_API_KEY"], max_tokens_to_sample=4096)
     else:
@@ -237,29 +237,37 @@ def build_glossary(glossary_file, fieldnames=None) -> VectorStoreRetriever:
     return retriever
 
 
-def glossary_rewrite(glossary: VectorStoreRetriever, text: str):
+def glossary_rewrite(chat: ChatOpenAI, glossary: VectorStoreRetriever, text: str):
     try:
         terms = glossary.invoke(input=text)
         print("\n".join([d.page_content for d in terms]))
         glossary_str = "\n\n".join([d.page_content.replace('\n', '. ') for d in terms])
-    except Exception as e:
-        print(e.__class__, e)
-        traceback.print_exc()
-        terms = []
 
-    if len(terms) > 0:
-        messages = [
-            SystemMessage(content=f"""You are a helpful assistant that writes about products for ecommerce websites. Make sure to write in English language."""),
-            HumanMessage(content=f"""Rewrite the following text using the terms in the glossary. 
+        if len(terms) > 0:
+            messages = [
+                SystemMessage(content=f"""You are a helpful assistant that writes about products for ecommerce websites. Make sure to write in English language."""),
+                HumanMessage(content=f"""Rewrite the following text using the terms in the glossary. 
 Preserve the original text as much as possible.
 Replace the terms in original text that match the definition with the corresponding terms in the glossary.
+Output only the rewritten text in markdown format.                         
 
 Terms, Definitions
-{glossary_str}                                                  
-"""),
-        ]
+{glossary_str}    
+
+Text to rewrite:
+{text}
+"""),]
+
+            print(f"HumanMessage={messages[1].content}")
+            response = chat.invoke(messages, temperature=0.0)
+            print(f"Response=\n{response.content}")
+            return response.content
+    except Exception as e:
+        print(e.__class__, e)
+        traceback.print_exc()
+        terms = []
 
-        print(f"HumanMessage={messages[1].content}")
+    return ""
 
 
 def generate(*data):
@@ -307,11 +315,11 @@ def generate(*data):
         intended_use = "Intended use: " + ", ".join(image_features["intended_use"])
         print(f"Detected features: {detected_features}, Intended use: {intended_use}, Alt text: {alt_texts}")
 
-    if glossary:
-        print("Getting terms")
-        terms = glossary.invoke(input=feature + detected_features)
-        for term in terms:
-            print(term)
+    # if glossary:
+    #     print("Getting terms")
+    #     terms = glossary.invoke(input=feature + detected_features)
+    #     for term in terms:
+    #         print(term)
 
     batch = []
     for i in range(visible + 1):
@@ -518,7 +526,11 @@ Excluded words: {excluded_words}"""),]
 
         descriptions.append(bestd)
 
-    # glossary_rewrite(glossary, descriptions[0])
+    print("\n\nRewriting with glossary")
+    rewrite = glossary_rewrite(chat=chat, glossary=glossary, text=descriptions[0])
+    if rewrite != "":
+        descriptions[0] = "Original:\n\n" + descriptions[0] + "\n\nRewritten:\n\n" + rewrite
+    print("\n\nDone rewriting with glossary\n\n")
 
     md_content = "\n\n---\n\n".join(descriptions)
 

fashion_glossary.csv

@@ -283,3 +283,4 @@ Watteau back,"The back of a garment in which box pleats are placed at the center
 weave types,"Fabrics made from yarns that are woven on a loom can be constructed in various ways. Lengthwise or warp yarns are placed on the loom first, and crosswise or weft (also called filling) yarns are interlaced with the warps. There are three basic weaves: plain weave, twill weave, and satin weave. In the plain weave, yarns in the first row of the weft cross over one warp yarn, under one, over one, etc., across the width of the fabric. In the second row, the weft passed under the first warp, over the second, under the third, etc. The third row follows the same pattern as the first. This is the simplest weave. By varying the kinds of yarns used, color of yarns, and size of yarns, many different fabrics can be produced. If several rows of weft yarns cross over several rows of warp yarns at a time, a variation of the plain weave called the basket weave can be made. In the twill weave, weft yarns interlace with warp yarns in a specified regular pattern. The resulting fabric has a diagonal line (called a wale) on the surface. For the satin weave, weft yarns float over then interlace with warp yarns in patterns that do not produce a diagonal line. By using loosely twisted filament fibers with high luster, the characteristic sheen of satin fabrics is produced. More complicated patterns produced on special looms, such as the dobby or jacquard, utilize combinations of the basic weaves to achieve very decorative fabrics."
 weft and warp knits,"Knitting is the formation of fabrics from yarns by creating interconnected loops. Knits may be made from one long, continuous yarn (as in hand knitting) or from sets of yarns. In weft knits, yarns run and interlock across the fabric. In warp knits, the yarns run or interlock in the lengthwise direction. Hand knitting is a form of weft knitting, but weft knits are also made on machines. Weft knits stretch more than warp knits. If stitches break, a run or ladder forms in the lengthwise direction. The most common types of weft knits are jersey (also called single or plain knits) in which all stitches are pulled to the same side of the fabric, so that one side of the fabric is smooth and the other side has loops. Knitted velour, terrycloth, fleece, imitation furs, plush, and other knitted pile fabrics are weft knits, as are doubleknits. Warp knits are more stable and resist runs. The most common fabrics made by warp knitting are tricots, which are made in varying weights, and raschel knits which can be made in complicated patterns that simulate lace or crochet."
 yarn types,"Yarns are made by twisting or otherwise binding fibers together and are used to construct fabrics. The fibers that are made into yarns can be long, continuous strands, called filaments, which can be twisted together loosely or more tightly. Filament yarns are made from either silk or manufactured fibers. Short, staple fibers (cotton, wool, linen, or manufactured fibers or silk cut into short lengths) must be twisted so that they will hold together to form a yarn. Yarns may be classified according to the number of parts. A single yarn is just one yarn, a ply yarn consists of two or more single yarns twisted together, and a cord yarn is made of two or more ply yarns twisted together. Yarns with a regular surface and diameter are called simple yarns. Those made to create decorative effects are known as novelty or fancy yarns. A novelty yarn can be a single yarn, such as a slub yarn in which some parts of the yarn are twisted tightly and other parts more loosely, or a ply yarn such as a bouclé yarn in which a decorative yarn makes irregular, decorative loops around a base yarn. By varying the structure of yarns, an enormous number of different types of yarns and decorative effects can be created."
+footware,"shoes, boots, sandals, and other items worn on the feet"