Added result screener + UI format

app.py

@@ -68,21 +68,21 @@ def init_leaderboard(dataframe):
             cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
             label="Select Columns to Display:",
         ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+        search_columns=[AutoEvalColumn.model.name],
         hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
         filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
+            # ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
+            # ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
+            # ColumnFilter(
+            #     AutoEvalColumn.params.name,
+            #     type="slider",
+            #     min=0.01,
+            #     max=150,
+            #     label="Select the number of parameters (B)",
+            # ),
+            # ColumnFilter(
+            #     AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
+            # ),
         ],
         bool_checkboxgroup_label="Hide models",
         interactive=False,
@@ -172,7 +172,7 @@ with demo:
                         interactive=True,
                     )
                     base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-                    ans_file = gr.File(label="Arena Hard Answer File", file_types=[".json"])
+                    ans_file = gr.File(label="Arena Hard Answer File", file_types=["json","jsonl"])
 
             submit_button = gr.Button("Submit Eval")
             submission_result = gr.Markdown()

src/about.py

@@ -13,7 +13,7 @@ class Task:
 class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard
     task0 = Task("arenahard", "score", "score")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
+    # task1 = Task("logiqa", "acc_norm", "LogiQA")
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------

src/display/utils.py

@@ -22,23 +22,29 @@ class ColumnContent:
 
 ## Leaderboard columns
 auto_eval_column_dict = []
+# ['results', 'model', 'score', 'lower', 'upper', 'avg_tokens', 'std_tokens', 'lc_score']
 # Init
 # auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("model", "markdown", True, never_hidden=True)])
 #Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+# auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 # Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+# auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+auto_eval_column_dict.append(["lower", ColumnContent, ColumnContent("lower", "number", True)])
+auto_eval_column_dict.append(["upper", ColumnContent, ColumnContent("upper", "number", True)])
+auto_eval_column_dict.append(["avg_tokens", ColumnContent, ColumnContent("avg_tokens", "number", True)])
+auto_eval_column_dict.append(["std_tokens", ColumnContent, ColumnContent("std_tokens", "number", True)])
+auto_eval_column_dict.append(["lc_score", ColumnContent, ColumnContent("lc_score", "number", True)])
+# auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+# auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+# auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+# auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+# auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+# auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
+# auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+# auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
@@ -52,6 +58,7 @@ class EvalQueueColumn:  # Queue column
     precision = ColumnContent("precision", "str", True)
     weight_type = ColumnContent("weight_type", "str", "Original")
     status = ColumnContent("status", "str", True)
+    answers_file = ColumnContent("answers_file", "str", True)
 
 ## All the model information that we might need
 @dataclass

src/leaderboard/read_evals.py

@@ -18,14 +18,14 @@ class EvalResult:
     """
     eval_name: str # org_model_precision (uid)
     full_model: str # org/model (path on hub)
-    org: str 
+    org: str
     model: str
     revision: str # commit hash, "" if main
     results: dict
     precision: Precision = Precision.Unknown
     model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
     weight_type: WeightType = WeightType.Original # Original or Adapter
-    architecture: str = "Unknown" 
+    architecture: str = "Unknown"
     license: str = "?"
     likes: int = 0
     num_params: int = 0
@@ -85,7 +85,7 @@ class EvalResult:
             org=org,
             model=model,
             results=results,
-            precision=precision,  
+            precision=precision,
             revision= config.get("model_sha", ""),
             still_on_hub=still_on_hub,
             architecture=architecture
@@ -172,25 +172,25 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
         for file in files:
             model_result_filepaths.append(os.path.join(root, file))
 
-    eval_results = {}
-    for model_result_filepath in model_result_filepaths:
-        # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
-        eval_result.update_with_request_file(requests_path)
-
-        # Store results of same eval together
-        eval_name = eval_result.eval_name
-        if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-        else:
-            eval_results[eval_name] = eval_result
-
-    results = []
-    for v in eval_results.values():
-        try:
-            v.to_dict() # we test if the dict version is complete
-            results.append(v)
-        except KeyError:  # not all eval values present
-            continue
-
-    return results
+    # eval_results = {}
+    # for model_result_filepath in model_result_filepaths:
+    #     # Creation of result
+    #     eval_result = EvalResult.init_from_json_file(model_result_filepath)
+    #     eval_result.update_with_request_file(requests_path)
+
+    #     # Store results of same eval together
+    #     eval_name = eval_result.eval_name
+    #     if eval_name in eval_results.keys():
+    #         eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
+    #     else:
+    #         eval_results[eval_name] = eval_result
+
+    # results = []
+    # for v in eval_results.values():
+    #     try:
+    #         v.to_dict() # we test if the dict version is complete
+    #         results.append(v)
+    #     except KeyError:  # not all eval values present
+    #         continue
+
+    return model_result_filepaths#results

src/populate.py

@@ -6,19 +6,22 @@ import pandas as pd
 from src.display.formatting import has_no_nan_values, make_clickable_model
 from src.display.utils import AutoEvalColumn, EvalQueueColumn
 from src.leaderboard.read_evals import get_raw_eval_results
+from src.envs import RESULTS_REPO
 
 
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
     raw_data = get_raw_eval_results(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
-
-    df = pd.DataFrame.from_records(all_data_json,columns=cols)
-    df['model']="nothing"
+    # all_data_json = [v.to_dict() for v in raw_data]
+    # print(raw_data)
+    df = pd.DataFrame.from_records(json.load(open(raw_data[0])))
+    print(list(df.columns))
+    # df['model']="nothing"
     # df.columns = cols
     # df.iloc[0]= create dummy
-    # df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    # df = df[cols].round(decimals=2)
+    # print(dir(AutoEvalColumn))
+    df = df.sort_values(by=[AutoEvalColumn.task0.name], ascending=False)
+    df = df[cols].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced
     # df = df[has_no_nan_values(df, benchmark_cols)]
@@ -27,25 +30,29 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
 
 def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
     """Creates the different dataframes for the evaluation queues requestes"""
-    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
+    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".") and not entry.endswith(".jsonl")]
     all_evals = []
 
     for entry in entries:
-        if ".json" in entry:
+        if ".json" in entry and 'toeval' not in entry:
             file_path = os.path.join(save_path, entry)
             with open(file_path) as fp:
+                print(file_path)
                 data = json.load(fp)
 
             data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
             data[EvalQueueColumn.revision.name] = data.get("revision", "main")
 
             all_evals.append(data)
-        elif ".md" not in entry:
+        elif ".md" not in entry and 'toeval' not in entry and 'results' not in entry:
             # this is a folder
             sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
             for sub_entry in sub_entries:
+                if 'toeval' in sub_entry:
+                    continue
                 file_path = os.path.join(save_path, entry, sub_entry)
                 with open(file_path) as fp:
+                    # print(file_path)
                     data = json.load(fp)
 
                 data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])

src/submission/submit.py

@@ -16,7 +16,7 @@ USERS_TO_SUBMISSION_DATES = None
 
 def add_new_eval(
     model: str,
-    base_model: str,
+    user_name: str,
     revision: str,
     precision: str,
     weight_type: str,
@@ -28,7 +28,7 @@ def add_new_eval(
     if not REQUESTED_MODELS:
         REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
 
-    user_name = ""
+    user_name = user_name
     model_path = model
     if "/" in model:
         user_name = model.split("/")[0]
@@ -75,10 +75,15 @@ def add_new_eval(
 
     # Seems good, creating the eval
     print("Adding new eval")
+    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+    os.makedirs(OUT_DIR, exist_ok=True)
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
+    out_path_upload = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}_toeval.json"
+
 
     eval_entry = {
         "model": model,
-        "base_model": base_model,
+        "user_name": user_name,
         "revision": revision,
         "precision": precision,
         "weight_type": weight_type,
@@ -87,8 +92,9 @@ def add_new_eval(
         "model_type": model_type,
         "likes": "",
         "params": "",
-        "license": license,
+        "license": "",
         "private": False,
+        "answers_file": str(out_path_upload),
     }
 
     # Check for duplicate submission
@@ -96,10 +102,6 @@ def add_new_eval(
         return styled_warning("This model has been already submitted.")
 
     print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
-    out_path_upload = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}_toeval.json"
 
     with open(out_path, "w") as f:
         f.write(json.dumps(eval_entry))