more ref, ci, debug

app.py

@@ -60,6 +60,7 @@ LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS,
 def init_leaderboard(dataframe):
     # if dataframe is None or dataframe.empty:
     #     raise ValueError("Leaderboard DataFrame is empty or None.")
+    # print(dataframe.columns)
     return Leaderboard(
         value=dataframe,
         datatype=[c.type for c in fields(AutoEvalColumn)],
@@ -102,45 +103,45 @@ with demo:
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
         with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
+            # with gr.Column():
+            #     with gr.Row():
+            #         gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+
+                # with gr.Column():
+                    # with gr.Accordion(
+                    #     f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
+                    #     open=False,
+                    # ):
+                    #     with gr.Row():
+                    #         finished_eval_table = gr.components.Dataframe(
+                    #             value=finished_eval_queue_df,
+                    #             headers=EVAL_COLS,
+                    #             datatype=EVAL_TYPES,
+                    #             row_count=5,
+                    #         )
+                    # with gr.Accordion(
+                    #     f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
+                    #     open=False,
+                    # ):
+                    #     with gr.Row():
+                    #         running_eval_table = gr.components.Dataframe(
+                    #             value=running_eval_queue_df,
+                    #             headers=EVAL_COLS,
+                    #             datatype=EVAL_TYPES,
+                    #             row_count=5,
+                    #         )
+
+                    # with gr.Accordion(
+                    #     f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
+                    #     open=False,
+                    # ):
+                    #     with gr.Row():
+                    #         pending_eval_table = gr.components.Dataframe(
+                    #             value=pending_eval_queue_df,
+                    #             headers=EVAL_COLS,
+                    #             datatype=EVAL_TYPES,
+                    #             row_count=5,
+                    #         )
             with gr.Row():
                 gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
 
@@ -171,7 +172,7 @@ with demo:
                         value="Original",
                         interactive=True,
                     )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
+                    base_model_name_textbox = gr.Textbox(label="Организация")
                     ans_file = gr.File(label="Arena Hard Answer File", file_types=["json","jsonl"])
 
             submit_button = gr.Button("Submit Eval")

src/display/utils.py

@@ -32,8 +32,9 @@ for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 # Model information
 # auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["lower", ColumnContent, ColumnContent("lower", "number", True)])
-auto_eval_column_dict.append(["upper", ColumnContent, ColumnContent("upper", "number", True)])
+auto_eval_column_dict.append(["CI", ColumnContent, ColumnContent("95% CI", "string", True)])
+auto_eval_column_dict.append(["lower", ColumnContent, ColumnContent("lower", "number", False)])
+auto_eval_column_dict.append(["upper", ColumnContent, ColumnContent("upper", "number", False)])
 auto_eval_column_dict.append(["avg_tokens", ColumnContent, ColumnContent("avg_tokens", "number", True)])
 auto_eval_column_dict.append(["std_tokens", ColumnContent, ColumnContent("std_tokens", "number", True)])
 auto_eval_column_dict.append(["lc_score", ColumnContent, ColumnContent("lc_score", "number", True)])

src/leaderboard/read_evals.py

@@ -167,9 +167,10 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
         try:
             files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
         except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
+            files = [sorted(files)[-1]]
 
-        for file in files:
+
+        for file in [files[-1]]:
             model_result_filepaths.append(os.path.join(root, file))
 
     # eval_results = {}

src/populate.py

@@ -16,11 +16,18 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     # print(raw_data)
     df = pd.DataFrame.from_records(json.load(open(raw_data[0])))
     print(list(df.columns))
+    df['95% CI'] = " "
     # df['model']="nothing"
     # df.columns = cols
     # df.iloc[0]= create dummy
     # print(dir(AutoEvalColumn))
     df = df.sort_values(by=[AutoEvalColumn.task0.name], ascending=False)
+    decimal = 1
+    for i,row in df.iterrows():
+        if 'lower' not in row:
+            continue
+        interval = '+'+str(round(row['upper'] - row['score'], decimal))+' / '+str(round(row['lower'] - row['score'], decimal))
+        df.at[i,'95% CI'] = interval
     df = df[cols].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced