import os os.system("wget https://raw.githubusercontent.com/Weyaxi/scrape-open-llm-leaderboard/main/openllm.py") from huggingface_hub import CommitOperationAdd, create_commit, HfApi, HfFileSystem, login from huggingface_hub.utils import RepositoryNotFoundError as deneme from openllm import * import gradio as gr import requests import pandas as pd api = HfApi() fs = HfFileSystem() data = get_json_format_data() finished_models = get_datas(data) df = pd.DataFrame(finished_models) def search(df, value): result_df = df[df["Model"] == value] return result_df.iloc[0].to_dict() if not result_df.empty else None def get_details_url(repo): author, model = repo.split("/") return f"https://huggingface.co/datasets/open-llm-leaderboard/details_{author}__{model}" def get_eval_results(repo): results = search(df, repo) text = f""" # [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) Detailed results can be found [here]({get_details_url(repo)}) | Metric | Value | |-----------------------|---------------------------| | Avg. | {results['Average ⬆️']} | | ARC (25-shot) | {results['ARC']} | | HellaSwag (10-shot) | {results['HellaSwag']} | | MMLU (5-shot) | {results['MMLU']} | | TruthfulQA (0-shot) | {results['TruthfulQA']} | | Winogrande (5-shot) | {results['Winogrande']} | | GSM8K (5-shot) | {results['GSM8K']} | """ return text desc = """ This is an automated PR created with https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr The purpose of this PR is to add evaluation results from the Open LLM Leaderboard to your model card. If you encounter any issues, please report them to https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr/discussions """ def commit(hf_token, repo): login(hf_token) try: try: # check if there is a readme already readme_text = fs.read_text(f"{repo}/README.md") + get_eval_results(repo) except: readme_text = get_eval_results(repo) liste = [CommitOperationAdd(path_in_repo="README.md", path_or_fileobj=readme_text.encode())] commit = (create_commit(repo_id=repo, operations=liste, commit_message=f"Adding Evaluation Results", commit_description=desc, repo_type="model", create_pr=True).__dict__['pr_url']) return commit except Exception as e: # unexpected error return e demo = gr.Interface(fn=commit, inputs=["text", "text"], outputs="text") demo.launch()