move inference api parameters

app_text_classification.py

@@ -2,8 +2,7 @@ import uuid
 
 import gradio as gr
 
-from io_utils import (get_logs_file, read_inference_type, read_scanners,
-                      write_inference_type, write_scanners)
+from io_utils import (get_logs_file, read_scanners, write_scanners)
 from text_classification_ui_helpers import (check_dataset_and_get_config,
                                             check_dataset_and_get_split,
                                             check_model_and_show_prediction,
@@ -65,15 +64,7 @@ def get_demo():
 
     with gr.Accordion(label="Model Wrap Advance Config (optional)", open=False):
         run_local = gr.Checkbox(value=True, label="Run in this Space")
-        run_inference = gr.Checkbox(value="False", label="Run with Inference API")
-
-        @gr.on(triggers=[uid_label.change], inputs=[uid_label], outputs=[run_inference])
-        def get_run_mode(uid):
-            return gr.update(
-                value=read_inference_type(uid) == "hf_inference_api"
-                and not run_local.value
-            )
-
+        run_inference = gr.Checkbox(value=False, label="Run with Inference API")
         inference_token = gr.Textbox(
             value="",
             label="HF Token for Inference API",
@@ -122,7 +113,7 @@ def get_demo():
 
     run_inference.change(
         select_run_mode,
-        inputs=[run_inference, inference_token, uid_label],
+        inputs=[run_inference],
         outputs=[inference_token, run_local],
     )
 
@@ -131,11 +122,7 @@ def get_demo():
         inputs=[run_local],
         outputs=[inference_token, run_inference],
     )
-
-    inference_token.change(
-        write_inference_type, inputs=[run_inference, inference_token, uid_label]
-    )
-
+    
     gr.on(
         triggers=[label.change for label in column_mappings],
         fn=write_column_mapping_to_config,
@@ -189,6 +176,8 @@ def get_demo():
             dataset_config_input,
             dataset_split_input,
             run_local,
+            run_inference,
+            inference_token,
             uid_label,
         ],
         outputs=[run_btn, logs],
@@ -204,6 +193,7 @@ def get_demo():
             dataset_split_input.change,
             run_inference.change,
             run_local.change,
+            inference_token.change,
             scanners.change,
         ],
         fn=enable_run_btn,

text_classification_ui_helpers.py

@@ -10,7 +10,7 @@ from transformers.pipelines import TextClassificationPipeline
 from wordings import get_styled_input
 
 from io_utils import (get_yaml_path, read_column_mapping, save_job_to_pipe,
-                      write_column_mapping, write_inference_type,
+                      write_column_mapping,
                       write_log_to_user_file)
 from text_classification import (check_model, get_example_prediction,
                                  get_labels_and_features_from_dataset)
@@ -51,10 +51,8 @@ def check_dataset_and_get_split(dataset_id, dataset_config):
         pass
 
 
-def select_run_mode(run_inf, inf_token, uid):
+def select_run_mode(run_inf):
     if run_inf:
-        if len(inf_token) > 0:
-            write_inference_type(run_inf, inf_token, uid)
         return (gr.update(visible=True), gr.update(value=False))
     else:
         return (gr.update(visible=False), gr.update(value=True))
@@ -213,10 +211,7 @@ def check_model_and_show_prediction(
         *column_mappings,
     )
 
-
-def try_submit(m_id, d_id, config, split, local, uid):
-    all_mappings = read_column_mapping(uid)
-
+def check_column_mapping_keys_validity(all_mappings):
     if all_mappings is None:
         gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
         return (gr.update(interactive=True), gr.update(visible=False))
@@ -224,6 +219,8 @@ def try_submit(m_id, d_id, config, split, local, uid):
     if "labels" not in all_mappings.keys():
         gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
         return (gr.update(interactive=True), gr.update(visible=False))
+
+def construct_label_and_feature_mapping(all_mappings):
     label_mapping = {}
     for i, label in zip(
         range(len(all_mappings["labels"].keys())), all_mappings["labels"].keys()
@@ -234,73 +231,88 @@ def try_submit(m_id, d_id, config, split, local, uid):
         gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
         return (gr.update(interactive=True), gr.update(visible=False))
     feature_mapping = all_mappings["features"]
+    return label_mapping, feature_mapping
+
+def try_submit(m_id, d_id, config, split, local, inference, inference_token, uid):
+    all_mappings = read_column_mapping(uid)
+    check_column_mapping_keys_validity(all_mappings)
+    label_mapping, feature_mapping = construct_label_and_feature_mapping(all_mappings)
 
     leaderboard_dataset = None
     if os.environ.get("SPACE_ID") == "giskardai/giskard-evaluator":
         leaderboard_dataset = "ZeroCommand/test-giskard-report"
+    
+    if local:
+        inference_type = "hf_pipeline"
+    if inference and inference_token:
+        inference_type = "hf_inference_api"
 
     # TODO: Set column mapping for some dataset such as `amazon_polarity`
-    if local:
-        command = [
-            "giskard_scanner",
-            "--loader",
-            "huggingface",
-            "--model",
-            m_id,
-            "--dataset",
-            d_id,
-            "--dataset_config",
-            config,
-            "--dataset_split",
-            split,
-            "--hf_token",
-            os.environ.get(HF_WRITE_TOKEN),
-            "--discussion_repo",
-            os.environ.get(HF_REPO_ID) or os.environ.get(HF_SPACE_ID),
-            "--output_format",
-            "markdown",
-            "--output_portal",
-            "huggingface",
-            "--feature_mapping",
-            json.dumps(feature_mapping),
-            "--label_mapping",
-            json.dumps(label_mapping),
-            "--scan_config",
-            get_yaml_path(uid),
-            "--leaderboard_dataset",
-            leaderboard_dataset,
-        ]
-        if os.environ.get(HF_GSK_HUB_KEY):
-            command.append("--giskard_hub_api_key")
-            command.append(os.environ.get(HF_GSK_HUB_KEY))
-            if os.environ.get(HF_GSK_HUB_URL):
-                command.append("--giskard_hub_url")
-                command.append(os.environ.get(HF_GSK_HUB_URL))
-            if os.environ.get(HF_GSK_HUB_PROJECT_KEY):
-                command.append("--giskard_hub_project_key")
-                command.append(os.environ.get(HF_GSK_HUB_PROJECT_KEY))
-            if os.environ.get(HF_GSK_HUB_HF_TOKEN):
-                command.append("--giskard_hub_hf_token")
-                command.append(os.environ.get(HF_GSK_HUB_HF_TOKEN))
-            if os.environ.get(HF_GSK_HUB_UNLOCK_TOKEN):
-                command.append("--giskard_hub_unlock_token")
-                command.append(os.environ.get(HF_GSK_HUB_UNLOCK_TOKEN))
-
-        eval_str = f"[{m_id}]<{d_id}({config}, {split} set)>"
-        logging.info(f"Start local evaluation on {eval_str}")
-        save_job_to_pipe(uid, command, eval_str, threading.Lock())
-        write_log_to_user_file(
-            uid,
-            f"Start local evaluation on {eval_str}. Please wait for your job to start...\n",
-        )
-        gr.Info(f"Start local evaluation on {eval_str}")
+    command = [
+        "giskard_scanner",
+        "--loader",
+        "huggingface",
+        "--model",
+        m_id,
+        "--dataset",
+        d_id,
+        "--dataset_config",
+        config,
+        "--dataset_split",
+        split,
+        "--hf_token",
+        os.environ.get(HF_WRITE_TOKEN),
+        "--discussion_repo",
+        os.environ.get(HF_REPO_ID) or os.environ.get(HF_SPACE_ID),
+        "--output_format",
+        "markdown",
+        "--output_portal",
+        "huggingface",
+        "--feature_mapping",
+        json.dumps(feature_mapping),
+        "--label_mapping",
+        json.dumps(label_mapping),
+        "--scan_config",
+        get_yaml_path(uid),
+        "--leaderboard_dataset",
+        leaderboard_dataset,
+        "--inference_type",
+        inference_type,
+        "--inference_token",
+        inference_token,
+    ]
+    if os.environ.get(HF_GSK_HUB_KEY):
+        command.append("--giskard_hub_api_key")
+        command.append(os.environ.get(HF_GSK_HUB_KEY))
+        if os.environ.get(HF_GSK_HUB_URL):
+            command.append("--giskard_hub_url")
+            command.append(os.environ.get(HF_GSK_HUB_URL))
+        if os.environ.get(HF_GSK_HUB_PROJECT_KEY):
+            command.append("--giskard_hub_project_key")
+            command.append(os.environ.get(HF_GSK_HUB_PROJECT_KEY))
+        if os.environ.get(HF_GSK_HUB_HF_TOKEN):
+            command.append("--giskard_hub_hf_token")
+            command.append(os.environ.get(HF_GSK_HUB_HF_TOKEN))
+        if os.environ.get(HF_GSK_HUB_UNLOCK_TOKEN):
+            command.append("--giskard_hub_unlock_token")
+            command.append(os.environ.get(HF_GSK_HUB_UNLOCK_TOKEN))
+
+    eval_str = f"[{m_id}]<{d_id}({config}, {split} set)>"
+    logging.info(f"Start local evaluation on {eval_str}")
+    save_job_to_pipe(uid, command, eval_str, threading.Lock())
+    print(command)
+    write_log_to_user_file(
+        uid,
+        f"Start local evaluation on {eval_str}. Please wait for your job to start...\n",
+    )
+    gr.Info(f"Start local evaluation on {eval_str}")
 
-        return (
-            gr.update(interactive=False),
-            gr.update(lines=5, visible=True, interactive=False),
-        )
+    return (
+        gr.update(interactive=False),
+        gr.update(lines=5, visible=True, interactive=False),
+    )
 
-    else:
-        gr.Info("TODO: Submit task to an endpoint")
 
-    return (gr.update(interactive=True), gr.update(visible=False))  # Submit button
+    # TODO: Submit task to an endpoint")
+
+    # return (gr.update(interactive=True), gr.update(visible=False))  # Submit button