|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
"""Main tasks functionality.""" |
|
|
|
from megatron import get_args, print_rank_0 |
|
from megatron.indexer import IndexBuilder |
|
from tasks.orqa.evaluate_utils import ORQAEvaluator |
|
|
|
def main(): |
|
""" |
|
Main program |
|
""" |
|
|
|
args = get_args() |
|
|
|
""" |
|
Create a BlockData data structure by running an IndexBuilder over an |
|
ICT Dataset and then evaluate on NQ task |
|
""" |
|
|
|
print_rank_0("Starting index builder!") |
|
|
|
index_builder = IndexBuilder() |
|
index_builder.build_and_save_index() |
|
print_rank_0("Build and save indices: done!") |
|
|
|
|
|
print_rank_0("Starting evaluations!") |
|
|
|
|
|
evaluator = ORQAEvaluator() |
|
|
|
|
|
if args.qa_data_dev is not None: |
|
evaluator.evaluate(args.qa_data_dev, "DEV") |
|
|
|
if args.qa_data_test is not None: |
|
evaluator.evaluate(args.qa_data_test, "TEST") |
|
|
|
|