From 43cfd932513709c62f310edc2425380d7d170495 Mon Sep 17 00:00:00 2001 From: Wenzhe Xue Date: Mon, 12 Aug 2024 23:53:40 -0700 Subject: [PATCH 1/2] add an test example uses pytest-json-report: - trace the log10 last completion url from session - save the json report --- tests/output.report.json | 1 + tests/test_use_json_report.py | 74 +++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 tests/output.report.json create mode 100644 tests/test_use_json_report.py diff --git a/tests/output.report.json b/tests/output.report.json new file mode 100644 index 0000000..7b02961 --- /dev/null +++ b/tests/output.report.json @@ -0,0 +1 @@ +{"created": 1723531310.846067, "duration": 17.749653100967407, "exitcode": 1, "root": "/Users/wenzhe/dev/log10_eval_example", "environment": {}, "summary": {"passed": 3, "failed": 1, "total": 4, "collected": 4}, "collectors": [{"nodeid": "", "outcome": "passed", "result": [{"nodeid": "tests/test_managed_eval.py", "type": "Module"}]}, {"nodeid": "tests/test_managed_eval.py", "outcome": "passed", "result": [{"nodeid": "tests/test_managed_eval.py::test_summarize_to_30_words[0]", "type": "Function", "lineno": 40}, {"nodeid": "tests/test_managed_eval.py::test_summarize_to_30_words[1]", "type": "Function", "lineno": 40}, {"nodeid": "tests/test_managed_eval.py::test_summarize_to_30_words[2]", "type": "Function", "lineno": 40}, {"nodeid": "tests/test_managed_eval.py::test_pass_rate_of_30_words", "type": "Function", "lineno": 66}]}], "tests": [{"nodeid": "tests/test_managed_eval.py::test_summarize_to_30_words[0]", "lineno": 40, "outcome": "passed", "keywords": ["test_summarize_to_30_words[0]", "parametrize", "pytestmark", "0", "test_managed_eval.py", "tests", "log10_eval_example", ""], "setup": {"duration": 0.0007630839245393872, "outcome": "passed"}, "metadata": {"log10": {"last_completion_id": "https://log10.io/app/test-0/completions/0191431e-b488-47ab-8d2c-554f865ae6ab"}}, "call": {"duration": 4.514427750022151, "outcome": "passed"}, "teardown": {"duration": 0.0011277500307187438, "outcome": "passed"}}, {"nodeid": "tests/test_managed_eval.py::test_summarize_to_30_words[1]", "lineno": 40, "outcome": "passed", "keywords": ["test_summarize_to_30_words[1]", "parametrize", "pytestmark", "1", "test_managed_eval.py", "tests", "log10_eval_example", ""], "setup": {"duration": 0.0020862920209765434, "outcome": "passed"}, "metadata": {"log10": {"last_completion_id": "https://log10.io/app/test-0/completions/fd8cf016-b6db-44dc-b93e-1385b90cad0a"}}, "call": {"duration": 5.654925833106972, "outcome": "passed"}, "teardown": {"duration": 0.0008063330315053463, "outcome": "passed"}}, {"nodeid": "tests/test_managed_eval.py::test_summarize_to_30_words[2]", "lineno": 40, "outcome": "failed", "keywords": ["test_summarize_to_30_words[2]", "parametrize", "pytestmark", "2", "test_managed_eval.py", "tests", "log10_eval_example", ""], "setup": {"duration": 0.0025581250665709376, "outcome": "passed"}, "metadata": {"log10": {"last_completion_id": "https://log10.io/app/test-0/completions/8092963e-aa35-4954-9249-0fa8577141e2"}}, "call": {"duration": 6.896540459012613, "outcome": "failed", "crash": {"path": "/Users/wenzhe/dev/log10_eval_example/tests/test_managed_eval.py", "lineno": 63, "message": "AssertionError: Number of words is 31, expected <= 30\nassert False"}, "traceback": [{"path": "tests/test_managed_eval.py", "lineno": 63, "message": "AssertionError"}], "longrepr": "data = [('Children of Time is a 2015 science fiction novel by author Adrian Tchaikovsky. The work was praised by Financial Ti...eneuve in 2016, it received critical acclaim and multiple awards, including an Academy Award for Best Sound Editing.')]\nsample_idx = 2\nresults_bag = ResultsBag:\n{'test_name': 'test_summarize_to_30_words_2', 'article': '\"Story of Your Life\" is a science fiction novell...pletions/8092963e-aa35-4954-9249-0fa8577141e2', 'num_words_less_than_30': False, 'cos_distance_less_than_02': np.True_}\njson_metadata = {'log10': {'last_completion_id': 'https://log10.io/app/test-0/completions/8092963e-aa35-4954-9249-0fa8577141e2'}}\nsession = \n\n @pytest.mark.parametrize(\"sample_idx\", range(3))\n def test_summarize_to_30_words(data: list, sample_idx: int, results_bag, json_metadata, session):\n article, expected_summary = data[sample_idx]\n output = summarize_to_30_words(article)\n cos_distance = cosine_distance(expected_summary, output)\n num_words = count_words(output)\n \n results_bag.test_name = f\"test_summarize_to_30_words_{sample_idx}\"\n results_bag.article = article\n results_bag.expected_summary = expected_summary\n results_bag.output = output\n results_bag.cos_sim = cos_distance\n results_bag.num_words = num_words\n results_bag.log10_completion_url = session.last_completion_url()\n json_metadata[\"log10\"] = {\"last_completion_id\": session.last_completion_url()}\n \n num_words_less_than_30 = num_words <= 30\n results_bag.num_words_less_than_30 = num_words_less_than_30\n \n cos_distance_less_than_02 = cos_distance < 0.2\n results_bag.cos_distance_less_than_02 = cos_distance_less_than_02\n \n> assert num_words_less_than_30, f\"Number of words is {num_words}, expected <= 30\"\nE AssertionError: Number of words is 31, expected <= 30\nE assert False\n\ntests/test_managed_eval.py:63: AssertionError"}, "teardown": {"duration": 0.00012041593436151743, "outcome": "passed"}}, {"nodeid": "tests/test_managed_eval.py::test_pass_rate_of_30_words", "lineno": 66, "outcome": "passed", "keywords": ["test_pass_rate_of_30_words", "test_managed_eval.py", "tests", "log10_eval_example", ""], "setup": {"duration": 0.0011677499860525131, "outcome": "passed"}, "call": {"duration": 0.0014591669896617532, "outcome": "passed"}, "teardown": {"duration": 7.26659782230854e-05, "outcome": "passed"}}]} \ No newline at end of file diff --git a/tests/test_use_json_report.py b/tests/test_use_json_report.py new file mode 100644 index 0000000..613820b --- /dev/null +++ b/tests/test_use_json_report.py @@ -0,0 +1,74 @@ +import pytest +from pytest_check import check +import sys +import os +import pandas as pd + +# Append the src directory to sys.path to make its modules available for import +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "src"))) + +import jsonlines +from log10.load import log10_session + +from my_llm import ( + summarize_to_30_words, +) +from my_eval_metrics import cosine_distance, count_words +from report_utils import ( + filter_results_by_test_name, + report_pass_rate, +) + + +@pytest.fixture +def session(): + with log10_session() as session: + assert session.last_completion_id() is None, "No completion ID should be found." + yield session + + +@pytest.fixture +def data(): + filename = "data.jsonl" + data = [] + with jsonlines.open(filename) as reader: + for obj in reader: + data.append((obj["article"], obj["summary"])) + return data + + +# @pytest.mark.repeat(2) +@pytest.mark.parametrize("sample_idx", range(3)) +def test_summarize_to_30_words(data: list, sample_idx: int, results_bag, json_metadata, session): + article, expected_summary = data[sample_idx] + output = summarize_to_30_words(article) + cos_distance = cosine_distance(expected_summary, output) + num_words = count_words(output) + + results_bag.test_name = f"test_summarize_to_30_words_{sample_idx}" + results_bag.article = article + results_bag.expected_summary = expected_summary + results_bag.output = output + results_bag.cos_sim = cos_distance + results_bag.num_words = num_words + results_bag.log10_completion_url = session.last_completion_url() + json_metadata["log10"] = {"last_completion_id": session.last_completion_url()} + + num_words_less_than_30 = num_words <= 30 + results_bag.num_words_less_than_30 = num_words_less_than_30 + + cos_distance_less_than_02 = cos_distance < 0.2 + results_bag.cos_distance_less_than_02 = cos_distance_less_than_02 + + assert num_words_less_than_30, f"Number of words is {num_words}, expected <= 30" + assert cos_distance_less_than_02, f"Cosine distance is {cos_distance}, expected < 0.2" + + +def test_pass_rate_of_30_words(module_results_df: pd.DataFrame): + #save module_results_df to csv + module_results_df.to_csv("module_results_df_080724.csv", index=False) + df = filter_results_by_test_name(module_results_df, "test_summarize_to_30_words") + + pass_rate, pass_rate_report_str = report_pass_rate(df) + + assert pass_rate > 0.66 \ No newline at end of file From 045d7de1ace5fea9cadf451363a3cd571860eceb Mon Sep 17 00:00:00 2001 From: wenzhe <145375501+wenzhe-log10@users.noreply.github.com> Date: Mon, 12 Aug 2024 23:59:29 -0700 Subject: [PATCH 2/2] format output.report.json --- tests/output.report.json | 180 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 179 insertions(+), 1 deletion(-) diff --git a/tests/output.report.json b/tests/output.report.json index 7b02961..e023261 100644 --- a/tests/output.report.json +++ b/tests/output.report.json @@ -1 +1,179 @@ -{"created": 1723531310.846067, "duration": 17.749653100967407, "exitcode": 1, "root": "/Users/wenzhe/dev/log10_eval_example", "environment": {}, "summary": {"passed": 3, "failed": 1, "total": 4, "collected": 4}, "collectors": [{"nodeid": "", "outcome": "passed", "result": [{"nodeid": "tests/test_managed_eval.py", "type": "Module"}]}, {"nodeid": "tests/test_managed_eval.py", "outcome": "passed", "result": [{"nodeid": "tests/test_managed_eval.py::test_summarize_to_30_words[0]", "type": "Function", "lineno": 40}, {"nodeid": "tests/test_managed_eval.py::test_summarize_to_30_words[1]", "type": "Function", "lineno": 40}, {"nodeid": "tests/test_managed_eval.py::test_summarize_to_30_words[2]", "type": "Function", "lineno": 40}, {"nodeid": "tests/test_managed_eval.py::test_pass_rate_of_30_words", "type": "Function", "lineno": 66}]}], "tests": [{"nodeid": "tests/test_managed_eval.py::test_summarize_to_30_words[0]", "lineno": 40, "outcome": "passed", "keywords": ["test_summarize_to_30_words[0]", "parametrize", "pytestmark", "0", "test_managed_eval.py", "tests", "log10_eval_example", ""], "setup": {"duration": 0.0007630839245393872, "outcome": "passed"}, "metadata": {"log10": {"last_completion_id": "https://log10.io/app/test-0/completions/0191431e-b488-47ab-8d2c-554f865ae6ab"}}, "call": {"duration": 4.514427750022151, "outcome": "passed"}, "teardown": {"duration": 0.0011277500307187438, "outcome": "passed"}}, {"nodeid": "tests/test_managed_eval.py::test_summarize_to_30_words[1]", "lineno": 40, "outcome": "passed", "keywords": ["test_summarize_to_30_words[1]", "parametrize", "pytestmark", "1", "test_managed_eval.py", "tests", "log10_eval_example", ""], "setup": {"duration": 0.0020862920209765434, "outcome": "passed"}, "metadata": {"log10": {"last_completion_id": "https://log10.io/app/test-0/completions/fd8cf016-b6db-44dc-b93e-1385b90cad0a"}}, "call": {"duration": 5.654925833106972, "outcome": "passed"}, "teardown": {"duration": 0.0008063330315053463, "outcome": "passed"}}, {"nodeid": "tests/test_managed_eval.py::test_summarize_to_30_words[2]", "lineno": 40, "outcome": "failed", "keywords": ["test_summarize_to_30_words[2]", "parametrize", "pytestmark", "2", "test_managed_eval.py", "tests", "log10_eval_example", ""], "setup": {"duration": 0.0025581250665709376, "outcome": "passed"}, "metadata": {"log10": {"last_completion_id": "https://log10.io/app/test-0/completions/8092963e-aa35-4954-9249-0fa8577141e2"}}, "call": {"duration": 6.896540459012613, "outcome": "failed", "crash": {"path": "/Users/wenzhe/dev/log10_eval_example/tests/test_managed_eval.py", "lineno": 63, "message": "AssertionError: Number of words is 31, expected <= 30\nassert False"}, "traceback": [{"path": "tests/test_managed_eval.py", "lineno": 63, "message": "AssertionError"}], "longrepr": "data = [('Children of Time is a 2015 science fiction novel by author Adrian Tchaikovsky. The work was praised by Financial Ti...eneuve in 2016, it received critical acclaim and multiple awards, including an Academy Award for Best Sound Editing.')]\nsample_idx = 2\nresults_bag = ResultsBag:\n{'test_name': 'test_summarize_to_30_words_2', 'article': '\"Story of Your Life\" is a science fiction novell...pletions/8092963e-aa35-4954-9249-0fa8577141e2', 'num_words_less_than_30': False, 'cos_distance_less_than_02': np.True_}\njson_metadata = {'log10': {'last_completion_id': 'https://log10.io/app/test-0/completions/8092963e-aa35-4954-9249-0fa8577141e2'}}\nsession = \n\n @pytest.mark.parametrize(\"sample_idx\", range(3))\n def test_summarize_to_30_words(data: list, sample_idx: int, results_bag, json_metadata, session):\n article, expected_summary = data[sample_idx]\n output = summarize_to_30_words(article)\n cos_distance = cosine_distance(expected_summary, output)\n num_words = count_words(output)\n \n results_bag.test_name = f\"test_summarize_to_30_words_{sample_idx}\"\n results_bag.article = article\n results_bag.expected_summary = expected_summary\n results_bag.output = output\n results_bag.cos_sim = cos_distance\n results_bag.num_words = num_words\n results_bag.log10_completion_url = session.last_completion_url()\n json_metadata[\"log10\"] = {\"last_completion_id\": session.last_completion_url()}\n \n num_words_less_than_30 = num_words <= 30\n results_bag.num_words_less_than_30 = num_words_less_than_30\n \n cos_distance_less_than_02 = cos_distance < 0.2\n results_bag.cos_distance_less_than_02 = cos_distance_less_than_02\n \n> assert num_words_less_than_30, f\"Number of words is {num_words}, expected <= 30\"\nE AssertionError: Number of words is 31, expected <= 30\nE assert False\n\ntests/test_managed_eval.py:63: AssertionError"}, "teardown": {"duration": 0.00012041593436151743, "outcome": "passed"}}, {"nodeid": "tests/test_managed_eval.py::test_pass_rate_of_30_words", "lineno": 66, "outcome": "passed", "keywords": ["test_pass_rate_of_30_words", "test_managed_eval.py", "tests", "log10_eval_example", ""], "setup": {"duration": 0.0011677499860525131, "outcome": "passed"}, "call": {"duration": 0.0014591669896617532, "outcome": "passed"}, "teardown": {"duration": 7.26659782230854e-05, "outcome": "passed"}}]} \ No newline at end of file +{ + "collectors": [ + { + "nodeid": "", + "outcome": "passed", + "result": [ + { + "nodeid": "tests/test_use_json_report.py", + "type": "Module" + } + ] + }, + { + "nodeid": "tests/test_use_json_report.py", + "outcome": "passed", + "result": [ + { + "lineno": 42, + "nodeid": "tests/test_use_json_report.py::test_summarize_to_30_words[0]", + "type": "Function" + }, + { + "lineno": 42, + "nodeid": "tests/test_use_json_report.py::test_summarize_to_30_words[1]", + "type": "Function" + }, + { + "lineno": 42, + "nodeid": "tests/test_use_json_report.py::test_summarize_to_30_words[2]", + "type": "Function" + }, + { + "lineno": 69, + "nodeid": "tests/test_use_json_report.py::test_pass_rate_of_30_words", + "type": "Function" + } + ] + } + ], + "created": 1723531836.549531, + "duration": 17.205761909484863, + "environment": {}, + "exitcode": 1, + "root": "/Users/wenzhe/dev/log10_eval_example", + "summary": { + "collected": 4, + "failed": 1, + "passed": 3, + "total": 4 + }, + "tests": [ + { + "call": { + "crash": { + "lineno": 0, + "message": "\u001b[0mNumber of words is 35, expected <= 30\nassert False\n\u001b[31mtest_use_json_report.py\u001b[0m:65 in test_summarize_to_30_words() -> with check:\n\u001b[31mtest_use_json_report.py\u001b[0m:66 in test_summarize_to_30_words -> assert num_words_less_than_30, f\"Number of words is {num_words}, expected <= 30\"\n\u001b[31mAssertionError: Number of words is 35, expected <= 30\nassert False\n\u001b[0m\n\n------------------------------------------------------------\nFailed Checks: 1", + "path": "tests/test_use_json_report.py::test_summarize_to_30_words[0]" + }, + "duration": 6.695120791089721, + "longrepr": "\u001b[31mFAILURE: \u001b[0mNumber of words is 35, expected <= 30\nassert False\n\u001b[31mtest_use_json_report.py\u001b[0m:65 in test_summarize_to_30_words() -> with check:\n\u001b[31mtest_use_json_report.py\u001b[0m:66 in test_summarize_to_30_words -> assert num_words_less_than_30, f\"Number of words is {num_words}, expected <= 30\"\n\u001b[31mAssertionError: Number of words is 35, expected <= 30\nassert False\n\u001b[0m\n\n------------------------------------------------------------\nFailed Checks: 1", + "outcome": "failed" + }, + "keywords": [ + "test_summarize_to_30_words[0]", + "parametrize", + "pytestmark", + "0", + "test_use_json_report.py", + "tests", + "log10_eval_example", + "" + ], + "lineno": 42, + "metadata": { + "log10": { + "last_completion_id": "https://log10.io/app/test-0/completions/45515561-5fc3-4709-8361-8eadf349e337" + } + }, + "nodeid": "tests/test_use_json_report.py::test_summarize_to_30_words[0]", + "outcome": "failed", + "setup": { + "duration": 0.0008069159230217338, + "outcome": "passed" + }, + "teardown": { + "duration": 0.0001188330352306366, + "outcome": "passed" + } + }, + { + "call": { + "duration": 5.655736291082576, + "outcome": "passed" + }, + "keywords": [ + "test_summarize_to_30_words[1]", + "parametrize", + "pytestmark", + "1", + "test_use_json_report.py", + "tests", + "log10_eval_example", + "" + ], + "lineno": 42, + "metadata": { + "log10": { + "last_completion_id": "https://log10.io/app/test-0/completions/29dded7a-2998-405c-a474-684ec0ba1682" + } + }, + "nodeid": "tests/test_use_json_report.py::test_summarize_to_30_words[1]", + "outcome": "passed", + "setup": { + "duration": 0.0003689579898491502, + "outcome": "passed" + }, + "teardown": { + "duration": 0.0006292080506682396, + "outcome": "passed" + } + }, + { + "call": { + "duration": 4.296570708043873, + "outcome": "passed" + }, + "keywords": [ + "test_summarize_to_30_words[2]", + "parametrize", + "pytestmark", + "2", + "test_use_json_report.py", + "tests", + "log10_eval_example", + "" + ], + "lineno": 42, + "metadata": { + "log10": { + "last_completion_id": "https://log10.io/app/test-0/completions/9d736109-d982-45e5-a013-db7415f66f2e" + } + }, + "nodeid": "tests/test_use_json_report.py::test_summarize_to_30_words[2]", + "outcome": "passed", + "setup": { + "duration": 0.002584167057648301, + "outcome": "passed" + }, + "teardown": { + "duration": 0.0004907089751213789, + "outcome": "passed" + } + }, + { + "call": { + "duration": 0.008413875009864569, + "outcome": "passed" + }, + "keywords": [ + "test_pass_rate_of_30_words", + "test_use_json_report.py", + "tests", + "log10_eval_example", + "" + ], + "lineno": 69, + "nodeid": "tests/test_use_json_report.py::test_pass_rate_of_30_words", + "outcome": "passed", + "setup": { + "duration": 0.006148124928586185, + "outcome": "passed" + }, + "teardown": { + "duration": 0.00025579100474715233, + "outcome": "passed" + } + } + ] +}