From 97ac3c4f3738b5d6e3e3e83dd0bef95b3d776142 Mon Sep 17 00:00:00 2001 From: Johnson Lai Date: Tue, 7 Jan 2025 10:01:34 +0800 Subject: [PATCH 1/6] add eval --- examples/examples-twitter-eval.py | 49 +++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 examples/examples-twitter-eval.py diff --git a/examples/examples-twitter-eval.py b/examples/examples-twitter-eval.py new file mode 100644 index 0000000..a590578 --- /dev/null +++ b/examples/examples-twitter-eval.py @@ -0,0 +1,49 @@ +import os +import requests +from virtuals_sdk import game + +def eval_tweet(original_tweet, responded_tweet): + headers = { + "x-api-key": "YOUR_API_KEY", # Replace with actual API key + "Content-Type": "application/json" + } + payload = { + "input_tweet": original_tweet, + "output_tweet": responded_tweet + } + response = requests.post( + "https://api.evaengine.ai/api/eval/evaluate-tweet", + headers=headers, + json=payload + ) + return response.json() + +agent = game.Agent( + api_key=os.environ.get("VIRTUALS_API_KEY"), + goal="reply tweet", + description="always reply to tweet and make sure read the tweet requirement carefully, please make sure your tweet are very interesting and clickbait and very provocative and very relevant to the tweet, Viral-worthy content, Perfect timing and context, Exceptional creativity/originality,Maximum engagement potential, Industry-leading example of effective tweeting", + world_info="You must always reply user's tweet" +) + +# applicable only for platform twitter +agent.list_available_default_twitter_functions() +agent.use_default_twitter_functions(["wait", "reply_tweet"]) + +# running reaction module only for platform twitter +result = agent.react( + session_id="session-twitter", + platform="twitter", + tweet_id="1869281466628349975", +) + +print(result) +event_request = result[0]["EVENT-REQUEST"] +tweet_request = result[-1]["TWEET-CONTENT"] + +original_tweet = event_request["event"].split("New tweet: ")[1] +print("original_tweet:", original_tweet) +responded_tweet = tweet_request["content"] +print("responded_tweet:", responded_tweet) + +eval_result = eval_tweet(original_tweet, responded_tweet) +print(eval_result) \ No newline at end of file From 98a841bdbd968c8f72f7825df3977f0c69e8f4f6 Mon Sep 17 00:00:00 2001 From: Johnson Lai Date: Tue, 7 Jan 2025 10:04:43 +0800 Subject: [PATCH 2/6] add eval example --- examples/examples-twitter-eval.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/examples/examples-twitter-eval.py b/examples/examples-twitter-eval.py index a590578..38d61c0 100644 --- a/examples/examples-twitter-eval.py +++ b/examples/examples-twitter-eval.py @@ -2,6 +2,7 @@ import requests from virtuals_sdk import game +# Checkout your eval dashboard here: https://evaengine.ai/virtuals (import your api key to view) def eval_tweet(original_tweet, responded_tweet): headers = { "x-api-key": "YOUR_API_KEY", # Replace with actual API key @@ -36,14 +37,11 @@ def eval_tweet(original_tweet, responded_tweet): tweet_id="1869281466628349975", ) -print(result) -event_request = result[0]["EVENT-REQUEST"] -tweet_request = result[-1]["TWEET-CONTENT"] +original_tweet = result[0]["EVENT-REQUEST"]["event"].split("New tweet: ")[1] +replied_tweet = result[-1]["TWEET-CONTENT"]["content"] -original_tweet = event_request["event"].split("New tweet: ")[1] print("original_tweet:", original_tweet) -responded_tweet = tweet_request["content"] -print("responded_tweet:", responded_tweet) +print("responded_tweet:", replied_tweet) -eval_result = eval_tweet(original_tweet, responded_tweet) +eval_result = eval_tweet(original_tweet, replied_tweet) print(eval_result) \ No newline at end of file From 23c596e7359d79da96f59fa4ae62d23959f83487 Mon Sep 17 00:00:00 2001 From: Johnson Lai Date: Tue, 7 Jan 2025 12:02:50 +0800 Subject: [PATCH 3/6] move eval into SDK --- ...amples-twitter-eval.py => example-eval.py} | 25 +++++-------------- src/virtuals_sdk/game.py | 7 ++++++ src/virtuals_sdk/sdk.py | 18 +++++++++++++ 3 files changed, 31 insertions(+), 19 deletions(-) rename examples/{examples-twitter-eval.py => example-eval.py} (61%) diff --git a/examples/examples-twitter-eval.py b/examples/example-eval.py similarity index 61% rename from examples/examples-twitter-eval.py rename to examples/example-eval.py index 38d61c0..71e4483 100644 --- a/examples/examples-twitter-eval.py +++ b/examples/example-eval.py @@ -1,31 +1,18 @@ import os -import requests + from virtuals_sdk import game -# Checkout your eval dashboard here: https://evaengine.ai/virtuals (import your api key to view) -def eval_tweet(original_tweet, responded_tweet): - headers = { - "x-api-key": "YOUR_API_KEY", # Replace with actual API key - "Content-Type": "application/json" - } - payload = { - "input_tweet": original_tweet, - "output_tweet": responded_tweet - } - response = requests.post( - "https://api.evaengine.ai/api/eval/evaluate-tweet", - headers=headers, - json=payload - ) - return response.json() +VIRTUALS_API_KEY = os.environ.get("VIRTUALS_API_KEY") agent = game.Agent( - api_key=os.environ.get("VIRTUALS_API_KEY"), + api_key=VIRTUALS_API_KEY, goal="reply tweet", description="always reply to tweet and make sure read the tweet requirement carefully, please make sure your tweet are very interesting and clickbait and very provocative and very relevant to the tweet, Viral-worthy content, Perfect timing and context, Exceptional creativity/originality,Maximum engagement potential, Industry-leading example of effective tweeting", world_info="You must always reply user's tweet" ) +agent.eval_react("Hello World", "Hello World") + # applicable only for platform twitter agent.list_available_default_twitter_functions() agent.use_default_twitter_functions(["wait", "reply_tweet"]) @@ -43,5 +30,5 @@ def eval_tweet(original_tweet, responded_tweet): print("original_tweet:", original_tweet) print("responded_tweet:", replied_tweet) -eval_result = eval_tweet(original_tweet, replied_tweet) +eval_result = agent.eval_react(original_tweet, replied_tweet) print(eval_result) \ No newline at end of file diff --git a/src/virtuals_sdk/game.py b/src/virtuals_sdk/game.py index 5a565ae..3e0d02a 100644 --- a/src/virtuals_sdk/game.py +++ b/src/virtuals_sdk/game.py @@ -256,6 +256,13 @@ def react(self, session_id: str, platform: str, tweet_id: str = None, event: str functions=self.enabled_functions, custom_functions=self.custom_functions ) + + def eval_react(self, input_tweet: str, output_tweet: str): + """ + Evaluate the agent reply with EvaEngine + Checkout your eval dashboard here: https://evaengine.ai/virtuals (import your api key to view) + """ + return self.game_sdk.eval_react(input_tweet, output_tweet) def deploy_twitter(self): """ diff --git a/src/virtuals_sdk/sdk.py b/src/virtuals_sdk/sdk.py index 64aa21f..f5f9504 100644 --- a/src/virtuals_sdk/sdk.py +++ b/src/virtuals_sdk/sdk.py @@ -89,6 +89,24 @@ def react(self, session_id: str, platform: str, goal: str, raise Exception(response.json()) return response.json()["data"] + + def eval_react(self, input_tweet: str, output_tweet: str): + """ + Evaluate the agent reply + Checkout your eval dashboard here: https://evaengine.ai/virtuals (import your api key to view) + """ + response = requests.post( + "https://api.evaengine.ai/api/eval/evaluate-tweet", + headers={"x-api-key": self.api_key}, + json={ + "input_tweet": input_tweet, + "output_tweet": output_tweet + } + ) + if (response.status_code != 200): + raise Exception(response.json()) + + return response.json() def deploy(self, goal: str, description: str, world_info: str, functions: list, custom_functions: list, main_heartbeat: int, reaction_heartbeat: int): """ From fd2f1f039914e478aeb4987b0c52670a5d9b180a Mon Sep 17 00:00:00 2001 From: Johnson Lai Date: Tue, 7 Jan 2025 14:14:17 +0800 Subject: [PATCH 4/6] clean up: --- README.md | 12 +++++++++ examples/example-eval.py | 53 ++++++++++++++++++++++++++++------------ src/virtuals_sdk/game.py | 6 +++-- 3 files changed, 54 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 547bc67..1e91933 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,18 @@ response = agent.react( ) ``` +We integrate with [EvaEngine by Chromia](https://evaengine.ai/virtuals) for response evaluation, enabling you to optimize your Character Card based on quantitative metrics (truth, accuracy, creativity, engagement) and scoring. +```python +response = agent.react( + session_id="567", # string identifier that you decide + tweet_id="xxxx", + platform="twitter", +) +original_tweet = response[0]["EVENT-REQUEST"]["event"].split("New tweet: ")[1] +replied_tweet = response[-1]["TWEET-CONTENT"]["content"] +eval_result = agent.eval_react(original_tweet, replied_tweet) +``` + Once you are happy, `deploy_twitter` will push your agent configurations to production and run your agent on Twitter/X autonomously. ```python # deploy agent! (NOTE: supported for Twitter/X only now) diff --git a/examples/example-eval.py b/examples/example-eval.py index 71e4483..4913fd7 100644 --- a/examples/example-eval.py +++ b/examples/example-eval.py @@ -17,18 +17,41 @@ agent.list_available_default_twitter_functions() agent.use_default_twitter_functions(["wait", "reply_tweet"]) -# running reaction module only for platform twitter -result = agent.react( - session_id="session-twitter", - platform="twitter", - tweet_id="1869281466628349975", -) - -original_tweet = result[0]["EVENT-REQUEST"]["event"].split("New tweet: ")[1] -replied_tweet = result[-1]["TWEET-CONTENT"]["content"] - -print("original_tweet:", original_tweet) -print("responded_tweet:", replied_tweet) - -eval_result = agent.eval_react(original_tweet, replied_tweet) -print(eval_result) \ No newline at end of file +# # running reaction module only for platform twitter +# result = agent.react( +# session_id="session-twitter", +# platform="twitter", +# tweet_id="1869281466628349975", +# ) + +# print("original_tweet:", original_tweet) +# print("responded_tweet:", replied_tweet) + +# # Checkout your eval dashboard here: https://evaengine.ai/virtuals (import your api key to view) +# eval_result = agent.eval_react(result) +# print(eval_result) + +# Run multiple test to get average eval score +eval_results = [] +for i in range(2): + result = agent.react( + session_id="session-twitter", + platform="twitter", + tweet_id="1869281466628349975", + ) + eval_result = agent.eval_react(result) + eval_results.append(eval_result) + +# Calculate averages from eval_results +final_scores = [result['final_score'] for result in eval_results] +truth_scores = [result['truth']['score'] for result in eval_results] +accuracy_scores = [result['accuracy']['score'] for result in eval_results] +creativity_scores = [result['creativity']['score'] for result in eval_results] +engagement_scores = [result['engagement']['score'] for result in eval_results] + +print(f"Average scores across {len(eval_results)} evaluations:") +print(f"Final Score: {(sum(final_scores) / len(final_scores)):.2f}") +print(f"Truth Score: {(sum(truth_scores) / len(truth_scores)):.2f}") +print(f"Accuracy Score: {(sum(accuracy_scores) / len(accuracy_scores)):.2f}") +print(f"Creativity Score: {(sum(creativity_scores) / len(creativity_scores)):.2f}") +print(f"Engagement Score: {sum(engagement_scores) / len(engagement_scores):.2f}") diff --git a/src/virtuals_sdk/game.py b/src/virtuals_sdk/game.py index 3e0d02a..e1a8162 100644 --- a/src/virtuals_sdk/game.py +++ b/src/virtuals_sdk/game.py @@ -257,12 +257,14 @@ def react(self, session_id: str, platform: str, tweet_id: str = None, event: str custom_functions=self.custom_functions ) - def eval_react(self, input_tweet: str, output_tweet: str): + def eval_react(self, response: List[Dict[str, Any]]): """ Evaluate the agent reply with EvaEngine Checkout your eval dashboard here: https://evaengine.ai/virtuals (import your api key to view) """ - return self.game_sdk.eval_react(input_tweet, output_tweet) + original_tweet = response[0]["EVENT-REQUEST"]["event"].split("New tweet: ")[1] + replied_tweet = response[-1]["TWEET-CONTENT"]["content"] + return self.game_sdk.eval_react(original_tweet, replied_tweet) def deploy_twitter(self): """ From c5a55da558baa255ccc60085b8d55351b9af9843 Mon Sep 17 00:00:00 2001 From: Johnson Lai Date: Tue, 7 Jan 2025 14:19:15 +0800 Subject: [PATCH 5/6] remove unused eval_react --- examples/example-eval.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/example-eval.py b/examples/example-eval.py index 4913fd7..75ee363 100644 --- a/examples/example-eval.py +++ b/examples/example-eval.py @@ -11,8 +11,6 @@ world_info="You must always reply user's tweet" ) -agent.eval_react("Hello World", "Hello World") - # applicable only for platform twitter agent.list_available_default_twitter_functions() agent.use_default_twitter_functions(["wait", "reply_tweet"]) From d190124998691df4e9af1cf5617d64fbcbb589c7 Mon Sep 17 00:00:00 2001 From: Johnson Lai Date: Tue, 7 Jan 2025 14:37:58 +0800 Subject: [PATCH 6/6] Enhance error handling in tweet evaluation and refactor API URL usage --- src/virtuals_sdk/game.py | 7 +++++-- src/virtuals_sdk/sdk.py | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/virtuals_sdk/game.py b/src/virtuals_sdk/game.py index e1a8162..a754eac 100644 --- a/src/virtuals_sdk/game.py +++ b/src/virtuals_sdk/game.py @@ -262,8 +262,11 @@ def eval_react(self, response: List[Dict[str, Any]]): Evaluate the agent reply with EvaEngine Checkout your eval dashboard here: https://evaengine.ai/virtuals (import your api key to view) """ - original_tweet = response[0]["EVENT-REQUEST"]["event"].split("New tweet: ")[1] - replied_tweet = response[-1]["TWEET-CONTENT"]["content"] + try: + original_tweet = response[0]["EVENT-REQUEST"]["event"].split("New tweet: ")[1] + replied_tweet = response[-1]["TWEET-CONTENT"]["content"] + except (KeyError, IndexError) as e: + raise ValueError("Invalid response format - missing tweet content. Please ensure the agent's goal includes replying to tweets.") return self.game_sdk.eval_react(original_tweet, replied_tweet) def deploy_twitter(self): diff --git a/src/virtuals_sdk/sdk.py b/src/virtuals_sdk/sdk.py index f5f9504..d92ccbe 100644 --- a/src/virtuals_sdk/sdk.py +++ b/src/virtuals_sdk/sdk.py @@ -4,6 +4,7 @@ class GameSDK: api_url: str = "https://game-api.virtuals.io/api" api_key: str + eval_api_url: str = "https://api.evaengine.ai/api" def __init__(self, api_key: str): self.api_key = api_key @@ -96,13 +97,14 @@ def eval_react(self, input_tweet: str, output_tweet: str): Checkout your eval dashboard here: https://evaengine.ai/virtuals (import your api key to view) """ response = requests.post( - "https://api.evaengine.ai/api/eval/evaluate-tweet", + f"{self.eval_api_url}/eval/evaluate-tweet", headers={"x-api-key": self.api_key}, json={ "input_tweet": input_tweet, "output_tweet": output_tweet } ) + if (response.status_code != 200): raise Exception(response.json())