From 5b10396d27472f515f14ab098054501655ad3a3c Mon Sep 17 00:00:00 2001 From: Masterain Date: Thu, 26 Jun 2025 14:18:55 -0700 Subject: [PATCH 01/14] introducing Cloudflare Secuity Tools --- .gitmodules | 4 ++++ cloudflare_security_utils | 1 + 2 files changed, 5 insertions(+) create mode 100644 .gitmodules create mode 160000 cloudflare_security_utils diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..3ec8791 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "cloudflare_security_utils"] + path = cloudflare_security_utils + url = https://github.com/DGP-Studio/cloudflare-api-security.git + branch = main diff --git a/cloudflare_security_utils b/cloudflare_security_utils new file mode 160000 index 0000000..d69e849 --- /dev/null +++ b/cloudflare_security_utils @@ -0,0 +1 @@ +Subproject commit d69e8490e450b50b41451781d4da630b67f1b300 From 856396cd44b88ca1a9b0bf7215097046ee9022b4 Mon Sep 17 00:00:00 2001 From: Masterain Date: Thu, 26 Jun 2025 16:44:04 -0700 Subject: [PATCH 02/14] sync submodule --- cloudflare_security_utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudflare_security_utils b/cloudflare_security_utils index d69e849..3fb8ce8 160000 --- a/cloudflare_security_utils +++ b/cloudflare_security_utils @@ -1 +1 @@ -Subproject commit d69e8490e450b50b41451781d4da630b67f1b300 +Subproject commit 3fb8ce8bf2563cc00e85e68b534b0886b63fba73 From 4ed59da4557a5cfb847b6596210bc205b345e417 Mon Sep 17 00:00:00 2001 From: Masterain Date: Thu, 26 Jun 2025 16:44:31 -0700 Subject: [PATCH 03/14] fix sourcing --- routers/enka_network.py | 2 +- routers/metadata.py | 3 ++- utils/dgp_utils.py | 37 ------------------------------------- 3 files changed, 3 insertions(+), 39 deletions(-) diff --git a/routers/enka_network.py b/routers/enka_network.py index 6c9b30f..3ba4689 100644 --- a/routers/enka_network.py +++ b/routers/enka_network.py @@ -1,7 +1,7 @@ from fastapi import APIRouter, Depends, Request from fastapi.responses import RedirectResponse from redis import asyncio as aioredis -from utils.dgp_utils import validate_client_is_updated +from cloudflare_security_utils.safety import validate_client_is_updated china_router = APIRouter(tags=["Enka Network"], prefix="/enka") diff --git a/routers/metadata.py b/routers/metadata.py index e3eb372..94e2fbc 100644 --- a/routers/metadata.py +++ b/routers/metadata.py @@ -2,7 +2,7 @@ from fastapi.responses import RedirectResponse from redis import asyncio as aioredis from mysql_app.schemas import StandardResponse -from utils.dgp_utils import validate_client_is_updated +from cloudflare_security_utils.safety import validate_client_is_updated from base_logger import get_logger import httpx import os @@ -10,6 +10,7 @@ china_router = APIRouter(tags=["Hutao Metadata"], prefix="/metadata") global_router = APIRouter(tags=["Hutao Metadata"], prefix="/metadata") fujian_router = APIRouter(tags=["Hutao Metadata"], prefix="/metadata") +logger = get_logger(__name__) async def fetch_metadata_repo_file_list(redis_client: aioredis.Redis) -> None: diff --git a/utils/dgp_utils.py b/utils/dgp_utils.py index 44a7cdb..cf9e255 100644 --- a/utils/dgp_utils.py +++ b/utils/dgp_utils.py @@ -1,9 +1,6 @@ import json import os import httpx -from fastapi import HTTPException, status, Header, Request -from redis import asyncio as aioredis -from typing import Annotated from base_logger import get_logger from config import github_headers, IS_DEBUG @@ -14,9 +11,6 @@ WHITE_LIST_REPOSITORIES = {} logger.error("Failed to load WHITE_LIST_REPOSITORIES from environment variable.") logger.info(os.environ.get("WHITE_LIST_REPOSITORIES")) -BYPASS_CLIENT_VERIFICATION = os.environ.get("BYPASS_CLIENT_VERIFICATION", "False").lower() == "true" -if BYPASS_CLIENT_VERIFICATION: - logger.warning("Client verification is bypassed in this server.") # Helper: HTTP GET with retry async def fetch_with_retry(url, max_retries=3): @@ -102,34 +96,3 @@ async def update_recent_versions(redis_client) -> list[str]: return new_user_agents -async def validate_client_is_updated(request: Request, user_agent: Annotated[str, Header()]) -> bool: - requested_hostname = request.headers.get("Host") - if "snapgenshin.cn" in requested_hostname: - return True - redis_client = aioredis.Redis.from_pool(request.app.state.redis) - if BYPASS_CLIENT_VERIFICATION: - logger.debug("Client verification is bypassed.") - return True - logger.info(f"Received request from user agent: {user_agent}") - if user_agent.startswith("Snap Hutao/2025"): - logger.info("Client is Snap Hutao Alpha, allowed.") - return True - if user_agent.startswith("PaimonsNotebook/"): - logger.info("Client is Paimon's Notebook, allowed.") - return True - if user_agent.startswith("Reqable/"): - logger.info("Client is Reqable, allowed.") - return True - - allowed_user_agents = await redis_client.get("allowed_user_agents") - if allowed_user_agents: - allowed_user_agents = json.loads(allowed_user_agents) - else: - # redis data is expired - logger.info("Updating allowed user agents from GitHub") - allowed_user_agents = await update_recent_versions(redis_client) - - if user_agent not in allowed_user_agents: - logger.info(f"Client is outdated: {user_agent}, not in the allowed list: {allowed_user_agents}") - raise HTTPException(status_code=status.HTTP_418_IM_A_TEAPOT, detail="Client is outdated.") - return True From affd0b916611e4182a720e1aefd0a1f8c3154f5e Mon Sep 17 00:00:00 2001 From: Masterain Date: Thu, 26 Jun 2025 17:04:22 -0700 Subject: [PATCH 04/14] add Cloudflare auto safety checker --- routers/client_feature.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/routers/client_feature.py b/routers/client_feature.py index a737e7e..6b03c70 100644 --- a/routers/client_feature.py +++ b/routers/client_feature.py @@ -1,6 +1,7 @@ -from fastapi import APIRouter, Request +from fastapi import APIRouter, Request, Depends from fastapi.responses import RedirectResponse from redis import asyncio as aioredis +from cloudflare_security_utils.safety import enhanced_safety_check china_router = APIRouter(tags=["Client Feature"], prefix="/client") @@ -8,7 +9,7 @@ fujian_router = APIRouter(tags=["Client Feature"], prefix="/client") -@china_router.get("/{file_path:path}") +@china_router.get("/{file_path:path}", dependencies=[Depends(enhanced_safety_check)]) async def china_client_feature_request_handler(request: Request, file_path: str) -> RedirectResponse: """ Handle requests to client feature metadata files. @@ -27,7 +28,7 @@ async def china_client_feature_request_handler(request: Request, file_path: str) return RedirectResponse(host_for_normal_files, status_code=301) -@global_router.get("/{file_path:path}") +@global_router.get("/{file_path:path}", dependencies=[Depends(enhanced_safety_check)]) async def global_client_feature_request_handler(request: Request, file_path: str) -> RedirectResponse: """ Handle requests to client feature metadata files. @@ -46,7 +47,7 @@ async def global_client_feature_request_handler(request: Request, file_path: str return RedirectResponse(host_for_normal_files, status_code=301) -@fujian_router.get("/{file_path:path}") +@fujian_router.get("/{file_path:path}", dependencies=[Depends(enhanced_safety_check)]) async def fujian_client_feature_request_handler(request: Request, file_path: str) -> RedirectResponse: """ Handle requests to client feature metadata files. From 56ae0f7eed2a9724ef221081384c1fdcefa2aecc Mon Sep 17 00:00:00 2001 From: Masterain Date: Thu, 26 Jun 2025 20:11:55 -0700 Subject: [PATCH 05/14] Update cloudflare_security_utils --- cloudflare_security_utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudflare_security_utils b/cloudflare_security_utils index 3fb8ce8..3d0d520 160000 --- a/cloudflare_security_utils +++ b/cloudflare_security_utils @@ -1 +1 @@ -Subproject commit 3fb8ce8bf2563cc00e85e68b534b0886b63fba73 +Subproject commit 3d0d520689fc33214247a582544216893ab1c868 From dbc1faa62863dfa970394da00c93673a0b477840 Mon Sep 17 00:00:00 2001 From: Masterain Date: Thu, 26 Jun 2025 20:19:55 -0700 Subject: [PATCH 06/14] Update cloudflare_security_utils --- cloudflare_security_utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudflare_security_utils b/cloudflare_security_utils index 3d0d520..eed85ea 160000 --- a/cloudflare_security_utils +++ b/cloudflare_security_utils @@ -1 +1 @@ -Subproject commit 3d0d520689fc33214247a582544216893ab1c868 +Subproject commit eed85eae563544370d0c3ea4eed6f44ce01f6be4 From 37c258db11ec60e7663c7352424e46c6941c65a8 Mon Sep 17 00:00:00 2001 From: Masterain Date: Thu, 26 Jun 2025 20:26:26 -0700 Subject: [PATCH 07/14] Update cloudflare_security_utils --- cloudflare_security_utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudflare_security_utils b/cloudflare_security_utils index eed85ea..49ebe1e 160000 --- a/cloudflare_security_utils +++ b/cloudflare_security_utils @@ -1 +1 @@ -Subproject commit eed85eae563544370d0c3ea4eed6f44ce01f6be4 +Subproject commit 49ebe1ef55264168ea923d5ca4b0b324b8bc0eda From a9be29195b75f368eccb5e8d96ac4af535570971 Mon Sep 17 00:00:00 2001 From: Masterain Date: Thu, 26 Jun 2025 20:40:21 -0700 Subject: [PATCH 08/14] Update cloudflare_security_utils --- cloudflare_security_utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudflare_security_utils b/cloudflare_security_utils index 49ebe1e..89b981e 160000 --- a/cloudflare_security_utils +++ b/cloudflare_security_utils @@ -1 +1 @@ -Subproject commit 49ebe1ef55264168ea923d5ca4b0b324b8bc0eda +Subproject commit 89b981e80d40ace59a683579382a6874ef06901e From 16ca5c5ca9fd287f18f38f52574159783faeb27a Mon Sep 17 00:00:00 2001 From: Masterain Date: Thu, 26 Jun 2025 20:47:58 -0700 Subject: [PATCH 09/14] Update cloudflare_security_utils --- cloudflare_security_utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudflare_security_utils b/cloudflare_security_utils index 89b981e..2b1be54 160000 --- a/cloudflare_security_utils +++ b/cloudflare_security_utils @@ -1 +1 @@ -Subproject commit 89b981e80d40ace59a683579382a6874ef06901e +Subproject commit 2b1be5413454b13063fd27d555473ddc8a375da2 From 0161cc56fc750481c5fccf300adbb4026f556f42 Mon Sep 17 00:00:00 2001 From: Masterain Date: Thu, 26 Jun 2025 21:05:45 -0700 Subject: [PATCH 10/14] Update config --- cloudflare_security_utils | 2 +- main.py | 3 +- routers/mgnt.py | 105 -------------------------------------- 3 files changed, 3 insertions(+), 107 deletions(-) delete mode 100644 routers/mgnt.py diff --git a/cloudflare_security_utils b/cloudflare_security_utils index 2b1be54..d2c64eb 160000 --- a/cloudflare_security_utils +++ b/cloudflare_security_utils @@ -1 +1 @@ -Subproject commit 2b1be5413454b13063fd27d555473ddc8a375da2 +Subproject commit d2c64ebf53c0d4ddb8a846bc0a543eebeb00a993 diff --git a/main.py b/main.py index d792f23..30ea02e 100644 --- a/main.py +++ b/main.py @@ -9,7 +9,8 @@ from datetime import datetime from contextlib import asynccontextmanager from routers import (enka_network, metadata, patch_next, static, net, wallpaper, strategy, crowdin, system_email, - client_feature, mgnt) + client_feature) +from cloudflare_security_utils import mgnt from base_logger import get_logger from config import (MAIN_SERVER_DESCRIPTION, TOS_URL, CONTACT_INFO, LICENSE_INFO, VALID_PROJECT_KEYS, IS_DEBUG, IS_DEV, SERVER_TYPE, REDIS_HOST, SENTRY_URL, BUILD_NUMBER, CURRENT_COMMIT_HASH) diff --git a/routers/mgnt.py b/routers/mgnt.py deleted file mode 100644 index aa22c3f..0000000 --- a/routers/mgnt.py +++ /dev/null @@ -1,105 +0,0 @@ -import os -from fastapi import APIRouter, Request, HTTPException, Depends -from starlette.responses import StreamingResponse -from utils.redis_tools import INITIALIZED_REDIS_DATA -from mysql_app.schemas import StandardResponse -from redis import asyncio as aioredis -from pydantic import BaseModel -from utils.authentication import verify_api_token -from utils.dgp_utils import update_recent_versions - - -router = APIRouter(tags=["Management"], prefix="/mgnt", dependencies=[Depends(verify_api_token)]) - - -class UpdateRedirectRules(BaseModel): - """ - Pydantic model for updating the redirect rules. - """ - rule_name: str - rule_template: str - - -@router.get("/redirect-rules", response_model=StandardResponse) -async def get_redirect_rules(request: Request) -> StandardResponse: - """ - Get the redirect rules for the management page. - - :param request: Request object from FastAPI, used to identify the client's IP address - - :return: Standard response with the redirect rules - """ - redis_client = aioredis.Redis.from_pool(request.app.state.redis) - current_dict = INITIALIZED_REDIS_DATA.copy() - for key in INITIALIZED_REDIS_DATA: - current_dict[key] = await redis_client.get(key) - return StandardResponse( - retcode=0, - message="success", - data=current_dict - ) - - -@router.post("/redirect-rules", response_model=StandardResponse) -async def update_redirect_rules(request: Request, update_data: UpdateRedirectRules) -> StandardResponse: - """ - Update the redirect rules for the management page. - - :param request: Request object from FastAPI, used to identify the client's IP address - :param update_data: Pydantic model for updating the redirect rules - - :return: Standard response with the redirect rules - """ - redis_client = aioredis.Redis.from_pool(request.app.state.redis) - if update_data.rule_name not in INITIALIZED_REDIS_DATA: - raise HTTPException(status_code=400, detail="Invalid rule name") - - await redis_client.set(update_data.rule_name, update_data.rule_template) - return StandardResponse( - retcode=0, - message="success", - data={ - update_data.rule_name: update_data.rule_template - } - ) - - -@router.get("/log", response_model=StandardResponse) -async def list_current_logs() -> StandardResponse: - """ - List the current logs of the API - - :return: Standard response with the current logs - """ - log_files = os.listdir("log") - return StandardResponse( - retcode=0, - message="success", - data=log_files - ) - - -@router.get("/log/{log_file}") -async def download_log_file(log_file: str) -> StreamingResponse: - """ - Download the log file specified by the user - - :param log_file: Name of the log file to download - - :return: Streaming response with the log file - """ - return StreamingResponse(open(f"log/{log_file}", "rb"), media_type="text/plain") - - -@router.post("/reset-version", response_model=StandardResponse) -async def reset_latest_version(request: Request) -> StandardResponse: - """ - Reset latest version information by updating allowed user agents. - """ - redis_client = aioredis.Redis.from_pool(request.app.state.redis) - new_versions = await update_recent_versions(redis_client) - return StandardResponse( - retcode=0, - message="Latest version information reset successfully.", - data={"allowed_user_agents": new_versions} - ) From c4f081a19fa6404d3470b3413ccbc7247f50053c Mon Sep 17 00:00:00 2001 From: Masterain Date: Thu, 26 Jun 2025 23:41:59 -0700 Subject: [PATCH 11/14] Update config --- cloudflare_security_utils | 2 +- config.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudflare_security_utils b/cloudflare_security_utils index d2c64eb..58c5bac 160000 --- a/cloudflare_security_utils +++ b/cloudflare_security_utils @@ -1 +1 @@ -Subproject commit d2c64ebf53c0d4ddb8a846bc0a543eebeb00a993 +Subproject commit 58c5bacd3418540070b9f17781ba016f30f9fb08 diff --git a/config.py b/config.py index 7db9c58..2e333fb 100644 --- a/config.py +++ b/config.py @@ -8,8 +8,8 @@ VALID_PROJECT_KEYS = ["snap-hutao", "snap-hutao-deployment"] IMAGE_NAME = os.getenv("IMAGE_NAME", "generic-api") -SERVER_TYPE = os.getenv("SERVER_TYPE", "[Unknown Server Type]") -IS_DEBUG = True if "alpha" in IMAGE_NAME.lower() or "dev" in IMAGE_NAME.lower() else False +SERVER_TYPE = os.getenv("SERVER_TYPE", "unknown").lower() +IS_DEBUG = True if "alpha" in SERVER_TYPE.lower() or "dev" in SERVER_TYPE.lower() else False IS_DEV = True if os.getenv("IS_DEV", "False").lower() == "true" or SERVER_TYPE in ["dev"] else False if IS_DEV: BUILD_NUMBER = "DEV" From ebb4d2e1afe911b8a2ab89783713740db990332a Mon Sep 17 00:00:00 2001 From: Masterain Date: Sun, 29 Jun 2025 20:27:01 -0700 Subject: [PATCH 12/14] Update --- cloudflare_security_utils | 2 +- routers/client_feature.py | 54 +++++++++++++++++---------------------- 2 files changed, 25 insertions(+), 31 deletions(-) diff --git a/cloudflare_security_utils b/cloudflare_security_utils index 58c5bac..16e33a2 160000 --- a/cloudflare_security_utils +++ b/cloudflare_security_utils @@ -1 +1 @@ -Subproject commit 58c5bacd3418540070b9f17781ba016f30f9fb08 +Subproject commit 16e33a219b7d57967afc90dcd4c86c0de1f160ff diff --git a/routers/client_feature.py b/routers/client_feature.py index 6b03c70..31819f4 100644 --- a/routers/client_feature.py +++ b/routers/client_feature.py @@ -9,17 +9,15 @@ fujian_router = APIRouter(tags=["Client Feature"], prefix="/client") -@china_router.get("/{file_path:path}", dependencies=[Depends(enhanced_safety_check)]) -async def china_client_feature_request_handler(request: Request, file_path: str) -> RedirectResponse: - """ - Handle requests to client feature metadata files. +@china_router.get("/{file_path:path}") +async def china_client_feature_request_handler( + request: Request, + file_path: str, + safety_check: bool | RedirectResponse = Depends(enhanced_safety_check) +) -> RedirectResponse: + if isinstance(safety_check, RedirectResponse): + return safety_check - :param request: Request object from FastAPI - - :param file_path: Path to the metadata file - - :return: HTTP 301 redirect to the file based on censorship status of the file - """ redis_client = aioredis.Redis.from_pool(request.app.state.redis) host_for_normal_files = await redis_client.get("url:china:client-feature") @@ -28,17 +26,15 @@ async def china_client_feature_request_handler(request: Request, file_path: str) return RedirectResponse(host_for_normal_files, status_code=301) -@global_router.get("/{file_path:path}", dependencies=[Depends(enhanced_safety_check)]) -async def global_client_feature_request_handler(request: Request, file_path: str) -> RedirectResponse: - """ - Handle requests to client feature metadata files. - - :param request: Request object from FastAPI +@global_router.get("/{file_path:path}") +async def global_client_feature_request_handler( + request: Request, + file_path: str, + safety_check: bool | RedirectResponse = Depends(enhanced_safety_check) +) -> RedirectResponse: + if isinstance(safety_check, RedirectResponse): + return safety_check - :param file_path: Path to the metadata file - - :return: HTTP 301 redirect to the file based on censorship status of the file - """ redis_client = aioredis.Redis.from_pool(request.app.state.redis) host_for_normal_files = await redis_client.get("url:global:client-feature") @@ -47,17 +43,15 @@ async def global_client_feature_request_handler(request: Request, file_path: str return RedirectResponse(host_for_normal_files, status_code=301) -@fujian_router.get("/{file_path:path}", dependencies=[Depends(enhanced_safety_check)]) -async def fujian_client_feature_request_handler(request: Request, file_path: str) -> RedirectResponse: - """ - Handle requests to client feature metadata files. - - :param request: Request object from FastAPI - - :param file_path: Path to the metadata file +@fujian_router.get("/{file_path:path}") +async def fujian_client_feature_request_handler( + request: Request, + file_path: str, + safety_check: bool | RedirectResponse = Depends(enhanced_safety_check) +) -> RedirectResponse: + if isinstance(safety_check, RedirectResponse): + return safety_check - :return: HTTP 301 redirect to the file based on censorship status of the file - """ redis_client = aioredis.Redis.from_pool(request.app.state.redis) host_for_normal_files = await redis_client.get("url:fujian:client-feature") From e9e85468ec3e7271fc442e5e19317f5994eded6b Mon Sep 17 00:00:00 2001 From: Masterain Date: Sun, 29 Jun 2025 23:50:52 -0700 Subject: [PATCH 13/14] add auto whitelist --- cloudflare_security_utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudflare_security_utils b/cloudflare_security_utils index 16e33a2..5e4bf8d 160000 --- a/cloudflare_security_utils +++ b/cloudflare_security_utils @@ -1 +1 @@ -Subproject commit 16e33a219b7d57967afc90dcd4c86c0de1f160ff +Subproject commit 5e4bf8d1b4a152c32cb63e8def5472ff218f04b5 From 3fbe11cb790e25226bf19aef9553dd7b6ae34517 Mon Sep 17 00:00:00 2001 From: Masterain Date: Mon, 30 Jun 2025 00:45:52 -0700 Subject: [PATCH 14/14] Safety update --- cloudflare_security_utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudflare_security_utils b/cloudflare_security_utils index 5e4bf8d..5f3708e 160000 --- a/cloudflare_security_utils +++ b/cloudflare_security_utils @@ -1 +1 @@ -Subproject commit 5e4bf8d1b4a152c32cb63e8def5472ff218f04b5 +Subproject commit 5f3708ebed49743b6f32208c5bfc6cc5e7623d4b