feat: add max_model_len for vllm

Author: terryyz

ADVANCED_USAGE.md

@@ -50,6 +50,7 @@ Below are all the arguments for `bigcodebench.evaluate` for the remote evaluatio
 - `--n_samples`: The number of samples, default to `1`
 - `--temperature`: The temperature, default to `0.0`
 - `--max_new_tokens`: The length of max new tokens, default to `1280`
+- `--max_model_len`: The length of max tokens in VLLM, default to `12800`
 - `--greedy`: Whether to use greedy decoding, default to `False`
 - `--strip_newlines`: Whether to strip newlines, default to `False`, set to `True` to strip newlines for some model series like StarCoder2
 - `--direct_completion`: Whether to use direct completion, default to `False`

bigcodebench/generate.py

@@ -132,6 +132,8 @@ def run_codegen(
     n_samples: int = 1,
     temperature: float = 0.0,
     max_new_tokens: int = 1280,
+    # vllm
+    max_model_len: int = 12800,
     greedy: bool = False,
     # openai
     reasoning_effort: str = "medium",
@@ -178,6 +180,7 @@ def run_codegen(
         lora_path=lora_path,
         temperature=temperature,
         max_new_tokens=max_new_tokens,
+        max_model_len=max_model_len,
         reasoning_effort=reasoning_effort,
         reasoning_budget=reasoning_budget,
         reasoning_beta=reasoning_beta,

bigcodebench/provider/__init__.py

@@ -10,6 +10,7 @@ def make_model(
     dataset: str = "bigcodebench",
     temperature: float = 0.0,
     max_new_tokens: int = 1280,
+    max_model_len: int = 12800,
     # openai only
     reasoning_effort: str = "medium",
     # anthropic only
@@ -42,6 +43,7 @@ def make_model(
             lora_path=lora_path,
             temperature=temperature,
             max_new_tokens=max_new_tokens,
+            max_model_len=max_model_len,
             revision=revision,
             dataset=dataset,
             direct_completion=direct_completion,

bigcodebench/provider/vllm.py

@@ -13,7 +13,7 @@
 )
 
 class VllmDecoder(DecoderBase):
-    def __init__(self, name: str, lora_path: str, dataset: str, tp: int, **kwargs) -> None:
+    def __init__(self, name: str, lora_path: str, dataset: str, tp: int, max_model_len: int, **kwargs) -> None:
         super().__init__(name, **kwargs)
 
         kwargs = {
@@ -41,8 +41,7 @@ def __init__(self, name: str, lora_path: str, dataset: str, tp: int, **kwargs) -
                 local_lora_path,
             )
 
-        # max_model_len is set to max_new_tokens * 10
-        self.llm = LLM(model=name, max_model_len=self.max_new_tokens * 10, enable_lora=True if self.lora_request else False, **kwargs)
+        self.llm = LLM(model=name, max_model_len=max_model_len, enable_lora=True if self.lora_request else False, **kwargs)
         self.llm.set_tokenizer(tokenizer=self.tokenizer)
 
     def is_direct_completion(self) -> bool: