ValueError: Cannot set a non-string value as the pad_token

[acDante]

Question

Hi, I would like to use inseq to get the attribution from Llama3.1-8b-instruct model, but I got the following error when initialising the inseq model:

This is my code:

model = inseq.load_model("meta-llama/Llama-3.1-8B-Instruct", "attention")
out = model.attribute(
    "Hello ladies and",
    generation_args={"max_new_tokens": 128},
    n_steps=500,
    internal_batch_size=50
).show()

ValueError                                Traceback (most recent call last)
Cell In[4], [line 1](vscode-notebook-cell:?execution_count=4&line=1)
----> [1](vscode-notebook-cell:?execution_count=4&line=1) model = inseq.load_model("meta-llama/Llama-3.1-8B-Instruct", "attention")
      [2](vscode-notebook-cell:?execution_count=4&line=2) out = model.attribute(
      [3](vscode-notebook-cell:?execution_count=4&line=3)     "Hello ladies and",
      [4](vscode-notebook-cell:?execution_count=4&line=4)     generation_args={"max_new_tokens": 128},
      [5](vscode-notebook-cell:?execution_count=4&line=5)     n_steps=500,
      [6](vscode-notebook-cell:?execution_count=4&line=6)     internal_batch_size=50
      [7](vscode-notebook-cell:?execution_count=4&line=7) ).show()

File /mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/__init__.py:47, in load_model(model, attribution_method, framework, **kwargs)
     [45](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/__init__.py:45) load_msg = f"Loading {model_name} {method_desc}"
     [46](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/__init__.py:46) with optional(not isnotebook(), Status(load_msg), logger.info, msg=load_msg):
---> [47](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/__init__.py:47)     return FRAMEWORKS_MAP[framework].load(model, attribution_method, **kwargs)

File /mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:163, in HuggingfaceModel.load(model, attribution_method, tokenizer, device, model_kwargs, tokenizer_kwargs, **kwargs)
    [159](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:159)     return HuggingfaceEncoderDecoderModel(
    [160](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:160)         model, attribution_method, tokenizer, device, model_kwargs, tokenizer_kwargs, **kwargs
    [161](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:161)     )
    [162](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:162) else:
--> [163](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:163)     return HuggingfaceDecoderOnlyModel(
    [164](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:164)         model, attribution_method, tokenizer, device, model_kwargs, tokenizer_kwargs, **kwargs
    [165](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:165)     )

File /mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:494, in HuggingfaceDecoderOnlyModel.__init__(self, model, attribution_method, tokenizer, device, model_kwargs, tokenizer_kwargs, **kwargs)
    [484](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:484) def __init__(
    [485](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:485)     self,
    [486](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:486)     model: Union[str, PreTrainedModel],
   (...)
    [492](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:492)     **kwargs,
    [493](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:493) ) -> NoReturn:
--> [494](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:494)     super().__init__(model, attribution_method, tokenizer, device, model_kwargs, tokenizer_kwargs, **kwargs)
    [495](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:495)     self.tokenizer.padding_side = "left"
    [496](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:496)     self.tokenizer.truncation_side = "left"

File /mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:122, in HuggingfaceModel.__init__(self, model, attribution_method, tokenizer, device, model_kwargs, tokenizer_kwargs, **kwargs)
    [120](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:120) self.pad_token = self._convert_ids_to_tokens(pad_token_id, skip_special_tokens=False)
    [121](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:121) if self.tokenizer.pad_token is None:
--> [122](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:122)     self.tokenizer.pad_token = self.pad_token
    [123](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:123) if self.model.config.pad_token_id is None:
    [124](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:124)     self.model.config.pad_token_id = pad_token_id

File /mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:1081, in SpecialTokensMixin.__setattr__(self, key, value)
   [1078](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:1078)         key = key_without_id
   [1080](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:1080)     if key != "additional_special_tokens" and not isinstance(value, (str, AddedToken)) and value is not None:
-> [1081](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:1081)         raise ValueError(f"Cannot set a non-string value as the {key}")
   [1082](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:1082)     self._special_tokens_map[key] = value
   [1083](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:1083) else:

ValueError: Cannot set a non-string value as the pad_token

[gsarti]

Hi @acDante, are you using the latest version of the library by installing with pip install git+https://github.com/inseq-team/inseq.git@main ? #287 was describing a similar issue iirc, and it should be fixed in main! Let me know!

[acDante]

Hi @gsarti, thanks for your prompt reply! Yes I am using the latest version of the library by installing from git repo, but I still got this issue. I also tried to initialise the model and tokenizer in another way and set a pad_token manually. However, I run into another issue in this case:

This is my code snippet:

model_name = "meta-llama/Llama-3.1-8B-Instruct"
model = AutoModelForCausalLM.from_pretrained(model_name,
                                             torch_dtype=torch.bfloat16,
                                             device_map="auto",
                                             use_auth_token=True,
                                             cache_dir="/mnt/ceph_rbd/llms")
tokenizer = AutoTokenizer.from_pretrained(model_name)
tokenizer.pad_token = "[PAD]"


model = inseq.load_model(model, "attention", tokenizer=tokenizer)
out = model.attribute(
    "Hello ladies and",
    generation_args={"max_new_tokens": 128},
    n_steps=500,
    internal_batch_size=50
).show()

It will cause this error message:

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[2], [line 12](vscode-notebook-cell:?execution_count=2&line=12)
      [8](vscode-notebook-cell:?execution_count=2&line=8) tokenizer.pad_token = "[PAD]"
     [11](vscode-notebook-cell:?execution_count=2&line=11) model = inseq.load_model(model, "attention", tokenizer=tokenizer)
---> [12](vscode-notebook-cell:?execution_count=2&line=12) out = model.attribute(
     [13](vscode-notebook-cell:?execution_count=2&line=13)     "Hello ladies and",
     [14](vscode-notebook-cell:?execution_count=2&line=14)     generation_args={"max_new_tokens": 128},
     [15](vscode-notebook-cell:?execution_count=2&line=15)     n_steps=500,
     [16](vscode-notebook-cell:?execution_count=2&line=16)     internal_batch_size=50
     [17](vscode-notebook-cell:?execution_count=2&line=17) ).show()

File /mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/attribution_model.py:474, in AttributionModel.attribute(self, input_texts, generated_texts, method, override_default_attribution, attr_pos_start, attr_pos_end, show_progress, pretty_progress, output_step_attributions, attribute_target, step_scores, include_eos_baseline, attributed_fn, device, batch_size, generate_from_target_prefix, skip_special_tokens, clean_special_chars, generation_args, **kwargs)
    [472](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/attribution_model.py:472) # If constrained decoding is not enabled, output texts are generated from input texts.
    [473](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/attribution_model.py:473) if not has_generated_texts or generate_from_target_prefix:
--> [474](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/attribution_model.py:474)     encoded_input = self.encode(
    [475](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/attribution_model.py:475)         input_texts,
    [476](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/attribution_model.py:476)         return_baseline=True,
    [477](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/attribution_model.py:477)         include_eos_baseline=include_eos_baseline,
    [478](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/attribution_model.py:478)         add_special_tokens=not skip_special_tokens,
    [479](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/attribution_model.py:479)     )
    [480](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/attribution_model.py:480)     if generate_from_target_prefix:
    [481](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/attribution_model.py:481)         decoder_input = self.encode(
    [482](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/attribution_model.py:482)             generated_texts, as_targets=True, add_special_tokens=not skip_special_tokens
    [483](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/attribution_model.py:483)         )

File /mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:282, in HuggingfaceModel.encode(self, texts, as_targets, return_baseline, include_eos_baseline, add_bos_token, add_special_tokens)
    [280](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:280) if as_targets and not self.is_encoder_decoder:
    [281](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:281)     raise ValueError("Decoder-only models should use tokenization as source only.")
--> [282](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:282) batch = self.tokenizer(
    [283](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:283)     text=texts if not as_targets else None,
    [284](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:284)     text_target=texts if as_targets else None,
    [285](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:285)     add_special_tokens=add_special_tokens,
    [286](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:286)     padding=True,
    [287](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:287)     truncation=True,
    [288](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:288)     return_tensors="pt",
    [289](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:289) ).to(self.device)
    [290](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:290) baseline_ids = None
    [292](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:292) # Fix: If two BOS tokens are present (e.g. when using chat templates), the second one is removed.

File /mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2887, in PreTrainedTokenizerBase.__call__(self, text, text_pair, text_target, text_pair_target, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, padding_side, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, **kwargs)
   [2885](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2885)     if not self._in_target_context_manager:
   [2886](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2886)         self._switch_to_input_mode()
-> [2887](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2887)     encodings = self._call_one(text=text, text_pair=text_pair, **all_kwargs)
   [2888](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2888) if text_target is not None:
   [2889](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2889)     self._switch_to_target_mode()

File /mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2975, in PreTrainedTokenizerBase._call_one(self, text, text_pair, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, padding_side, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, split_special_tokens, **kwargs)
   [2970](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2970)         raise ValueError(
   [2971](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2971)             f"batch length of `text`: {len(text)} does not match batch length of `text_pair`:"
   [2972](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2972)             f" {len(text_pair)}."
   [2973](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2973)         )
   [2974](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2974)     batch_text_or_text_pairs = list(zip(text, text_pair)) if text_pair is not None else text
-> [2975](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2975)     return self.batch_encode_plus(
   [2976](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2976)         batch_text_or_text_pairs=batch_text_or_text_pairs,
   [2977](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2977)         add_special_tokens=add_special_tokens,
   [2978](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2978)         padding=padding,
   [2979](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2979)         truncation=truncation,
   [2980](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2980)         max_length=max_length,
   [2981](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2981)         stride=stride,
   [2982](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2982)         is_split_into_words=is_split_into_words,
   [2983](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2983)         pad_to_multiple_of=pad_to_multiple_of,
   [2984](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2984)         padding_side=padding_side,
   [2985](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2985)         return_tensors=return_tensors,
   [2986](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2986)         return_token_type_ids=return_token_type_ids,
   [2987](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2987)         return_attention_mask=return_attention_mask,
   [2988](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2988)         return_overflowing_tokens=return_overflowing_tokens,
   [2989](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2989)         return_special_tokens_mask=return_special_tokens_mask,
   [2990](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2990)         return_offsets_mapping=return_offsets_mapping,
   [2991](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2991)         return_length=return_length,
   [2992](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2992)         verbose=verbose,
   [2993](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2993)         split_special_tokens=split_special_tokens,
   [2994](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2994)         **kwargs,
   [2995](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2995)     )
   [2996](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2996) else:
   [2997](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2997)     return self.encode_plus(
   [2998](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2998)         text=text,
   [2999](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2999)         text_pair=text_pair,
   (...)
   [3017](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:3017)         **kwargs,
   [3018](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:3018)     )

File /mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:3168, in PreTrainedTokenizerBase.batch_encode_plus(self, batch_text_or_text_pairs, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, padding_side, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, split_special_tokens, **kwargs)
   [3151](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:3151) """
   [3152](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:3152) Tokenize and prepare for the model a list of sequences or a list of pairs of sequences.
   [3153](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:3153) 
   (...)
   [3164](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:3164)         details in `encode_plus`).
   [3165](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:3165) """
   [3167](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:3167) # Backward compatibility for 'truncation_strategy', 'pad_to_max_length'
-> [3168](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:3168) padding_strategy, truncation_strategy, max_length, kwargs = self._get_padding_truncation_strategies(
   [3169](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:3169)     padding=padding,
   [3170](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:3170)     truncation=truncation,
   [3171](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:3171)     max_length=max_length,
   [3172](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:3172)     pad_to_multiple_of=pad_to_multiple_of,
   [3173](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:3173)     verbose=verbose,
   [3174](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:3174)     **kwargs,
   [3175](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:3175) )
   [3177](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:3177) return self._batch_encode_plus(
   [3178](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:3178)     batch_text_or_text_pairs=batch_text_or_text_pairs,
   [3179](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:3179)     add_special_tokens=add_special_tokens,
   (...)
   [3196](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:3196)     **kwargs,
   [3197](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:3197) )

File /mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2788, in PreTrainedTokenizerBase._get_padding_truncation_strategies(self, padding, truncation, max_length, pad_to_multiple_of, verbose, **kwargs)
   [2785](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2785)             max_length = self.model_max_length
   [2787](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2787) # Test if we have a padding token
-> [2788](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2788) if padding_strategy != PaddingStrategy.DO_NOT_PAD and (self.pad_token is None or self.pad_token_id < 0):
   [2789](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2789)     raise ValueError(
   [2790](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2790)         "Asking to pad but the tokenizer does not have a padding token. "
   [2791](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2791)         "Please select a token to use as `pad_token` `(tokenizer.pad_token = tokenizer.eos_token e.g.)` "
   [2792](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2792)         "or add a new pad token via `tokenizer.add_special_tokens({'pad_token': '[PAD]'})`."
   [2793](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2793)     )
   [2795](https://vscode-remote+tunnel-002bxiaotang-002dpod.vscode-resource.vscode-cdn.net/mnt/ceph_rbd/miniconda3/envs/inseq/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2795) # Check that we will truncate to a multiple of pad_to_multiple_of if both are provided

TypeError: '<' not supported between instances of 'NoneType' and 'int'

[gsarti]

@acDante Can you try the following? This works for me on trl-internal-testing/tiny-random-LlamaForCausalLM, which uses the same architecture:

import inseq
from transformers import AutoTokenizer, AutoModelForCausalLM

model_name = "meta-llama/Llama-3.1-8B-Instruct"
m = AutoModelForCausalLM.from_pretrained(model_name)
t = AutoTokenizer.from_pretrained(model_name)

# Fix the mismatch between the tokenizer and model pad token
m.config.pad_token_id = t.pad_token_id
m.config.pad_token = t.pad_token

model = inseq.load_model(m, "attention", tokenizer=t)
out = model.attribute(
    "Hello ladies and",
    generation_args={"max_new_tokens": 128, "skip_special_tokens": True},
).show()

P.S. n_steps and internal_batch_size are parameters of integrated_gradients, using them with attention leads to them being ignored during attribution, so you can omit them.

[acDante]

Hi, sorry for getting back late! I am afraid this code snippet does not work for Llama3.1-8b-instruct model. Here is the error information I get:

Code snippet

import inseq
from transformers import AutoTokenizer, AutoModelForCausalLM

model_name = "meta-llama/Llama-3.1-8B-Instruct"
m = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", torch_dtype=torch.bfloat16,)
t = AutoTokenizer.from_pretrained(model_name)

# Fix the mismatch between the tokenizer and model pad token
m.config.pad_token_id = t.pad_token_id
m.config.pad_token = t.pad_token

model = inseq.load_model(m, "attention", tokenizer=t)
out = model.attribute(
    "Hello ladies and",
    generation_args={"max_new_tokens": 128, "skip_special_tokens": True},
).show()

Error

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[1], [line 21](vscode-notebook-cell:?execution_count=1&line=21)
     [18](vscode-notebook-cell:?execution_count=1&line=18) inputs = t.encode(input_text, return_tensors="pt")
     [20](vscode-notebook-cell:?execution_count=1&line=20) model = inseq.load_model(m, "attention", tokenizer=t)
---> [21](vscode-notebook-cell:?execution_count=1&line=21) out = model.attribute(
     [22](vscode-notebook-cell:?execution_count=1&line=22)     input_text,
     [23](vscode-notebook-cell:?execution_count=1&line=23)     generation_args={"max_new_tokens": 128, "skip_special_tokens": True},
     [24](vscode-notebook-cell:?execution_count=1&line=24) ).show()

File ~/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/attribution_model.py:420, in AttributionModel.attribute(self, input_texts, generated_texts, method, override_default_attribution, attr_pos_start, attr_pos_end, show_progress, pretty_progress, output_step_attributions, attribute_target, step_scores, include_eos_baseline, attributed_fn, device, batch_size, generate_from_target_prefix, generation_args, **kwargs)
    [418](https://vscode-remote+ssh-002dremote-002bprivate.vscode-resource.vscode-cdn.net/home/xiaotang/Project/context-faithful-llm/guided-cad/notebooks/~/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/attribution_model.py:418) # If constrained decoding is not enabled, output texts are generated from input texts.
    [419](https://vscode-remote+ssh-002dremote-002bprivate.vscode-resource.vscode-cdn.net/home/xiaotang/Project/context-faithful-llm/guided-cad/notebooks/~/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/attribution_model.py:419) if not has_generated_texts or generate_from_target_prefix:
--> [420](https://vscode-remote+ssh-002dremote-002bprivate.vscode-resource.vscode-cdn.net/home/xiaotang/Project/context-faithful-llm/guided-cad/notebooks/~/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/attribution_model.py:420)     encoded_input = self.encode(input_texts, return_baseline=True, include_eos_baseline=include_eos_baseline)
    [421](https://vscode-remote+ssh-002dremote-002bprivate.vscode-resource.vscode-cdn.net/home/xiaotang/Project/context-faithful-llm/guided-cad/notebooks/~/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/attribution_model.py:421)     if generate_from_target_prefix:
    [422](https://vscode-remote+ssh-002dremote-002bprivate.vscode-resource.vscode-cdn.net/home/xiaotang/Project/context-faithful-llm/guided-cad/notebooks/~/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/attribution_model.py:422)         decoder_input = self.encode(generated_texts, as_targets=True)

File ~/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:273, in HuggingfaceModel.encode(self, texts, as_targets, return_baseline, include_eos_baseline, add_bos_token, add_special_tokens)
    [271](https://vscode-remote+ssh-002dremote-002bprivate.vscode-resource.vscode-cdn.net/home/xiaotang/Project/context-faithful-llm/guided-cad/notebooks/~/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:271)     baseline_ids = torch.ones_like(batch["input_ids"]).long() * self.tokenizer.unk_token_id
    [272](https://vscode-remote+ssh-002dremote-002bprivate.vscode-resource.vscode-cdn.net/home/xiaotang/Project/context-faithful-llm/guided-cad/notebooks/~/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:272) else:
--> [273](https://vscode-remote+ssh-002dremote-002bprivate.vscode-resource.vscode-cdn.net/home/xiaotang/Project/context-faithful-llm/guided-cad/notebooks/~/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:273)     baseline_ids_non_eos = batch["input_ids"].ne(self.eos_token_id).long() * self.tokenizer.unk_token_id
    [274](https://vscode-remote+ssh-002dremote-002bprivate.vscode-resource.vscode-cdn.net/home/xiaotang/Project/context-faithful-llm/guided-cad/notebooks/~/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:274)     baseline_ids_eos = batch["input_ids"].eq(self.eos_token_id).long() * self.eos_token_id
    [275](https://vscode-remote+ssh-002dremote-002bprivate.vscode-resource.vscode-cdn.net/home/xiaotang/Project/context-faithful-llm/guided-cad/notebooks/~/miniconda3/envs/inseq/lib/python3.10/site-packages/inseq/models/huggingface_model.py:275)     baseline_ids = baseline_ids_non_eos + baseline_ids_eos

TypeError: ne() received an invalid combination of arguments - got (list), but expected one of:
 * (Tensor other)
      didn't match because some of the arguments have invalid types: (!list of [int, int, int]!)
 * (Number other)
      didn't match because some of the arguments have invalid types: (!list of [int, int, int]!)