diff --git a/surprisal/model.py b/surprisal/model.py index 0a10d6a..0671a02 100644 --- a/surprisal/model.py +++ b/surprisal/model.py @@ -321,7 +321,22 @@ def surprise( mask_mask = torch.eye(n, n)[1:, :].repeat(b, 1).bool() ids_with_bos_token[mask_mask] = self.tokenizer.mask_token_id - import IPython + # below is from ckauf and neuranna? + # if "within_word_l2r" == PLL_metric: + # """ + # Future tokens belonging to the same word as the target token are masked during token inference as well. + # """ + # mask_indices = [ + # [mask_pos] + # + [ + # j + # for j in range(mask_pos + 1, effective_length + 2) + # if word_ids[j] == word_ids[mask_pos] + # ] + # if word_ids[mask_pos] is not None + # else [mask_pos] + # for mask_pos in range(effective_length + 2) + # ] raise NotImplementedError diff --git a/test.py b/test.py index c6fa57d..3ed2654 100644 --- a/test.py +++ b/test.py @@ -5,11 +5,12 @@ import surprisal -g = surprisal.AutoHuggingFaceModel.from_pretrained(model_id="gpt2") -# b = surprisal.AutoHuggingFaceModel.from_pretrained(model_id="bert-base-uncased") +# m = surprisal.AutoHuggingFaceModel.from_pretrained(model_id="gpt2") +m = surprisal.AutoHuggingFaceModel.from_pretrained(model_id="bert-base-uncased") stims = [ + "The hooligan wrecked my new car!", "I am a cat on the mat", # "The cat sat on the mat.", # "The cat sat on the pizza.", @@ -20,7 +21,7 @@ # "How likely is a spicy burrito?", ] -surps = [*g.surprise(stims), *g.surprise(stims, use_bos_token=False)] +surps = [*m.surprise(stims), *m.surprise(stims, use_bos_token=False)] f, a = plt.subplots()