diff --git a/.gitignore b/.gitignore index a2fc05f..32b2e7d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ testing.ipynb dist/ *.gitignore mlp_map_test/ -attn_test/ simulation.py -test.pt \ No newline at end of file +attn_test/ +test.pt +mlp/ + diff --git a/README.md b/README.md index fe07539..122b34a 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This library is in an alpha state, it probably has some bugs. Please let me know ``` -from tiny_model import TinyModel, tokenizer +from tinymodel import TinyModel, tokenizer lm = TinyModel() diff --git a/pyproject.toml b/pyproject.toml index 9c3874a..e2e2fbf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,10 @@ [tool.poetry] name = "tinymodel" -version = "0.1.2-1" +version = "0.1.2-2" description = "A small TinyStories LM with SAEs and transcoders" authors = ["Noa Nabeshima "] readme = "README.md" -packages = [{include = "tiny_model"}] +packages = [{include = "tinymodel"}] [tool.poetry.dependencies] python = "^3.11" @@ -19,7 +19,7 @@ datasets = "^2.20.0" [[tool.poetry.source]] -name = "tiny_model" +name = "tinymodel" url = "https://github.com/noanabeshima/tiny_model" priority = "explicit" diff --git a/tiny_model/lm.py b/tiny_model/lm.py deleted file mode 100644 index 24af344..0000000 --- a/tiny_model/lm.py +++ /dev/null @@ -1,261 +0,0 @@ -import re -from textwrap import dedent - -import torch -import torch.distributions as dists -import torch.nn as nn -import torch.nn.functional as F -from huggingface_hub import hf_hub_download - -from .lm_modules import TransformerBlock -from .sparse_mlp import SparseMLP -from .tokenization.tokenization import dec, enc - -DEFAULT_SPARSE_MLPS = { - # "M0": "mlp_map_test/M0_S-6_R2_P2", - # "M1": "mlp_map_test/M1_S-4_R8_P2", - - "M0": "mlp/M0_S-2_R1_P0", - "M1": "mlp/M1_S-2_R1_P0", - "M2": "mlp/M2_S-2_R1_P0", - "M3": "mlp/M3_S-1_B0_P0", - - "A0": "attn/A0_S-2_B0_P1", - "A1": "attn/A1_S-1_B0_P1", - "A2": "attn/A2_S-2_B0_P1", - "A3": "attn/A3_S-1_B2_P1", -} - - -def parse_mlp_tag(mlp_tag): - defaults_tag_pat = re.compile( - r"(?P(M|Rm|Ra|A|Mo))(?P\d+)(\D(?P\d+))?" - ) - defaults_match = defaults_tag_pat.fullmatch(mlp_tag) - file_tag_pat = re.compile(r'(?P(?P(Mo|M|A|Rm|Ra))(?P\d+)_S[-\d]+.{0,6}_P\d+)([^\d](?P\d+))?') - full_file_match = file_tag_pat.fullmatch(mlp_tag) - - if defaults_match: - match_groups = defaults_match.groupdict() - mlp_type, layer, feature_idx = ( - match_groups["mlp_type"], - int(match_groups["layer"]), - match_groups["feature_idx"] - ) - - - feature_idx = None if feature_idx is None else int(feature_idx) - - assert mlp_type+str(layer) in DEFAULT_SPARSE_MLPS - return DEFAULT_SPARSE_MLPS[mlp_type+str(layer)], mlp_type, layer, feature_idx - elif full_file_match: - # try interpreting the mlp_tag as a filename - - mlp_type_to_file = { - # 'Mo': 'mlp_out', - 'A': 'attn_test', - 'M': 'mlp_map_test', - # 'Ra': 'res_pre_attn', - # 'Rm': 'res_pre_mlp' - } - - match_groups = full_file_match.groupdict() - - full_name, mlp_type, layer, feature_idx = match_groups['full_name'], match_groups['mlp_type'], int(match_groups['layer']), match_groups['feature_idx'] - file = mlp_type_to_file[mlp_type] + '/' + full_name - - feature_idx = None if feature_idx is None else int(feature_idx) - - return file, mlp_type, layer, feature_idx - else: - return False - - - - - -class TinyModel(nn.Module): - def __init__( - self, - d_model=768, - n_layers=4, - n_heads=16, - max_seq_len=256, - vocab_size=10_000, - from_pretrained="tiny_model", - ): - super().__init__() - self.embed = nn.Embedding(vocab_size, d_model) - self.embed.weight = nn.Parameter( - 1e-4 * torch.randn(self.embed.weight.shape, requires_grad=True) - ) - self.pos_embed = nn.Parameter( - 1e-4 * torch.randn(1, max_seq_len, d_model, requires_grad=True) - ) - - self.torso = nn.Sequential( - *[ - TransformerBlock( - d_model=d_model, n_heads=n_heads, max_seq_len=max_seq_len - ) - for _ in range(n_layers) - ] - ) - self.lm_head = nn.Linear(d_model, vocab_size) - - self.d_model = d_model - self.n_layers = n_layers - self.n_heads = n_heads - self.vocab_size = vocab_size - self.max_seq_len = max_seq_len - - if isinstance(from_pretrained, str): - self.load_state_dict(get_state_dict(from_pretrained)) - else: - assert ( - from_pretrained is False - ), "from_pretrained kwarg must be False or a string specifying model" - - # Dict from mlp_tag to sparse mlp - self.sparse_mlps = nn.ModuleDict() - - @property - def dtype(self): - return self.embed.weight.dtype - - @property - def device(self): - return self.embed.weight.device - - def forward(self, tok_ids, return_idx=None): - T = tok_ids.shape[-1] - x = self.embed(tok_ids) + self.pos_embed[:, :T] - if return_idx is not None: - assert isinstance(return_idx, int) - assert 0 <= return_idx and return_idx <= self.n_layers - for layer_idx, layer in enumerate(self.torso): - if layer_idx == return_idx: - return x - x = layer(x) - else: - x = self.torso(x) - logits = self.lm_head(x) - return F.log_softmax(logits, dim=-1) - - def generate(self, prompt, n_toks=50, temperature=0.8, break_on_end=True): - assert temperature >= 0.0 - toks = enc(prompt, add_begin=True).to(self.lm_head.weight.device) - - for _ in range(n_toks): - with torch.no_grad(): - logprobs = self.forward(toks)[0, -1] - if temperature == 0: - next_tok = logprobs.argmax().item() - else: - next_tok = dists.Categorical( - logits=logprobs * (1 / temperature) - ).sample() - toks = torch.cat((toks, torch.tensor([[next_tok]]).to(toks.device)), dim=-1) - if break_on_end and next_tok == enc("[END]").item(): - break - if toks.shape[1] >= self.max_seq_len: - break - return dec(toks[:, 1:])[0] - - def sparse_mlp(self, mlp_tag=None, mlp=None): - ''' - Returns `get_sparse_mlp_acts`, which takes in tok_ids and returns sparse mlp activations. It optionally allows `indices`. - ''' - assert not (mlp_tag is None and mlp is None) - - parse_output = parse_mlp_tag(mlp_tag) - - - if parse_output is False: - assert False, dedent( - 'Failed to parse mlp.' - ) - # assert False, dedent( - # """ - # [STUB] - # That\'s not a valid MLP tag. Here are some examples of MLP tags: - # M0, A2, Rm0, Ra1, Mo3 - # They start with a string in [M, A, Rm, Ra, Mo] - # representing mlp map, attn out SAE, residual pre-mlp SAE, residual pre-attn SAE, and MLP out SAE respectively. - # and they end with a number representing the layer. - - # You can also specify individual feature_idxs, e.g. lm['A2.100'](tok_ids) to get the activations of neuron 100. - # """ - # ) - else: - file, mlp_type, layer, feature_idx = parse_output - mlp_tag = mlp_type + str(layer) - if mlp is None: - sparse_mlp = SparseMLP.from_pretrained(file).to(device=self.device, dtype=self.dtype) - else: - sparse_mlp = mlp.to(device=self.device, dtype=self.dtype) - # else: - # assert False, dedent( - # """ - # mlp_tag {mlp_tag} not found in tiny_model.sparse_mlps or DEFAULT_SPARSE_MLPS - - # [STUB]: unimplemented - # To add a sparse_mlp, do e.g. - # tiny_model.set_saes({ - # \'M2\': SparseMLP.from_pretrained(\'mlp_map/M0_S-1_B0_P0\') - # }) - - # Available keys (of form {mlp_type}{layer}) are: - # M0..3 (for MLPs) - # A0..3 (for Attn out) - # Rm0..3 (for SAE on the residual before MLP) - # Ra0..3 (for SAE on the residual stream before attn) - # Mo0..3 (for SAE on MLP out) - - # See https://huggingface.co/noanabeshima/tiny_model/tree/main for available sparse MLPs. - # """ - # ) - - def get_sparse_mlp_acts(tok_ids, indices=feature_idx): - x = self.forward(tok_ids, return_idx=layer) - if mlp_type == "Ra": - return sparse_mlp.get_acts(x, indices=indices) - attn_out = self.torso[layer].attn(x) - if mlp_type == "A": - return sparse_mlp.get_acts(attn_out, indices=indices) - x = attn_out + x - if mlp_type in {"M", "Rm"}: - return sparse_mlp.get_acts(x, indices=indices) - else: - assert mlp_type == "Mo", "mlp_type must be one of Ra, A, M, Rm, Mo" - mlp_out = self.torso[layer].mlp(x) - return sparse_mlp.get_acts(mlp_out, indices=indices) - - return get_sparse_mlp_acts - - def __getitem__(self, mlp_tag): - """ - To be used like: - sparse_acts = lm['A0'](tok_ids, indices=[1,5,100]) - sparse_acts = lm['M1'](tok_ids, indices=slice(0,100)) - sparse_acts = lm['M3'](tok_ids, indices=0) - - or for single neurons - - sparse_acts = lm['M2N100'](tok_ids) - sparse_acts = lm['M2.100'](tok_ids) - """ - return self.sparse_mlp(mlp_tag) - - -def get_state_dict(model_fname="tiny_model"): - assert model_fname in [ - "tiny_model", - "tiny_model_2L_1E", - "tiny_model_2L_3E" - ], "There are 3 models available: `tiny_model`, `tiny_model_2L_1E`, and `tiny_model_2L_3E`." - state_dict = torch.load( - hf_hub_download(repo_id="noanabeshima/tiny_model", filename=f"{model_fname}.pt"), - map_location=torch.device('cpu') - ) - return state_dict diff --git a/tiny_model/sparse_mlp.py b/tiny_model/sparse_mlp.py deleted file mode 100644 index cb3df1e..0000000 --- a/tiny_model/sparse_mlp.py +++ /dev/null @@ -1,38 +0,0 @@ -import torch -import torch.nn as nn -from huggingface_hub import hf_hub_download - - -class SparseMLP(nn.Module): - def __init__(self, d_model, n_features): - super().__init__() - self.d_model = d_model - self.n_features = n_features - - self.encoder = nn.Linear(d_model, n_features) - self.act = nn.ReLU() - self.decoder = nn.Linear(n_features, d_model) - - def get_acts(self, x, indices=None): - """Indices are either a slice, an int, or a list of ints""" - if indices is None: - return self.act(self.encoder(x)) - preacts = x @ self.encoder.weight.T[:, indices] + self.encoder.bias[indices] - return self.act(preacts) - - def __call__(self, x): - x = self.encoder(x) - x = self.act(x) - x = self.decoder(x) - return x - - @classmethod - def from_pretrained(self, state_dict_path: str, repo_id="noanabeshima/tiny_model"): - """Uses huggingface_hub to download an SAE/sparse MLP.""" - state_dict = torch.load( - hf_hub_download(repo_id=repo_id, filename=state_dict_path + ".pt") - ) - n_features, d_model = state_dict["encoder.weight"].shape - mlp = SparseMLP(d_model=d_model, n_features=n_features) - mlp.load_state_dict(state_dict) - return mlp diff --git a/tiny_model/__init__.py b/tinymodel/__init__.py similarity index 51% rename from tiny_model/__init__.py rename to tinymodel/__init__.py index 156cdda..3969591 100644 --- a/tiny_model/__init__.py +++ b/tinymodel/__init__.py @@ -1,3 +1,3 @@ -from .lm import TinyModel +from .lm import TinyModel, parse_mlp_tag from .sparse_mlp import SparseMLP -from .tokenization.tokenization import dec, enc, tok_see, tokenizer, raw_toks, toks \ No newline at end of file +from .tokenization.tokenization import dec, enc, tok_see, tokenizer, raw_toks, pretty_toks, multi_tok_words, words \ No newline at end of file diff --git a/tinymodel/lm.py b/tinymodel/lm.py new file mode 100644 index 0000000..bbf9d21 --- /dev/null +++ b/tinymodel/lm.py @@ -0,0 +1,411 @@ +import re +from textwrap import dedent + +import torch +import torch.distributions as dists +import torch.nn as nn +import torch.nn.functional as F +from huggingface_hub import hf_hub_download + +from .lm_modules import TransformerBlock +from .sparse_mlp import SparseMLP, get_sliced_mlp, get_masked_mlp +from .tokenization.tokenization import dec, enc + +DEFAULT_SPARSE_MLPS = { + "T0": "mlp_map_test/M0_S-3_R1_P0", + "T1": "mlp_map_test/M1_S-3_R1_P0", + "T2": "mlp_map_test/M2_S-3_R1_P0", + "T3": "mlp_map_test/M3_S-3_R1_P0", + + "M0": "mlp/M0_S-4_R1_P0", + "M1": "mlp/M1_S-4_R1_P0", + "M2": "mlp/M2_S-4_R1_P0", + "M3": "mlp/M3_S-4_R1_P0", + + "A0": "attn/A0_S-1_R1_P0", + "A1": "attn/A1_S-1_R1_P0", + "A2": "attn/A2_S-3_R1_P0", + "A3": "attn/A3_S-3_R1_P0", + + "Ra0": "res_pre_attn/Ra0_S-3_R1_P0", + "Ra1": "res_pre_attn/Ra1_S-3_R1_P0", + "Ra2": "res_pre_attn/Ra2_S-3_R1_P0", + "Ra3": "res_pre_attn/Ra3_S-3_R1_P0", + + "Rm0": "res_pre_mlp/Rm0_S-3_R1_P0", + "Rm1": "res_pre_mlp/Rm1_S-3_R1_P0", + "Rm2": "res_pre_mlp/Rm2_S-3_R1_P0", + "Rm3": "res_pre_mlp/Rm3_S-3_R1_P0", +} + + +def parse_mlp_tag(mlp_tag): + defaults_tag_pat = re.compile( + r"(?P(T|M|Rm|Ra|A|Mo))(?P\d+)(\D(?P\d+))?" + ) + defaults_match = defaults_tag_pat.fullmatch(mlp_tag) + + # file_tag_pat = re.compile(r'(?P(?P(T|Mo|M|A|Rm|Ra))(?P\d+)_S[-\d]+.{0,6}_P\d+)([^\d](?P\d+))?') + file_tag_pat = re.compile(r'(?P(?P.+/)?(?P(T|Mo|M|A|Rm|Ra))(?P\d+)(_N[\d]+)?(_S[-\d]+)(.{0,6}_P\d+)?([^\d](?P\d+))?)') + full_file_match = file_tag_pat.fullmatch(mlp_tag) + + if defaults_match: + match_groups = defaults_match.groupdict() + mlp_type, layer, feature_idx = ( + match_groups["mlp_type"], + int(match_groups["layer"]), + match_groups["feature_idx"] + ) + + + feature_idx = None if feature_idx is None else int(feature_idx) + + assert mlp_type+str(layer) in DEFAULT_SPARSE_MLPS + + return DEFAULT_SPARSE_MLPS[mlp_type+str(layer)], mlp_type, layer, feature_idx + elif full_file_match: + # try interpreting the mlp_tag as a filename + + mlp_type_to_file = { + 'M': 'mlp', + 'Mo': 'mlp', + 'A': 'attn_test', + 'T': 'mlp_map_test', + 'Ra': 'res_pre_attn', + 'Rm': 'res_pre_mlp' + } + + match_groups = full_file_match.groupdict() + + full_name, mlp_type, layer, feature_idx = match_groups['full_name'], match_groups['mlp_type'], int(match_groups['layer']), match_groups['feature_idx'] + + if 'base_dir' in match_groups: + file = full_name + else: + file = mlp_type_to_file[mlp_type] + '/' + full_name + + feature_idx = None if feature_idx is None else int(feature_idx) + + return file, mlp_type, layer, feature_idx + else: + return False + + +class TinyModel(nn.Module): + def __init__( + self, + d_model=768, + n_layers=4, + n_heads=16, + max_seq_len=256, + vocab_size=10_000, + from_pretrained="tiny_model", + ): + super().__init__() + self.embed = nn.Embedding(vocab_size, d_model) + self.embed.weight = nn.Parameter( + 1e-4 * torch.randn(self.embed.weight.shape, requires_grad=True) + ) + self.pos_embed = nn.Parameter( + 1e-4 * torch.randn(1, max_seq_len, d_model, requires_grad=True) + ) + + self.torso = nn.Sequential( + *[ + TransformerBlock( + d_model=d_model, n_heads=n_heads, max_seq_len=max_seq_len + ) + for _ in range(n_layers) + ] + ) + self.lm_head = nn.Linear(d_model, vocab_size) + + self.d_model = d_model + self.n_layers = n_layers + self.n_heads = n_heads + self.vocab_size = vocab_size + self.max_seq_len = max_seq_len + + if isinstance(from_pretrained, str): + self.load_state_dict(get_state_dict(from_pretrained)) + else: + assert ( + from_pretrained is False + ), "from_pretrained kwarg must be False or a string specifying model" + + # mlp tag list + self._sparse_tags = [] + + ''' + If using NNSight, you can't return NNsight wrapped modules via self.sparse_mlps + unless self.envoy is set. E.G. + + lm = TinyModel() + model = NNSight(lm) + lm.nnsight_proxy = model + ''' + self.nnsight_proxy = None + + + @property + def dtype(self): + return self.embed.weight.dtype + + @property + def device(self): + return self.embed.weight.device + + @property + def proxy(self): + return self if self.nnsight_proxy is None else self.nnsight_proxy + + @property + def sparse_mlps(self): + res = {} + for layer in range(4): + for mlp_type in ['A', 'T', 'M', 'Mo', 'Ra', 'Rm']: + mlp_tag = f"{mlp_type}{layer}" + if mlp_tag in self._sparse_tags: + if mlp_type == 'Ra': + res[mlp_tag] = self.proxy.torso[layer].res_pre_attn_sae + elif mlp_type == 'A': + res[mlp_tag] = self.proxy.torso[layer].attn_sae + elif mlp_type == 'Rm': + res[mlp_tag] = self.proxy.torso[layer].res_pre_mlp_sae + elif mlp_type == 'T': + res[mlp_tag] = self.proxy.torso[layer].transcoder + elif mlp_type in {'M', 'Mo'}: + res[mlp_tag] = self.proxy.torso[layer].mlp_sae + else: + raise ValueError(f'mlp_tag `{mlp_tag}` not found.') + return res + + + def get_upstream(self, downstream_tag): + assert downstream_tag in self.sparse_mlps, f'downstream_tag `{downstream_tag}` not found in self.sparse_mlps' + res = {} + for mlp_tag, mlp in self.sparse_mlps.items(): + if mlp_tag == downstream_tag: + break + else: + res[mlp_tag] = mlp + return res + + def get_downstream(self, upstream_tag): + assert upstream_tag in self.sparse_mlps, f'upstream_tag `{upstream_tag}` not found in self.sparse_mlps' + res = {} + mlp_tags = list(self.sparse_mlps.keys()) + downstream_tags = mlp_tags[mlp_tags.index(upstream_tag)+1:] + res = {ds_tag: self.sparse_mlps[ds_tag] for ds_tag in downstream_tags} + return res + + + def forward(self, tok_ids, return_idx=None, disable_flashattn=False): + T = tok_ids.shape[-1] + x = self.embed(tok_ids) + self.pos_embed[:, :T] + + for layer_idx, layer in enumerate(self.torso): + if layer_idx == return_idx: + return x + x = layer(x, disable_flashattn=disable_flashattn) + logits = self.lm_head(x) + return F.log_softmax(logits, dim=-1) + + def generate(self, prompt, n_toks=50, temperature=0.8, break_on_end=True): + assert temperature >= 0.0 + toks = enc(prompt, add_begin=True).to(self.lm_head.weight.device) + + for _ in range(n_toks): + with torch.no_grad(): + logprobs = self.forward(toks)[0, -1] + if temperature == 0: + next_tok = logprobs.argmax().item() + else: + next_tok = dists.Categorical( + logits=logprobs * (1 / temperature) + ).sample() + toks = torch.cat((toks, torch.tensor([[next_tok]]).to(toks.device)), dim=-1) + if break_on_end and next_tok == enc("[END]").item(): + break + if toks.shape[1] >= self.max_seq_len: + break + return dec(toks[:, 1:])[0] + + def get_sparse_act_fn(self, mlp_tag=None, mlp=None): + ''' + Returns `get_sparse_mlp_acts`, which takes in tok_ids and returns sparse mlp activations. It optionally allows `indices`. + ''' + assert not (mlp_tag is None and mlp is None) + + parse_output = parse_mlp_tag(mlp_tag) + + + if parse_output is False: + assert False, dedent( + 'Failed to parse mlp.' + ) + # assert False, dedent( + # """ + # [STUB] + # That\'s not a valid MLP tag. Here are some examples of MLP tags: + # M0, A2, Rm0, Ra1, Mo3 + # They start with a string in [M, A, Rm, Ra, Mo] + # representing mlp map, attn out SAE, residual pre-mlp SAE, residual pre-attn SAE, and MLP out SAE respectively. + # and they end with a number representing the layer. + + # You can also specify individual feature_idxs, e.g. lm['A2.100'](tok_ids) to get the activations of neuron 100. + # """ + # ) + else: + file, mlp_type, layer, feature_idx = parse_output + mlp_tag = mlp_type + str(layer) + if mlp is None: + sparse_mlp = SparseMLP.from_pretrained(file).to(device=self.device, dtype=self.dtype) + else: + sparse_mlp = mlp.to(device=self.device, dtype=self.dtype) + # else: + # assert False, dedent( + # """ + # mlp_tag {mlp_tag} not found in tiny_model.sparse_mlps or DEFAULT_SPARSE_MLPS + + # [STUB]: unimplemented + # To add a sparse_mlp, do e.g. + # tiny_model.set_saes({ + # \'M2\': SparseMLP.from_pretrained(\'mlp_map/M0_S-1_B0_P0\') + # }) + + # Available keys (of form {mlp_type}{layer}) are: + # M0..3 (for MLPs) + # A0..3 (for Attn out) + # Rm0..3 (for SAE on the residual before MLP) + # Ra0..3 (for SAE on the residual stream before attn) + # Mo0..3 (for SAE on MLP out) + + # See https://huggingface.co/noanabeshima/tiny_model/tree/main for available sparse MLPs. + # """ + # ) + + def get_sparse_mlp_acts(tok_ids, indices=feature_idx): + x = self.forward(tok_ids, return_idx=layer) + if mlp_type == "Ra": + return sparse_mlp.get_acts(x, indices=indices) + attn_out = self.torso[layer].attn(x) + if mlp_type == "A": + return sparse_mlp.get_acts(attn_out, indices=indices) + x = attn_out + x + if mlp_type in {"T", "Rm"}: + return sparse_mlp.get_acts(x, indices=indices) + else: + assert mlp_type in {"M", "Mo"}, "mlp_type must be one of Ra, A, M, Rm, T" + mlp_out = self.torso[layer].mlp(x) + return sparse_mlp.get_acts(mlp_out, indices=indices) + + return get_sparse_mlp_acts + + def __getitem__(self, index): + """ + To be used like: + sparse_acts = lm['A0'](tok_ids, indices=[1,5,100]) + sparse_acts = lm['M1'](tok_ids, indices=slice(0,100)) + sparse_acts = lm['M3'](tok_ids, indices=0) + + or for single neurons + + sparse_acts = lm['M2N100'](tok_ids) + sparse_acts = lm['M2.100'](tok_ids) + """ + if isinstance(index, int): + return self.torso[index] + else: + mlp_tag = index + return self.get_sparse_act_fn(mlp_tag) + + def register_sparse(self, mlp_tag=None, mlp=None, include_error=True, detach_error=True, detach_pred=False, feat_mask=None): + assert not (mlp_tag is None and mlp is None) + + # If mlp_tag is a list, interpret mlp_tag and mlp as being lists of mlps to add + if isinstance(mlp_tag, list): + assert all([isinstance(tag, str) for tag in mlp_tag]),\ + 'if mlp_tag is a list, must be a list of strings (mlp tags)' + assert mlp is None or len(mlp) == len(mlp_tag) + + if mlp is None: + mlp = [None for _ in range(len(mlp_tag))] + + # make arguments plural to make clear that they're lists + mlp_tags, provided_mlps = mlp_tag, mlp + + kwargs = dict(include_error=include_error, detach_error=detach_error, detach_pred=detach_pred) + for tag, provided_mlp in zip(mlp_tags, provided_mlps): + self.register_sparse(mlp_tag=tag, mlp=provided_mlp, **kwargs) + + return + + + parse_output = parse_mlp_tag(mlp_tag) + + if parse_output is False: + assert False, dedent( + 'Failed to parse mlp.' + ) + else: + file, mlp_type, layer, feature_idx = parse_output + assert feature_idx is None + mlp_tag = mlp_type + str(layer) + if mlp is None: + sparse_mlp = SparseMLP.from_pretrained( + file, + include_error=include_error, + detach_error=detach_error, + detach_pred=detach_pred + ) + else: + sparse_mlp = mlp + + if feat_mask is not None: + sparse_mlp = get_masked_mlp(sparse_mlp, mask=feat_mask) + + sparse_mlp = sparse_mlp.to(device=self.device, dtype=self.dtype) + + if mlp_type != "T": + sparse_mlp.register_full_backward_hook(lambda m, grad_in, grad_out: (grad_out,)) + else: + # [STUB] - warning that transcoders don't backward through transcoder errors. + pass + + transformer_block = self.proxy.torso[layer] + if mlp_type == 'Ra': + transformer_block.res_pre_attn_sae = sparse_mlp + elif mlp_type == "A": + transformer_block.attn_sae = sparse_mlp + elif mlp_type == 'Rm': + transformer_block.res_pre_mlp_sae = sparse_mlp + elif mlp_type == "T": + transformer_block.transcoder = sparse_mlp + elif mlp_type == 'M': + transformer_block.mlp_sae = sparse_mlp + + + else: + raise ValueError(f'mlp_type {mlp_type} is unsupported.') + + + self._sparse_tags.append(mlp_tag) + + def wipe_sparse(self): + for block in self.proxy.torso: + block.attn_sae = None + block.transcoder = None + block.mlp_sae = None + self._sparse_tags = [] + + + + +def get_state_dict(model_fname="tiny_model"): + state_dict = torch.load( + hf_hub_download(repo_id="noanabeshima/tiny_model", filename=f"{model_fname}.pt"), + map_location=torch.device('cpu'), + weights_only=True + ) + return state_dict diff --git a/tiny_model/lm_modules.py b/tinymodel/lm_modules.py similarity index 74% rename from tiny_model/lm_modules.py rename to tinymodel/lm_modules.py index b989d92..0a59688 100644 --- a/tiny_model/lm_modules.py +++ b/tinymodel/lm_modules.py @@ -10,7 +10,7 @@ def __init__(self, name=None): super().__init__() self.name = name - def forward(self, x): + def forward(self, x, *args, **kwargs): return x def __repr__(self): @@ -67,7 +67,7 @@ def Wv(self): def Wo(self): return self.O.weight.detach() - def forward(self, x): + def forward(self, x, disable_flashattn=False): x = self.attn_inp(x) # hookpoint q, k, v = self.Q(x), self.K(x), self.V(x) @@ -80,17 +80,23 @@ def forward(self, x): vs = einops.rearrange(v, "b s (h d) -> b h s d", h=self.n_heads) vs = self.vs(vs) # hookpoint - + # force torch to use flash attention 2 - if x.dtype == torch.float16 or x.dtype == torch.bfloat16: + if (x.dtype == torch.float16 or x.dtype == torch.bfloat16) and not disable_flashattn: with torch.backends.cuda.sdp_kernel( enable_flash=True, enable_math=False, enable_mem_efficient=False ): head_writeouts = F.scaled_dot_product_attention( qs, ks, vs, is_causal=True ) + elif disable_flashattn: + with torch.backends.cuda.sdp_kernel( + enable_flash=False, enable_math=True, enable_mem_efficient=True + ): + head_writeouts = F.scaled_dot_product_attention(qs, ks, vs, is_causal=True) else: head_writeouts = F.scaled_dot_product_attention(qs, ks, vs, is_causal=True) + head_writeouts = self.head_writeouts(head_writeouts) # hookpoint catted_head_writeouts = einops.rearrange(head_writeouts, "b h q d -> b q (h d)") @@ -129,6 +135,12 @@ def forward(self, x): return mlp_out +# res_attn is just an autoencoder with identity x baseline +# res_mlp is just.. an autoencoder with identity x baseline +# mlp is.. a sparse mlp with identity y baseline +# attn is a sparse mlp with identity y baseline + + class TransformerBlock(nn.Module): def __init__(self, d_model, n_heads, max_seq_len): super().__init__() @@ -149,17 +161,39 @@ def __init__(self, d_model, n_heads, max_seq_len): self.res_mlp = HookPoint() self.res_final = HookPoint() - def forward(self, x): + self.res_pre_attn_sae = None + self.attn_sae = None + self.res_pre_mlp_sae = None + self.transcoder = None + self.mlp_sae = None + + + def forward(self, x, disable_flashattn=False): x = self.res_attn(x) # hookpoint - attn_x = self.attn(x) - x = attn_x + x + if self.res_pre_attn_sae is not None: + x = self.res_pre_attn_sae(x) + + attn_out = self.attn(x, disable_flashattn=disable_flashattn) + if self.attn_sae is not None: + attn_out = self.attn_sae(x=attn_out, target=attn_out) + + x = attn_out + x x = self.res_mlp(x) # hookpoint - mlp_x = self.mlp(x) - x = mlp_x + x + if self.res_pre_mlp_sae is not None: + x = self.res_pre_mlp_sae(x) - x = self.res_final(x) # hookpoint + mlp_out = self.mlp(x) + + if self.transcoder is not None: + mlp_out = self.transcoder(x=x, target=mlp_out) + if self.mlp_sae is not None: + mlp_out = self.mlp_sae(mlp_out, target=mlp_out) + + x = mlp_out + x + + x = self.res_final(x) # hookpoint return x diff --git a/tinymodel/sparse_mlp.py b/tinymodel/sparse_mlp.py new file mode 100644 index 0000000..627b1b0 --- /dev/null +++ b/tinymodel/sparse_mlp.py @@ -0,0 +1,98 @@ +import torch +import torch.nn as nn +from huggingface_hub import hf_hub_download + + +class SparseModelError(nn.Module): + def __init__(self): + super().__init__() + def forward(self, pred, target): + return (target - pred.detach()) + + +class SparseMLP(nn.Module): + def __init__(self, d_model, n_features, include_error=False, detach_error=False, detach_pred=False): + super().__init__() + self.d_model = d_model + self.n_features = n_features + + self.encoder = nn.Linear(d_model, n_features) + self.act = nn.ReLU() + self.decoder = nn.Linear(n_features, d_model) + + self.include_error = include_error + self.detach_error = detach_error + self.detach_pred = detach_pred + + if self.include_error: + self.get_eps = SparseModelError() + + def get_acts(self, x, indices=None): + """Indices are either a slice, an int, or a list of ints""" + if indices is None: + return self.act(self.encoder(x)) + preacts = x @ self.encoder.weight.T[:, indices] + self.encoder.bias[indices] + return self.act(preacts) + + def __call__(self, x, target=None): + preacts = self.encoder(x) + acts = self.act(preacts) + pred = self.decoder(acts) + + if self.detach_pred: + pred = pred.detach() + + if self.include_error: + error = self.get_eps(pred, target) + + if self.detach_error: + error = error.detach() + + return pred + error + else: + return pred + + @classmethod + def from_pretrained(self, state_dict_path: str, repo_id="noanabeshima/tiny_model", include_error=True, detach_error=True, detach_pred=False, **kwargs): + """Uses huggingface_hub to download an SAE/sparse MLP.""" + state_dict = torch.load( + hf_hub_download(repo_id=repo_id, filename=state_dict_path + ".pt"), + weights_only=True + ) + n_features, d_model = state_dict["encoder.weight"].shape + + if 'n_features' in kwargs: + assert kwargs['n_features'] == n_features + if 'd_model' in kwargs: + assert kwargs['d_model'] == d_model + + mlp = SparseMLP(d_model=d_model, n_features=n_features, include_error=include_error, detach_error=detach_error, detach_pred=detach_pred) + mlp.load_state_dict(state_dict) + return mlp + +def get_sliced_mlp(mlp: SparseMLP, start: int, end: int): + assert isinstance(start, int) and isinstance(end, int), (start, end) + assert end > start + assert end <= mlp.n_features and start >= 0 + new_mlp = SparseMLP(d_model=mlp.d_model, n_features=end-start, include_error=mlp.include_error, detach_error=mlp.detach_error, detach_pred=mlp.detach_pred) + new_mlp.encoder.weight.data = mlp.encoder.weight.data[start:end] + new_mlp.encoder.bias.data = mlp.encoder.bias.data[start:end] + new_mlp.decoder.weight.data = mlp.decoder.weight.data[:,start:end] + new_mlp.decoder.bias.data = mlp.decoder.bias.data + + return new_mlp + +from typing import Iterable + +def get_masked_mlp(mlp: SparseMLP, mask: list[int]): + assert len(mask) > 0 + assert isinstance(mask, Iterable) + assert all([isinstance(it, int) for it in mask]) + + new_mlp = SparseMLP(d_model=mlp.d_model, n_features=len(mask), include_error=mlp.include_error, detach_error=mlp.detach_error, detach_pred=mlp.detach_pred) + new_mlp.encoder.weight.data = mlp.encoder.weight.data[mask] + new_mlp.encoder.bias.data = mlp.encoder.bias.data[mask] + new_mlp.decoder.weight.data = mlp.decoder.weight.data[:,mask] + new_mlp.decoder.bias.data = mlp.decoder.bias.data + + return new_mlp diff --git a/tiny_model/tokenization/neo_tok_ids_to_ts.pt b/tinymodel/tokenization/neo_tok_ids_to_ts.pt similarity index 100% rename from tiny_model/tokenization/neo_tok_ids_to_ts.pt rename to tinymodel/tokenization/neo_tok_ids_to_ts.pt diff --git a/tiny_model/tokenization/tokenization.py b/tinymodel/tokenization/tokenization.py similarity index 95% rename from tiny_model/tokenization/tokenization.py rename to tinymodel/tokenization/tokenization.py index 751264b..d6792fd 100644 --- a/tiny_model/tokenization/tokenization.py +++ b/tinymodel/tokenization/tokenization.py @@ -25,8 +25,8 @@ "unk_token": "[UNK]", }, ) -neo_tok_ids_to_ts = torch.load(f"{current_dir}/neo_tok_ids_to_ts.pt") -ts_tok_ids_to_neo = torch.load(f"{current_dir}/ts_tok_ids_to_neo.pt") +neo_tok_ids_to_ts = torch.load(f"{current_dir}/neo_tok_ids_to_ts.pt", weights_only=True) +ts_tok_ids_to_neo = torch.load(f"{current_dir}/ts_tok_ids_to_neo.pt", weights_only=True) def clean_text(text): @@ -188,7 +188,9 @@ def __call__( tokenizer = Tokenizer() - - raw_toks = np.array([dec(tok_id) for tok_id in range(10_000)]) -toks = np.array([tok.replace("\n", "↵").replace(" ", "⋅") for tok in raw_toks]) \ No newline at end of file +pretty_toks = np.array([tok.replace("\n", "↵").replace(" ", "⋅") for tok in raw_toks]) +import json +with open(f"{current_dir}/words.json", "r") as f: + multi_tok_words = np.array(json.load(f)) +words = np.concatenate((raw_toks, multi_tok_words), axis=0) \ No newline at end of file diff --git a/tiny_model/tokenization/ts_tok_ids_to_neo.pt b/tinymodel/tokenization/ts_tok_ids_to_neo.pt similarity index 100% rename from tiny_model/tokenization/ts_tok_ids_to_neo.pt rename to tinymodel/tokenization/ts_tok_ids_to_neo.pt diff --git a/tinymodel/tokenization/words.json b/tinymodel/tokenization/words.json new file mode 100644 index 0000000..7400516 --- /dev/null +++ b/tinymodel/tokenization/words.json @@ -0,0 +1 @@ +[" didn't", "Lily", " don't", " yummy", " Fluffy", " couldn't", " mommy", "Sue", "Don't", " can't", " Bobo", "Mia", " Timmy", "Lucy", "I'm", " swam", " I'm", " clapped", " barked", "Mommy", " Mommy", " Mimi", " puddle", " wagged", " Whiskers", "Kitty", " kite", " teddy", " Doggy", " Lila", " Momo", "Sally", " smelly", " Ollie", " grumpy", " beak", " Chirpy", " bossy", " crayons", " wasn't", " snowman", " splashed", " wouldn't", " Grandma", " won't", " dizzy", " nosy", " Ducky", " Ella", " jeep", " vase", " jolly", " chubby", " sunflower", "Mum", " clap", "Bobo", "Sara", " flute", " swan", " caterpillar", " dependable", " I'll", " Don't", " cupboard", " eraser", " jellyfish", " parrot", " envious", " decorate", " mop", " cauliflower", " melon", " armchair", " leopard", " crocodile", " barber", " muffin", " cactus", "Molly", " alligator", " twig", "Fluffy", " flea", " pebble", " Poppy", " icecream", "Mama", " Lulu", "Timmy", " scooter", " radish", " crayon", " hippo", " aeroplane", " bathtub", " Bessie", " tripped", " penguin", " bookcase", " celery", " otter", " acorn", "Tommy", "Ouch", " kayak", " meadow", " sparkly", " oyster", " cobweb", " mule", " waffle", " bathe", " tummy", " zebra", " bathrobe", " dishwasher", " lollipop", " sneeze", " octopus", " blackboard", " lotion", " unpack", " napkin", " bookshelf", " Birdy", " panda", " Susie", " thermometer", " chimney", " Binky", " kangaroo", " bandage", " Buzzy", " Pippin", " faucet", " mitten", "Daisy", " fireman", " Dolly", " Kiki", " birdcage", " peeked", " oasis", "Mummy", " ashtray", " apron", " rhinoceros", "Buddy", " squirrels", " rained", " Bubbles", " cherries", " igloo", " hanger", " Speedy", " blouse", " Pandy", " reindeer", " Zoomy", " Mandy", "Yuck", " excitedly", " blueberries", " firework", " spade", " giraffe", "You're", " giggled", " ostrich", " berry", "Mimi", " shiver", " grandpa", " sneezed", " Hoppy", " Elly", " you're", " shivering", " pastel", " zoomed", " tremble", " meowed", " puddles", "Emma", " xray", " Mittens", " sweetie", "Remy", " Fifi", " sparkled", " Grandpa", " prune", "Jill", " clapping", " Finny", "Woof", " yawned", "Lila", " splashing", " sniffed", " twisty", " jogged", " Gigi", " flapped", " Bluey", "Bella", " blueberry", " Mummy", " Squeaky", "Ow", " bouncy", "Grandma", " lemons", " Tuna", "Bobby", " Lizzy", " Pinky", " shopkeeper", " Lola", " licked", " dressup", " shined", " zigzag", " vroom", " beep", " marbles", " pinched", " sweetheart", " prunes", "Momo", " jogging", " safari", " Zara", " whistled", " zipped", " Tilly", "Meow", " wagging", " waffles", " Chicky", " kneeled", " Toto", " threeyearold", "Yum", " spoons", " pears", " barks", " twirled", " kitty", " Froggy", " mama", " chewed", " acorns", " olives", " mailman", " growled", " doesn't", " Snowy", " muffins", " yucky", " napping", " Millie", " Suzy", " daisy", " excite", " sandcastle", "Jenny", " Rosie", " wiggle", " twigs", "Mandy", " Milly", "Ollie", " hadn't", "Whiskers", "Doggy", " petals", " Stitch", " knelt", " Beep", " giggle", " Zigzag", "Ella", " Pippa", " peaches", " Skelly", "Yay", " peered", " slimy", " grownups", " seesaw", " Ava", " pillows", " oat", "Yummy", "I'll", " Gogo", " isn't", " mittens", " shouldn't", " you'll", "Ducky", "Leo", " bravely", " giraffes", " whistling", " bandaid", "Tiny", " snowballs", " zooming", "We're", " Wiggly", " tickle", " naps", " kneel", " scooped", "Luna", " ladybug", "Roar", " polishing", " ribbons", " Bouncy", "Benny", " lily", " pebbles", " wiggled", " 3yearold", " Birdie", " grownup", "Sammy", " doggy", "Achoo", " Robby", " wag", " Pup", " pointy", " admiring", " radishes", " he'd", "Susie", " meow", " pastels", " faraway", " comfy", " Paws", " twirl", "Beep", " scrubbed", " lemonade", " prettiest", "Bunny", "Bye", " sparkle", "Chirpy", " glowed", "Grandpa", "Poppy", " chirped", "Surprise", " hideandseek", " You're", " Catty", " sparkles", " smiley", " Ziggy", " amazement", " bunnies", " Abigail", " limes", " Jilly", " Twinkle", " funnylooking", " ornaments", " weren't", " pencils", " Hoot", " Fido", " Buggy", " playroom", " wags", " chirping", " hissed", " honk", " Brownie", " playtime", " Rolly", "Vroom", " tugged", " pretends", " penguins", " Whistle", " pumpkins", " Olly", " Bop", " I've", " shivered", " snowmen", " Foxy", " scarves", "Tina", " woof", " twinkling", " mane", " Dodo", " Cuddles", " Dizzy", " sausages", " toaster", " siren", "Zoom", " necklaces", "Excuse", "Toby", " Hippy", "Pete", " scaring", " sneaked", " sprinkles", " pastries", " crunchy", " wobbled", "Grace", "Shh", " she'd", " bloomed", " snuggled", " windy", " growl", " zipping", " beamed", " twirling", "Annie", " flapping", " splashes", " We'll", "Polly", "Guess", " piggy", " Biggie", " booboo", " Rollie", "Ew", " swooped", " Moo", " paddled", " Mixy", " dived", " cobwebs", " we'll", "Milly", " sailboat", "Zoomy", " comforted", " avocados", " Lizzie", "Sunny", " bumpy", "Lilly", "Lola", " untie", " fairies", " growling", " kites", " tickled", " hider", "Maggie", " raincoat", " buzzed", " itchy", " Zippy", " Beaky", " patted", "Kiki", " Woof", " scolded", " raindrops", " peeking", " pigeons", " Titi", "Dolly", " bracelets", " Abbie", "Lulu", "Betty", " Kayla", " giggling", " chases", " Cara", " Missy", " geese", " cuddly", " quack", " lollipops", " tiptoed", " bedtime", " stomped", "Ribbit", " blooming", " frosting", " sneezing", " zebras", "You'll", " candies", "Penny", " grassy", " squealed", " Barky", " quarreling", " sunflowers", "Joey", " zookeeper", "Sophie", " mints", " licks", " Isn't", " playhouse", "Sandy", " grandchild", "Mmm", " Dara", "Oops", " Nana", "Spike", " sewed", " pedaled", " Quack", " unpacked", " wiggly", " starfish", "Hurry", "We'll", " Wooly", " Blinky", " seagull", " scurried", " Henny", " untied", "Wake", " Matilda", "Bessie", " nectar", "Speedy", " rainbows", " You'll", " crumbs", " claps", "Whee", " rustling", " Tweetie", " Snuggles", " Biggy", "Millie", " pennies", " clowns", " quacked", " melons", " cuddle", " napkins", "Silly", " Peanut", " rumble", "Ellie", " wobble", "Uhoh", " firefly", "Buzzy", " prettier", " trampoline", " twinkle", " Fuzzy", " princesses", " Mikey", "Teddy", " skunk", " fluttered", "Pippin", "Moo", "I've", " paintbrush", " firemen", "Elly", "Binky", " Sleepy", " Willy", " Mira", "Coco", " shovels", " Ugly", " swans", " budge", "Freddy", " Toot", " purred", "Olive", " genie", " Joanna", " wobbly", "Quack", " Inky", " daisies", " burrow", " plucked", " pajamas", " decorating", " farthest", " Abi", " Janie", " Jojo", " Softy", " Suzie", "Jazz", " surfboard", " threeyearsold", " trembled", "Chloe", " snuck", " Puppy", "Ahoy", " rubs", " overjoyed", " Sparky", " Mina", " growls", " cushions", "Clara", " Bingo", "Owl", " I'd", " rowed", " scooters", " cosy", " Bibi", "Momma", " beehive", " forgave", " Pinchy", " goodnight", " raccoon", "Finn", " treehouse", " envelopes", " Zee", "Arrr", " peeped", " screwdriver", " squishy", " we're", " dollhouse", " Chew", "Uncle", "Goodbye", "Lucky", " gracefully", "Brrr", " drawers", "Lizzy", " fluttering", " waddled", " pouted", "Finny", " Rufus", " Settle", " pandas", " swimsuit", " Swimmy", "Pandy", " Chirp", " pecked", " mischievous", " petted", " hedgehog", " forts", " campfire", "Bluey", "Pinky", " Fishy", " Kip", "Katie", " doorbell", "Zara", " snowflakes", " Dina", " glittering", " Nutty", " quarreled", " cashier", " mesmerized", " Elsie", " storekeeper", " aeroplanes", " Mousy", " Elle", " Chewy", " Piggy", " Maddie", " Frosty", "Suzy", " Cally", " Davey", " creams", " Waffle", " Cammy", " hangers", " purr", " sweetest", " Ratty", "zoom", "Hoppy", " Vicky", " Tweety", "Fifi", " Zoey", " sniffing", "Honey", "Birdy", " swims", "vroom", " grasshopper", " librarian", "Shrink", " pinching", "Whose", "Gina", " streamers", " Jonny", "Beth", " Granny", " winked", " haircuts", "Abby", " zigzags", " Microscope", " Puffy", " We're", " chirp", " Startled", " beeped", " frowning", " aren't", "Aunt", " spooky", "meow", " tricycle", " honked", "Bounce", " postman", " Snowman", " crackers", " meowing", " Bubu", " umbrellas", " squinted", " stung", " crowns", " aprons", " magnifying", " Greeny", " smelt", "Splash", " Liza", "Abigail", " matey", " Allie", " bluebird", " oysters", " Vroom", " Mabel", " scraped", " flutter", " Minnie", " sniffs", " cloths", " meadows", "Tilly", " stretchy", " sandcastles", " ticktock", " Gertie", " Silly", " seaweed", " Vivi", "Hannah", " massages", " Flap", " Winky", " Dilly", " swimmer", " Alarm", " spoonful", "Cindy", " Zipper", " rumbling", " shinier", " Tia", " Bumpy", " Jacky", " Momma", " merrygoround", " Jumpy", "Boys", " Flutter", " Jody", " suns", "Grow", "Choo", " It'll", " Muffin", "Pippa", " Sadie", " glee", " Gilly", "Gigi", " peck", " beaks", " Yummy", " gardener", " scurrying", " Yawn", " dreamt", " squeak", " cubs", " mustn't", " wept", " they'd", "Boom", " Excited", " scampered", " riverbank", " tumbled", " Grumpy", "Rabbit", "Frog", " thorns", " Quacky", " Sofia", " Cozy", " combed", "Nina", " Cici", " bravest", " tidying", "Pip", " hiss", " mister", " trotted", " messes", " picnics", " zips", " cheetah", "Startled", " squeaky", " squished", " Flicker", " weirdlooking", " Marcy", " pours", " Xray", " pedaling", " Maddy", "Mittens", " vroomvroom", " shrank", " yawning", " twinkled", " wiggling", " waded", " Coby", " loudest", " raking", "Jessie", " Horsy", "Catch", "Surrender", "freeze", "Squeaky", " strolled", "Robby", " Quickly", " glimmering", " Lina", " moat", " soapy", " giggles", " scarecrow", " Tippy", " Skeleton", "Ava", " shimmering", " stethoscope", " chocolates", " dresser", " Roly", " Freya", "Chicky", " cuddled", " fleas", " Mixie", "Zigzag", " tidied", " yelped", " Joke", " playdate", " Gemma", " Wiggles", " napped", " zooms", " stroked", "Slowly", " Ria", " unpacking", " ache", " Moppy", " roars", " Graceful", " Jetty", " shyly", " eyeshadow", " otters", " Eliza", "Isn't", " JoJo", " Crabby", " bubbling", " Ronny", " gobbled", " mattresses", " toolbox", "Dino", " papa", "Curious", "poison", "Bubbles", "Climb", " Roo", " Ami", " erasers", " Janey", " stomping", " Slinky", " Reliable", " glittered", "Zoe", " cowboys", "Wally", " Blacky", " inchworm", " bushy", "Gogo", " seagulls", " Scissors", " Hops", "Rosie", " Floppy", " cupboards", " haven't", " scoops", " pail", " foggy", " miller", " hooves", " sweaters", "Julie", " Cocoa", " XRay", " thunderstorm", " intently", "Lizzie", " backpacks", " squeaked", " munch", " foxes", "Sophia", "Tara", "I'd", "Papa", "Cara", " choo", " grandkids", "Ahchoo", " momma", " Chomp", " paddling", " Flopsy", " grumbled", "Snowy", " swimsuits", " glided", "Maya", " Mindy", " windowsill", "Tuna", " Icy", " knobs", " beaver", " pepperoni", " Zeb", " sneezes", " tickling", "Afterwards", " poppies", " bandages", " quacking", " Bubba", " banged", " mugs", " laces", " nestled", "Eww", " bumping", " Woolly", "Velvet", " yummiest", " Peep", " hummed", " Bebe", " marshmallows", " Twisty", " Hanger", " Tidy", "Everyday", " kangaroos", " Spidey", " puffed", " braver", " teapot", " wriggled", " pancake", " dragonfly", " springing", "Blink", " Puff", "Whoa", " sweetly", " Auntie", " Flexible", "Liz", " Shelly", "guess", " wrinkled", "Freeze", " Twinkie", " unwrapped", " Horsie", " marveled", " sneaks", " warmly", " ostriches", " Lenny", " roamed", " duckling", "Reverse", "Foxy", "beep", " Josie", " cheerfully", " Jasmine", " Jemma", " hairdresser", " Ringo", " Hurry", " Cate", " forgets", " obeyed", " vests", " cranes", " Cari", "Maddie", "Excitedly", " obediently", " squeezes", " mermaid", " lunchtime", " Flora", " Faye", " aisles", " yum", "Froggy", "Dare", " swaying", "Claire", " Nappy", "Ally", " scowled", "Surprised", " sweeties", " squawked", "Twinkle", "Bop", " Nelly", " prickly", " scrubbing", " Spicy", " toppings", " Mona", " sternly", " groaned", "Bless", " Achoo", " bendy", " Flick", " inched", " mommies", "Liza", "Fido", " Riri", " binoculars", " Gracie", " dirtier", " sparrow", " toothbrush", " pout", " mustered", " Broccoli", " Whee", "Squirrel", "Stir", " Carefully", "Pepper", "welcome", " skips", "Jilly", " piglet", " nudged", " twirls", " funniest", " tickles", " you'd", " quizzes", " snowflake", " Crusty", " Dusty", " munching", " saxophone", " Printy", " birdhouse", " whistles", " bubbly", " minding", "Spray", " sleepover", " Flute", "They're", " Whistler", " Layla", " Chippy", " thud", "Noah", " Tuesdays", "Baa", " fetched", " pizzas", " croaked", " clothesline", " glittery", " they're", " squatted", " unzip", " Hilda", "Karen", " seriouslooking", " Cacky", " bellies", " beanstalk", " racers", " nibble", "Jackie", " sirens", " watermelon", " Snip", " checkup", " honking", " Squeak", " gulped", " Penelope", " swirls", " untangle", "Maddy", " Zog", " Abbi", " raked", " braid", " They're", " Callie", " Nellie", " shooed", " crickets", " tugging", "Birdie", " Cluck", "Mmmm", " petal", "Ooh", " Excitedly", " earrings", "Mira", " tugofwar", "Catty", " whoosh", " tiring", " Owly", " yearold", "Rocky", "Rolly", " kayaking", "Whistle", " Dotty", " Roxy", " Crawl", " Teaspoon", "Promise", " spoiling", " gorillas", " slithering", " hammers", " chipmunk", "Chirp", " spied", "Kayla", " Inchy", " squirmed", " Chairy", "Hippy", " dripped", " crouched", " Jimbo", "Milo", "Bonnie", " munched", " Gabby", " ladybird", " gruff", " softest", "Dizzy", " Blueberry", " parrots", "Amelia", " marshmallow", " cutest", " woodcutter", " mesmerised", " petting", " Maisy", " antlers", " Crocodile", " Baa", " ribbit", " moonlight", "Mina", " Clarissa", "It'll", "Toto", " hare", " stitched", " Muffy", " sprays", " Bossy", "Olly", " Breezy", " ladybugs", "Rollie", " dozed", " snoring", " thump", "Nearby", " timidly", " mooed", " icecreams", " keyhole", " He'd", " hopscotch", " snip", " daddies", " Anty", " trudged", "Ami", " rags", " crocodiles", "Davey", " practised", "Danger", "Patty", " Carla", " juggle", "Amber", " beaming", "Ralph", " Spooky", " Clap", " Patches", " ripples", "Skelly", " wheelbarrow", " Bobble", " whiskers", " stormy", "Charlotte", " gliding", "Excited", " Anya", " wobbling", " She'd", " surfed", " Pebby", "Paws", "Feeling", " unload", " Spinny", " wideeyed", " Forky", " lovingly", " Rina", "Bouncy", "Wiggly", " flaps", " softness", "Yoga", " whined", " messier", " hardworking", " joyfully", " crazylooking", " o'clock", "Linda", " tiptoes", "Stella", " squirted", " moo", "Toys", "Mara", " Rollo", " dolly", "Fiona", " Nell", "Bree", " Isabella", " fireflies", " fixer", " razors", " ducklings", " snowing", " Piney", " springtime", "Goodnight", "Careful", "Joanna", "Lena", " gooey", " wavy", " journeyed", " tastiest", " reappeared", "Pup", " riddle", " Bess", " hopster", " Zoomie", " shoo", "Frustrated", "Abbie", " carefree", " huddled", "Oliver", " Trudy", " lolly", "okay", " Joanne", " meows", " comets", " frisbee", " plopped", " quacks", " Wanda", " crawls", " stirs", " breadcrumbs", " halo", " leaped", " rips", "Wee", " tripping", "Copper", " Salty", " Gail", " doves", "Julia", "Princess", " sawed", "Shoo", "Poison", " ballerina", " Fluff", " birthdays", " Jax", " Sari", " Mop", " untwist", " sharpening", "Mae", " Can't", " Johny", " cupcakes", " kayaks", " shrimps", " Gwen", " capes", " pecking", " blushed", " Furry", " juiciest", " bathed", "Peanut", " woken", " grumble", " tightrope", "Knock", " Spotty", " Steak", " washes", "Nana", " glistening", " Melinda", " Kri", " pantry", " winks", "Shall", "boo", " Zizi", " Kiko", " Annabel", "Matilda", " Toe", " Instantly", "Jonny", "Mister", "Manny", " BunBun", " stroller", " Lyla", " rattling", " pew", "Mumma", " braids", " shoreline", " Mustache", " robin", " brightened", " raindrop", " rustled", " Kanga", " Lazy", " zippers", " Jellyfish", "Mabel", "Sadie", " walker", " boomed", " pounced", " darted", " rumbled", "Zippy", "Wisdom", " Chickie", " carpets", "Swing", "Zoey", " meaner", " snuggle", " Dora", " Deaf", " flutes", " Infant", " Tink", " threeyear", " planks", "Dara", " Boxy", "Soup", " Cleo", "Butterfly", " hissing", " rotted", " Dory", " Lotion", "Nora", " chatted", " Socky", " chews", " sleigh", "Pam", " Mamma", "Martha", " cork", " Ouch", " scuttled", " Grandad", " rainwater", "Lina", " Flappy", " Slowpoke", " perch", " perked", "Sharing", " playfully", " pea", " wildflowers", " mops", " Fireworks", " accidently", "moo", "Gus", "Robbie", " Wipey", " unwrap", " misty", "Betsy", " owls", "Missy", " sawing", " sprout", "boom", " Jodie", " Frustrated", " Jennie", " 3year", "Fuzzy", " gulp", " peels", " crackling", " huffed", " Nemo", " Sami", "Bubba", " Alana", "Becky", " parades", " hesitantly", " twitched", " Kitten", "Drink", "Shrimp", " Sock", " nicest", "Judy", " alligators", " Dolls", " rowing", " Fruity", " glows", " spiky", " Gia", "Izzy", "Nellie", " gummy", " you've", " peeks", " Gramps", " birdie", " dryer", " Oasis", " dears", " spits", " cupcake", " it'll", " peacefulness", " ouch", " tummies", " commotion", " townspeople", " Ravi", " perse", " scooping", " steaming", " raincoats", " hiked", " insides", "Beef", " rustle", " ketchup", " baa", " spiciest", " Intrigued", "Holly", " Bea", " adored", " Maisie", " pegs", " chalks", " strolling", " yanked", " Cleaning", " tortoise", "Mindy", "goodbye", " Trina", " Annabelle", " saidI", " Matty", "Janet", " clucked", " frowns", " Flea", " Tiki", " flippers", " Ema", "ayla", " Nibbles", " critter", "Johny", " lizards", " Whistling", " doorknob", " handlebars", " seatbelt", " bouquet", "Beaky", " Suki", " badger", " mopping", " porridge", " cucumbers", "Janie", " Johnnie", " friendlylooking", " trusty", " warmest", " Yeti", " angrier", " inquisitive", " crumb", " hoof", " hovered", " creak", " bowing", " cuddling", "Halfway", " grandparent", "Torn", " auntie", " slurped", " peacock", "Mixy", " tshirt", " tumbling", "Zee", " purses", "Spark", " Oink", "Minnie", " sleds", " panting", "Rosa", " MooMoo", " Shoe", " spades", " Becca", " kneels", " scrunched", " Pops", " nibbled", " furrowed", " combing", "Sofia", " disagreeing", " carton", " peering", " grumbling", "Brownie", " summertime", "Granny", "Jojo", " thoughtfully", " chugging", " buckled", " Choo", "Graceful", " whimpered", " groan", " noone", "don't", " Delia", " Maia", " seahorse", "Samantha", " Ballie", " Elmo", "Dina", "Ruth", " Moony", " cacti", "choo", "Shoot", "Tessa", " sweeter", "Biggie", " marveling", " rattle", "Bingo", " deliciouslooking", "Dance", " learner", " wriggling", " housework", " squirt", "Buggy", " Bibo", " Pillow", " bales", " sniffled", "Carla", " sipping", " Prune", " Fizzy", " pokes", " Bethany", " coop", " sharpened", "Lion", "Cammy", " Allowed", " passerby", "Rebecca", " Pumpy", "Hmmm", " oars", " Mumma", "Auntie", " hisses", "Eddie", " Cowy", "Elle", "Robot", " hee", " Moly", " Apples", " hoses", " practising", "Muffy", " Ribbit", " Bobbi", "Ruff", " Whistly", " Ernie", " Starry", " Snail", " Bathing", " imagines", " Jellie", " sipped", "Ding", " Tic", "Barney", " Jakey", " Flop", " coughed", "Bert", " Cactus", " applauding", " Crabbie", " weaved", "ticktock", "Daughter", " Glue", " droplets", "Spicy", " bobbing", " mopped", " bumble", " mailboxes", "Frightened", " fanciest", " buckles", " Dependable", " Bumble", " Grilly", " squawking", " patting", " sunbeam", " squish", "Caw", " captivated", " screeched", " Simba", " Amie", " glistened", " Squeezy", " clutched", " Alfie", "yield", " Gently", " Cacty", " Dishy", " cooed", " Slippy", " bobbed", "Duck", " Dinah", "Nancy", "university", " Tena", " looped", " tuck", "Rufus", " bristles", " strangelooking", " Flossy", "choochoo", " Peek", " calmer", " Isla", " Sharky", " Vanny", " wheeled", " tantrum", "Moral", " sunsets", " collars", " kindhearted", " beepbeep", " jiggled", "Barry", "Arr", "Ziggy", " panicking", "Jemma", " scratchy", " Carrot", " Raya", " Rude", " Climb", "Chocolate", "Carry", " sadder", " windmill", "Jenna", " triceratops", " Whispy", " Trish", " scuttling", " snails", " climber", " slithered", " Thumb", " canes", " scowling", " Denny", "Moe", "Birds", " whizzing", " Libby", " popsicles", " roundabout", " unscrewed", " Ashtray", " sta", " bandaids", "Shake", "Settle", " Carty", " Sticky", "Teaspoon", " Roby", " jeeps", "Lucas", "Whale", " hooted", " zap", " goodbyes", "Toot", "Pine", "Suzie", " cocoon", " heartshaped", " motioned", " Iggy", " faucets", "Confused", " Addy", " blinks", " Brainy", " Grump", " blackbird", " Croco", "colina", " beeping", " suitcases", "Woah", " whimper", " puffy", " clung", "Lets", " ticklish", " howl", " Fireman", " Needle", " CarCar", " Wednesdays", " fetching", " Kitey", "Crab", " puffing", "Meh", " Hugs", "Allie", " pucker", " Ostrich", "kyard", " ladders", " Zai", "Sweetheart", "Missile", "surrender", "Choochoo", " scribbles", " Boring", " toe", "Tia", " makebelieve", " Oxygen", " Goldie", " squeaking", "Sasha", " You've", " jacko", " snuggly", "Eliza", "Jimbo", " sandals", " darlings", " Delilah", " rhino", " scaly", "Miles", " girly", "Leah", " pawed", "Maisy", " mules", " hermit", "Janey", "Beware", " yummylooking", " Brushy", " granny", " Truckie", " ironing", "Dogs", "Lana", " railings", "Blinky", " Peppa", " Mojo", " squeal", " Splashy", " gleefully", " farmyard", " Ela", " mushy", " Yara", " Alli", "Dinner", " Soap", " Oat", " Fae", " sips", "Merry", " Ruffy", " Purry", " skates", " koala", " limped", " sundae", " cuddles", " wiggles", " stings", " Rino", " sponges", " saidWhat", " seaside", "Restore", " caw", " Wipe", " choochoo", " farmhouse", "Nate", " creaking", "Jumpy", "Wink", "Jellyfish", "Henny", "Louise", " Climbing", " hotdog", " Tooty", " heartbroken", " combs", "Rover", " Nod", " godmother", " sizzle", " Trusty", " ducked", " Curly", " Katerina", " cocked", "Jacky", "Paula", " Celly", "Kip", " pricked", "stupid", " snowed", " balm", " dials", " carousel", " nibbling", " polka", " strummed", "Crane", "Puppy", " sunken", " Shopi", " plump", " littlest", " goo", " dusted", "Gabby", "Josie", " trunks", " Mry", " gratefully", "Emmy", " sprinted", " andy", " whines", " Yacht", "C'mon", " cupped", " Vinny", "Miffy", " honks", "Oink", " Octopus", " silliest", " somers", " Cacti", " tickly", " tiptop", " diaries", " gurgling", " chattered", " Didi", " gazes", " cluck", "Joke", " craziest", " stomachs", " caterpillars", "Mikey", "Cheese", " shriek", " Sheepy", " fragrant", "Dee", " chilli", " sprinkling", "Pinchy", "Monkey", " jogger", " Midge", " Sparkly", " Hailey", " strum", " Cee", "Spiders", " Belinda", "Disappear", " slurping", " snuggling", " gobble", " Kaitlyn", " Kimmy", " jugs", " namedie", " puppets", " wriggle", " bossing", " tulip", " Marla", "Willy", " shaker", " Petey", " antelope", " Pinny", " toddled", " orchard", " greener", " highpitched", " Leafy", "vanish", "Mona", " unwell", "Dot", " fiercelooking", " achy", " unzipped", " wrinkle", " seashells", " clumsily", " Spiny", " matchbox", " crev", " macaroni", "Kenny", " BaaBaa", " Snaily", " Buttons", " Flyer", " droopy", "Mack", " nuzzled", " hoot", " gardeners", " snout", " Pencil", " Lillie", "ewie", " Giggles", " Jessy", " Toma", "Alicia", " breakable", "Bess", "Wheat", " hungrier", " Sink", " sicker", " antennae", " Micky", " moles", "Biggy", " cookout", " talons", "Stretching", " Tessa", " walkietalkie", " Goosey", " Petal", " booms", " Patcy", "Barky", " Moomoo", " Phoebe", " Hana", " cookbook", " Bessy", " Kani", "Hilda", " Roar", " Madie", " anxiously", "Johnnie", " runny", " spooned", " roughlooking", " crumpled", " chattering", " paddles", " reds", "Nurse", " Seagull", " hippos", " sacking", " Gertrude", "Ana", " Isabelle", " Jumping", " Giraffe", " Nosy", " Whose", "Vicky", " Lolly", " leafy", " polishes", " Bobbie", " blossoms", " flitting", " Bury", " neared", " impatiently", " hurriedly", " Halfway", " glimmered", " whirred", " trumpets", " Ron", " shh", " boing", "Penelope", " Gruff", " Haddie", " Tabby", " tablecloth", "Annabelle", " whirled", " sillier", " floaties", "Everywhere", " plumber", "Cautiously", " saidLet", " entranced", "Wendy", " Zola", "Zap", " dizziness", "Sandra", "Rudy", " tippytoes", " firewood", " they", " Reddy", " stinky", "Tiger", "Dilly", " strutted", " Oaty", " Knob", "Ned", " Janice", "Roxy", "Cats", "Pow", " Passport", " scarylooking", " Bubbly", " Cooly", "Cally", " glinting", "Herb", " plow", " lullaby", " sighs", "Quickly", " Zane", " Greta", " Nicky", "CRACK", " glimmer", " saws", "Winky", " hurrying", " lumpy", "Tidy", "Avery", " leaky", " tutu", " Fuzz", " wetness", "Ducks", " Melty", "Becca", " drags", " hoo", " sobbing", "Erin", " hens", " Surprised", " Andi", " Donny", " grime", " jogs", " Hopper", " jingle", "Zoomie", " Mrks", "Yippee", " tippy", "Boat", " chewy", " bleeds", " spool", " lunchbox", " Marry", " dustpan", " Flugel", " Burt", " Rana", " reddest", "Waffle", " scrapes", "Chew", " Jenaya", " bathrobes", " guiltylooking", "Sweetie", " That'll", "Gerry", "Brr", " soothed", " Jada", " dandelions", " Biki", "Abi", " Peeking", " thekyard", "Honk", " toothpaste", " Makeup", " Snuggy", " closets", " Fudge", "Elsie", " campsite", " Kelsie", "buzz", " Elmer", "Kelsey", " zoomzoom", " crows", " sprinkler", " admirer", "Ugh", " Lili", " stomps", "Jasmine", " dialed", "Pizza", " slither", " Simmo", " mixers", " Sprinkles", " Jules", " sooo", " Jie", " quarrels", " Aali", " tock", "Hooray", "Anita", " scribble", "Yucky", " Tammie", "Sheila", " poof", "Whoosh", " Tigger", " Peppy", " errands", " pincher", " 3yearsold", " Sissy", " treasured", " Tooly", " Cubie", " squashes", " Elisa", " busily", "Ll", " firetruck", " dangerouslooking", " rugs", " rafting", " Marley", " saidNo", "Plant", " Tomy", " Thermometer", "Joan", " ringmaster", " Everytime", " Sofie", " tinkling", " tiara", " We'd", "Vanish", " Luxury", " bestie", " Mixer", " Tippe", " floaty", " sandpit", " playdough", "Bury", " stoves", "Nope", " glues", " tutors", " Riles", " they'll", " Hoop", "Katy", " clucking", " fishbowl", " coconuts", " vacuumed", " Peaches", " Scruffy", "POP", " Pointing", " Poofy", " drumming", " Lilli", " parkkeeper", " rudely", " rummaged", " pondered", "Hee", "Bibi", " spinner", "Wooly", " whirl", " Tonya", " longlost", "Crack", " grunted", " cuter", " scolding", " Thumpy", " Octo", " rinsed", " pounce", " anded", " Doggo", "shrink", " saidI'm", " entalled", " Soar", " toad", " twitching", "pinch", " Lottie", " Underneath", " parlor", " valuing", "Swamp", " elated", " washer", " Smokey", "Aye", "Angela", " stooped", " booboos", " Pickles", " slurp", " Spiders", " Wriggles", "Aha", " lilyp", " darken", " gestured", " Grr", "Smile", "Kitten", " nightstand", " Frisky", " campground", " sharpen", " Icey", " pigtails", " yawns", " carnival", "Pew", " Angie", "Stberry", " boar", " sweetsmelling", " shrugs", "Jennie", " grumpily", " ponytail", " scarier", " bookshelves", " couches", "Ivy", " Eggy", " outsmart", " winking", " bloop", " expensivelooking", " snipped", "Cal", " shiniest", "Fairy", " Pippy", " funnyshaped", "Titi", " gasps", "Rise", " juggled", " Fzy", "Spin", " andangles", " hamburger", "Panda", "Ashley", " furthest", "Balloon", " drumsticks", " huff", " bossed", " squawk", "bitter", "Noon", " stalks", " Tasty", " smoky", " Sacky", " Sprayy", " hotelier", " yogurts", " Jaya", " cornfield", "Tweetie", " anthill", " Lacey", " joyous", " Honk", "Sew", " Mrney", " stubbed", " surfboards", " snorted", " Alba", " quilt", " undist", "Frogs", " Corny", "Caroline", "Alfie", " peep", "Squawk", " silvery", " yachts", " Frightened", "Jody", " oatberries", "Brill", " kindest", " Pinch", " shopped", " Rizzo", " namedy", " lifeguard", "Stitch", " angrylooking", "Ronny", " Heed", "Gertie", " Luca", " Singy", "Shine", " Addie", " Baking", " ficked", "Lamp", " whizzed", " trotting", " confetti", "Chess", "Stanley", "fragile", " conditioner", "Dodo", "Randy", " stubbornly", "Muffin", "Ginger", " bulldog", " Smiths", " dayun", "Arf", " ruffled", "Dora", " Mopsy", "Wear", " Pia", " frilly", " Chompy", " Queeny", " passag", " madder", " cutters", "Gemma", " energized", " handkerchief", " sharpener", " Popcorn", " foreb", " Glidey", " Peeky", " Franky", "Behave", " Grumbly", "Sheep", " Nearby", "woof", "Coby", " Biscuit", "oxygen", " Ocky", " Roberta", " raspberries", " mateys", " andge", " marcher", " germs", " Emmie", "Clap", " wondrous", " saucers", " theey", " Dink", " longnecked", " crunching", " Glowbug", "Helen", "Softy", " theest", "slap", "Yield", "Chick", "shelf", " babysitter", " scold", " gnarled", " andb", " disobeyed", "Twist", " Pepi", " pickles", " Compy", " splattered", "Annabel", " clover", " everytime", " golfers", "Candy", "Gail", " Inch", " HopHop", " storybook", "Bunnies", " smacked", " twinkly", "whoosh", " lilies", " harshlooking", " webbed", "Blow", " byebye", " Elina", "Yup", " honeycomb", " saidOh", "Wheeee", " ponytails", " spotless", " feathered", " Mica", " shrieked", " feelers", " Tutor", " Leila", "Greta", " cheeky", " Sweetie", " Kola", " trickort", "Mixie", " Carley", "Frankie", "Goldy", " layed", " oatmeal", " glint", " Needles", "Oxygen", "Bubble", " colouring", " scurry", " neighed", " Guppy", " Feathers", " veils", " squeaks", " Oaky", " doggie", " Fridgey", "usely", "Sell", " Miffy", " Minty", " galloped", "Micky", "We've", " Kippy", " Caterpillar", " Socks", "Grumpy", " Springy", " booed", " baseballs", " Humble", " Hooty", " helplessly", "splash", " beady", " dents", " grizzly", " Celery", " Snook", " friendliest", "Ringo", "Deliver", "Mud", " leopards", " hairstyle", " notred", " footballs", " nana", "Percy", "Clover", " screech", " erases", " cauliflowers", " Racy", "dangerous", " Coaly", " Greenie", " Otter", " drips", " Hootie", "Jolly", "Heidi", " Wherever", " Mrchief", "Zebra", " lovedators", " chirps", " crinkled", " purring", " pinchers", " hamster", " flopped", "Bike", " Pastels", " choppy", " Boats", "Seal", "Trish", "Ricky", " gingerly", " grilling", "spicy", " gluing", " cools", " feathery", " Peety", " saidCome", " Timon", " trays", " looping", " creaked", "Kleo", "Kara", " soggy", " Nola", " outrun", "Carefully", "Flopsy", " carelessly", "Months", "Wise", " Trex", " Smiling", "Vivi", "Swim", "Heather", " itched", " grandad", " jiggle", " pinks", "Swinging", " moonrock", " brighten", " mustaches", " examsams", " postcard", " ouchy", " whimpering", " stumbles", " wriggly", " untidy", " Beatrice", " Mailman", " doghouse", " Tima", " Cuddle", " strumming", " uniting", " merci", " hallways", " unicorns", " tulips", " aye", " wetter", "Tweety", "Climbing", " Mrrobe", " CawCaw", " wilder", " Mousey", "Mellie", "Tick", " Windy", " pines", " yuck", " Yuck", "Seat", " shimmered", " ticked", " frighten", " Yolanda", "Lan", " she'll", " disobey", " daydreaming", " Sorting", " scrunch", " Sheed", "Hopkins", " Chipmunk", " Batty", " tidier", "Bugs", " seashell", " archer", " tosses", " oohed", " gatekeeper", "Hanna", "Sparky", "Miri", "Liam", " Pippip", "Cozy", " Squirrels", "Spinny", " sillylooking", " By", "Moppy", " massaged", " Hoply", " octopuses", " turkeys", " merrily", " Mila", " wrinkly", " spicysmelling", "We'd", " ovens", " sizzling", " jays", " terriblelooking", " Dodi", " rakes", " raisins", " waggy", " glinted", " Margo", "Haha", "Couch", " speeded", "roar", " coughs", " basketballs", " fours", " unwrapping", "Squeak", "Timothy", " chimneys", " noodle", " croaking", " shuffled", " sofas", " ands", " Jacklyn", " scampering", " Pebbles", "Peach", " Cauliflower", "Kathy", " Rani", " paws", " peekaboo", " flipper", " grooves", " feeder", " smoothness", " scooted", " unusuallooking", " oinked", "Weee", " Gorilla", "Lemon", " Imani", "Wherever", " donkeys", " buns", " basking", "Oooh", " Laila", "Tammy", " airfield", "eterred", "Oxy", "Louie", " tightest", "Kyla", "Victory", "Moments", "Peep", " hasn't", " fished", " Melly", " bubbled", " tastylooking", " kinder", "Cathy", " mermaids", " creamcolored", " deafness", " Gretel", "Marry", " zigzagging", "Amanda", " dayie", "Hang", " critters", " Cows", " coolness", " lapping", " moos", " broomstick", "Someday", " hummingbird", " stammered", " contentedly", " wands", " teddies", " Stitchy", " Eddy", "weewoo", "Delicious", " Moolie", "Lia", " yelp", "Coffee", "Sparkle", " cucumber", "Lunch", "Bees", "Shoes", " xrays", " Fluffles", " Trucko", " partied", " Mags", " slowest", " hammering", "Reliable", "Swimmy", " tingly", " ripest", " longed", " rhinoceroses", " Jayne", " Magda", " swished", "Cars", " Avocado", "Susy", " Pipo", " whisked", " Beezy", " andited", " lawman", "Riley", "Gracie", " quiting", " squint", " delicately", " buzzy", " sprayer", " Sisi", "Blech", " pinata", " Yogurt", " thunderous", " Lia", "Kiss", " digger", " Liana", " dozing", " Nia", "Squirrels", "applaud", " Carrots", " sharpest", "Rae", " flashlights", " chugged", " chug", "Ike", "Megan", " pry", " fasten", "weeooo", " chalkboard", "Cauliflower", "emergency", " He'll", " Jemima", " powdery", " longneck", " Slithers", " Singing", " howling", " canary", " jingling", "uryn", " Hippo", " Twig", " wisest", " pouncing", " parakeet", " Rhys", " Woofy", " pluck", "Mop", "Roaring", " snoozed", "Genna", " tingle", " Daniella", " momWhat", " mayise", " zuc", " earthworm", " Fibi", " smudge", " surfer", " Rhinoceros", "Popcorn", "Rita", " whooshing", "Patches", " Salma", " Maxine", " Microphone", " Jokey", " madey", " wrappers", " Moll", " rainbowcoloured", "Aww", " patterned", "Dangerous", " Cheepy", " pedalling", " Comets", " houseboat", " Folly", " ferret", "Maxine", " hideout", " Squiggly", " Snowball", " scribbled", " Lipp", " unrolled", " ding", " eachother", " Katelyn", " sledding", " Bally", " birdy", " outstretched", " erasing", "wisdom", " Goldy", "Tin", "Emilia", " Nona", " squinting", " Volly", " Pasta", "Cari", " pendant", " thoughtfulness", "Sari", "Theo", " greenest", " Rope", "Kaya", "Jelly", " eagerness", " Eager", " fastened", " faroff", " clattered", " Binks", " skated", " jingled", " hamburgers", " shoelaces", " Shells", "Flowers", "weigh", " igloos", " Spoil", " Kace", "Inky", " brokendown", "Freddie", "oggan", " Lea", " pinky", " rhyme", " Amazingly", " sadlooking", " wakeup", " Pappy", " seedling", " daydreamed", " rowboat", "Piggy", " Wilbur", "you're", " Swans", " copilot", " Scared", "Peek", " coals", "Chewy", " Zena", " headband", " blossomed", "Cly", "Peggy", " wagons", " toughlooking", " barbershop", "Grandfather", " Annalise", " whirling", " swish", "Melissa", " Tiddles", " mewed", " leashes", " luckiest", " whirring", " Mints", "Sister", "Cuddles", " scribbly", "Mole", " Marcia", " woodman", " Lexi", " rung", "Kippy", "Yeehaw", " bakers", "Trudy", " cawed", " nightingale", " pinees", "fetch", " Bip", " sighing", "Yawn", " squealing", " waddle", " saidYes", " PlayDoh", " Jannie", "Sunshine", " dunes", "Carol", " Queenie", "Lightning", "Zipper", " pitterpatter", " Lissie", " scribbling", " paintbrushes", " saidThat", " Jy", " thumping", " Lenna", "Leopard", " imaginations", " cooing", " zoos", " blooms", " thely", " beeps", " inseparable", " Tles", " Fisherman", " croaks", " misbehave", " dayy", " Giddy", " Twinkly", " Layland", " camels", " swishing", " Socksy", " gleaming", "Jax", " Lawyer", " swingset", " thunderclap", " rippling", " grumpiness", " courageously", " snore", " Fins", " rooster", "veterin", " Ka", " Stuffy", " Smiley", " ached", " dinnertime", " splat", " dreary", " Quiz", "Bessy", " Pina", " earring", " songbird", " sunning", " swum", "Hoot", " prodded", " dares", " squiggles", " squirm"] \ No newline at end of file