diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..1278349 Binary files /dev/null and b/.DS_Store differ diff --git a/.gitmodules b/.gitmodules index c0bb54c..261bb34 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,12 @@ [submodule "dependency_repos/github-downloader"] path = dependency_repos/github-downloader url = https://github.com/EleutherAI/github-downloader +[submodule "dependency_repos/apps"] + path = dependency_repos/apps + url = https://github.com/hendrycks/apps.git +[submodule "dependency_repos/human-eval"] + path = dependency_repos/human-eval + url = https://github.com/openai/human-eval +[submodule "dependency_repos/CodeXGLUE"] + path = dependency_repos/CodeXGLUE + url = https://github.com/microsoft/CodeXGLUE diff --git a/dependency_repos/CodeXGLUE b/dependency_repos/CodeXGLUE new file mode 160000 index 0000000..3e7bfe6 --- /dev/null +++ b/dependency_repos/CodeXGLUE @@ -0,0 +1 @@ +Subproject commit 3e7bfe6dc4a88534c7803ce1bd8d1733c1d16888 diff --git a/dependency_repos/apps b/dependency_repos/apps new file mode 160000 index 0000000..f834ca7 --- /dev/null +++ b/dependency_repos/apps @@ -0,0 +1 @@ +Subproject commit f834ca7d7405935376aabb5830edd0c42635824e diff --git a/dependency_repos/human-eval b/dependency_repos/human-eval new file mode 160000 index 0000000..463c980 --- /dev/null +++ b/dependency_repos/human-eval @@ -0,0 +1 @@ +Subproject commit 463c980b59e818ace59f6f9803cd92c749ceae61 diff --git a/metrics/.DS_Store b/metrics/.DS_Store new file mode 100644 index 0000000..66f0e1e Binary files /dev/null and b/metrics/.DS_Store differ diff --git a/metrics/bleu.py b/metrics/bleu.py index 49c56ca..17a06ce 100644 --- a/metrics/bleu.py +++ b/metrics/bleu.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -# Took the following from CodeXGlue Repository - https://github.com/microsoft/CodeXGLUE/blob/main/Code-Code/code-to-code-trans/evaluator/CodeBLEU/bleu.py +# The following code is taken from CodeXGlue Repository - https://github.com/microsoft/CodeXGLUE/blob/main/Code-Code/code-to-code-trans/evaluator/CodeBLEU/bleu.py """Python implementation of BLEU and smooth-BLEU. @@ -28,98 +28,106 @@ def _get_ngrams(segment, max_order): - """Extracts all n-grams upto a given maximum order from an input segment. - - Args: - segment: text segment from which n-grams will be extracted. - max_order: maximum length in tokens of the n-grams returned by this - methods. - - Returns: - The Counter containing all n-grams upto max_order in segment - with a count of how many times each n-gram occurred. - """ - ngram_counts = collections.Counter() - for order in range(1, max_order + 1): - for i in range(0, len(segment) - order + 1): - ngram = tuple(segment[i:i+order]) - ngram_counts[ngram] += 1 - return ngram_counts - - -def compute_bleu(reference_corpus, translation_corpus, max_order=4, - smooth=True): - """Computes BLEU score of translated segments against one or more references. - - Args: - reference_corpus: list of lists of references for each translation. Each - reference should be tokenized into a list of tokens. - translation_corpus: list of translations to score. Each translation - should be tokenized into a list of tokens. - max_order: Maximum n-gram order to use when computing BLEU score. - smooth: Whether or not to apply Lin et al. 2004 smoothing. - - Returns: - 3-Tuple with the BLEU score, n-gram precisions, geometric mean of n-gram - precisions and brevity penalty. - """ - matches_by_order = [0] * max_order - possible_matches_by_order = [0] * max_order - reference_length = 0 - translation_length = 0 - for (references, translation) in zip(reference_corpus, - translation_corpus): - reference_length += min(len(r) for r in references) - translation_length += len(translation) - - merged_ref_ngram_counts = collections.Counter() - for reference in references: - merged_ref_ngram_counts |= _get_ngrams(reference, max_order) - translation_ngram_counts = _get_ngrams(translation, max_order) - overlap = translation_ngram_counts & merged_ref_ngram_counts - for ngram in overlap: - matches_by_order[len(ngram)-1] += overlap[ngram] - for order in range(1, max_order+1): - possible_matches = len(translation) - order + 1 - if possible_matches > 0: - possible_matches_by_order[order-1] += possible_matches - - precisions = [0] * max_order - for i in range(0, max_order): - if smooth: - precisions[i] = ((matches_by_order[i] + 1.) / - (possible_matches_by_order[i] + 1.)) + """Extracts all n-grams upto a given maximum order from an input segment. + + Args: + segment: text segment from which n-grams will be extracted. + max_order: maximum length in tokens of the n-grams returned by this + methods. + + Returns: + The Counter containing all n-grams upto max_order in segment + with a count of how many times each n-gram occurred. + """ + ngram_counts = collections.Counter() + for order in range(1, max_order + 1): + for i in range(0, len(segment) - order + 1): + ngram = tuple(segment[i : i + order]) + ngram_counts[ngram] += 1 + return ngram_counts + + +def compute_bleu(reference_corpus, translation_corpus, max_order=4, smooth=True): + """Computes BLEU score of translated segments against one or more references. + + Args: + reference_corpus: list of lists of references for each translation. Each + reference should be tokenized into a list of tokens. + translation_corpus: list of translations to score. Each translation + should be tokenized into a list of tokens. + max_order: Maximum n-gram order to use when computing BLEU score. + smooth: Whether or not to apply Lin et al. 2004 smoothing. + + Returns: + 3-Tuple with the BLEU score, n-gram precisions, geometric mean of n-gram + precisions and brevity penalty. + """ + matches_by_order = [0] * max_order + possible_matches_by_order = [0] * max_order + reference_length = 0 + translation_length = 0 + for (references, translation) in zip(reference_corpus, translation_corpus): + reference_length += min(len(r) for r in references) + translation_length += len(translation) + + merged_ref_ngram_counts = collections.Counter() + for reference in references: + merged_ref_ngram_counts |= _get_ngrams(reference, max_order) + translation_ngram_counts = _get_ngrams(translation, max_order) + overlap = translation_ngram_counts & merged_ref_ngram_counts + for ngram in overlap: + matches_by_order[len(ngram) - 1] += overlap[ngram] + for order in range(1, max_order + 1): + possible_matches = len(translation) - order + 1 + if possible_matches > 0: + possible_matches_by_order[order - 1] += possible_matches + + precisions = [0] * max_order + for i in range(0, max_order): + if smooth: + precisions[i] = (matches_by_order[i] + 1.0) / ( + possible_matches_by_order[i] + 1.0 + ) + else: + if possible_matches_by_order[i] > 0: + precisions[i] = ( + float(matches_by_order[i]) / possible_matches_by_order[i] + ) + else: + precisions[i] = 0.0 + + if min(precisions) > 0: + p_log_sum = sum((1.0 / max_order) * math.log(p) for p in precisions) + geo_mean = math.exp(p_log_sum) else: - if possible_matches_by_order[i] > 0: - precisions[i] = (float(matches_by_order[i]) / - possible_matches_by_order[i]) - else: - precisions[i] = 0.0 - - if min(precisions) > 0: - p_log_sum = sum((1. / max_order) * math.log(p) for p in precisions) - geo_mean = math.exp(p_log_sum) - else: - geo_mean = 0 - - ratio = float(translation_length) / reference_length - - if ratio > 1.0: - bp = 1. - else: - bp = math.exp(1 - 1. / ratio) - bleu = geo_mean * bp - print(geo_mean) - bleu_score_dict = {"bleu":bleu,"precision":precisions,"bp":bp,"ratio":ratio,"trans_len":translation_length,"ref_len":reference_length} - return bleu_score_dict#(bleu, precisions, bp, ratio, translation_length, reference_length) + geo_mean = 0 + + ratio = float(translation_length) / reference_length + + if ratio > 1.0: + bp = 1.0 + else: + bp = math.exp(1 - 1.0 / ratio) + bleu = geo_mean * bp + bleu_score_dict = { + "bleu": bleu, + "precision": precisions, + "bp": bp, + "ratio": ratio, + "trans_len": translation_length, + "ref_len": reference_length, + } + return bleu_score_dict # (bleu, precisions, bp, ratio, translation_length, reference_length) + def bleu_test_case(): """A simple functionality test case to evaluate BLEU""" - generated = [[["a","=","b","\n","y","=","a","+","1"]]] - reference = [["a","=","b","\n","print","a"]] - score_dict = compute_bleu(generated,reference,smooth=False) + generated = [[["a", "=", "b", "\n", "y", "=", "a", "+", "1"]]] + reference = [["a", "=", "b", "\n", "print", "a"]] + score_dict = compute_bleu(generated, reference, smooth=False) return score_dict + if __name__ == "__main__": score_dict = bleu_test_case() - print(score_dict) \ No newline at end of file + print(score_dict) diff --git a/metrics/calc_code_bleu.py b/metrics/calc_code_bleu.py new file mode 100644 index 0000000..0b601b0 --- /dev/null +++ b/metrics/calc_code_bleu.py @@ -0,0 +1,128 @@ + +#Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +# -*- coding:utf-8 -*- +import json +import argparse +import bleu +import weighted_ngram_match +import syntax_match +import dataflow_match + + +def python_process(tokens): + new_tokens = [] + indent_count = 0 + num_tokens = len(tokens) + tidx = 0 + while tidx < num_tokens: + tok = tokens[tidx] + tok = tok.strip() + if tok in ["NEW_LINE"]: + new_tokens.append("\n") + if tidx + 1 < num_tokens: + next_token = tokens[tidx + 1] + if next_token == "INDENT": + indent_count += 1 + tidx += 1 + elif next_token == "DEDENT": + indent_count -= 1 + tidx += 1 + for ic in range(indent_count): + new_tokens.append("\t") + else: + new_tokens.append(tok) + tidx += 1 + return new_tokens + pass + + +def php_process(tokens): + new_tokens = [] + num_tokens = len(tokens) + tidx = 0 + while tidx < num_tokens: + tok = tokens[tidx] + tok = tok.strip() + if tok == "$": + if tidx + 1 < num_tokens: + tok += tokens[tidx + 1].strip() + tidx += 1 + pass + pass + tidx += 1 + new_tokens.append(tok) + return new_tokens + + +def language_specific_processing(tokens, lang): + if lang == 'python': + return python_process(tokens) + elif lang == 'php': + return php_process(tokens) + else: + return tokens + + +parser = argparse.ArgumentParser() +parser.add_argument('--ref', type=str, required=True, + help='reference file') +parser.add_argument('--hyp', type=str, required=True, + help='hypothesis file') +parser.add_argument('--lang', type=str, required=True, + choices=['java', 'js', 'c_sharp', 'php', 'go', 'python', 'ruby'], + help='programming language') +parser.add_argument('--params', type=str, default='0.25,0.25,0.25,0.25', + help='alpha, beta and gamma') + +args = parser.parse_args() + +lang = args.lang +if lang == 'js': + lang = 'javascript' +alpha, beta, gamma, theta = [float(x) for x in args.params.split(',')] + +# preprocess inputs +references = [json.loads(x.strip())[lang] for x in open(args.ref, 'r', encoding='utf-8').readlines()] +hypothesis = [x.strip() for x in open(args.hyp, 'r', encoding='utf-8').readlines()] + +assert len(hypothesis) == len(references) + +# calculate ngram match (BLEU) +tokenized_hyps = [language_specific_processing(x.split(), lang) for x in hypothesis] +tokenized_refs = [[language_specific_processing(x.split(), lang) for x in reference] for reference in references] + +ngram_match_score = bleu.corpus_bleu(tokenized_refs, tokenized_hyps) + +# calculate weighted ngram match +keywords = [x.strip() for x in open('keywords/' + lang + '.txt', 'r', encoding='utf-8').readlines()] + + +def make_weights(reference_tokens, key_word_list): + return {token: 1 if token in key_word_list else 0.2 \ + for token in reference_tokens} + + +tokenized_refs_with_weights = [[[reference_tokens, make_weights(reference_tokens, keywords)] \ + for reference_tokens in reference] for reference in tokenized_refs] + +weighted_ngram_match_score = weighted_ngram_match.corpus_bleu(tokenized_refs_with_weights, tokenized_hyps) + +# calculate syntax match +syntax_match_score = syntax_match.corpus_syntax_match(references, hypothesis, lang) + +# calculate dataflow match +dataflow_match_score = dataflow_match.corpus_dataflow_match(references, hypothesis, lang) + +# print('ngram match: {0}, weighted ngram match: {1}, syntax_match: {2}, dataflow_match: {3}'. \ +# format(ngram_match_score, weighted_ngram_match_score, syntax_match_score, dataflow_match_score)) +print('Ngram match:\t%.2f\nWeighted ngram:\t%.2f\nSyntax match:\t%.2f\nDataflow match:\t%.2f' % ( \ + ngram_match_score * 100, weighted_ngram_match_score * 100, syntax_match_score * 100, dataflow_match_score * 100)) + +code_bleu_score = alpha * ngram_match_score \ + + beta * weighted_ngram_match_score \ + + gamma * syntax_match_score \ + + theta * dataflow_match_score + +print('CodeBLEU score: %.2f' % (code_bleu_score * 100.0)) \ No newline at end of file diff --git a/metrics/extrinsic_eval.py b/metrics/extrinsic_eval.py index 24ee15a..c8f28ad 100644 --- a/metrics/extrinsic_eval.py +++ b/metrics/extrinsic_eval.py @@ -1,6 +1,10 @@ from metrics.bleu import compute_bleu +from metrics.parse_check import check_parse -def compute_metrics(references,generated) -> dict: +Parser = check_parse() # Initializing parser + + +def compute_metrics(references, generated, lang) -> dict: """ Calculates various metrics and returns the calculated dict of these matrics. args: @@ -8,11 +12,12 @@ def compute_metrics(references,generated) -> dict: reference should be tokenized into a list of tokens. translation: list of translations to score. Each translation should be tokenized into a list of tokens. + lang(str) : The language generated code belongs to returns: A dicitonary with different metrics intact. """ - metrics_dict = {} #Update as in new metrics are added over here. - metrics_dict["smoothed_bleu_4"] = compute_bleu(references,generated,smooth=True) - metrics_dict["bleu_4"] = compute_bleu(references,generated,smooth=False) - - return metrics_dict \ No newline at end of file + metrics_dict = {} # Update as in new metrics are added over here. + metrics_dict["smoothed_bleu_4"] = compute_bleu(references, generated, smooth=True) + metrics_dict["bleu_4"] = compute_bleu(references, generated, smooth=False) + metrics_dict["parse_score"] = Parser(generated, lang)["parse_score"] + return metrics_dict diff --git a/metrics/parse_check.py b/metrics/parse_check.py new file mode 100644 index 0000000..7f5e750 --- /dev/null +++ b/metrics/parse_check.py @@ -0,0 +1,53 @@ +from tree_sitter import Language, Parser + +def load_tree_sitter_languages(): + """Loads language Grammars to evaluate""" + py_parser = Parser() + py_parser.set_language(Language('./tree_sitter_utils/build/my-languages.so', 'python')) + js_parser = Parser() + js_parser.set_language(Language('./tree_sitter_utils/build/my-languages.so', 'javascript')) + cpp_parser = Parser() + cpp_parser.set_language(Language('./tree_sitter_utils/build/my-languages.so', 'cpp')) + go_parser = Parser() + go_parser.set_language(Language('./tree_sitter_utils/build/my-languages.so', 'go')) + java_parser = Parser() + java_parser.set_language(Language('./tree_sitter_utils/build/my-languages.so', 'java')) + return { + "py" : py_parser, + "js" : js_parser, + "cpp" : cpp_parser, + "go" : go_parser, + "java": java_parser + } + +class check_parse: + def __init__(self): + self.language_dict = load_tree_sitter_languages() + def __call__(self,batch,lang): + """ + args: + batch : list[str] of code generated by the model + lang : lang should be one of the above language_dict keys + + returns: + dict( + parse_score = averaged out score on how many datapoints are parsed + index_parse = check if corresponding index is parsed + ) + """ + cumulative_parse_score = 0 + index_parse_list = [] + parser = self.language_dict[lang] + for inp in batch: + parsed = parser.parse(bytes(inp,"utf-8")) + inp_ind_score = int("ERROR" not in parsed.root_node.sexp()) + cumulative_parse_score+=inp_ind_score + index_parse_list.append(inp_ind_score) + return {"parse_score":cumulative_parse_score,"index_parse":index_parse_list} +if __name__ == "__main__": + Parse = check_parse() + score = Parse([""" +def a(): + if bar: + baz()"""],"py") + print(score) \ No newline at end of file diff --git a/metrics/syntax_match.py b/metrics/syntax_match.py new file mode 100644 index 0000000..dd81c15 --- /dev/null +++ b/metrics/syntax_match.py @@ -0,0 +1,73 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from parser import DFG_python,DFG_java,DFG_ruby,DFG_go,DFG_php,DFG_javascript,DFG_csharp +from parser import (remove_comments_and_docstrings, + tree_to_token_index, + index_to_code_token, + tree_to_variable_index) +from tree_sitter import Language, Parser + +dfg_function={ + 'python':DFG_python, + 'java':DFG_java, + 'ruby':DFG_ruby, + 'go':DFG_go, + 'php':DFG_php, + 'javascript':DFG_javascript, + 'c_sharp':DFG_csharp, +} + +def calc_syntax_match(references, candidate, lang): + return corpus_syntax_match([references], [candidate], lang) + +def corpus_syntax_match(references, candidates, lang): + JAVA_LANGUAGE = Language('parser/my-languages.so', lang) + parser = Parser() + parser.set_language(JAVA_LANGUAGE) + match_count = 0 + total_count = 0 + + for i in range(len(candidates)): + references_sample = references[i] + candidate = candidates[i] + for reference in references_sample: + try: + candidate=remove_comments_and_docstrings(candidate,'java') + except: + pass + try: + reference=remove_comments_and_docstrings(reference,'java') + except: + pass + + candidate_tree = parser.parse(bytes(candidate,'utf8')).root_node + + reference_tree = parser.parse(bytes(reference,'utf8')).root_node + + def get_all_sub_trees(root_node): + node_stack = [] + sub_tree_sexp_list = [] + depth = 1 + node_stack.append([root_node, depth]) + while len(node_stack) != 0: + cur_node, cur_depth = node_stack.pop() + sub_tree_sexp_list.append([cur_node.sexp(), cur_depth]) + for child_node in cur_node.children: + if len(child_node.children) != 0: + depth = cur_depth + 1 + node_stack.append([child_node, depth]) + return sub_tree_sexp_list + cand_sexps = [x[0] for x in get_all_sub_trees(candidate_tree)] + ref_sexps = get_all_sub_trees(reference_tree) + + # print(cand_sexps) + # print(ref_sexps) + + for sub_tree, depth in ref_sexps: + if sub_tree in cand_sexps: + match_count += 1 + total_count += len(ref_sexps) + + score = match_count / total_count + return score diff --git a/metrics/tree_sitter_utils/.DS_Store b/metrics/tree_sitter_utils/.DS_Store new file mode 100644 index 0000000..643fb35 Binary files /dev/null and b/metrics/tree_sitter_utils/.DS_Store differ diff --git a/metrics/weighted_ngram_match.py b/metrics/weighted_ngram_match.py new file mode 100644 index 0000000..f12b71b --- /dev/null +++ b/metrics/weighted_ngram_match.py @@ -0,0 +1,558 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +# Natural Language Toolkit: BLEU Score +# +# Copyright (C) 2001-2020 NLTK Project +# Authors: Chin Yee Lee, Hengfeng Li, Ruxin Hou, Calvin Tanujaya Lim +# Contributors: Björn Mattsson, Dmitrijs Milajevs, Liling Tan +# URL: +# For license information, see LICENSE.TXT + +"""BLEU score implementation.""" + +import math +import sys +from fractions import Fraction +import warnings +from collections import Counter + +from utils import ngrams +import pdb + + +def sentence_bleu( + references, + hypothesis, + weights=(0.25, 0.25, 0.25, 0.25), + smoothing_function=None, + auto_reweigh=False, +): + """ + Calculate BLEU score (Bilingual Evaluation Understudy) from + Papineni, Kishore, Salim Roukos, Todd Ward, and Wei-Jing Zhu. 2002. + "BLEU: a method for automatic evaluation of machine translation." + In Proceedings of ACL. http://www.aclweb.org/anthology/P02-1040.pdf + >>> hypothesis1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', + ... 'ensures', 'that', 'the', 'military', 'always', + ... 'obeys', 'the', 'commands', 'of', 'the', 'party'] + >>> hypothesis2 = ['It', 'is', 'to', 'insure', 'the', 'troops', + ... 'forever', 'hearing', 'the', 'activity', 'guidebook', + ... 'that', 'party', 'direct'] + >>> reference1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', + ... 'ensures', 'that', 'the', 'military', 'will', 'forever', + ... 'heed', 'Party', 'commands'] + >>> reference2 = ['It', 'is', 'the', 'guiding', 'principle', 'which', + ... 'guarantees', 'the', 'military', 'forces', 'always', + ... 'being', 'under', 'the', 'command', 'of', 'the', + ... 'Party'] + >>> reference3 = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the', + ... 'army', 'always', 'to', 'heed', 'the', 'directions', + ... 'of', 'the', 'party'] + >>> sentence_bleu([reference1, reference2, reference3], hypothesis1) # doctest: +ELLIPSIS + 0.5045... + If there is no ngrams overlap for any order of n-grams, BLEU returns the + value 0. This is because the precision for the order of n-grams without + overlap is 0, and the geometric mean in the final BLEU score computation + multiplies the 0 with the precision of other n-grams. This results in 0 + (independently of the precision of the othe n-gram orders). The following + example has zero 3-gram and 4-gram overlaps: + >>> round(sentence_bleu([reference1, reference2, reference3], hypothesis2),4) # doctest: +ELLIPSIS + 0.0 + To avoid this harsh behaviour when no ngram overlaps are found a smoothing + function can be used. + >>> chencherry = SmoothingFunction() + >>> sentence_bleu([reference1, reference2, reference3], hypothesis2, + ... smoothing_function=chencherry.method1) # doctest: +ELLIPSIS + 0.0370... + The default BLEU calculates a score for up to 4-grams using uniform + weights (this is called BLEU-4). To evaluate your translations with + higher/lower order ngrams, use customized weights. E.g. when accounting + for up to 5-grams with uniform weights (this is called BLEU-5) use: + >>> weights = (1./5., 1./5., 1./5., 1./5., 1./5.) + >>> sentence_bleu([reference1, reference2, reference3], hypothesis1, weights) # doctest: +ELLIPSIS + 0.3920... + :param references: reference sentences + :type references: list(list(str)) + :param hypothesis: a hypothesis sentence + :type hypothesis: list(str) + :param weights: weights for unigrams, bigrams, trigrams and so on + :type weights: list(float) + :param smoothing_function: + :type smoothing_function: SmoothingFunction + :param auto_reweigh: Option to re-normalize the weights uniformly. + :type auto_reweigh: bool + :return: The sentence-level BLEU score. + :rtype: float + """ + return corpus_bleu( + [references], [hypothesis], weights, smoothing_function, auto_reweigh + ) + + +def corpus_bleu( + list_of_references, + hypotheses, + weights=(0.25, 0.25, 0.25, 0.25), + smoothing_function=None, + auto_reweigh=False, +): + """ + Calculate a single corpus-level BLEU score (aka. system-level BLEU) for all + the hypotheses and their respective references. + Instead of averaging the sentence level BLEU scores (i.e. marco-average + precision), the original BLEU metric (Papineni et al. 2002) accounts for + the micro-average precision (i.e. summing the numerators and denominators + for each hypothesis-reference(s) pairs before the division). + >>> hyp1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', + ... 'ensures', 'that', 'the', 'military', 'always', + ... 'obeys', 'the', 'commands', 'of', 'the', 'party'] + >>> ref1a = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', + ... 'ensures', 'that', 'the', 'military', 'will', 'forever', + ... 'heed', 'Party', 'commands'] + >>> ref1b = ['It', 'is', 'the', 'guiding', 'principle', 'which', + ... 'guarantees', 'the', 'military', 'forces', 'always', + ... 'being', 'under', 'the', 'command', 'of', 'the', 'Party'] + >>> ref1c = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the', + ... 'army', 'always', 'to', 'heed', 'the', 'directions', + ... 'of', 'the', 'party'] + >>> hyp2 = ['he', 'read', 'the', 'book', 'because', 'he', 'was', + ... 'interested', 'in', 'world', 'history'] + >>> ref2a = ['he', 'was', 'interested', 'in', 'world', 'history', + ... 'because', 'he', 'read', 'the', 'book'] + >>> list_of_references = [[ref1a, ref1b, ref1c], [ref2a]] + >>> hypotheses = [hyp1, hyp2] + >>> corpus_bleu(list_of_references, hypotheses) # doctest: +ELLIPSIS + 0.5920... + The example below show that corpus_bleu() is different from averaging + sentence_bleu() for hypotheses + >>> score1 = sentence_bleu([ref1a, ref1b, ref1c], hyp1) + >>> score2 = sentence_bleu([ref2a], hyp2) + >>> (score1 + score2) / 2 # doctest: +ELLIPSIS + 0.6223... + :param list_of_references: a corpus of lists of reference sentences, w.r.t. hypotheses + :type list_of_references: list(list(list(str))) + :param hypotheses: a list of hypothesis sentences + :type hypotheses: list(list(str)) + :param weights: weights for unigrams, bigrams, trigrams and so on + :type weights: list(float) + :param smoothing_function: + :type smoothing_function: SmoothingFunction + :param auto_reweigh: Option to re-normalize the weights uniformly. + :type auto_reweigh: bool + :return: The corpus-level BLEU score. + :rtype: float + """ + # Before proceeding to compute BLEU, perform sanity checks. + + p_numerators = Counter() # Key = ngram order, and value = no. of ngram matches. + p_denominators = Counter() # Key = ngram order, and value = no. of ngram in ref. + hyp_lengths, ref_lengths = 0, 0 + + assert len(list_of_references) == len(hypotheses), ( + "The number of hypotheses and their reference(s) should be the " "same " + ) + + # Iterate through each hypothesis and their corresponding references. + for references, hypothesis in zip(list_of_references, hypotheses): + # For each order of ngram, calculate the numerator and + # denominator for the corpus-level modified precision. + for i, _ in enumerate(weights, start=1): + p_i_numeraotr, p_i_denominator = modified_recall(references, hypothesis, i) + p_numerators[i] += p_i_numeraotr + p_denominators[i] += p_i_denominator + + # Calculate the hypothesis length and the closest reference length. + # Adds them to the corpus-level hypothesis and reference counts. + hyp_len = len(hypothesis) + hyp_lengths += hyp_len + ref_lengths += closest_ref_length(references, hyp_len) + + # Calculate corpus-level brevity penalty. + bp = brevity_penalty(ref_lengths, hyp_lengths) + + # Uniformly re-weighting based on maximum hypothesis lengths if largest + # order of n-grams < 4 and weights is set at default. + if auto_reweigh: + if hyp_lengths < 4 and weights == (0.25, 0.25, 0.25, 0.25): + weights = (1 / hyp_lengths,) * hyp_lengths + + # Collects the various recall values for the different ngram orders. + p_n = [ + (p_numerators[i], p_denominators[i]) + for i, _ in enumerate(weights, start=1) + ] + + # Returns 0 if there's no matching n-grams + # We only need to check for p_numerators[1] == 0, since if there's + # no unigrams, there won't be any higher order ngrams. + if p_numerators[1] == 0: + return 0 + + # If there's no smoothing, set use method0 from SmoothinFunction class. + if not smoothing_function: + smoothing_function = SmoothingFunction().method1 + # Smoothen the modified precision. + # Note: smoothing_function() may convert values into floats; + # it tries to retain the Fraction object as much as the + # smoothing method allows. + p_n = smoothing_function( + p_n, references=references, hypothesis=hypothesis, hyp_len=hyp_lengths + ) + # pdb.set_trace() + s = (w_i * math.log(p_i[0]/p_i[1]) for w_i, p_i in zip(weights, p_n)) + s = bp * math.exp(math.fsum(s)) + return s + + +def modified_recall(references, hypothesis, n): + """ + Calculate modified ngram recall. + :param references: A list of reference translations. + :type references: list(list(str)) + :param hypothesis: A hypothesis translation. + :type hypothesis: list(str) + :param n: The ngram order. + :type n: int + :return: BLEU's modified precision for the nth order ngram. + :rtype: Fraction + """ + # Extracts all ngrams in hypothesis + # Set an empty Counter if hypothesis is empty. + # pdb.set_trace() + numerator = 0 + denominator = 0 + + counts = Counter(ngrams(hypothesis, n)) if len(hypothesis) >= n else Counter() + # Extract a union of references' counts. + # max_counts = reduce(or_, [Counter(ngrams(ref, n)) for ref in references]) + max_counts = {} + for reference_and_weights in references: + reference = reference_and_weights[0] + weights = reference_and_weights[1] + reference_counts = ( + Counter(ngrams(reference, n)) if len(reference) >= n else Counter() + ) + # for ngram in reference_counts: + # max_counts[ngram] = max(max_counts.get(ngram, 0), counts[ngram]) + clipped_counts = { + ngram: min(count, counts[ngram]) for ngram, count in reference_counts.items() + } + # reweight + if n == 1 and len(weights) == len(reference_counts): + def weighted_sum(weights, counts): + sum_counts = 0 + for ngram, count in counts.items(): + sum_counts += count * (weights[ngram[0]] if ngram[0] in weights else 1) + return sum_counts + + numerator += weighted_sum(weights, clipped_counts) + denominator += max(1, weighted_sum(weights, reference_counts)) + + else: + numerator += sum(clipped_counts.values()) + denominator += max(1, sum(reference_counts.values())) + + # # Assigns the intersection between hypothesis and references' counts. + # clipped_counts = { + # ngram: min(count, max_counts[ngram]) for ngram, count in counts.items() + # } + + # numerator += sum(clipped_counts.values()) + # # Ensures that denominator is minimum 1 to avoid ZeroDivisionError. + # # Usually this happens when the ngram order is > len(reference). + # denominator += max(1, sum(counts.values())) + + #return Fraction(numerator, denominator, _normalize=False) + return numerator, denominator + + +def closest_ref_length(references, hyp_len): + """ + This function finds the reference that is the closest length to the + hypothesis. The closest reference length is referred to as *r* variable + from the brevity penalty formula in Papineni et. al. (2002) + :param references: A list of reference translations. + :type references: list(list(str)) + :param hyp_len: The length of the hypothesis. + :type hyp_len: int + :return: The length of the reference that's closest to the hypothesis. + :rtype: int + """ + ref_lens = (len(reference) for reference in references) + closest_ref_len = min( + ref_lens, key=lambda ref_len: (abs(ref_len - hyp_len), ref_len) + ) + return closest_ref_len + + +def brevity_penalty(closest_ref_len, hyp_len): + """ + Calculate brevity penalty. + As the modified n-gram precision still has the problem from the short + length sentence, brevity penalty is used to modify the overall BLEU + score according to length. + An example from the paper. There are three references with length 12, 15 + and 17. And a concise hypothesis of the length 12. The brevity penalty is 1. + >>> reference1 = list('aaaaaaaaaaaa') # i.e. ['a'] * 12 + >>> reference2 = list('aaaaaaaaaaaaaaa') # i.e. ['a'] * 15 + >>> reference3 = list('aaaaaaaaaaaaaaaaa') # i.e. ['a'] * 17 + >>> hypothesis = list('aaaaaaaaaaaa') # i.e. ['a'] * 12 + >>> references = [reference1, reference2, reference3] + >>> hyp_len = len(hypothesis) + >>> closest_ref_len = closest_ref_length(references, hyp_len) + >>> brevity_penalty(closest_ref_len, hyp_len) + 1.0 + In case a hypothesis translation is shorter than the references, penalty is + applied. + >>> references = [['a'] * 28, ['a'] * 28] + >>> hypothesis = ['a'] * 12 + >>> hyp_len = len(hypothesis) + >>> closest_ref_len = closest_ref_length(references, hyp_len) + >>> brevity_penalty(closest_ref_len, hyp_len) + 0.2635971381157267 + The length of the closest reference is used to compute the penalty. If the + length of a hypothesis is 12, and the reference lengths are 13 and 2, the + penalty is applied because the hypothesis length (12) is less then the + closest reference length (13). + >>> references = [['a'] * 13, ['a'] * 2] + >>> hypothesis = ['a'] * 12 + >>> hyp_len = len(hypothesis) + >>> closest_ref_len = closest_ref_length(references, hyp_len) + >>> brevity_penalty(closest_ref_len, hyp_len) # doctest: +ELLIPSIS + 0.9200... + The brevity penalty doesn't depend on reference order. More importantly, + when two reference sentences are at the same distance, the shortest + reference sentence length is used. + >>> references = [['a'] * 13, ['a'] * 11] + >>> hypothesis = ['a'] * 12 + >>> hyp_len = len(hypothesis) + >>> closest_ref_len = closest_ref_length(references, hyp_len) + >>> bp1 = brevity_penalty(closest_ref_len, hyp_len) + >>> hyp_len = len(hypothesis) + >>> closest_ref_len = closest_ref_length(reversed(references), hyp_len) + >>> bp2 = brevity_penalty(closest_ref_len, hyp_len) + >>> bp1 == bp2 == 1 + True + A test example from mteval-v13a.pl (starting from the line 705): + >>> references = [['a'] * 11, ['a'] * 8] + >>> hypothesis = ['a'] * 7 + >>> hyp_len = len(hypothesis) + >>> closest_ref_len = closest_ref_length(references, hyp_len) + >>> brevity_penalty(closest_ref_len, hyp_len) # doctest: +ELLIPSIS + 0.8668... + >>> references = [['a'] * 11, ['a'] * 8, ['a'] * 6, ['a'] * 7] + >>> hypothesis = ['a'] * 7 + >>> hyp_len = len(hypothesis) + >>> closest_ref_len = closest_ref_length(references, hyp_len) + >>> brevity_penalty(closest_ref_len, hyp_len) + 1.0 + :param hyp_len: The length of the hypothesis for a single sentence OR the + sum of all the hypotheses' lengths for a corpus + :type hyp_len: int + :param closest_ref_len: The length of the closest reference for a single + hypothesis OR the sum of all the closest references for every hypotheses. + :type closest_ref_len: int + :return: BLEU's brevity penalty. + :rtype: float + """ + if hyp_len > closest_ref_len: + return 1 + # If hypothesis is empty, brevity penalty = 0 should result in BLEU = 0.0 + elif hyp_len == 0: + return 0 + else: + return math.exp(1 - closest_ref_len / hyp_len) + + +class SmoothingFunction: + """ + This is an implementation of the smoothing techniques + for segment-level BLEU scores that was presented in + Boxing Chen and Collin Cherry (2014) A Systematic Comparison of + Smoothing Techniques for Sentence-Level BLEU. In WMT14. + http://acl2014.org/acl2014/W14-33/pdf/W14-3346.pdf + """ + + def __init__(self, epsilon=0.1, alpha=5, k=5): + """ + This will initialize the parameters required for the various smoothing + techniques, the default values are set to the numbers used in the + experiments from Chen and Cherry (2014). + >>> hypothesis1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', 'ensures', + ... 'that', 'the', 'military', 'always', 'obeys', 'the', + ... 'commands', 'of', 'the', 'party'] + >>> reference1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', 'ensures', + ... 'that', 'the', 'military', 'will', 'forever', 'heed', + ... 'Party', 'commands'] + >>> chencherry = SmoothingFunction() + >>> print(sentence_bleu([reference1], hypothesis1)) # doctest: +ELLIPSIS + 0.4118... + >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method0)) # doctest: +ELLIPSIS + 0.4118... + >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method1)) # doctest: +ELLIPSIS + 0.4118... + >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method2)) # doctest: +ELLIPSIS + 0.4489... + >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method3)) # doctest: +ELLIPSIS + 0.4118... + >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method4)) # doctest: +ELLIPSIS + 0.4118... + >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method5)) # doctest: +ELLIPSIS + 0.4905... + >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method6)) # doctest: +ELLIPSIS + 0.4135... + >>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method7)) # doctest: +ELLIPSIS + 0.4905... + :param epsilon: the epsilon value use in method 1 + :type epsilon: float + :param alpha: the alpha value use in method 6 + :type alpha: int + :param k: the k value use in method 4 + :type k: int + """ + self.epsilon = epsilon + self.alpha = alpha + self.k = k + + def method0(self, p_n, *args, **kwargs): + """ + No smoothing. + """ + p_n_new = [] + for i, p_i in enumerate(p_n): + if p_i[0] != 0: + p_n_new.append(p_i) + else: + _msg = str( + "\nThe hypothesis contains 0 counts of {}-gram overlaps.\n" + "Therefore the BLEU score evaluates to 0, independently of\n" + "how many N-gram overlaps of lower order it contains.\n" + "Consider using lower n-gram order or use " + "SmoothingFunction()" + ).format(i + 1) + warnings.warn(_msg) + # When numerator==0 where denonminator==0 or !=0, the result + # for the precision score should be equal to 0 or undefined. + # Due to BLEU geometric mean computation in logarithm space, + # we we need to take the return sys.float_info.min such that + # math.log(sys.float_info.min) returns a 0 precision score. + p_n_new.append(sys.float_info.min) + return p_n_new + + def method1(self, p_n, *args, **kwargs): + """ + Smoothing method 1: Add *epsilon* counts to precision with 0 counts. + """ + return [ + ((p_i[0] + self.epsilon), p_i[1]) + if p_i[0] == 0 + else p_i + for p_i in p_n + ] + + def method2(self, p_n, *args, **kwargs): + """ + Smoothing method 2: Add 1 to both numerator and denominator from + Chin-Yew Lin and Franz Josef Och (2004) Automatic evaluation of + machine translation quality using longest common subsequence and + skip-bigram statistics. In ACL04. + """ + return [ + (p_i[0] + 1, p_i[1] + 1) + for p_i in p_n + ] + + def method3(self, p_n, *args, **kwargs): + """ + Smoothing method 3: NIST geometric sequence smoothing + The smoothing is computed by taking 1 / ( 2^k ), instead of 0, for each + precision score whose matching n-gram count is null. + k is 1 for the first 'n' value for which the n-gram match count is null/ + For example, if the text contains: + - one 2-gram match + - and (consequently) two 1-gram matches + the n-gram count for each individual precision score would be: + - n=1 => prec_count = 2 (two unigrams) + - n=2 => prec_count = 1 (one bigram) + - n=3 => prec_count = 1/2 (no trigram, taking 'smoothed' value of 1 / ( 2^k ), with k=1) + - n=4 => prec_count = 1/4 (no fourgram, taking 'smoothed' value of 1 / ( 2^k ), with k=2) + """ + incvnt = 1 # From the mteval-v13a.pl, it's referred to as k. + for i, p_i in enumerate(p_n): + if p_i.numerator == 0: + p_n[i] = 1 / (2 ** incvnt * p_i.denominator) + incvnt += 1 + return p_n + + def method4(self, p_n, references, hypothesis, hyp_len=None, *args, **kwargs): + """ + Smoothing method 4: + Shorter translations may have inflated precision values due to having + smaller denominators; therefore, we give them proportionally + smaller smoothed counts. Instead of scaling to 1/(2^k), Chen and Cherry + suggests dividing by 1/ln(len(T)), where T is the length of the translation. + """ + hyp_len = hyp_len if hyp_len else len(hypothesis) + for i, p_i in enumerate(p_n): + if p_i.numerator == 0 and hyp_len != 0: + incvnt = i + 1 * self.k / math.log( + hyp_len + ) # Note that this K is different from the K from NIST. + p_n[i] = incvnt / p_i.denominator + return p_n + + def method5(self, p_n, references, hypothesis, hyp_len=None, *args, **kwargs): + """ + Smoothing method 5: + The matched counts for similar values of n should be similar. To a + calculate the n-gram matched count, it averages the n−1, n and n+1 gram + matched counts. + """ + hyp_len = hyp_len if hyp_len else len(hypothesis) + m = {} + # Requires an precision value for an addition ngram order. + p_n_plus1 = p_n + [modified_precision(references, hypothesis, 5)] + m[-1] = p_n[0] + 1 + for i, p_i in enumerate(p_n): + p_n[i] = (m[i - 1] + p_i + p_n_plus1[i + 1]) / 3 + m[i] = p_n[i] + return p_n + + def method6(self, p_n, references, hypothesis, hyp_len=None, *args, **kwargs): + """ + Smoothing method 6: + Interpolates the maximum likelihood estimate of the precision *p_n* with + a prior estimate *pi0*. The prior is estimated by assuming that the ratio + between pn and pn−1 will be the same as that between pn−1 and pn−2; from + Gao and He (2013) Training MRF-Based Phrase Translation Models using + Gradient Ascent. In NAACL. + """ + hyp_len = hyp_len if hyp_len else len(hypothesis) + # This smoothing only works when p_1 and p_2 is non-zero. + # Raise an error with an appropriate message when the input is too short + # to use this smoothing technique. + assert p_n[2], "This smoothing method requires non-zero precision for bigrams." + for i, p_i in enumerate(p_n): + if i in [0, 1]: # Skips the first 2 orders of ngrams. + continue + else: + pi0 = 0 if p_n[i - 2] == 0 else p_n[i - 1] ** 2 / p_n[i - 2] + # No. of ngrams in translation that matches the reference. + m = p_i.numerator + # No. of ngrams in translation. + l = sum(1 for _ in ngrams(hypothesis, i + 1)) + # Calculates the interpolated precision. + p_n[i] = (m + self.alpha * pi0) / (l + self.alpha) + return p_n + + def method7(self, p_n, references, hypothesis, hyp_len=None, *args, **kwargs): + """ + Smoothing method 7: + Interpolates methods 4 and 5. + """ + hyp_len = hyp_len if hyp_len else len(hypothesis) + p_n = self.method4(p_n, references, hypothesis, hyp_len) + p_n = self.method5(p_n, references, hypothesis, hyp_len) + return p_n diff --git a/nbs/data_processing.ipynb b/nbs/data_processing.ipynb index 4cd8119..45fff5c 100644 --- a/nbs/data_processing.ipynb +++ b/nbs/data_processing.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 4, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -16,25 +16,25 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 2, "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "File exists: ../data/repo_infos.csv\n" ] }, { - "output_type": "execute_result", "data": { "text/plain": [ "'../data/repo_infos.csv'" ] }, + "execution_count": 2, "metadata": {}, - "execution_count": 5 + "output_type": "execute_result" } ], "source": [ @@ -49,9 +49,18 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/nathan/nathan_venv/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3441: DtypeWarning: Columns (3) have mixed types.Specify dtype option on import or set low_memory=False.\n", + " exec(code_obj, self.user_global_ns, self.user_ns)\n" + ] + } + ], "source": [ "our_repos = pd.read_csv(data_path/\"repo_infos.csv\", parse_dates=True)\n", "eleuther_repos = pd.read_csv(\n", @@ -61,69 +70,206 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 4, "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ - "array([ 28043.1 , 70708.05, 338060.77])" + "array(['GNU General Public License v3.0', 'Other',\n", + " 'GNU General Public License v2.0', 'MIT License', 'The Unlicense',\n", + " 'GNU Lesser General Public License v3.0', 'Apache License 2.0',\n", + " 'BSD 2-Clause Simplified License',\n", + " 'BSD 3-Clause New or Revised License',\n", + " 'GNU Lesser General Public License v2.1',\n", + " 'Open Software License 3.0',\n", + " 'Do What The F*ck You Want To Public License',\n", + " 'Microsoft Public License', 'zlib License',\n", + " 'GNU Affero General Public License v3.0',\n", + " 'Mozilla Public License 2.0',\n", + " 'Creative Commons Attribution Share Alike 4.0 International',\n", + " 'Boost Software License 1.0', 'European Union Public License 1.2',\n", + " 'ISC License', 'Creative Commons Zero v1.0 Universal',\n", + " 'Eclipse Public License 2.0', 'Eclipse Public License 1.0',\n", + " 'Creative Commons Attribution 4.0 International',\n", + " 'PostgreSQL License', 'BSD Zero Clause License',\n", + " 'Microsoft Reciprocal License', 'BSD 3-Clause Clear License',\n", + " 'Artistic License 2.0', 'SIL Open Font License 1.1',\n", + " 'University of Illinois/NCSA Open Source License',\n", + " 'European Union Public License 1.1',\n", + " 'Educational Community License v2.0',\n", + " 'Universal Permissive License v1.0', 'Academic Free License v3.0',\n", + " 'BSD 4-Clause Original or Old License', 'MIT No Attribution',\n", + " 'ODC Open Database License v1.0'], dtype=object)" ] }, + "execution_count": 4, "metadata": {}, - "execution_count": 21 + "output_type": "execute_result" } ], "source": [ - "np.percentile(our_repos[\"size\"].values, [90, 95, 99])" + "our_repos.license.unique()" ] }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 5, "metadata": {}, "outputs": [ { - "output_type": "execute_result", + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/nathan/nathan_venv/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3441: DtypeWarning: Columns (3) have mixed types.Specify dtype option on import or set low_memory=False.\n", + " exec(code_obj, self.user_global_ns, self.user_ns)\n" + ] + }, + { "data": { "text/plain": [ - "(544340, 517122)" + "" ] }, + "execution_count": 5, "metadata": {}, - "execution_count": 22 + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAIWCAYAAABJMsq1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAC3iElEQVR4nOydd5glRdXGf2eXHJYMkpMEAckiCH4kERAUVIIEQUBBCaKoiBEFkSBBQUFRoqIkUXKSHCTssgtLZmGJkiSbQPD9/jjVO3V7OtSduTuz7Nb7PP3c29XVVae7q/tUnWiSyMjIyMjIqMOI4SYgIyMjI2PKRmYUGRkZGRmNyIwiIyMjI6MRmVFkZGRkZDQiM4qMjIyMjEZMN9wE9BrzzjuvllhiieEmIyMjI+NdhTFjxvxd0nxVx6Y6RrHEEkswevTo4SYjIyMj410FM3ui7lgWPWVkZGRkNCIzioyMjIyMRmRGkZGRkZHRiMwoMjIyMjIakRlFRkZGRkYjMqPIyMjIyGhEZhQZGRkZGY3IjCIjIyMjoxGZUWRkZGRkNGKq88wusMRBl/Yre/yILYaBkoyMjIx3N/KKIiMjIyOjEZlRZGRkZGQ0IjOKjIyMjIxGZEaRkZGRkdGIzCgyMjIyMhqRGUVGRkZGRiMyo8jIyMjIaERmFBkZGRkZjciMIiMjIyOjEZlRZGRkZGQ0IjOKjIyMjIxGZEaRkZGRkdGIzCgyMjIyMhqRGUVGRkZGRiMyo8jIyMjIaERmFBkZGRkZjciMIiMjIyOjEZlRZGRkZGQ0IjOKjIyMjIxGtDIKM1vUzK4zs/vN7D4z2z+U/8DMnjGzcWH7WHTOt8xsgpk9ZGabRuWbhbIJZnZQVL6kmd0eys8xsxlC+Yxhf0I4vkRPrz4jIyMjoxUpK4q3ga9JWgFYG9jHzFYIx46TtGrYLgMIxz4DrAhsBpxoZiPNbCTwC2BzYAVgh6idI0Nb7wVeAfYI5XsAr4Ty40K9jIyMjIwhRCujkPSspLvC/zeAB4CFG07ZCjhb0puSJgITgLXCNkHSY5LeAs4GtjIzAzYCzg/nnwFsHbV1Rvh/PrBxqJ+RkZGRMUToSkcRRD+rAbeHon3N7B4zO9XM5gplCwNPRac9HcrqyucBXpX0dqm8o61w/LVQv0zXnmY22sxGv/jii91cUkZGRkZGC5IZhZnNBvwR+Iqk14GTgKWBVYFngWMmB4EpkHSypDUlrTnffPMNFxkZGRkZUyWSGIWZTY8zibMkXQAg6XlJ70j6H/BrXLQE8AywaHT6IqGsrvwlYE4zm65U3tFWOD5HqJ+RkZGRMURIsXoy4BTgAUnHRuULRtU+Cdwb/l8EfCZYLC0JLAPcAdwJLBMsnGbAFd4XSRJwHbBNOH9X4MKorV3D/22Aa0P9jIyMjIwhwnTtVVgX+Cww3szGhbJv41ZLqwICHgf2ApB0n5mdC9yPW0ztI+kdADPbF7gSGAmcKum+0N43gbPN7EfAWJwxEX5/a2YTgJdx5pKRkZGRMYRoZRSSbgaqLI0uazjnMOCwivLLqs6T9Bh9oqu4/D/Atm00ZmRkZGRMPmTP7IyMjIyMRmRGkZGRkZHRiMwoMjIyMjIakRlFRkZGRkYjMqPIyMjIyGhEZhQZGRkZGY3IjCIjIyMjoxGZUWRkZGRkNCIzioyMjIyMRmRGkZGRkZHRiMwoMjIyMjIakRlFRkZGRkYjMqPIyMjIyGhEZhQZGRkZGY3IjCIjIyMjoxGZUWRkZGRkNCIzioyMjIyMRmRGkZGRkZHRiMwoMjIyMjIakRlFRkZGRkYjMqPIyMjIyGhEZhQZGRkZGY3IjCIjIyMjoxGZUWRkZGRkNCIzioyMjIyMRmRGkZGRkZHRiMwoMjIyMjIakRlFRkZGRkYjWhmFmS1qZteZ2f1mdp+Z7R/K5zazq83skfA7Vyg3MzvezCaY2T1mtnrU1q6h/iNmtmtUvoaZjQ/nHG9m1tRHRkZGRsbQIWVF8TbwNUkrAGsD+5jZCsBBwDWSlgGuCfsAmwPLhG1P4CTwjz5wMPBBYC3g4OjDfxLwhei8zUJ5XR8ZGRkZGUOEVkYh6VlJd4X/bwAPAAsDWwFnhGpnAFuH/1sBZ8pxGzCnmS0IbApcLellSa8AVwObhWOjJN0mScCZpbaq+sjIyMjIGCJ0paMwsyWA1YDbgQUkPRsOPQcsEP4vDDwVnfZ0KGsqf7qinIY+ynTtaWajzWz0iy++2M0lZWRkZGS0IJlRmNlswB+Br0h6PT4WVgLqMW0daOpD0smS1pS05nzzzTc5ycjIyMiY5pDEKMxsepxJnCXpglD8fBAbEX5fCOXPAItGpy8SyprKF6kob+ojIyMjI2OIkGL1ZMApwAOSjo0OXQQUlku7AhdG5bsE66e1gdeC+OhK4KNmNldQYn8UuDIce93M1g597VJqq6qPjIyMjIwhwnQJddYFPguMN7NxoezbwBHAuWa2B/AEsF04dhnwMWAC8C9gNwBJL5vZocCdod4hkl4O//cGTgdmBi4PGw19ZGRkZGQMEVoZhaSbAas5vHFFfQH71LR1KnBqRfloYKWK8peq+sjIyMjIGDpkz+yMjIyMjEZkRpGRkZGR0YjMKDIyMjIyGpEZRUZGRkZGIzKjyMjIyMhoRGYUGRkZGRmNyIwiIyMjI6MRmVFkZGRkZDQiM4qMjIyMjEZkRpGRkZGR0YjMKDIyMjIyGpEZRUZGRkZGIzKjyMjIyMhoRGYUGRkZGRmNyIwiIyMjI6MRmVFkZGRkZDQiM4qMjIyMjEZkRpGRkZGR0YjMKDIyMjIyGpEZRUZGRkZGIzKjyMjIyMhoRGYUGRkZGRmNyIwiIyMjI6MRmVFkZGRkZDQiM4qMjIyMjEZkRpGRkZGR0YjMKDIyMjIyGpEZRUZGRkZGI1oZhZmdamYvmNm9UdkPzOwZMxsXto9Fx75lZhPM7CEz2zQq3yyUTTCzg6LyJc3s9lB+jpnNEMpnDPsTwvElenbVGRkZGRnJSFlRnA5sVlF+nKRVw3YZgJmtAHwGWDGcc6KZjTSzkcAvgM2BFYAdQl2AI0Nb7wVeAfYI5XsAr4Ty40K9jIyMjIwhRiujkHQj8HJie1sBZ0t6U9JEYAKwVtgmSHpM0lvA2cBWZmbARsD54fwzgK2jts4I/88HNg71MzIyMjKGEIPRUexrZvcE0dRcoWxh4KmoztOhrK58HuBVSW+XyjvaCsdfC/X7wcz2NLPRZjb6xRdfHMQlZWRkZGSUMVBGcRKwNLAq8CxwTK8IGggknSxpTUlrzjfffMNJSkZGRsZUhwExCknPS3pH0v+AX+OiJYBngEWjqouEsrryl4A5zWy6UnlHW+H4HKF+RkZGRsYQYkCMwswWjHY/CRQWURcBnwkWS0sCywB3AHcCywQLpxlwhfdFkgRcB2wTzt8VuDBqa9fwfxvg2lA/IyMjI2MIMV1bBTP7A7ABMK+ZPQ0cDGxgZqsCAh4H9gKQdJ+ZnQvcD7wN7CPpndDOvsCVwEjgVEn3hS6+CZxtZj8CxgKnhPJTgN+a2QRcmf6ZwV5sRkZGRkb3aGUUknaoKD6loqyofxhwWEX5ZcBlFeWP0Se6isv/A2zbRl9GRkZGxuRF9szOyMjIyGhEZhQZGRkZGY3IjCIjIyMjoxGZUWRkZGRkNCIzioyMjIyMRmRGkZGRkZHRiMwoMjIyMjIakRlFRkZGRkYjMqPIyMjIyGhEZhQZGRkZGY3IjCIjIyMjoxGZUWRkZGRkNCIzioyMjIyMRmRGkZGRkZHRiMwoMjIyMjIakRlFRkZGRkYjMqPIyMjIyGhEZhQZGRkZGY3IjCIjIyMjoxGZUWRkZGRkNCIzioyMjIyMRmRGkZGRkZHRiMwoMjIyMjIakRlFRkZGRkYjMqPIyMjIyGhEZhQZGRkZGY3IjCIjIyMjoxGtjMLMTjWzF8zs3qhsbjO72sweCb9zhXIzs+PNbIKZ3WNmq0fn7BrqP2Jmu0bla5jZ+HDO8WZmTX1kZGRkZAwtUlYUpwOblcoOAq6RtAxwTdgH2BxYJmx7AieBf/SBg4EPAmsBB0cf/pOAL0TnbdbSR0ZGRkbGEKKVUUi6EXi5VLwVcEb4fwawdVR+phy3AXOa2YLApsDVkl6W9ApwNbBZODZK0m2SBJxZaquqj4yMjIyMIcRAdRQLSHo2/H8OWCD8Xxh4Kqr3dChrKn+6orypj34wsz3NbLSZjX7xxRcHcDkZGRkZGXUYtDI7rATUA1oG3IekkyWtKWnN+eabb3KSkpGRkTHNYaCM4vkgNiL8vhDKnwEWjeotEsqayhepKG/qIyMjIyNjCDFQRnERUFgu7QpcGJXvEqyf1gZeC+KjK4GPmtlcQYn9UeDKcOx1M1s7WDvtUmqrqo+MjIyMjCHEdG0VzOwPwAbAvGb2NG69dARwrpntATwBbBeqXwZ8DJgA/AvYDUDSy2Z2KHBnqHeIpEJBvjduWTUzcHnYaOgjIyMjI2MI0cooJO1Qc2jjiroC9qlp51Tg1Iry0cBKFeUvVfWRkZGRkTG0yJ7ZGRkZGRmNyIwiIyMjI6MRmVFkZGRkZDQiM4qMjIyMjEZkRpGRkZGR0YjMKDIyMjIyGpEZRUZGRkZGIzKjyMjIyMhoRGYUGRkZGRmNyIwiIyMjI6MRmVFkZGRkZDQiM4qMjIyMjEZkRpGRkZGR0YjW6LFTM5Y46NKO/ceP2GJAdTIyMjKmZuQVRUZGRkZGIzKjyMjIyMhoRGYUGRkZGRmNyIwiIyMjI6MRmVFkZGRkZDQiM4qMjIyMjEZkRpGRkZGR0YjMKDIyMjIyGpEZRUZGRkZGIzKjyMjIyMhoRGYUGRkZGRmNyIwiIyMjI6MRmVFkZGRkZDQiM4qMjIyMjEYMilGY2eNmNt7MxpnZ6FA2t5ldbWaPhN+5QrmZ2fFmNsHM7jGz1aN2dg31HzGzXaPyNUL7E8K5Nhh6MzIyMjK6Ry9WFBtKWlXSmmH/IOAaScsA14R9gM2BZcK2J3ASOGMBDgY+CKwFHFwwl1DnC9F5m/WA3oyMjIyMLjA5RE9bAWeE/2cAW0flZ8pxGzCnmS0IbApcLellSa8AVwObhWOjJN0mScCZUVsZGRkZGUOEwTIKAVeZ2Rgz2zOULSDp2fD/OWCB8H9h4Kno3KdDWVP50xXl/WBme5rZaDMb/eKLLw7mejIyMjIyShhsKtT1JD1jZvMDV5vZg/FBSTIzDbKPVkg6GTgZYM0115zs/WVkZGRMSxjUikLSM+H3BeBPuI7h+SA2Ivy+EKo/Aywanb5IKGsqX6SiPCMjIyNjCDFgRmFms5rZ7MV/4KPAvcBFQGG5tCtwYfh/EbBLsH5aG3gtiKiuBD5qZnMFJfZHgSvDsdfNbO1g7bRL1FZGRkZGxhBhMKKnBYA/BYvV6YDfS7rCzO4EzjWzPYAngO1C/cuAjwETgH8BuwFIetnMDgXuDPUOkfRy+L83cDowM3B52DIyMjIyhhADZhSSHgNWqSh/Cdi4olzAPjVtnQqcWlE+GlhpoDQOFZY46NKO/ceP2GKYKMnIyMjoPbJndkZGRkZGIzKjyMjIyMhoRGYUGRkZGRmNyIwiIyMjI6MRmVFkZGRkZDQiM4qMjIyMjEZkRpGRkZGR0YjMKDIyMjIyGpEZRUZGRkZGIzKjyMjIyMhoRGYUGRkZGRmNyIwiIyMjI6MRmVFkZGRkZDQiM4qMjIyMjEZkRpGRkZGR0YjMKDIyMjIyGpEZRUZGRkZGIzKjyMjIyMhoRGYUGRkZGRmNGHDO7Ix0lHNqQ/+82il1MjIyMoYDeUWRkZGRkdGIzCgyMjIyMhqRGUVGRkZGRiMyo8jIyMjIaERmFBkZGRkZjciMIiMjIyOjEdk89l2EsgltlflsSp2MjIyMbpAZxTSIzEwyMjK6wRTPKMxsM+BnwEjgN5KOGGaSpnr0ykEwM6SMjKkDUzSjMLORwC+ATYCngTvN7CJJ9w8vZRm9QmYmGRlTPqZoRgGsBUyQ9BiAmZ0NbAVkRjENoRe6mbxKysgYOEzScNNQCzPbBthM0ufD/meBD0rat1RvT2DPsLsc8FB0eF7g7y1dTUl1piRaUupMSbT0qs6UREtKnSmJlpQ6UxItKXWmJFp6Vafq+OKS5qusLWmK3YBtcL1Esf9Z4OddtjH63VRnSqJlaqR3arymKYmWTO+7o05KG/E2pftRPAMsGu0vEsoyMjIyMoYIUzqjuBNYxsyWNLMZgM8AFw0zTRkZGRnTFKZoZbakt81sX+BK3Dz2VEn3ddnMye+yOlMSLSl1piRaelVnSqIlpc6UREtKnSmJlpQ6UxItvaqT0sYkTNHK7IyMjIyM4ceULnrKyMjIyBhmZEaRkZGRkdGIzCgGCTOb28zmHm46MjKmBeT3bXgw1ekozGwB4MfAQpI2N7MVgHUknRKOzwFsBiwcTnkGuFLSq1EbI4C1Jd1a08diwFHAxsCrgAGjgGuBgyQ9XqJnUl+Sni+1tSxwErCApJXMbGXgE5J+FGj9FrA1MD8g4AXgQuCIEs2zAF8DFpP0BTNbBlhO0iWp191GT1RncWAZSX8xs5mB6SS9EY4tj3vPx/1cJOmBqntZ6ns3SaeZ2cXhWish6ROl82rpieqsF+qcZmbzAbNJmhiONT6j1Dot19b4cZP0clS37Vm2janWZ5A6HmquZTZJ/4j22965xjEV6F0YuL3U7maSrujmfauhdzdJp0X78wFfAJYgMuiRtHsivcn3rmncheO1z9rMpgP2AD4JLBT1dSFwiqT/Ru0YsBOwlKRDwj17D3BXahuN6Mbp4t2wAZcD2wF3h/3pgPHh/y7Ao2EQfDdsvwxlu5TaGdvQx1+B7YGRUdlI3Hz3trC/KnAb8ADwl7A9GMpWj867AQ9VMjYquzf8Xgl8E3hPdOw9oeyqEk3nAAdG584CjBvAddfSE/5/ATdbfjTsLwNcE/5/ExgHHATsHLaDirKEZ/dk+F2/aSudU0tPVOdg4GLg4bC/EHBLyjNKrPMG8HrF9gbweqgzEXgs/Ja3x1KeZSItrc+gm/HQ9JxS3rmEMf5lPJLCn4HHga2iOnelvm9d0nsrcGSg+dPFlkhvN+9S5bjr4r39Q+hnbdyHbJHw/yTgnFI7J+Fx8R4I+3Ph70VyG433MLXiu2UD7gy/8UMubvxDwJwV58xVPMyo7OgwgKyi/iMN/T9S9ImHGykfX7t4oVLobejnodL+6Ip2ihe3m+uupSe6rhlKxwtG/DAwfUU/M0T35Z6abTzw5gCedy09pTpWqnNPyjNKfY49HsOVzzKR3pRn0DoegANqtq8BL3c5ZprG+Hh8lg0+wx8N7B/XJ+19Sx5XMW01bfbqG1I57lKedfEsG2gs93VXVTvdtNG0TdF+FAPEP81sHoLowszWBl4Lx4xqkcb/wrEYe+Evxjtm9u/iXEmjgDFmdiJwBvBUqL8osCswNuzPKun2ckeSbjOzWaOiv5vZ0hG92wDPhmNPmNmBwBkK4oWwzP9c1G+Bt4LYpWhnaeDNAVx3Ez3gL91bvtKFsDwu2v4fPmt6otTmguEYwALApsArpTqGz/T6CnwZfjiwAjBTUS5pqUR6CrwlSWZWXFNx/1OeUepzjOmev0Tvk6Xjc+Ern7jOjTG9Nc9yrgRaUp5Bynj4MfAT4O2KemXdZtM7B81jaoSCuEnS42a2AXB+ECcWtKS8b8njCrjEzD4m6bKKa2ujt5t3qW7cddRpeG9fNrNtgT9K+l84PgLYtuI6/xuibRftzBdo6qaNeqRylHfLBqyOixVeC78PAyuHY7vSt2z8dtiKZePnuuhjBuBLwBX4jGU8vvzeG5gx1DkeuBRfMn8obNuHsp9HbS2FixD+hcsObwaWUN8s5UhcvPBy2B4IZXOXaNoEXzK/CJyFL+M36Pa6m+gJx48K5z8Y+vwTcFg4thkwIdyLk8N2RSjbLNQ5BViv5r7+vrR/My6XvgdYHPgBcEipTi09UZ2vA7/CRT9fwEUZ+6U8o9TnGOp+AngE+CcuUvofcF+pzufDeHkFuA74N3BtyrNMpDflGbSOB/zjukbNc3oq9Z1LGOPXAquW2psOOBN4p4v3rZtx9UZ4Nv8J/yeJCBPo7eZdqhx3Xby3S+CiqRfDPX0Y11GeAyxZamcnPGrF08Bh+Mpn227aaNqmOmU2TJpVLodz+IfUqfSZC595lBVRr5TaKJRDS0o61MwWBRaUdEcXdGxOtVKx30wmzDZGqKSE7QZhVrc2ft23Sfp7dCzputvoCbORPYCPhn6uxAM3Kjq+VqmfOyW9M4DrGSNpDTMbL+n9cVkqPVG9TeI6kq4O5a3PKPU5mtndwEbAXyStZmYbAjtL2iOqMx74AP58Vg2K3B9L+lSprcpnmUhv6zNoGw9mthzwUjyGonMXUH8Feu07F9XpN6bMbBHgbUnPVdRfV9It5fKhQsM7kPwu1Y27Up3a97ZUB0kvNdC7PD6xMlxP90DpeGsbtUjlKO+WDeeis4f/3wUuIFIed9FOpXIo4bwtu+xnf9yCw4Df4FYKH004b/XS/rq4mARcgXksHja42+tOpgeYm2jmOBme5a24mOMCYF/ccqNJb1NJDzArQRGKf8w+QYUcvwf0FvLmu/EPDJT0GPTJv8fRNxsurzp68iyHamt75wY6xhP77up9i877BK6HPLrcRq/oTRl3A33WRAYuYX/paDxtgBsJzNlNG411h3uQ9XojKIuA9fCl/Ra42V3beSeX9iuVQwnt/DChzp7lNvEZyp+AFYu+W9r4dfm6w8BeJQzsfYAbBnDdjfQA14eXaG5cvHI7cFxCP5d0Wwefec+GW2qcFj5Aa5fqtNIDjMGtSRYOdc4Dzkp9Rql1cHHFbMAJuLXJz4BbS3X+BMyJi9FuxM0ULxvss0ykN+UZnJxQp3zdje/cIMZ4Cr0p71t5XB0BXAPsHrargcN7QG/5XWodd4N4by8t7Y/DRXbvxcVOPymPq7Y2GuumVny3bPRZShwO7BiXtZy3Rmn/dtwEr2AY86W0k0jjXvFACb8/Az6ZSm9FmwWd3wf2iMu6vO5GeqL7+/niJaVkyVHTz4KDqYOvLEY1PO9aeqJ7sx9wYPg/LvUZpdYhzCDDC7srPqubp+H89fFZ5gyDfZaJ9KY8g0q9RMt1F8+g8p0b6BhPoTdlK7eDf5xHRPsj4zEzCHrL71LruBvoe1vRd9HOgQQ9yEC+I7Xt96qhKWUDLqFPgTQnMCMDMGOkRjkUHV8L+ED4vwJuIfWxAfRzGnAVrgSdBZgdGBMdfw9hiYgzq08BK1a0cwPunPdwOGcEJTPRHtEzHreguSq6/lZGMcBn+Xt8tTArnv72aeAbpTqt9OCWMevg/gYrFucN4xgdiVsmLVZsk+NZDuH1NL5zbWMqsY+evG/F+CAyBsFXo/f0kt7UcTfQZ00wKY72bwd2AO4lKKmJ/J9S2misO9yDbDIM2lnCx3SZsL8gQb4YXtC9gEOBdUvnfbeireXxpeC+wPui8oPDwx+Nz6KuBb6HixK+Uzp/44qHuln0fwRuNTJn2J+HPiutvfAl6+O41cftuHXHQ4TZR9TOe8LL8+GwvxjBASgcK3Qu8+Bij/HAufSfbdXSE/a3DS/aiWF/Kdz0DvyjfjjwW8LMMjrvxNQ60f648LsTcAwwPf2ZQC09UZ3/w5n+N6M6x7eMo90Sxtpupf1P4R+X1yg53EV19sNTUN5HnwVP+ZqanmXjmArHL8ctoZYGTse9me8gjGG6fA8Srrv2nUsZUw39XN7N+9bWTrS/A24+fDpucjsR2D7xnUy+dynjrulZt1xT2YlwBdwqboewv2TRb2obTdvUavU0Ererjt3znzSz3+CD+g48reoNkg4I59wlafXEdsbjXrIzAs8Bi0h6PdhD3y5pZTP7Ms5kHgh195d0YVVfZrYwbv4Z93Nj6OeDwMz4wH6vpOeC1cV1klZNvB9X4B+OWYEdcTO83+OhQT4iaatS/Up6Evr5I/6hvA2X/f4XZwZvFtecUidq7z783v0eN/+8wczulrRKynUPBmb2pKTFuqljZhOAj6shXEmo80ENwPIkZUyZ2Y24fHo2XBb/TdwUckvgK5I27vY9aLvuUFb5rkTH68Z4XV+G6xYWTHzfWtsp0bsgrgMDuEMlq6sGegd177qBmR1QdwhnkK0xr3rRBkzhiYsGAjPbD5+BPE+fg5GAlYG1JK0c6v0cONHMLsBnGNbQzjv0OdqsjJvzvQP8y8welfQ6gKR/m1nR5xdwmeU/zGwJ3IloCUk/i/sysyNxW/j7Qz8FvTcC/5X0r6if50I/rxROPFE7n8L9K+YP7ccOggtIOiHU21vSkeG0E8xsj1I7TfS0xclZWtKnQ9Gfzew7wLVmFsdmSqlT4Jf4aupu4MbghPV6id7GuD2hzrK4TXtHHTzBfBUM/+hhZve01YnwfBOTCHiKTme0/g3XPEt8stA2pmaXdHFo51BJZ4fyi83sh+F/63vQzXW3vHNtY+pOXPxSdlYDF2NB2vvW2o6ZLS/pwYipPB1+FzKzhSTdlUBvN9+QynEnaaOoTtN7m+z0aGbr4pKCgrkVY2ah1DaaMNUxCty0bbmaGdsMxR9JbwN7mtn38aXsbF2085aZzRI+4rFN/xz0vSgpHqfgs/rlJL1Jf8jMppfbpG8R9TMT/R/yUdTPZuO6ZzYca6MH3ErnJtzCp+wbMaOZjVDwAJV0mJk9g79gs3VRp/AFeF7SwlHZk8CGXdBT4Dyc6fymVOcy2r15u/H4HW1m5+BxiybdP0kXRHUeA643s0tLdY6N6lQ+SzO7L2FMjYxOiduEvvGf8h50c91N7wo0j6kHcOX4I+UDZlZ4Yae8byntHADsiYsxyxDuA9NGbzffkLpxF6Ppvb0L+LOkMRXX9PlS0SnAV3FLq7ivi7toox6pMqp3y4ab501Xc+x3RPqBqPzz+Ow9tZ0Za8rnBd4f/rd6nIayy6lRKuHyyn404OZ2HymV3VLVRjh2SFUfuCnd+aWyWnrC8XENx44q0xXKN6MvJk9rnahsdMLzrqUnqlOpiCTBmzelTrR/WsV2aqnOwVVbyrNMGVO4/LzuWf809T3o8rpr35WEMb4N/lGuOrZ1+E1531rbifZnqqgzUyK93XxDWhXgLe/tcsC8NccWKO1XugB000bTNtXpKMzsFPzmNM3Yms4vZHortrVjZscDZ6siHHmqx2mQ2a+C23XH/Xy5RNM5kp5poPtnuGLsz9TMZs1spFo8pNvoMbMf4b4BdXFykpBIyxG44vccPCxGQUsckruVHjP7AR624E90XtPLdedMbpjZbIGGf1Qcq3yWuFx8ivNibnvnUsZ4Yj+171uX7VTpIyeV9ZDeH9Ay7hLf29UVxGINfR2BryYvKLVTiNNa22hsfypkFAdXlUv6YVV56vl9zeiQqO6uuCxzOXwwnC1pdKm9xsEd2qjq6IwSTdvhsZ7OAc5T/xAKp9XQG8vqn8Tj5ZyDxxfq9/Db6DGzN3Cl+Fthi2WqyUikZWLNNS0V1WmlJ6WdXiBMDk7AvW3BRWL7S3o6qrMSbvFVKBH/jlu53BfVaX2WUxLa3rmUMZ7YT+v71nL+e/DV+O9wo45CXDcK+KWk5VPpDTqcs/EJ3KM1/aWM35T39jqcmZwf+ru3oq/ratrZKLWNRqQuPd5tGzDLIM/fNqUslM+NK1Svob/4ZFdcFv4oHi5gzYrzZ6Zm2VyqtzLu0/EgHk+o63uCM5wLcCXxz6kQL6TSM9jnU0HLh4d73Azymq4GdsPFQdPhUX6vLtW5Fdgw2t+Akvf2u3Vreud6Oaaa3reW83bFxWRvhN9iuwj4VDf04krjA3GdwJ240nqxgVxPIu3vwR04b8FNqlvNmHvZxrAPrslwQ9fBrRWKJDirULLPT2ynn3dkVVkoXwtXkE0ALq6pUzm4gY/jfhETw/6qeJC3uge9X3jQZdv7ZUPbRQKUlZsGAh67qkNfkkIPPgvbGfhe2F8UtwQZzDOro2UWPHbQyWF/GfrH5WmlJ6WdHo29cW1lVDh/lsu6fZbDvbW9c92M8cT+Wt+3lvM/3XK8K3rDeBro+O32vX0/viJ9q1S+AK5XKnxPVqDka9XWRuM9Ge5B1usNd0pblJoMbWG/+Lh8P+wvVnxcgM1x8cHzuANLsZ2O21vH7RyF+wRcgc8e52ygq3Jw4zOSOVro3RuPaXQfbgK3QkX7N9CQmS4qWx84Ebe+Obf80rTRQ0KwxPCCfI8Qj6rqBUmkpTb7V5f0pLSzOEHJjs8mZ6+gt7EO/sLvjMuKR4b/5Wx7fwr3ZomwfRf4U7fPMoGW1g8HDe9Bl301vnNtYyp1zJDwviW2czAeMqNj64be6L4Uq4o7gK8NYNylPOv30eckez3ufDt/qU5blsHWNpq2If+QT+6NoP2nIZgfDR8XfDa0K+7Ys2/YvohHLp2r1M5e1FgUpA5u+lKnxvSWVwuHU7J2qeinMctY2H8c/1DtQIhYWdFOIz0kBEtMfEFSaKnN/tUlPY3tkJZONaXO4rgY40Vcifln+ofnmAufeNwVtp9VjKu2jHEptDR+ONregy77anzn2sZUF2Mm5X1Laedr0fYdPE/EqV3Se3t4fgfheaoHOn5T3tu/4ibICzVcd9uYaW2jaZsa/SieMrMPEXwQ8JtTtlH+oNyLdSxMcmAr7KPvx0NGzIB7DoPPtE7DY9pMgqRfJdDzKJ5ovl+M+YD7zGxHYKR5RrcvU7JTl/SthH7aMtOBhyF4vd+Z3dFTl0krxtKStjezHQL9/zKzsiNUCi1N2b+6oaetnX3wWd3tgd5HzLPU0U0dSU/gQf5qIc9Z0GY90/YsU+idV9K5ZvatUOdtMytbmTW9B9301fbOtY5xEsZM4vuW0k6HH4WZHY3nMemG3l0kPdRCS8r4bX1vJa3T0g+0ZBlMbKMWUyOj+CI+S1sYTyhyFT7YYzR9XI7CHWcWV0hYYmaj6Itdv383xCQM7v3wWc2beKiKK4EfddNHwD54NrPlgwPbRFysENPS9mFOoed4fCUwv5kdhtuvf7fURusLkkjLD/CV2KJmdhZuTfS5Up0Ueg5uaSclnWptHTM7UNJRZnZCxXlI+rKZ/VTSV8zs4po6MYNpe5Yp9LalJ4U0JpvSV9s7lzLGUz6qKRhIO7PgoeyT6U1gEtA+7iDhvU3EAfhqdmkzuwUPILrNANqpxFRnHpsCM9sJN7NbHQ8Ktg2uQDrPzB4BllXpxoQX6kFJyww5wV3AepApL6GPtkxam+Af6xXwj8a6eJrI6wfQV0r2r0Z62toxs6PwwHm74B+JvYH7JX0npY6ZfVzSxU1mlWa2hqQxZrZ+TZ0bKmiuy7CWQu/quK5tJTyi6HzANpLuierUvgfd9NUL9GrMpLRjHjuqeL9H4vfmEEk/H9xVVNLTOn5DvV5kuGzNMjhgDEReNSVv+ICfM9qfi5J3bCiviwz7cEPb/Y7hyVp2C//no4s8tOGcqyvovbKi3uI0KxR/XNHOjwZw/xrpwQf97NH+KFyEUW5nHjzsyJa0yJUbaLmYBh1GKj24fmmOaH9OIm9dPIzJF/CQC+eH/1ZqI6VOq0k17ldRrrN/ab/xWabQEupNhzuOrkRNRr+696DL625857oY461jJuV9a2snvEvFtjAlr/JUehPGb+O4S3nWpbpN5sf7VLSzdzdtNF7LQE6akjcqknWUy2hIG4grIPuF+cWXgxeVyg7GP2YPh/2FqHDJbxrcifSmKBSr2rmrtJ9iCdNIDx5j36L9ERX9tKZ3TKRlfdwq6gn8I7UNpfALifSMSxknoXxuWkJg19Up91vzDKrqlJ9367NMoKU1JXDTe9BlX61jJuGaU8ZM6/uW0k44tnq43v2A1bq5nrCfYl3VOu5SnjXwIVpM/tv6SmmjaUuq9G7a8Eijc0X7c9Pf2mMcNWkD8RnG7bgJ2TFhuwE3f1u4oh2j2TqicXDjpnWLRfuLVwyUcbhyPe6nfE33EMXEwVcd5VzMKZYwjfTUDMjyNd8T7ssq1KR3TKElqjsS2AQ3oS3nd0iip6JObDp4Pe3pVGvrkGBSja+MLsaD7F0UbdfRn+k3PstEeltTApOQPjOxr8Z3rm1MdTFmxtH+vqW0833cTPSHYbubyHchkd4U66rGcdfFe5ti8j+ezgnTyNKYaW2jaZsaldnHAH81s/PwAbMN7s0c439yK5BP4XkOTogsP54BPmhmG+HLdvCX55qKvt6SJAshv4OcsYxPAqvhgxZJfzOz2aPj3wFuNrMiRPKH8QiXMVIUimcB10QhAXbDRQIxUixh2uh5zDwvwklhf2/cDyLG2+G+bAX8QtIpVgpnnkgLQTH5cTpl6TFS6BltZsfipqDgH484muYc8vwGnwfOlHSw9Q+z3VTnb3hSnU+U2n0Dj+gJbjXzLB7I7phSnXJfbc8yhd7iXm6Bz3ovNY+LFaP2Peiyr7Z3LmWMp4yZlPctpZ2dgFUk/Se0cwTOhIr7k0JvimVf27iDtPcWSU+Vmi+/K1cA55hZYTyzVyjrpo16pHKUd9OGizEKH4gq57Tb6TJtYE0/X6cvBeQXcFvl/Up1ihllYe8/K/1nQfPi8tQ6mepRwLfx0B2b4FY+h1XU25w+66xNK45fj8tvC1rWpiKRexM9eNz8s3E/gedxq5Cy888NtKR3TKEFX0E8jodq3pAoz3GX9MyKJ/EZTV+WtFmj4ynpVFPqVOoBBjiGa59lIi2tKYFJeA9S+kp859rGeMqYSXnfUtq5jk55/px4vLFu6L0Vn/0X43dp+jvkNo67Lt7b83HR0V14lsev43Gu4jojcCe688O2FzCymzYax2OvBvZwb8Co8Dt31VYxqI+ni7SBDf1ugi/ZjwY2qTheObiB5cPx1au2ikHQqrxMoHV1PPzHa+H3YfpSPCbTk9BPa3rHJlqiOpvGg30yjp2UdKopdZYJz+f+8LwfAx4r1Vkb1zf9Aw9i+A4lcVqP6G1MT5r6HjT1Rcs71+UYT0oJSvv71pRG9oRwvX/GzXhPx/2jnsZ1ON3QuwnOlF7EVwWPAxtMpvE5b+jjeXxC9DtgnqFsY6oxjzWzSyRtaR6xMb6oIppoTyOFhj5nBf4j6R0zWw43TbtcJbO0YLL30UDLlZKuNrOTJe1pLVEfK/qcG08FeU/Yv1nSeuZRVKuue1Tp/EoTOjP7taQv1NEDXKEWX4EqepvQQMtGkq4NIpH+xEgXTAbfhUHDzG7GdVLH4eKy3fBV0PejOqOBz+BMf03c7HRZSd/q9lkm0NOYnnSwaHvn8MCVXY/xhv6S3reG83dtqbJuN/TWmb6mjLtePWszO1fSdiWT37ivlVPaae1namEUTTCzhRXlcrCatIHdMhMzG4PLL+cCbsaXl29J2imq0/XgNrO1Jd0W7V+Py7+nw2WcL+ARR79a3UJtu9viH/w3zOy7+EzpR0qIU9/iKyBJZ0Z1P0V9esdWWszsh3JZ+Gn0hyTtnkJPqu+CpaVTTakzRtIaZjZe0vvjsqjOaElrmtk96kupOVbSalU0ViGRlsr0pPGHI+U9SOmrhsaOd67ieHmMp4yZlPettZ2BoILedXHl9T/NbGd8/P5M0hOp466Lvo/C9Sf/xvUOKwNflfQ7M1tQ0rPmWQ6r+nqirY0kIrpZvrxbN4JJWLT/IC4XnB+Xk89Dl0u50E4hn9wPODD8H1eqMwYXAyyMW42cB5zVJb1jw+/ngR+G//3kxAnt1FrC4GKK2q2ln6NL+xOosMlPpaUHz/vohDrnRP9vxT8u2wGfLrZS/dQ6I3Axxr64IcNDpTo34hZsZ+K6p69SEVG26Vkm0jKhbUynvAcpfaWMvYSxmTJmUt632naAc8PveFyc1rF1+y7RYl3VNu666Gtc+P0kblI+R+KYuWWwbRTb1Gj1VIWyNcJrki7vRbtmtg5uRVFYVows15FbROwBnCQXl4xra7e0P52ZLYi/rN14xJbbabKE+XhDO8I/fnXYDtfFFHhe1TmAU2kBwMxmxD9MS9A5mz2kpe0yPVWIY9/MIumbLfVT6uyPTwq+DByK52Aur3g+izOTfXEmsSh+jW2In2UKLU/RP2RHGSnvQUpfVSiPvbbjKWMm5X1ramf/8LtlSz+VfZf2U6yrqpASc6nf+x9+t8CTlr3W38CqEov1oI2Ok6d2lOVr15nZT6hJG9gF9sctLP4k6T4zWwqfHcdIGdxt9B6Cx5u5WdKdoZ9+SeQT2nkmmM9tAhwZPsQjACTtltBeHarMAs+hIb1jEy0RLsQ/dmPoLu5P+hvguMTMPqbm9K6tdSTdGf7+A9dPVNV5Ivz9D27Dn4r4WabQ+xhwvZk1pQROeQ9S+mqjN+V4yphJed9q25GLaEYCp0vasJuLqaD3DXPT7p2B/zOzEbg1US9Q7usSM3sQFxt9KYgD/9NlOwNtA5iKdBR1Sk38o7GrOmWdPVGuJdL1f/js9hZJR4bB/RXcwqSO3o0kVdmIV7V/QN0h4DuS5o7qzgJshpsLPhJWKe+XdFVUp3IWj2egq+vnbkmTgqo16Ra6pOVeSSvVXPfcVeUxPebxjurqXCJpwdBWSjrVlDrLAt+gT+ZfXPhGUZ2yXqCos1Tqs0yk5eCqhhSlBE55D5r6anvncKugpDGeMmZSkDj2rsHFqR0rrjrlcw2978HTqd4p6SYzWwy3ejozZdx1896G/ubGV4DvhHdnlKTnrMbgI7TzS0nztbVRc35nY1MRo2i0aFCX+XkT+1wWZwJLUPNhaDi3UtEVtXFDVLdWoVj3QYjqdMxarcUSxsyuoG8WHzvk7Iu/RFWzdWkAVmUJtJwMnCBpfMW5haVNLT01H8K4UrezykaY2d24z0fHvZM0JqrzIC5yKtd5qdtnmUjTLJL+1e15iW23WRE93nRQ3St1B/y+ldq5EHeCvRr4Z3Toj03npdKbMu4G8N5+iP7XfWYNY4zbmbSyrWuj6fxJ504tjKIbmNkCeDCuhSRtbmYr4DkjTumynZQPw6AHt5ndCtxU0U/jwK5oJ8USpnYW30U/y+Ke0gtIWsnMVgY+IelHUZ0UWu7Hw0tMxMUIVq7TC5gLa3fCnc4ONbNFgQUl3dFlnQ4Lp5q+bpf0wSGgdx1caTmbpMXMbBVgL0l7R3Va34OUvnqBxDGT+r61tdNquZdA72Sxrqrp67e4Q984+q5b6sIkfdBtKFHrPTVtdBFnqKWdMQl17sY9JtcC1ii2LvsZ16PrTrGEORkXAQ2mnxtoT++YQsviVdtkGA8pmd5S6vwADyGyIPXOnkfgDmPrMHCHxhRaUuID9SQLXo+eQcqYSXnfUtrZv+K8fmUt/bRaafXw3jwA3TvY9rKNaUWZXUZSnKEEXGxme+MhNWLF2ctRnbclndTvzO4wUIViGSmWMOsBnwuinYHO4meRdId1WlW8nUpLpH8oYvMLeFVhxE8GpGR6S6lTzFS/EZUJ92ae1E74XbNUpxvxSQotqD22T6+y4PUCKWMm5X1LaWdXPNFSjM9VlDUhxUqrV7gX9zgvZ6wcsjamGkZhZvsqPfFISvavFKR8GFIGdxv2B75tZrXKy0SkWMJs3mWbVUhJy9pEyxj66x9mC6KHz0t6vAc0xkjJ9NZYx9zq5SBJ59R1Es6/SNJxQ0BvSkrgXmXBq0U490hJbebKKWMm5X2rbcc8gN+OwJJmdlF0zuxA4/toZkeXriHFSqtXmBe438zuKPXVTWSBQbUx1egozOwuSXXWBuW6q9OS/auHdE2sKBZwHw0mhMkPsN56omjn2KhurSWM9bci6pjFm9kH8Bloh929mX0Mn13FcuKlcBHWh/Cw2hOBneMPfBMtddcS5MJ7StosKlsbD6ccp619n6TbK85dL1zXzZL+FB1LyfSWUme0pHilUHUNd0haq6XOj4GjJL0a9ucCvibpu13QMi8+Q/4IznCvwsUrL0V1Wt+DxL4a9QJmdpuktVuuuXXMpKCpHXPv5SXx4HwHRae9gTvclVcecbtPSlos2q+1rrJ6q6ei0iTz40Q9UaXhi6QbrN7qqahzQVsbTedPomFaZBSh/qDTBpqbmB2Ax67f0zwR+3KSLkk4t3hwn8KXhIUr/Q74h/erUd1ahWL0wV0O+ACe4wDcge4OSf3y71qFJYxVWxHNhutYPg+ciidfeqJ03uLAaapQzltCescqWppQfs5BJLJ6xNBGAKNLdU7EleJ/CEXb40mg9onqpKRTbUsBewTwdzxXwSRrmnj1aGbH4fb25Trxx2OsSiE9Kq67ld4UpLwHCdd9Az7L/1VBt0VGEWZ2Eh6Z4LzSNfebfTeNmW7et5Z2jlTJibCqrHT8KUmL1h0v1W2yepI6zY8vxwMTfkfSKuF5jFUIARPVWxwP8PiXcB9GysPfFAxrfpw5Xhv2N8TD/GzZ1kbKNU12RcxQbbgc8vWK7Q36J7tJShuY0GdK8pJZ8AxjJ4f9jkxY+Eet3O7o0n6K8vJGOlOCzg7cWKqzDl1mucIZ2RXl/kp1yiGuW9M7DpCW2Sru77gEeh6EflnwHoj2U9KpptSZWLGVo8deV7GVQ1zfQ3PiohRazqh4BqeW6rS+B4l93Rl+x1Y9F/xDWN7KtKSMmZT3LaWdqiyD91AfBXce4OlS/WWBayJaViZKfpS6td27sJ+S4fIqfPJY7C9IZwrj1jYa6ez2wqbUjZrUljV1xw3m/Oic0RUPuRzzv3Fw43LjpaL9JYk+YqHsroR+HqLz4zIj/eMMDSjLFR7LZkLD8Qml/X73svxyNtGCzxrL26H46uYLpXYuwENmTB+2/YE/l+pcQmQthVtPXRzTS39GUqa3tU4Px/I38aB3e4TtZkJso27obRvjKe9BYl+X46aXxTjdhpDitotrThkzKe9bbTu49eF44F90xnmaiIfgnojrzlIYfqt1VShbCbcs26XYSsevpz0vyzjaM1yWvxnlyVBrG03bVKPM7hIjzcwU7lZQuA3EkuMt8wxsRTtL0z/URFsmrK/iSt3H8KX94njSkRgpCsUzgTvM7E9hf2sGlimrA2Y2Gz7o/mJmh+GzpoIOw8NQXFs6baSZzSjpzVBvZpxxpdIye7kq8Bwuay47330RzzHw3VDvGkI2MuvztJ0deCAo8oRbHsV+AJPGQqDrf0EEQDd1UkQjKTJpuQf/Pbi4B+BQSVd2Se8IM5tL0iuh37npb7yS8h6k9LUPrhdY3syeIegFomueCWd4KwIzRW3FXtcpYyblfWtq5/c4U+uno1B3xiWQYF0VxMIb4Hk/LsMNRW7G39UCB+Di4qXN7BaCnqjUV0qGy2vM7Eo6xat/6bKNWkxNjOK89iqT0Jo2MBEHh/MWNbOz8MTunyvVaRzckq4IH5TlQ9GDxSCPcDxuNTV/+Fhvg38YJ0HSYUHe+eFQtJuksaV2ai1hrFopPhce3vzn+Ev2G2CC9QU1XBVfzn6+dF5KesdaWtSFB7KkF/D8DlU4OrGZlHSqKXVOwy22PhT2n8HHZSxDPz3UK4I7PoyvOjucPeVGA3UB+1JoSUkJnPIetPYl6THgIw16gd/i4r9N8bhlO9HfAitlzKS8b7XtyEN2vIbrAYFJ7+M+ZvYZSSuGd3Un/OMOHsr8fElvlfpJsdLaBhepjpW0W5gkdIT1lofVX59mPdENZvZtYGbz3DZ74/nX43b2NVdsF+//yYoMNlLaaMLUpMyuizsDgCIPxKDw3Iu+GdvVwG8kde1LYTXJS6Ljm+Af9RVwOeK6wOckXR+OF7PQxeWJgyoVdInK1vVwZdVpYdUxm6SJ0fFaSxjrb4Uk4CVczzE+amMp+nKJ3xc+ElX3ZXOi+1uaESdZ5TTBEhIXpbQT2pofZ8Yb0bcq+UpgQt3UKXJNjFWfUvduSatEde6U9IFSnXGSVrXEZDYptIR6K9Dnn3GtpPtLx1vfg8Tr3h9nfm8Av8YtpA5SiNtVXKuFHBxhYnCTSpZQbWMm1Gl831LaMbOF8Bn3jsD78RVGoVi/CNf3FVZ8a+Dv7ibA19VneZZi2XeHpLXM82hsGO7PA5KWj+rsg6cdeDXsz4VnHDwxqjMCX5FNSn6GP6fkj/eg20iVUU3pG25jXbtNpj4/iSefL/bnBLauqDcPHt63Kgd1ioIuRaF4MD5DeDjsL0QUj74H17p46Vo3xD/0BwAzDMPz/njTcw/Hbg6/b9Bi4NAjmlLyKF9PQt7yQdCQnBK4h30Wnt2b4ivfFYn0C/Tljb8Rl9nPS0nmn9hP0vvWcP6euPHAw3gSn5WBidHx66hOr/oRPF3qlRXHZo3fzdKxEwONX8SjPY/FLQTjOuMqzhtb0Uec/3okLvqKx3bjGG9qI+neTY6BM6VuDCJxSU17KQ+5cXCTqKCjXaE4Dp8pxO2UrX9aLWEarvV2XK4OLnL6O/C10OZvQnnyh3kwtPRwPBTJb07AZ80dW2qdqL2P0j+P8oalOqvTP1f4KuFYndVNsaXQe0n4nUiUt5tIIUvCe9DldRfn/Az4ZMV4/nx4vusHWl4AvjiAMVP7vqW0gzur3gCsGZ3/WPT/wYaxMpHow0qCdVXp/CUo5YSPnkH8bo8ksnALZbfh0oFifzbc9LWbsT6oNqYaHYV1elr2g9yBbf+wO5DEJVUYUVFWvqcHK5IVSno1iHn+HIpSFHQpCsW3JMnMinaqwpSvrLDEDe28YmarVdSrwsyS/hb+74x/1I8JS9pxob31wm9ZGV2FWlrCte2BM9mFQpVn8PwUp0j6r9WHhC7a+4TVhyIvUIjvRlc10UWdos+rgpihEI3sr/6ikfvwD+YkmTR946jKIz3uqxi/tbQo2M1LWrJcwcwWDn9T3oPk6wbGmNlVuMXet8xsdiJjC0m/CX9voNOLutsxU/u+JbazILAtcIx5mPBz6cwhMSJWhBcIyvj/qtPfZ3NJ346u4xVz59PvRud9Ehf5vSZ3+JvTzLaW9OeonStp1xPNJOkfUV//CCLrmMYN6RMJ36sg2u6mjUZ0w5Wm5A2fxd2FO/78H/4yTtoSzu9aTIM7oR2LixiWDv9PL9Xpt1IhMkvDZZ/lWegGpfopJqBfB36Fz9i+APwV2K9U525grmh/bhJN5Eo03wVs2nSNFeeX0zvW0oJbbpyEf3AXCdvaoeycUGf9pi3UKWbVEyu2RtEHaelUyylg+9mll8uotuEftJltIr2N6UlT34OK6x6Br5TmDPvzEM2e8VDypxBMZnGZ/x4DGDOt71vqdYcx9TWcET6ArxC+S39T6iVwvcX3Suc3+rqEsnEV/Y4t7Rsumjo/bHsRiYiKZ0IUOBLXm/w1/F8YX+3fEO7HseH/HcDCKW0kja/BDtApZcOXbJvhIo2xuAxyxS7Of2oAfc6KRwMdHbbDgVm7Hdw06DDC8fmBs/El+/O4BdL8FfU2wSOTHk21rHUX3Prk0HB/HgQ+W6pT6UiEixXODb8TgenD8QWpcBpsu79NtBD0LDXt9DuGm3SujCsme6IvIe2jWjgLzoQzurtxEUQhKlqCIM7Ave/XwD9Kq9EXOXYDKkQeuKPjsbj10tY9ord1jCfWKX/A/69qi44PKFpzxZhpfd8Gck1hzH8//N8XeBIXrb4EPEFpwhXqNfq6hDptk8SRVc++4pwPAI/iqQZuxiPXrhGO/Qk3jql61y9MaSPpfejm5Xm3bLjd9OfwWfq+iee0vmgDpKVxcONWULOG/zuHj8Pik/HerBBehn2BFSqOVzoS4TOfz+B+H/FMZTWi1UU397eOFlyeui1ualmUjcAtVW4vtbEFHon2+kD7k7hYIK5zDfCxUtnJLfQmf1TxFd5EXGQYr2DuLsYfrmS/DpeZXxdtF+HZ1uJ2T8StwHYL2xV4XubB0pvM/LrpCzeiKLarcf3LtdHxVu/jgdKSOOa7bgf3valUUkd1NscnZUdXvQOkTRIvxP1u2uiZHjcEWIkwSQvlDzWcU3a4rWwjZZtqdBQA5mk8t8DtpJegz/+gOP6pulPxpWNqPz+V9JU6ObmigH6S/kmnc08ZJwGrmCeWOQBfop8JrJ9iAppiUmlmoyS9HmT2z+ErkuJa5lZCmGb5SDu7goaxUVtVvhgFLbOFOim0fAZPCnOimb0SDs+Jf1jLPhPH4ArjCaGNpYFL6fRBWBL4ppl9QH0+Gms26DAsbKTUkfQz4Gdmtp+kE6oqyzMsnmFmn1Z7wqmN8MCGCjScAdyXSG+dmbjh9zDpPUjpq4Ckj3dU8FhkP42KaqPUJo6Z1vctpZ02mNnH8VXAE/I4St83s0/jq4r9FZmah36bfF0A9gO+h1s2gjPRfUp15sKf7R10xsH6hJltJOnaiue1rJkhj5VVpbcpzGFHJrbRiqmGUZjZmTinvAz4oaR7K6p9vKKsQGsgvwi/Db+1Dl1dMJO3JcnMtsJnjaeY2R7hWIpCcZfQXpMS7/e4WKtQlk4ik8QwzRWMqKMNuY1/Ew0/S6VFbou+feh/nnB9df4VbxRMIuAx+vJYFHgVt6s/PjyPwmu4SXn8Vhd1CDSeYO3pJi8xsx0r6hwS1ZkALIZ/oMBDnUxIpKVqrFA6lvIeJF93BZ4G3hftN3kfp4yZ1vctsZ02HIbrwjCzLfFxsgO+av4lsGmqrwv0TRKDcl+KlMkRvtdAz/p41IOq5yVcd3mJmf0a9235Z6B9VuA4/FuY0kYrpiaHu//Rx5EbH+BkpuMceciONSSNsZbwvuaRN6/ARQz/h+sh7lYpemRFP0dL+rqF9Jtmdo2kjZvOqWlnYUnPRPutjkSTC2Vaauq8R51J5TfBfTzOxZ/7trioIU75OVZ9zm2fw5WYc0lapMf0/5aWdJNWk5NcbkFWTCrmwGXKHSFHJG3QS3p7hdIqZgRuPv24osjF1oNozTV9nyNp+wGc9wn8fQP3Y7nYIudIMzs10Hlk2O8qOnU45/24dKBYnf0d9/GpmsR2S/+nJf3R3HnxcFzUXkwsFsN1td9Wf4/yfm0k9Te1MIopBVaKW19TZ9LgDmZ6O+Jy3JvMbDHc6unMljaelOdCHouHifgSPovogDqTEiXTawkhwnuNxHt3qaQtrLuk8ntJ+lW0vwawjzpjDQ0aZvYArmupfamsISd53aSigBJzBww1rDMH9ds4k7ilQcQFkCz2aOm7dcxUnHM4roc7KxTtgIei2RKfIP0LnyB9WtLocM79klaoaK6WFvNc99+RdF3Y3wD4saQPRXXilckMuB7hn20T24q+ZsZD6YNHiG0N3d/NvZtqRE/vMqxT/JH0HK7kKvafpDNoWB0KkcBn8ACA09G8/G5rx3dcz/Npgmik0FWURCOTC1Vijg5I2iL87tZWNzrnV6X9MUBPmURASrrJW83s/eof3HCKZQRtCPqXKjSJuJLFHpMBWwCrSvofTNIBjcX1KuNwR70HIiaxGmkpRMvjd9aCSQBIut5K/k2xyNj8ZduKIP7qpi9J/8ad97pB6/tWIDOKAcDqM1gZnQ48TW2kKKFTFKkPAUeax9FpUqzVoTz7vZA+0UjZ8W9yo3V5a2azxbLesLKo0gFNDkbQhpR0k605yQc6yxwumMcnOxy3Youjwy5Ve1J37Q/6favAnPSlP50DQNKp5hFY58ct1go8h4uG21Aeh4+Z2ffo07HsTP/gjX0n+0r0z+YOuU0GMFV9DQTJbUyTjMLcI/FruFlabSC+BhzTcOzB0Efj4FaaJ2mrQtHMdpb0O2AFM3tfuZKkY1MsYSIsoijV6EBgDak8u6SlCvfjMtgC8TObCffm/htdwtoDKqakXP1BQletOcnbZpkptFhLetJQJyUg3W8lfTamr6LsNDzW2HF4DLDdcC/nA4DXFIVQD+fvgZue/jQqa0r/2vq+JbZT4HBgrHkmOsN1FQcBSHrGzP4G7GRmS4WV9PT4SvFJ6866anc8DP8F+Ji/idJKtiSeGwGsCfwnHBtP/buyQNSG4e/tU/0qJrbRhmlSR2GeFH0MnkRkpcA4bpW0ag/7uK7puKQNQ73jgT9I+usA+9lL0q+sOR/2rlXHojqTRAdmdjJwQpVopAuaJimPo7K7JK2eQkvLy/gdSbWhOczNAm+O5cAJ9B6Mv6DLSVrWPLroeZLWja+JlpSrLX00hhNRSz6EkkK+lRZrSU8a9seVx3z52ZWVuOY5K8bH8nrrM6gYr2CEYR7KBGBtlRTXZjZDoDdeRdWOmab70nSfmtoxswVxgwFwQ4HnomMn4SFINpL0vsBsrpJH/a18zwoomF+H+/SX4j1voDfWtb2NR2b4taQXzFOXNvVVKK+J732p/eQ2mjBNrihoTyY0aLQNkAhjgO+Z2XK4z8fZhWw0sZ9fhd8fNtSpkyFPgpndi78c0wG7mSdSqhSNJKA2eUwKLXg4hZ9QSgQTUGk3HmEZXHTQDT6Jm0DeBSDpb+YmjTFq421ViA8nnUOfxV1bHKdJYpqmWWYbLRFaE+vQkLjIzL4FFPkLXo+u5y3cKi7Gm4FZPWJm++JxuWYD/lNmEoHetyret6RkVwmobcfcUCTGuPA7g5ktJtcPgkdmXj0wZORxnGYI/2vfsxiS3jGz/5nZHPI8GHX1akVaqR/xgLvM/YTuHEQbtZhWGUVKIL4hgfocsebGlchHhkG7TMr5YUXS1H5qXoaFcbPGXiAlCU0T7sJjWY0pHzCzz5f2yx/p5/DwCt0gJaBibQKfFvEhoc6SXdATK4GLWeZWKbRESEmsU5u4SNLhwOFmdrikb7XQuz8eHv/LeEiWjXBP9F+b2QKSno8rmyfwKWOwYyalnUvpz6yF+3XMj4fUgLSMkin4BzDezK6m05kuNpdeBI/QW6xeb8Kd+57usq8P4uKyJ0JfA5ng1WJaFT01JhMaJprWwp3MtsItLposRuLzksVKLe10vcxvaa81CU3DucsBL6k6KU2/D89gYWZfx1cim+Dy692B3yvysrbEZEE9oGU+3C9kgqLouqU6KcmEqvxhdiqJK0aQkMDLPOrs4nQ6CN6YcC274Mzja4TVGh7v6ifAz8tjczBjZiDtmNkS+KTiI3jo9BNC+U74u7g6zmS2wVMAd5NFs/bdLIl6r8adUGOF906SNumyr0oRU69WFNMkowCwhExZCW18khBGOOzPiftA/LmLNo7CRR+P4iEy/lz+QFiaIpXomNSl/4OZPU1kpluGWvwxJgfMbHVJd9UcWwb31F0aj+T5DbU469W0Y3gk0eWJsn9JunrAhA8QYbX0Y3wsLAnsKakxfH5NOyOBI+UOmYPyhzGzI3AT7PvpdCL8hKWFet8cVxSvFOreBxyhAVjohWe1E+69f0gQJb1H0h0tp5bbWQZPRftBXFF+RiEiC8xzbdwiamOozihpZiPLDLWmrxnwsSXcge+t0vEqPVFV2cy48c1DDX2tQl8q1Jsk3V063tpGbdvTMKMY0Cyp1EaKMrCRmZjZXsAfmxiVpSkv18StT2bHB/erwO6x+MbcXnx/dVqEHCNpdzN7FhdlVOpqUmSz1kV4gyZaojrX4dYm5+Phxe+Njt2E+5vciOf1XkdSo4NXA92VisBwrGcpVxPouBfYUNKLYUVwlqR1ouPJtJjZbSqlGo2OnStpO6uxiFGnkvkhPGR4P9Gs9cBBsMsx06RkTjE3XwlnECsCR+FGJP0+9uV3uIbux4A/4hnr7q+p8zE89P+jgY4lgb1iJmlm1+Dv7R9C0Q54vvuNozofxydFM0ha0sxWBQ5RZHZtno72C/T5pnwSD3x5QmobTZgmdRRmdiS+tLyPPtmj8I9ON+hF4qJfAztaMMWrmSWlKC9PBfaWdBOAubnnaXj47QJNiYue1SCd6tSjxEVR2YbmnuvbAb8Kq6Vz5Caes0v6daj6EzOrXHkkolIRGNAUb6sDQfZeJAd6piwiCzPiteI6uMVN8WzfkvQigKTHzJ0fB0QLbv55Ee61H8vHL6C7BF6P4eahVTq8+4H5yh9K81zdLya03e2YaVIyp7RzNx5p+FL8OaxlkU49YrTXmAcDvCB+70pYBV9p/SZM3E7FDVFej+ocC60BK3fHdRRFVIVb6O+z8YNA7/WBznFmVtZ57YHfnyLe05F4TpoTumijFtMko8A9mZermiV1idFmdizwi7C/D31J2Qu0MZNfEGZJwCF4QLs/0me6B2nKy3cKJgEg6WYzK1u5jDCzuSS9AmCuQC9oGbTVl3VnAtpES3zOc3gwv+vw3OLfx/NXzBQYS0H3zPF+nciqBrWKQEkXh/ZqdT1hdvZL3HGrEH8tYmav4sz7LjP7KB4+/JG4DvBeM9tb0lXhnNg4oWO/+JA10RJhJjyfwkZRmfCPX6HU3ltSh+I/fGDisn8B48LMN3Yi/DL+ETqR/pgH1wHu2EZkl2OmVsmc2E6qE+ZeeCDDt83sP1SsbuSivF/jCvv1cT3DcWZ2PnBoYA6tASvlOoS2Wf1/Jb1mnYZiZQZmRLHDwv/4hJQ2ajGtMoqmWVI3SAkj3MZMamdJEb6IKy+/S5/ycs9SnRvMrVf+EOpsD1xvwfEvfDiPAf5qZufhg2gbPGIm9Cn/BoNkE9AWWgAwdyDcHrcGewm/z18Lh5+lU6cSh0IRnR/INmxad8AS5PDA6bhIoUNnZK5bOg2fff4M+IhKwRXDrO4yPNrqN0rNjynVTaGl+J/iSbwJ/S3ENi+VXRS2KrxXFeJaecyyk6pOqEA3Y6ZIG7CAmR1GUDKntpPIYJNWN4FhbYHP/pfAx/NZuI7gMjwZ0mgzu4zOgJV3mps/fwQ4TtIjYaV5Cj7On8ANa+KJzn3mEYdHmutXvgzcWiLpNOB2MyukF1uHNrtpo/5661dWUx8i2e7C+MtbNUvqdZ+z4szkI6HoajwJe7FEvB23TLkzMIz5cLlro4y0op8mBz9J2ijUW4G+j+i1dfLVoUAbLWb2V1zBf5768nVPTnrmpzMExZMpcngze0Q15sxmNkHSe83sEdwA4e3S8RmA+yW9t+r8Ut1knYB5nuc9cHl8fE27m9mX8FXp0nj48gKz46lQd47KChqXDbuTIr+a2UOSlquhddKxIJL7MbCQpM3Dc19HJY/tFJjZ8jQomXsBq46q8BrwRPH8go7iOjyH+62l84+X54o5raKdAlvjWSr/Gz7gX8ONKVbDxdWFUhpzh+DvhOPgebZ/JCn2rSnoXi/s3qTOXDFxGxbaOLTcRh2mNUbRK1PSnyoxcVFCW1WmeN+TdK71SJFqncmC+kEtXsEDQZg1rYfTfZP6lPdDTksbzENOHwMshId5Xxw3UV6xVK/SgiWIh5bGletFGIVF8VwhEyXta+7Ath3O+OI6nwHOlfstdENzmzXNeXh4ix1xkeZO4Zr2N7M58IQ5h9MZU+iN8v03sw3wcfk4/oFZFA+VfaOZXYrnULmsdM7mwJclbR72L8dnvN+RtIq5fm2sSgYEdWOmVKcx1EpqO00ws9vw97GITvB+PODjHMCXJF1lZutJurl03rqSbknsY5IhjJn9Hs/c+LOwX/aGr7X+GwjCamjWkj6l+ZxpiVEUCLP8/yhYPIQbN6MSQvOG+q25JrphJuVZEp5P4Z9m9nF5nPwUe+w58Q/TEnRacn3ZzC6RtKV5ILoqi5CeBG+LaDkRD3lcWHJsj4c+3meoaUmBmd2Nr2z+Imk1M9sQz8GxR1RnC1wPUWnBEj6OW9GpqL4o/ogGUVpVna5WdW20hDpjw7XcI2ll87wFNymyhDJXrj4t6c3AEFYGzlRkZGAeimNHBZNK8xhSf5CH7FgGV87eSp+YbE08OvKWkh4O59wpt0waq75wIh0Wg01jJqpzMO2hVprG3r6Sfp5wfy/AJ2v3hf0VcGZ7IK7jWbX8MQ/1ijA1KZkp78JFV6/g4qaNov4ekPS+qN1a679UBGb0RVx3cScwCviZpJ8kNaAe5KR9t214TubZov3Z8FhPveyjSH6+ftUWji2MD/wZwv78+BL9bwPo71ZcRr8b7hW7Kz7zG477+yBhEhL2R+Cz2WF/9jX0jg6/dxPydOPJo8rX9N5of2ngwclEz7pNZSm04NZU4JZ8K+GRbR8r1RmHTyreCzyMO8JdVqpzTwUt90T/Zwxj7piw7Q7MVKp/Pa7gvivsr40nC+pqzAR6jc7c2/ektlP0n3D/760rC/fpa/iq8IBo+0ExZoCPh99dq7ZwbEt8ovAcHtup6Gd94NKK/t+D6xVuwVc63+1yTI0LvzuF5zR91bOt26ZVZfZMikJVS/pHkOElwZojMkpuLTMmtF1pS25mX8FlhhOAGcNM6EhcfLFGqJOsvAzXVBlMr0bmGrfTs2VtQF0qzwHRYmazhWNVqSQL0VFHtrIu6X019HEjcJaZvUBkUhqQknK1irY9JZVjI5Xr/EDSD6KiE3DRBzVlKbScbO5n8D1cGT0b/dNu/k/S20FUc4I8levYUp3RZvYb4Hdhfyci81y55WCTLB6aU6EWqB0zEVJCraS004b7zJXxRY747YEHzM2V/4ffy3L+l9eLa1KCpZykS8y9qWdXsPwLGB36K9evs/5LxfRhVbk17hX/3+I+pmBaZRT/jOV+5hnP/t3F+a32523MBL/3y0l62dx34mF81hhbujTlCC7jt2b2BTzsdqygf5nmMM3dWgjVImJss+MvVkcqz1AtmRbrTCVpZvYipVSS1j9b2ZfNbB1J3+6C9K3w5/9V/EM4By5qKOTdUGPBktB2itnxmNDXOrhhw3zWGUF3FG6tkkyLpN+EvzfQaTkU47/mgTF3oS++VDm/w5dwS71CH3YT1SaxtZCbB69PRSrUxDFT4Fxzy745w1jfHTdRTW1nZesLcBijbP76OVzZ/5Wwfwu+ivgvsGaYWJ6ulvAY5uE5tlWnU+nZkjYN9+VtXPQU36vyBKXR+s/SAlKCO/49jq+abwxMKusommBmH8BnC3/Db+h7gO1VEYQuoa334B8q4ZZLz4XyxVtO/ZM6FVaT8vXW9NOmvNwHNy99lb6BIw2hzL9OZ1OgbnXV0F5KKsl76MxWNhIXTSQHQzPPj3CjpEcqjjXOltVFpr0EOv4P2BCXJf8yOvQGcDEezTWJFjN7FBex3oTrJu6r6G+F0NdfJf3B3FR3O0lHxvJ8M1ux6vwurmtb4ApJb5jZd/GV0Y8iBtJ0TTeENhpDraS0Ywke16Gt/RUUy+Uy6073OE4tkRtSYG79dw5u9NAz6z8zm04lK7zautMiowAIy7DCtG9Ayd7NY/N8H7gWH7jr427xp5bq9WMmQbxxdlTtM/G+OsMxpCgvHwPWUnMokJnwmdIkixDgl0o0keslUmipYp7lssAoNggrJ8ytqa7vklH8ELd/XxJf+t+If1zHDfDyMLNrFUySG+o8LGnZaP8aSRtbCLEx0L5DWzPis+kP40Evl8Nl0p9MPH+SsrZKcVtRv9KENhwrFOrr4dFljwa+L+mDXV5TbaiVxPNTGUWVonqs3Dig1ZAlOmcM8EmF8OVh8vintns5GFiFiXd0bAv6m0snRWOYVkVP4C9OkbpxdTND0pldtvENYDVJLwGYBxq8FXfnJ5SVmckJZnYILc5VJRwDraEAJuBetE04E5+dFm79O+JRK7dtOa8rWFoqzxRaalNJmtlVkj5KQ7ayVEg6OLQ5Mx4v5xt4/uQi7HSxsqiaQe4emFUMA5YtysNHMr4nhThqlqI83JsFzexDwPut0+u86KsQldbSEu2+g4tK3sHl6i+EDesi1lOJ3kpYhQmtme2qPme8wmN4C1xxe6mZ/ajURsqYaQq1ktJOY/TXIIbbEVjSPPxJgVGEtKlq0T2W8B3gZvMkUoYz7bKjLFYTd67hOU3ShUZtVJp444wBM/slHgp+Q+A3uD4lOZjiNMkozM3sNsAZxWW4N+rN+MerG7xEpxLxjVAWo5KZqMZRqQYpyst/4qEWrqPeiXAlRZnJgOvMrOcOd0pLGJ9Cy+7Up5KcL/T1BzO7nr6QJ99UlK0sBUEcsi6upBwLfD30FaMp5erjuLz3R7iuw8L5caj40/BUr99QiAFlZhPVmafi+7jCeRH6R/KN9Tcp6V9fx61jjsU/zvG43D/8Nuna5jQPaDkCGGWdyZSQx4wqcAzwUZVMaAlGGcAz5rqFTfB8KzNSCm2TOGY+COxsZo9Tk3OhpZ0FrTl/yzG4x/+8dOrS3sAjFE+Cma2LWzoVH/h+5t2SrjA33ij6/0p5xW99cec6ovPiq9qU51Tg0NBPh4l3dPxDYcJyjzzr5TF0TjQbMU2KngKHXgWXZa9i7jn6OyXGgLc+ReOquDPOhfjD3Qpf3n8uqnsrLhopnLNmwEUjrak6o5dzE3xAxsrLJyXtHdXdtaoNdfpa/A63eLgt7H8Q2EfSLu1XPTiUl/1ttFhLKskgavt6XX+lD1kbbXfhCYIuxZW/f1VLHDArpVwNH9WvAkdLusjMHlNJP2RuNPETPCDkz/GcE/10SGb2PUmHdkF/By2hbCtcrLcWnpXuVlwPc01U50hVxHqS9E1r1s1InVF+7ymvQuIyc4vCzfAUqo+YpyF9vzy+VdN1FeKexeRe8pV6P7UrlYt2Kt+RqJ0zQv1ZgX/Lg28ui+tFLi+J0x7En/cYohhLMUMOjKoxLLo1ROeN6qTQM1rSmuY+QauFupPEtGZ2u6QPmjsTfgqf0N6nhIgAMI2uKOi76W+bRyR9ATejS0Uxa3k0bAUuLP5EzGQCHoOlg5kk9hPPSJ/HdSDgkTlnjiuqOWhdsXSdHrjVzJ4M+4tTSk7fC1hawvhGWtSeSnIOfKZVF9snmVHInaRG4auKTXDT0hcUIpLWoCPlqqQ/mdlVwKHmyvFyvC6CbPsjwL44Q5qpXCfUO9Q6TX6vl3RJVd0qWkIbFwIXmjtzbo5b8BxI57ipjfWk7pT0Y6zBhBafoY8GsL50pB3jrmnM4Ix1dUlPmNkfJX26jpCmdprekRJuBD5sIYw5blG2fbiuAq+pPafGiSQE/KQ97lwKPa9as4n3JeZOuT/Bk0iJYDGWgmmVUYwON+3X+IzgH3hI3iQoLW9uCjMZiYc6OK6mn9aXNVHenLJ07SViBldO5dkNLU2pJJ8oyeQHDPM8BR/GGfGauDPVTaU6ZTPEfilX5aaNB5gnkFmHCsits443D7FRqVi1/ia/+5vZhxRMflNoMbM/4qvmR/GPxy7A7eHYpFhP1qlfmR03Be0WX6TZhDZOQToTbjTwEEF+HtA0ZuLJQJsVX2071ql36Af1WSyZpH8Fhn+i3Mt6XKn6dWb2E3xCEot6Yz+glICfTdF5C6TQsxXOEPuZeIf2ihXqH83sEtzvqmoCVolpUvQUwzwd4ihJqbP8+NzrqP44J/slmNkdktZqqdOkSF1Q0rMpy3Lrn1y+qPNkVflAYAmpPFNpaRKnlUVZg0F4cW7E9VR3agAWcL2E9cbkdz1chPZOVDajPFzHHCTGekroZyQuwli+i3NWx0Ocfz7sN44ZS7TASmjnRXwS8AecaZaNBQpT3LE4Iz0O2EPSfVayuLLqIJyK331LCPjZNMajOq30tMH6WxneDJykVItHTYYQBO+GDZfTHYsrrT45wDbWiLZ1Q3tHlepch1s8dWzR8eNwefWHcfvy1fFldtzGp6NtJzzmy/E1NM2DKzfXqDg2Hhd7jcfzIryNv+S9uqefx8V4f8VnuZ9oqJtECy4qWa6ifKUe0v2VirL9w+8y+CrwXjznwMJDMDbvAeaO9ucOZcm0UBGuIi7DLbp6EoIk0LRYl+eMTx0zuA7gdVx083b4X+y/3kU7I3FdyRm40cKPgBUr6q2Pe5J/M+wvVfe+tVzjTqGdp3Efp4dwP5WkMd4NPfj37BE8ym3HvQnHz8XDjm8Ytl/jcbKSrmWaXFFYQgCyQbTdsUIICswCM+Ef+7clHRiOt85MKvqYpLwMs+GDJN0blIR34fLgpfFUiD9taKdjZjdYWEsqz5Zz+9Fig0zf2AXdTXbzN9GjlKtd0LMDcAQ+yYhNfvdto8XcZ2dhXF+wI32z5lG4n8ryUd0Lgf3UsqI0N9ldgk7zzTOj4zfiYrQ76BQRfiIcj73MR+CToXkkbTqYMVOisat2zC2vdsBl9j9URbBAawgdY2bfr2pXJb8Eqwn4GR3vaoyHd382lSK/mtkEPMZUZdh1M7tfnVaGlWV1mFZ1FBvhuQEEYJ6/uWvPU+sMlT0CX1nMEddRf2/vW8zDCxTHK616WhArL5dUX0iL3YCrJe1iZoW8+ad1jcg9Y7tyempBWyrPWtTQ8gP6p2/smae59dnNL1WSX89OsJuny5SrQRSzAJ0f1Vic1moJoxqTXzP7VgItm+IhKBbBV8sFo3id/p7dc+FxjSo/8IHe3+KTjnF0mm/GpuTlGFJlxDGRCuuyP4b9AY+ZEpLaCeVb4ExiCfqSIcV1qkLH7KJO7/RYUTwTrnt7IGpjYWBB3AryQXNHuK/gz2ah6Nwf0DLGrSLyq5mVI78+X8ckAu4ys7XVaWWYklIXmHYZRS8Ch0Gnk9zbwEQ8WcwktDGTIC8+mCioHT6jeC2q06S8jOXpGxMsGeThEv5XoqVqZtfLhEBJqTy7oKUqfWP5mvZXTbiFBHpvo91uPjnlqpnthz/L5+nMxR7rFlotYcxt9MfJzWx3Bg40s58l0rKCPNf4dpLObbn+tg88uHJ/hWJSVQV5eIzF8TwRfzE3hx0ZHW8y/kgeMy1obcfMzsQj6V6GryLqwnX/CjhAnaFjfo3rG4r2OmKWmdnReDIgLCHgZ4TWMY7f/9fNc9dcjq8ux+CroQKjzewc3EIstqD6IT2weJxWGUU5cNha+I2+CDpnVE1Qp7NUHdqYyam4zLkI2fBZ3DlrkkhBzakZnwofqKfxj+0VAOZexuUAb00zu16gG2/zFFpS0jfuiqcZjfG5irIqnC/Pq/Av1XvadpNydX9c1lx2uoyRYglzErCKufXUAbhs+cxEWj5mZgfhH5NGRlG+ZnMF+A74ZKXAvXgstGepgXmAvj3xGfjSuOjrl4T0uuZK3APpHz5iI7obM01IaWdnfCWwPx48ctIl0OcdD57UZ5JIWNL1Vh2pNsYs+CoO/F60BfwskDLGUyK/jsItqD4alYkeWTxOq4yiUr6YirBsOxl/KcYDu9ct+xKYydLqtAv/oQXTtzBwjg793IN79T5TOn8PfGb6ETyw4auhvMjXXNC8Ki5eu69liTpgKD1DYCot++Ezszdx5e2VhNDKVh9uIRYbtWGEmX0bD7dxQPmgpGO7FA0+hSsTm/DfIJ4qxJ7z0X8G+bYkmTvN/ULSKWa2R1mPUoMr8Iiks5lHSo2nqvHHkND/avh93BafxJSZ9bzA/WFSFZtvxpOpffDJ1u3h2CNB1FLgLDyo3Za4CGVX3BcoecwEWpcEnlWw1AmToQUkPZ7SjqQRiV3Vho6JaInN0UfikQIK/cR/FKzH5I6CD9UwCegc438gpCgt1WmN/KpEvxcrxYJKxTSpzC6jmEkpUZltZqOBb9GnVPy8QujgqE4SMzGPDPkNhbSKQexwtKR1rEeKVHPF2874LOuDwOGRrHtIkUKLmc2immyDZrakpInhZVmSChNPXC7cGhXTzJbDZ2lfoTNaK5DsLxO3dwoeQ+xSOj+qx0Z1qlLfflfSeVGdG/AP/u64NdwLeFKcbswhL5S0Vc2xZfGVww7A3/GP+Ncl9TOxtrTgd4XX71i5AcB0uIVV4Zk9JqzcYm/tOyV9oKrthmsajYeiiKMc3NJtOwn9zIWLbOKAlT9UlDfCOs3R38Z1BEU+7eSAn1F7o/yQWnOchPrTyXOJJKVLtsR0v3WYVlcUKTOpJoxQCG8MnGeeD7mMX+AhJoqP/E9xRWMZXwTODLoK8NlgYVvdlSK1Advjdvn/Mo81dQVdeGX2GCm0vGZmP8ZfzvJs+48EL11cx9S1hUwBeWyiI8MHLDnuTQOeDNsMVHhmB5yPM8nCEmZrXKcRY3t8bO4elNiL0SmPboWkrcxD0xQf0dsLZS8um74JT1daBJr8ak07jfqHgBvCymxmM9sEt9e/ODpe6NGeNY9g+jdcTNUtplMUXl/SWxViuwHD3Nfgi7hF5Hjga6rxqZF7ic+F6zenAxYwDyx6F12I08xTHpxKEMWa2Wv4cx9jZjtL+l3VajfgWPoU6G2K6bZYUI2YphhFzUzKuhQvgAdM+1TdvjzOUAozQdLduDx6VNiPl5TJitQWvFnM0CW9ZG5iN1xIoeUxfCV2i5ntKGlidKxD6xfu+5G4FZjRX96cglvN7FgaDApSUKxArDkj3wXA1pIeDHUXBK4mUnIG5nAW8AEz2xJPa3pmRVu1MM8BcTRuTWN41OJvSDof1399BvcuvgKf7VaFQmnVPwQchItAxwN74cri30THfxQmQl/DowWPwj2Iu8WLZvYJSRcF2rbC3+Ne4Qycqd2EhzJ5H33JizpgZofiurBH6ZvJC899nSxOw/VPe0u6KbS7Hi4yXhko9CK1OkolZNML+G/xvpnZCEnXmdlPU4mcpkRP5lZAN+HejcVMql/wtoR2Tms4LLnHdDlo3dHxvhKC1lm1j0XcT5IHuJm9iq9sgEnhjov9ZOV9KsLMbA/6Ky93T6HF+pLU74yLlr5TfCit5PNgLfbjifT+EVfaFi/bZ4FVYjGfWVKAt5VwuXYxW/47JbPK8OH9GC5yWhR3pPq6ogB5ZrYdvoK4PrpHxUc+lZa7gU0kFaHF58Nnk6tEdWbFQz/sgCvCz8TzJcS0jCPoHxQ8ii14BZvZcmFVVnVP15WUHA4k0PcF+vtr7B7VWRrXdywU7stT+P2d0E07DTRM8nYO4rM7VO8F/hAe2PCtquOpsIoIA+UxntjOxfQXPb2GrzR+hUcc3hp/n+bFxU8fUEJwUpj2GMXW+ExqXVzkcTbwG6VZL3XbVysz6XF/y+LWMgtIWsnMVsa9U39UJ2eOiKmz+BkoLefh4o0dcQXfTrg8dP8UWqwzZMMS+Mf3GXy2el2JUdwiad1B0jtO/TORdZSZ51D+Hz5jfF8QO1wVy8ctISNfKN8H9xBeAk9AdWvpeONHPpGWcsiJETToOUIb2+IGERtH5bX6hzDx+i0e9fcfpfbuwv14msxqY3PpW/FJXDkaaz+RcNOKrZt2Ks4tT0KawoX8EfhS8YwGijCrnxlXZAsXO/6HEGBR7l+0JK70XoJO5hf7u/wMV6jHTsSvhzZH4SK1f+Om6EUsqN8pMWTLNMUoCqTMpN5tMFeAfgP4VTTzu1fSSsNAS/FRKTKbTY9njCvnF2g8P9ofgdv87wrMLGnB6NjPcPPNP9OpQO4mzHitQUFUp1jljI3ubznbXm1GvpKc2fAgfffgoSTKCu/Gj3wiLT/BxRfxh+MelcKKJ9ybo/D0urvgH6u9gfslfcfc8ucinMHsouDMFc4bS4OzZ7jmOJ5RP2YdHWuU1ZfuXW07bTCzd+hzpDP8A/4vKsSZZrYmfeFUKq3BqlZV5bIUqUGYOJyCi/b+Fx2MDQr6GQcUZWZ2H3BJ+dlbRZj5OkxTOooCchf63wO/j2ZS38RD+A4pzOxmXCZ+E27BkWT1UIFZJN1hnY47SflwJwMKBeCrQRzzHKUw2C24NN6RK7R/aGZX4p6sMersx5MZBZ0GBYab136uVCfFrLXJrLIsZ76gphzginCt8Uf+sm5okfQNc/1NESr9ZEkdHsiJaNI//DcwjCvx0NZn4Lmw/xdoOwc3yHgxbjDQWx7nl5jZxyRdRn+0yuoT22mEpLKSvgln4Lqxjo93CSfg1m21ZUrTj/5H0vEtdWazkLcDIIgjZwvH3qIhpHxC/9PmimJKQlhWfjhsa+Ozk5skdaXsM7PL8VhA54XZ5ja4LmbzXtOcQMvnceuklXHF3Gx4juR+JqhTEqzaoKA4lmLWGptVgjP/Hygyq0ygwXDHrQ/E7cQf+RRaorrz4mPrSdXb8rfRNAOeLEd4PuzCPDUWEc6Jiz4Xw0UbF+Dy8SvKqzvzJE8flfSlqOwNnCG8Rd9Eo2MWn0hrT9pJ6KfWvNfM1sG9uL+CB/0sMAoPQFqIEFfCpQCFiep9+Ep2fKm9HfGwPVdRE9LczD6GGxk8ik92lsRXf+/FmcEcdKY7mB2fmCZZPmVGMQBYKS1kGd2IPUJ7C+IRIj8MbIi/1JuZB8pr6iceKEvhfhsfwk1sJwI7S3q8G1rebbAGxXnCuR/HxTFPhP3v40Ebn8Cjx04s1e8I8KYuFOhm9lNJX7FqpWNZZNEaQrqOFqsPErkUnhL1p6k0h/a2oP8HaC9Jl5dFhKH+rnik1JmBxyWVQ1YU9e5Tog1/dE6rrH6oYG4l9yYueuv4eJvZ/+Hv8Rfp9M95A7hY7pC4FW7gcjh9pq1r4v5ZX5cnnir6Ohw3sHiUKDSMSsYs5nGsiqCPD0n6j/UopHxmFAOApVk9JTETM3sUt475PT4DHae+PARdWz0F/cuIWIRV93GKGurJi1YnQ476ObbXtFiD4jzh3HuAteU+HVviduk74JFQt1XkRGlucfO0PJ/DBvhq6UxJr6YwATNbQ24bn+LAdgYequHOGrqbaJn0ATb3bVheUZBIdZHTIrTxIJ3+FksDl0pa3sz2lnRixTlL4SE71pf0vpp2Hygfs5asfimy+pR2eoGad7PQKVwjaWMLScVqzr8b2Ko8kTM33rhQnfqmCXi8p34WVma2kaRr6743Fau5Ds9sJeaimSZ1FIOF0tzlP95wLJahH4+LGIoP1A1mdqOkRxPllwCY2f64mOcN4NdhNXKQXEF/dGo7g0SKDHlAtFhNeGXgvZK2NbOt5AmNCoabAqnPA/xTwClBPDPGzPYu1f0jsKaZvRc3N7wIZ+4fo08nUXtthdin/FGrwQeBnczsCVy5WihTi498Ey3JQSIT8YYi81Nc5/JGaLMfkwjljwFfNLMbzGwtRWa7AOZOZmW9xRG4uC3O6reupNj/qFVWn9jOoNHybi5oHpr9/dbpB1WcexfuPPh4RbuPmxt/xLgXmBM3aS1jfTzHTdX3ZtJ3Jqyej6XkmU1nlsFa5BXFIBGW5mWxxyH1Z9S2MxseJvzrwCIqKdaCPHOFUj9nRscL65pN8SXvd4Hfqkt77CkJVhFeGegIr2wh/4d5ToS9ccX5HUrwjQkrig/hyvCJwKclFbmdO2L1W5+l0YF4zvUTyqIXa4hkazWpaunPBLCWbIVNtIRVzVV4kMhT8TD0r5rHRRo9AHHPSfhH5dxA/7a49/lfAk21YlYzWyucdzp93slr4hZUn5F0e1T3Hlqy+iXK6lvb6QXMdTK70F8M9mUL+kF8Alj2mC5WHXfj/j8dM/rw7C8uXff1+KrxTiosrMIkahs1RAsO/W1EyTNb0h5158TIK4pBwMx+iUeN3BC3BNkGT95SrlfLTMzsGHxAzYZHjfw+/fM1HwxsgDOKy3BrhZvpzAlQzFo+hosh7jPrNIEyDzJ4OP0ZTlcOh3WwzjDP/aBOu/kUWlLCK59srkT+Hj6zno208Nng5pvjcHvzByImsRr9o6X+1zwQ4S70zd7KM7+mSLbdRPFcEA+Y+EagZxTuJfxEAi1JQSK7wEx4iJFCZPYirn/4OC3WZXIrvLXwoIGfC8X34hF0q2bHc9IX0HGOiuPvx2X1G9EZxr0sgm1rpxe4DA9T38/qSe4Yeb6ZfU99uarLOBj4i3mompiJHkR/S6SDmwiR9L8waWiKFjwoz2w0mdI5TgsbrgiNf2fDLVTiOr/EP+hPhQc+HhdxFMe3wZ3kmvoZjzvK3B32F8ATFMV1TsNnWo/gzGt2YEypzs24OOIefJb4AzxURa/ux65NW7e04FYg0wPn4fJuonvwUzw0+6BSk+IhKVbD9TpF2YKUUnviDO14PHgkuFK3SE25Ax7b6BWcWRXbdbiiuan/9fAIsXHZWMJqP+yPoDONaS0tw/guzAfMl1Bv9ZryHXBGeDpuyTURZ3RxnQl4Jrim9lvb6dH19ks1W1PvE7hI8mhc1xMfWwX/NowJ25l4RIC4zkgSUtbiGRG/jnv7z11s0fG/4N+nn+Nm1z8Dbk2+3uEcXO/2DRdxgM8sFgJmxBO7x3UamUn4COwMfC/sLwasVdPPGFz8YuXBQ1/ynznD/jzAyqU6Y8Lv+HLZMNy7Vlrw2PzP4LM3wxnKTeHYvrhc/vGw/T6UdXz0e0zzDLgI4P3xByvQtQGer3n9aFsdl0WX21kNXxU9jjOT/UrHx1Wcc08KLT2+3s3x8Cp/D9sNwMei44Yz+L/jM/hX8BXH9xvarP3A4gz6E2F7T8XxPwPzJ9Dd2E6P7s1X8VAhC1LxYQ51DsfTn+4etqtxT/1u+7qQlpzkOEMsb49Fx2cN34jp8Inbl/F0tEk0ZNHT4HBxkFX+BDdDFP0jof4n/P7LzBYCXsIHV4Ff0Jfx7FAqMp7hSZXmDG2PAf6Bf5RiFDb3K5ckTjHeDPLMR8xsX/wjPFtd5W6RYv3TDS1yxWUsznoiyFaR5zj+eeh3IVzX8CH8BZ4PZ6g9g1WYiZrZXpIuV0IkW+suIOVjZvZl3C8BXPdSOO410jLIy4zp/QLuYHcgneabR5jZIpJOxu/1unjMoInhvKWAk8zsq5KOq2q61M/y8lShhS7t6fC7kJktpM7Al3MCD5pZP1l9l+30Am/h7/136AwKGItOt6BTX3IGvlosp6Rtw1y0pKxVexii6fGVKMCf1WXAy6zMHiDCR25thTg95jbMM5UfgLmn7gm4mOUXBGYi6fvheGM4hqBnWETSU2F/CWCUpHtK/cRhnWfCA7mNUWRCG6xNHsBfuEPxj+lPFIVeGAysOxPQVlrCPf00/RWGhX7H8Bn1h/AP1gr4jPav6jKXRMK11ZqJRnXeoO+jMQP+cv5T0ijrIiCluQnj8fjkQfis9Cvqi/3USksPrvd+YD2VbO3NQ8PfLI8xNRaPSfX3Up358NhTHT4W4djWkv4c7Z8saU9rMDeN6taOq27a6QXMg36uVb72Up17gA2Ke2ieFvl6dW+i3Pg+hfGyD52Oe7+Q9EJ4h36FBwScSN/K/E/AF5UY1DCvKAYIuQLpF7gYAUlv0pmrtmAm18gVin80d4YqM5PGcAySZGaX4R9EVONAJ6nDPM7MFiWKtRP62F7S1/EVSYqJb1dQn+fvYvisJfbl2DL6n0rLhXgEzDH0v7dX48xlHC76+7EGGEHWzH4r6bMtZbVmogUUpawNTGwrXIkMXYT2DgzhMw0kt9Jig4iiWjRRZhLh/JeiFev0VR9KSS9aMPE0s/eEsucCTSPMbEWFiLqS9gy/rabgasiN0U07PcIE3FquCYcDYwPzMty346DmU3wSpciHpum6zeOS/R7XyRTGLWsAdwQjkE3xCcui6jOOmB2ftH6PRMOPvKIYBMwTqv8VuEA1N9IqvFdLx1NCQzQ6YNW0a7jlTGzieZsSA/MNBuahxB/Hla2Fx3A5MmcrLdYQ1NDMfoXL6P+NM4q/4iuJrvMTVNA2EtedxPduQGai5edvDQEpLT1bWSstNogoqqGP24E95flS4vJV8BXxWuX7Vqp3Fz6TPQj/SB6JWz7di4tJj5J0SumcD9GfscUm4JNyY0ha2txy7peKot2mtNMLmNmf8Bn8dXSKwb5cqrcgfWLkOyQ9V9PeCvSJJl+VtGZ0rPa6zew2PIrt2FJ7q+L3f1Z85fOv0vHZgNvq3q9+9GVGMXBYX1yZd/APVlWUyRRm0hgaIoga3ovLwascsCh9XEYAq+IhFHaO6pyEW/mcR6ess6uQI20IIok9cEe0H0g6r+KD2UqLmZ0MnKBS7JtSX6PwWfuHwu98wL2Sdk2g81u4vLiIEgp+b9/Cg+h9K6p7WkNTUn9v/BG4TH99RVFoS/0XASm3Dy/9xyVdbB4Go6qTM7qgZZwGGEU19LEe7rR2Gp3mm7vi9vc3W2e01Y7TcfHng7jz4Mz42H1vWFnMhYeLn0Sfmf0WT4w0jj7GJnWaVI+jJjdGN+30Am3PKNRZFzdM+Kd5bpXVcT+gwh9mCfqYw39x5r9mWWrQdN1W8vcpnXc/nn+9UtRVvndNyKKnQSAWNTRgL+AA4B0zq2Qm8mxnDza0UZVCtYzYsedt4A/qnzhmJlyZHstrRXeRVlMgecyb9YE/mOcPL0fmTKFlPeBzZjYRn7X1Y5Ch/F84o34TD6iXlB5T0uHA4WZ2uFo8d9W9N/7b+Kpqq4Y2X8Hjc50c9pOylSXSMuAoqqGPm62/D8T9uF7uuVCnMdpqWHH8CzfkeDQ67xUzK0+a1sT9Zppmrm/K058W7U9H/5VXSjuDhjwKwAzAsqHoIfVPm3oSnr1yFfwbcAq+glzfPLT9KFwE+Wl5/KeJZSYR0HTdZmZzqRR40lwfMgJQYMxVYs5kT/3MKAaBIN7ZCfd+PTToBRZUFLKgjplYp+LTov/T4eaO8bL5iTDDW0bSaUHWW7YQavy4hDo910vU4NnQ39/NPcWPBDqWuIm01Ea+NbPj8FXEMrglyV9xS6Bd1edklopLzGzWqplfqjgo/B/U/TWzi1qq3JxKC7A/8G0zewtfIfWboLRB0vO4A2gdvbPgjlz/DfvL4Q6fj8uj3crMpg/Ht4jOmwn/iMW4F88rUnZ0jHGDNefmTm1n0DCPsXUGPhkwYFEz21XSjVG1tyXJPADgLySdYmaFJ/Tz+Ip6AXwV/AjVnvvg1/0dqq/7OOAqM/s6bnkJrqM4Mhz7Fr4irGIUycw0i54GAUvLNNbKTEK92fDZ2164zPpr0bGD8ZnScpKWNTcHPU9RZjdL8HS2hix4vbonFdeEqjORJdESZmMfDrs3KcjMzc1Hb8GX9u8wCJhbp6yC6zxOx73st5O0fqo4KLSzFO7ItDb+Ev4V+Ko89lEKHS/ijpl/AG6n/8s9KpWWoYB52JQ9wmz4vXhUgrPwMXgncCLwN0lvl85bGHifpL9EZdfh4tI7qE8ENAIXaX4UvzdX4hkq1U07vYCZjQF2VEgFG8bzHxRFyzVPJnYF7kPxYTzGUpyAag7cyGEHfMIzJ7Bpxbeh8brNDUUOxHUmwld+PylWqD2BJoMzyrSyEZyH8FgyRdndpTon4RYGD4T9uYA7o+Nz4k5LjwE/osIJBpe3WqmfsgNWiqfzDbisM27n3slwX1bCZ/lP4ErWMcCK3dKCz4rvxUNSHIJ7qO83GegtnuP38Q/fpLIu27kNDzExXdh2xuXK5XqLAx8J/2fGk/uAi+c2w2eqY8N4WLFbOkJbRqcj56KUHDl7cN9iZ8lDCR7muOhvfJdtrV+1DYCmnrST0M89bWX4yuYA4MNhfzE8E2BVe/PjIdRvAZ6Kyn6K57s+HJ8o9PQ6UrcsehocUrKefVDBTwImyWdnME8o8zXc4ulUYDXVO8G8JUmFXNfccqaMmSVdY2YmV5b9IMx6YtHBUGXBOxk4QJ25o3+Ni4q6oWUP/P79M7RzJD5LP6HH9L4RFNufBT4cZnCFeWc3YdFnkfTbaP93ZvaNuL5FFiy40nURXGS2sXxldAWe4W5GfKZ5vZn9UNLPu6TlRDodOf+BT1gqk+10AzM7Wm7aHNOyESEGl1ye3lWkWjVE1bX6gIrFuStH/2vb6TFGm9lvCLmtcabcEQBQrrg/C/hAmPXfoRrrK7lJ9AnACdYXFLII73ECHivsePpnXhwSZEYxOByPO67Mb2aHEUxbS3XqmMkTuHPYabgydo/4o6koDzBwrrk56JzhQ7M7/T3AU7yu/27unFXQsg2TR5Y7a8EkACRdX8HcUmgxItPO8L/W7XwQ2B7PZ7F7eLkXoy/wYGtY9KA4BLjczA7CFZSifwpTcPHiWrhoCbnYZv6orRlxef4OuIlnMcaSaIlQOUHp4vwmbIfHFbrH3KrvGdwq7yoA8ygCSSjp6gj//46bnX5T0kv0BVTcJ/wWzHhn+sZPSju9xJcCPYVe6Eb6POkJNG2Hj6Pr8XF7gpl9Q9L54T2+Pjx/wyeLn8K/C7uG3wUlfSc0d6W5yfGwIOsoBglrN22t9JOgT55YCZU8i4MSa5KMUtLVpeNlT+c5cFv12NN5KYYgC565jflddL7Qa0j6ZDe0mCdC2pW+D+XWwOmKsrSFj3o/KDEhS9TOAnTau79QOj4rHtI7Dl89ozzp0UT8WVYqDNWpJ7pd0getLyz4dLiYa2UzOxMX210GnC3p3hpaa2mJ+8Hv7Z2BYdR6S3cLM3tK0qLmocv3x0PSnKo+/dGHgKXj1VXoH5XyZ9e0Pxc+c/6QpG2j8rFl+q3Zl6OyncEgXMd8ku4vla8IvBBfn3lo703U51E/Hx7mexUzuxeXIvzXPHz61/D3ezXgYEkfDudvQN+4ui7eVxcZ6kL/I3HleWwok/aeDJfMa2rZcLnyQrj8cTEqgnfh6Qn3wYPWvW8KoHlWglx8MrU/Fz4TvitsPwPmGggtOIP9cthWqzg+HtfLjMctR97GHQ27oXc7fAZ3Br7cn4jH94/r3IYnTir2Z6OL6JvReUfhvhsP4gnv/wQcFo79D/ewfgMPfV5sbwCvd0MLbkBxET7bPwx4CM/al0rn3DXbPHh2vbrz5qVvAmp0GTSw1NZdpf1xwLrR/oeoCKDY1s4gx/bZwP9VlH8Y+H15bJb2RxRlMd24Z/X+0f7Y8Ps4rrucWLE9Vmp7Adz89vKwvwJB3xb29wvP4b7wroynQs9St+UVxSBgZvvhocOfp08sIpUcXAbFyf38T+HmbvOHPiaZOnYjt7aW2ElDiVRawqxw0VKd2iW4eVC4vSV9vgtaamd+UZ1xKjmwFWXWRTrKIGb4PA2WOwn01tJSKitWuwDXqrsc302rJCQtaWZr4+GtX8ZXsb/FGcUIPFfGCriJ854qBQ0ErlB10MCi/+nxWGWxU+kauIhmjkDXK7i4sGk89GtnMDCz0Yq8pkvHOiIJmNlPcEu6P4Si7fGP8zeDGGmLcA1P4JaT94Xz+qWJTaDrclyM/R35imU6nOEUFlYTcHHkgERwWUcxOOyPm6zW3vw6ZoIPoFQchWfDqnrRu5Fb18ZO6gWsxQ9AncrWVlrM7FBcdPAofcxQ9E9UE/dxl7mDXzcYoU5R00v0t/P/p5mtXnyUwkfr3+HY+qSloxyJr3aWp7+OqRs00RKjiAkk3LoqGWqPRgoevffb+If7WmBzSbcFBlV8HDuCBkp6zNxX5SrguBrmOhf+UT2/RNMY3IFtjrA/yfijm3YGiSYn20mJrMKE4HhcnFlEdj5Z7l8CbmQyGn8+F0VMYn2iSMFdYF5J55obZSDpbXPP+QJP4e/bgJAZxeCQcvNbmUkCnm+YDd5Ptcy0iKQaYxFJmw2CjjasQ7MfQLe0bIfLumsjXAY9RoEiJ8ff0sidhCvM7Eo6Z35lJfRXgPPM7G/4db2HELhP0sHBkOByNaSjlPSOmT1kZot1s6KsQC0tBczs+3h4kD+GOqeZ2XlK9JmxvnDdlVBf3udCgX2Igj5MHu4bEoIG0p+5CmfUP5N0aQVdk7JFhj6KVWhX7QwCE6zC493MNif6wEsezDPM6PtFPpB0ibl10+zq9Kq+Ex9/3eKf5pF9C+X+2nR+mx7DLegupdO/5FgSkBnF4JBy8xuZSVCi/hhYSNLm4QO/jjoDpo02s3PwxC1xPxfgpnNVSe7nwZXmO0Zlt5rZ+9UQO2mQeA8ud98h9Hsp7oR0X0XdFFqaksoXiGd4b4c+kwLfFZD0jTAjrZr5FXXuDDPl5UJRR8gGpaWjhITcAgn0NtISsBOeLe0/AGZ2BC7jT3WuPKaJBDrTkUL/FY1wj/A6vAXdebNbQ+rhbtoZJL4CXGpu0RTHwFqH/ulu77JSJNgCYVVl6jSnBhfHvoPrLbrBAbhOamkzuwX39t4mOv5k2GYgMcRNB71ZRzFwmHtM94MiiyUzOwV/oSuZSZtsMdQ5rbob7d6lzPR+3IxxIvWxk3oC6/MD+AnwQ3miofh4Ky1mtiYuorqXFi9ba/ACT6R3AdxsVURWTxZCeIT/26ozqu+PJX072j+CvqREMRN4OaqzflX/SrD/75KW64BPKoQzMTdZvUA9zMtgfUEBjf6BFYswHbVBAyWVc4639XeP3Dqs+J0NX8V9uPXkHiKM7R3pC0tzH67I/k+pXm0wT3OrtI3L49Xcou1GSWtYn9l1JdQ/V8h0+LfGqJ48YGazqBRJNgWZUfQATTe/jZmY2Z2SPmCdiYv6KSYb+n5I0nIpx6zPkadMyxMpfSXSU/YDuAg3nXymVK+VFjO7Dw+VPJ7OHB03RHVWwpWoxUv1dzzeU6VpaQ3NZXv3DwOFvfsk80vrH468vD+x+pL6JycaCLqk5c+4fPxqnPltgs++nw5ENUZTNTe3fkohkJ+Z7YLPdp/AIwJ3ZZrZC1ifafFtuM/BS7jO571DTUsKmsZ4+XmVzisYYTdm19viBgJvmNl3cRHsj9Snx1oHt4qaTdJi5qFx9pK0d9LFqEdmY9Pihi837weeDPurACfW1J2lpvx6XExUhJFYG7ihVGdZPMvZvWF/ZTxnBfhK5WMV7W5OMJUrla8H7Bb+z4fHoOrV/TgTN4f9EbBSQv1GWohCnTS0cSuwYbS/AV2arQJ3E+ViDrTcHf6PjcrHls4r789U0fZMpf21cTn0P3DxyztEpq8tdHZDy65NW0JfdxFyQOMJd/6GM4pDgfMT6Z0F11MU+8vh6VM/OcDx9T1cFPlp4DncQfPQXo3fXm/hWc8e7Y/CLY/AfZ5mrThnduDBAfR1T/hdD/e32IIofAyuM1yUAYbvGfab+W7eUm4+LcwE5/y34HqMW4CHgZVLbdTGRcKDiT2MB7PbL2xnhLJlS+0cjEedfDjsLwTc0sP7keQHkEoLcCwe42adcJ9WB1Yv1bm7go5+ZS10N9m73xWVl+36G/dr6ozGxRFjcYuX3YDDE+nshpaZcNHISlQwsIS+7o7+/wJfRRT74xLbuBGPeEy45pdxndo1wBGluoU459u4RdD3afC3CPXnqClPbmdybuEZW2lcFRPCrwOXA4tHx5fAJ37fqGhrLvwb8H/FVu4r/B6OByucVBb+315RlvyeZGX2ICHpKeuMV1SOZPpTPJ/ERaH+3Wb2f9H5Rd6GJtlibVwkeQiA99MpM70BX1b+p7MZPol7ft4Vzv2beVrEnkBS2aS0CSm0FF64cSa8QpFa4DHzvOSxF3i35oVNVk+rmNnrBDl8+A99cnjM030uHI6vRp+oYBQ+q+6ApAlmNlIe2+k08zAbjfkwuqBlOtw4YndcTGR4COxCD9ZPbl2DkWY2nTzy68Z4fKoCqd+NuSQ9Ev7vihs27GceSmQMnWlBU8yl9wHOkvSqpDfNbBYz21vSid20M4QwhS8yTDJ4mC78P9rM/gHcGHQthk+ojpBUDgXyedx6chHcIGFtPOZZ/B48Yx7mZxPgyCACjt/Hp8w95hUszvbHVzVJyIxicEi6+U3MJJIt3lfIFs1skmwxoDEukjxf92lRm1tWMAlICy44VGilRQ35j83ziO+NfxB/SJ8J4o2hLBlqsHpSS3KegE1xf49FcGuh4mG/js9sY/wrfCjHmdlR+HNMYrCJtPwEF18sqb4cyaNwf5uj8TGagj/geRD+jls03RTaei/p9viK/rcFDUwxl/6CpF9MatzjV32BTqu/yWoCbvUBCqsMQx4zD4dffPj3ptOE9pfAL4sJkqIc8yXsj+ubbpO0obnF249LdbbDIw8fLelV8xSscUDKL+IREhbGxYhX0hc7qx3DsSSbWjbcC/Us3JnuBTyS5DylOufjoQbuwh1yvo7H8SmON8oWw7Gl8HzI/8JDMtxMtGStoKsyZEHo+1f4YP0CPivpedjuxHvXSgsuj/4yLoI6vtjCsW1x8dp3iOTgPXqmNsBzD6woK+tdFsdn/6Nw8duxeIrQXtH/SBX9uJjrkS7bWhtf+c0alS1LSfzXcP7vcOb01fCOzBI913I4/pOB97e0N55OUU7hwNhVO4O8v4s3baW68+MhP14I1/97gi4M91yv3Urt3Bl+x+Exvai47sWqtl5dd7Z6mswwDyf+M+Aj+KzjKjyuy0vh+Fh5cLjDcbn4760i+FmoOyvuRVw38yjqVZ4fjjUGFxxKtNFiZrficY3KVk9nhOOz4QrOzXDRU1yn1ZHIWkJQSLqiy+vpZ8liZmMUJbOZ3DCzhyUt2+2xyURLN0EDU8ylf4J/kH8VivbCLbO+1k07UwLMsxRW4RPAwooyXJoH2dwN9+HYCA/7Mb2kj0V1ipVOIYZcEhdjrxiODy6pVmYU3cM8UuTSki4K+8fhYQwAfq6G2DMVbV2CrxI2wZW1/8bt+Fcxs0WAJSTdHOoeQF/o8N9LmlDT5loqZcl6N6LJhDAcnwGXc++I+y7EjOKHdedF54+mLwTFyZRCUNQx24p2lse9hY+ic7k/CldMrhjVXRcPlLc4nfGremVC+2fcX+LMUvnOeNa+ZMe+yYEwcXpJpQ9Porn0CFxX8pFQdDXwa4Uouqnt9AJhknEC8D7cgW0k8E95/LXk9LmhLcMdJL+JG74cJumemn7Xx8frFWqOWNAR8yyYFP+CPj3cZ/AVfFq4m8m1RJuaN9xa50PR/v24yd5ngT+HshXx1J5FnePwgGanEi3dcWXnp+izDlkQ+Gj4/wdgy6juQ3g44u/hSr2ifBTOuMp0rhx+9yCypMBt6QtrpC8O8b1LpgUXWXwh3JNJEUzDsc3CfT+CGtPjBFrGRf8fKB0b20U7W+E6opfCb7Edj3vZx3UfxE2X58fNouehIqvhIO7vwrg13vW4vuQY3LjhDnymOpTPeu1AxwW4YcK9uFnrC8BmNefMT6LoBPd3+cVg2xngtdVar+Fx2aDFNBmfKHw+jInT8VA/VX0tTZ/IaQNcHDtnAo1xBsKqjHzJVk9DNmimpg0YXdq/Lfp/c/htZSahvFa2SH+Tx7HR/5vC73a4cmoc7iH6gahOYYp3Z/wxos+UbiZKPhtDcO+SacGVba/i4ZYnEoVXxpWrA0oTWr4/Nfd6IKlQy0xhJkqhvalIjTqZ7vNG9JlLbzyUzziiYTQuWtwWF5esHcqXp7/fxydw/co/w3P+HxXh4nGGc1QYE9fRX6+V1E4vri383hOVje3i/H1wHdtJuNSgqe44nKm8N5zzE+CyUp0Dou3ruD7kyuj4kfjqewl8NXsgbko7afLVtGWrp4Ghw4xTUmy+WWQrW1DSrVH565L+CGBme0Xll1IhWyQEPiv1u3H0f97w+208KdCzZrYW8Fsz+5bcaqewvjF1BiU8L9D9nyBHHkp0Q8vXcEVvVWC5XoRtaDU37QaS/moeIXZT3DN9E9zw4DzrC7J3XZC1X0BnWJJkcWUiLdfiEV2HE21BA2Mciq9A/iLX2W2ImzpjZsvi93MH+kKkmKqt4mrb6TFqrdcsLYryCfjKaj1g3eh+VOlU/iePBvtJ4ARJJwST6hhtMc+2C7/xtwdcBCXcYKYWmVEMDH8zsw9Kuj0uDHLLInJpCjNBUUyn0MbquBkdeC7nZSU9HOq+HOosj4tqAEZKejYcvyO8GJeY2aL0yUfnLNHy49DOCPoYzlChG1om0Bc/qOdQmrlpEoLseEfgY7iYZ11gKfWFdikH2Yvjc4mG0OnvYrQFDYzxX0kvmdkIMxsh6Toz+2k49iC+gtxSQS9nZl+t6bOpnV7iszhj2BcXkS6KSwwgLYpyShj3Av81sx1w0VURJbcjTpZadHJKCxtfi8woBoZvAueY2ekEhzFgDfxBFiGCU5hJP6gzn8LB+Ef/sFI/36bPHv4NM1ta0qPh/GfNbAM80myhRL0q+GaU83kfQshzPITohpZ/4jO26+icfTfGKRpqmNnTeGTOk4Cvy+PtTIyYBDWz36kd3azYXg1WbDcBZ5nZC/QFFPwUPvO9zsyuwE1O60LYN7XTM6hPOf4f3I8nRkoU5ZOBK/AwOw+2dLcb7gdxmKSJZrYkfQ6mAJgn2zqQkiRC0kbmOdn3oe97cB+u22mKytyBbPU0QISbvy/9b/7z4fha+BL5dCqYiYJVklXnU5hH0qbh+Er0DQBwheBPFILemQf3+qdKFlDmDoDbSTormNX+BnfauTtUWQWXIX9eA4y4OhB0Q4uZ7VrVhoJ5bKizuaTLS318Ue7MNCQIM9at8Wfze9w7eLw6g7Z9HJdnPxH2v09fkL0vq8d5y99tCOPi3/g7sBNu2XNWLKYMdbbCP8Ab4bHF/lSIt1Lb6RG9SdZrVhNF2dybf7OwLYuvPK7ARWb/jM4fCZwpaacWeq7Cvzdfx5nKrng+movwMXk6fWHRi+/QTpJuSbrezCgmH9qYSahzcHTK27iS7o+q9qweLD1LRbTcX6xChgOptAQ5cGH73y+8ibmvxXeDTB7znBAbStp88lBejWDiuAH+UfgY/oHaA1c6/sPM7sGVuf8ysy1xR7sdcOXstsXEYFpGMG1dRtJfzGwWXKxa6TNkniJ3W3zStfFA2xkErQ/iIqcxRJEW1OcflRRFOdQdAXwQt4bbGGd0V6kvpPzNeKrUJnPYMfLQ5PcU+g0zuzPQ9iVJY0v1VwV+pUTz2MwoMqZYBBHaGTjzNFwOvKukG6M68wKX4P4Lm+EWNTs0vVSTG2E1Vyi0N5U0r5ndrZCD28xOxZnekWG/0V9kWoB5KI49cQucpc1sGeCXZSYwVO0k9HN73UfWzM7E465dhkdhSA55H86fFx83Z0XtvQ9nNnGek2Ojc26TtLZ5zLLjcfH2+cCbklao6ef+umP96mZGMbxoki0OG1FTCMxsDB4J86Gwvywu612jVG9+PMTJGGB3TUGD2sxmlvTvsKL4EK6cnwh8WtLoUCf5hZ1aYWbj8Oiot6svL8v4srHHULWT0M8RuP9EP+s18zhWxQc9HouFRdOoqJ2j8LD8/8ZFTyvjHtO/i+rEUodJUGeCtC1xvcyiuEXVKFx3ciRupv9Kif658XD8y6dcb1ZmDz/OwmWLW9IpW+wagekgaUDnT4GYvmASAJIeDrN1zOwN+syKhXvHLgVsY2YdL+NwQlJh7fNT3B7+ddy5r2ASqxEFeJyG8aY8WCAA5lFWB8Lwe9VOG4rVRD/rNXUXRfmjkg40N319HFfc34jHyfJG+5Kc1SZIk3RJ+PsaMMlwIuhCrjKzr9OpKz0SdwJOQmYUA4CZXUzn4BNu331dPBNIxDySTjGz/eWZ224IssW4v0XwWcJ6oa+b8HhRTwfZ+MG4LmSEV7e3cXvrQ6I2igBqSTOIyYkuaBltZr+h76XZGVd6I6ln4dGHApJODWKB+elT4oN7Ku82PFRNUbjBzL6NW0dtgpuIXzyM7TSih1ZsxTd4C+A8Sa9ZycfEoux0QEd2OqsJExLR+WUz+xvuX7JiqHs/nv0u+b5k0dMAYNV5j+fGP2SPSDoolZnUyRYlLR3VuRq3XIhzLuwkaZNgNbU5sKekiaH+Urip5hWSjovauRD3ZH1ykLdg0EihJSgE96Ev/PeNwEnysOpFnU8C10p6LezPCWwg6c+TifQkBAXlbJJeb62cUdyvPYiCRAK/6VaM2Kt2EvqZA5+gFbllbgAOKcZhF+0cgVvM/RsXmc0JXBLrP8zza28DXBSJ0+6VtJLVWAYWUGQhOBhkRtFDhJnyGEmrpjCTcE6lbFEh4GCoM06lHNpFmbmH5iYqeS8HMdRVigLbmdmNuJXNHXQqxYY8UFwTLYH2+STdXzpnReCFWLRWc2/GKjGgXy9hZr/HxYfv4KFKRgE/k/SToabl3Yhg4bY8Pql6aKAGCb1qp6WPP+Lm0MWH+LPAKpI+NYC25gZek/ROsNIapZCrPBwvcoWPjRjF3fLAoTPh6VZfLLU5H/CGemQ9mUVPPUR40MX/G6rqmLv3T8ruVSdbLOEl8+ifReTHHfAAdOBy/KoQFy8W8vwI30u8lKFAEy0n0JmMpsDceP6JHaOyKnnwcI3rFSS9bmY74WkuD8KfdWYULTCzLYBfAo/iK4ElzWwvlXxkhqqdBCwt6dPR/g+DIj0JZraRpGvNE2YVZXGVC6L/TQnSjseV4HF98FX4R4EvpdLUhMwoBoAwAyhjLjzpyH0VxyahYCYpssVod3f843lcOOdW+uTaTbOljmOSbrAKG/MmeicXWmh5ryIT2Oicm8zspFLxaDM7Fg+hDC6qGsPwYPrwIm+Nh5v/r4UMfjHCynMBOh21hl0cOMw4Bvd/KUJ0LI17NHf7ge9VO234t5mtp74UAOvSP0xJE9bHY3F9vOKY6Pzwx9npnsEjGBTZ6daQtCclSPqTmf2oC3oakRnFwDCGPosbwv+X8GiWX4IkZjI6paPwUflxg3ioCJPQ71RKYRIssjHHQxcvjM++empjnoIWWpoU1eVV0n746uQc/DlcTTcpHnuLX+GWK3fjuZAXx62cJsHM9sNl28/TFwtJuFnktIw31Bld4DH64pkNRztt+BJwRtBVGJ786nOpJ0s6OPymGDKY6j2z++VkjzAi6DBfk3RKR4Nme+Aiq5+m0Jt1FJMJZjaRembyI3y2nyRbtATPzESaxjEENuaDpcXMLsU92C8rnbM5Hu5i87C/oDy21Xtime6UBDObTtLb0f4E4IPqcUiJdzvCSnFx4Fz8XdkWj5/1FwBJZdHKZG2nC7pHhXYHZLSQohQ3s4fxCcg5eNSGV6NjN+D5XToSlZnZB/DV1ax4RIByRIMZ8FDpSROUvKIYAMzs/5qOS7pRLdEazawICpYiW3wMuCXoNzo8M4PI5r/FQDCz5fAQEo/LQ43HGCob8xQ00fIV4FIz244+MdKaeFTOLaM2DjGzb+GOReXwyUOOYKX1aTxkQ/xuHRL9fwrXR2V0YiZ8lVUYgbwIzIyLZsqimKFopxJmtrOk31lnjLZJ+gUlpOAt4VRcKV6EAf8snvRqku5C0rLmseM+A3zHPN3r2cF68hvAueYBSuN3ZZdQ/+QykwhtvmXWP9Z7HTKjGBi+UVFWiA8WBUa2MRO6ky0+GrYR9BfLXIGbAz5iZu/Fc+GeBWxpHr32oKjuDTYENuaJqKVF0iNm9n5cab1SUR+3Hf8PQDALfBK3mjrVzHZRKf3nMOBCnAmMIfLWLeEx4Pqwaoo9erv9wExVSBTBDFk7DZg1/FaJRwcy6UpSiocVwx1m9mM8TtgZwO/kqQXWwsWtnwvV78VXrS+Yh1tfQFF8OQAzW6AbIrPoqQcIiqzv4jqIwyRdbO5HUUbMTB6W9L6a9h6oOmYVnpmx6MjMDsVj3OwTlpZjYrGSDZGNeQoGS0uQ/6+HW4D9HrhFPc6L3C0s2La31GkNxzAtwvqcStcNRZOcSoejnYR+1lUp8mpVWUI7f8VFR7FS/GhJ60R1RgGfxFcISwN/As6VVGm0YWarKyTCMrNd8NSpX6PTM/snuMFFmp+FhiFF4tSy4YrX63G9wyYtddfFLS9uw5fBNwBrVdT7AHBjqWwd3JvyybC/CnBi+B+nYrwF2Drar82JiyuRVx7uezgYWnDl94jiXgz3hucYeH9i3dlwh7xhp3tK2HAjhN1wKcd0+Oz46uFqJ6Gffqlyq8oS2lkFN354HA85Pxb3x4jrTMQtHtcZCG24Q+4NuI70pfB/827ozCuKASDYan8HFzMcpjAbqKm7MW6VI9x66epQvhaucDudCtmiooRH1uyZ+Ts8DMQzuN3+kvJQ1nPiOahXidq5Hs8pPF3o8wU8MFhdtrDJhsHSUlYSTwkIsuP34i/2m9A/raV5fpHf4swR3Ft/F3UmtZnmYA1OpcPRTkP76+DBHb9CZ6ykUcAn4/ety3ZrleJmHrysi7bGqscOp1lHMTBcDDyNc+cDzXMgTILcuzhmJt8tMxO1yBbLHUp6qqR7KmLgfwF3wFkCDzBWiKZWAI4uNTOH3CHs83gylIPNo5oOB5JoMc+jvZii4IABd+BJnjCzEyTtN/lJbkVKDoyTgQMkXQdgHkr91/jHZ1pGk1PpcLRThxnw1eB0dOopXscnc12hbAARKcUPMbOfSvoKcJFV+OOo3mS+Q4xpnh55X9xbHdxZ7+eSrk+lMzOKgSElIFgrMwkMYZLMOsgWq9IT1npmyqOTHhFXNo9n/1dJt5bamc7MFsQtLL6TcA2TE620mGeFOxp/OZc0T7ZySHhBYq65bsXpQw5JT5gHbPtwKLpJ0t2larMWTCKcc715VrZpHU1OpcPRTiXUF7jzdPVGJ9ZkAFHEditP+NqwFp4KuZB+/By3vDsEf29Wxw1A9lXJBL0OmVEMAKoJz1HCQKJL/oYwSy6h1jPTPAf3EbjDz6H44JoXd7bZRdIVUTuH4ErjmyXdaR488JEB0NkLpNDyA3zQXw8gaZx5vmAYPrPeWpjZ/vgKrzDB/J2ZnSzphKjaY2b2PToDPD42hGROcbB2p9IhbScR/zKznzD4PDKLSNqs6oCCsrrpe2Nmx5eLgM+a5w0HN57ZujRhGWdmo3GGmsQohl2J9W7c8Ly9+0T7t+Mv+2PANoNod2xp/8jwu23DOaNxy6FtgVdw5xrwZebYgdIyJWzAbeX7QlDe4wmA7gHGR/+L/XuGks6YNnzFUOzPWqYFt4w7HrdAuQufAMw13Pd6uDfgZmCGKaWdhH6uwq32HsB9Nk4t3tcu22k1gMBXzFcDD4dvzETgsXDsKTwM/y54Lptdcd+R4v+DDe3WHutXd7gHyLtxw62LFo32xwHzAIsB14SyrpkJkcVS2B+PzxBqrSmAcdH/B0rHxpb2j8KVbtMD14QBtfMw3cNWWvAY/DuGD/Ay+Azol+HY4k3bMF3TeGCmaH8mYPxwj9d3wwaciUfc/R5wQLENVzsJ/YwJv7HV4Z0DaOd+PErDQ9RMdIAHcf3X/OE7Mw+exwZcT/JT3ER8oVD2WJnOpmtI2bLoaWCYQdJT0f7N8pAML0Xy5gNxu+cCM+Kmr7Pinpfnm2efQtJz5qE7RpjZiuqzgLkCXyXMZh7PqcjmFqdU/F/URzkoWVk805pNawiRQst+uP7iTfxFuBIPf4KG2WeiBqcBt5vZn8L+1jizo1BMWv88JcDwhHqfwtDkVDoc7bSh8HZ+NugB/kafJVs3SDGAeE010W8lvQF8xczWAM4KjpxxROWlzSM6lGF4RsgkZPPYAcDMJkh6b82xR+VJ3e+U9IGo/OeS9g3/b8M/KgfhD+xI3PLpXtyJ7Ch51rsZJb1pZhdK2qqmv3fwsB6GhyoorJ4Mn91OH9UtTGp/gydHusJCXPvB3I+BIIWW2HHo3QIzW52+REs3SRobyteQNMaq85SgNL1XxhQCS8gj03L+KLnVXyVzkfRyVLc2P3epTcMjHKwjaedQVjneojaSxl1mFAOAmZ0FXC/p16XyvfDsaju0MRP8g/5B/OP+BB5a+zkzmwvPgreqmd0laXUz+62kz/aA7tZsWkOFFFrM7DrgPcD5wDmS7h1qOrtFeH6L0hlC/K7o+P6SflY6p1/ZtIbwrKtWWl0ph3vVzuSGmV0iaUvrHzwUXFqwVFT3un4NeJ3kazJPcFR8jyaoy4RGmVEMAGY2P25+9iadbvEz4nqG59uYCbCcpMIPoDyTHitpNTO7F/gxbs3UL76UBhAJ0zqzac2KR7AdlsirKbQE8dx2wPb4rO0cST8q1anztRhSmIdQ+Rwu+iherI4XumD+pfPGahgy8k1JCKKTAjPhvgVvSzqw5pTJ2k5CP2fgoUFeDftzAcdI2r2X/QwW5sE2f4ybDT+BM6RFcYnGd1QRMLCyncwoBg4z2wg3jwO4T9K10bFGZoKbpa0tT26ziEIsmsD5b5enOVwP2An/UJaXtOp2UJpHmj0A/6juaWbL4AzrkpZTe45uaTEPEnggsL2kGaLySb4Wksq+FkMKM3sIt2DpFw7ezHbAFfPr4SKLArMD/5M05DlBpnSY2R2S1ppS2im12Y+5d8Pwg4iyFpLuslKEWnzy8XdcJzoxsZ/j8DH21aDPKLzAjwb+LWn/lHayMnsQCIzh2ppjLwAfKjGTSwtmEpS4CnXjgGXz4AG8kHtz32xmoxUlHgkz6DjcdipOwx17Ci/gZ4DzgCFnFCm0mNn78JXEp3HHxXMI9ybCD6j3tRhq3IuL0KqcJm8FnsV9XI6Jyt/ArV2maZRk9SPwSdUcw9VOAkaY2VySXon67eZ7ekzDMQEbUa2MXwIPNf4DSWfHB4JBDOrMcbMlsKyiFUHQjXwJt6bKjGJKQB0zUU3qS0nP4B/NuOyU4Ey0KR6SYBPcXvy8LslZWtL2YXaLPCZUckz6HiOFllNx5rCppL/VtPNfSa+VTh2uZfLhwNggMoyVjp+QW2k9gQd4zOiPOGvk27ivwB7D2E4bjgH+ambnhb62AQ5LPVlSq0OuaiIKB6b0F+Ds8M4cjIfoGOGH7W3gBEmHeDP9xUZB3Jv8nmRGMYUjWC3siCcjugN3vllKpXDjiXgrrEYU2l6a+rwJkxuttCgKtdyA+8xsRzwHyDJ4SOVy6JKhwhm4Bdt4Os2WJ8HM3qCPkc2A+5H8U27qPM1CLYm+hrqdhH7ONPduLvRPn5J0f+r5ZvappuNN+kdJL0eTqq/i34QPFOIo8ygHJ5nZV4H7rSJXi3k8rAeT6c06iikXZvY0npznJODPkt4ws4kDfRnMEwR9Fw8YeBU+wD6nLoKD9QpNtJjZuZK2M7PxdK4OqqKxzoL7WsR5LQ7t1qqjF7CSSXRCfcMdM9dWZ4KpaQZmdqCko8L/bSWdFx37saRvD2U7XdC9WFV5naSg4vzTGg436h/Ng/x9T9JGZjYWT3Hw91Kd+fD3akvcrPbfdEapnhmPdtshvajtMzOK4UeNbBEz+ymu+L4Xdzi7EPf0TXaUqehrHmBt/KN6Gy4Cur35rMmDOlqsLxf24lXnacp0tsPMjsVXRRfRYO9ecd40a/UUW4GVLcKqLMQmdztd0B1PYmYGlgQekrRi/VmD6qPA3Lhz3y6SHrSGZFnxsZKu9H5J13RDSxY9DRMSZIvIPXm/ipvT7oCHvZjDPJf0ZZL+0W2/cg/ySyM67sJDjww56miR9Gwo2lvSN+NzzOxI4JvR/rLA1ynlqdbw2M0XH/u1o7JCMQn0EzmMwGd3Q776mYJgNf+r9oeinSQoyhwJk6yY9k49v8Kiqdz+sfQ3WBHwkqR/RmX9LOyqjjUZ3qQgM4rhQ6NsUdJx4GtQPIPedeYhxguF9om4Bc1gMVzK7CqUadmEiCkEbF4qOw/PdPcb+nJ0DAtSFJR4dsMCb+PhSyq97qcRqOZ/1f5QtDMgBHPWbhxXW8OLJK6cVzEP71OGEUW1HSyy6GmY0CZbbBNFmNnM8lwUg6XjSUnDsqIoo6AlmO7tjceieTSqMjueG3vn6JwxktZgCoB5VsFd6L+6+fIwkTTFw7oIQTMU7XRBd7wiGIGnB5hH0qa97GdKQV5RDB+mLzMJcD1FWDk0ohsmYTWB6PCXaJ7UdnqBRFp+j+cXPxyPh1XgDYUYOJG9/MVmtjeecD7WC7zM0OMyXNfSZPW0FB5afG38PvwVd4aaJnNSSBo5JbXTBeIVwdu4CPWPQ0xDYczxXwUPazNbDreQfFzSnxpP7qafvKIYHjQp2HqtfLMeBQYbLlrMvdzj5DBPWnWMnKjKwBX+A0XKczMPCPkL+lJ1fgbYT8MQbyvj3Q8zuxHYQ9IjZvZe3IT+LNya8M5eWdNlRjFMiJbK/Q4xGZbK70aYh+c4FlgI93ZeHM+5saKZrSPpr8NKYAnB8OAfuHd55erGzO6JzXtD2bBE8M3oHg0rYiA9XLyFQJBmtq6kWwZBz/hCsW4ea2xuSfuY2Qx4von3N7eQhix6GiakLJUrBmUR6+U6ScORQ2Ko8SNcRPMXeZDEDfHUoeCz8p6aPPYAbwE/wf06JgUFBJaKRGWXm9lBwNnh2PakpqPMmBJQ5K/+FB7ZuHgPdwCe76Kd3XAR5AkMbhzH34eN8PGHpLfMrFL8ORDkFcUwIUW2WCOmmRv/WD4ytTtpmce4WtPM7gZWk/S/YvY9JfoemNljwFpVuqcpUVSWMXAUY7OtrOH8P+Cm0QvRabDRz6m0pZ3fAc/hYX8OApaUh8OZE7ihVyvVvKIYPlyBx6ApZIt/xWWLW5rZByUdVKc7MM9YNYZORe/UiFfNk8TfhGfveoE+cd2SVp25Cxi2jHET6LO26YCGKLRExpBhVjNbqjBCMA9EOWvLOZMgz1nzHjySwGDG6hfwwH5L4Fkji/G3An2rn0EjryiGCYOVLZrZOEmrJvY1xYiwuqHFPEfFv3Hzw53wKKBnSXrJzB4BPl/Xz1Aq6AuYp0BdEfd7iXUUXzazjSRdazUxfjSA3CIZwwcz2ww4GXgMXwUsDuwl6coBtDUDsGzYfUiJOSIa2psXd8zr2cc9ryiGD62yRatOkzgXbqt/X8WxOlTNLOYGdjazlYZYhJVMi6R/hjAey0g6I4jrCt3OG8PBDFrw57BVYX3cM/bjFceEx+PJeJdAnrp3GWD5UPSgpK4DbAbx8pm446UBi5rZrpJuTDx/beAI4GU8wdlvcUfcEebBAK/olqZKSMrbMGy4Euxo3EP7eWCWUD4ncHf4PxGfsUyM/t+Jh/IY1QMaRgLjhvte1NGCL6vvBB4N+8sA14T/Fww3zTXXMQOwUtimLx0bAWw33DTmbVDP98Do/7alYz8eQHtj8IRdxf6yuEQh9fzReEDMbYFX8ACT4AxsbK+uO4uehgnmIbb3BxYETpV0dyj/EB4c77dDRMc4JYqwJjfKtJjZODwp0e0KiutYZDelwcw2wEONPw6TUk7uqmh22I3CM2PKg/U4+GCNuXS/sobzJ70zZvaApPdFx8aqRwYfWfQ0TJB7Vh8RlwXZ4l8l3Rr2/6+ljdTlaa9EWINGl7S8KRfFFedOx/AlJUrBMbhC8SGYFLDwD3iWtQJ/MbOv4wmZJvnRaHg8yTO6R6+DD442s9/QZ2a7E75KSEVsAluO1pB1FO92JMoWv1FxqoCV8dlqatiCOOtX0cZLuNL1SwO9hgGiG1puMLNvAzOb56/YG7h4qAgdAKYvmASApIcrwrFsH373icqEx7XKmPLR6+CDX8LHQhEP7CY84GcqiqCAhr8nRYDAHBRwaoB5dqxv45Y8JwObS7rNzJYH/lC1ZDSzdfFkP3MBh0makj+ag4b5UuLzdCYl+o2iQWtm10jauHRev7KhgJmdis/witnhzsAIRUlozGwmlZIqVZVlTJmwIQ4+OKUgM4phQjeyRTPbGPgePmP5saSru+yrJyKsXiCVFvMc4fdJWr6qnpnNBMyCr0Q2oG+FMgq4ou68yQkzmxGfHa4Xim4ETlJkDVMlxx6IbDsjYyiRRU/Dh1bZopltgYeDeA34rqSbB9hXr0RYvUASLfLk7w+Z2WKqTi+5F/AV3LN1DH2M4nXg570mugnmoeHnk+dMPjZsmNmKOON6MThXLYyLB1ajk7HNMpT0ZmR0i7yiGCakLGGDP8XTwN1UyD81QO/jKUmE1USLeWTM1fCImLHi9xNRnf0knTB0FPeHmZ0NnFhemZnZh4EvSdrRzHYFPoeHbbiTTsZ2hrLD3TQJM3u/pPHDTUcbMqOYglET62kS1KXD2WBFWL1ECi1111++bjNbCQ9ZEIciP7OnBDegyeTVSjmNzexASUeV6iypkOUwY9qCmd0EzAicjkcdeG14KapGZhTTAEoirMMGIcIaElpC7KsFVAq/bGbrAc9KejQqOxjXUayAR2HdHLhZ0jaT7SL60/uQpOVSjtXoKKaYLH0ZQ4/g4b077jR3B3DacE7iqpAZxRQMM9sKWETSL8L+7cB84fCBks5PbGeyiLAGghRazOwS4FvlJbmZvR9ffXw8KhsPrIJ7oa5iZgsAv5O0yeS8jhJdlwK/kHRZqXxz4MuSNg/WbCviXvWxnmYU8A1JKw4VvRlTHoLxxtbA8bg40uD/27v3YLvq8ozj34dgICjXclFRINyEci0Xy60tSKODFQamMyCBljoDOELlUlumop0ySoc6Kg0TpxSh00ILFloNV0FLwXAHE25JwACWiw6xIJYYKASMT//4rQP77Oyz905ystY6nOczc2ayfnvtw+uMZ797/S7vy7ltmZLMYna7nUPpgDZiPWB/SpXKfwKGShTAYeMc15oYJpates3b2l4gabuu4ddcyo//StJGlAZHHxyHOFfFWcBNko6lLKxDWYs4EPhEdf2h6t+bMLre0zL6FDeMdzZJe1J6U/wB8J/AkbYflPR+SkXpJIoYaKrtn3Rc32X7JeAllcqqQ1nVtYy1achYNunz2rSu63kqtfcvpXxIv0L5A6uNSxvKPYCZlBpPAHMp1URfr+65DrhOXZ35qm2+vQoFxuQwG7iM8vTw1u5H289L+mJzYY2WqacWk/SU7R3HeO3HtncY8veMyxTWeBgmFpWmLrfZvrTrvScDM2wfRw/V08ZGth9da/8DxkE1zfAxSle0GdS8phLtIeks27O6xs60fVFDIfWURNFikq4EftDjA/PTwKG2jx/y99wNfHLk6aQqtnc41RRWnaeYh4mlWmeYQ2kt2jmVMxU4xvbPun7n1pR+AG89Idd5iHBY1S6umZROhg8ABwPb++1mMzHJjLG5YdSB2zbI1FO7nQ1cK2km8GA1ti9lreLoVfg94zKFNU4GxmL7f4CDVHpkj0zl3GT7tu5fJukrlPpJjwErqmFTTkW3hqSfAs8BFwN/bnuZpKeTJCYnScdTvjRM1+hOjRtS6r+1Sp4oJgBJH6HsmIFS1mKlD8wB7x+XKazxMN6xSFoM7OnVaBqzNqiUj9/GHcUBq/FZlOS+ELgKuA5Y4PTKnpRUGnJNBy5gdEvjZcCjtn/VSGBjSKKYBMZrCquNsUi6mdJA5pVxDHO1SDqS0oxqqu3pkvYGvjSy/ViSKGc+jqdMP21M6Zv+3TbEHzGWJIpJQNKWlBady+kxhVVN9UyoWCTNpkwxbU05R/FfdPWpHr+ohyNpPqWt7Q88oNGSSvnxkQXtj9nevNZgo1GS7rJ9iKRljD5PJMC2N2ootJ6SKCaRNZ3CalMsVe2kMdm+fHVjW12S7rN9QOdipIboViZpWufWyIi2yWL2JFJ9GDeWHDqNQyxLgXtsvzBOIY2HRdXGgylVWYYzgHsGvSlJYvKStAPwU9vLVVrp7glcYfvlJuPqtk7TAUSsphOBhyQ9KelySadWxQGb9FnKU9JyyoL1Usqp7YixfBtYUdU3+yalqsBVzYa0skw9xYRWHbI7qPo5ENgG+KHtjzcQyz62Hxx851v3rwO8x/YvB94c70gj5ygk/QXwuu3ZbTxHkSeKmNBsP0NZFH8IeJhS66m7zEddvi7pcUlfHuvpRtJVkjaqzowsBB6rPiRicnqzOlNxEnBjNda6dqpJFDEhSTpX0g2S7gM+Tzm1/Q3KmYpGiiBW/93DgBeBSyQt6FGv5zerJ4ijgZspe+n/qNZAo00+RXkS/hvbT0uaDvxLwzGtJFNPMSFJ+hGl690NlAXj+92ipi9VkcBzgONsT+0YXwTsTZmH/obtuZIesb1XM5FGU6qaX1fYPqHpWAbJE0VMSLZ3oRTUm0c5xDZH0gOSLpX0qSZikrSrpPOqHhmzKQnsA123XQI8Q6ltdUd1QjdrFJOQ7RXAtpKmDry5YXmiiAlP0rqUQ3u/C3wamG57SgNx3AtcDVxj+/lVeN+6bSvZEPWQdAWwK3A9o/vCX9hYUD3kHEVMSJKOoux0OpiyJXURcDfwOYY4u7A22D5w0D2S1gP+ENiO0X9/X1pLYUW7/bj6WYdSELCV8kQRE5Kk71ASwz3AfNtvNBjLNbaPraacepVj2LPj3lso5yvm83a1W2x/va54o30kbdDmSsJJFBFrSNL7bC+p1htWYvvZjnsX2m76YGC0hKQDgX+knKfZRtJelM6IpzUc2ihZzI5YQ7aXVP88zfaznT9A9x/8PdWOqAiAWZTikC8B2H6EstbWKkkUEeNnRo+xI7quDwHmS1os6dHqrEWrW7fG2tXVyAs6piTbIovZMeFV2wt3ri4X236z5v/+ZyhPDtt3fehvSFlH6dSdOGJy+4mkgwBXpefPBB5vOKaVZI0iJrSq4ubllLMJohRVO6nOntmSNgY2pUe3MtsrtbWs5qF/p7q8s5puiElI0ubARcDvU/7/+33gzKo9cGskUcSEVjULmjnSelTSzsC3bO/bYExbAuuPXNt+ruO1M4FTgO9UQ8cA37Q9u9YgoxUkbWH7xabjGCSJIia0Xo2BhmkWtJZiORK4EHg/pTjhtsDjtnfruOdR4EDbr1bX7wbubSLeaJ6kJyhPw1cD325bH4oRWcyOiW6epMskHVr9XEop69GE84EDgCdsTwcOB+7rukeMXqxcUY3FJGR7Z+CLlEOjD0q6UdKJDYe1kjxRxIRWnXQ+nbKbCOBO4O9tLx/7XWstlnm295P0CPBbtn/dXfBP0p9RSkrPqYaOBv7Z9qy64412qdYrLgROaKIETT9JFDHhSdoCoOm5Xkm3Uj74/xb4Dcr00/62D+q6bx86Epvth+qMM9pD0kaUdapPAjtQvkBcY3t+o4F1SaKICUmSgL8G/pS3p1BXALNtN1I3qVpveK2K5wRgY+DK7h0skjal7M56a3v6qnTGi3cOSU8D11KSw70NhzOmJIqYkKopnCOAU20/XY1tD1wM3GL77xqKa1tgJ9u3StoAmGJ7WcfrXwb+hFIIbuSPz7Y/Unuw0ThJ8gT4EE6iiAlJ0kPADNs/7xrfAvh+Ez2HJZ0CnApsZnsHSTsB/2D78I57FgN7NFnEMJonaZbtsyTdwOhCkgDYPqqBsMaUk9kxUb2rO0lAWaeoTrg24XTgw8D9VSxPVmcqOi0ENqGsX8TkNdLu9GuNRjGkJIqYqPp9I2/q2/py22+U5ZO3Gip1f1u8AHhI0kLgrZ1ZbfsGGWvXyGK17blNxzKMJIqYqPaS1KuFqOg4FV2zuZLOBaZJmkGp/3RD1z2XA18BFgC/rjm+aBlJBwPnUQ5nrsvbPUy2bzKublmjiBgn1U6sk4GPUv7gvwdc1rlYKemHtvdvKMRoGUk/As5m5UZWqfUU8U4jaQqwyPYuA+67kDLldD2jp56yPXYSknS/7d9uOo5BMvUUMQ5sr6h6TGzTWQSwh5HdWAd0vh3I9tjJ6XZJX6UUiWztF4c8UUSME0l3UBLBA8CrI+NZqI6xSLq9x3DrztUkUUSME0m/12u8c2eLpE2APwa2Y/TJ7DPWcngRqy1TTxFrSNKOwFbdWx0lHQIs6br9u5SKstn1NIlVlQU6Gfg5cNdIpYE2SaKIWHOzgM/3GF9avXZkx9j6trs/JGLy2bDH2HbAFySdZ/vfao6nr0w9RayhflteJS2wvUfH9dnAK8CNjF68XKllakw+kjYDbrW9T9OxdMoTRcSa26TPa9O6rt8Avgp8gY6igECrDlhFM2z/QiNH+1skiSJizc2TdIrtSzsHJZ1MOUjV6XPAjr3qVEVIOgz436bj6JZEEbHmzgLmSDqBtxPDfsBUSlOaTk8B/1dfaNFGkhawch2wzYDnKbviWiVrFBHjpPo2uHt1ucj2bT3umUPpj3w7o9cosj12Eqn6lnQy8JLtV3vd37QkiogaSTqp17jty+uOJWJYSRQRNZM0Fdi5ulxs+80m44kYJIkiokaSDqWUGn+GUmH2g8BJtu9oLqqI/pIoImokaT4w0/bi6npn4Fu29202soixrdN0ABGTzLtGkgSA7SeAplq3Rgwl22Mj6jVP0mXAv1bXJwLzGownYqBMPUXUSNJ6wOnAIdXQHcDFtpeP/a6IZiVRRNRA0hbAFrYf6xrfDXjB9ovNRBYxWNYoIuoxG9i8x/hmwEU1xxKxSvJEEVEDSfNs7zfGawtt797rtYg2yBNFRD169R8YkV1P0WpJFBH1eErSx7sHJR0B/HcD8UQMLVNPETWQtBNwE3APoyvMHgh8ojpPEdFKSRQRNam2xs6ko8IscJXt15uLKmKwJIqIiOgraxQREdFXEkVERPSVRBFRM0nTJH2o6TgihpVEEVEjSUcCDwO3VNd7S7q+0aAiBkiiiKjXecCHgZcBbD8MTG8unIjBkigi6vWm7aVdY9l6GK2WfhQR9VokaSYwpTqEdwblEF5Ea+WJIqJenwV2A5YDVwFLgbOaDChikBy4i6iRpH1sP9h0HBGrIokiokaSbgfeC/wHcLXthQ2HFDFQEkVEzSS9FzgWOA7YiJIwzm82qoixJVFENETSHsA5wHG2pzYdT8RYspgdUSNJu0o6T9ICSnvUe4APNBxWRF95ooiokaR7gauBa2w/33Q8EcNIooiIiL5y4C6iBpKusX1sNeXU+e1MgG3v2VBoEQPliSKiBpLeZ3uJpG17vW772bpjihhWFrMjamB7SfXP02w/2/kDnNZkbBGDJFFE1GtGj7Ejao8iYhVkjSKiBpI+Q3ly2F7Sox0vbQjc3UxUEcPJGkVEDSRtDGwKXAD8ZcdLy2z/opmoIoaTRBHRAElbAuuPXNt+rsFwIvrKGkVEjSQdKelJ4GlgLvAMcHOjQUUMkEQRUa/zgQOAJ2xPBw4H7ms2pIj+kigi6vWm7ZeAdSStY/t2YL+mg4roJ7ueIur1sqT3AHcCV0p6AXi14Zgi+spidkSNJL0beI3yNH8CsDFwZfWUEdFKSRQRNavKeOxk+1ZJGwBTbC9rOq6IsWSNIqJGkk6htEG9pBraGri2sYAihpBEEVGv04GDgV8C2H4S2LLRiCIGSKKIqNdy22+MXEhal9FlxyNaJ4kiol5zJZ0LTJM0A/h34IaGY4roK4vZETWSJOBk4KOUpkXfAy5z/hCjxZIoImoiaQqwyPYuTccSsSoy9RRRE9srgMWStmk6lohVkZPZEfXaFFgk6QE6TmTbPqq5kCL6S6KIqNdfNR1AxKpKooiogaQdga1sz+0aPwRY0vtdEe2QNYqIesyiOmTXZWn1WkRrJVFE1GMr2wu6B6ux7eoPJ2J4SRQR9dikz2vT6goiYnUkUUTUY15VEHAUSScD8xuIJ2JoOXAXUQNJWwFzgDd4OzHsB0wFjrH9s6ZiixgkiSKiRpIOA3avLhfZvq3JeCKGkUQRERF9ZY0iIiL6SqKIiIi+kigiIqKvJIqIiOjr/wHYRuyWwbHeFgAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" } ], "source": [ - "len(our_repos), len(our_repos[our_repos[\"size\"] < 70708])" + "our_repos = pd.read_csv(data_path/\"repo_infos.csv\", parse_dates=True)\n", + "\n", + "# display bar chart of the most popular licenses\n", + "our_repos.license.value_counts().plot(kind=\"bar\")" ] }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ - "our_filtered_repos = our_repos[our_repos[\"size\"] < 70708]" + "our_repos = pd.read_csv(data_path/\"repo_infos.csv\", parse_dates=True)\n", + "\n", + "# number of our repos with an MIT or Apache license\n", + "mit_apache_repos = our_repos[(our_repos.license == \"MIT License\") | (our_repos.license == \"Apache License 2.0\")]" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([ 19163.2 , 48317.6 , 218033.08])" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "np.percentile(mit_apache_repos[\"size\"].values, [90, 95, 99])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(357429, 339557)" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(mit_apache_repos), len(mit_apache_repos[mit_apache_repos[\"size\"] < 48317.6])" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "mit_apache_repos_filtered = mit_apache_repos[mit_apache_repos[\"size\"] < 48317.6]" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([ 19163.2 , 48317.6 , 218033.08])" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "np.percentile(our_repos[\"size\"].values, [90, 95, 99])" ] }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 22, "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ - "648023" + "(544340, 517122)" ] }, + "execution_count": 22, "metadata": {}, - "execution_count": 24 + "output_type": "execute_result" } ], + "source": [ + "len(our_repos), len(our_repos[our_repos[\"size\"] < 70708])" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "our_filtered_repos = our_repos[our_repos[\"size\"] < 70708]" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], "source": [ "# Combine our repos and EleutherAI's repos\n", "def combine_repos(ours, eleuthers):\n", @@ -136,8 +282,17 @@ " dedup_combined = combined[~combined[\"name\"].duplicated(keep=\"last\")]\n", " return dedup_combined\n", "\n", - "combined = combine_repos(our_filtered_repos, eleuther_repos)\n", - "len(combined)" + "# combined = combine_repos(our_filtered_repos, eleuther_repos)\n", + "# len(combined)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "mit_apache_combined = combine_repos" ] }, { @@ -155,10 +310,11 @@ "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stderr", + "output_type": "stream", "text": [ - "/home/nathan/.local/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3441: DtypeWarning: Columns (1,3,4,6,7,12,13,14,15,16,17,18,19,20,21,22,23,24,26) have mixed types.Specify dtype option on import or set low_memory=False.\n exec(code_obj, self.user_global_ns, self.user_ns)\n" + "/home/nathan/.local/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3441: DtypeWarning: Columns (1,3,4,6,7,12,13,14,15,16,17,18,19,20,21,22,23,24,26) have mixed types.Specify dtype option on import or set low_memory=False.\n", + " exec(code_obj, self.user_global_ns, self.user_ns)\n" ] } ], @@ -167,11 +323,11 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "Shard the dataset into manageable pieces since EleutherAI's downloader has a memory leak." - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -202,8 +358,175 @@ "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
namefork projectcommitsbranchesdefault branchreleasescontributorslicensewatchersstargazers...total issuesopen issuestotal pull requestsopen pull requestslast commitlast commit SHAhas wikiis archivedlanguageslabels
00-1-0/lightblue-0.4False8.01master0.04GNU General Public License v3.014.086...98502020-10-18 21:26:07.09a4f7b37e923b262d2a29894676ff8ed8cde6237TrueFalseNaNNaN
10-14n/ndroidFalse131.01master0.02Other5.050...11212015-03-17 13:10:07.04e5dbe69855a7fda8b74e61d9db5aa61e6ba9ee8TrueFalseC,C++,Objective-C,Shell,Assembly,Haxe,Groff,Py...bug,duplicate,enhancement,help wanted,invalid,...
30-sec/zero-crackFalse4.01main1.01GNU General Public License v3.00.062...00002021-05-12 02:03:08.070ee16550a81b396333565515723d5abab87c719TrueFalsePythonbug,documentation,duplicate,enhancement,good f...
40-tikaro/minimum-viable-startpageFalse15.01master0.0?MIT License4.056...00212019-04-21 09:11:12.0a4fb4aea4474d635c4e4738f7d8c1a485d5d74c8TrueFalseJavaScript,CSS,HTMLbug,duplicate,enhancement,good first issue,hel...
50-u-0/dugon-media-serverFalse52.01master5.0?MIT License2.014...51002020-05-16 04:11:45.01d6bb1c589e51d2c34b11be20d34dae4bb0c7779TrueFalseJavaScript,Dockerfilebug,documentation,duplicate,enhancement,featur...
\n", + "

5 rows × 27 columns

\n", + "
" + ], "text/plain": [ " name fork project commits branches \\\n", "0 0-1-0/lightblue-0.4 False 8.0 1 \n", @@ -255,11 +578,11 @@ "5 bug,documentation,duplicate,enhancement,featur... \n", "\n", "[5 rows x 27 columns]" - ], - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
namefork projectcommitsbranchesdefault branchreleasescontributorslicensewatchersstargazers...total issuesopen issuestotal pull requestsopen pull requestslast commitlast commit SHAhas wikiis archivedlanguageslabels
00-1-0/lightblue-0.4False8.01master0.04GNU General Public License v3.014.086...98502020-10-18 21:26:07.09a4f7b37e923b262d2a29894676ff8ed8cde6237TrueFalseNaNNaN
10-14n/ndroidFalse131.01master0.02Other5.050...11212015-03-17 13:10:07.04e5dbe69855a7fda8b74e61d9db5aa61e6ba9ee8TrueFalseC,C++,Objective-C,Shell,Assembly,Haxe,Groff,Py...bug,duplicate,enhancement,help wanted,invalid,...
30-sec/zero-crackFalse4.01main1.01GNU General Public License v3.00.062...00002021-05-12 02:03:08.070ee16550a81b396333565515723d5abab87c719TrueFalsePythonbug,documentation,duplicate,enhancement,good f...
40-tikaro/minimum-viable-startpageFalse15.01master0.0?MIT License4.056...00212019-04-21 09:11:12.0a4fb4aea4474d635c4e4738f7d8c1a485d5d74c8TrueFalseJavaScript,CSS,HTMLbug,duplicate,enhancement,good first issue,hel...
50-u-0/dugon-media-serverFalse52.01master5.0?MIT License2.014...51002020-05-16 04:11:45.01d6bb1c589e51d2c34b11be20d34dae4bb0c7779TrueFalseJavaScript,Dockerfilebug,documentation,duplicate,enhancement,featur...
\n

5 rows × 27 columns

\n
" + ] }, + "execution_count": 45, "metadata": {}, - "execution_count": 45 + "output_type": "execute_result" } ], "source": [ @@ -469,8 +792,8 @@ "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" }, "kernelspec": { - "name": "python3", - "display_name": "Python 3.8.10 64-bit" + "display_name": "Python 3.8.10 64-bit", + "name": "python3" }, "language_info": { "codemirror_mode": { diff --git a/scripts/evaluation/apps_utils/generate_gpt_codes.py b/scripts/evaluation/apps_utils/generate_gpt_codes.py new file mode 100644 index 0000000..f26ca22 --- /dev/null +++ b/scripts/evaluation/apps_utils/generate_gpt_codes.py @@ -0,0 +1,102 @@ +# MIT License + +# Copyright (c) 2021 Dan Hendrycks and contributors. + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +""" +Run a tranined model to generate Python code. +""" + +import io +import json +import logging +import math +import random +import numpy as np +import os +import pprint +import sys +import time +import transformers +import torch + +from apps_utils.reindent import run as run_reindent + +# for timing and debugging +from datetime import datetime, date +from tqdm import tqdm + + +def reindent_code(codestr): + """ + Given code string, reindent it in the same way that the + Github dataset was indented + """ + codestr = io.StringIO(codestr) + ret = io.StringIO() + + run_reindent( + codestr, + ret, + config={ + "dry-run": False, + "help": False, + "to": 10, + "from": -1, + "tabs": True, + "encoding": "utf-8", + "is-tabs": False, + "tabsize": 10, + "all-tabs": False, + }, + ) + + return ret.getvalue() + + +def generate_prompt( + test_case_path, prompt_path, solutions_path, tokenizer, starter_path=None +): + _input = "\nQUESTION:\n" + with open(prompt_path, "r") as f: + data = f.readlines() + data = "".join(data) + _input += data + if starter_path != None: + with open(starter_path, "r") as f: + data = f.readlines() + data = "".join(data) + data = "\n" + data # + "\n" + _input += data + else: + # _input += "\n\n" + pass + + with open(test_case_path, "r") as f: + data = json.load(f) + if not data.get("fn_name"): + _input += "\nUse Standard Input format" # \n" + else: + _input += "\nUse Call-Based format" # \n" + + _input += "\nANSWER:\n" + + return _input diff --git a/scripts/evaluation/apps_utils/reindent.py b/scripts/evaluation/apps_utils/reindent.py new file mode 100644 index 0000000..78f7141 --- /dev/null +++ b/scripts/evaluation/apps_utils/reindent.py @@ -0,0 +1,227 @@ +# MIT License + +# Copyright (c) 2021 Dan Hendrycks and contributors. + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +""" +Reindent files. +""" + +from __future__ import print_function +import sys +import getopt +import codecs +import tempfile +import shutil +import os + + +def _find_indentation(line, config): + if len(line) and line[0] in (" ", "\t") and not line.isspace(): + if line[0] == "\t": + config['is-tabs'] = True + # Find indentation + i = 0 + for char in list(line): + if char not in (" ", "\t"): + break + i += 1 + config["from"] = i + + +def find_indentation(line, config): + # Find indentation level used in file + if config['from'] < 0: + _find_indentation(line, config) + + if config['from'] >= 0: + # Set old indent + indent = " " if not config['is-tabs'] else "\t" + indent = indent * config['from'] + + # Set new indent + newindent = " " if not config['tabs'] else "\t" + if not config['tabs']: + newindent = newindent * config['to'] + + return indent, newindent + + # Continue to the next line, indentation not found + return False + + +def replace_inline_tabs(content, config): + newcontent = "" + imagined_i = 0 + for i in range(0, len(content)): + char = content[i] + if char == '\t': + spaces = config['tabsize']-(imagined_i % config['tabsize']) + newcontent += " " * spaces + imagined_i += spaces + else: + newcontent += char + imagined_i += 1 + return newcontent + + +def run(fd_in, fd_out, config): + from reindent_4_spaces import Reindenter + import io + + inter = io.StringIO() + ri = Reindenter(fd_in) + ri.run() + ri.write(inter) + fd_in = inter + fd_in.seek(0) + + while True: + line = fd_in.readline() + if not line: + break + line = line.rstrip('\r\n') + + # Find indentation style used in file if not set + if config['from'] < 0: + indent = find_indentation(line, config) + if not indent: + print(line, file=fd_out) + continue + indent, newindent = indent + + # Find current indentation level + level = 0 + while True: + whitespace = line[:len(indent) * (level + 1)] + if whitespace == indent * (level + 1): + level += 1 + else: + break + + content = line[len(indent) * level:] + if config['all-tabs']: + content = replace_inline_tabs(content, config) + + line = (newindent * level) + content + print(line, file=fd_out) + # print(config) + + +def run_files(filenames, config): + for filename in filenames: + with codecs.open(filename, encoding=config['encoding']) as fd_in: + if config['dry-run']: + print("Filename: %s" % filename) + fd_out = sys.stdout + else: + fd_out = tempfile.NamedTemporaryFile(mode='wb', delete=False) + fd_out.close() + fd_out = codecs.open(fd_out.name, "wb", encoding=config['encoding']) + + run(fd_in, fd_out, config) + + if not config["dry-run"]: + fd_out.close() + shutil.copy(fd_out.name, filename) + os.remove(fd_out.name) + + +def main(args): + config = { + "dry-run": False, + "help": False, + "to": 4, + "from": -1, + "tabs": False, + "encoding": "utf-8", + "is-tabs": False, + "tabsize": 4, + "all-tabs": False + } + possible_args = { + "d": "dry-run", + "h": "help", + "t:": "to=", + "f:": "from=", + "n": "tabs", + "e:": "encoding=", + "s:": "tabsize=", + "a": "all-tabs", + } + optlist, filenames = getopt.getopt( + args[1:], + "".join(possible_args.keys()), + possible_args.values() + ) + + shortargs, longargs = [], [] + for shortarg in possible_args: + shortargs.append(shortarg.rstrip(":")) + longargs.append(possible_args[shortarg].rstrip("=")) + + for opt, val in optlist: + opt = opt.lstrip("-") + if opt in shortargs: + opt = longargs[shortargs.index(opt)] + if isinstance(config[opt], bool): + config[opt] = True + elif isinstance(config[opt], int): + config[opt] = int(val) + else: + config[opt] = val + + if config['help']: + help = """ + Usage: %s [options] filename(s) + Options: + -h, --help Show this message + -d, --dry-run Don't save anything, just print + the result + -t , --to Convert to this number of spaces + (default: 4) + -f , --from Convert from this number of spaces + (default: auto-detect, will also + detect tabs) + -n, --tabs Don't convert indentation to spaces, + convert to tabs instead. -t and + --to will have no effect. + -a, --all-tabs Also convert tabs used for alignment + in the code (Warning: will replace + all tabs in the file, even if inside + a string) + -s , --tabsize Set how many spaces one tab is + (only has an effect on -a, default: 4) + -e , --encoding Open files with specified encoding + (default: utf-8) + """ % args[0] + + # Also removes 8 leading spaces to remove our indentation + print("\n".join([x[8:] for x in help[1:].split("\n")])) + sys.exit(0) + + if filenames: + run_files(filenames, config) + else: + run(sys.stdin, sys.stdout, config) + +if __name__ == "__main__": + main(sys.argv) diff --git a/scripts/evaluation/apps_utils/test_one_solution.py b/scripts/evaluation/apps_utils/test_one_solution.py new file mode 100644 index 0000000..fafc307 --- /dev/null +++ b/scripts/evaluation/apps_utils/test_one_solution.py @@ -0,0 +1,158 @@ +# MIT License + +# Copyright (c) 2021 Dan Hendrycks and contributors. + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +Run solutions from one problem. +""" + +import io +import json +import logging +import math +import numpy as np +import os +import pprint +import sys +import apps_utils.testing_util as test_util +import time + +# for timing debugging +from datetime import datetime, date +from pathlib import Path +from tqdm import tqdm + +from typing import List + + +def print_results(results, args): + res = [] + per_prob_res = [] + all_correct = [] + for index in results: + res.extend(results[index]) + per_prob_res.append(np.mean(results[index])) + all_correct.append(np.all(results[index])) + tmp_results = res + compile_errors = len(tmp_results[tmp_results == -2]) + runtime_errors = len(tmp_results[tmp_results == -1]) + failures = len(tmp_results[tmp_results == False]) + successes = len(tmp_results[tmp_results == True]) + total_testcases = len(res) + if args.debug: + print( + f"number of compile errors = {compile_errors} avg = {compile_errors / total_testcases }" + ) + print( + f"number of runtime errors = {runtime_errors} avg = {runtime_errors / total_testcases}" + ) + print(f"number of test cases run = {total_testcases}") + + print( + f"Test Case Average (average accuracy over problems) = {np.mean(per_prob_res)}" + ) + print( + f"Strict Accuracy (all test cases passed / total problems) = {np.mean(all_correct)}" + ) + + +def eval_and_save_problems(test_loc, save): + test_path = Path(test_loc) + problems = list(test_path.glob("*/")) + + print(len(problems)) + gpt_codes = {} + gpt_bleu = {} + gpt_codebleu = {} + results = {} + codes_loc = os.path.join(save, f"all_codes.json") + # if not os.path.exists(codes_loc): + # codes_loc = os.path.join(args.save, f"{args.start}-{args.end}_codes.json") + + if os.path.exists(codes_loc): + results_loc = os.path.join(save, f"all_results.json") + print(codes_loc, results_loc) + + with open(codes_loc, "r") as f: + gpt_codes = json.load(f) + + # main eval loop + for index, problem in enumerate(tqdm(problems[:2])): + try: + # if args.debug: + # print(f"\n\nproblem path = {problem}") + output_str = gpt_codes[str(index)] + except: + print("CANNOT FIND OUTPUT_STR FOR", problem) + continue + prob_path = problem # os.path.join(args.root, problem) + + # with open(os.path.join(prob_path, "solutions.json"), "r") as f: + # sols = json.load(f) + + if not os.path.exists(save): + os.makedirs(save) + + res = [] + # for o_idx, o in enumerate(output_str): + # print(o) + # if args.debug: + # print(f"\nTesting solution {o_idx}") + curr_res = [-2] + try: + curr_res = test_util.run_test( + prob_path=prob_path, test=output_str, debug=False # args.debug + ) + fixed = [] + for e in curr_res: + if isinstance(e, np.ndarray): + e = e.item(0) + if isinstance(e, np.bool_): + e = bool(e) + fixed.append(e) + curr_res = fixed + if not np.all(curr_res): + print(f"Results were not all True: {curr_res}") + except Exception as e: + print(f"test framework exception = {repr(e)}{e}\n") + break + finally: + assert isinstance(curr_res, list) + res.append(curr_res) + + # if args.debug: + # print( + # f"\nHow to read results [-2] = compile error, [-1] = runtime error [False] = failed test case [True] = passed test case" + # ) + # print(f"results = {res}") + + results[index] = res + + with open(results_loc, "w") as f: + try: + f.write(json.dumps(results)) + except Exception as e: + import pdb + + pdb.set_trace() + print("didn't save problem due to {e}") + + return results diff --git a/scripts/evaluation/apps_utils/testing_util.py b/scripts/evaluation/apps_utils/testing_util.py new file mode 100644 index 0000000..2c5444f --- /dev/null +++ b/scripts/evaluation/apps_utils/testing_util.py @@ -0,0 +1,544 @@ +import argparse +import json +import os +import sys +import io +import faulthandler + +# used for debugging to time steps +from datetime import datetime + +# to run the solution files we're using a timing based approach +import signal + +import numpy as np +# for capturing the stdout +from io import StringIO +from typing import get_type_hints +from typing import List, Tuple +# used for testing the code that reads from input +from unittest.mock import patch, mock_open + +from pyext import RuntimeModule + +from enum import Enum +class CODE_TYPE(Enum): + call_based = 0 + standard_input = 1 + +# stuff for setting up signal timer +class TimeoutException(Exception): + pass +def timeout_handler(signum, frame): + print("alarm went off") + #return + raise TimeoutException +signal.signal(signal.SIGALRM, timeout_handler) +timeout = 4 # seconds + +# used to capture stdout as a list +# from https://stackoverflow.com/a/16571630/6416660 +# alternative use redirect_stdout() from contextlib +class Capturing(list): + def __enter__(self): + self._stdout = sys.stdout + sys.stdout = self._stringio = StringIO() + # Make closing the StringIO a no-op + self._stringio.close = lambda x: 1 + return self + def __exit__(self, *args): + self.extend(self._stringio.getvalue().splitlines()) + del self._stringio # free up some memory + sys.stdout = self._stdout + + +def parse_args(): + parser = argparse.ArgumentParser(description="Utility for testing code generation.") + parser.add_argument("-v", "--verbosity-level", action="store", type=int, + help="") + parser.add_argument("-s", "--source", type=str, default="leetcode", + choices=["leetcode", "atcoder", "codewars",], + help="which data source to gather from.") + parser.add_argument("-d", "--data", type=str, default="question", + choices=["question", "q", "solutions", "sol", "s", "starter", "tests", "t"], + help="which type of data to receive.") + parser.add_argument("-n", "--number", type=int, default=0, + help="which problem to query.") + + args = parser.parse_args() + return args + + +def get_valid_problems(data_dir="leetcode"): + # these are unnecessary atm + if data_dir == "leetcode": + root = os.path.join(args.source, "data") + elif data_dir == "atcoder": + pass + + root = os.path.join(data_dir, "data") + if os.path.exists(os.path.join(data_dir, "valid_problems.json")): + with open(os.path.join(data_dir, "valid_problems.json"), "r") as f: + return json.load(f) + + # after we compute it once let's save it and load that instead + # TODO determine if might be better to reload each time + tmp = os.listdir(root) + valid_probs = [] + for folder in tmp: + prob_path = os.path.join(root, folder) + files = os.listdir(prob_path) + #TODO add more validity checks + if "input_output.json" in files or "sols.json" in files: + valid_probs.append(prob_path) + valid_probs = sorted(valid_probs) + #with open(os.path.join(args.source,"valid_problems.json"), "w") as f: + # json.dump(valid_probs, f) + return valid_probs + + +def get_question(problem_list, prob_index): + root = problem_list[prob_index] + #print("get q", root) + if os.path.exists(os.path.join(root, "question.txt")): + with open(os.path.join(root, "question.txt")) as f: + question = f.readlines() + else: + print("question prompt not found") + question = "" + question = "".join(question) + return question + + +def get_solutions(problem_list, prob_index): + root = problem_list[prob_index] + if os.path.exists(os.path.join(root, "solutions.json")): + with open(os.path.join(root, "solutions.json")) as f: + sols = json.load(f) + return sols + + +def run_test(prob_path:str=None, problem_list:List[str]=None, prob_index:int=None, + test:str=None, debug:bool=False): + """ + if test is not None it'll try to run the code. + otherwise it'll just return an input and output pair. + """ + if prob_path is None and problem_list is None: + print("please provide either prob_path or problem_list") + exit() + + if debug: + print(f"start = {datetime.now().time()}") + if prob_path is not None: + root = prob_path + elif problem_list is not None: + root = problem_list[prob_index] + + if os.path.exists(os.path.join(root, "input_output.json")): + with open(os.path.join(root, "input_output.json")) as f: + in_outs = json.load(f) + if debug: + print(f"test cases json = {in_outs['inputs']} {in_outs['outputs']}") + + if in_outs.get("fn_name") is None: + which_type = CODE_TYPE.standard_input # Standard input + method_name = None + else: + which_type = CODE_TYPE.call_based # Call-based + method_name = in_outs["fn_name"] + if debug: + print(f"loaded json = {datetime.now().time()}") + + #else: + # continue + if test is None: + return in_outs + elif test is not None: + results = [] + sol = "import sys\nimport time\nimport itertools\nfrom itertools import accumulate, product, permutations, combinations\nimport collections\nfrom collections import Counter, OrderedDict, deque, defaultdict, ChainMap\nfrom functools import lru_cache\nimport math\nfrom math import sqrt, sin, cos, tan, ceil, fabs, floor, gcd, exp, log, log2\nimport fractions\nfrom typing import List, Tuple\nimport numpy as np\nimport random\nimport heapq\nfrom heapq import *\n" + if debug: + print(f"loading test code = {datetime.now().time()}") + + if which_type == CODE_TYPE.call_based: + sol += test + if debug: # or True: + print(f"sol = {sol}") + signal.alarm(timeout) + try: + tmp_sol = RuntimeModule.from_string("tmp_sol", "", sol) + if "class Solution" not in test: + tmp = tmp_sol + else: + tmp = tmp_sol.Solution() + signal.alarm(0) + except Exception as e: + signal.alarm(0) + print(f"type 0 compilation error = {e}") + results.append(-2) + return results + signal.alarm(0) + + elif which_type == CODE_TYPE.standard_input: + # sol + tmp_test = test.split("\n") + + new_test = [] + for x in tmp_test: + if (not x.startswith("from ")) and (not x.startswith("import ")): + new_test.append("\t" + x + "\n") + else: + new_test.append(x + "\n") + tmp_test = new_test + + new_test = "" + started = False + for i in tmp_test: + if i.startswith("\t") and not started: + new_test += "stdin = sys.stdin\nstdout = sys.stdout\n" + new_test += "def code():\n" + new_test += i + started = True + elif started and ((i.startswith("from ")) or (i.startswith("import "))): + new_test += "\t" + i + else: + new_test += i + tmp_test = new_test + + sol += tmp_test + if debug: + print(f"sol = {sol}") + # print(f"{o}") + method_name = "code" + signal.alarm(timeout) + try: + tmp_sol = RuntimeModule.from_string("tmp_sol", "", sol) + tmp = tmp_sol + signal.alarm(0) + except Exception as e: + signal.alarm(0) + print(f"type 1 compilation error = {e}") + results.append(-2) + return results + signal.alarm(0) + if debug: + print(f"get method = {datetime.now().time()}") + + try: + method = getattr(tmp, method_name) # get_attr second arg must be str + except: + signal.alarm(0) + e = sys.exc_info() + print(f"unable to get function error = {e}") + return results + + for index, inputs in enumerate(in_outs["inputs"]): + # JSON forces dictionaries to have string keys; this undoes this (assuming a singleton list) + try: + if isinstance(inputs[0], dict): + inputs = [{int(k): v for k,v in inputs[0].items()}] + except: + True + try: + if isinstance(in_outs["outputs"][index], dict): + in_outs["outputs"][index] = [{int(k): v for k,v in in_outs["outputs"][index].items()}] + except: + True + try: + if isinstance(in_outs["outputs"][index][0], dict): + in_outs["outputs"][index] = [{int(k): v for k,v in in_outs["outputs"][index][0].items()}] + except: + True + + if debug: + print(f"time: {datetime.now().time()} testing index = {index} inputs = {inputs}, {type(inputs)}. type = {which_type}") + if which_type == CODE_TYPE.call_based: # Call-based + signal.alarm(timeout) + faulthandler.enable() + try: + # print("------------") + # print(inputs) + output = method(*inputs) + + # ground truth sequences are not tuples + if isinstance(output, tuple): + output = list(output) + + tmp_result = output == in_outs["outputs"][index] + if isinstance(in_outs["outputs"][index], list) and in_outs["outputs"][index]: + tmp_result = tmp_result or (output == in_outs["outputs"][index][0]) + + # ground truth sequences are not tuples + try: + if isinstance(output[0], tuple): + tmp_result = tmp_result or ([list(x) for x in output] == in_outs["outputs"][index][0]) + except: + True + results.append(tmp_result) + + # reset the alarm + signal.alarm(0) + except Exception as e: + signal.alarm(0) + faulthandler.disable() + print(f"Standard input runtime error or time limit exceeded error = {e}") + results.append(-1) + continue + faulthandler.disable() + signal.alarm(0) + if debug: + print(f"outputs = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}") + elif which_type == CODE_TYPE.standard_input: # Standard input + faulthandler.enable() + signal.alarm(timeout) + passed = False + + if isinstance(inputs, list): + inputs = "\n".join(inputs) + if isinstance(in_outs['outputs'][index], list): + in_outs['outputs'][index] = "\n".join(in_outs['outputs'][index]) + + with Capturing() as output: + try: + call_method(method, inputs) + # reset the alarm + signal.alarm(0) + passed = True + except Exception as e: + # runtime error or took too long + signal.alarm(0) + print(f"Call-based runtime error or time limit exceeded error = {repr(e)}{e}") + results.append(-1) + signal.alarm(0) + + if not passed: + if debug: + nl = "\n" + if not isinstance(inputs, list): + print(f"not passed output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]}") + else: + print(f"not passed output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}") + continue + + if passed and debug: + print(f"==> output = {output}, test outputs = {in_outs['outputs'][index]}") + + if custom_compare_(output, in_outs['outputs'][index]): + tmp_result = True + results.append(tmp_result) + continue + + # ground truth sequences are expressed as lists not tuples + if isinstance(output, tuple): + output = list(output) + + tmp_result = False + try: + tmp_result = (output == [in_outs["outputs"][index]]) + if isinstance(in_outs["outputs"][index], list): + tmp_result = tmp_result or (output == in_outs["outputs"][index]) + if isinstance(output[0], str): + tmp_result = tmp_result or ([e.strip() for e in output] == in_outs["outputs"][index]) + except Exception as e: + print(f"Failed check1 exception = {e}") + pass + + if tmp_result == True: + results.append(tmp_result) + continue + + # try one more time without \n + if isinstance(in_outs["outputs"][index], list): + for tmp_index, i in enumerate(in_outs["outputs"][index]): + in_outs["outputs"][index][tmp_index] = i.split("\n") + in_outs["outputs"][index][tmp_index] = [x.strip() for x in in_outs["outputs"][index][tmp_index] if x] + else: + in_outs["outputs"][index] = in_outs["outputs"][index].split("\n") + in_outs["outputs"][index] = list(filter(len, in_outs["outputs"][index])) + in_outs["outputs"][index] = list(map(lambda x:x.strip(), in_outs["outputs"][index])) + + try: + tmp_result = (output == [in_outs["outputs"][index]]) + if isinstance(in_outs["outputs"][index], list): + tmp_result = tmp_result or (output == in_outs["outputs"][index]) + except Exception as e: + print(f"Failed check2 exception = {e}") + pass + + if tmp_result == True: + results.append(tmp_result) + continue + + # try by converting the output into a split up list too + if isinstance(output, list): + output = list(filter(len, output)) + + if debug: + nl = "\n" + if not isinstance(inputs, list): + print(f"output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]}") + else: + print(f"output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}") + + if tmp_result == True: + results.append(tmp_result) + continue + + try: + tmp_result = (output == [in_outs["outputs"][index]]) + if isinstance(in_outs["outputs"][index], list): + tmp_result = tmp_result or (output == in_outs["outputs"][index]) + except Exception as e: + print(f"Failed check3 exception = {e}") + pass + + try: + output_float = [float(e) for e in output] + gt_float = [float(e) for e in in_outs['outputs'][index]] + tmp_result = tmp_result or ((len(output_float) == len(gt_float)) and np.allclose(output_float, gt_float)) + except Exception as e: + pass + try: + if isinstance(output[0], list): + output_float = [float(e) for e in output[0]] + gt_float = [float(e) for e in in_outs['outputs'][index][0]] + tmp_result = tmp_result or ((len(output_float) == len(gt_float)) and np.allclose(output_float, gt_float)) + except Exception as e: + pass + + if tmp_result == True: + results.append(tmp_result) + continue + + # try by converting the stuff into split up list + if isinstance(in_outs["outputs"][index], list): + for tmp_index, i in enumerate(in_outs["outputs"][index]): + in_outs["outputs"][index][tmp_index] = set(i.split()) + else: + in_outs["outputs"][index] = set(in_outs["outputs"][index].split()) + + try: + tmp_result = (output == in_outs["outputs"][index]) + except Exception as e: + print(f"Failed check4 exception = {e}") + continue + + if tmp_result == True: + results.append(tmp_result) + continue + + # try by converting the output into a split up list too + if isinstance(output, list): + for tmp_index, i in enumerate(output): + output[tmp_index] = i.split() + output = list(filter(len, output)) + for tmp_index, i in enumerate(output): + output[tmp_index] = set(i) + else: + output = output.split() + output = list(filter(len, output)) + output = set(output) + + try: + tmp_result = (set(frozenset(s) for s in output) == set(frozenset(s) for s in in_outs["outputs"][index])) + except Exception as e: + print(f"Failed check5 exception = {e}") + + + # if they are all numbers, round so that similar numbers are treated as identical + try: + tmp_result = tmp_result or (set(frozenset(round(float(t),3) for t in s) for s in output) ==\ + set(frozenset(round(float(t),3) for t in s) for s in in_outs["outputs"][index])) + except Exception as e: + print(f"Failed check6 exception = {e}") + + if tmp_result == True and debug: + print("PASSED") + + results.append(tmp_result) + + if debug: + nl = "\n" + if not isinstance(inputs, list): + print(f"output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]}") + else: + print(f"output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}") + + + return results + +def custom_compare_(output, ground_truth): + + if isinstance(output, list): + output_1 = "\n".join(output) + if stripped_string_compare(output_1, ground_truth): + return True + + if isinstance(output, list): + output_2 = [o.lstrip().rstrip() for o in output] + output_2 = "\n".join(output_2) + if stripped_string_compare(output_2, ground_truth): + return True + + return False + +def stripped_string_compare(s1, s2): + s1 = s1.lstrip().rstrip() + s2 = s2.lstrip().rstrip() + return s1 == s2 + +def call_method(method, inputs): + + if isinstance(inputs, list): + inputs = "\n".join(inputs) + + inputs_line_iterator = iter(inputs.split("\n")) + + # sys.setrecursionlimit(10000) + + # @patch('builtins.input', side_effect=inputs.split("\n")) + @patch('builtins.open', mock_open(read_data=inputs)) + @patch('sys.stdin', StringIO(inputs)) + @patch('sys.stdin.readline', lambda *args: next(inputs_line_iterator)) + @patch('sys.stdin.readlines', lambda *args: inputs.split("\n")) + @patch('sys.stdin.read', lambda *args: inputs) + # @patch('sys.stdout.write', print) + def _inner_call_method(_method): + try: + return _method() + except SystemExit as e: + pass + finally: + pass + return _inner_call_method(method) + +def main(args): + print(args) + problem_list = sorted(get_valid_problems(args.source)) + print(f"number of problems = {len(problem_list)}") + prob_index = args.number + print(f"problem is {problem_list[prob_index]}") + + # This checks it correctly loaded. remove this later + assert prob_index < len(problem_list) + + if args.data == "q" or args.data == "question": + tmp = get_question(problem_list, prob_index) + print("q", tmp) + elif args.data in ["solutions", "sol", "s",]: + tmp = get_solutions(problem_list, prob_index) + print("sol", tmp) + elif args.data == "starter": + tmp = get_starter(problem_list, prob_index) + print("starter", tmp) + elif args.data in ["test", "t"]: + # test it with sols + sols = get_solutions(problem_list, prob_index) + tmp = run_test(problem_list, prob_index, test=sols[0]) + + print("results = ", tmp) + print("-2 = compile error, -1 is runtime error, False failed test, True passed test") + +if __name__ == "__main__": + args = parse_args() + main(args) diff --git a/scripts/evaluation/code_search_net.py b/scripts/evaluation/code_search_net.py new file mode 100644 index 0000000..49ac5ad --- /dev/null +++ b/scripts/evaluation/code_search_net.py @@ -0,0 +1,4 @@ +from datasets import load_dataset + +dataset = load_dataset("code_x_glue_ct_code_to_text", "go") +print(dataset) diff --git a/scripts/evaluation/concode.py b/scripts/evaluation/concode.py new file mode 100644 index 0000000..dd90f34 --- /dev/null +++ b/scripts/evaluation/concode.py @@ -0,0 +1,23 @@ +import pandas as pd + +from datasets import load_dataset, load_metric +from fastcore.script import * +from pathlib import Path + +bleu = load_metric("sacrebleu") + +predictions = ["hello there kenobi", "foo bar foobar"] +references = [ + ["hello there general kenobi"], + ["foo bar foobar"], # , "hello there !"], # , "foo bar foobar"], +] + + +@call_parse +def main(concode_path: Param("Path to the concode data in CodeXGLUE", str)): + concode_path = Path(concode_path) + dataset = load_dataset("json", data_files=str(concode_path / "test.json")) + print(dataset) + results = bleu.compute(predictions=predictions, references=references) + print(list(results.keys())) + print(round(results["score"], 1)) diff --git a/scripts/evaluation/evaluate.py b/scripts/evaluation/evaluate.py new file mode 100644 index 0000000..4fbd421 --- /dev/null +++ b/scripts/evaluation/evaluate.py @@ -0,0 +1,165 @@ +import json +import torch +import pandas as pd + +# import apps.eval.reident + +from apps_utils.generate_gpt_codes import generate_prompt +from apps_utils.test_one_solution import eval_and_save_problems +from datasets import load_dataset, load_metric +from fastcore.script import * +from human_eval.data import write_jsonl, read_problems +from pathlib import Path +from metrics.extrinsic_eval import compute_metrics +from subprocess import check_output +from transformers import AutoTokenizer, AutoModelWithLMHead + +bleu = load_metric("sacrebleu") +tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neo-125M") +model = AutoModelWithLMHead.from_pretrained( + "/home/nathan/gpt-code-clippy/data/APPS/models/1.5B" +) + + +def generate_text(prompt): + # print(prompt) + input_ids = torch.LongTensor(tokenizer.encode(prompt, verbose=False)).unsqueeze( + 0 + ) # .cuda() + output_ids = model.generate( + input_ids, + num_beams=2, + early_stopping=True, + max_length=1024 - len(input_ids), + ) + output_str = tokenizer.decode(output_ids[0]) + return output_str + # # "a", "=", "b", "\n", "y", "=", "a", "+", "1" + # return "a = b \n y = a + 1" + + +def _eval_concode(path): + # TODO: format input to model same as App and OpenAI HumanEval datasets are formatted + data = load_dataset("json", data_files=str(path / "test.json"))["train"] + predictions = [[]] + references = [] + for example in data: + output = generate_text(example["nl"]) + predictions[0].append(output.split(" ")) + references.append(example["code"].split(" ")) + results = compute_metrics(predictions, references) + print(f"Bleu score for Concode dataset: {results}") + + +def _eval_apps(path): + gpt_codes = {} + prob_paths = sorted(path.glob("*/")) + # map prob_paths to strings and save as a json file + str_paths = [str(p) for p in prob_paths] + with open(path / "test.json", "w") as f: + json.dump(str_paths, f) + for index, prob_path in enumerate(prob_paths[:2]): + test_case_path = prob_path / "input_output.json" + prompt_path = prob_path / "question.txt" + starter_path = prob_path / "starter_code.py" + solutions_path = prob_path / "solutions.json" + if not starter_path.exists(): + starter_path = None + if not test_case_path.exists() or not prompt_path.exists(): + continue + prompt = generate_prompt( + Args(), + test_case_path, + prompt_path, + solutions_path, + tokenizer, + starter_path=starter_path, + ) + output = generate_text(prompt) + print(output) + # print(output) + gpt_codes[index] = output + # print(output) + + with open(path.parent / "all_codes.json", "w") as f: + json.dump(gpt_codes, f) + + eval_and_save_problems(path, path.parent) + + # execute bash command to run eval script + # results = check_output( + # [ + # # python3 test_one_solution.py -t /path/to/apps/test --save /path/to/save_dir --print_results + # "python", + # "./apps_utils/test_one_solution.py", + # "-t", + # str(path), + # "--save", + # str(path.parent), + # "--print_results", + # ] + # ).decode("utf-8") + + +# test_case_path = os.path.join(prob_path, "input_output.json") +# prompt_path = os.path.join(prob_path, "question.txt") +# starter_path = os.path.join(prob_path, "starter_code.py") +# solutions_path = os.path.join(prob_path, "solutions.json") +# generate_prompt(args, test_case_path, prompt_path, solutions_path, tokenizer, starter_path=None) + + +def _eval_human_eval(path): + problems = read_problems() + num_samples_per_task = 1 + samples = [ + dict( + task_id=task_id, + completion=generate_text(problems[task_id]["prompt"]), + ) + for task_id in problems + for _ in range(num_samples_per_task) + ] + write_jsonl("human_eval.jsonl", samples) + # execute bash command to run eval script + results = check_output( + [ + "python", + path / "evaluate_functional_correctness.py", + "human_eval.jsonl", + ] + ).decode("utf-8") + + print(results) + + +@call_parse +def main( + concode_path: Param("Path to the concode data in CodeXGLUE", str), + apps_path: Param("Path to the the App dataset", str), + human_eval_path: Param("Path to the human eval dataset", str), +): + concode_path = Path(concode_path) + apps_path = Path(apps_path) + human_eval_path = Path(human_eval_path) + # _eval_concode(concode_path) + # _eval_human_eval(human_eval_path) + _eval_apps(apps_path) + # dataset = load_dataset("json", data_files=str(concode_path / "test.json")) + # print(dataset) + # results = bleu.compute(predictions=predictions, references=references) + # print(list(results.keys())) + # print(round(results["score"], 1)) + + +# problems = read_problems() +# print(problems) +# num_samples_per_task = 200 +# samples = [ +# dict( +# task_id=task_id, +# completion=generate_text(problems[task_id]["prompt"]), +# ) +# for task_id in problems[:1] +# for _ in range(num_samples_per_task) +# ] +# write_jsonl("human_eval.jsonl", samples) diff --git a/scripts/evaluation/human_eval.jsonl b/scripts/evaluation/human_eval.jsonl new file mode 100644 index 0000000..beafda1 --- /dev/null +++ b/scripts/evaluation/human_eval.jsonl @@ -0,0 +1,164 @@ +{"task_id": "HumanEval/0", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/1", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/2", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/3", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/4", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/5", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/6", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/7", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/8", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/9", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/10", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/11", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/12", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/13", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/14", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/15", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/16", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/17", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/18", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/19", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/20", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/21", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/22", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/23", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/24", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/25", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/26", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/27", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/28", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/29", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/30", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/31", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/32", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/33", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/34", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/35", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/36", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/37", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/38", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/39", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/40", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/41", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/42", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/43", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/44", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/45", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/46", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/47", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/48", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/49", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/50", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/51", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/52", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/53", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/54", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/55", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/56", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/57", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/58", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/59", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/60", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/61", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/62", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/63", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/64", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/65", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/66", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/67", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/68", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/69", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/70", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/71", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/72", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/73", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/74", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/75", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/76", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/77", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/78", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/79", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/80", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/81", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/82", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/83", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/84", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/85", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/86", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/87", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/88", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/89", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/90", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/91", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/92", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/93", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/94", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/95", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/96", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/97", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/98", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/99", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/100", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/101", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/102", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/103", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/104", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/105", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/106", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/107", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/108", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/109", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/110", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/111", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/112", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/113", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/114", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/115", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/116", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/117", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/118", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/119", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/120", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/121", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/122", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/123", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/124", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/125", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/126", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/127", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/128", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/129", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/130", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/131", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/132", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/133", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/134", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/135", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/136", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/137", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/138", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/139", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/140", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/141", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/142", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/143", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/144", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/145", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/146", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/147", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/148", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/149", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/150", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/151", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/152", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/153", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/154", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/155", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/156", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/157", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/158", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/159", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/160", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/161", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/162", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/163", "completion": "a = b \n y = a + 1"} diff --git a/scripts/evaluation/human_eval.jsonl_results.jsonl b/scripts/evaluation/human_eval.jsonl_results.jsonl new file mode 100644 index 0000000..8c14355 --- /dev/null +++ b/scripts/evaluation/human_eval.jsonl_results.jsonl @@ -0,0 +1,164 @@ +{"task_id": "HumanEval/0", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/1", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/2", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/3", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/4", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/5", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/6", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/7", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/8", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/9", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/10", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 21)", "passed": false} +{"task_id": "HumanEval/11", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/12", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/13", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/14", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 10)", "passed": false} +{"task_id": "HumanEval/15", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/16", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/17", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 18)", "passed": false} +{"task_id": "HumanEval/18", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/19", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/20", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/21", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/22", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/23", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/24", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 9)", "passed": false} +{"task_id": "HumanEval/25", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/26", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/27", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 9)", "passed": false} +{"task_id": "HumanEval/28", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/29", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/30", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/31", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 21)", "passed": false} +{"task_id": "HumanEval/32", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 25)", "passed": false} +{"task_id": "HumanEval/33", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/34", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 9)", "passed": false} +{"task_id": "HumanEval/35", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/36", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/37", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/38", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 19)", "passed": false} +{"task_id": "HumanEval/39", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 18)", "passed": false} +{"task_id": "HumanEval/40", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 21)", "passed": false} +{"task_id": "HumanEval/41", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/42", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/43", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 20)", "passed": false} +{"task_id": "HumanEval/44", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/45", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 9)", "passed": false} +{"task_id": "HumanEval/46", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 19)", "passed": false} +{"task_id": "HumanEval/47", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/48", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/49", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/50", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/51", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 20)", "passed": false} +{"task_id": "HumanEval/52", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/53", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/54", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 20)", "passed": false} +{"task_id": "HumanEval/55", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/56", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/57", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/58", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/59", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/60", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/61", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/62", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/63", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 18)", "passed": false} +{"task_id": "HumanEval/64", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 19)", "passed": false} +{"task_id": "HumanEval/65", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/66", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/67", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/68", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 37)", "passed": false} +{"task_id": "HumanEval/69", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/70", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/71", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/72", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 21)", "passed": false} +{"task_id": "HumanEval/73", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/74", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/75", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/76", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/77", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/78", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 21)", "passed": false} +{"task_id": "HumanEval/79", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/80", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/81", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 28)", "passed": false} +{"task_id": "HumanEval/82", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/83", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 8)", "passed": false} +{"task_id": "HumanEval/84", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/85", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 10)", "passed": false} +{"task_id": "HumanEval/86", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/87", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 23)", "passed": false} +{"task_id": "HumanEval/88", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 18)", "passed": false} +{"task_id": "HumanEval/89", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/90", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/91", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/92", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 21)", "passed": false} +{"task_id": "HumanEval/93", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/94", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/95", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/96", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/97", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/98", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/99", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 21)", "passed": false} +{"task_id": "HumanEval/100", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/101", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/102", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/103", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/104", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/105", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 25)", "passed": false} +{"task_id": "HumanEval/106", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/107", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 26)", "passed": false} +{"task_id": "HumanEval/108", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/109", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 30)", "passed": false} +{"task_id": "HumanEval/110", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/111", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/112", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/113", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/114", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/115", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 38)", "passed": false} +{"task_id": "HumanEval/116", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/117", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/118", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 18)", "passed": false} +{"task_id": "HumanEval/119", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 18)", "passed": false} +{"task_id": "HumanEval/120", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 28)", "passed": false} +{"task_id": "HumanEval/121", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/122", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/123", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 20)", "passed": false} +{"task_id": "HumanEval/124", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 23)", "passed": false} +{"task_id": "HumanEval/125", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/126", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 19)", "passed": false} +{"task_id": "HumanEval/127", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 23)", "passed": false} +{"task_id": "HumanEval/128", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/129", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 33)", "passed": false} +{"task_id": "HumanEval/130", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 20)", "passed": false} +{"task_id": "HumanEval/131", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/132", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/133", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/134", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/135", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/136", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/137", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/138", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 10)", "passed": false} +{"task_id": "HumanEval/139", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/140", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/141", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/142", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/143", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 23)", "passed": false} +{"task_id": "HumanEval/144", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/145", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/146", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/147", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/148", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 19)", "passed": false} +{"task_id": "HumanEval/149", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 18)", "passed": false} +{"task_id": "HumanEval/150", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/151", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/152", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 18)", "passed": false} +{"task_id": "HumanEval/153", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 20)", "passed": false} +{"task_id": "HumanEval/154", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/155", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 10)", "passed": false} +{"task_id": "HumanEval/156", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/157", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/158", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/159", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 32)", "passed": false} +{"task_id": "HumanEval/160", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 28)", "passed": false} +{"task_id": "HumanEval/161", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/162", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 10)", "passed": false} +{"task_id": "HumanEval/163", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} diff --git a/scripts/evaluation/human_eval_bench.py b/scripts/evaluation/human_eval_bench.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/evaluation/metrics/bleu.py b/scripts/evaluation/metrics/bleu.py new file mode 100644 index 0000000..17a06ce --- /dev/null +++ b/scripts/evaluation/metrics/bleu.py @@ -0,0 +1,133 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# The following code is taken from CodeXGlue Repository - https://github.com/microsoft/CodeXGLUE/blob/main/Code-Code/code-to-code-trans/evaluator/CodeBLEU/bleu.py + + +"""Python implementation of BLEU and smooth-BLEU. + +This module provides a Python implementation of BLEU and smooth-BLEU. +Smooth BLEU is computed following the method outlined in the paper: +Chin-Yew Lin, Franz Josef Och. ORANGE: a method for evaluating automatic +evaluation metrics for machine translation. COLING 2004. +""" + +import collections +import math + + +def _get_ngrams(segment, max_order): + """Extracts all n-grams upto a given maximum order from an input segment. + + Args: + segment: text segment from which n-grams will be extracted. + max_order: maximum length in tokens of the n-grams returned by this + methods. + + Returns: + The Counter containing all n-grams upto max_order in segment + with a count of how many times each n-gram occurred. + """ + ngram_counts = collections.Counter() + for order in range(1, max_order + 1): + for i in range(0, len(segment) - order + 1): + ngram = tuple(segment[i : i + order]) + ngram_counts[ngram] += 1 + return ngram_counts + + +def compute_bleu(reference_corpus, translation_corpus, max_order=4, smooth=True): + """Computes BLEU score of translated segments against one or more references. + + Args: + reference_corpus: list of lists of references for each translation. Each + reference should be tokenized into a list of tokens. + translation_corpus: list of translations to score. Each translation + should be tokenized into a list of tokens. + max_order: Maximum n-gram order to use when computing BLEU score. + smooth: Whether or not to apply Lin et al. 2004 smoothing. + + Returns: + 3-Tuple with the BLEU score, n-gram precisions, geometric mean of n-gram + precisions and brevity penalty. + """ + matches_by_order = [0] * max_order + possible_matches_by_order = [0] * max_order + reference_length = 0 + translation_length = 0 + for (references, translation) in zip(reference_corpus, translation_corpus): + reference_length += min(len(r) for r in references) + translation_length += len(translation) + + merged_ref_ngram_counts = collections.Counter() + for reference in references: + merged_ref_ngram_counts |= _get_ngrams(reference, max_order) + translation_ngram_counts = _get_ngrams(translation, max_order) + overlap = translation_ngram_counts & merged_ref_ngram_counts + for ngram in overlap: + matches_by_order[len(ngram) - 1] += overlap[ngram] + for order in range(1, max_order + 1): + possible_matches = len(translation) - order + 1 + if possible_matches > 0: + possible_matches_by_order[order - 1] += possible_matches + + precisions = [0] * max_order + for i in range(0, max_order): + if smooth: + precisions[i] = (matches_by_order[i] + 1.0) / ( + possible_matches_by_order[i] + 1.0 + ) + else: + if possible_matches_by_order[i] > 0: + precisions[i] = ( + float(matches_by_order[i]) / possible_matches_by_order[i] + ) + else: + precisions[i] = 0.0 + + if min(precisions) > 0: + p_log_sum = sum((1.0 / max_order) * math.log(p) for p in precisions) + geo_mean = math.exp(p_log_sum) + else: + geo_mean = 0 + + ratio = float(translation_length) / reference_length + + if ratio > 1.0: + bp = 1.0 + else: + bp = math.exp(1 - 1.0 / ratio) + bleu = geo_mean * bp + bleu_score_dict = { + "bleu": bleu, + "precision": precisions, + "bp": bp, + "ratio": ratio, + "trans_len": translation_length, + "ref_len": reference_length, + } + return bleu_score_dict # (bleu, precisions, bp, ratio, translation_length, reference_length) + + +def bleu_test_case(): + """A simple functionality test case to evaluate BLEU""" + generated = [[["a", "=", "b", "\n", "y", "=", "a", "+", "1"]]] + reference = [["a", "=", "b", "\n", "print", "a"]] + score_dict = compute_bleu(generated, reference, smooth=False) + return score_dict + + +if __name__ == "__main__": + score_dict = bleu_test_case() + print(score_dict) diff --git a/scripts/evaluation/metrics/extrinsic_eval.py b/scripts/evaluation/metrics/extrinsic_eval.py new file mode 100644 index 0000000..d609870 --- /dev/null +++ b/scripts/evaluation/metrics/extrinsic_eval.py @@ -0,0 +1,46 @@ +from metrics.bleu import compute_bleu + + +def compute_exact_match(references, generated) -> float: + """ + Computes Exact Match Accuracy. + args: + reference: list of lists of references for each translation. Each + reference should be tokenized into a list of tokens. + translation: list of translations to score. Each translation + should be tokenized into a list of tokens. + returns: + exact_match_accuracy : Float + """ + assert ( + len(references[0]) == len(generated), + "Number of Samples should be equal in References and Synthesized Outputs..", + ) + exact_match_count = 0.0 + for gen, ref in zip(generated, references[0]): + if gen == ref: + exact_match_count += 1 + exact_match_acc = exact_match_count / len(generated) + return exact_match_acc + + +def compute_metrics(references, generated) -> dict: + """ + Calculates various metrics and returns the calculated dict of these matrics. + args: + reference: list of lists of references for each translation. Each + reference should be tokenized into a list of tokens. + translation: list of translations to score. Each translation + should be tokenized into a list of tokens. + returns: + A dicitonary with different metrics intact. + """ + metrics_dict = { + "smoothed_bleu_4": None, + "bleu_4": None, + "exact_match_acc": None, + } # Update as in new metrics are computed. + metrics_dict["smoothed_bleu_4"] = compute_bleu(references, generated, smooth=True) + metrics_dict["bleu_4"] = compute_bleu(references, generated, smooth=False) + metrics_dict["exact_match_acc"] = compute_exact_match(references, generated) + return metrics_dict diff --git a/scripts/get_license_info.py b/scripts/get_license_info.py new file mode 100644 index 0000000..6c1b266 --- /dev/null +++ b/scripts/get_license_info.py @@ -0,0 +1,31 @@ +import os + +import pandas as pd + +from fastcore.script import * +from ghapi.all import GhApi + +GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN") + + +# Open issue on repo using custom title and body +def get_license_info(owner, repo): + api = GhApi(owner=owner, repo=repo, token=GITHUB_TOKEN) + license = api.licenses.get_for_repo(owner=owner, repo=repo) + return license.license.name + +@call_parse +def main(repos_path: Param("Path to the csv containing all of the repos", str)): + """ + Use pandas dataframe from the repos path to open issues in each of them. + """ + repos_path = Path(repos_path) + df = pd.read_csv(repos_path) + + # Loop through repos and get their license + licenses = [] + for _, row in df.iterrows(): + owner, repo = row["name"].split("/") + licenses.append(get_license_info(owner, repo)) + df["license"] = licenses + df.to_csv(repos_path.parent/f"{repos_path.stem}_with_license.csv", index=False)