diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..894404cdb526068b3387b5394587c6cad1757d96 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +asset/design.png filter=lfs diff=lfs merge=lfs -text diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..0606c719b9774012708f505d061d9c6b0b903db1 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Jiajun Shen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/__pycache__/parser.cpython-312.pyc b/__pycache__/parser.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7edfef26251e989d3319e0d9f4682e7d46f62098 Binary files /dev/null and b/__pycache__/parser.cpython-312.pyc differ diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..19bbd7e81b3277c2526cd6deaf42c72644cfbb3b --- /dev/null +++ b/app.py @@ -0,0 +1,104 @@ +from flask import Flask, request, jsonify, send_file, Response +from flask_cors import CORS +import openai +import sys +import os +from methods.self_RAG_demo import pipeline, graph +from citekit.utils.utils import parse_html_config + +app = Flask(__name__) +CORS(app) # 允许跨域请求 + + +@app.route("/") +def index(): + return send_file("index.html") + + +@app.route("/run_pipeline", methods=["POST"]) +def run_pipeline(): + data = request.json + if not data: + return jsonify({"error": "Invalid input data"}), 400 + + try: + result = pipeline(data) # 直接调用 pipeline 处理数据 + print(result) + return jsonify(result) # 返回 JSON 结果 + except Exception as e: + return jsonify({"error": str(e)}), 500 + +@app.route("/get_nodes", methods=["POST"]) +def get_nodes(*args, **kwargs): + graph.update() + try: + return jsonify(graph.get_json()) + except Exception as e: + return jsonify({"error": str(e)}), 500 + +@app.route("/update", methods=["POST"]) +def update(): + + data = request.json + update_info = data.get("update_info") + update_object = data.get('update_object') + print(update_info, update_object) + try: + config, update_info = parse_html_config(update_info) + print('GOT CONFIG', config, update_info) + pipeline.update(update_object, config, update_info) + return jsonify({}) + except Exception as e: + return jsonify({"error": str(e)}), 500 + + + +@app.route("/get_config", methods=["POST"]) +def get_config(): + data = request.json + config = data.get("config").lower() + module_name = data.get("module_name") + module = pipeline.get_module_by_name(module_name) + + try: + if config in ['prompt', 'destination', 'max turn', 'global prompt', 'parallel']: + return jsonify(module.get_json_config(config)) + else: + raise NotImplementedError + + + except Exception as e: + return jsonify({"error": str(e)}), 500 + +@app.route("/chat", methods=["POST"]) +def chat(): + data = request.json + api_key = data.get("api_key") + user_message = data.get("message") + + if not api_key or not user_message: + return jsonify({"error": "API Key and message are required"}), 400 + + try: + openai.api_key = api_key + response = openai.ChatCompletion.create( + model="gpt-4o", + messages=[ + {"role": "system", "content": "You are a helpful assistant that follows the instructions of the user. You will be given a pipeline and (maybe) some datapoints in json format. You will be asked questions about the pipeline or the datapoints. Refuse to answer questions that are not about the pipeline or the datapoints."}, + {"role": "user", "content": user_message} + ], + stream=True # 启用流式输出 + ) + + def generate(): + for chunk in response: + if "choices" in chunk and chunk["choices"]: + yield chunk["choices"][0]["delta"].get("content", "") + + + return Response(generate(), content_type="text/event-stream") # 使用流式响应 + except Exception as e: + return jsonify({"error": str(e)}), 500 + +if __name__ == '__main__': + app.run(host="0.0.0.0", port=7860) diff --git a/asset/design.png b/asset/design.png new file mode 100644 index 0000000000000000000000000000000000000000..7e30b732c874e5fa5135f534aa2dbba5fd0c5b61 --- /dev/null +++ b/asset/design.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c1faa6bd26eb8d540ea26d0658f04d95d6e1f9169bbf1ca007686b961be9741 +size 1432892 diff --git a/citekit/Dataset/Dataset.py b/citekit/Dataset/Dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..d07848100c4c172245c02e0ef51a1ab3d765de03 --- /dev/null +++ b/citekit/Dataset/Dataset.py @@ -0,0 +1,41 @@ +from torch.utils.data import Dataset +import json + +default_get = lambda key : lambda data: data[key] + +class PromptDataset(Dataset): + + def __init__(self,data_dir,*keys,**projections) -> None: + self.data = [] + for d in data_dir: + list_contents = {key:default_get(key)(d) for key in keys if key in d.keys()} + dict_contents = {projection:projections[projection](d) for projection in projections.keys()} + self.data.append({**list_contents,**dict_contents}) + + def __getitem__(self, index) -> dict: + + return self.data[index] + + def __len__(self): + return len(self.data) + +class FileDataset(PromptDataset): + + def __init__(self,data_dir,*keys,**projections) -> None: + with open(data_dir,'r',encoding='utf-8') as file: + data_dir = json.load(file) + if not keys: + keys = data_dir[0].keys() + + self.data = [] + for d in data_dir: + list_contents = {key:default_get(key)(d) for key in keys if key in d.keys()} + dict_contents = {projection:projections[projection](d) for projection in projections.keys()} + self.data.append({**list_contents,**dict_contents}) + + def __getitem__(self, index) -> dict: + + return self.data[index] + + def __len__(self): + return len(self.data) \ No newline at end of file diff --git a/citekit/Dataset/__pycache__/Dataset.cpython-310.pyc b/citekit/Dataset/__pycache__/Dataset.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c0ddd17e20bcca13869976f8dc24eaa65dd40230 Binary files /dev/null and b/citekit/Dataset/__pycache__/Dataset.cpython-310.pyc differ diff --git a/citekit/Dataset/__pycache__/Dataset.cpython-312.pyc b/citekit/Dataset/__pycache__/Dataset.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c24210d5ca7409047fc953648ea3c0f176421c4f Binary files /dev/null and b/citekit/Dataset/__pycache__/Dataset.cpython-312.pyc differ diff --git a/citekit/__init__.py b/citekit/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c3961685ab8defd1a28f2a2a7bb3b74fcafa4d6b --- /dev/null +++ b/citekit/__init__.py @@ -0,0 +1 @@ +from __future__ import absolute_import diff --git a/citekit/__pycache__/__init__.cpython-312.pyc b/citekit/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d2a46b3bd138beb8d4731685e360767dc15baeb8 Binary files /dev/null and b/citekit/__pycache__/__init__.cpython-312.pyc differ diff --git a/citekit/attribute/__init__.py b/citekit/attribute/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/citekit/attribute/__pycache__/__init__.cpython-312.pyc b/citekit/attribute/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2922645817600b423187a026dee67e7e3bfbd231 Binary files /dev/null and b/citekit/attribute/__pycache__/__init__.cpython-312.pyc differ diff --git a/citekit/attribute/__pycache__/attribute.cpython-312.pyc b/citekit/attribute/__pycache__/attribute.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4b1e3e32c49d672e2c3e3004fa658282b407288b Binary files /dev/null and b/citekit/attribute/__pycache__/attribute.cpython-312.pyc differ diff --git a/citekit/attribute/attribute.py b/citekit/attribute/attribute.py new file mode 100644 index 0000000000000000000000000000000000000000..c4a8ed3a9d07b45a0b7d748a34dd4fa5af366575 --- /dev/null +++ b/citekit/attribute/attribute.py @@ -0,0 +1,222 @@ +import json +from context_cite import ContextCiter +import re +import torch +from transformers import LlamaForCausalLM, LlamaTokenizer + + +def all_normalize(obj): + all_values = [] + for output_sent_result in obj: + for each_doc in output_sent_result: + for each_span in each_doc: + all_values.append(each_span[1]) + max_val = max(all_values) + min_val = min(all_values) + for output_sent_result in obj: + for i, each_doc in enumerate(output_sent_result): + for j, each_span in enumerate(each_doc): + each_span = (each_span[0], (each_span[1] - min_val) / (max_val - min_val)) + output_sent_result[i][j] = each_span + return obj + +def all_normalize_in(obj): + for output_sent_result in obj: + all_values = [] + for each_doc in output_sent_result: + for each_span in each_doc: + all_values.append(each_span[1]) + max_val = max(all_values) + min_val = min(all_values) + for i, each_doc in enumerate(output_sent_result): + for j, each_span in enumerate(each_doc): + each_span = (each_span[0], (each_span[1] - min_val) / (max_val - min_val)) + output_sent_result[i][j] = each_span + return obj + +def load_json(file_path): + + with open(file_path, 'r') as file: + data = file.read() + if file_path.endswith('.jsonl'): + data = f'[{'},{'.join(data.split("}\n{"))}]' + objects = json.loads(data) + return objects + +def ma(text): + pattern = r"Document \[\d+\]\(Title:[^)]+\)" + + match = re.search(pattern, text) + + if match: + index = match.end() + return index + else: + return 0 + +def write_json(file_path, data): + with open(file_path, 'w') as json_file: + json.dump(data, json_file, indent=4) + + + +def load_model(model_name_or_path): + from transformers import AutoModelForCausalLM, AutoTokenizer + model = AutoModelForCausalLM.from_pretrained( + model_name_or_path, + device_map='auto', + token = 'your token' + ) + + tokenizer = AutoTokenizer.from_pretrained(model_name_or_path) + model.eval() + return model, tokenizer + + +def compute_log_prob(model, tokenizer, input_text, output_text): + inputs = tokenizer(input_text, return_tensors="pt") + output_tokens = tokenizer(output_text, return_tensors="pt")["input_ids"] + + with torch.no_grad(): + logits = model(**inputs).logits[:, -output_tokens.shape[1]-1:-1, :] + + log_probs = torch.nn.functional.log_softmax(logits, dim=-1) + output_log_probs = log_probs.gather(2, output_tokens.unsqueeze(-1)).squeeze(-1) + return output_log_probs.sum().item() + +def compute_contributions(model, tokenizer, question, docs, output): + full_input = question + '\n\n' + '\n'.join(docs) + base_prob = compute_log_prob(model, tokenizer, full_input, output) + + contributions = [] + for i in range(len(docs)): + reduced_docs = docs[:i] + docs[i+1:] + reduced_input = question + '\n\n' + '\n'.join(reduced_docs) + reduced_prob = compute_log_prob(model, tokenizer, reduced_input, output) + contributions.append(base_prob - reduced_prob) + + return contributions + +class InterpretableAttributer: + + def __init__(self, levels=['doc', 'span', 'word'], model = 'gpt-2'): + for level in levels: + assert level in ['doc', 'span', 'word'], f'Invalid level: {level}' + # span before doc + self.levels = sorted(levels, key=lambda x: ['span', 'doc', 'word'].index(x)) + #self.model, self.tokenizer = load_model(model) + + + def attribute(self, question, docs, output): + attribute_results = {} + for level in self.levels: + attribute_result = [] + for sentence in output: + attribute_result.append(self._attribute(question, docs, sentence, level)) + attribute_results[level] = attribute_result + return attribute_results + + + def _attribute(self, question, docs, output, level): + if level == 'doc': + return self.doc_level_attribution(question, docs, output) + elif level == 'span': + return self.span_level_attribution(question, docs, output) + elif level == 'word': + return self.word_level_attribution(question, docs, output) + else: + raise ValueError(f'Invalid level: {level}') + + def span_level_attribution(self, question, docs, output): + # USE CONTEXT CITE + context = '\n\n'.join(docs) + response = output + + cc = ContextCiter(self.model, self.tokenizer, context, question) + _, prompt = cc._get_prompt_ids(return_prompt=True) + cc._cache["output"] = prompt + response + result = cc.get_attributions(as_dataframe=True, top_k=1000).data.to_dict(orient='records') + return result + + + def parse_attribution_results(self, docs, results): + context = '\n\n'.join(docs) + lens = [len(doc) for doc in docs] + len_sep = len('\n\n') + final_results = {} + for level, result in results.items(): + if level == 'span': + ordered_all_sents = [] + for output_sent_result in result: + final_end_for_span = {} + all_span_results = [] + for each_span in output_sent_result: + span_text = each_span["Source"] + span_score = each_span["Score"] + start = 0 + if span_text in final_end_for_span: + start = final_end_for_span[span_text] + span_start = context.find(span_text, start) + span_end = span_start + len(span_text) + final_end_for_span[span_text] = span_end + # locate the document + doc_idx = 0 + while span_start > lens[doc_idx]: + span_start -= lens[doc_idx] + len_sep + span_end -= lens[doc_idx] + len_sep + doc_idx += 1 + all_span_results.append((span_start, span_score, doc_idx)) + ordered = [[] for _ in range(len(docs))] + for span_start, span_score, doc_idx in all_span_results: + ordered[doc_idx].append((span_start, span_score)) + for i in range(len(docs)): + doc = docs[i] + real_start = ma(doc) + ordered[i] = sorted(ordered[i], key=lambda x: x[0]) + ordered[i][0] = (real_start, ordered[i][0][1]) + + ordered_all_sents.append(ordered) + final_results[level+'_level'] = all_normalize_in(ordered_all_sents) + elif level == 'doc': + self.span_to_doc(result) + else: + raise NotImplementedError(f'Parsing for {level} not implemented yet') + return final_results + + def span_to_doc(self, results): + import numpy as np + span_level = results['span_level'] + doc_level = [] + for output_sent_result in span_level: + doc_level.append([np.mean([span[1] for span in doc]) for doc in output_sent_result]) + results['doc_level'] = doc_level + + + def attribute_for_result(self, result): + docs = result['doc_cache'] + question = result['data']['question'] + output = result['output'] + attribution_results = self.attribute(question, docs, output) + parsed_results = self.parse_attribution_results(docs, attribution_results) + result.update(parsed_results) + + if 'doc' not in self.levels: + # if doc is not in the levels, we need to convert the span level to doc level + print('Converting span level to doc level...') + try: + self.span_to_doc(result) + print('Conversion successful') + except Exception as e: + print(f'Error converting span level to doc level: {e}') + + def attribute_for_results(self, results): + for result in results: + self.attribute_for_result(result) + return results + + +if __name__ == '__main__': + attributer = InterpretableAttributer(levels=['span']) + results = load_json('res_attr.json') + attributer.attribute_for_results(results) + write_json('res_attr_span.json', results) \ No newline at end of file diff --git a/citekit/cite_modules/LLM.py b/citekit/cite_modules/LLM.py new file mode 100644 index 0000000000000000000000000000000000000000..19088dd5909052994adc6e9280453c02303442b6 --- /dev/null +++ b/citekit/cite_modules/LLM.py @@ -0,0 +1,427 @@ +import torch +from citekit.prompt.prompt import Prompt +import re +from citekit.utils.utils import one_paragraph, first_sentence, make_as +import random +import os + + + +class Module: + module_count = 1 + def __init__(self,prompt_maker: Prompt = None, pipeline = None, self_prompt = {}, iterative = False, merge = False, max_turn =6, output_as = None, parallel = False) -> None: + self.self_prompt = self_prompt + self.use_head_prompt = True + self.connect_to(pipeline) + self.prompt_maker = prompt_maker + self.last_message = '' + self.destinations = [] + self.conditions = {} + self.head_key = None + self.parallel = parallel + self.iterative = iterative + self.merge = merge + self.head_process = one_paragraph + self.max_turn = max_turn + self.multi_process = False + self.output_cond = {} # {cond : {'post_processing':post, 'end':end}} + self.count = Module.module_count + Module.module_count += 1 + self.if_add_output_to_head = False + self.turns = 0 + self.end = False + + def __str__(self) -> str: + if self.model_type: + return f'{self.model_type}-[{self.count}]' + else: + return f'Unknown-type module-[{self.count}]' + + def get_json_config(self, config): + print('get_json_config:',config) + avaliable_mapping = { + 'max turn': 'max_turn', + 'prompt': 'prompt', + 'destination': 'destination', + 'global prompt': 'head_key', + } + if config == 'prompt': + prompt_info = { + 'template': self.prompt_maker.template, + 'components': self.prompt_maker.components + } + self_info = self.self_prompt + + return { + 'prompt_info': prompt_info, + 'self_info': self_info + } + elif config == 'destination': + return { + 'destination': str(self.destinations[0]) + } + elif config in ['max turn','global prompt']: + config = avaliable_mapping[config] + print('getting the config:',config) + return getattr(self, config) + else: + raise NotImplementedError(f'get_json_config for {config} is not implemented') + + def get_destinations(self): + return self.destinations + + def update(self, config, update_info): + + if config == 'prompt': + template = update_info['template'] + components = update_info['components'] + self_prompt = update_info['self_prompt'] + import copy + # avoid changing the original prompt_maker + self.prompt_maker = copy.deepcopy(self.prompt_maker) + + self.prompt_maker.update(template=template, components=components) + self.self_prompt = self_prompt + + elif config == 'destination': + print('update destination:',update_info[0], 'post_processing:',update_info[1]) + if update_info[1] == 'None': + self.set_target(update_info[0]) + else: + self.set_target(update_info[0], post_processing=make_as(update_info[1])) + + elif config == 'delete_destination': + for i, d in enumerate(self.destinations): + if str(d) == str(update_info): + self.destinations.remove(d) + del self.conditions[d] + break + elif config == 'header': + self.add_to_head(update_info, sub = True) + elif config == 'max turn': + self.max_turn = update_info + else: + raise NotImplementedError(f'update for {config} is not implemented') + + def end_multi(self): + return + + def set_use_head_prompt(self,use): + assert isinstance(use,bool) + self.use_head_prompt = use + + def reset(self): + self.end = False + self.turns = 0 + + def change_to_multi_process(self,bool_value): + if bool_value: + self.last_message = [] + else: + self.last_message = '' + self.multi_process = bool_value + @property + def get_use_head_prompt(self): + return self.use_head_prompt + + def generate(self, head_prompt: dict = {}, dynamic_prompt: dict = {}): + raise NotImplementedError + + def send(self): + for destination in self.destinations: + cond = self.conditions[destination]['condition'] + if cond(self): + return destination + return None + + def set_target(self,destination, condition = lambda self: True, post_processing = lambda x:x) -> None: + self.conditions[destination] = {'condition': condition, 'post_processing' : post_processing} + self.destinations = [destination] + self.destinations + destination.connect_to(self.pipeline) + + def clear_destination(self): + self.destinations = [] + self.conditions = {} + + def add_output_to_head(self, outputs): + if self.if_add_output_to_head: + if not self.head_sub: + if self.head_key not in self.pipeline.head.keys(): + self.pipeline.head.update({self.head_key: self.head_process(outputs)}) + else: + self.pipeline.head[self.head_key] += '\n' + self.pipeline.head[self.head_key] += self.head_process(outputs) + else: + self.pipeline.head[self.head_key] = self.head_process(outputs) + + def connect_to(self, pipeline = None) -> None: + self.pipeline = pipeline + if pipeline: + pipeline.module.append(self) + + def output(self): + outed = False + for cond, post_and_end in self.output_cond.items(): + if cond(self): + if not outed: + if not self.merge: + self.pipeline.output.append(post_and_end['post_processing'](self.last_message)) + else: + self.pipeline.output.append(post_and_end['post_processing'](''.join(self.last_message))) + outed = True + if post_and_end['end']: + self.end = True + + def set_output(self, cond = lambda self: True, post_processing = lambda x:x, end = True): + self.output_cond[cond] = {'post_processing': post_processing, 'end' : end} + + def get_first_module(self): + return self + + def add_to_head(self, datakey, sub = False, process = None): + self.if_add_output_to_head = True + self.head_key = datakey + self.head_sub = sub + if process: + self.head_process = process + + +def load_model(model_name_or_path,dtype = torch.float16): + from transformers import AutoModelForCausalLM, AutoTokenizer + model = AutoModelForCausalLM.from_pretrained( + model_name_or_path, + torch_dtype=dtype, + device_map='auto', + ) + + tokenizer = AutoTokenizer.from_pretrained(model_name_or_path) + model.eval() + return model, tokenizer + + +class LLM(Module): + model_type = 'Generator' + def __init__(self, model = None, prompt_maker: Prompt =None, pipeline = None, post_processing = None, self_prompt = {}, device = 'cpu',temperature = 0.5 ,stop = None, max_turn = 6, share_model_with = None, iterative = False, auto_cite = False, output = None,merge = False, noisy = True, parallel = False, output_as ='Answer', auto_cite_from = 'docs') -> None: + super().__init__(prompt_maker,pipeline,self_prompt, iterative, merge, parallel = parallel) + self.max_turn = max_turn + if post_processing: + self.post_processing = post_processing + else: + self.post_processing = lambda x: {output_as:x} + if model: + self.model_name = model + self.stop = stop + self.multi_process = False + self.noisy = noisy + self.head_process = one_paragraph + self.auto_cite = auto_cite + if auto_cite: + self.cite_from = auto_cite_from + if model: + if 'gpt' not in model.lower(): + if not share_model_with: + print('loading model...') + self.model, self.tokenizer = self.load_model(model) + else: + print('sharing model...') + self.model, self.tokenizer = share_model_with.model, share_model_with.tokenizer + self.temperature = temperature + self.device = device + else: + self.openai_key = os.getenv('OPENAI_API_KEY') + self.output_cond = {} # {cond : {'post_processing':post, 'end':end}} + self.if_add_output_to_head = False + + self.token_used = 0 + + def reset(self): + self.end = False + self.turns = 0 + self.token_used = 0 + + + def __str__(self) -> str: + if self.model_name: + return f'{self.model_name}-[{self.count}]' + else: + return 'unknown model' + + def __repr__(self) -> str: + return (f'{self.prompt_maker}\n|\n|\nV\n{self}\n|\n|\nV\n'+ '/'.join([str(des) for des in self.destinations]+['output'])) + + def load_model(self, model_name_or_path,dtype = torch.float16): + from transformers import AutoModelForCausalLM, AutoTokenizer + model = AutoModelForCausalLM.from_pretrained( + model_name_or_path, + torch_dtype=dtype, + device_map='auto', + ) + + tokenizer = AutoTokenizer.from_pretrained(model_name_or_path) + model.eval() + return model, tokenizer + + def set_cite(self,key): + self.cite_from = key + self.auto_cite = True + + def generate_content(self, prompt): + if 'gpt' in self.model_name.lower(): + import openai + openai.api_key = self.openai_key + prompt = [ + {'role': 'system', + 'content': "You are a good helper who follow the instructions"}, + {'role': 'user', 'content': prompt} + ] + response = openai.ChatCompletion.create( + model=self.model_name, + messages=prompt, + max_tokens=500, + stop = self.stop + ) + self.token_used += response['usage']['completion_tokens'] + response['usage']['prompt_tokens'] + return response['choices'][0]['message']['content'] + + else: + inputs = self.tokenizer(prompt, return_tensors="pt").to(self.model.device) + stop = [] if self.stop is None else self.stop + + outputs = self.model.generate( + **inputs, + do_sample = True, + max_new_tokens = 200, + temperature = self.temperature + ) + self.token_used += len(outputs[0]) + + outputs = self.tokenizer.decode(outputs[0][inputs['input_ids'].size(1):], skip_special_tokens=True) + return one_paragraph(outputs) + print(outputs) + + + def generate(self, head_prompt: dict = {}, dynamic_prompt: dict = {}): + if self.use_head_prompt: + #print(head_prompt,self.self_prompt,dynamic_prompt) + prompt = self.prompt_maker(head_prompt,self.self_prompt,dynamic_prompt) + else: + prompt = self.prompt_maker(self.self_prompt,dynamic_prompt) + if self.noisy: + print(f'prompt to {str(self)}:\n',prompt,'\n\n') + self.turns += 1 + + outputs = self.generate_content(prompt) + #print('DEBUG:',outputs) + if self.noisy: + print('OUTPUT:') + print(outputs) + if self.auto_cite: + outputs = self.cite_from_prompt({**head_prompt,**self.self_prompt,**dynamic_prompt},outputs) + if self.multi_process: + self.last_message.append(outputs) + else: + self.last_message = outputs + + + self.add_output_to_head(outputs) + + destination = self.send() + + if self.turns > self.max_turn: + self.end = True + if destination in self.conditions: + return self.conditions[destination]['post_processing'](outputs) + else: + return self.post_processing(outputs) + + def add_output_to_head(self, outputs): + if self.if_add_output_to_head: + if not self.head_sub: + if self.head_key not in self.pipeline.head.keys(): + self.pipeline.head.update({self.head_key: self.head_process(outputs)}) + else: + self.pipeline.head[self.head_key] += '\n' + self.pipeline.head[self.head_key] += self.head_process(outputs) + else: + self.pipeline.head[self.head_key] = self.head_process(outputs) + + def output(self): + outed = False + for cond, post_and_end in self.output_cond.items(): + if cond(self): + if not outed: + if not self.merge and not self.iterative: + self.pipeline.output.append(post_and_end['post_processing'](self.last_message)) + else: + self.pipeline.output.append(post_and_end['post_processing'](' '.join(self.last_message))) + outed = True + if post_and_end['end']: + self.end = True + + def set_output(self, cond = lambda self: True, post_processing = lambda x:x, end = True): + self.output_cond[cond] = {'post_processing': post_processing, 'end' : end} + + + def cite_from_prompt(self,prompt_dict,input): + input = first_sentence(input) + cite_docs = prompt_dict[self.cite_from] + refs = re.findall(r'\[\d+\]', cite_docs) + pattern = r'([.!?])\s*$' + if refs: + cite = ''.join(refs) + else: + cite = '' + output = re.sub(pattern, rf' {cite}\1 ', input) + if cite not in output: + output += cite + return output + def add_to_head(self, datakey, sub = False, process = None): + self.if_add_output_to_head = True + self.head_key = datakey + self.head_sub = sub + if process: + self.head_process = process + + + +class TestLLM(LLM): + def __init__(self, model='gpt-4', prompt_maker: Prompt = None, pipeline=None, post_processing=lambda x: x, self_prompt={}, device='cpu', temperature=0.5, stop=None, max_turn=6,share_model_with = None, iterative= False, ans = None) -> None: + super().__init__(model,prompt_maker,pipeline,self_prompt=self_prompt,share_model_with=share_model_with,iterative=iterative) + self.max_turn = max_turn + self.post_processing = post_processing + self.model_name = model + self.last_message = '' + self.stop = stop + self.output_cond = {} # {cond : {'post_processing':post, 'end':end}} + self.if_add_output_to_head = False + + self.token_used = 0 + self.ans = 'Strain[1], turns:, heat[2][4]. Sent2[5]. Sent3.\n\n rdd' if not ans else ans + def generate_content(self, prompt): + return self.ans + + +class AutoAISLLM(LLM): + def __init__(self, model=None, prompt_maker: Prompt = None, pipeline=None, post_processing=None, self_prompt={}, device='cpu', temperature=0.5, stop=None, max_turn=6, share_model_with=None, iterative=False, auto_cite=False, output=None, merge=False, noisy=False, output_as='Answer') -> None: + super().__init__(model, prompt_maker, pipeline, post_processing, self_prompt, device, temperature, stop, max_turn, share_model_with, iterative, auto_cite, output, merge, noisy, output_as) + + self.prompt_maker = Prompt('\n Answer: ',components={ + 'INST':'{INST}\n\n', + 'premise':'Premise: {premise}\n\n', + 'claim':'Claim: {claim}\n', + }) + self.self_prompt={'INST': 'In this task, you will be presented a premise and a claim. If the premise entails the claim, output "1", otherwise output "1". Your answer should only contains one number without any other letters and punctuations.'} + + def generate(self, premise, claim): + dict_answer = super().generate({'premise':premise,'claim':claim}) + return dict_answer.get('Answer') + + + +if __name__ == '__main__': + prompt = Prompt(template='',components={'INST':'{INST}\n\n', + 'Question':'Question:{Question}\n\n', + 'Docs':'{Docs}\n', + 'feedback':'Here is the feed back of your last response:{feedback}\n', + 'Answer':'Here is answer and you have to give feedback:{Answer}'}) + m = LLM('gpt') \ No newline at end of file diff --git a/citekit/cite_modules/Retrieve.py b/citekit/cite_modules/Retrieve.py new file mode 100644 index 0000000000000000000000000000000000000000..8df9c120000d511ffc6da46da01c2d8bc539804f --- /dev/null +++ b/citekit/cite_modules/Retrieve.py @@ -0,0 +1,99 @@ +import argparse +import csv +import json +import os +import time +import pickle + +import numpy as np +import torch +from tqdm import tqdm +from rank_bm25 import BM25Okapi +from sentence_transformers import SentenceTransformer + +def gtr_build_index(encoder, docs): + with torch.inference_mode(): + embs = encoder.encode(docs, show_progress_bar=True, normalize_embeddings=True) + embs = embs.astype("float16") + + GTR_EMB = os.environ.get("GTR_EMB") + with open(GTR_EMB, "wb") as f: + pickle.dump(embs, f) + return embs + + +class DPRRetriever: + def __init__(self, DPR_WIKI_TSV, GTR_EMB = None, emb_model = "sentence-transformers/gtr-t5-xxl") -> None: + device = "cuda" if torch.cuda.is_available() else "cpu" + self.device = device + self.encoder = SentenceTransformer(emb_model, device = device) + self.docs = [] + print("loading wikipedia file...") + with open(DPR_WIKI_TSV) as f: + reader = csv.reader(f, delimiter="\t") + for i, row in enumerate(reader): + if i == 0: + continue + self.docs.append(row[2] + "\n" + row[1]) + if not GTR_EMB: + print("gtr embeddings not found, building...") + embs = gtr_build_index(self.encoder, self.docs) + else: + print("gtr embeddings found, loading...") + with open(GTR_EMB, "rb") as f: + embs = pickle.load(f) + + self.gtr_emb = torch.tensor(embs, dtype=torch.float16, device=device) + + def retrieve(self, question, topk): + with torch.inference_mode(): + query = self.encoder.encode(question, batch_size=4, show_progress_bar=True, normalize_embeddings=True) + query = torch.tensor(query, dtype=torch.float16, device=self.device) + query = query.to(self.device) + scores = torch.matmul(self.gtr_emb, query) + score, idx = torch.topk(scores, topk) + ret = [] + for i in range(idx.size(0)): + title, text = self.docs[idx[i].item()].split("\n") + ret.append({"id": str(idx[i].item() + 1), "title": title, "text": text, "score": score[i].item()}) + + return ret + + def __repr__(self) -> str: + return 'DPR Retriever' + + def __str__(self) -> str: + return repr(self) + +class BM25Retriever: + def __init__(self, DPR_WIKI_TSV): + self.docs = [] + self.tokenized_docs = [] + print("loading wikipedia file...") + with open(DPR_WIKI_TSV) as f: + reader = csv.reader(f, delimiter="\t") + for i, row in enumerate(reader): + if i == 0: + continue + self.docs.append(row[2] + "\n" + row[1]) + self.tokenized_docs.append((row[2] + " " + row[1]).split()) + + print("BM25 index building...") + self.bm25 = BM25Okapi(self.tokenized_docs) + + def retrieve(self, question, topk): + query = question.split() + scores = self.bm25.get_scores(query) + topk_indices = scores.argsort()[-topk:][::-1] + ret = [] + for idx in topk_indices: + title, text = self.docs[idx].split("\n", 1) + ret.append({"id": str(idx + 1), "title": title, "text": text, "score": scores[idx]}) + + return ret + def __repr__(self) -> str: + return 'BM25 Retriever' + + def __str__(self) -> str: + return repr(self) + \ No newline at end of file diff --git a/citekit/cite_modules/__pycache__/LLM.cpython-310.pyc b/citekit/cite_modules/__pycache__/LLM.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3b106caeef231a283657daa18d51c379bec72498 Binary files /dev/null and b/citekit/cite_modules/__pycache__/LLM.cpython-310.pyc differ diff --git a/citekit/cite_modules/__pycache__/LLM.cpython-312.pyc b/citekit/cite_modules/__pycache__/LLM.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b4381db22159a0c815bec458f9a9a62f866efa89 Binary files /dev/null and b/citekit/cite_modules/__pycache__/LLM.cpython-312.pyc differ diff --git a/citekit/cite_modules/__pycache__/Retrieve.cpython-310.pyc b/citekit/cite_modules/__pycache__/Retrieve.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4387f9c1796bf921b9646d2d71415761946984b4 Binary files /dev/null and b/citekit/cite_modules/__pycache__/Retrieve.cpython-310.pyc differ diff --git a/citekit/cite_modules/__pycache__/Retrieve.cpython-312.pyc b/citekit/cite_modules/__pycache__/Retrieve.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..501b2b12631d560b4c66e2b6294eb237121f3ef9 Binary files /dev/null and b/citekit/cite_modules/__pycache__/Retrieve.cpython-312.pyc differ diff --git a/citekit/cite_modules/__pycache__/augment_model.cpython-310.pyc b/citekit/cite_modules/__pycache__/augment_model.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..35c2b7746804d62463782423f79f1d9320e0289a Binary files /dev/null and b/citekit/cite_modules/__pycache__/augment_model.cpython-310.pyc differ diff --git a/citekit/cite_modules/__pycache__/augment_model.cpython-312.pyc b/citekit/cite_modules/__pycache__/augment_model.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5d2b2a05cbdca8ef709b1fc6e21f6b51c5b32c9e Binary files /dev/null and b/citekit/cite_modules/__pycache__/augment_model.cpython-312.pyc differ diff --git a/citekit/cite_modules/augment_model.py b/citekit/cite_modules/augment_model.py new file mode 100644 index 0000000000000000000000000000000000000000..ae23d51c0e5f205dade48cd99c9f379279540d55 --- /dev/null +++ b/citekit/cite_modules/augment_model.py @@ -0,0 +1,455 @@ +from citekit.cite_modules.LLM import Module,LLM +from citekit.cite_modules.Retrieve import DPRRetriever +from citekit.evaluator.evaluator import _run_nli_autoais, Evaluator +from citekit.prompt.prompt import Prompt +from citekit.utils.utils import one_paragraph, make_as +from sentence_transformers import SentenceTransformer +import re +import random + + +class Retriever(Module): + model_type = 'retriever' + def __init__(self, documents = None ,retrieve_by = 'index', prompt_maker = None, pipeline = None, post_processing = lambda input, output: {'RetrievedDocs':output}, self_prompt = {},topk = 3,add_id = True, merge = False, tsv_path = 'None', emb_path = 'None', retrieve_from_data = True, parallel = False) -> None: + super().__init__(prompt_maker,pipeline,self_prompt,merge=merge, parallel=parallel) + self.retrieve_by = retrieve_by + self.use_head_prompt = False + self.dataset_documents = None + self.documents = None + self.default_doc_key = 'docs' + self.retrieve_from_data = retrieve_from_data + if not documents: + self.documents = self.pipeline.doc_cache + else: + self.dataset_documents = documents + self.post_processing = post_processing + self.if_add_output_to_head = False + self.topk = topk + self.add_id = add_id + if retrieve_by =='bm25': + self.bm25_module_loaded = False + from rank_bm25 import BM25Okapi + import nltk + nltk.download('punkt') + from nltk.tokenize import word_tokenize + self.word_tokenize = word_tokenize + self.BM25Okapi = BM25Okapi + self.bm25_module_loaded = True + elif retrieve_by == 'dpr': + self.dpr_retriever = DPRRetriever(DPR_WIKI_TSV=tsv_path, + GTR_EMB=emb_path) + + def generate(self,head_prompt: dict = {}, dynamic_prompt: dict = {}): + index = self.pipeline.data_index + if self.dataset_documents: + if isinstance(self.dataset_documents[0], list): + # Each data has a document + self.documents = self.dataset_documents[index] + else: + self.documents = self.dataset_documents + else: + if not self.retrieve_from_data: + self.documents = self.pipeline.doc_cache.show_docs() + else: + def _stringtfy(doc): + if isinstance(doc, str): + return doc + return f"({doc['title']}) {doc['text']}" + self.documents = list(map(_stringtfy, self.pipeline.current_data[self.default_doc_key])) + # query + if self.use_head_prompt: + prompt = self.prompt_maker(head_prompt,self.self_prompt,dynamic_prompt) + else: + prompt = self.prompt_maker(self.self_prompt,dynamic_prompt) + + retrieved_docs = [] + if self.retrieve_by == 'index': + # query : Document [1][2] + indice = [int(r[1:]) - 1 for r in re.findall(r"\[\d+",prompt)] + for index in indice: + retrieved_docs.append(self.documents[index]) + if len(retrieved_docs) > self.topk: + retrieved_docs = retrieved_docs[:self.topk] + elif self.retrieve_by =='bm25': + # natural language query + tokenized_docs = [self.word_tokenize(doc.lower()) for doc in self.documents] + bm25 = self.BM25Okapi(tokenized_docs) + tokenized_query = self.word_tokenize(prompt.lower()) + doc_scores = bm25.get_scores(tokenized_query) + if self.topk > len(doc_scores): + self.topk = len(doc_scores) - 1 + top_k_idx = sorted(range(len(doc_scores)), key=lambda i: doc_scores[i], reverse=True)[:self.topk] + retrieved_docs = [self.documents[index] for index in top_k_idx] + retrieved_docs_new = [] + for re_doc in retrieved_docs: + self.pipeline.doc_cache.add_doc(re_doc,self.add_id) + retrieved_docs_new.append(self.pipeline.doc_cache.get_last()) + retrieved_docs = retrieved_docs_new + #raise NotImplementedError + + + elif self.retrieve_by =='gtr': + docs_dict = self.dpr_retriever.retrieve(prompt,topk=self.topk) + retrieved_docs = [f"({d['title']}) {d['text']}" for d in docs_dict] + retrieved_docs_new = [] + for re_doc in retrieved_docs: + self.pipeline.doc_cache.add_doc(re_doc,self.add_id) + retrieved_docs_new.append(self.pipeline.doc_cache.get_last()) + retrieved_docs = retrieved_docs_new + + retrieved_docs_prompt = '\n'.join(retrieved_docs) + destination = self.send() + if self.multi_process: + self.last_message.append(retrieved_docs_prompt) + else: + self.last_message = retrieved_docs_prompt + #print(self.last_message) + + if self.if_add_output_to_head: + self.pipeline.head.update({self.head_key:retrieved_docs_prompt}) + if destination in self.conditions: + try: + return self.conditions[destination]['post_processing'](prompt,retrieved_docs_prompt) + except: + return self.conditions[destination]['post_processing'](retrieved_docs_prompt) + + else: + return retrieved_docs_prompt + raise NotImplementedError + + +class EvalModule(Module, Evaluator): + model_type = 'evaluator' + def __init__(self, prompt_maker = None, pipeline=None, self_prompt={},criteria = None, iterative = False, max_turn =6 ,parallel = False) -> None: + Module.__init__(self,prompt_maker, pipeline, self_prompt,iterative=iterative,max_turn=max_turn, parallel=parallel) + Evaluator.__init__(self,criteria,pipeline) + + def generate(self, head_prompt: dict = {}, dynamic_prompt: dict = {}): + result = {} + p_data = {**head_prompt, **self.self_prompt,**dynamic_prompt} + for criteria, get_data in self.get_data.items(): + data_dict = {} + for k, v in get_data.items(): + if v == 'doc_cache': + data_dict[k] = self.pipeline.doc_cache.show_docs() + else: + if v in p_data.keys(): + data_dict[k] = p_data[v] + else: + data_dict[k] = self.pipeline.current_data[v] + eval_func = Evaluator.eval_criteria[criteria] + data = [data_dict] + result[criteria] = eval_func(data) + + if self.multi_process: + self.last_message.append(result) + else: + self.last_message = result + destination = self.send() + if destination in self.conditions: + return self.conditions[destination]['post_processing'](result) + else: + return result + + +class CitationSimplyfier(Module): + ''' + Simplify the citation of the 'answer' in prompt. + Argument can be changed to fit into different name of key in prompts + By Defaut, the simplifier simplifies the 'answer' and output the sring with citation simplified. + ''' + model_type = 'simplifier' + def __init__(self, prompt_maker = None, pipeline=None, self_prompt={}, criteria = None, key = 'answer', test = False, allow_empty = True) -> None: + Module.__init__(self,prompt_maker, pipeline, self_prompt) + if not test: + self.entail = _run_nli_autoais + else: + self.entail = lambda p,c : random.randint(0,1) + self.docs = ['0'] * 100 + self.key = key + self.allow_empty = allow_empty + def generate(self, head_prompt: dict = {}, dynamic_prompt: dict = {}) -> str: + docs = self.pipeline.doc_cache + prompt = {**head_prompt, **dynamic_prompt} + answer = prompt[self.key] + + refs = re.findall(r'\[\d+\]', answer) + last_ref_index = None + for match in re.finditer(r'\[\d+\]', answer): + last_ref_index = match.end() + + if not refs: + return answer + + refs_str = ''.join(refs) + + def simplify(sentence, refs, docs): + ref_numbers = [int(ref.strip('[]')) for ref in refs] + + docs_combined = ''.join(docs[ref - 1] for ref in ref_numbers if ref - 1 < len(docs)) + + if not self.entail(docs_combined, sentence): + if self.allow_empty: + return '' + return ''.join(refs) + + if len(ref_numbers) == 1: + return ''.join(f'[{num}]' for num in ref_numbers) + + def remove_and_test(ref_numbers): + for i, ref in enumerate(ref_numbers): + new_ref_numbers = ref_numbers[:i] + ref_numbers[i+1:] + new_docs_combined = ''.join(docs[r - 1] for r in new_ref_numbers if r - 1 < len(docs)) + if self.entail(new_docs_combined, sentence): + if len(new_ref_numbers) == 1: + return new_ref_numbers + return remove_and_test(new_ref_numbers) + return ref_numbers + + simplified_ref_numbers = remove_and_test(ref_numbers) + + simplified_refs = ''.join(f'[{num}]' for num in simplified_ref_numbers) + return simplified_refs + + simplified_refs = simplify(answer,refs,docs) + + sentence_without_refs = re.sub(r'\[\d+\]', '', answer) + + if last_ref_index is not None: + output = sentence_without_refs[:last_ref_index - len(refs_str)] + simplified_refs + sentence_without_refs[last_ref_index - len(refs_str):] + else: + output = sentence_without_refs + simplified_refs + + if not simplified_refs and self.allow_empty: + output = '' + + + if self.multi_process: + self.last_message.append(output) + else: + self.last_message = output + + return output + + +class Verifier(Module): + + ''' + Verifier is currently only used for single sentence/single target answer, not for parallel or iterative answer. + Verifier will return dynamic prompt, not like other modules returning output. It is a judger only to decide the target module. + By default, the verifoer verifies whether the documents support the answer. + ''' + model_type = 'verifier' + def __init__(self, prompt_maker = None, pipeline=None, self_prompt={}, criteria = None, key = 'answer', test = False) -> None: + Module.__init__(self,prompt_maker, pipeline, self_prompt) + if not test: + self.entail = _run_nli_autoais + else: + self.entail = lambda p,c : random.randint(0,1) + self.docs = ['s'] * 100 + self.key = key + self.test = test + + # Overcite this function to + def verifier_judge(self,**kargs): + docs = self.pipeline.doc_cache + answer = kargs[self.key] + refs = re.findall(r'\[\d+\]', answer) + if not refs: + return False + ref_numbers = [int(ref.strip('[]')) for ref in refs] + + docs_combined = ''.join(docs[ref - 1] for ref in ref_numbers if ref - 1 < len(docs)) + return bool(self.entail(docs_combined, re.sub(r'\[\d+\]', '', answer))) + + + def generate(self, head_prompt: dict = {}, dynamic_prompt: dict = {}): + prompt = {**head_prompt, **dynamic_prompt} + out = self.verifier_judge(**prompt) + + self.last_message = out + + self.turns += 1 + return dynamic_prompt + + +class AugmentCluster(): + def __init__(self, module_list = []) -> None: + self.module_list = module_list + module_count = len(module_list) + for i in range(module_count - 1): + assert isinstance(module_list[i],LLM) and isinstance(module_list[i+1],LLM) + module_list[i].set_target(module_list[i+1], post_processing = module_list[i].post_processing) + + def get_first_module(self): + return self.module_list[0] + + def get_destinations(self): + return self.module_list[-1].get_destinations() + + def reset(self): + for module in self.module_list: + module.reset() + + def update(self, config, update_info): + print('updating the AugmentCluster', update_info) + self.module_list[-1].update(config, update_info) + + def __str__(self): + print('getting str of the cluster', ' -> '.join([str(module) for module in self.module_list])) + return ' -> '.join([str(module) for module in self.module_list]) + + def set_target(self,destination, condition = lambda self: True, post_processing = lambda x: x) -> None: + self.module_list[-1].set_target(destination, condition, post_processing) + + def set_output(self, cond = lambda self: True, post_processing = lambda x:x, end = True): + self.module_list[-1].set_output(cond, post_processing, end) + + def connect_to(self, pipeline = None) -> None: + for module in self.module_list: + module.connect_to(pipeline) + pipeline.stored_clusters.append(self) + + +class Attribute_post_select(LLM): + noisy = False + model_name = 'function' + def generate(self, head_prompt: dict = {}, dynamic_prompt: dict = {}): + print('post_select', head_prompt, dynamic_prompt) + docs = head_prompt['docs'] + ans_docs = one_paragraph(dynamic_prompt['span']).split("\n") + spans = [ans_doc[14:].split("") for ans_doc in ans_docs] + msg = '' + span_list = {} + doc_map = {} + j = 1 + i = 1 + for doc in spans: + if doc!= [] : + span_list[f'{i}'] = [] + msg += f'Document [{i}]:\n' + for span in doc: + if len(span)> 3: + msg += f'{j}. {span}\n' + span_list[f'{i}'].append(f'{j}. {span.strip()}') + doc_map[str(j)] = str(i) + j+=1 + docs = docs.replace(span.strip(), f'{span.strip()}') + i+=1 + self.pipeline.head['doc_map'] = doc_map + self.pipeline.head['docs'] = docs + self.pipeline.head['span'] = msg + self.pipeline.head['span_list'] = span_list + return {'span_list': Prompt.UNABLE,'doc_map': Prompt.UNABLE} + +class Attribute_post_cluster(LLM): + noisy = False + model_name = 'function' + def generate(self, head_prompt: dict = {}, dynamic_prompt: dict = {}): + print('f1', head_prompt, dynamic_prompt) + span_ls = head_prompt['span_list'] + doc_map = head_prompt['doc_map'] + span_list = [item for sublist in head_prompt['span_list'].values() for item in sublist] + clusters = eval(one_paragraph(dynamic_prompt['cls'].strip())) + self.pipeline.head['clusters'] = clusters + def _form(cls): + text = '' + doc_list = cls['cluster'] + for doc_num in span_ls.keys(): + pieces = [str(i) for i in doc_list if doc_map.get(str(i),'None') == doc_num] + if pieces: + text += f'Document [{doc_num}]: \n' + '\n'.join([span_list[int(num)-1] for num in pieces]) + '\n' + + return(text) + #print([{'span': _form(cls)} for cls in clusters]) + return [{'span': _form(cls),'span_list': Prompt.UNABLE,'doc_map': Prompt.UNABLE,'clusters':Prompt.UNABLE,'docs':Prompt.UNABLE} for cls in clusters if _form(cls)] + +class Ranker(EvalModule): + + def __init__(self, prompt_maker=None, pipeline=None, self_prompt={}, criteria=None,iterative = True, max_turn = 3, parallel = False, post_processing = lambda x : x, fixed_turn = None) -> None: + super().__init__(prompt_maker, pipeline, self_prompt, criteria, iterative = iterative, max_turn = max_turn, parallel = parallel) + self.compare = [] + self.post_processing = post_processing + if fixed_turn: + self.fixed_turn = fixed_turn + else: + self.fixed_turn = max_turn + def generate(self, head_prompt: dict = {}, dynamic_prompt: dict = {}): + + self.turns += 1 + result = {} + p_data = {**head_prompt, **self.self_prompt,**dynamic_prompt} + for criteria, get_data in self.get_data.items(): + data_dict = {} + for k, v in get_data.items(): + if v == 'doc_cache': + data_dict[k] = self.pipeline.doc_cache.show_docs() + else: + if v in p_data.keys(): + data_dict[k] = p_data[v] + else: + data_dict[k] = self.pipeline.current_data[v] + eval_func = self.eval_criteria[criteria] + data = [data_dict] + result[criteria] = eval_func(data) + + result = sum([value for key, value in result.items()]) + self.compare.append((result,dynamic_prompt)) + output = max(self.compare,key = lambda x:x[0])[1] + destination = self.send() + self.last_message = output + if len(self.compare) == self.fixed_turn: + self.compare = [] + if destination in self.conditions: + return self.conditions[destination]['post_processing'](output) + else: + return self.post_processing(output) + + return {} + + def end_multi(self): + self.compare = [] + return super().end_multi() + + + +class AttributingModule(AugmentCluster): + model_type = 'attributer' + demo ={ + "selection_instruction":"In this task, you are presented with several documents, which need to be summarized. As an intermediate step, you need to identify salient content within the documents. For each document, copy verbatim the salient spans, and use as a delimiter between each consecutive span. IMPORTANT: The output must be of the format Document []: -delimited consecutive salient spans. IMPORTANT: Each salient content must be a single consecutive verbatim span from the corresponding passages. IMPORTANT: make sure the total number of copied words (from all documents) is around 200 words, and not more than 900.", + "selection_shot":"Document [1]: Cherrapunji Cherrapunji ( with the native name Sohra being more commonly used, and can also be spelled Cherrapunjee or Cherrapunji) is a subdivisional town in the East Khasi Hills district in the Indian state of Meghalaya. It is the traditional capital of aNongkhlaw \"hima\" (Khasi tribal chieftainship constituting a petty state), both known as Sohra or Churra. Cherrapunji has often been credited as being the wettest place on Earth, but for now nearby Mawsynram currently holds that distinction. Cherrapunji still holds the all-time record for the most rainfall in a calendar month for July 1861 and most rain in a year from August 1860 to July 1861, however: it received in\" \nDocument [2]: Radio relay station known as Akashvani Cherrapunji. It broadcasts on FM frequencies. Cherrapunji Cherrapunji (; with the native name Sohra being more commonly used, and can also be spelled Cherrapunjee or Cherrapunji) is a subdivisional town in the East Khasi Hills district in the Indian state of Meghalaya. It is the traditional capital of aNongkhlaw \"hima\" (Khasi tribal chieftainship constituting a petty state), both known as Sohra or Churra. Cherrapunji has often been credited as being the wettest place on Earth, but for now nearby Mawsynram currently holds that distinction. Cherrapunji still holds the all-time record for the most rainfall\" \nDocument [3]: \"Mawsynram Mawsynram () is a village in the East Khasi Hills district of Meghalaya state in north-eastern India, 65 kilometres from Shillong. Mawsynram receives one of the highest rainfalls in India. It is reportedly the wettest place on Earth, with an average annual rainfall of 11,872 mm, but that claim is disputed by Lloró, Colombia, which reported an average yearly rainfall of 12,717 mm between 1952 and 1989 and López de Micay, also in Colombia, which reported an annual 12,892 mm per year between 1960 and 2012. According to the \"Guinness Book of World Records\", Mawsynram received of rainfall in 1985. Mawsynram is located at 25° 18′\" \n\nAnswer: \nDocument [1]: Cherrapunji has often been credited as being the wettest place on Earth still holds the all-time record for the most rainfall in a calendar month for July 1861 and most rain in a year from August 1860 to July 1861 \nDocument [2]: Cherrapunji has often been credited as being the wettest place on Earthstill holds the all-time record for the most rainfall \nDocument [3]: Mawsynram receives one of the highest rainfalls in India but that claim is disputed by Lloró, Colombia, which reported an average yearly rainfall of 12,717 mm between 1952 and 1989 López de Micay, also in Colombia, which reported an annual 12,892 mm per year between 1960 and 2012. ", + "clustering_instruction":"In this task, you are presented with several passages, where some parts are \"highlighted\" (namely, there are and tokens before and after each such span). The goal is to fuse all those highlights into a single summary. As an intermediate step, you need to cluster highlights that can be merged into a sentence (namely, each cluster will be later merged into one sentence). Make sure the clusters are in the same order you would then write the corresponding summary sentences. IMORTANT: make sure there are at most 3 clusters, and no more than 3 highlights per cluster. IMPORTANT: The output must be of the format [\"cluster\":[comma-delimited highlights indices]]", + "clustering_shot":"Document [1]: Cherrapunji Cherrapunji ( with the native name Sohra being more commonly used, and can also be spelled Cherrapunjee or Cherrapunji) is a subdivisional town in the East Khasi Hills district in the Indian state of Meghalaya. It is the traditional capital of aNongkhlaw \"hima\" (Khasi tribal chieftainship constituting a petty state), both known as Sohra or Churra. Cherrapunji has often been credited as being the wettest place on Earth, but for now nearby Mawsynram currently holds that distinction. Cherrapunji still holds the all-time record for the most rainfall in a calendar month for July 1861 and most rain in a year from August 1860 to July 1861, however: it received in\" \nDocument [2]: Radio relay station known as Akashvani Cherrapunji. It broadcasts on FM frequencies. Cherrapunji Cherrapunji (; with the native name Sohra being more commonly used, and can also be spelled Cherrapunjee or Cherrapunji) is a subdivisional town in the East Khasi Hills district in the Indian state of Meghalaya. It is the traditional capital of aNongkhlaw \"hima\" (Khasi tribal chieftainship constituting a petty state), both known as Sohra or Churra. Cherrapunji has often been credited as being the wettest place on Earth, but for now nearby Mawsynram currently holds that distinction. Cherrapunji still holds the all-time record for the most rainfall\" \nDocument [3]: \"Mawsynram Mawsynram () is a village in the East Khasi Hills district of Meghalaya state in north-eastern India, 65 kilometres from Shillong. Mawsynram receives one of the highest rainfalls in India. It is reportedly the wettest place on Earth, with an average annual rainfall of 11,872 mm, but that claim is disputed by Lloró, Colombia, which reported an average yearly rainfall of 12,717 mm between 1952 and 1989 and López de Micay, also in Colombia, which reported an annual 12,892 mm per year between 1960 and 2012. According to the \"Guinness Book of World Records\", Mawsynram received of rainfall in 1985. Mawsynram is located at 25° 18′\" \n\nThe highlighted spans are: \nDocument [1]: 1. Cherrapunji has often been credited as being the wettest place on Earth \n2. still holds the all-time record for the most rainfall in a calendar month for July 1861 and most rain in a year from August 1860 to July 1861 \nDocument [2]: \n3. Cherrapunji has often been credited as being the wettest place on Earth \n4. still holds the all-time record for the most rainfall \nDocument [3]: \n5. Mawsynram receives one of the highest rainfalls in India \n6. but that claim is disputed by Lloró, Colombia, which reported an average yearly rainfall of 12,717 mm between 1952 and 1989 \n7. López de Micay, also in Colombia, which reported an annual 12,892 mm per year between 1960 and 2012. \n\nAnswer: \nThe highlighted spans are clustered as follows: \n[{\"cluster\":[6,7]}, {\"cluster\":[5]},{\"cluster\":[1,2]}]", + "gen_instruction":"In this task, you are presented with several passages, where some parts are \"highlighted\" (namely, there are and tokens before and after each such span). You may also be presented with a prefix of the answer. You job is to generate the next sentence of the answer, that covers all and only the \"highlighted\" spans. Make sure it connects well with the prefix(if eixists), and that it covers all and only the \"highlighted\" spans. Always cite for any factual claim. When citing several search results, use [1][2][3]. Cite at least one document and at most three documents in each sentence. If multiple documents support the sentence, only cite a minimum sufficient subset of the documents.", + "gen_shot":"Document [1]: Cherrapunji Cherrapunji ( with the native name Sohra being more commonly used, and can also be spelled Cherrapunjee or Cherrapunji) is a subdivisional town in the East Khasi Hills district in the Indian state of Meghalaya. It is the traditional capital of aNongkhlaw \"hima\" (Khasi tribal chieftainship constituting a petty state), both known as Sohra or Churra. Cherrapunji has often been credited as being the wettest place on Earth, but for now nearby Mawsynram currently holds that distinction. Cherrapunji still holds the all-time record for the most rainfall in a calendar month for July 1861 and most rain in a year from August 1860 to July 1861, however: it received in\" \nDocument [2]: Radio relay station known as Akashvani Cherrapunji. It broadcasts on FM frequencies. Cherrapunji Cherrapunji (; with the native name Sohra being more commonly used, and can also be spelled Cherrapunjee or Cherrapunji) is a subdivisional town in the East Khasi Hills district in the Indian state of Meghalaya. It is the traditional capital of aNongkhlaw \"hima\" (Khasi tribal chieftainship constituting a petty state), both known as Sohra or Churra. Cherrapunji has often been credited as being the wettest place on Earth, but for now nearby Mawsynram currently holds that distinction. Cherrapunji still holds the all-time record for the most rainfall\" \nDocument [3]: \"Mawsynram Mawsynram () is a village in the East Khasi Hills district of Meghalaya state in north-eastern India, 65 kilometres from Shillong. Mawsynram receives one of the highest rainfalls in India. It is reportedly the wettest place on Earth, with an average annual rainfall of 11,872 mm, but that claim is disputed by Lloró, Colombia, which reported an average yearly rainfall of 12,717 mm between 1952 and 1989 and López de Micay, also in Colombia, which reported an annual 12,892 mm per year between 1960 and 2012. According to the \"Guinness Book of World Records\", Mawsynram received of rainfall in 1985. Mawsynram is located at 25° 18′\" \n\nPrefix: Several places on Earth claim to be the most rainy, such as Lloró, Colombia, which reported an average annual rainfall of 12,717 mm between 1952 and 1989, and López de Micay, Colombia, which reported an annual 12,892 mm between 1960 and 2012 [3]. \n\nThe highlighted spans are: \nDocument [3]: \n5. Mawsynram receives one of the highest rainfalls in India \n\nAnswer: \nThe next sentence is: \nHowever, the official record is held by Mawsynram, India with an average annual rainfall of 11,872 mm [3]." + } + PARA_SEP = '\n\n' + selection_shot = demo['selection_instruction'] + PARA_SEP + demo['selection_shot'] + PARA_SEP + cls_shot = demo['clustering_instruction'] + PARA_SEP + demo['clustering_shot'] + PARA_SEP + gen_shot = demo['gen_instruction'] + PARA_SEP + demo['gen_shot'] + PARA_SEP + prompt = Prompt(template='', + components={'INST':'{INST}\n\n', + 'shot':'{shot}', + 'question':'Question:{question}\n\n', + 'docs':'{docs}\n', + 'span':'The highlighted spans are: \n{span}\n\n', + 'prefix':'Prefix: {prefix}\n\n', + 'add':'Answer: \n{add}' + }) + def __init__(self, model) -> None: + module_list = [] + select = LLM(model = model, prompt_maker = self.prompt, self_prompt={'INST':self.demo['selection_instruction'],'shot':self.selection_shot,'add':''}, post_processing=make_as('span'),noisy= False) + post_select = Attribute_post_select() + clustering = LLM(model = model, prompt_maker = self.prompt, self_prompt={'INST':self.demo['clustering_instruction'],'shot':self.cls_shot, 'add':'The highlighted spans are clustered as follows:'},share_model_with=select, post_processing=make_as('cls'),noisy=False) + post_cls = Attribute_post_cluster() + module_list = [select,post_select,clustering,post_cls] + super().__init__(module_list) + +MODEL_TYPE_MAPPING = { + 'retrieve': Retriever, + 'eval': EvalModule, + 'simplify': CitationSimplyfier, + 'verify': Verifier, + 'rank': Ranker, + 'attributing': AttributingModule +} \ No newline at end of file diff --git a/citekit/evaluator/__init__.py b/citekit/evaluator/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/citekit/evaluator/__pycache__/__init__.cpython-310.pyc b/citekit/evaluator/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ca6ad93ae6f59f34b2b0d43c490c4d221c7bf92f Binary files /dev/null and b/citekit/evaluator/__pycache__/__init__.cpython-310.pyc differ diff --git a/citekit/evaluator/__pycache__/__init__.cpython-312.pyc b/citekit/evaluator/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ec909f201dee202c5e8cdf70585458e530dd6d01 Binary files /dev/null and b/citekit/evaluator/__pycache__/__init__.cpython-312.pyc differ diff --git a/citekit/evaluator/__pycache__/evaluator.cpython-310.pyc b/citekit/evaluator/__pycache__/evaluator.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7c1791bb37250b7e7cf406ea12f80b1a530cba9d Binary files /dev/null and b/citekit/evaluator/__pycache__/evaluator.cpython-310.pyc differ diff --git a/citekit/evaluator/__pycache__/evaluator.cpython-312.pyc b/citekit/evaluator/__pycache__/evaluator.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f987a73df9d8e32181fa52149f309de04f859372 Binary files /dev/null and b/citekit/evaluator/__pycache__/evaluator.cpython-312.pyc differ diff --git a/citekit/evaluator/evaluator.py b/citekit/evaluator/evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..49fe523aa7e23f9c42b2cb9cb4f502ac563e762f --- /dev/null +++ b/citekit/evaluator/evaluator.py @@ -0,0 +1,1118 @@ +from nltk import sent_tokenize +import nltk +nltk.download('punkt') +import re +import random +import transformers +import numpy as np +from citekit.utils.utils import * +from rouge import Rouge +import torch +from transformers import AutoModelForSeq2SeqLM, AutoTokenizer +import copy +import torch +from tqdm import tqdm +import sys +import logging +import random +from itertools import product,combinations +import time +import logging +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +PIPELINE_OUTPUT = 'output' +PIPELINE_DOC_CACHE = 'doc_cache' + +global autoais_model, autoais_tokenizer +autoais_model = None +autoais_tokenizer = None +get_docs_by_index = lambda i,docs: docs[i] if i < len(docs) else None +ais_LLM = None + +QA_MODEL = "gaotianyu1350/roberta-large-squad" +AUTOAIS_MODEL = "google/t5_xxl_true_nli_mixture" +AUTOAIS_MODEL_ABSOLUTE = "/mnt/usercache/huggingface/t5_xxl_true_nli_mixture" + +def get_cite(sent): + return re.sub(r"\[\d+", "", re.sub(r" \[\d+", "", sent)).replace(" |", "").replace("]", ""),[int(r[1:]) - 1 for r in re.findall(r"\[\d+", sent)] + + +def entail(premise, claim): + + """ + Run inference for assessing AIS between a premise and hypothesis. + Adapted from https://github.com/google-research-datasets/Attributed-QA/blob/main/evaluation.py + """ + global autoais_model, autoais_tokenizer + input_text = "premise: {} hypothesis: {}".format(premise, claim) + input_ids = autoais_tokenizer(input_text, return_tensors="pt").input_ids.to(autoais_model.device) + with torch.inference_mode(): + outputs = autoais_model.generate(input_ids, max_new_tokens=10) + result = autoais_tokenizer.decode(outputs[0], skip_special_tokens=True) + inference = 1 if result == "1" else 0 + return inference + +def load_auto_ais(): + global autoais_model, autoais_tokenizer + print('Initializing eval model for citation precision and recall...') + try: + autoais_model = AutoModelForSeq2SeqLM.from_pretrained(AUTOAIS_MODEL, torch_dtype=torch.bfloat16, device_map="auto") + autoais_tokenizer = AutoTokenizer.from_pretrained(AUTOAIS_MODEL, use_fast=False) + + except: + print('Unable to load model from hub, trying to load from local path...') + autoais_model = AutoModelForSeq2SeqLM.from_pretrained(AUTOAIS_MODEL_ABSOLUTE, torch_dtype=torch.bfloat16, device_map="auto") + autoais_tokenizer = AutoTokenizer.from_pretrained(AUTOAIS_MODEL_ABSOLUTE, use_fast=False) + print('Done!') + +def compute_mauve(data): + """Compute Mauve score.""" + + logger.info("Computing MAUVE...") + human_data = [] + model_data = [] + for item in data: + # Remove ending punctuations + # Remove any new lines + # Truncate by 100 words + human_data.append( + ' '.join((item['question'] + " " + item['answer'].strip()).split()[:100]).rstrip(string.punctuation)) + model_data.append( + ' '.join((item['question'] + " " + item['output'].strip()).split()[:100]).rstrip(string.punctuation)) + + import mauve + out = mauve.compute_mauve( + p_text=human_data, + q_text=model_data, + device_id=0, + max_text_length=512, + verbose=True, + batch_size=8, + featurize_model_name="gpt2-large" + ) + return out.mauve * 100 + + +def compute_rouge_l(data): + total = len(data) + res = { + "r": 0.0, + "p": 0.0, + "f": 0.0 + } + for item in data: + if item['output'] and item['answer']: + rouge = Rouge() + scores = rouge.get_scores(item['output'], item['answer']) + res['r'] += scores[0]['rouge-l']['r'] + res['p'] += scores[0]['rouge-l']['p'] + res['f'] += scores[0]['rouge-l']['f'] + else: + print('Warning: no hypothesis or references') + res['r'] /= total + res['p'] /= total + res['f'] /= total + + return res + +def compute_qa(question, output, short_answers, qa_pipeline=None): + """Compute QA-based accuracy. + Args: + + Returns: + QA metrics (QA-EM, QA-F1, QA-Hit) + """ + + # Load model + if not qa_pipeline: + qa_pipeline = transformers.pipeline("question-answering", model=QA_MODEL, device='mps') + + # Get prediction + em, f1, bins = 0,0,0 + context = output if len(output) > 0 else " " + result = qa_pipeline(question=question, context=context, handle_impossible_answer=True) + loc_counter, loc_em, loc_f1 = 0, 0, 0 + print(result) + prediction = result["answer"] + + loc_em = max([compute_exact(a, prediction) for a in short_answers]) + loc_f1 = max([compute_f1(a, prediction) for a in short_answers]) + loc_counter += 1 + + em= loc_em / loc_counter + f1= loc_f1 / loc_counter + bins = int(loc_em == loc_counter) + return em, f1, bins + +def compute_qa(data): + """Compute QA-based accuracy. + Args: + data: requires filed `qa_pairs/short_answers` and `output` + Returns: + QA metrics (QA-EM, QA-F1, QA-Hit) + """ + + if 'qa_pairs' not in data[0] or data[0]['qa_pairs'] is None: + #logger.warn("Warning: no QA pairs found in data") + return { + 'QA-EM': 0, + 'QA-F1': 0, + 'QA-Hit': 0, + } + + # Load model + #logger.info("Loading the RoBERTa-large SQuAD model for QA-based accuracy...") + global qa_pipeline + if not qa_pipeline: + qa_pipeline = transformers.pipeline("question-answering", model=QA_MODEL) + #logger.info("Done") + + # Get prediction + #logger.info("Computing the QA-based accuracy...") + em, f1, bins = [], [], [] + for item in tqdm(data): + question = [qa_pair['question'] for qa_pair in item['qa_pairs']] + context = item['output'] if len(item['output']) > 0 else " " + results = qa_pipeline(question=question, context=context, handle_impossible_answer=True) + loc_counter, loc_em, loc_f1 = 0, 0, 0 + + for idx, res in enumerate(results): + answers = item["qa_pairs"][idx]["short_answers"] + prediction = res["answer"] + + loc_em += max([compute_exact(a, prediction) for a in answers]) + loc_f1 += max([compute_f1(a, prediction) for a in answers]) + loc_counter += 1 + + em.append(loc_em / loc_counter) + f1.append(loc_f1 / loc_counter) + bins.append(loc_em == loc_counter) + + return { + 'QA-EM': 100 * np.mean(em), + 'QA-F1': 100 * np.mean(f1), + 'QA-Hit': 100 * np.mean(bins) + } + + +def cite_pr(sent_with_cite, docs = None, get_docs = get_docs_by_index, get_cite = get_cite, max_cite= None,rich_return = False): + """ + : sent_with_cite: ONE sentence with citation like [1][2][3] + : get_docs: by default like [1][2], get ids + : docs: List, all the COMPLETE documents with TITLE + + : return + number of citations, integer + recall (0 or 1) + precision (number of relevent documents) + + optional; + multi_cite + mcite_support + mcite_overcite + """ + if rich_return: + raise NotImplementedError + + result = {'num_cites': 0,'recall':0,'precision':0,'multi_cite':0,'mcite_support' :0,'mcite_overcite':0} + sent, cites= get_cite(sent_with_cite) + + if not cites: + return (0, 0, 0) if not rich_return else result # no citations + if max_cite: + cites = cites[:max_cite] + num_cites = len(cites) + result['num_cites'] = num_cites + + refs = [get_docs(cite, docs) for cite in cites] + if None in refs: + return (num_cites, 0, 0) if not rich_return else result# wrong citation(s) + + # recall + recall = entail(premise=''.join(refs),claim=sent) + result['recall'] = recall + + # precision + precision = 0 + if num_cites == 1: + precision = recall + else: + for idx, ref in enumerate(refs): + if entail(premise=ref,claim=sent): + precision += 1 + else: + if not entail(premise=''.join([refs[i] for i in range(len(refs)) if i != idx]), claim = sent): + precision += 1 + elif recall: + result['mcite_overcite'] = 1 + result['precision'] = precision + + #other + if num_cites > 1: + result['multi_cite'] = 1 + if recall: + result['mcite_support'] = 1 + + + return (num_cites, recall, precision) if not rich_return else result + + +def cite_pr_answer(answer, docs = None, get_docs = get_docs_by_index, get_cite = get_cite, max_cite= None,rich_return = False): + epsilon = 1e-8 + num_c = 0 + recall = 0 + precision = 0 + sents = sent_tokenize(answer) + for sent in sents: + c,r,p = cite_pr(sent,get_docs=get_docs,docs=docs,get_cite=get_cite,max_cite=max_cite,rich_return=rich_return) + num_c += c + recall += r + precision += p + # diveded by Zero! + return recall/(len(sents)+ epsilon), precision/(num_c+epsilon) + + +def _run_nli_autoais(passage, claim, test = False): + """ + Run inference for assessing AIS between a premise and hypothesis. + Adapted from https://github.com/google-research-datasets/Attributed-QA/blob/main/evaluation.py + """ + if not test: + global autoais_model, autoais_tokenizer + if not autoais_model: + load_auto_ais() + input_text = "premise: {} hypothesis: {}".format(passage, claim) + input_ids = autoais_tokenizer(input_text, return_tensors="pt").input_ids.to(autoais_model.device) + with torch.inference_mode(): + outputs = autoais_model.generate(input_ids, max_new_tokens=10) + result = autoais_tokenizer.decode(outputs[0], skip_special_tokens=True) + inference = 1 if result == "1" else 0 + return inference + else: + res = random.randint(0,1) + + return res + +def _run_llm_autoais(passage, claim): + global ais_LLM + assert(ais_LLM) + return int(ais_LLM.generate(premise = passage, claim = claim)) + +def test_compute_autoais(data): + print(data[0]['docs'][:5]) + print(data[0]['output'][:5]) + return { + "citation_rec": random.randint(0,100), + "citation_prec": random.randint(0,100), + } + +def compute_autoais(data, + decontext=False, + concat=False, + qampari=False, + at_most_sents = 3, + at_most_citations=3, + entail_function = _run_nli_autoais): + """ + Compute AutoAIS score. + + Args: + data: requires field `output` and `docs` + - docs should be a list of items with fields `title` and `text` (or `phrase` and `sent` for QA-extracted docs) + citation: check citations and use the corresponding references. + decontext: decontextualize the output + """ + + global autoais_model, autoais_tokenizer + + + ais_scores = [] + ais_scores_prec = [] + + sent_total = 0 + sent_mcite = 0 + sent_mcite_support = 0 + sent_mcite_overcite = 0 + autoais_log = [] + for item in tqdm(data): + # Get sentences by using NLTK + if qampari: + print('now qampari...') + sents = [item['question'] + " " + x.strip() for x in + item['output'].rstrip().rstrip(".").rstrip(",").split(",")] + else: + sents = sent_tokenize(item['output'])[:at_most_sents] + if len(sents) == 0: + ais_scores.append(0.0) + ais_scores_prec.append(0.0) # len(sents)) + continue + + target_sents = [remove_citations(sent).strip() for sent in sents] + + entail = 0 + entail_prec = 0 + total_citations = 0 + for sent_id, sent in enumerate(sents): + target_sent = target_sents[sent_id] # Citation removed and (if opted for) decontextualized + joint_entail = -1 # Undecided + + # Find references + #ref = [int(r[1:]) - 1 for r in re.findall(r"\[\d+", sent)] # In text citation id starts from 1 + matches = re.findall(r"\[(\d+(?:,\s*\d+)*)\]", sent) + ref = [int(num)-1 for match in matches for num in match.replace(' ', '').split(',')] + if len(ref) == 0: + # No citations + joint_entail = 0 + elif any([ref_id >= len(item['docs']) for ref_id in ref]): + # Citations out of range + joint_entail = 0 + else: + if at_most_citations is not None: + ref = ref[:at_most_citations] + total_citations += len(ref) + joint_passage = '\n'.join([(item['docs'][psgs_id]) for psgs_id in ref]) + + # If not directly rejected by citation format error, calculate the recall score + if joint_entail == -1: + joint_entail = entail_function(joint_passage, target_sent) + autoais_log.append({ + #"question": item['question'], + "output": item['output'], + "claim": sent, + "passage": [joint_passage], + "model_type": "NLI", + "model_output": joint_entail, + }) + + entail += joint_entail + if len(ref) > 1: + sent_mcite += 1 + + # calculate the precision score if applicable + if joint_entail and len(ref) > 1: + sent_mcite_support += 1 + # Precision check: did the model cite any unnecessary documents? + for psgs_id in ref: + # condition A + passage = item['docs'][psgs_id] + nli_result = entail_function(passage, target_sent) + + # condition B + if not nli_result: + subset_exclude = copy.deepcopy(ref) + subset_exclude.remove(psgs_id) + passage = '\n'.join([item['docs'][pid] for pid in subset_exclude]) + nli_result =entail_function(passage, target_sent) + if nli_result: # psgs_id is not necessary + flag = 0 + sent_mcite_overcite += 1 + else: + entail_prec += 1 + else: + entail_prec += 1 + else: + entail_prec += joint_entail + sent_total += len(sents) + ais_scores.append(entail / len(sents)) + ais_scores_prec.append(entail_prec / total_citations if total_citations > 0 else 0) # len(sents)) + + if sent_mcite > 0 and sent_mcite_support > 0: + print( + "Among all sentences, %.2f%% have multiple citations, among which %.2f%% are supported by the joint set, among which %.2f%% overcite." % ( + 100 * sent_mcite / sent_total, + 100 * sent_mcite_support / sent_mcite, + 100 * sent_mcite_overcite / sent_mcite_support + )) + + return { + "citation_rec": 100 * np.mean(ais_scores), + "citation_prec": 100 * np.mean(ais_scores_prec), + } + +def compute_claims_test(data): + print(data[0]['claims']) + print(data[0][PIPELINE_OUTPUT]) + return random.randint(1,100) + +def compute_claims(data): + global autoais_model, autoais_tokenizer + if autoais_model is None: + #logger.info("Loading AutoAIS model...") + # autoais_model = AutoModelForSeq2SeqLM.from_pretrained(AUTOAIS_MODEL, torch_dtype=torch.bfloat16, max_memory=get_max_memory(), device_map="auto") + autoais_model = AutoModelForSeq2SeqLM.from_pretrained(AUTOAIS_MODEL, torch_dtype=torch.bfloat16, + device_map="auto") + # autoais_model = AutoModelForSeq2SeqLM.from_pretrained(AUTOAIS_MODEL, torch_dtype=torch.bfloat16, max_memory=get_max_memory(), device_map="auto",offload_folder= "/data/hongbang/zsf/projects/ALCE/ALCE/model/t5_xxl_true_nli_mixture/offload1") + autoais_tokenizer = AutoTokenizer.from_pretrained(AUTOAIS_MODEL, use_fast=False) + #logger.info("Computing claims...") + scores = [] + for item in tqdm(data): + normalized_output = remove_citations(item['output']) + entail = 0 + claims = item["claims"] + for claim in claims: + entail += _run_nli_autoais(normalized_output, claim) + scores.append(entail / len(claims)) + return 100 * np.mean(scores) + + +#citation appropriateness +def check_if_citations_needed(passages, answer, grain): + + def _format_document(doc): + """Format document for AutoAIS. + + if "sent" in doc: + # QA-extracted docs + return "Title: %s\n%s" % (doc['title'], doc['sent']) + else: + return "Title: %s\n%s" % (doc['title'], doc['text']) + """ + return doc + + global autoais_model, autoais_tokenizer + if autoais_model is None and False: + #logger.info("Loading AutoAIS model...") + # autoais_model = AutoModelForSeq2SeqLM.from_pretrained(AUTOAIS_MODEL, torch_dtype=torch.bfloat16, max_memory=get_max_memory(), device_map="auto") + autoais_model = AutoModelForSeq2SeqLM.from_pretrained(AUTOAIS_MODEL, torch_dtype=torch.bfloat16, + device_map="auto") + # autoais_model = AutoModelForSeq2SeqLM.from_pretrained(AUTOAIS_MODEL, torch_dtype=torch.bfloat16, max_memory=get_max_memory(), device_map="auto",offload_folder= "/data/hongbang/zsf/projects/ALCE/ALCE/model/t5_xxl_true_nli_mixture/offload1") + autoais_tokenizer = AutoTokenizer.from_pretrained(AUTOAIS_MODEL, use_fast=False) + + if grain=="over_fine" or grain=="more_over_fine": + num_passages = len(passages) + passages_per_chunk = num_passages // 5 # Divide passages evenly into 5 chunks + remainder = num_passages % 5 # Handle remaining passages + passages_five=[] + start_idx = 0 + for i in range(5): + end_idx = start_idx + passages_per_chunk + if remainder > 0: + end_idx += 1 + remainder -= 1 + chunk_passages = passages[start_idx:end_idx] + passages_five.append('\n'.join([_format_document(p) for p in chunk_passages])) + start_idx = end_idx + passages=passages_five + combinations_3 = combinations(passages, 3) # 获取所有三个passage的组合 + for combination in combinations_3: + joint_passage = '\n'.join( + [passage for passage in combination]) # 将三个passage连接为一个字符串，并保留格式 + entail = _run_nli_autoais(joint_passage, answer) + if entail == 1: + return 1 + return 0 + + else: + if len(passages)>=3:#正常粒度 + combinations_3 = combinations(passages, 3) + for combination in combinations_3: + joint_passage = '\n'.join( + [_format_document(passage) for passage in combination]) + entail = _run_nli_autoais(joint_passage, answer) + if entail == 1: + return 1 + return 0 + else:#粗粒度 + joint_passage = '\n'.join( + [_format_document(passage) for passage in passages]) + entail = _run_nli_autoais(joint_passage, answer) + if entail == 1: + return 1 + else: + return 0 + + +#citaion granularity +def find_permutations(n, m): + ''' + :param n: 最大数量总和 + :param m: 位长度 + :return: + ''' + # Generate all possible sequences of length m + all_sequences = list(product(range(n + 1), repeat=m)) + #print('all_sequences', all_sequences) + + # Filter sequences where the sum of digits equals n + valid_sequences = [seq for seq in all_sequences if sum(seq) == n] + return valid_sequences + + +def get_subspans(list_span, span_count): + list_subspan = [] + for i in range(0, len(list_span) - span_count + 1): + list_subspan.append(list_span[i: i + span_count]) + + return list_subspan + + +def get_all_span_comb(list_list_span, target_span_count=-1): + if target_span_count == -1: # 所有子集 + max_span_count = len(sum(list_list_span, [])) + doc_count = len(list_list_span) + list_span_comb_all = [] + for span_count in range(1, max_span_count + 1): + list_comb = find_permutations(span_count, doc_count)#给定数量的子串在文本中的所有可能组合 + + list_span_comb = [] # 最终当前长度的所有可能组合 + for comb in list_comb: + list_list_subspan = [] + + for idx_doc, span_count_doc in enumerate(comb): + list_subspan = get_subspans(list_list_span[idx_doc], span_count_doc) + if len(list_subspan) == 0: + list_list_subspan = None + break + list_list_subspan.append(list_subspan) + + if list_list_subspan: + list_span_comb_cur = [sum(list(combination), []) for combination in product(*list_list_subspan)] + list_span_comb_cur = list(set([tuple(span_comb) for span_comb in list_span_comb_cur])) + + list_span_comb += list_span_comb_cur + list_span_comb_all += list_span_comb + list_span_comb_all = set(list_span_comb_all) + else: # 当前长度的组合 + doc_count = len(list_list_span) + list_comb = find_permutations(target_span_count, doc_count) + + list_span_comb = [] # 最终当前长度的所有可能组合 + for comb in list_comb: + list_list_subspan = [] + + for idx_doc, span_count_doc in enumerate(comb): + list_subspan = get_subspans(list_list_span[idx_doc], span_count_doc) + if len(list_subspan) == 0: + list_list_subspan = None + break + list_list_subspan.append(list_subspan) + + if list_list_subspan: + list_span_comb_cur = [combination for combination in product(*list_list_subspan)] + for idx in range(len(list_span_comb_cur)): + list_span_comb_cur[idx] = tuple([tuple(span_comb) for span_comb in list_span_comb_cur[idx]]) + + list_span_comb += list_span_comb_cur + list_span_comb_all = list_span_comb + list_span_comb_all = set(list_span_comb_all) + return list_span_comb_all + + +def run_converge_2(list_list_span=None, sentence=None): + ''' + 基于假设：更长的text不能蕴含，则其任何子串都不能蕴含 + span数量递减（提供更多的剪枝选项） + 最终gold可能有一个span的误差 + ''' + ###### + #print('origin nli count', len(get_all_span_comb(list_list_span, target_span_count=-1)))#给定文本的所有可能的子串组合 + max_span_count = len(sum(list_list_span, [])) # span总数 + + set_comb_hash = set([]) + + ### span数量二分 + nli_count = 0 + skip_count = 0 + list_list_span_gold = copy.copy(list_list_span) # 当前能够精准蕴含的span + + span_count_min, span_count_max = 1, max_span_count + start_time=time.time() + timeout=300 + while span_count_min < span_count_max:#每次迭代中不断寻找更小的子串组合 + span_count_cur = span_count_max - 1 + flag_find = False + if time.time() - start_time > timeout: + print('timeout!') + list_list_span_gold=[] + break + ### 存在可蕴含，继续找更少的span + ### 不存在可蕴含，继续找更多的span + # 长度为span_count_max - 1的所有可能的子串组合 + set_comb_cur = get_all_span_comb(list_list_span, target_span_count=span_count_cur) + + list_comb_cur = list(set_comb_cur) + random.shuffle(list_comb_cur) + for comb in list_comb_cur: + list_list_span_cur = [list(t) for t in comb] + list_span_cur = sum(list_list_span_cur, []) + str_text = ' '.join(list_span_cur) # TODO: 统一字符串化的方式 + + if hash(str_text) in set_comb_hash: + skip_count += 1 + continue + + #### ⚠️ 注意在这里替换nli函数 + nli_label = _run_nli_autoais(str_text, sentence) # TODO: nli label function + nli_count += 1 + + if nli_label == 1: # 只要存在可蕴含，直接继续找更少的span + list_list_span_gold = copy.copy(list_list_span_cur) + span_count_max = span_count_cur#更新span数量上限 + flag_find = True + # print(f"find nli!, nli_count: {nli_count}, skip_count: {skip_count}, len(set_comb_hash): {len(set_comb_hash)}", ) + break + else: # 不能蕴含，剪枝所有子集 + set_comb_cur_del = get_all_span_comb(list_list_span_cur, target_span_count=-1) + set_comb_hash_cur = set([hash(' '.join(list(tuple_comb_))) for tuple_comb_ in set_comb_cur_del]) # TODO: 统一字符串化的方式 + + set_comb_hash |= set_comb_hash_cur + if flag_find == False: + print(f"CAN'T find nli!, nli_count: {nli_count}, skip_count: {skip_count}, len(set_comb_hash): {len(set_comb_hash)}", ) + break + span_count_gold = span_count_max # gold的span数量 + print('len(set_comb_del)', len(set_comb_hash)) + print('nli_count', nli_count, 'skip_count', skip_count, 'span_count_gold', span_count_gold) + return list_list_span_gold + + +def compute_autoais_grained(data, + at_most_citations=3,method='ALCE',grain='default'): + + """ + Compute AutoAIS score. + + Args: + data: requires field `output` and `docs` + - docs should be a list of items with fields `title` and `text` (or `phrase` and `sent` for QA-extracted docs) + citation: check citations and use the corresponding references. + decontext: decontextualize the output + """ + global autoais_model, autoais_tokenizer + if autoais_model is None and False: + #logger.info("Loading AutoAIS model...") + # autoais_model = AutoModelForSeq2SeqLM.from_pretrained(AUTOAIS_MODEL, torch_dtype=torch.bfloat16, max_memory=get_max_memory(), device_map="auto") + autoais_model = AutoModelForSeq2SeqLM.from_pretrained(AUTOAIS_MODEL, torch_dtype=torch.bfloat16, + device_map="auto") + # autoais_model = AutoModelForSeq2SeqLM.from_pretrained(AUTOAIS_MODEL, torch_dtype=torch.bfloat16, max_memory=get_max_memory(), device_map="auto",offload_folder= "/data/hongbang/zsf/projects/ALCE/ALCE/model/t5_xxl_true_nli_mixture/offload1") + autoais_tokenizer = AutoTokenizer.from_pretrained(AUTOAIS_MODEL, use_fast=False) + def _format_document(doc): + + """Format document for AutoAIS.""" + if isinstance(doc, dict): + if "sent" in doc: + # QA-extracted docs + return "Title: %s\n%s" % (doc['title'], doc['sent']) + else: + return "Title: %s\n%s" % (doc['title'], doc['text']) + elif isinstance(doc,str): + return doc + + + #logger.info(f"Running AutoAIS...") + + ais_scores_need = [] # 是否需要引用 + ais_scores = [] # quote_recall + ais_doc_scores=[]#doc_recall + + sent_total = 0 + + autoais_log = [] + granularity_list = [] + skipped =0 + for item in tqdm(data): + output = item['output'] + + if method=='baseline': + model_answer=item['output_parse']['answer'] + answer = '' + reference = {} + span_contents = {} + if not model_answer["text"].endswith("."): + model_answer["text"] += "." + answer += " " + model_answer["text"] + spans = model_answer['reference'] + for span in spans: + match = re.match(r'^(\d+)\.', span) + if match: + span_number = match.group(1) + span_content = span.split('. ', 1)[1].strip() # 获取1. 后面的内容 + span_contents[span_number] = span_content + reference.update(span_contents) + + item['output_answer'] = answer.strip() + item['output_ref_span'] = reference + output = item['output_answer'] + + elif method=='ALCE': + # 匹配 According to Document + pattern_doc = r"According to Document \[(\d+)\]" + # 匹配 (Title: Godfrey Chitalu) + pattern_title = r"\(Title: [^\)]+\)" + + output = re.sub(pattern_doc, r"[\1]", output) + output = re.sub(pattern_title, "", output) + output=output.strip().split("\n")[0] + output=output.replace("<|im_end|>", "") + # Get sentences by using NLTK + sents = sent_tokenize(output)[:3] + if len(sents) == 0: + continue + + target_sents = [remove_citations(sent).strip() for sent in sents] + output_ref_span = item.get('output_ref_span', {}) + # sent_joint_passage = '\n'.join([_format_document(doc) for doc in item['docs']]) + + entail = 0 + entail_doc=0 + total_citations = 0 + need_citations_sentences = 0 # 一个回答中需要引用的句子数量 + correct_predictions = 0 # 新增：记录正确的预测是否需要引用的子句数量 + + for sent_id, sent in enumerate(sents): + target_sent = target_sents[sent_id] # Citation removed and (if opted for) decontextualized + joint_entail = -1 # Undecided + joint_doc_entail=-1 + + # 1. appropriatness + # 每句话是否需要引用 + need_citations = check_if_citations_needed(item['docs'], target_sent,grain) + + + if method=='baseline': + # Find references number + ref_mark = [int(r[1:]) for r in re.findall(r"\{\d+", sent)] + # 引用的span（拼接）match document + ref, ref_span = match_document(ref_mark, output_ref_span) + #logger.info(f"For `{target_sent}`, find citations {ref}") + ref_id = [x -1 for x in ref] + processed_refs = set() + ref_passage = [] + for psgs_id in ref_id: + if 0 <= psgs_id < len(item['docs']) and psgs_id not in processed_refs: + ref_passage.append(_format_document(item['docs'][psgs_id])) + processed_refs.add(psgs_id) + elif psgs_id in processed_refs: + print("Warning: psgs_id already processed:", psgs_id + 1) + else: + print("Error: psgs_id out of range:", psgs_id+1) + + joint_span = '\n'.join(ref_span) + joint_passage = '\n'.join(ref_passage) + + elif method=='ALCE': + ref = list(set([int(r[1:]) for r in re.findall(r"\[\d+", sent)])) + #logger.info(f"For `{target_sent}`, find citations {ref}") + ref_id=list(set([int(r[1:])-1 for r in re.findall(r"\[\d+", sent)])) + processed_refs = set() + ref_passage = [] + for psgs_id in ref_id: + if 0 <= psgs_id < len(item['docs']) and psgs_id not in processed_refs: + ref_passage.append(_format_document(item['docs'][psgs_id])) + processed_refs.add(psgs_id) + elif psgs_id in processed_refs: + print("Warning: psgs_id already processed:", psgs_id+1) + else: + print("Error: psgs_id out of range:", psgs_id+1) + ref_span=ref_passage + joint_passage = '\n'.join(ref_passage) + joint_span=joint_passage + + + autoais_log.append({ + "question": item['question'], + "output_answer": item['output'], + "docs": item['docs'], + "claim": { + "sentence": sent, + "if_citations_needed": need_citations, + "has_reference": ref, + "doc_recall": None, + "quote_recall": None, + "granularity_score":None, + "granularity_span":None + } + }) + + if len(ref) == 0: + # No citations + joint_entail = 0 + joint_doc_entail=0 + elif any([ref_id > len(item['docs']) for ref_id in ref]): + # Citations out of range + joint_entail = 0 + joint_doc_entail=0 + else: + if at_most_citations is not None: + ref = ref[:at_most_citations] + total_citations += len(ref) + + # 更新正确预测是否需要引用的数量 + if_citations_needed = autoais_log[-1]["claim"]["if_citations_needed"] + has_reference = autoais_log[-1]["claim"]["has_reference"] + if (if_citations_needed == 1 and has_reference) or (if_citations_needed == 0 and not has_reference): + correct_predictions += 1 + #logger.info("citation appropriateness finished") + + # 2. 在需要引用的情况下才计算citation correctness + if need_citations and has_reference:#需要引用且引用了才考虑后两个指标 + start_time = time.time() + need_citations_sentences += 1 + # 2.(1):quote_corr + # If not directly rejected by citation format error, calculate the recall score + if joint_entail == -1: + # φ(premise, hypothesis)判断所有引用span的拼接是否entail模型的回答output + joint_entail = _run_nli_autoais(joint_span, target_sent) + entail += joint_entail + autoais_log[-1]["claim"]["quote_recall"] = joint_entail + #logger.info(f"citation recall finished, recall is {joint_entail}") + + #2.(2):doc_corr + if joint_doc_entail == -1: + if method=='ALCE': + joint_doc_entail=joint_entail + elif method=='baseline': + joint_doc_entail=_run_nli_autoais(joint_passage, target_sent) + entail_doc+=joint_doc_entail + autoais_log[-1]["claim"]["doc_recall"] = joint_doc_entail + #print(f"the total time for two recall is {time.time() - start_time}") + + + + # 4. 只有quote_corr=1（当该条数据，所有引用的拼接可以entail模型output的时候，）才计算引用粒度granularity + start_time=time.time() + if joint_entail: + all_clauses = [] + clauses_first_three = [] + # 遍历每个不同的this_span + #logger.info("calculating granularity") + if len(ref_span)>5: + print("Too many quotations!") + autoais_log[-1]["claim"]["granularity_score"] = None + autoais_log[-1]["claim"]["granularity_span"] = 0 + else: + for idx, this_span in enumerate(ref_span): + #logger.info(f"this span is {this_span}") + # 分割引用跨度为子句 + clauses = re.split(r'([,.])', this_span) + clauses = [clause.strip() for clause in clauses if + clause.strip() and any(char.isalnum() for char in clause.strip())] + all_clauses.append(clauses) + if idx<3: + clauses_first_three.append(clauses) + + max_span_count = len(sum(all_clauses, [])) + if max_span_count==0: + continue + doc_count = len(all_clauses) + min_comb_length=float('inf') + + if method=="ALCE" and grain=="default": + gold_span_res=run_converge_2(clauses_first_three,target_sent) + else: + gold_span_res = run_converge_2(all_clauses, target_sent) + # gold结果 + merged_gold_span_res = [] + + # 遍历嵌套列表，并将其中的子列表合并到大列表中 + for sublist in gold_span_res: + merged_gold_span_res.extend(sublist) + autoais_log[-1]["claim"]["granularity_span"] = merged_gold_span_res + min_comb_length=len(merged_gold_span_res) + if min_comb_length!=float('inf'): + granularity_score = min_comb_length / max_span_count + granularity_list.append(granularity_score) + autoais_log[-1]["claim"]["granularity_score"] = granularity_score + + + print(autoais_log[-1]["claim"]["granularity_span"]) + print(autoais_log[-1]["claim"]["granularity_score"]) + print(f"the total time for granularity is {time.time() - start_time}") + else:#不需要引用或没有引用 + autoais_log[-1]['claim']['recall']=None + autoais_log[-1]["claim"]["granularity_score"]=None + autoais_log[-1]["claim"]["granularity_span"]=None + + + sent_total += len(sents) + ais_scores_need.append(correct_predictions / len(sents)) #是否正确判断需不需要引用：正确判断/总 + if need_citations_sentences!=0: # recall：能entail的/需要引用的 + ais_scores.append(entail / need_citations_sentences) + ais_doc_scores.append(entail_doc / need_citations_sentences) + + #过滤None + granularity_list = [value for value in granularity_list if value is not None] + + #logger.info(f"skipped {skipped}") + #autoais_log.append(f"skipped {skipped}") + ##print(autoais_log) + # print(ais_scores_need,ais_doc_scores,ais_scores,granularity_list) + return { + "citation_correct_prediction": 100 * np.mean(ais_scores_need), + "citation_doc_rec":100 * np.mean(ais_doc_scores), + "citation_quote_rec": 100 * np.mean(ais_scores), + "citation_granularity": 100 * np.mean(granularity_list) + } #autoais_log + +def compute_qampari_f1(data, cot=False): + prec = [] + rec = [] + rec_top5 = [] + f1 = [] + f1_top5 = [] + + num_preds = [] + for item in data: + if cot: + if ":" in item['output']: + o = ':'.join(item['output'].split(":")[1:]) # try to separate the COT part and the answer list part. + else: + o = "" + else: + o = item['output'] + preds = [normalize_answer(x.strip()) for x in remove_citations(o).rstrip().rstrip(".").rstrip(",").split(",")] + preds = [p for p in preds if len(p) > 0] # delete empty answers + #print(preds) + num_preds.append(len(preds)) + answers = [[normalize_answer(x) for x in ans] for ans in item['answers']] + flat_answers = [item for sublist in answers for item in sublist] + #print(flat_answers) + prec.append(sum([p in flat_answers for p in preds]) / len(preds) if len(preds) > 0 else 0) + #print(prec) + rec.append(sum([any([x in preds for x in a]) for a in answers]) / len(answers)) + rec_top5.append(min(5, sum([any([x in preds for x in a]) for a in answers])) / min(5, len(answers))) + if (prec[-1] + rec[-1]) == 0: + f1.append(0) + else: + f1.append(2 * prec[-1] * rec[-1] / (prec[-1] + rec[-1])) + if (prec[-1] + rec_top5[-1]) == 0: + f1_top5.append(0) + else: + f1_top5.append(2 * prec[-1] * rec_top5[-1] / (prec[-1] + rec_top5[-1])) + + return { + "num_preds": np.mean(num_preds), + "qampari_prec": 100 * np.mean(prec), + "qampari_rec": 100 * np.mean(rec), + "qampari_rec_top5": 100 * np.mean(rec_top5), + "qampari_f1": 100 * np.mean(f1), + "qampari_f1_top5": 100 * np.mean(f1_top5), + } + +def compute_length(data): + return sum(len(item['output'].split(' '))for item in data)/(len(data)) + + +if __name__ =='__main__': + #question = "Why did New York City try to ban food donations to the poor?" + #output = "New York City, under Mayor Michael Bloomberg's administration, tried to ban food donations to the poor mainly due to concerns about the nutritional content of the donated food. The city argued that it couldn't inspect donated food for its salt, fat, and fiber content, thereby making it hard to control the nutritional quality of the food served to its homeless population [1][2][3]. Critics of this policy, however, have claimed such an approach demonstrated excessive control over people's eating habits and lacked common sense [2]. Despite the ban, many organizations like the New York City Rescue Mission continued to serve needy citizens through food donations [5]." + #compute_qa(question,output,['','']) + pass + + + +class Evaluator(): + autoais_model_load = False + + eval_criteria = {'test_pr':test_compute_autoais,'cite_recall_precision':compute_autoais, 'pr':compute_autoais,'qa':compute_qa,'rouge': compute_rouge_l,'claims':compute_claims, 'qampari':compute_qampari_f1,'length':compute_length,'str_em':compute_str_em,'grained':compute_autoais_grained,'cite_recall_precision_llm':lambda data: compute_autoais(data=data,entail_function=_run_llm_autoais),'mauve':compute_mauve} + def __init__(self,criteria= None, pipeline = None, ais_model = None) -> None: + self.eval_criteria = Evaluator.eval_criteria + self.pipeline = pipeline + self.get_data = {} + self.ais_model = ais_model + global ais_LLM + ais_LLM = ais_model + + + + def set_eval(self, eval_c, **data_get_key): + if eval_c in self.get_data.keys(): + print(f'Already set! {eval_c}') + return + if eval_c in self.eval_criteria.keys(): + self.get_data[eval_c] = data_get_key + if eval_c == 'cite_recall_precision': + global autoais_model, autoais_tokenizer + if not Evaluator.autoais_model_load: + print('Initializing eval model for citation precision and recall...') + try: + autoais_model = AutoModelForSeq2SeqLM.from_pretrained(AUTOAIS_MODEL, torch_dtype=torch.bfloat16, device_map="auto") + autoais_tokenizer = AutoTokenizer.from_pretrained(AUTOAIS_MODEL, use_fast=False) + + except: + print('Unable to load model from hub, trying to load from local path...') + autoais_model = AutoModelForSeq2SeqLM.from_pretrained(AUTOAIS_MODEL_ABSOLUTE, torch_dtype=torch.bfloat16, device_map="auto") + autoais_tokenizer = AutoTokenizer.from_pretrained(AUTOAIS_MODEL_ABSOLUTE, use_fast=False) + Evaluator.autoais_model_load = True + if eval_c == 'qa': + global qa_pipeline + qa_pipeline = transformers.pipeline("question-answering", model=QA_MODEL) + + else: + raise KeyError('eval_criteria unavailable') + + def new_eval(self, name, eval_func, **data_get_key): + self.eval_criteria[name] = eval_func + self.set_eval(name, **data_get_key) + + def __call__(self,data_from_pipeline= None): + result = {} + + for criteria, get_data in self.get_data.items(): + if not data_from_pipeline: + data_dict = {} + for k, v in get_data.items(): + if isinstance(v,str): + if v == 'output': + data_dict[k] = ' '.join(self.pipeline.output) + elif v == 'doc_cache': + data_dict[k] = self.pipeline.doc_cache + else: + data_dict[k] = self.pipeline.dataset[self.pipeline.data_index][v] + else: + data_dict[k] = v + else: + data_dict = data_from_pipeline + + eval_func = self.eval_criteria[criteria] + data = [data_dict] + result[criteria] = eval_func(data) + return result + + + +class DefaultEvaluator(Evaluator): + def __init__(self, args = None, criteria= None, pipeline = None) -> None: + super().__init__(criteria,pipeline) + if args: + if hasattr(args,'str_em') and args.str_em: + self.set_eval('str_em',output = PIPELINE_OUTPUT, qa_pairs = 'qa_pairs') + if hasattr(args,'pr') and args.pr: + self.set_eval('cite_recall_precision', output = PIPELINE_OUTPUT, docs = PIPELINE_DOC_CACHE, question = 'question') + if hasattr(args,'mauve') and args.mauve: + self.set_eval('mauve', output = PIPELINE_OUTPUT, answer = 'answer' ,question = 'question') + if hasattr(args,'rouge') and args.rouge: + if (hasattr(args, 'dataset') and 'qampari' not in args.dataset.lower()) or not hasattr(args, 'dataset'): + self.set_eval('rouge', output = PIPELINE_OUTPUT, answer = 'answer') + if hasattr(args,'qa') and args.qa: + if (hasattr(args, 'dataset') and 'asqa' in args.dataset.lower()) or not hasattr(args, 'dataset'): + self.set_eval('qa',output = PIPELINE_OUTPUT, qa_pairs = 'qa_pairs') + if hasattr(args,'claims') and args.claims: + if (hasattr(args, 'dataset') and 'eli5' in args.dataset.lower()) or not hasattr(args, 'dataset'): + self.set_eval('claims',output = PIPELINE_OUTPUT, claims = 'claims') + if hasattr(args,'qampari') and args.qampari: + if (hasattr(args, 'dataset') and 'qampari' in args.dataset.lower()) or not hasattr(args, 'dataset'): + self.set_eval('qampari',output = PIPELINE_OUTPUT, answers = 'answers') + if hasattr(args,'length') and args.length: + self.new_eval('length',lambda data: len(data[0]['output'].split(' ')), output = PIPELINE_OUTPUT) + + elif criteria: + if 'cite_recall_precision' in criteria: + self.set_eval('cite_recall_precision', output = PIPELINE_OUTPUT, docs = PIPELINE_DOC_CACHE, question = 'question') + if hasattr(args,'mauve') and args.mauve: + self.set_eval('mauve', output = PIPELINE_OUTPUT, answer = 'answer' ,question = 'question') + if 'rouge' in criteria: + self.set_eval('rouge', output = PIPELINE_OUTPUT, answer = 'answer') + if 'qa' in criteria: + self.set_eval('qa',output = PIPELINE_OUTPUT, qa_pairs = 'qa_pairs') + if 'str_em' in criteria: + self.set_eval('str_em',output = PIPELINE_OUTPUT, qa_pairs = 'qa_pairs') + if 'claims' in criteria: + self.set_eval('claims',output = PIPELINE_OUTPUT, claims = 'claims') + if 'qampari' in criteria: + self.set_eval('qampari',output = PIPELINE_OUTPUT, answers = 'answers') + if 'length' in criteria: + self.new_eval('length',lambda data: len(data[0]['output'].split(' ')), output = PIPELINE_OUTPUT) + + else: + self.new_eval('length',lambda data: len(data[0]['output'].split(' ')), output = PIPELINE_OUTPUT) \ No newline at end of file diff --git a/citekit/pipeline/__pycache__/pipeline.cpython-310.pyc b/citekit/pipeline/__pycache__/pipeline.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..63af846bb28ab250ccd5ffa3a8739f21d883d72e Binary files /dev/null and b/citekit/pipeline/__pycache__/pipeline.cpython-310.pyc differ diff --git a/citekit/pipeline/__pycache__/pipeline.cpython-312.pyc b/citekit/pipeline/__pycache__/pipeline.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5ac6ac8287db1a46af01b5cabf0e9c837e896acb Binary files /dev/null and b/citekit/pipeline/__pycache__/pipeline.cpython-312.pyc differ diff --git a/citekit/pipeline/__pycache__/pipeline_inter.cpython-310.pyc b/citekit/pipeline/__pycache__/pipeline_inter.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7f674c7689f580a2e412b369dc57467eb5be09bf Binary files /dev/null and b/citekit/pipeline/__pycache__/pipeline_inter.cpython-310.pyc differ diff --git a/citekit/pipeline/pipeline.py b/citekit/pipeline/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..f8b9c6b29f1d61e6b81bc92574234317f4cb5afc --- /dev/null +++ b/citekit/pipeline/pipeline.py @@ -0,0 +1,423 @@ +from citekit.cite_modules.LLM import LLM,Module +from citekit.cite_modules.augment_model import AugmentCluster, AttributingModule, MODEL_TYPE_MAPPING +from citekit.prompt.prompt import ALCEVanillaPrompt, DocPrompt +import logging +import json +from tqdm import tqdm +import traceback +import copy +from citekit.utils.utils import flatten_dict +import csv + + + +def merge_str_dicts(dicts): + result = {} + for dictionary in dicts: + for key, value in dictionary.items(): + if key in result: + result[key] += ' ' + value + else: + result[key] = value + return result + +PIPELINE_OUTPUT = 'output' +PIPELINE_DOC_CACHE = 'doc_cache' + +class DocCache(): + def __init__(self) -> None: + self.__docs = list() + + def __len__(self): + return len(self.__docs) + + def __getitem__(self,index): + if index>=0 and index int: + if not isinstance(doc, str): + assert isinstance(doc, dict) and 'text' in doc and 'title' in doc + doc = f'(Title: {doc["title"]}){doc["text"]}' + if add_id: + doc_head = f'Document [{len(self)+1}]' + else: + doc_head = '' + self.__docs.append(doc_head + doc) + return len(self) + + def load_docs(self, docs, add_id = False): + for doc in docs: + self.add_doc(doc, add_id) + return len(self) + + def clear(self): + self.__docs = list() + + def show_docs(self): + return self.__docs + + +class Pipeline(): + def __init__(self,save_path = None, sequence = None, head_prompt_maker = None, llm = None, module= None, retriever = None, evaluator = None, dataset = None, rich_eval = False, train_data = False, attributer = None) -> None: + self.save_path = save_path + self.train_data = train_data + self.head_prompt_maker = head_prompt_maker + self.table_head = True + self.attributer = attributer + self.llm = llm + self.initial_docs = None + self.data_keys = None + self.stored_clusters = [] + self.module = [] + if llm: + llm.connect_to(self) + if not isinstance(module,list) and module is not None: + if module: + module.connect_to(self) + else: + if isinstance(module, list): + for i in module: + if isinstance(i, AugmentCluster) or isinstance(i, Module): + i.connect_to(self) + self.dataset = dataset + + self.data_index = 0 + self.retriever = retriever + if retriever: + retriever.pipeline = self + + self.eval = evaluator + if evaluator: + evaluator.pipeline = self + self.output = [] + self.log = [] + self.doc_cache = DocCache() + self.head = {} + self.result = {} + self.rich_eval = rich_eval + self.initial_module = None + + def load_data(self, dataset): + self.data = dataset + + def set_initial_module(self, module): + self.initial_module = module + def get_initial_module(self): + return self.initial_module + + def set_data_keys(self, keys): + self.data_keys = keys + def get_data_keys(self): + return self.data_keys + + + def update(self, update_object, config, update_info): + print(f'Updating {update_object} with {config} and {update_info}') + module = self.get_module_by_name(update_object) + if config in ['prompt', 'header']: + module.update(config, update_info) + elif config in ['destination']: + module.update(config, [self.get_module_by_name(update_info[0]), update_info[1]]) + elif config in ['delete_destination']: + module.update(config, self.get_module_by_name(update_info)) + elif config in ['new_model']: + model_type, model, key = update_info + print('Creating new model:', model_type, model) + new_model_class = MODEL_TYPE_MAPPING[model_type] + print('New model class:', new_model_class) + new_model = new_model_class(model) + new_model.connect_to(self) + print('Created new model:', new_model) + module.update('destination', [new_model, key]) + + else: + raise NotImplementedError + + + + def set_initial_docs(self, d): + self.initial_docs = d + def get_initial_docs(self): + return self.initial_docs + + def run_on_dataset(self,datakeys,init_docs=None,initial_module= None,start=0): + if self.initial_module and not initial_module: + initial_module = self.initial_module + if self.save_path: + for i in range(start,len((self.dataset))): + self.data_index = i + try: + self.run(datakeys,init_docs,initial_module,train=self.train_data) + except Exception as e: + print(f'Error: {e}, skipping data {i}') + traceback.print_exc() + else: + for i in range(start,len((self.dataset))): + self.data_index = i + try: + self.run(datakeys,init_docs,initial_module,write=False,train=self.train_data) + except Exception as e: + print(f'Error: {e}, skipping data {i}') + traceback.print_exc() + + + + def form_eval_data(self) -> dict: + """To write rich eval, you can use data from: + pipeline.dataset, doc_cache and output + to post_process data as a argument dict for evaluation + """ + raise NotImplementedError('You have to write to apply rich eval with designed arguments.') + + def direct_run(self, dynamic_prompt= {}, module = None): + if not module: + module = self.llm + if isinstance(module, AugmentCluster): + module = module.get_first_module() + while isinstance(module, Module): + if isinstance(dynamic_prompt,dict): + module.change_to_multi_process(False) + dynamic_prompt = module.generate(self.head,dynamic_prompt=dynamic_prompt) + elif isinstance(dynamic_prompt,list) and all([isinstance(d,dict) for d in dynamic_prompt]): + module.change_to_multi_process(True) + if module.parallel: + dynamic_prompt = [module.generate(self.head,d) for d in dynamic_prompt] + if module.merge: + dynamic_prompt = merge_str_dicts(dynamic_prompt) + module.add_output_to_head(module.last_message) + elif not module.iterative and not module.merge: + for d in dynamic_prompt: + self.direct_run(dynamic_prompt = d, module = copy.copy(module)) + #dynamic_prompt = [module.generate(self.head,d) for d in dynamic_prompt] + break + elif module.iterative: + iter_dynamic = {} + for d in dynamic_prompt: + iter_dynamic = module.generate(self.head,{**d,**iter_dynamic}) + dynamic_prompt = iter_dynamic + module.end_multi() + else: + print(type(dynamic_prompt)) + raise TypeError(str(dynamic_prompt)) + self.log.append(f'{module} -> {module.send()}\n: {module.last_message}') + if isinstance(module, Module): + module.output() + print('DEBUG:', str(module), module.end, module.turns, module.max_turn) + if module.end or module.turns > module.max_turn: + break + module = module.send() + if isinstance(module, AugmentCluster): + module = module.get_first_module() + + + def __call__(self, data): + # run only one data + # backup + dataset_backup = self.dataset + current_data_index_backup = self.data_index + if hasattr(self,'current_data'): + current_data_backup = self.current_data + else: + current_data_backup = None + + # set data and run + dataset = [data] + self.dataset = dataset + self.data_index = 0 + result = self.run(datakeys = self.data_keys, init_docs = self.initial_docs, initial_module = self.initial_module, write = False, train = False) + + # restore + self.data_index = current_data_index_backup + self.current_data = current_data_backup + self.dataset = dataset_backup + + + return result + + def run(self, datakeys, init_docs = None, initial_module = None, write = True, train = False): + + # get data + self.current_data = self.dataset[self.data_index] + data = self.current_data + + # from head prompt from specific data + head = dict() + for key in datakeys: + if isinstance(data[key],str): + head[key] = data[key] + else: + assert isinstance(data[key],list) + assert all([isinstance(item, str) for item in data[key]]) + head[key] = ''.join(data[key]) + + #init + self.head = head + self.output = [] + self.doc_cache.clear() + if init_docs: + self.doc_cache.load_docs(data[init_docs]) + self.llm.reset() + if self.module: + for i in self.module: + i.reset() + self.log = [] + # run only one data, and add data_index by 1 + dynamic_prompt = {} + if not initial_module: + module = self.llm + else: + module = initial_module + if isinstance(module, AugmentCluster): + module = module.get_first_module() + while isinstance(module, Module): + if isinstance(dynamic_prompt,dict): + module.change_to_multi_process(False) + dynamic_prompt = module.generate(self.head,dynamic_prompt=dynamic_prompt) + elif isinstance(dynamic_prompt,list) and all([isinstance(d,dict) for d in dynamic_prompt]): + module.change_to_multi_process(True) + if module.parallel: + dynamic_prompt = [module.generate(self.head,d) for d in dynamic_prompt] + if module.merge: + dynamic_prompt = merge_str_dicts(dynamic_prompt) + module.add_output_to_head(module.last_message) + elif not module.iterative and not module.merge: + for d in dynamic_prompt: + self.direct_run(dynamic_prompt = d, module = copy.copy(module)) + #dynamic_prompt = [module.generate(self.head,d) for d in dynamic_prompt] + break + + elif module.iterative: + iter_dynamic = {} + for d in dynamic_prompt: + iter_dynamic = module.generate(self.head,{**d,**iter_dynamic}) + dynamic_prompt = iter_dynamic + module.end_multi() + else: + print(type(dynamic_prompt)) + raise TypeError(str(dynamic_prompt)) + self.log.append(f'{module} -> {module.send()}\n: {module.last_message}') + if isinstance(module, Module): + module.output() + if module.end or module.turns > module.max_turn: + break + module = module.send() + if isinstance(module, AugmentCluster): + module = module.get_first_module() + + # if eval, send to evaluation + if self.eval: + if not self.rich_eval: + self.result = self.eval() + else: + self.result = self.eval(self.form_eval_data()) + else: + self.result = {} + if write: + self.write() + if train: + self.export_training_data() + + #self.logs = self.delete_inner_cluster_logs(self.log) + res = {'data':self.get_data(), 'doc_cache':self.doc_cache.show_docs(), 'log': self.log.copy(),'output':self.output,'result': self.result} + if self.attributer: + self.attributer.attribute_for_result(res) + self.data_index += 1 + return res + + def delete_inner_cluster_logs(self, logs): + print(logs) + for cluster in self.stored_clusters: + cluster_name = str(cluster) + print('Combining logs for cluster:', cluster_name) + in_cluster = False + for i, log in enumerate(logs): + in_out_names = log.split('\n')[0] + if in_out_names in cluster_name: + # This is the inner log + if not in_cluster: + in_cluster = True + log_start = i + else: + continue + elif in_cluster: + # This is the outer log + in_cluster = False + log_end = i + cluster_output = logs[log_end] + next_module = in_out_names.split('->')[1].strip() + cluster_log = f"{cluster_name} -> {next_module}\n: {cluster_output}" + logs = logs[:log_start] + [cluster_log] + logs[log_end+1:] + print('Final logs:', logs) + return logs + + + + + def get_data(self): + return self.dataset[self.data_index] + + def write(self): + '''Default writing''' + llm_token_used = self.llm.token_used + write_down = {'data':self.get_data(), 'doc_cache':self.doc_cache.show_docs(), 'log': self.log.copy(),'output':self.output,'result': self.result,'token_used':llm_token_used} + + if self.attributer: + self.attributer.attribute_for_result(write_down) + + with open(self.save_path, 'a', encoding='utf-8') as file: + json_line = json.dumps(write_down, indent=4) + file.write(json_line + '\n') + + def get_module_by_name(self, name): + print('Getting module by name:', name) + for module in self.module: + if str(module) == name: + return module + if str(self.llm) == name: + return self.llm + + for cluster in self.stored_clusters: + print('trying cluster:', cluster) + if str(cluster) == name: + print('found cluster:', cluster) + return cluster + + return None + + def export_training_data(self): + flattened_data = [flatten_dict(self.result)] + header = set() + for item in flattened_data: + header.update(item.keys()) + header = sorted(header) + with open('output.csv', mode='a', newline='') as file: + writer = csv.DictWriter(file, fieldnames = header) + if self.table_head: + writer.writeheader() + self.table_head = False + + for row in flattened_data: + writer.writerow(row) + + + def __str__(self) -> str: + return 'pipeline output' + +class Sequence(Pipeline): + def __init__(self, save_path=None, sequence=None, head_prompt_maker=None, retriever=None, evaluator=None, dataset=None, rich_eval=False) -> None: + first_module = sequence[0] + other = sequence[1:] + super().__init__(save_path, sequence, head_prompt_maker, first_module, other, retriever, evaluator, dataset, rich_eval) + for i in range(len(sequence)-1): + module = sequence[i] + assert isinstance(module, Module) or isinstance(module,AugmentCluster) + module.set_target(sequence[i+1],post_processing=lambda x: {module.output_as: x}) + sequence[-1].set_output() + diff --git a/citekit/prompt/__pycache__/prompt.cpython-310.pyc b/citekit/prompt/__pycache__/prompt.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d0aff3e1183b936b0f9d64a18ad413319bca794a Binary files /dev/null and b/citekit/prompt/__pycache__/prompt.cpython-310.pyc differ diff --git a/citekit/prompt/__pycache__/prompt.cpython-312.pyc b/citekit/prompt/__pycache__/prompt.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6e2afcf3b6c6523617911b23e4f858673aee3c67 Binary files /dev/null and b/citekit/prompt/__pycache__/prompt.cpython-312.pyc differ diff --git a/citekit/prompt/prompt.py b/citekit/prompt/prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..625d4caac643969c434a9859508b90ee04c702df --- /dev/null +++ b/citekit/prompt/prompt.py @@ -0,0 +1,294 @@ +import json + + +truncate = lambda x, l: x[:l] +token_len = len + +def combine(*args): + if all([isinstance(arg,dict) for arg in args]): + if len(args) == 1: + return args[0] + else: + combined = args[0].copy() + combined.update(combine(*args[1:])) + return combined + +default_get = lambda key : lambda data: data[key] + + + +class Prompt: + components = {} + template = "" + truncate = lambda x, l: x[:l] + UNABLE = 'prompt_unable' + + + def update(self, **kargs): + try: + for key in kargs.keys(): + if key == 'template': + arg_template = kargs[key] + + if key == 'components': + arg_components = kargs[key] + test_prompt = Prompt(arg_template, arg_components) + except Exception as e: + print(e) + print('Update Rejected due to invalid template or components') + return + + self.template = arg_template + self.components = arg_components + + def __init__(self,template='', components={}, max_token=8000) -> None: + + ''' + Args: + template: The way to order and organize each components, use to represent a component, .... + components: The content of a component, use {NAME} to represent the placeholder of corresponding data + max_token: a list as long as components, representing the max number of tokens for each component, or a int representing the same max_token for all components + ''' + + # template + self.template = template + + # components + if isinstance(components,dict): + for key in components.keys(): + if f'<{str(key)}>' not in self.template: + raise Exception('component name not in template!') + self.components = components + + # max_token + self.max_token = {} + if isinstance(max_token,list): + if len(components)==len(max_token): + self.max_token = {att:val for (att,val) in zip(components.keys(),max_token)} + else: + raise Exception('max_token is not corresponding to components') + elif isinstance(max_token,int): + self.max_token_init = max_token + self.max_token = {att:max_token for att in components} + else: + raise TypeError('max_token should be int or list') + + def __repr__(self) -> str: + prompt = self.template + for key in self.components.keys(): + prompt = prompt.replace(f'<{str(key)}>',self.components[key]) + return prompt + + def __str__(self) -> str: + return repr(self) + + def part_template(self,**kargs): + ''' + Add components in to the prompt. + ''' + for part in kargs.keys(): + if f'<{str(part)}>' in self.template: + self.components[part] = kargs[part] + else: + raise Exception('component name not in template!') + + def __call__(self, *args,**kargs) -> str: + return self.make_prompt(*args, **kargs) + + + def __str__(self): + + input = {} + for key in self.components.keys(): + input[key] = self.components[key] + + + return self.make_prompt(input) + + + def make_prompt(self,*args,**kargs) -> str: + ''' + arg: a dictionary containing all contents to the placeholder of the prompt + kargs: use NAME=value to pass arguments + ''' + + if args: + args = combine(*args) + args = args.copy() + args.update(kargs) + else: + args = kargs + prompt = self.template + + for key in self.components.keys(): + if key not in args or args[key] == Prompt.UNABLE: + prompt = prompt.replace(f'<{str(key)}>',"") + else: + prompt = prompt.replace(f'<{str(key)}>', self.components[key]) + + prompt_args = {} + for key in args.keys(): + if key in self.components.keys(): + if self.max_token.get(key): + max_token = self.max_token.get(key) + else: + max_token = min(4096,self.max_token_init) + if token_len(args[key])> max_token: + args[key] = self.truncate(args[key],max_token) + + + return prompt.format(**args) + + def set_max_token(self,**kargs) -> None: + for key in kargs.keys(): + if key in self.components.keys(): + self.max_token[key] = kargs.get(key) + else: + raise KeyError(f'{key} not in Template!') + + def load_data(self,data_loader,*keys,**projections): + ''' + load data to make prompts from a data loader + projections: the function to get the information from a data. + ''' + + prompts = [] + for data in data_loader: + l_contents = {key:default_get(key)(data) for key in keys} + d_contents = {projection:projections[projection](data) for projection in projections.keys()} + prompts.append(self.make_prompt({**l_contents, **d_contents})) + + return prompts + + + + + +class DocPrompt(Prompt): + ''' + Containing Doc ID, Title and Passage in order: + + Document:[{ID}] + (Title:{Title}) + {Passage} + ''' + def __init__(self, template='<Passage>', components={'ID':'Document[{ID}]: ','Title':'(Title:{Title})','Passage':'{Passage}\n'}, max_token=4096) -> None: + super().__init__(template, components, max_token) + + +class ALCEDocPrompt(Prompt): + ''' + Containing Doc ID, Title and Passage in order: + + Document:[{ID}] + (Title:{Title}) + {Passage} + ''' + def __init__(self, template='<ID><title><text>', components={'ID':'Document [{ID}]','title':'(Title:{title}): ','text':'{text}\n'}, max_token=4096) -> None: + super().__init__(template, components, max_token) + + def default_load_data(self,data_loader, text = 'text', from_idx = 0): + return super().load_data(list(enumerate(data_loader)),text = lambda data: data[1][text],ID = lambda data: str(data[0]+1 + from_idx),title = lambda data: data[1]['title']) + + def default_load_data_wo_ID(self,data_loader): + return super().load_data(list(enumerate(data_loader)),text = lambda data: data[1]['text'],title = lambda data: data[1]['title']) + def default_load_data_wo_title(self,data_loader): + return super().load_data(list(enumerate(data_loader)),text = lambda data: data[1]['text'],ID = lambda data: str(data[0]+1)) + def default_load_data_extraction(self,data_loader): + return super().load_data(list(enumerate(data_loader)),text = lambda data: data[1]['extraction'],ID = lambda data: str(data[0]+1),title = lambda data: data[1]['title']) + def default_load_data_summary(self,data_loader): + return super().load_data(list(enumerate(data_loader)),text = lambda data: data[1]['summary'],ID = lambda data: str(data[0]+1),title = lambda data: data[1]['title']) + +class ALCEVanillaPrompt(Prompt): + ''' + Containing INST(Instruction), Question, Doc and Answer in order: + + {INST} + + Question:{Question} + + {Doc} + Answer:{Answer} + ''' + def __init__(self, + template="<INST><Question><Doc><Answer>\n", + components={'INST':'{INST}\n\n', 'Question':'Question:{Question}\n\n','Doc':'{Doc}\n','Answer':'Answer:{Answer}'}, + max_token=4096) -> None: + super().__init__(template, components, max_token) + +class NewALCEVanillaPrompt(Prompt): + ''' + Containing INST(Instruction), Question, Doc and Answer in order: + + {INST} + + Question:{Question} + + {Doc} + Answer:{Answer} + ''' + def __init__(self, + template="<INST><question><docs><answer>\n", + components={'INST':'{INST}\n\n', 'question':'Question:{question}\n\n','docs':'{docs}\n','answer':'Answer:{answer}'}, + max_token=4096) -> None: + super().__init__(template, components, max_token) + + + +class AGEEPrompt(Prompt): + ''' + Containing INST(Instruction), Question and Doc in order: + + {INST} + + Question:{Question} + + Search Results:{Doc} + ''' + def __init__(self, + template="<INST><Question><Doc>\n", + components={'INST':'{INST}\n\n', 'Question':'Question:\n{Question}\n','Doc':'Search Results:\n{Doc}\n'}, + max_token=4096) -> None: + super().__init__(template, components, max_token) + + + + +alce_prompt= ALCEVanillaPrompt() +#alce_prompt.set_max_token(INST = 10,Doc = 100,Answer = 15) +DocP= DocPrompt() + + +#print(content['demos']) + + +#print(data[0]) +''' +pps = alce_prompt.load_data(content['demos'], + INST = lambda _: content['instruction'], + Question = lambda data: data['question'], + Doc = lambda data: ''.join(DocPrompt().load_data(list(enumerate(data['docs'])), + ID = lambda data: str(data[0]), + Title = lambda data: data[1]['title'], + Passage = lambda data: data[1]['text'])), + Answer = lambda data: data['answer']) + +''' + +#print(pps[0]) + +''' +data_loader = [] +with open('data.txt','r',encoding='utf-8') as f: + content = f.readlines() + for i,c in enumerate(content): + if i%3 == 0: + data_loader.append({'Q':c.strip(),'A':content[i+1].strip()}) +print(data_loader) + + +pps = Dp.load_data(data_loader, + INST= lambda data: "Instruction: Write an accurate, engaging, and concise answer for the given question", + Question= lambda data: data['Q'], + Answer= lambda data: data['A']) +''' \ No newline at end of file diff --git a/citekit/utils/__pycache__/utils.cpython-310.pyc b/citekit/utils/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1f04fdf982f3909b7f1c9e6c08077d4949d9c2e7 Binary files /dev/null and b/citekit/utils/__pycache__/utils.cpython-310.pyc differ diff --git a/citekit/utils/__pycache__/utils.cpython-312.pyc b/citekit/utils/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5cf662193b0fdf4d05c746e3d76df14e36cf0efe Binary files /dev/null and b/citekit/utils/__pycache__/utils.cpython-312.pyc differ diff --git a/citekit/utils/utils.py b/citekit/utils/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..425e90b0daac03e0e7a5b0c893fecf549f3377a5 --- /dev/null +++ b/citekit/utils/utils.py @@ -0,0 +1,317 @@ +import numpy as np +import string +import re +import collections +import torch +import nltk + +def one_paragraph(text): + paras = text.lstrip('\n').split('\n\n') + if not paras: + return '' + else: + return paras[0].rstrip('\n') + +def strong_one_paragraph(text): + paras = text.lstrip('\n').split('\n') + if not paras: + return '' + else: + return paras[0].rstrip('\n') + +def compute_str_em(data): + """Compute STR-EM metric (only for ASQA) + Args: + data: requires field `qa_pairs/short_answers` and `output` + Returns: + STR-EM and STR-EM-HIT () + """ + if 'qa_pairs' not in data[0] or data[0]['qa_pairs'] is None: + return 0 + + acc = [] + hit = [] + + for item in data: + loc_acc = [] + for qa_pair in item['qa_pairs']: + loc_acc.append(exact_presence(qa_pair['short_answers'], item["output"])) + acc.append(np.mean(loc_acc)) + hit.append(int(np.mean(loc_acc) == 1)) + + return 100 * np.mean(acc) + return 100 * np.mean(acc), 100 * np.mean(hit) + +def average(func): + def avg_func(dataset): + print(len(dataset)) + results = [func(*data) for data in dataset] if dataset else [] + if results: + return np.mean(np.array(results), axis=0).tolist() + else: + return None + return avg_func + +def normalize_answer(s): + def remove_articles(text): + return re.sub(r"\b(a|an|the)\b", " ", text) + + def white_space_fix(text): + return " ".join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return "".join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + + return white_space_fix(remove_articles(remove_punc(lower(s)))) + +def compute_f1(a_gold, a_pred): + """Compute F1 score between two strings.""" + + def _get_tokens(s): + if not s: + return [] + return normalize_answer(s).split() + + gold_toks = _get_tokens(a_gold) + pred_toks = _get_tokens(a_pred) + + common = collections.Counter(gold_toks) & collections.Counter(pred_toks) + num_same = sum(common.values()) + + if len(gold_toks) == 0 or len(pred_toks) == 0: + # If either is no-answer, then F1 is 1 if they agree, 0 otherwise + return int(gold_toks == pred_toks) + + if num_same == 0: + return 0 + + precision = 1.0 * num_same / len(pred_toks) + recall = 1.0 * num_same / len(gold_toks) + f1 = (2 * precision * recall) / (precision + recall) + + return f1 + + +def compute_exact(a_gold, a_pred): + """Check whether two strings are equal up to normalization.""" + + return int(normalize_answer(a_gold) == normalize_answer(a_pred)) + + +def exact_presence(short_answers, context): + """Verify if any of the answers is present in the given context. + Args: + short_answers: list of short answers to look for in the context + context: a paragraph to search for short answers + Returns: + true if any of the short answers is present in the context + """ + + n_short_answers = [normalize_answer(sa) for sa in short_answers] + n_context = normalize_answer(context) + + for ans in n_short_answers: + if ans in n_context: + return True + + return False + +def output_begin_with(word): + def f(self) -> bool: + return self.last_message.strip().lower()[:len(word)] == word + return f + +def output_end_with(word): + def f(self) -> bool: + return strong_one_paragraph(self.last_message.strip())[-len(word):] == word + return f + + +def make_as(datakey): + def f(passage): + return {datakey:passage} + return f + +def cut_and_make_as(datakey): + def f(passage): + return {datakey:one_paragraph(passage)} + return f + +def remove_citations(sent): + return re.sub(r"{\d+", "", re.sub(r" {\d+", "", sent)).replace(" |", "").replace("}", "").replace("{", "") + +def remove_citations(sent): + return re.sub(r"\[\d+", "", re.sub(r" \[\d+", "", sent)).replace(" |", "").replace("]", "") + + +def match_document(ref_mark, output_ref_span): + ref = set() + ref_span = [] + for num in ref_mark: + ref_str = str(num) + if ref_str in output_ref_span: + ref_parts = output_ref_span[ref_str].split("[") + if len(ref_parts) > 1: + ref_id_parts = ref_parts[1].split("]") + if len(ref_id_parts) > 0: + ref_id = ref_id_parts[0].strip() + if ref_id.isdigit(): + ref.add(int(ref_id)) # 添加Document id + + ref_span_parts = output_ref_span[ref_str].split(":",1)#第一个冒号后面的片段 + if len(ref_span_parts) > 1: + ref_span.append(ref_span_parts[1].strip()) # 添加后面的句子片段 + else: + ref_span.append('') + return list(ref), ref_span + +def get_max_memory(): + """Get the maximum memory available for the current GPU for loading models.""" + free_in_GB = int(torch.cuda.mem_get_info()[0]/1024**3) + max_memory = f'{free_in_GB-6}GB' + n_gpus = torch.cuda.device_count() + max_memory = {i: max_memory for i in range(n_gpus)} + return max_memory + +def each_make_as(key): + def function(output): + sents = nltk.sent_tokenize(one_paragraph(output)) + if len(sents)>3: + sents = sents[:3] + return [make_as(key)(sent) for sent in sents] + return function + +def each_par_make_as(key): + def function(output): + sents = one_paragraph(output).split('\n') + if len(sents)>3: + sents = sents[:3] + return [make_as(key)(sent) for sent in sents] + return function + +def sentence(key): + def function(output): + sents = nltk.sent_tokenize(one_paragraph(output)) + for sent in sents: + refs = re.findall(r'\[\d+\]', sent) + if refs: + return make_as(key)(sent) + return make_as(key)('') + return function + +def sentences(key): + def function(output): + sents = nltk.sent_tokenize(one_paragraph(output)) + return [make_as(key)(sent) for sent in sents][:1] + return function + +def three_sentences(key): + def function(output): + sents = nltk.sent_tokenize(one_paragraph(output)) + return [make_as(key)(sent) for sent in sents][:3] + return function + +def first_sentence(text): + sents = nltk.sent_tokenize(one_paragraph(text)) + for sent in sents: + return sent + return '' + +def flatten_dict(d, parent_key='', sep='_'): + items = [] + for k, v in d.items(): + new_key = f'{parent_key}{sep}{k}' if parent_key else k + if isinstance(v, dict): + items.extend(flatten_dict(v, new_key, sep=sep).items()) + else: + items.append((new_key, v)) + return dict(items) + + + +import re +from bs4 import BeautifulSoup + +def parse_html_prompt(input_str): + soup = BeautifulSoup(input_str, "html.parser") + + # 处理 <p></p> 内的内容 + p_content = soup.find("p").decode_contents().replace("<br>", "\n") + p_content = re.sub(r'<span[^>]*>(.*?)</span>', r'<\1>', p_content) + template = p_content.strip().replace(' <br/>', '').replace(' ', '').replace('<br/>', '') + + # 解析 component-item + components = {} + for item in soup.find_all("div", class_="component-item"): + key_span = item.find("div", class_="component-key").find("span") + key = key_span.get_text(strip=True) if key_span else "" + value_div = item.find("div", class_="component-value") + value_content = value_div.decode_contents() + value_content = re.sub(r'<span[^>]*>(.*?)</span>', r'{\1}', value_content) + components[key] = value_content.strip().replace(' <br/>', '').replace('<br/>', '') + + # 解析 self-info-item + self_prompt = {} + for item in soup.find_all("div", class_="self-info-item"): + key_span = item.find("div", class_="component-key").find("span") + key = key_span.get_text(strip=True) if key_span else "" + value_div = item.find("div", class_="component-value") + value = value_div.get_text(strip=True) if value_div else "" + self_prompt[key] = value.replace(' <br/>', '').replace('<br/>', '') + + return { + 'template': template, + 'components': components, + 'self_prompt': self_prompt + } + + +def parse_html_destination(input_str): + soup = BeautifulSoup(input_str, "html.parser") + destination = soup.find("destination").get_text(strip=True) + prompt_key = soup.find("prompt_key").get_text(strip=True) + return destination, prompt_key + +def parse_html_new_model(input_str): + soup = BeautifulSoup(input_str, "html.parser") + model_type = soup.find("model_type").get_text(strip=True) + model_name = soup.find("model").get_text(strip=True) + key = soup.find("prompt_key").get_text(strip=True) + return model_type, model_name, key + +def parse_delete_destination(input_str): + soup = BeautifulSoup(input_str, "html.parser") + destination = soup.find("deletedestination").get_text(strip=True) + return destination + +def parse_html_header(input_str): + soup = BeautifulSoup(input_str, "html.parser") + header = soup.find("to_head").get_text(strip=True) + return header + +def parse_html_config(info): + config = '' + if 'class="component-value"' in info: + func = parse_html_prompt + config = 'prompt' + elif '</destination>' in info: + func = parse_html_destination + config = 'destination' + elif '<model_type>' in info: + func = parse_html_new_model + config = 'new_model' + elif 'deletedestination' in info: + config = 'delete_destination' + func = parse_delete_destination + elif 'to_head' in info: + config = 'header' + func = parse_html_header + else: + raise NotImplementedError + result = func(info) + print(info, 'parsed as', config) + return config, result \ No newline at end of file diff --git a/context_cite/__init__.py b/context_cite/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ba2d0b3ceeaab8b45bb883676773804aec8157ab --- /dev/null +++ b/context_cite/__init__.py @@ -0,0 +1,4 @@ +from .context_citer import ContextCiter + +__version__ = "0.0.1" +VERSION = __version__ diff --git a/context_cite/__pycache__/__init__.cpython-312.pyc b/context_cite/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7aaec5dd0fe31959c418f01b34d6b65849eb7b90 Binary files /dev/null and b/context_cite/__pycache__/__init__.cpython-312.pyc differ diff --git a/context_cite/__pycache__/__init__.cpython-39.pyc b/context_cite/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7fc77eec341678eac63fecfbe28bb6d77ad37b5e Binary files /dev/null and b/context_cite/__pycache__/__init__.cpython-39.pyc differ diff --git a/context_cite/__pycache__/context_citer.cpython-312.pyc b/context_cite/__pycache__/context_citer.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..40e4c4f3c35dd3fcc11da09bdb055b3163cfbf71 Binary files /dev/null and b/context_cite/__pycache__/context_citer.cpython-312.pyc differ diff --git a/context_cite/__pycache__/context_citer.cpython-39.pyc b/context_cite/__pycache__/context_citer.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d22d07bfe018cb78e96e9886d59fa5889d0f8b12 Binary files /dev/null and b/context_cite/__pycache__/context_citer.cpython-39.pyc differ diff --git a/context_cite/__pycache__/context_partitioner.cpython-312.pyc b/context_cite/__pycache__/context_partitioner.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..63149f81a81ee7000a7a9b17f3fba5a352a2269b Binary files /dev/null and b/context_cite/__pycache__/context_partitioner.cpython-312.pyc differ diff --git a/context_cite/__pycache__/context_partitioner.cpython-39.pyc b/context_cite/__pycache__/context_partitioner.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a147f1799fc9d7050cb09bcd9fe82591abef7681 Binary files /dev/null and b/context_cite/__pycache__/context_partitioner.cpython-39.pyc differ diff --git a/context_cite/__pycache__/solver.cpython-312.pyc b/context_cite/__pycache__/solver.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..65d2806a94e27cfb7e5e3bd832b2f9538d918fd0 Binary files /dev/null and b/context_cite/__pycache__/solver.cpython-312.pyc differ diff --git a/context_cite/__pycache__/solver.cpython-39.pyc b/context_cite/__pycache__/solver.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..76784241b11a9c0ecc453f0c4c9a87bef538cda1 Binary files /dev/null and b/context_cite/__pycache__/solver.cpython-39.pyc differ diff --git a/context_cite/__pycache__/utils.cpython-312.pyc b/context_cite/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d6c2854bf4f43c2a370ff2002bec2ec6f723129e Binary files /dev/null and b/context_cite/__pycache__/utils.cpython-312.pyc differ diff --git a/context_cite/__pycache__/utils.cpython-39.pyc b/context_cite/__pycache__/utils.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c53e5ed7fef958cb72243ff814fe7f2e60907f3b Binary files /dev/null and b/context_cite/__pycache__/utils.cpython-39.pyc differ diff --git a/context_cite/context_citer.py b/context_cite/context_citer.py new file mode 100644 index 0000000000000000000000000000000000000000..57d05a89e3bd9639ffcf076a166cc7fe8db4c35e --- /dev/null +++ b/context_cite/context_citer.py @@ -0,0 +1,375 @@ +import numpy as np +import pandas as pd +import torch as ch +from numpy.typing import NDArray +from typing import Dict, Any, Optional, List +import logging +from transformers import AutoTokenizer, AutoModelForCausalLM +from .context_partitioner import BaseContextPartitioner, SimpleContextPartitioner +from .solver import BaseSolver, LassoRegression +from .utils import ( + get_masks_and_logit_probs, + aggregate_logit_probs, + split_text, + highlight_word_indices, + get_attributions_df, + char_to_token, +) + + +DEFAULT_GENERATE_KWARGS = {"max_new_tokens": 512, "do_sample": False} +DEFAULT_PROMPT_TEMPLATE = "Context: {context}\n\nQuery: {query}" + + +class ContextCiter: + def __init__( + self, + model: Any, + tokenizer: Any, + context: str, + query: str, + source_type: str = "sentence", + generate_kwargs: Optional[Dict[str, Any]] = None, + num_ablations: int = 64, + ablation_keep_prob: float = 0.5, + batch_size: int = 1, + solver: Optional[BaseSolver] = None, + prompt_template: str = DEFAULT_PROMPT_TEMPLATE, + partitioner: Optional[BaseContextPartitioner] = None, + ) -> None: + """ + Initializes a new instance of the ContextCiter class, which is designed + to assist in generating contextualized responses using a given machine + learning model and tokenizer, tailored to specific queries and contexts. + + Arguments: + model (Any): + The model to apply ContextCite to (a HuggingFace + ModelForCausalLM). + tokenizer (Any): + The tokenizer associated with the provided model. + context (str): + The context provided to the model + query (str): + The query to pose to the model. + source_type (str, optional): + The type of source to partition the context into. Defaults to + "sentence", can also be "word". + generate_kwargs (Optional[Dict[str, Any]], optional): + Additional keyword arguments to pass to the model's generate + method. + num_ablations (int, optional): + The number of ablations used to train the surrogate model. + Defaults to 64. + ablation_keep_prob (float, optional): + The probability of keeping a source when ablating the context. + Defaults to 0.5. + batch_size (int, optional): + The batch size used when performing inference using ablated + contexts. Defaults to 1. + solver (Optional[Solver], optional): + The solver to use to compute the linear surrogate model. Lasso + regression is used by default. + prompt_template (str, optional): + A template string used to create the prompt from the context + and query. + partitioner (Optional[BaseContextPartitioner], optional): + A custom partitioner to split the context into sources. This + will override "source_type" if specified. + """ + + self.model = model + self.tokenizer = tokenizer + if partitioner is None: + self.partitioner = SimpleContextPartitioner( + context, source_type=source_type + ) + else: + self.partitioner = partitioner + if self.partitioner.context != context: + raise ValueError("Partitioner context does not match provided context.") + self.query = query + self.generate_kwargs = generate_kwargs or DEFAULT_GENERATE_KWARGS + self.num_ablations = num_ablations + self.ablation_keep_prob = ablation_keep_prob + self.batch_size = batch_size + self.solver = solver or LassoRegression() + self.prompt_template = prompt_template + + self._cache = {} + self.logger = logging.getLogger("ContextCite") + self.logger.setLevel(logging.DEBUG) # TODO: change to INFO later + + if self.tokenizer.pad_token is None: + self.tokenizer.pad_token = self.tokenizer.eos_token + + @classmethod + def from_pretrained( + cls, + pretrained_model_name_or_path, + context: str, + query: str, + device: str = "cuda", + model_kwargs: Dict[str, Any] = {}, + tokenizer_kwargs: Dict[str, Any] = {}, + **kwargs: Dict[str, Any], + ) -> "ContextCiter": + """ + Load a ContextCiter instance from a pretrained model. + + Arguments: + pretrained_model_name_or_path (str): + The name or path of the pretrained model. This can be a local + path or a model name on the HuggingFace model hub. + context (str): + The context provided to the model. The context and query will be + used to construct a prompt for the model, using the prompt template. + query (str): + The query provided to the model. The context and query will be + used to construct a prompt for the model, using the prompt template. + device (str, optional): + The device to use. Defaults to "cuda". + model_kwargs (Dict[str, Any], optional): + Additional keyword arguments to pass to the model's constructor. + tokenizer_kwargs (Dict[str, Any], optional): + Additional keyword arguments to pass to the tokenizer's constructor. + **kwargs (Dict[str, Any], optional): + Additional keyword arguments to pass to the ContextCiter constructor. + + Returns: + ContextCiter: + A ContextCiter instance initialized with the provided model, + tokenizer, context, query, and other keyword arguments. + """ + model = AutoModelForCausalLM.from_pretrained( + pretrained_model_name_or_path, **model_kwargs + ) + model.to(device) + tokenizer = AutoTokenizer.from_pretrained( + pretrained_model_name_or_path, **tokenizer_kwargs + ) + tokenizer.padding_side = "left" + return cls(model, tokenizer, context, query, **kwargs) + + def _get_prompt_ids( + self, + mask: Optional[NDArray] = None, + return_prompt: bool = False, + ): + context = self.partitioner.get_context(mask) + prompt = self.prompt_template.format(context=context, query=self.query) + messages = [{"role": "user", "content": prompt}] + chat_prompt = self.tokenizer.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True + ) + chat_prompt_ids = self.tokenizer.encode(chat_prompt, add_special_tokens=False) + + if return_prompt: + return chat_prompt_ids, chat_prompt + else: + return chat_prompt_ids + + @property + def _response_start(self): + prompt_ids = self._get_prompt_ids() + return len(prompt_ids) + + @property + def _output(self): + if self._cache.get("output") is None: + prompt_ids, prompt = self._get_prompt_ids(return_prompt=True) + input_ids = ch.tensor([prompt_ids], device=self.model.device) + output_ids = self.model.generate(input_ids, **self.generate_kwargs)[0] + # We take the original prompt because sometimes encoding and decoding changes it + raw_output = self.tokenizer.decode(output_ids) + prompt_length = len(self.tokenizer.decode(prompt_ids)) + self._cache["output"] = prompt + raw_output[prompt_length:] + return self._cache["output"] + + @property + def _output_tokens(self): + return self.tokenizer(self._output, add_special_tokens=False) + + @property + def _response_ids(self): + return self._output_tokens["input_ids"][self._response_start :] + + @property + def response(self): + """ + The response generated by the model (excluding the prompt). This + property is cached. + + Returns: + str: + The response generated by the model. + """ + output_tokens = self._output_tokens + char_response_start = output_tokens.token_to_chars(self._response_start).start + return self._output[char_response_start:] + + @property + def response_with_indices(self, split_by="word", color=True) -> [str, pd.DataFrame]: + """ + The response generated by the model, annotated with the starting index + of each part. + + Arguments: + split_by (str, optional): + The method to split the response by. Can be "word" or "sentence". + Defaults to "word". + color (bool, optional): + Whether to color the starting index of each part. Defaults to True. + + Returns: + str: + The response with the starting index of each part highlighted. + """ + start_indices = [] + parts, separators, start_indices = split_text(self.response, split_by) + separated_str = highlight_word_indices(parts, start_indices, separators, color) + return separated_str + + @property + def num_sources(self) -> int: + """ + The number of sources within the context. I.e., the number of sources + that the context is partitioned into. + + Returns: + int: + The number of sources in the context. + """ + return self.partitioner.num_sources + + @property + def sources(self) -> List[str]: + """ + The sources within the context. I.e., the context as a list + where each element is a source. + + Returns: + List[str]: + The sources within the context. + """ + return self.partitioner.sources + + def _char_range_to_token_range(self, start_index, end_index): + output_tokens = self._output_tokens + response_start = self._response_start + offset = output_tokens.token_to_chars(response_start).start + ids_start_index = char_to_token(output_tokens, start_index + offset) + ids_end_index = char_to_token(output_tokens, end_index + offset - 1) + 1 + return ids_start_index - response_start, ids_end_index - response_start + + def _indices_to_token_indices(self, start_index=None, end_index=None): + if start_index is None or end_index is None: + start_index = 0 + end_index = len(self.response) + if not (0 <= start_index < end_index <= len(self.response)): + raise ValueError( + f"Invalid selection range ({start_index}, {end_index}). " + f"Please select any range within (0, {len(self.response)})." + ) + return self._char_range_to_token_range(start_index, end_index) + + def _compute_masks_and_logit_probs(self) -> None: + self._cache["reg_masks"], self._cache["reg_logit_probs"] = ( + get_masks_and_logit_probs( + self.model, + self.tokenizer, + self.num_ablations, + self.num_sources, + self._get_prompt_ids, + self._response_ids, + self.ablation_keep_prob, + self.batch_size, + ) + ) + + @property + def _masks(self): + if self._cache.get("reg_masks") is None: + self._compute_masks_and_logit_probs() + return self._cache["reg_masks"] + + @property + def _logit_probs(self): + if self._cache.get("reg_logit_probs") is None: + self._compute_masks_and_logit_probs() + return self._cache["reg_logit_probs"] + + def _get_attributions_for_ids_range(self, ids_start_idx, ids_end_idx) -> tuple: + outputs = aggregate_logit_probs(self._logit_probs[:, ids_start_idx:ids_end_idx]) + num_output_tokens = ids_end_idx - ids_start_idx + weight, bias = self.solver.fit(self._masks, outputs, num_output_tokens) + return weight, bias + + def get_attributions( + self, + start_idx: Optional[int] = None, + end_idx: Optional[int] = None, + as_dataframe: bool = False, + top_k: Optional[int] = None, + verbose: bool = True, + ): + """ + Get the attributions for (part of) the response. + + Arguments: + start_idx (int, optional): + Start index of the part to attribute to. If None, defaults to + the start of the response. + end_idx (int, optional): + End index of the part to attribute to. If None, defaults to the + end of the response. + as_dataframe (bool, optional): + If True, return the attributions as a stylized dataframe in + sorted order. Otherwise, return them as a numpy array where + the ith element corresponds to the score of the ith source + within the context. Defaults to False. + top_k (int, optional): + Only used if as_dataframe is True. Number of top attributions to + return. If None, all attributions are returned. Defaults to None. + verbose (bool, optional): + If True, print the selected part of the response. Defaults to + True. + + Returns: + NDArray | Any: + If as_dataframe is False, return a numpy array where the ith element + corresponds to the score of the ith source within the context. + Otherwise, return a stylized dataframe in sorted order. + """ + if self.num_sources == 0: + self.logger.warning("No sources to attribute to!") + return np.array([]) + + if not as_dataframe and top_k is not None: + self.logger.warning("top_k is ignored when not using dataframes.") + + ids_start_idx, ids_end_idx = self._indices_to_token_indices(start_idx, end_idx) + selected_text = self.response[start_idx:end_idx] + selected_tokens = self._response_ids[ids_start_idx:ids_end_idx] + decoded_text = self.tokenizer.decode(selected_tokens) + if selected_text.strip() not in decoded_text.strip(): + self.logger.warning( + "Decoded selected tokens do not match selected text.\n" + "If the following look close enough, feel free to ignore:\n" + "What you selected: %s\nWhat is being attributed: %s", + selected_text.strip(), + decoded_text.strip(), + ) + + if verbose: + print(f"Attributed: {decoded_text.strip()}") + + # _bias is the bias term in the l1 regression + attributions, _bias = self._get_attributions_for_ids_range( + ids_start_idx, + ids_end_idx, + ) + if as_dataframe: + return get_attributions_df(attributions, self.partitioner, top_k=top_k) + else: + return attributions diff --git a/context_cite/context_partitioner.py b/context_cite/context_partitioner.py new file mode 100644 index 0000000000000000000000000000000000000000..9a93abe71e3463d2ccdb9806dc7937e1b87b9e08 --- /dev/null +++ b/context_cite/context_partitioner.py @@ -0,0 +1,101 @@ +import numpy as np +from numpy.typing import NDArray +from typing import Optional, List +from abc import ABC, abstractmethod +from .utils import split_text + + +class BaseContextPartitioner(ABC): + """ + A base class for partitioning a context into sources. + + Attributes: + context (str): + The context to partition. + + Methods: + num_sources(self) -> int: + Property. The number of sources within the context. + split_context(self) -> None: + Split the context into sources. + get_source(self, index: int) -> str: + Get a represention of the source corresponding to a given index. + get_context(self, mask: Optional[NDArray] = None) -> str: + Get a version of the context ablated according to the given mask. + sources(self) -> List[str]: + Property. A list of all sources within the context. + """ + + def __init__(self, context: str) -> None: + self.context = context + + @property + @abstractmethod + def num_sources(self) -> int: + """The number of sources.""" + + @abstractmethod + def split_context(self) -> None: + """Split the context into sources.""" + + @abstractmethod + def get_source(self, index: int) -> str: + """Get a represention of the source corresponding to a given index.""" + + @abstractmethod + def get_context(self, mask: Optional[NDArray] = None): + """Get a version of the context ablated according to the given mask.""" + + @property + def sources(self) -> List[str]: + """A list of all sources.""" + return [self.get_source(i) for i in range(self.num_sources)] + + +class SimpleContextPartitioner(BaseContextPartitioner): + """ + A simple context partitioner that splits the context into sources based on + a separator. + """ + + def __init__(self, context: str, source_type: str = "sentence") -> None: + super().__init__(context) + self.source_type = source_type + self._cache = {} + + def split_context(self): + """Split text into parts and cache the parts and separators.""" + parts, separators, _ = split_text(self.context, self.source_type) + self._cache["parts"] = parts + self._cache["separators"] = separators + + @property + def parts(self): + if self._cache.get("parts") is None: + self.split_context() + return self._cache["parts"] + + @property + def separators(self): + if self._cache.get("separators") is None: + self.split_context() + return self._cache["separators"] + + @property + def num_sources(self) -> int: + return len(self.parts) + + def get_source(self, index: int) -> str: + return self.parts[index] + + def get_context(self, mask: Optional[NDArray] = None): + if mask is None: + mask = np.ones(self.num_sources, dtype=bool) + separators = np.array(self.separators)[mask] + parts = np.array(self.parts)[mask] + context = "" + for i, (separator, part) in enumerate(zip(separators, parts)): + if i > 0: + context += separator + context += part + return context diff --git a/context_cite/solver.py b/context_cite/solver.py new file mode 100644 index 0000000000000000000000000000000000000000..7af5c89a7997a354f9abc57676194c3e1a28ee1a --- /dev/null +++ b/context_cite/solver.py @@ -0,0 +1,54 @@ +import numpy as np +from numpy.typing import NDArray +from typing import Tuple +from abc import ABC, abstractmethod +from sklearn.preprocessing import StandardScaler +from sklearn.linear_model import Lasso +from sklearn.pipeline import make_pipeline + + +class BaseSolver(ABC): + """ + A base solver class. + + Methods: + fit(self, masks: NDArray, outputs: NDArray, num_output_tokens: int) -> Tuple[NDArray, NDArray]: + Fit the solver to the given data. + """ + + @abstractmethod + def fit( + self, masks: NDArray, outputs: NDArray, num_output_tokens: int + ) -> Tuple[NDArray, NDArray]: ... + + +class LassoRegression(BaseSolver): + """ + A LASSO solver using the scikit-learn library. + + Attributes: + lasso_alpha (float): + The alpha parameter for the LASSO regression. Defaults to 0.01. + + Methods: + fit(self, masks: NDArray, outputs: NDArray, num_output_tokens: int) -> Tuple[NDArray, NDArray]: + Fit the solver to the given data. + """ + + def __init__(self, lasso_alpha: float = 0.01) -> None: + self.lasso_alpha = lasso_alpha + + def fit( + self, masks: NDArray, outputs: NDArray, num_output_tokens: int + ) -> Tuple[NDArray, NDArray]: + X = masks.astype(np.float32) + Y = outputs / num_output_tokens + scaler = StandardScaler() + lasso = Lasso(alpha=self.lasso_alpha, random_state=0, fit_intercept=True) + # Pipeline is ((X - scaler.mean_) / scaler.scale_) @ lasso.coef_.T + lasso.intercept_ + pipeline = make_pipeline(scaler, lasso) + pipeline.fit(X, Y) + # Rescale back to original scale + weight = lasso.coef_ / scaler.scale_ + bias = lasso.intercept_ - (scaler.mean_ / scaler.scale_) @ lasso.coef_.T + return weight * num_output_tokens, bias * num_output_tokens diff --git a/context_cite/utils.py b/context_cite/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d5d3ffd39b2650031b06e570088bfe3079f0928f --- /dev/null +++ b/context_cite/utils.py @@ -0,0 +1,224 @@ +import nltk +import numpy as np +import pandas as pd +import torch as ch +from numpy.typing import NDArray +from spacy.lang.en import English +from tqdm.auto import tqdm +from typing import Any, List, Optional, Tuple +from datasets import Dataset +from torch.utils.data import DataLoader +from transformers import DataCollatorForSeq2Seq + +nltk.download("punkt_tab") + + +def split_text(text: str, split_by: str) -> Tuple[List[str], List[str], List[str]]: + """Split response into parts and return the parts, start indices, and separators.""" + parts = [] + separators = [] + start_indices = [] + + for line in text.splitlines(): + if split_by == "sentence": + parts.extend(nltk.sent_tokenize(line)) + elif split_by == "word": + tokenizer = English().tokenizer + parts = [token.text for token in tokenizer(text)] + else: + raise ValueError(f"Cannot split response by '{split_by}'") + + cur_start = 0 + for part in parts: + cur_end = text.find(part, cur_start) + separator = text[cur_start:cur_end] + separators.append(separator) + start_indices.append(cur_end) + cur_start = cur_end + len(part) + + return parts, separators, start_indices + + +def highlight_word_indices(words, indices, separators, color: bool): + formatted_words = [] + + # ANSI escape code for red color + if color: + RED = "\033[36m" # ANSI escape code for light gray + RESET = "\033[0m" # Reset color to default + else: + RED = "" + RESET = "" + + for word, idx in zip(words, indices): + # Wrap index with red color + formatted_words.append(f"{RED}[{idx}]{RESET}{word}") + + result = "".join(sep + word for sep, word in zip(separators, formatted_words)) + return result + + +def _create_mask(num_sources, alpha, seed): + random = np.random.RandomState(seed) + p = [1 - alpha, alpha] + return random.choice([False, True], size=num_sources, p=p) + + +def _create_regression_dataset( + num_masks, num_sources, get_prompt_ids, response_ids, alpha, base_seed=0 +): + masks = np.zeros((num_masks, num_sources), dtype=bool) + data_dict = { + "input_ids": [], + "attention_mask": [], + "labels": [], + } + for seed in range(num_masks): + mask = _create_mask(num_sources, alpha, seed + base_seed) + masks[seed] = mask + prompt_ids = get_prompt_ids(mask=mask) + input_ids = prompt_ids + response_ids + data_dict["input_ids"].append(input_ids) + data_dict["attention_mask"].append([1] * len(input_ids)) + data_dict["labels"].append([-100] * len(prompt_ids) + response_ids) + return masks, Dataset.from_dict(data_dict) + + +def _compute_logit_probs(logits, labels): + batch_size, seq_length = labels.shape + # [num_tokens x vocab_size] + reshaped_logits = logits.reshape(batch_size * seq_length, -1) + reshaped_labels = labels.reshape(batch_size * seq_length) + correct_logits = reshaped_logits.gather(-1, reshaped_labels[:, None])[:, 0] + cloned_logits = reshaped_logits.clone() + cloned_logits.scatter_(-1, reshaped_labels[:, None], -ch.inf) + other_logits = cloned_logits.logsumexp(dim=-1) + reshaped_outputs = correct_logits - other_logits + return reshaped_outputs.reshape(batch_size, seq_length) + + +def _make_loader(dataset, tokenizer, batch_size): + collate_fn = DataCollatorForSeq2Seq(tokenizer=tokenizer, padding="longest") + loader = DataLoader( + dataset, + batch_size=batch_size, + collate_fn=collate_fn, + ) + return loader + + +def _get_response_logit_probs(dataset, model, tokenizer, response_length, batch_size): + if batch_size > 1: + assert tokenizer.padding_side == "left", "Tokenizer must use left padding" + loader = _make_loader(dataset, tokenizer, batch_size) + logit_probs = ch.zeros((len(dataset), response_length), device=model.device) + + start_index = 0 + for batch in tqdm(loader): + batch = {key: value.to(model.device) for key, value in batch.items()} + with ch.no_grad(), ch.cuda.amp.autocast(): + output = model(**batch) + logits = output.logits[:, -(response_length + 1) : -1] + labels = batch["labels"][:, -response_length:] + batch_size, _ = labels.shape + cur_logit_probs = _compute_logit_probs(logits, labels) + logit_probs[start_index : start_index + batch_size] = cur_logit_probs + start_index += batch_size + + return logit_probs.cpu().numpy() + + +def get_masks_and_logit_probs( + model, + tokenizer, + num_masks, + num_sources, + get_prompt_ids, + response_ids, + ablation_keep_prob, + batch_size, + base_seed=0, +): + masks, dataset = _create_regression_dataset( + num_masks, + num_sources, + get_prompt_ids, + response_ids, + ablation_keep_prob, + base_seed=base_seed, + ) + logit_probs = _get_response_logit_probs( + dataset, model, tokenizer, len(response_ids), batch_size + ) + return masks, logit_probs.astype(np.float32) + + +def aggregate_logit_probs(logit_probs, output_type="logit_prob"): + """Compute sequence-level outputs from token-level logit-probabilities.""" + logit_probs = ch.tensor(logit_probs) + log_probs = ch.nn.functional.logsigmoid(logit_probs).sum(dim=1) + if output_type == "log_prob": + return log_probs.numpy() + elif output_type == "logit_prob": + log_1mprobs = ch.log1p(-ch.exp(log_probs)) + return (log_probs - log_1mprobs).numpy() + elif output_type == "total_token_logit_prob": + return logit_probs.mean(dim=1).numpy() + else: + raise ValueError(f"Cannot aggregate log probs for output type '{output_type}'") + + +def _color_scale(val, max_val): + start_color = (255, 255, 255) + end_color = (80, 180, 80) + if val == 0: + return f"background-color: rgb{start_color}" + elif val == max_val: + return f"background-color: rgb{end_color}" + else: + fraction = val / max_val + interpolated_color = tuple( + start_color[i] + (end_color[i] - start_color[i]) * fraction + for i in range(3) + ) + return f"background-color: rgb{interpolated_color}" + + +def _apply_color_scale(df): + # A score of np.log(10) means that the ablating this sources causes the + # logit-probability to drop by np.log(10), which (roughly) corresponds to + # a decrease in probability of 10x. + max_val = max([df["Score"].max(), np.log(10)]) + return df.style.applymap(lambda val: _color_scale(val, max_val), subset=["Score"]) + + +def get_attributions_df( + attributions: NDArray[Any], + context_partitioner, + top_k: Optional[int] = None, +) -> Any: + order = attributions.argsort()[::-1] + selected_attributions = [] + selected_sources = [] + + if top_k is not None: + order = order[:top_k] + + for i in order: + selected_attributions.append(attributions[i]) + selected_sources.append(context_partitioner.get_source(i)) + + df = pd.DataFrame.from_dict( + {"Score": selected_attributions, "Source": selected_sources} + ) + df = _apply_color_scale(df).format(precision=3) + return df + + +# The Llama 3 char_to_token is buggy (start and end chars for a given token +# are often the same), so we implement our own +def char_to_token(output_tokens, char_index): + for i in range(len(output_tokens["input_ids"]) - 1): + if char_index < output_tokens.token_to_chars(i + 1).start: + return i + return i + 1 diff --git a/example_result.json b/example_result.json new file mode 100644 index 0000000000000000000000000000000000000000..81836622f21fe8e6feabd880c6e4f6ffdee1aa2b --- /dev/null +++ b/example_result.json @@ -0,0 +1,15559 @@ +[ + { + "data": { + "question": "Who has the highest goals in world football?", + "answer": "The players with the highest all-time goals and highest men's and women's international football goals differ. The player with the highest all-time men's football goals is Josef Bican, who in 2020 was recognized by FIFA, the international governing body of football, as the record scorer with an estimated 805 goals. Christine Sinclair has the highest goals in women's international football with 187 and is the all-time leader for international goals scored for men or women. Cristiano Ronaldo and Ali Daei are currently tied for leading goalscorer in the history of men's international football with 109.", + "qa_pairs": [ + { + "context": "No context provided", + "question": "Who has the highest goals in men's world international football?", + "short_answers": [ + "Daei", + "Ali Daei" + ], + "wikipage": null + }, + { + "context": "No context provided", + "question": "Who has the highest goals all-time in men's football?", + "short_answers": [ + "Bican", + "Josef Bican" + ], + "wikipage": null + }, + { + "context": "The first player to reach 100 international goals was Italian Elisabetta Vignotto. Abby Wambach scored 100 goals in 9 years, while Christine Sinclair reached the milestone in just under 10 years while Mia Hamm is the youngest player to score 100 international goals at the age of 26 years 185 days. Most played exclusively in the forward position, with Kristine Lilly and Michelle Akers having also played as midfielder. All players scored at a high average rate of more than one goal every three matches. International goals in this list should not include goals scored in penalty-shoot-out; see Penalty shootout (association football). Players who are currently active at international level are indicated in bold type background.", + "question": "Who has the highest goals in women's world international football?", + "short_answers": [ + "Sinclair", + "Christine Sinclair" + ], + "wikipage": "List of women's footballers with 100 or more international goals ..." + } + ], + "docs": [ + "Document [1](Title:Argentina\u2013Brazil football rivalry): \"Football Player of the Century\", by IFFHS International Federation of Football History and Statistics, 1999, \"South America Football Player of the Century\", by IFFHS International Federation of Football History and Statistics. Pel\u00e9's 1281 goals are recognized by FIFA as the highest total achieved by a professional footballer, although the Soccer Statistic Foundation (rssf) recognizes only 767 goals in official mode, occupying the third place after Josef Bican (805) and Romario (772). For his part, Maradona has been named the best soccer player in World Cup history both by The Times and FourFourTwo, publication that also rewarded him as the \"Best\n", + "Document [2](Title:Godfrey Chitalu): have beaten Gerd M\u00fcller's record of 85 goals in a year, the Football Association of Zambia claimed that the world record actually pertained to Godfrey Chitalu who had scored 116 goals (possibly 117) during the 1972 calendar year and 107 during the 1972 season. The difference of goals is due to first 9 goals being scored before the season officially started. The Football Association of Zambia presented the evidence to FIFA but a spokesperson responded that they would ratify neither Lionel Messi's nor Chitalu's records as they do not keep statistical track of domestic competitions. Nonetheless, it could constitute the\n", + "Document [3](Title:Godfrey Chitalu): highest official tally claimed by a national football association. Chitalu made his international debut on 29 June 1968 in a friendly match against Uganda in Lusaka which Zambia won 2\u20131. He scored his first goal in a 2\u20132 draw against the same team five days later. Chitalu played a prominent role during the World Cup qualification matches against Sudan with Zambia being eliminated on a strange rule which was peculiar to Africa and favoured the team that won the second leg. Despite the aggregate score being tied at 6\u20136 after Zambia won the first leg 4\u20132 and lost the return\n", + "Document [4](Title:Wartan Ghazarian): goals (4 in World Cup qualifiers, 3 in Asian Cup qualifiers, 12 in friendlies). His record was later broken by Roda Antar, after Roda scored his 20th goal in 2018 FIFA World Cup qualification match against Laos. On 16 November 2008, during Round 6 of the Lebanese Football League, at the age of 39 years, Vartan scored his 130th goal in the Lebanese first division against Tadamon Tyre, becoming officially the highest all-time scorer in the history of Lebanese football. Some officials do not recognize the 12 goals he scored in the 2000\u20132001 season which was canceled. However, his remaining\n", + "Document [5](Title:Josef Bican): for Christmas, but died less than 2 weeks before that, at the age of 88. Josef Bican Josef \"Pepi\" Bican (25 September 1913 \u2013 12 December 2001) was a Czech-Austrian professional footballer who played as a striker. Rec.Sport.Soccer Statistics Foundation (RSSSF) estimates that he scored at least 805 goals in all competitive matches, which would make him the most prolific scorer of all time. Having scored a record 600 league goals and at least 1468 goals overall, the International Federation of Football History & Statistics (IFFHS) awarded Bican the \"Golden Ball\" as the greatest goalscorer of the last century. He\n" + ] + }, + "doc_cache": [ + "Document [1](Title:Humberto Suazo)announced his retirement from football on January 14, 2016. Suazo had also become a fixture with his international side. In 2006, he scored 17 goals in national and international matches, surpassing Peter Crouch by one goal for the \"World's Top Goal Scorer Award\". His four international goals all came in friendlies. He scored goals against New Zealand and Sweden. His other two goals came in the form of penalties versus Ivory Coast and Colombia. In January 2007, he was awarded world's top goal scorer of 2006 by the International Federation of Football History & Statistics in Salzburg, Austria. He was\n", + "Document [2](Title:FIFA Puska\u0301s Award)forward European football produced in first-division football, and scored a once world record of 84 goals in 85 internationals. He is one of the top scorers of the 20th century, with 512 goals in 528 matches. \"It is important to preserve the memory of those footballing greats who have left their mark on our history. Ferenc Pusk\u00e1s was not only a player with immense talent who won many honours, but also a remarkable man. FIFA is therefore delighted to pay tribute to him by dedicating this award to his memory,\" said Blatter at the inauguration of the award in Budapest.\n", + "Document [3](Title:FIFA Puska\u0301s Award)forward European football produced in first-division football, and scored a once world record of 84 goals in 85 internationals. He is one of the top scorers of the 20th century, with 512 goals in 528 matches. \"It is important to preserve the memory of those footballing greats who have left their mark on our history. Ferenc Pusk\u00e1s was not only a player with immense talent who won many honours, but also a remarkable man. FIFA is therefore delighted to pay tribute to him by dedicating this award to his memory,\" said Blatter at the inauguration of the award in Budapest.\n", + "Document [4](Title:Oktay Dereliog\u0306lu)Oktay Derelio\u011flu Oktay Derelio\u011flu (born 17 December 1975) is Turkish former international footballer and current football coach. Derelio\u011flu played as a forward and is notable for his goal against Belgium during the qualifying rounds of the 1998 FIFA World Cup. The goal started at midfield before Derelio\u011flu jinked past six Belgian defenders, one of them twice, mimicking Diego Maradona's \"Goal of the Century\". Derelio\u011flu holds the record for most goals in European competitions for Be\u015fikta\u015f with 14. Derelio\u011flu was born in Istanbul and began his club career with local club Fatih Karag\u00fcmr\u00fck. He made his debut at the age of\n", + "Document [5](Title:FIFA World Cup top goalscorers)Germany or West Germany. In total, 64 footballers came from UEFA (Europe), 29 from CONMEBOL (South America), and only four from elsewhere: Cameroon, Ghana, Australia, and the United States. Fontaine holds the record for the most goals scored at a single tournament, with 13 goals in 1958. The players that came closest to this record were Kocsis in 1954, M\u00fcller in 1970 and Portugal's Eus\u00e9bio in 1966, with 11, 10 and 9, respectively. The lowest scoring tournament's top scorer was in 1962, when six players tied at only four goals each. Across the 21 editions of the World Cup, 30\n", + "Document [6](Title:Argentina\u2013Brazil football rivalry)\"Football Player of the Century\", by IFFHS International Federation of Football History and Statistics, 1999, \"South America Football Player of the Century\", by IFFHS International Federation of Football History and Statistics. Pel\u00e9's 1281 goals are recognized by FIFA as the highest total achieved by a professional footballer, although the Soccer Statistic Foundation (rssf) recognizes only 767 goals in official mode, occupying the third place after Josef Bican (805) and Romario (772). For his part, Maradona has been named the best soccer player in World Cup history both by The Times and FourFourTwo, publication that also rewarded him as the \"Best\n" + ], + "log": [ + "gpt-3.5-turbo-[1] -> retriever-[4]\n: Query: \"Top goal scorers in international football history\"", + "retriever-[4] -> gpt-3.5-turbo-[3]\n: Document [1](Title:Humberto Suazo)announced his retirement from football on January 14, 2016. Suazo had also become a fixture with his international side. In 2006, he scored 17 goals in national and international matches, surpassing Peter Crouch by one goal for the \"World's Top Goal Scorer Award\". His four international goals all came in friendlies. He scored goals against New Zealand and Sweden. His other two goals came in the form of penalties versus Ivory Coast and Colombia. In January 2007, he was awarded world's top goal scorer of 2006 by the International Federation of Football History & Statistics in Salzburg, Austria. He was\n\nDocument [2](Title:FIFA Puska\u0301s Award)forward European football produced in first-division football, and scored a once world record of 84 goals in 85 internationals. He is one of the top scorers of the 20th century, with 512 goals in 528 matches. \"It is important to preserve the memory of those footballing greats who have left their mark on our history. Ferenc Pusk\u00e1s was not only a player with immense talent who won many honours, but also a remarkable man. FIFA is therefore delighted to pay tribute to him by dedicating this award to his memory,\" said Blatter at the inauguration of the award in Budapest.\n", + "gpt-3.5-turbo-[3] -> evaluator-[2]\n: ['Humberto Suazo was awarded the title of the world\\'s top goal scorer of 2006, surpassing Peter Crouch by one goal for the \"World\\'s Top Goal Scorer Award\"[1]. ', 'Ferenc Pusk\u00e1s holds the record for the highest number of goals in world football, scoring 84 goals in 85 internationals and a total of 512 goals in 528 matches[2]. ']", + "evaluator-[2] -> gpt-3.5-turbo-[1]\n: {'answer': 'Humberto Suazo was awarded the title of the world\\'s top goal scorer of 2006, surpassing Peter Crouch by one goal for the \"World\\'s Top Goal Scorer Award\"[1]. '}", + "gpt-3.5-turbo-[1] -> retriever-[4]\n: Query: \"Current leading goal scorers in top European football leagues\"", + "retriever-[4] -> gpt-3.5-turbo-[3]\n: Document [3](Title:FIFA Puska\u0301s Award)forward European football produced in first-division football, and scored a once world record of 84 goals in 85 internationals. He is one of the top scorers of the 20th century, with 512 goals in 528 matches. \"It is important to preserve the memory of those footballing greats who have left their mark on our history. Ferenc Pusk\u00e1s was not only a player with immense talent who won many honours, but also a remarkable man. FIFA is therefore delighted to pay tribute to him by dedicating this award to his memory,\" said Blatter at the inauguration of the award in Budapest.\n\nDocument [4](Title:Oktay Dereliog\u0306lu)Oktay Derelio\u011flu Oktay Derelio\u011flu (born 17 December 1975) is Turkish former international footballer and current football coach. Derelio\u011flu played as a forward and is notable for his goal against Belgium during the qualifying rounds of the 1998 FIFA World Cup. The goal started at midfield before Derelio\u011flu jinked past six Belgian defenders, one of them twice, mimicking Diego Maradona's \"Goal of the Century\". Derelio\u011flu holds the record for most goals in European competitions for Be\u015fikta\u015f with 14. Derelio\u011flu was born in Istanbul and began his club career with local club Fatih Karag\u00fcmr\u00fck. He made his debut at the age of\n", + "gpt-3.5-turbo-[3] -> evaluator-[2]\n: ['Ferenc Pusk\u00e1s holds the once world record of scoring 84 goals in 85 internationals in European football, making him one of the top scorers of the 20th century[3]. ', 'Oktay Derelio\u011flu holds the record for most goals in European competitions for Be\u015fikta\u015f with 14[4]. ']", + "evaluator-[2] -> gpt-3.5-turbo-[1]\n: {'answer': 'Ferenc Pusk\u00e1s holds the once world record of scoring 84 goals in 85 internationals in European football, making him one of the top scorers of the 20th century[3]. '}", + "gpt-3.5-turbo-[1] -> retriever-[4]\n: Query: \"Players with most goals in FIFA World Cup history\"", + "retriever-[4] -> gpt-3.5-turbo-[3]\n: Document [5](Title:FIFA World Cup top goalscorers)Germany or West Germany. In total, 64 footballers came from UEFA (Europe), 29 from CONMEBOL (South America), and only four from elsewhere: Cameroon, Ghana, Australia, and the United States. Fontaine holds the record for the most goals scored at a single tournament, with 13 goals in 1958. The players that came closest to this record were Kocsis in 1954, M\u00fcller in 1970 and Portugal's Eus\u00e9bio in 1966, with 11, 10 and 9, respectively. The lowest scoring tournament's top scorer was in 1962, when six players tied at only four goals each. Across the 21 editions of the World Cup, 30\n\nDocument [6](Title:Argentina\u2013Brazil football rivalry)\"Football Player of the Century\", by IFFHS International Federation of Football History and Statistics, 1999, \"South America Football Player of the Century\", by IFFHS International Federation of Football History and Statistics. Pel\u00e9's 1281 goals are recognized by FIFA as the highest total achieved by a professional footballer, although the Soccer Statistic Foundation (rssf) recognizes only 767 goals in official mode, occupying the third place after Josef Bican (805) and Romario (772). For his part, Maradona has been named the best soccer player in World Cup history both by The Times and FourFourTwo, publication that also rewarded him as the \"Best\n", + "gpt-3.5-turbo-[3] -> evaluator-[2]\n: ['At the FIFA World Cup, Just Fontaine holds the record for the most goals scored at a single tournament with 13 goals in 1958[5]. ', 'Pel\u00e9 holds the record for the highest number of goals in world football, with 1281 goals recognized by FIFA[6]. ']", + "evaluator-[2] -> gpt-3.5-turbo-[1]\n: {'answer': 'At the FIFA World Cup, Just Fontaine holds the record for the most goals scored at a single tournament with 13 goals in 1958[5]. '}" + ], + "output": [ + "Humberto Suazo was awarded the title of the world's top goal scorer of 2006, surpassing Peter Crouch by one goal for the \"World's Top Goal Scorer Award\" [1].", + "Ferenc Pusk\u00e1s holds the once world record of scoring 84 goals in 85 internationals in European football, making him one of the top scorers of the 20th century [3]. ", + "At the FIFA World Cup, Just Fontaine holds the record for the most goals scored at a single tournament with 13 goals in 1958 [5]. " + ], + "result": { + "length": 82 + }, + "token_used": 4774, + "doc_level": [ + 4.18450927734375, + 1.68487548828125, + 1.68487548828125, + 4.11444091796875, + 1.137939453125, + -9.610931396484375 + ], + "span_level": [ + [ + [ + [ + 34, + 0.9752396407005616 + ], + [ + 93, + 0.8480042981883524 + ], + [ + 154, + 0.3376145269353801 + ], + [ + 298, + 0.11908060659579556 + ], + [ + 351, + 0.3241283355659681 + ], + [ + 399, + 0.9765835857569739 + ], + [ + 482, + 0.2033701772798625 + ], + [ + 633, + 0.014042915539730028 + ] + ], + [ + [ + 38, + 0.8493001101032331 + ], + [ + 165, + 0.03312713523794175 + ], + [ + 246, + 0.08075571253349467 + ], + [ + 355, + 0.9898912949824789 + ], + [ + 460, + 0.8904421041367327 + ], + [ + 610, + 0.38480494719036606 + ] + ], + [ + [ + 38, + 0.29215553448463405 + ], + [ + 165, + 0.12807777408352983 + ], + [ + 246, + 0.06531646698441584 + ], + [ + 355, + 0.11867147295547309 + ], + [ + 460, + 0.5567021984669929 + ], + [ + 610, + 0.769231598164667 + ] + ], + [ + [ + 37, + 0.6135003633724169 + ], + [ + 165, + 0.48081133019211375 + ], + [ + 297, + 0.20916909199233424 + ], + [ + 448, + 0.7491566612724427 + ], + [ + 538, + 0.6739085072731161 + ], + [ + 630, + 0.7128352408130009 + ] + ], + [ + [ + 50, + 0.841831548325789 + ], + [ + 74, + 0.1593109111401093 + ], + [ + 239, + 0.9448484481403537 + ], + [ + 338, + 0.27484095387772056 + ], + [ + 484, + 0.019643740514884156 + ], + [ + 587, + 0.35951595078893017 + ] + ], + [ + [ + 53, + 0.7208818124952807 + ], + [ + 280, + 0.33648496430693486 + ], + [ + 542, + 0.28298701121624115 + ] + ] + ], + [ + [ + [ + 34, + 0.32801372770469195 + ], + [ + 93, + 0.3282807482104053 + ], + [ + 154, + 0.19525296544159376 + ], + [ + 298, + 0.31462690471047305 + ], + [ + 351, + 0.8780142232937335 + ], + [ + 399, + 0.5121432207142949 + ], + [ + 482, + 0.25737530987754453 + ], + [ + 633, + 0.17070633532620094 + ] + ], + [ + [ + 38, + 0.7800650772989878 + ], + [ + 165, + 0.751948512683764 + ], + [ + 246, + 0.41083403364590065 + ], + [ + 355, + 0.4771115777641406 + ], + [ + 460, + 0.810289699661572 + ], + [ + 610, + 0.37181088824095476 + ] + ], + [ + [ + 38, + 0.5979186460202017 + ], + [ + 165, + 0.7910254693757455 + ], + [ + 246, + 0.2274304127360015 + ], + [ + 355, + 0.36624432828148545 + ], + [ + 460, + 0.9274672849162501 + ], + [ + 610, + 0.5804309247346146 + ] + ], + [ + [ + 37, + 0.37758393784304867 + ], + [ + 165, + 0.6970956771842183 + ], + [ + 297, + 0.7330666086878526 + ], + [ + 448, + 0.8237479033231114 + ], + [ + 538, + 0.44881986527093976 + ], + [ + 630, + 0.9568261717729746 + ] + ], + [ + [ + 50, + 0.8227421675305857 + ], + [ + 74, + 0.8748392816298317 + ], + [ + 239, + 0.20195965051336318 + ], + [ + 338, + 0.4678488458434905 + ], + [ + 484, + 0.7803994731054034 + ], + [ + 587, + 0.45935028011585455 + ] + ], + [ + [ + 53, + 0.04659832953061516 + ], + [ + 280, + 0.40258405550994536 + ], + [ + 542, + 0.7387206933659894 + ] + ] + ], + [ + [ + [ + 34, + 0.9311796538221773 + ], + [ + 93, + 0.09802311563476718 + ], + [ + 154, + 0.1841608640590997 + ], + [ + 298, + 0.21184359592076385 + ], + [ + 351, + 0.7484600515692544 + ], + [ + 399, + 0.7919269234079847 + ], + [ + 482, + 0.3311662245548356 + ], + [ + 633, + 0.8470746844279172 + ] + ], + [ + [ + 38, + 0.05761980918988596 + ], + [ + 165, + 0.014965766885068899 + ], + [ + 246, + 0.6398679520746595 + ], + [ + 355, + 0.662434743823984 + ], + [ + 460, + 0.9793310465832624 + ], + [ + 610, + 0.31189926257479517 + ] + ], + [ + [ + 38, + 0.8506306128929363 + ], + [ + 165, + 0.4719436802640077 + ], + [ + 246, + 0.7503823882284352 + ], + [ + 355, + 0.9476525124608852 + ], + [ + 460, + 0.42555540314000206 + ], + [ + 610, + 0.31139389580505994 + ] + ], + [ + [ + 37, + 0.4908693853907372 + ], + [ + 165, + 0.4647473222376487 + ], + [ + 297, + 0.3512389954420536 + ], + [ + 448, + 0.4303600907067554 + ], + [ + 538, + 0.7533081385975575 + ], + [ + 630, + 0.8350390947391708 + ] + ], + [ + [ + 50, + 0.3629466053000354 + ], + [ + 74, + 0.47486379653628497 + ], + [ + 239, + 0.43182660962066555 + ], + [ + 338, + 0.8070052354697811 + ], + [ + 484, + 0.6039868781551592 + ], + [ + 587, + 0.3578713826249794 + ] + ], + [ + [ + 53, + 0.2561411484400733 + ], + [ + 280, + 0.7442471914015426 + ], + [ + 542, + 0.9008671728122173 + ] + ] + ] + ], + "word_level": [ + [ + [ + [ + 34, + 0.9726499810603795 + ], + [ + 44, + 0.08605671020733052 + ], + [ + 48, + 0.7596409861809017 + ], + [ + 59, + 0.05763882612999671 + ], + [ + 64, + 0.5145239051825758 + ], + [ + 73, + 0.45700032932647927 + ], + [ + 76, + 0.16541894551748604 + ], + [ + 84, + 0.5532297599051686 + ], + [ + 88, + 0.43057656720854987 + ], + [ + 94, + 0.14214088498206523 + ], + [ + 100, + 0.8758168922684791 + ], + [ + 104, + 0.20081190189567377 + ], + [ + 109, + 0.9356991116217385 + ], + [ + 116, + 0.7219574313352661 + ], + [ + 118, + 0.8708936103353166 + ], + [ + 126, + 0.09109601532609823 + ], + [ + 131, + 0.42478689455399443 + ], + [ + 135, + 0.9532934971578741 + ], + [ + 149, + 0.9217453358083582 + ], + [ + 155, + 0.7332662377274012 + ], + [ + 158, + 0.8796839520635138 + ], + [ + 164, + 0.6275774828812761 + ], + [ + 167, + 0.6734740068382253 + ], + [ + 174, + 0.6371931070574532 + ], + [ + 177, + 0.13655041255713818 + ], + [ + 183, + 0.28302491354819126 + ], + [ + 186, + 0.8787614548337237 + ], + [ + 195, + 0.9899222403661093 + ], + [ + 199, + 0.6945864751528461 + ], + [ + 213, + 0.9552826041096931 + ], + [ + 222, + 0.583279356201901 + ], + [ + 233, + 0.9941956623758189 + ], + [ + 239, + 0.48195696524188913 + ], + [ + 246, + 0.6677641280707589 + ], + [ + 249, + 0.7327067274655747 + ], + [ + 253, + 0.640038487613746 + ], + [ + 258, + 0.6762132799133805 + ], + [ + 262, + 0.5008711689992287 + ], + [ + 266, + 0.8340187102575166 + ], + [ + 275, + 0.8529102970249229 + ], + [ + 279, + 0.8697714238530159 + ], + [ + 284, + 0.2753275960783972 + ], + [ + 291, + 0.19085553946188516 + ], + [ + 299, + 0.4304598693894228 + ], + [ + 303, + 0.7311499877171561 + ], + [ + 308, + 0.1983888594019283 + ], + [ + 322, + 0.6189171026425311 + ], + [ + 328, + 0.2633704071272944 + ], + [ + 332, + 0.7764581833090596 + ], + [ + 337, + 0.0047831781014120445 + ], + [ + 340, + 0.9517417934051873 + ], + [ + 352, + 0.17670655740992414 + ], + [ + 355, + 0.5730771523543212 + ], + [ + 362, + 0.27374706562798024 + ], + [ + 368, + 0.30172016092127685 + ], + [ + 376, + 0.9076941653477242 + ], + [ + 380, + 0.7007656495300092 + ], + [ + 388, + 0.9118009978077177 + ], + [ + 392, + 0.6101883977635021 + ], + [ + 400, + 0.9372889641903638 + ], + [ + 404, + 0.8401677216195338 + ], + [ + 410, + 0.6423756646375247 + ], + [ + 414, + 0.051317570730764706 + ], + [ + 420, + 0.04104655183006123 + ], + [ + 425, + 0.6447578663016221 + ], + [ + 428, + 0.9253858884115785 + ], + [ + 432, + 0.06469230538917103 + ], + [ + 437, + 0.7059482176331049 + ], + [ + 440, + 0.1979618935792936 + ], + [ + 450, + 0.7946809861922456 + ], + [ + 457, + 0.3861910212203763 + ], + [ + 463, + 0.596413727876745 + ], + [ + 469, + 0.678685363839173 + ], + [ + 473, + 0.49727031812918976 + ], + [ + 483, + 0.8449105859428789 + ], + [ + 486, + 0.4863451116791887 + ], + [ + 494, + 0.4022791995796927 + ], + [ + 500, + 0.738801668038707 + ], + [ + 503, + 0.3752619227763957 + ], + [ + 507, + 0.48756856950671845 + ], + [ + 515, + 0.23666096661853975 + ], + [ + 523, + 0.9717143834132929 + ], + [ + 527, + 0.3015668769856851 + ], + [ + 532, + 0.4106303810200229 + ], + [ + 539, + 0.8240988498744837 + ], + [ + 542, + 0.5130940235821316 + ], + [ + 547, + 0.8835523732699343 + ], + [ + 550, + 0.5091943652363315 + ], + [ + 554, + 0.8363470240292314 + ], + [ + 568, + 0.8422641425363049 + ], + [ + 579, + 0.16864565697441403 + ], + [ + 582, + 0.4769481421525644 + ], + [ + 591, + 0.97665055636654 + ], + [ + 599, + 0.2481313029375718 + ], + [ + 601, + 0.435512214938918 + ], + [ + 612, + 0.3262569046900523 + ], + [ + 615, + 0.3887858238150872 + ], + [ + 625, + 0.4116203910375821 + ], + [ + 634, + 0.20804166314868577 + ], + [ + 637, + 0.998009136720274 + ] + ], + [ + [ + 38, + 0.01895191794983786 + ], + [ + 46, + 0.8091549259035867 + ], + [ + 55, + 0.22371531154224544 + ], + [ + 64, + 0.04170305585498946 + ], + [ + 73, + 0.7577407985746973 + ], + [ + 76, + 0.5606045725204206 + ], + [ + 91, + 0.2021533517584695 + ], + [ + 101, + 0.9593754788838803 + ], + [ + 105, + 0.5116245004131195 + ], + [ + 112, + 0.3831201417660244 + ], + [ + 114, + 0.3527125414730288 + ], + [ + 119, + 0.7386062631099651 + ], + [ + 125, + 0.7990031572361751 + ], + [ + 132, + 0.6375414755678795 + ], + [ + 135, + 0.1986568214797113 + ], + [ + 138, + 0.2990939923338106 + ], + [ + 144, + 0.042350902993752104 + ], + [ + 147, + 0.7923706775051015 + ], + [ + 150, + 0.5650679729269246 + ], + [ + 166, + 0.1876496745408287 + ], + [ + 169, + 0.9566224077010641 + ], + [ + 172, + 0.09018415352300413 + ], + [ + 176, + 0.565513881421995 + ], + [ + 179, + 0.1958777987756306 + ], + [ + 183, + 0.2778699853420834 + ], + [ + 187, + 0.42248651482377897 + ], + [ + 195, + 0.4950370421279131 + ], + [ + 198, + 0.09944192654106454 + ], + [ + 202, + 0.8355734164643888 + ], + [ + 207, + 0.849422416524635 + ], + [ + 216, + 0.9025131530244342 + ], + [ + 221, + 0.2246783253758975 + ], + [ + 225, + 0.6567175368851491 + ], + [ + 231, + 0.7061986337567662 + ], + [ + 234, + 0.9172364160432521 + ], + [ + 238, + 0.9291620251673874 + ], + [ + 247, + 0.35835364150186755 + ], + [ + 251, + 0.19806664327112145 + ], + [ + 254, + 0.817528899263466 + ], + [ + 264, + 0.6818162944034358 + ], + [ + 267, + 0.3471461344822452 + ], + [ + 276, + 0.24402369364302656 + ], + [ + 280, + 0.15965250543770604 + ], + [ + 287, + 0.11910002417525267 + ], + [ + 290, + 0.8414386419662918 + ], + [ + 296, + 0.3693408189901478 + ], + [ + 308, + 0.03007754373349747 + ], + [ + 315, + 0.7652163147590383 + ], + [ + 319, + 0.9630074508590972 + ], + [ + 324, + 0.06632505255957166 + ], + [ + 329, + 0.6930792074511971 + ], + [ + 335, + 0.41169847826129946 + ], + [ + 340, + 0.089983276967049 + ], + [ + 343, + 0.9221129577424041 + ], + [ + 347, + 0.024352377304205364 + ], + [ + 356, + 0.8363505282373193 + ], + [ + 363, + 0.7775150374476623 + ], + [ + 370, + 0.0069677471496412124 + ], + [ + 374, + 0.19400039836301552 + ], + [ + 378, + 0.13912023964891063 + ], + [ + 383, + 0.4061422221890222 + ], + [ + 385, + 0.6396258935475556 + ], + [ + 392, + 0.15742931839217533 + ], + [ + 397, + 0.2671888605480571 + ], + [ + 405, + 0.5029253150782247 + ], + [ + 412, + 0.7820817314198254 + ], + [ + 416, + 0.9298850123676912 + ], + [ + 420, + 0.730881373753405 + ], + [ + 425, + 0.15083904029380713 + ], + [ + 434, + 0.11458460764916611 + ], + [ + 438, + 0.8203304115935969 + ], + [ + 443, + 0.05505432706003832 + ], + [ + 445, + 0.18991814324255174 + ], + [ + 456, + 0.789323812722833 + ], + [ + 461, + 0.08249958198216845 + ], + [ + 466, + 0.7474939904482418 + ], + [ + 469, + 0.2321942352038885 + ], + [ + 479, + 0.6523289419822869 + ], + [ + 489, + 0.042554905862500436 + ], + [ + 492, + 0.6484626644520137 + ], + [ + 496, + 0.09481440010070763 + ], + [ + 504, + 0.7484695642032869 + ], + [ + 507, + 0.7946473942871619 + ], + [ + 511, + 0.7196758040157935 + ], + [ + 514, + 0.20146651780318037 + ], + [ + 525, + 0.2566481277219588 + ], + [ + 530, + 0.516785834040892 + ], + [ + 536, + 0.9657973341980828 + ], + [ + 539, + 0.16493605065462802 + ], + [ + 543, + 0.24896831106607964 + ], + [ + 552, + 0.133105467995994 + ], + [ + 557, + 0.4495983478008049 + ], + [ + 565, + 0.08028646005213202 + ], + [ + 568, + 0.7061667025527271 + ], + [ + 572, + 0.46326981753103535 + ], + [ + 585, + 0.40632475728473716 + ], + [ + 588, + 0.22393808944445026 + ], + [ + 592, + 0.2877759699302114 + ], + [ + 598, + 0.14091002413323384 + ], + [ + 601, + 0.4754504181304122 + ] + ], + [ + [ + 38, + 0.9794652317785165 + ], + [ + 46, + 0.7592533367173389 + ], + [ + 55, + 0.15029712452277344 + ], + [ + 64, + 0.7262454247566498 + ], + [ + 73, + 0.5179997577367508 + ], + [ + 76, + 0.08462317188116808 + ], + [ + 91, + 0.5267665429059917 + ], + [ + 101, + 0.2992592195180812 + ], + [ + 105, + 0.4133018610033953 + ], + [ + 112, + 0.942428206399702 + ], + [ + 114, + 0.7781863816223291 + ], + [ + 119, + 0.38582786134814384 + ], + [ + 125, + 0.5156602790786449 + ], + [ + 132, + 0.8926193689742762 + ], + [ + 135, + 0.5488150895385591 + ], + [ + 138, + 0.6737410553077283 + ], + [ + 144, + 0.5398186070880724 + ], + [ + 147, + 0.5152492017639415 + ], + [ + 150, + 0.3901711046270677 + ], + [ + 166, + 0.029766350051515733 + ], + [ + 169, + 0.20663417157776365 + ], + [ + 172, + 0.7288940548079337 + ], + [ + 176, + 0.7344787705705658 + ], + [ + 179, + 0.6229041369512515 + ], + [ + 183, + 0.9260335313847975 + ], + [ + 187, + 0.6477851651074203 + ], + [ + 195, + 0.4076175883589801 + ], + [ + 198, + 0.5767604149545458 + ], + [ + 202, + 0.20272054804405792 + ], + [ + 207, + 0.42307064574956366 + ], + [ + 216, + 0.6422707404844868 + ], + [ + 221, + 0.5261544740747952 + ], + [ + 225, + 0.48358792850477894 + ], + [ + 231, + 0.2615922892969095 + ], + [ + 234, + 0.1089786668351792 + ], + [ + 238, + 0.6399359365157113 + ], + [ + 247, + 0.7151429307962669 + ], + [ + 251, + 0.6894405827717566 + ], + [ + 254, + 0.05739548760055557 + ], + [ + 264, + 0.3883109323975241 + ], + [ + 267, + 0.9803369286933918 + ], + [ + 276, + 0.20182822729204963 + ], + [ + 280, + 0.5334362762069331 + ], + [ + 287, + 0.2603792884306666 + ], + [ + 290, + 0.02265491900456451 + ], + [ + 296, + 0.5723929074083922 + ], + [ + 308, + 0.8290600915093783 + ], + [ + 315, + 0.6556228427425396 + ], + [ + 319, + 0.22863902437526407 + ], + [ + 324, + 0.41073426672828317 + ], + [ + 329, + 0.693002119124838 + ], + [ + 335, + 0.30202850028750283 + ], + [ + 340, + 0.8445374136110521 + ], + [ + 343, + 0.08917522423474677 + ], + [ + 347, + 0.5714662784385183 + ], + [ + 356, + 0.2768355861289127 + ], + [ + 363, + 0.08349113497294103 + ], + [ + 370, + 0.05071155820628903 + ], + [ + 374, + 0.753990553339965 + ], + [ + 378, + 0.8917792065894531 + ], + [ + 383, + 0.3229700214611694 + ], + [ + 385, + 0.6017303429942582 + ], + [ + 392, + 0.9433062412100356 + ], + [ + 397, + 0.7093642077812163 + ], + [ + 405, + 0.4387203468807386 + ], + [ + 412, + 0.9137111697964068 + ], + [ + 416, + 0.5185792052653937 + ], + [ + 420, + 0.23895882737279361 + ], + [ + 425, + 0.3789222896282899 + ], + [ + 434, + 0.3383545103692106 + ], + [ + 438, + 0.6237546870972888 + ], + [ + 443, + 0.4478093437008268 + ], + [ + 445, + 0.8730950451877334 + ], + [ + 456, + 0.9843639356651239 + ], + [ + 461, + 0.6164389340883191 + ], + [ + 466, + 0.4512979468735716 + ], + [ + 469, + 0.35787370575737076 + ], + [ + 479, + 0.1813561722378061 + ], + [ + 489, + 0.7023033967532876 + ], + [ + 492, + 0.23272337126755438 + ], + [ + 496, + 0.6075361189595367 + ], + [ + 504, + 0.9358718504466712 + ], + [ + 507, + 0.6826051938380512 + ], + [ + 511, + 0.01681792279553651 + ], + [ + 514, + 0.6038252851495565 + ], + [ + 525, + 0.8422884785841935 + ], + [ + 530, + 0.5711223845833082 + ], + [ + 536, + 0.34628068509472 + ], + [ + 539, + 0.6634623105289472 + ], + [ + 543, + 0.6520574099131884 + ], + [ + 552, + 0.023118082819996943 + ], + [ + 557, + 0.08254306442643256 + ], + [ + 565, + 0.7991224776739679 + ], + [ + 568, + 0.06812939578506794 + ], + [ + 572, + 0.6933946560110053 + ], + [ + 585, + 0.7751816211920642 + ], + [ + 588, + 0.4719612811355067 + ], + [ + 592, + 0.6385257568754354 + ], + [ + 598, + 0.2027583817092793 + ], + [ + 601, + 0.5310095810091856 + ] + ], + [ + [ + 37, + 0.6786180923777138 + ], + [ + 43, + 0.7356796981712884 + ], + [ + 54, + 0.3122624183076076 + ], + [ + 60, + 0.6960494822400738 + ], + [ + 71, + 0.15055360315844535 + ], + [ + 77, + 0.4650389443155072 + ], + [ + 80, + 0.3238184922255305 + ], + [ + 89, + 0.7516940222095565 + ], + [ + 95, + 0.04539187097069175 + ], + [ + 98, + 0.9652899114059557 + ], + [ + 106, + 0.5557162320991196 + ], + [ + 113, + 0.93032680725156 + ], + [ + 127, + 0.567558913809352 + ], + [ + 138, + 0.5018757460910768 + ], + [ + 142, + 0.39583999956108173 + ], + [ + 150, + 0.18811390405119 + ], + [ + 159, + 0.5526108608245548 + ], + [ + 166, + 0.5607599981278294 + ], + [ + 177, + 0.639715257789935 + ], + [ + 184, + 0.2866673539359168 + ], + [ + 187, + 0.12031800704191531 + ], + [ + 189, + 0.9801948217129368 + ], + [ + 197, + 0.5936832049036366 + ], + [ + 201, + 0.5597963379805618 + ], + [ + 204, + 0.06429644050515182 + ], + [ + 212, + 0.7830986671175035 + ], + [ + 216, + 0.10297368806592178 + ], + [ + 220, + 0.009071092071019304 + ], + [ + 225, + 0.395915278184511 + ], + [ + 233, + 0.9618035411308056 + ], + [ + 241, + 0.5271786312897606 + ], + [ + 248, + 0.8895340283488209 + ], + [ + 252, + 0.2352986752761781 + ], + [ + 263, + 0.6123150016530776 + ], + [ + 270, + 0.3841032651074002 + ], + [ + 273, + 0.7839371360987384 + ], + [ + 277, + 0.17580201465906675 + ], + [ + 282, + 0.535494650333246 + ], + [ + 287, + 0.8418923054436044 + ], + [ + 293, + 0.9786186844928716 + ], + [ + 298, + 0.14803181622976813 + ], + [ + 302, + 0.9855370806350663 + ], + [ + 307, + 0.7433888801183093 + ], + [ + 315, + 0.34361312415503853 + ], + [ + 318, + 0.042822262077160556 + ], + [ + 327, + 0.352962714409533 + ], + [ + 334, + 0.8531925329491449 + ], + [ + 345, + 0.4332054261440911 + ], + [ + 352, + 0.11664314069391213 + ], + [ + 357, + 0.9806071645369893 + ], + [ + 361, + 0.8410292477122081 + ], + [ + 369, + 0.24147037549446226 + ], + [ + 380, + 0.7431827362279382 + ], + [ + 384, + 0.7586627544696628 + ], + [ + 387, + 0.31432726090688146 + ], + [ + 392, + 0.9215318718781551 + ], + [ + 399, + 0.6066644289228255 + ], + [ + 409, + 0.3426301344417091 + ], + [ + 415, + 0.15318413972607248 + ], + [ + 426, + 0.9936992737784514 + ], + [ + 432, + 0.9168614614913766 + ], + [ + 435, + 0.410322332376718 + ], + [ + 439, + 0.29025666713439335 + ], + [ + 449, + 0.2656483776458468 + ], + [ + 460, + 0.7939536573715431 + ], + [ + 466, + 0.23925041705171124 + ], + [ + 470, + 0.7643591618713277 + ], + [ + 477, + 0.5153722370211955 + ], + [ + 481, + 0.6415445619502043 + ], + [ + 486, + 0.5384996847469026 + ], + [ + 492, + 0.29896405132460746 + ], + [ + 495, + 0.7307102179473692 + ], + [ + 504, + 0.14217340812378376 + ], + [ + 517, + 0.003339393350253017 + ], + [ + 521, + 0.45156981921232253 + ], + [ + 530, + 0.7355754723301862 + ], + [ + 535, + 0.19204499683207 + ], + [ + 539, + 0.5058963573199146 + ], + [ + 550, + 0.07530362469826835 + ], + [ + 554, + 0.13473159760964692 + ], + [ + 559, + 0.8842680623215865 + ], + [ + 562, + 0.4003265968065788 + ], + [ + 571, + 0.2500603578159315 + ], + [ + 575, + 0.8857690385941768 + ], + [ + 581, + 0.6726743051429795 + ], + [ + 585, + 0.8563962875675292 + ], + [ + 590, + 0.6757159081925681 + ], + [ + 597, + 0.7585621678845341 + ], + [ + 602, + 0.42496674443286775 + ], + [ + 608, + 0.36906409880961966 + ], + [ + 613, + 0.45559683377485727 + ], + [ + 619, + 0.3097425746420145 + ], + [ + 631, + 0.007828954910206853 + ], + [ + 634, + 0.7712049790122333 + ], + [ + 639, + 0.9756689855037052 + ], + [ + 643, + 0.6565913655130314 + ], + [ + 649, + 0.3273860558570758 + ], + [ + 652, + 0.08695398500152751 + ], + [ + 656, + 0.11115207992931708 + ], + [ + 660, + 0.21888978587399288 + ] + ], + [ + [ + 50, + 0.4181276173985091 + ], + [ + 58, + 0.22358256875618876 + ], + [ + 61, + 0.8428357644094672 + ], + [ + 66, + 0.07895283485215976 + ], + [ + 75, + 0.832648782274282 + ], + [ + 78, + 0.8449929241209239 + ], + [ + 85, + 0.6584257332091348 + ], + [ + 88, + 0.17492752108361387 + ], + [ + 100, + 0.09675999690622195 + ], + [ + 105, + 0.7716554892649223 + ], + [ + 110, + 0.7115584342294312 + ], + [ + 115, + 0.11527010768059043 + ], + [ + 125, + 0.4732062209830158 + ], + [ + 128, + 0.15341365678925434 + ], + [ + 133, + 0.14633899700124475 + ], + [ + 142, + 0.5347904100729775 + ], + [ + 149, + 0.7684896594208711 + ], + [ + 159, + 0.079850026024186 + ], + [ + 163, + 0.38816043411066536 + ], + [ + 168, + 0.6818105453267002 + ], + [ + 173, + 0.8252105470104025 + ], + [ + 178, + 0.4908887946599344 + ], + [ + 189, + 0.5065778347092146 + ], + [ + 199, + 0.6348193683334751 + ], + [ + 206, + 0.21549879074603184 + ], + [ + 217, + 0.7687890049662857 + ], + [ + 221, + 0.5725686535030459 + ], + [ + 225, + 0.3836186927038474 + ], + [ + 232, + 0.6006526539962808 + ], + [ + 240, + 0.045687993584656694 + ], + [ + 249, + 0.4398687287162385 + ], + [ + 255, + 0.7340205160386013 + ], + [ + 259, + 0.3331977478453423 + ], + [ + 266, + 0.8646282116020737 + ], + [ + 270, + 0.4868806585463368 + ], + [ + 274, + 0.5922812386454627 + ], + [ + 279, + 0.021955731053276528 + ], + [ + 285, + 0.5937530491539885 + ], + [ + 292, + 0.2073257803631705 + ], + [ + 295, + 0.11047876430716452 + ], + [ + 297, + 0.6780533483992277 + ], + [ + 304, + 0.5338115459008669 + ], + [ + 316, + 0.6673120036770543 + ], + [ + 321, + 0.6753018690987888 + ], + [ + 324, + 0.40200274252480783 + ], + [ + 330, + 0.8498680485854615 + ], + [ + 333, + 0.7631680250486181 + ], + [ + 339, + 0.21364832955951873 + ], + [ + 343, + 0.7119872680988123 + ], + [ + 351, + 0.9161671862600897 + ], + [ + 356, + 0.9298029467253388 + ], + [ + 361, + 0.010687990855487883 + ], + [ + 369, + 0.9283025539274666 + ], + [ + 372, + 0.4340117187969056 + ], + [ + 377, + 0.8900909042273508 + ], + [ + 384, + 0.2138711682285921 + ], + [ + 389, + 0.2579528792821434 + ], + [ + 396, + 0.17704807663823663 + ], + [ + 399, + 0.39858301459241097 + ], + [ + 405, + 0.3025286136075479 + ], + [ + 412, + 0.3443583794015198 + ], + [ + 415, + 0.17733489469957564 + ], + [ + 420, + 0.8249570400727799 + ], + [ + 424, + 0.39554238549450427 + ], + [ + 435, + 0.018804878657817103 + ], + [ + 443, + 0.1666952078023889 + ], + [ + 446, + 0.5287648550754626 + ], + [ + 452, + 0.2517074888144806 + ], + [ + 457, + 0.18380530314453425 + ], + [ + 461, + 0.8352231114666506 + ], + [ + 464, + 0.07630747586006958 + ], + [ + 468, + 0.3774003546970661 + ], + [ + 471, + 0.8875923306116066 + ], + [ + 485, + 0.15362065249011259 + ], + [ + 489, + 0.016613189077949952 + ], + [ + 496, + 0.2957986416693388 + ], + [ + 504, + 0.8646000106947709 + ], + [ + 517, + 0.07017434826811253 + ], + [ + 521, + 0.5167609162738179 + ], + [ + 528, + 0.7761046244708356 + ], + [ + 532, + 0.9736986047136892 + ], + [ + 535, + 0.24459705279663302 + ], + [ + 541, + 0.9699143521402658 + ], + [ + 546, + 0.08631147057753963 + ], + [ + 550, + 0.9329536461710488 + ], + [ + 558, + 0.4029094662189612 + ], + [ + 563, + 0.48950213814290533 + ], + [ + 566, + 0.5720925927347997 + ], + [ + 571, + 0.06653739284208882 + ], + [ + 576, + 0.9369613636588959 + ], + [ + 582, + 0.6335837590905603 + ], + [ + 588, + 0.4927572074779326 + ], + [ + 595, + 0.743473566610875 + ], + [ + 599, + 0.08764534745472508 + ], + [ + 602, + 0.33164384004392455 + ], + [ + 611, + 0.7018519136829592 + ], + [ + 614, + 0.3332136079757716 + ], + [ + 618, + 0.9181048304139274 + ], + [ + 624, + 0.029015313742146498 + ], + [ + 629, + 0.5887503014423027 + ] + ], + [ + [ + 53, + 0.7098624436551949 + ], + [ + 63, + 0.27350597663329 + ], + [ + 70, + 0.3778543171535603 + ], + [ + 73, + 0.6788789518251402 + ], + [ + 77, + 0.10726250942979199 + ], + [ + 87, + 0.8394934840325272 + ], + [ + 90, + 0.13144659671593306 + ], + [ + 96, + 0.14916667941771888 + ], + [ + 110, + 0.6081108713582365 + ], + [ + 121, + 0.7754722094408412 + ], + [ + 124, + 0.048850174508976885 + ], + [ + 133, + 0.9249341678422579 + ], + [ + 141, + 0.0806650159535468 + ], + [ + 145, + 0.8178204511915984 + ], + [ + 157, + 0.23806659840467403 + ], + [ + 163, + 0.9636346881028524 + ], + [ + 170, + 0.7348125410146231 + ], + [ + 178, + 0.695970320773431 + ], + [ + 187, + 0.2683538153629337 + ], + [ + 194, + 0.027531170222428236 + ], + [ + 197, + 0.9897871581832495 + ], + [ + 201, + 0.9349219621561988 + ], + [ + 211, + 0.20826974781683527 + ], + [ + 214, + 0.40830153079211073 + ], + [ + 220, + 0.9522543601443701 + ], + [ + 234, + 0.48722825670149117 + ], + [ + 245, + 0.2189698220892068 + ], + [ + 248, + 0.7894018204012684 + ], + [ + 257, + 0.11658783085727709 + ], + [ + 265, + 0.7612179589301148 + ], + [ + 269, + 0.25687250457509336 + ], + [ + 281, + 0.7915486959101194 + ], + [ + 288, + 0.6620038717870498 + ], + [ + 293, + 0.17793247221924535 + ], + [ + 299, + 0.8874161227734442 + ], + [ + 303, + 0.1845146677702384 + ], + [ + 314, + 0.42922325616576695 + ], + [ + 317, + 0.7297133293653615 + ], + [ + 322, + 0.22172246028971176 + ], + [ + 325, + 0.405231052472529 + ], + [ + 329, + 0.6403644213120393 + ], + [ + 337, + 0.8296638150344615 + ], + [ + 343, + 0.9234366987033985 + ], + [ + 352, + 0.24389357853446325 + ], + [ + 355, + 0.8054120495364607 + ], + [ + 357, + 0.2261120490782631 + ], + [ + 370, + 0.2090667153511956 + ], + [ + 382, + 0.6598685218022208 + ], + [ + 391, + 0.7674729035384641 + ], + [ + 395, + 0.9908798421289623 + ], + [ + 402, + 0.24050918953720957 + ], + [ + 412, + 0.5033565750721927 + ], + [ + 423, + 0.10195103581721132 + ], + [ + 430, + 0.48872813200214615 + ], + [ + 441, + 0.549337304909546 + ], + [ + 446, + 0.3665262549052506 + ], + [ + 450, + 0.5454383881404011 + ], + [ + 456, + 0.30445706687802043 + ], + [ + 459, + 0.2857412394338058 + ], + [ + 468, + 0.8965825738674744 + ], + [ + 474, + 0.5961864023812716 + ], + [ + 484, + 0.3506319466997945 + ], + [ + 488, + 0.9566836717788472 + ], + [ + 494, + 0.614234466994785 + ], + [ + 500, + 0.42844585050155337 + ], + [ + 506, + 0.14717073025662597 + ], + [ + 512, + 0.3449488647931255 + ], + [ + 518, + 0.3568748346660552 + ], + [ + 524, + 0.9628157595798801 + ], + [ + 528, + 0.8971519829989567 + ], + [ + 536, + 0.21282978468896085 + ], + [ + 543, + 0.5048568894709257 + ], + [ + 547, + 0.809505242754497 + ], + [ + 551, + 0.404914214074809 + ], + [ + 557, + 0.9863217837460115 + ], + [ + 566, + 0.009213726267557787 + ], + [ + 570, + 0.7146586576269167 + ], + [ + 575, + 0.3220464582074578 + ], + [ + 581, + 0.7102890944830219 + ], + [ + 585, + 0.6449417276264502 + ], + [ + 590, + 0.5784561547249889 + ], + [ + 597, + 0.6175460902863847 + ], + [ + 604, + 0.6964329578800111 + ], + [ + 607, + 0.6507577711812726 + ], + [ + 613, + 0.18064408949669108 + ], + [ + 617, + 0.4927123476219185 + ], + [ + 625, + 0.07186669634357168 + ], + [ + 630, + 0.940838267869826 + ], + [ + 633, + 0.15705888252056666 + ], + [ + 637, + 0.635591687954609 + ], + [ + 643, + 0.6464760716695656 + ], + [ + 647, + 0.10630347815627617 + ], + [ + 660, + 0.5190274367083727 + ], + [ + 672, + 0.6925411729540314 + ], + [ + 677, + 0.06048738781455276 + ], + [ + 682, + 0.5395952027074449 + ], + [ + 691, + 0.550134321974646 + ], + [ + 695, + 0.841719121163114 + ], + [ + 698, + 0.7938554667851336 + ], + [ + 702, + 0.9392368087223598 + ] + ] + ], + [ + [ + [ + 34, + 0.7890569315564715 + ], + [ + 44, + 0.34833360797833524 + ], + [ + 48, + 0.23310670203749118 + ], + [ + 59, + 0.10275006708192691 + ], + [ + 64, + 0.97349460252762 + ], + [ + 73, + 0.11341489632136548 + ], + [ + 76, + 0.5005021251608037 + ], + [ + 84, + 0.10080262327645861 + ], + [ + 88, + 0.7057930822155167 + ], + [ + 94, + 0.6671977752330589 + ], + [ + 100, + 0.27742499712242763 + ], + [ + 104, + 0.05879260321763102 + ], + [ + 109, + 0.6867775661796186 + ], + [ + 116, + 0.8961247400426459 + ], + [ + 118, + 0.7451806366818057 + ], + [ + 126, + 0.5649536420386371 + ], + [ + 131, + 0.5478797376490352 + ], + [ + 135, + 0.5520351101773397 + ], + [ + 149, + 0.359549602214737 + ], + [ + 155, + 0.7706253301123137 + ], + [ + 158, + 0.17622559182261788 + ], + [ + 164, + 0.1723096796991308 + ], + [ + 167, + 0.9194426318991031 + ], + [ + 174, + 0.29398725467038445 + ], + [ + 177, + 0.8543548849533317 + ], + [ + 183, + 0.17475980276352232 + ], + [ + 186, + 0.31541291279916905 + ], + [ + 195, + 0.08876444151372109 + ], + [ + 199, + 0.9299259304403733 + ], + [ + 213, + 0.1443561245915812 + ], + [ + 222, + 0.7763669289734871 + ], + [ + 233, + 0.8092179054280617 + ], + [ + 239, + 0.4243292377445288 + ], + [ + 246, + 0.5753206683531717 + ], + [ + 249, + 0.32035625650257094 + ], + [ + 253, + 0.12439978464843215 + ], + [ + 258, + 0.4234276559626138 + ], + [ + 262, + 0.6973360806652288 + ], + [ + 266, + 0.17190614283098815 + ], + [ + 275, + 0.7470478621483679 + ], + [ + 279, + 0.8906130893055474 + ], + [ + 284, + 0.04323681805940083 + ], + [ + 291, + 0.27735355451116883 + ], + [ + 299, + 0.5517245544053182 + ], + [ + 303, + 0.9299521135290709 + ], + [ + 308, + 0.6967870645268078 + ], + [ + 322, + 0.16348400211186676 + ], + [ + 328, + 0.146858094033943 + ], + [ + 332, + 0.7683391053402875 + ], + [ + 337, + 0.49708211060653185 + ], + [ + 340, + 0.13032249800068885 + ], + [ + 352, + 0.7034000833108022 + ], + [ + 355, + 0.3348032001824297 + ], + [ + 362, + 0.7335572085931431 + ], + [ + 368, + 0.8297282542996905 + ], + [ + 376, + 0.9091750363509475 + ], + [ + 380, + 0.9480042082107611 + ], + [ + 388, + 0.5157719166272101 + ], + [ + 392, + 0.4498340428048001 + ], + [ + 400, + 0.32065773904952244 + ], + [ + 404, + 0.7356036622976159 + ], + [ + 410, + 0.6743563799767319 + ], + [ + 414, + 0.6007528393800347 + ], + [ + 420, + 0.3314474275334316 + ], + [ + 425, + 0.7761390531361201 + ], + [ + 428, + 0.7443898732023506 + ], + [ + 432, + 0.07624013605948221 + ], + [ + 437, + 0.311582912457708 + ], + [ + 440, + 0.42557166487562614 + ], + [ + 450, + 0.9877386816445454 + ], + [ + 457, + 0.5272789637644623 + ], + [ + 463, + 0.811916139737507 + ], + [ + 469, + 0.6214926563240035 + ], + [ + 473, + 0.38155171117913533 + ], + [ + 483, + 0.016273613123231057 + ], + [ + 486, + 0.47113637116926343 + ], + [ + 494, + 0.9897673645970705 + ], + [ + 500, + 0.2487503589994795 + ], + [ + 503, + 0.5540829249848721 + ], + [ + 507, + 0.08958239496129661 + ], + [ + 515, + 0.19572452991574718 + ], + [ + 523, + 0.9795019261831417 + ], + [ + 527, + 0.14527644667934325 + ], + [ + 532, + 0.3499402780039973 + ], + [ + 539, + 0.021265322037981704 + ], + [ + 542, + 0.9099254878829618 + ], + [ + 547, + 0.11797028477615568 + ], + [ + 550, + 0.6968430370568808 + ], + [ + 554, + 0.1580095061675706 + ], + [ + 568, + 0.7693688223587518 + ], + [ + 579, + 0.09026817384677954 + ], + [ + 582, + 0.7929368137849006 + ], + [ + 591, + 0.3378386266204926 + ], + [ + 599, + 0.21694987999357185 + ], + [ + 601, + 0.6018431410757022 + ], + [ + 612, + 0.6672460618027466 + ], + [ + 615, + 0.5971250824814032 + ], + [ + 625, + 0.10007799042948928 + ], + [ + 634, + 0.3404581184176648 + ], + [ + 637, + 0.34453347465320106 + ] + ], + [ + [ + 38, + 0.4474508735131496 + ], + [ + 46, + 0.6173468090368438 + ], + [ + 55, + 0.43569101970742796 + ], + [ + 64, + 0.2660048812850917 + ], + [ + 73, + 0.9105305294237704 + ], + [ + 76, + 0.20133702563552835 + ], + [ + 91, + 0.8431824808304823 + ], + [ + 101, + 0.39242147668620475 + ], + [ + 105, + 0.4523651702455115 + ], + [ + 112, + 0.5025991007172359 + ], + [ + 114, + 0.9869319972850592 + ], + [ + 119, + 0.6505647072855929 + ], + [ + 125, + 0.27903360528920096 + ], + [ + 132, + 0.08731932969559009 + ], + [ + 135, + 0.3884063737134462 + ], + [ + 138, + 0.026617118148956953 + ], + [ + 144, + 0.8998392229481966 + ], + [ + 147, + 0.2141054275903428 + ], + [ + 150, + 0.9539375465280628 + ], + [ + 166, + 0.650296476547029 + ], + [ + 169, + 0.9467686652257072 + ], + [ + 172, + 0.8860464587053568 + ], + [ + 176, + 0.7204309611438177 + ], + [ + 179, + 0.5220835420329334 + ], + [ + 183, + 0.8668190208141641 + ], + [ + 187, + 0.7461309322619611 + ], + [ + 195, + 0.2408011610688514 + ], + [ + 198, + 0.19083236798586178 + ], + [ + 202, + 0.34742758197134427 + ], + [ + 207, + 0.6383805012025395 + ], + [ + 216, + 0.10391665776135739 + ], + [ + 221, + 0.8347054339269112 + ], + [ + 225, + 0.43627018017125474 + ], + [ + 231, + 0.8223928300403988 + ], + [ + 234, + 0.1569636235773324 + ], + [ + 238, + 0.7553043446621545 + ], + [ + 247, + 0.9991102860142057 + ], + [ + 251, + 0.8370294246230476 + ], + [ + 254, + 0.5633964291714767 + ], + [ + 264, + 0.1711744688368977 + ], + [ + 267, + 0.1197465030716971 + ], + [ + 276, + 0.7214710914747487 + ], + [ + 280, + 0.8730046136761482 + ], + [ + 287, + 0.004680736782659345 + ], + [ + 290, + 0.6514818895313449 + ], + [ + 296, + 0.7265939042782648 + ], + [ + 308, + 0.35072817332768425 + ], + [ + 315, + 0.509532497256609 + ], + [ + 319, + 0.23702464477799312 + ], + [ + 324, + 0.26775784493550436 + ], + [ + 329, + 0.5862771208752532 + ], + [ + 335, + 0.9226897967936665 + ], + [ + 340, + 0.9924423589735745 + ], + [ + 343, + 0.9705702072722224 + ], + [ + 347, + 0.17385858335942872 + ], + [ + 356, + 0.7375280113008328 + ], + [ + 363, + 0.5370579250712801 + ], + [ + 370, + 0.5096066515774006 + ], + [ + 374, + 0.3026193963951479 + ], + [ + 378, + 0.744546771426147 + ], + [ + 383, + 0.17813370150353847 + ], + [ + 385, + 0.40484831905046326 + ], + [ + 392, + 0.7863284993585248 + ], + [ + 397, + 0.5357557855679704 + ], + [ + 405, + 0.3656328425624815 + ], + [ + 412, + 0.5791704518952997 + ], + [ + 416, + 0.1302520449085095 + ], + [ + 420, + 0.45361529060189 + ], + [ + 425, + 0.13919444569045403 + ], + [ + 434, + 0.5866746485538024 + ], + [ + 438, + 0.8081423761008428 + ], + [ + 443, + 0.16992182969890557 + ], + [ + 445, + 0.5465276495143276 + ], + [ + 456, + 0.8998816614613986 + ], + [ + 461, + 0.10777714316124698 + ], + [ + 466, + 0.1681655477447167 + ], + [ + 469, + 0.34987661138864024 + ], + [ + 479, + 0.15535949595187803 + ], + [ + 489, + 0.3757914607234336 + ], + [ + 492, + 0.2894894181396652 + ], + [ + 496, + 0.22934716715178394 + ], + [ + 504, + 0.2245356214959584 + ], + [ + 507, + 0.33532189175623917 + ], + [ + 511, + 0.7363271827278677 + ], + [ + 514, + 0.6972419906319351 + ], + [ + 525, + 0.21701061469009597 + ], + [ + 530, + 0.22648002551906332 + ], + [ + 536, + 0.9655385419126402 + ], + [ + 539, + 0.6011661386585093 + ], + [ + 543, + 0.8753815007412826 + ], + [ + 552, + 0.06014784364721837 + ], + [ + 557, + 0.14280920818668497 + ], + [ + 565, + 0.41597597793906715 + ], + [ + 568, + 0.9082852794108708 + ], + [ + 572, + 0.3661377260303065 + ], + [ + 585, + 0.20771362920034397 + ], + [ + 588, + 0.7614214032350628 + ], + [ + 592, + 0.605107440558353 + ], + [ + 598, + 0.16347158872749246 + ], + [ + 601, + 0.1704727549155275 + ] + ], + [ + [ + 38, + 0.44223362740057326 + ], + [ + 46, + 0.9280547524038262 + ], + [ + 55, + 0.9034089510910916 + ], + [ + 64, + 0.3095912719501891 + ], + [ + 73, + 0.24585514816742682 + ], + [ + 76, + 0.5397430397763092 + ], + [ + 91, + 0.655343282168449 + ], + [ + 101, + 0.08860800579137951 + ], + [ + 105, + 0.654978832506852 + ], + [ + 112, + 0.8400418661223789 + ], + [ + 114, + 0.8957113239762413 + ], + [ + 119, + 0.8427544709335894 + ], + [ + 125, + 0.28985461809442115 + ], + [ + 132, + 0.7558537845057792 + ], + [ + 135, + 0.8684569811462889 + ], + [ + 138, + 0.9484454664108761 + ], + [ + 144, + 0.5379735850620495 + ], + [ + 147, + 0.5014763132418425 + ], + [ + 150, + 0.5678434404591313 + ], + [ + 166, + 0.6520788962194506 + ], + [ + 169, + 0.23184137819664719 + ], + [ + 172, + 0.975243465645978 + ], + [ + 176, + 0.4765016425093468 + ], + [ + 179, + 0.685695923774693 + ], + [ + 183, + 0.04411807915427313 + ], + [ + 187, + 0.17154494744554327 + ], + [ + 195, + 0.9870644223493374 + ], + [ + 198, + 0.48672308554197696 + ], + [ + 202, + 0.11182799710902858 + ], + [ + 207, + 0.849294217698516 + ], + [ + 216, + 0.26463403807142627 + ], + [ + 221, + 0.6572949966504793 + ], + [ + 225, + 0.8705604820148202 + ], + [ + 231, + 0.9998271654633966 + ], + [ + 234, + 0.055561167557820035 + ], + [ + 238, + 0.5265122837185042 + ], + [ + 247, + 0.8795068353399654 + ], + [ + 251, + 0.8286267641217387 + ], + [ + 254, + 0.16680918130025335 + ], + [ + 264, + 0.029303041082990866 + ], + [ + 267, + 0.01631659330665891 + ], + [ + 276, + 0.7342031455937434 + ], + [ + 280, + 0.916753860836387 + ], + [ + 287, + 0.14993438226629008 + ], + [ + 290, + 0.2324555461946166 + ], + [ + 296, + 0.14412313518608755 + ], + [ + 308, + 0.4148133054311651 + ], + [ + 315, + 0.38313557809137877 + ], + [ + 319, + 0.5912985712885397 + ], + [ + 324, + 0.3084532351825161 + ], + [ + 329, + 0.2626734465106423 + ], + [ + 335, + 0.6496635758964893 + ], + [ + 340, + 0.5735613446979964 + ], + [ + 343, + 0.6145812190368586 + ], + [ + 347, + 0.8280822248061036 + ], + [ + 356, + 0.0802347596520121 + ], + [ + 363, + 0.5489584675860743 + ], + [ + 370, + 0.1496634456274345 + ], + [ + 374, + 0.790379971748317 + ], + [ + 378, + 0.8563851669925344 + ], + [ + 383, + 0.07260133819233983 + ], + [ + 385, + 0.359146273632164 + ], + [ + 392, + 0.9875119736672606 + ], + [ + 397, + 0.0010048374462110488 + ], + [ + 405, + 0.8768785631742823 + ], + [ + 412, + 0.6390138897009717 + ], + [ + 416, + 0.6702662976304804 + ], + [ + 420, + 0.37123299858531567 + ], + [ + 425, + 0.5985473170947941 + ], + [ + 434, + 0.7274824270781993 + ], + [ + 438, + 0.17930792251885608 + ], + [ + 443, + 0.11585138168612297 + ], + [ + 445, + 0.3542090975184625 + ], + [ + 456, + 0.6169097550976957 + ], + [ + 461, + 0.40886719041661623 + ], + [ + 466, + 0.17305083058611526 + ], + [ + 469, + 0.7275878633454099 + ], + [ + 479, + 0.37219549597331436 + ], + [ + 489, + 0.033353248620686005 + ], + [ + 492, + 0.9121686998529818 + ], + [ + 496, + 0.14788561593654248 + ], + [ + 504, + 0.7956804487463186 + ], + [ + 507, + 0.22629374156939908 + ], + [ + 511, + 0.491929807631321 + ], + [ + 514, + 0.7633822197033119 + ], + [ + 525, + 0.8634766748901725 + ], + [ + 530, + 0.018129181905868874 + ], + [ + 536, + 0.7635377465854861 + ], + [ + 539, + 0.026559657817842974 + ], + [ + 543, + 0.8052449135417911 + ], + [ + 552, + 0.9629753671776455 + ], + [ + 557, + 0.17177489540973312 + ], + [ + 565, + 0.5860837122340373 + ], + [ + 568, + 0.9099135384449513 + ], + [ + 572, + 0.9792669981821831 + ], + [ + 585, + 0.8429095476588548 + ], + [ + 588, + 0.5597289537548925 + ], + [ + 592, + 0.5898246302489015 + ], + [ + 598, + 0.3978855787411818 + ], + [ + 601, + 0.8666192781340598 + ] + ], + [ + [ + 37, + 0.48732145806003135 + ], + [ + 43, + 0.7315389182587093 + ], + [ + 54, + 0.7346138137733782 + ], + [ + 60, + 0.791522838015047 + ], + [ + 71, + 0.18195736644326188 + ], + [ + 77, + 0.35697200873315327 + ], + [ + 80, + 0.9233267381378623 + ], + [ + 89, + 0.5194318225323206 + ], + [ + 95, + 0.22660113007996685 + ], + [ + 98, + 0.657942742848426 + ], + [ + 106, + 0.5520156569255555 + ], + [ + 113, + 0.7283308804898909 + ], + [ + 127, + 0.855074350464641 + ], + [ + 138, + 0.024644201734533078 + ], + [ + 142, + 0.49746366194427527 + ], + [ + 150, + 0.6559447356188574 + ], + [ + 159, + 0.11427698263984032 + ], + [ + 166, + 0.1736951553985978 + ], + [ + 177, + 0.22407257720246776 + ], + [ + 184, + 0.03509654016049246 + ], + [ + 187, + 0.794032855225741 + ], + [ + 189, + 0.8890970130262511 + ], + [ + 197, + 0.19735847534869033 + ], + [ + 201, + 0.12462554183011332 + ], + [ + 204, + 0.7848573846433101 + ], + [ + 212, + 0.23957492131038105 + ], + [ + 216, + 0.9274059868185438 + ], + [ + 220, + 0.5978834990800633 + ], + [ + 225, + 0.6398212705297761 + ], + [ + 233, + 0.4687133261214791 + ], + [ + 241, + 0.9608768853171389 + ], + [ + 248, + 0.8114157081430573 + ], + [ + 252, + 0.3590144858714962 + ], + [ + 263, + 0.6817101427091221 + ], + [ + 270, + 0.7095178203646221 + ], + [ + 273, + 0.1673861870523642 + ], + [ + 277, + 0.11653279908183722 + ], + [ + 282, + 0.7298884293063613 + ], + [ + 287, + 0.07169441501234297 + ], + [ + 293, + 0.9629308785122198 + ], + [ + 298, + 0.2936764705129362 + ], + [ + 302, + 0.08099748615118574 + ], + [ + 307, + 0.13470815750882104 + ], + [ + 315, + 0.1861520657337956 + ], + [ + 318, + 0.09642572354107981 + ], + [ + 327, + 0.0868199658530866 + ], + [ + 334, + 0.7496740459061344 + ], + [ + 345, + 0.36588958478232625 + ], + [ + 352, + 0.5833815986780084 + ], + [ + 357, + 0.5164128122826452 + ], + [ + 361, + 0.8681044387306317 + ], + [ + 369, + 0.5556029430440954 + ], + [ + 380, + 0.06985286216672626 + ], + [ + 384, + 0.2601032901187196 + ], + [ + 387, + 0.8064223748398224 + ], + [ + 392, + 0.523700877153728 + ], + [ + 399, + 0.21393520864874005 + ], + [ + 409, + 0.8933881149332326 + ], + [ + 415, + 0.7599273305849825 + ], + [ + 426, + 0.9183367161617594 + ], + [ + 432, + 0.0945655190082928 + ], + [ + 435, + 0.02184027158112578 + ], + [ + 439, + 0.265318312570446 + ], + [ + 449, + 0.8485020415964083 + ], + [ + 460, + 0.4426380762786035 + ], + [ + 466, + 0.5299663098990308 + ], + [ + 470, + 0.25011117822767226 + ], + [ + 477, + 0.541226019879165 + ], + [ + 481, + 0.9960845696119888 + ], + [ + 486, + 0.5733265068360727 + ], + [ + 492, + 0.393271503311305 + ], + [ + 495, + 0.1229780997077572 + ], + [ + 504, + 0.4108992856472402 + ], + [ + 517, + 0.8491335702308893 + ], + [ + 521, + 0.20381313802331047 + ], + [ + 530, + 0.2158719856177248 + ], + [ + 535, + 0.2734639885830148 + ], + [ + 539, + 0.49026614281676295 + ], + [ + 550, + 0.15607270191293132 + ], + [ + 554, + 0.1486443303981022 + ], + [ + 559, + 0.2100724327675879 + ], + [ + 562, + 0.3090396245764352 + ], + [ + 571, + 0.21905180265059887 + ], + [ + 575, + 0.8517598840808955 + ], + [ + 581, + 0.7026969072298469 + ], + [ + 585, + 0.05112447583666746 + ], + [ + 590, + 0.5786512058922415 + ], + [ + 597, + 0.18198441080102112 + ], + [ + 602, + 0.07706488776076181 + ], + [ + 608, + 0.6787417286256527 + ], + [ + 613, + 0.16325673110718686 + ], + [ + 619, + 0.9972789429762873 + ], + [ + 631, + 0.4537956751197344 + ], + [ + 634, + 0.5125121509447853 + ], + [ + 639, + 0.46469297697024536 + ], + [ + 643, + 0.41655212576143974 + ], + [ + 649, + 0.0037973950714607074 + ], + [ + 652, + 0.6633751282998047 + ], + [ + 656, + 0.10757093043061772 + ], + [ + 660, + 0.338893725753324 + ] + ], + [ + [ + 50, + 0.46329562499023536 + ], + [ + 58, + 0.16645052615652 + ], + [ + 61, + 0.23279229373333654 + ], + [ + 66, + 0.8874775658699657 + ], + [ + 75, + 0.35852094977349924 + ], + [ + 78, + 0.7123989022131302 + ], + [ + 85, + 0.03191142584116824 + ], + [ + 88, + 0.11294809300439423 + ], + [ + 100, + 0.8674428435670353 + ], + [ + 105, + 0.2618953090979549 + ], + [ + 110, + 0.9654296486707095 + ], + [ + 115, + 0.740593506110645 + ], + [ + 125, + 0.8671278333134325 + ], + [ + 128, + 0.9371048574358949 + ], + [ + 133, + 0.5822008087842866 + ], + [ + 142, + 0.5426908406935381 + ], + [ + 149, + 0.4579294006700333 + ], + [ + 159, + 0.9872539446091603 + ], + [ + 163, + 0.6522262964951245 + ], + [ + 168, + 0.701208762229819 + ], + [ + 173, + 0.539651122434288 + ], + [ + 178, + 0.5205971563303006 + ], + [ + 189, + 0.7910740688808839 + ], + [ + 199, + 0.880707545901061 + ], + [ + 206, + 0.42411548104058716 + ], + [ + 217, + 0.7288803035308135 + ], + [ + 221, + 0.935122773844572 + ], + [ + 225, + 0.18607254650758875 + ], + [ + 232, + 0.14851879159265857 + ], + [ + 240, + 0.6439653702786083 + ], + [ + 249, + 0.25214938115435515 + ], + [ + 255, + 0.5787773104041685 + ], + [ + 259, + 0.21170691366237548 + ], + [ + 266, + 0.2281767295730729 + ], + [ + 270, + 0.7282859114876636 + ], + [ + 274, + 0.6927018853358593 + ], + [ + 279, + 0.3149043477308541 + ], + [ + 285, + 0.37786853627529815 + ], + [ + 292, + 0.9621863241354417 + ], + [ + 295, + 0.22654863189104157 + ], + [ + 297, + 0.1950885421191273 + ], + [ + 304, + 0.7035432388336782 + ], + [ + 316, + 0.9868707975181374 + ], + [ + 321, + 0.44489616163056334 + ], + [ + 324, + 0.9962101293432382 + ], + [ + 330, + 0.4057357576448518 + ], + [ + 333, + 0.3816366350967332 + ], + [ + 339, + 0.09632930174882737 + ], + [ + 343, + 0.21041083337242017 + ], + [ + 351, + 0.44440712708069463 + ], + [ + 356, + 0.17164252340076802 + ], + [ + 361, + 0.27224777073991924 + ], + [ + 369, + 0.2746708244330234 + ], + [ + 372, + 0.5042090949750971 + ], + [ + 377, + 0.26893772188424614 + ], + [ + 384, + 0.13703162466306829 + ], + [ + 389, + 0.6709946993478709 + ], + [ + 396, + 0.7425031025858074 + ], + [ + 399, + 0.07367990763711374 + ], + [ + 405, + 0.5737329087841551 + ], + [ + 412, + 0.42283768445053327 + ], + [ + 415, + 0.14538208626702298 + ], + [ + 420, + 0.7385864330759113 + ], + [ + 424, + 0.5352434565723423 + ], + [ + 435, + 0.9562950262489279 + ], + [ + 443, + 0.37689879630749135 + ], + [ + 446, + 0.9800283875284765 + ], + [ + 452, + 0.27194862024284394 + ], + [ + 457, + 0.9238213995157135 + ], + [ + 461, + 0.7697259871148568 + ], + [ + 464, + 0.8196762336433535 + ], + [ + 468, + 0.08948938931763672 + ], + [ + 471, + 0.8910718142313844 + ], + [ + 485, + 0.4414001787442354 + ], + [ + 489, + 0.04360567041065011 + ], + [ + 496, + 0.9270145494284555 + ], + [ + 504, + 0.042545759396008465 + ], + [ + 517, + 0.9521675973975832 + ], + [ + 521, + 0.008791207278368351 + ], + [ + 528, + 0.2685486830216608 + ], + [ + 532, + 0.9200116043998585 + ], + [ + 535, + 0.17764129517234362 + ], + [ + 541, + 0.20861002583786192 + ], + [ + 546, + 0.741317459447591 + ], + [ + 550, + 0.3459646744396313 + ], + [ + 558, + 0.8155329648321876 + ], + [ + 563, + 0.6424834691936478 + ], + [ + 566, + 0.37024342286240786 + ], + [ + 571, + 0.6222972219125422 + ], + [ + 576, + 0.5601816460629756 + ], + [ + 582, + 0.9843033745506632 + ], + [ + 588, + 0.33742712567610544 + ], + [ + 595, + 0.8322894387460966 + ], + [ + 599, + 0.05355159325555381 + ], + [ + 602, + 0.8699304334693828 + ], + [ + 611, + 0.6706494662230841 + ], + [ + 614, + 0.4756513274513827 + ], + [ + 618, + 0.31775158476108134 + ], + [ + 624, + 0.6611687751408006 + ], + [ + 629, + 0.8608485985316154 + ] + ], + [ + [ + 53, + 0.699102494724101 + ], + [ + 63, + 0.800372721585696 + ], + [ + 70, + 0.5323197752654771 + ], + [ + 73, + 0.2962854028122266 + ], + [ + 77, + 0.3228392073801596 + ], + [ + 87, + 0.5540922604007493 + ], + [ + 90, + 0.6351743082939401 + ], + [ + 96, + 0.8311140720562941 + ], + [ + 110, + 0.2556625214416398 + ], + [ + 121, + 0.4513850797741661 + ], + [ + 124, + 0.22907572689690436 + ], + [ + 133, + 0.5380537917758845 + ], + [ + 141, + 0.6408602129882965 + ], + [ + 145, + 0.9445972392341314 + ], + [ + 157, + 0.6199553571763167 + ], + [ + 163, + 0.554232805563841 + ], + [ + 170, + 0.9013474997291948 + ], + [ + 178, + 0.8918302072501937 + ], + [ + 187, + 0.8909589946359359 + ], + [ + 194, + 0.46741390005375927 + ], + [ + 197, + 0.23394093920211934 + ], + [ + 201, + 0.7554857786207828 + ], + [ + 211, + 0.6716484338360507 + ], + [ + 214, + 0.6982625810142954 + ], + [ + 220, + 0.07756829617786964 + ], + [ + 234, + 0.10543133801207416 + ], + [ + 245, + 0.8212706147019501 + ], + [ + 248, + 0.7819417976845624 + ], + [ + 257, + 0.4273771639622532 + ], + [ + 265, + 0.6528465272724773 + ], + [ + 269, + 0.9638939836964517 + ], + [ + 281, + 0.7450668218566974 + ], + [ + 288, + 0.05525705408212278 + ], + [ + 293, + 0.21389849085183343 + ], + [ + 299, + 0.4636204567619403 + ], + [ + 303, + 0.9966123272587805 + ], + [ + 314, + 0.22428482939894145 + ], + [ + 317, + 0.7921600916608913 + ], + [ + 322, + 0.3038124297513308 + ], + [ + 325, + 0.7563636611097031 + ], + [ + 329, + 0.11444958049492016 + ], + [ + 337, + 0.994068284057543 + ], + [ + 343, + 0.7317072006838083 + ], + [ + 352, + 0.36369310428714974 + ], + [ + 355, + 0.11292761408657626 + ], + [ + 357, + 0.5286592496178362 + ], + [ + 370, + 0.06028885848596577 + ], + [ + 382, + 0.8470438298307269 + ], + [ + 391, + 0.5776859639730917 + ], + [ + 395, + 0.5251835536712407 + ], + [ + 402, + 0.45990716537290444 + ], + [ + 412, + 0.7428870061519649 + ], + [ + 423, + 0.7510285213638478 + ], + [ + 430, + 0.4352733619724859 + ], + [ + 441, + 0.2147695920996574 + ], + [ + 446, + 0.501240001459065 + ], + [ + 450, + 0.667244416534853 + ], + [ + 456, + 0.297215479510085 + ], + [ + 459, + 0.8939244300696457 + ], + [ + 468, + 0.4683704890372028 + ], + [ + 474, + 0.2948296859509618 + ], + [ + 484, + 0.5558978008927696 + ], + [ + 488, + 0.3187845041977845 + ], + [ + 494, + 0.8358701346915884 + ], + [ + 500, + 0.7021760106841318 + ], + [ + 506, + 0.09997668241321533 + ], + [ + 512, + 0.5363629667239129 + ], + [ + 518, + 0.6095471050954253 + ], + [ + 524, + 0.06645269858275804 + ], + [ + 528, + 0.0004996794999616583 + ], + [ + 536, + 0.18583913985147682 + ], + [ + 543, + 0.2422029115331914 + ], + [ + 547, + 0.8878040824945956 + ], + [ + 551, + 0.0005814220018177663 + ], + [ + 557, + 0.06492637518782718 + ], + [ + 566, + 0.9356451130645658 + ], + [ + 570, + 0.27496408425900665 + ], + [ + 575, + 0.8149729817986036 + ], + [ + 581, + 0.7210780924167814 + ], + [ + 585, + 0.1510244300524528 + ], + [ + 590, + 0.20294576451397206 + ], + [ + 597, + 0.1553282518502751 + ], + [ + 604, + 0.4912031029658829 + ], + [ + 607, + 0.8394845881297451 + ], + [ + 613, + 0.7167414521575795 + ], + [ + 617, + 0.6258405704852249 + ], + [ + 625, + 0.1432368900797274 + ], + [ + 630, + 0.002903796951071125 + ], + [ + 633, + 0.20218923597005156 + ], + [ + 637, + 0.8332197633190558 + ], + [ + 643, + 0.8234395593505321 + ], + [ + 647, + 0.6349695464514649 + ], + [ + 660, + 0.6012635910462582 + ], + [ + 672, + 0.9362407256861722 + ], + [ + 677, + 0.22284153273302432 + ], + [ + 682, + 0.151976768826095 + ], + [ + 691, + 0.8250511870651384 + ], + [ + 695, + 0.5892820741377013 + ], + [ + 698, + 0.25155675749236817 + ], + [ + 702, + 0.742731328132861 + ] + ] + ], + [ + [ + [ + 34, + 0.9636600537618455 + ], + [ + 44, + 0.7437945944807257 + ], + [ + 48, + 0.8167755204855559 + ], + [ + 59, + 0.4921773678349116 + ], + [ + 64, + 0.8053567616220777 + ], + [ + 73, + 0.5700045151611166 + ], + [ + 76, + 0.7630111189110091 + ], + [ + 84, + 0.88365565859156 + ], + [ + 88, + 0.2233792197327109 + ], + [ + 94, + 0.7386905488629014 + ], + [ + 100, + 0.37523879679703187 + ], + [ + 104, + 0.1784340102949673 + ], + [ + 109, + 0.33578723681849654 + ], + [ + 116, + 0.545012726220743 + ], + [ + 118, + 0.3277119915329856 + ], + [ + 126, + 0.35126075314636085 + ], + [ + 131, + 0.24592953074169677 + ], + [ + 135, + 0.006141596161829521 + ], + [ + 149, + 0.078418659674236 + ], + [ + 155, + 0.05163370682889579 + ], + [ + 158, + 0.9078188180121576 + ], + [ + 164, + 0.40364899609502813 + ], + [ + 167, + 0.875723422854441 + ], + [ + 174, + 0.1902459583262993 + ], + [ + 177, + 0.47800210901623597 + ], + [ + 183, + 0.78738848908616 + ], + [ + 186, + 0.6462206526354649 + ], + [ + 195, + 0.5599955063440741 + ], + [ + 199, + 0.4119511106798204 + ], + [ + 213, + 0.06903002557771654 + ], + [ + 222, + 0.7847289018575392 + ], + [ + 233, + 0.3075813157578189 + ], + [ + 239, + 0.7629280971179497 + ], + [ + 246, + 0.540157297334651 + ], + [ + 249, + 0.530198079020022 + ], + [ + 253, + 0.35168666422334294 + ], + [ + 258, + 0.2208951506629111 + ], + [ + 262, + 0.3314450632464927 + ], + [ + 266, + 0.9619402242871128 + ], + [ + 275, + 0.24937888882457504 + ], + [ + 279, + 0.46324533269749 + ], + [ + 284, + 0.9040467515838522 + ], + [ + 291, + 0.5615296680496175 + ], + [ + 299, + 0.8539580258622507 + ], + [ + 303, + 0.07360629092607218 + ], + [ + 308, + 0.9217870208950464 + ], + [ + 322, + 0.45025230717064535 + ], + [ + 328, + 0.7413576004604062 + ], + [ + 332, + 0.3676868225662393 + ], + [ + 337, + 0.9568505036804705 + ], + [ + 340, + 0.18032789101920532 + ], + [ + 352, + 0.5209139443916063 + ], + [ + 355, + 0.332870464442223 + ], + [ + 362, + 0.509244063757699 + ], + [ + 368, + 0.825667793299011 + ], + [ + 376, + 0.9055345060630526 + ], + [ + 380, + 0.4413353736406316 + ], + [ + 388, + 0.7144565042653581 + ], + [ + 392, + 0.2677189870911154 + ], + [ + 400, + 0.16986184341191557 + ], + [ + 404, + 0.0162948263290561 + ], + [ + 410, + 0.11925579193360836 + ], + [ + 414, + 0.006545359502425185 + ], + [ + 420, + 0.24793733793349504 + ], + [ + 425, + 0.035782021601621694 + ], + [ + 428, + 0.8058232222224312 + ], + [ + 432, + 0.11160633573976575 + ], + [ + 437, + 0.8785918499614999 + ], + [ + 440, + 0.47396294743805045 + ], + [ + 450, + 0.17272158928731962 + ], + [ + 457, + 0.48890922599812725 + ], + [ + 463, + 0.40641396640679905 + ], + [ + 469, + 0.7317979087049525 + ], + [ + 473, + 0.022496394301480538 + ], + [ + 483, + 0.6959406470731367 + ], + [ + 486, + 0.563754028973515 + ], + [ + 494, + 0.6773912152907232 + ], + [ + 500, + 0.013363086227264676 + ], + [ + 503, + 0.35886327121950623 + ], + [ + 507, + 0.045743575352879096 + ], + [ + 515, + 0.7127070952580636 + ], + [ + 523, + 0.36466298249342854 + ], + [ + 527, + 0.3221549319243515 + ], + [ + 532, + 0.6765505496499103 + ], + [ + 539, + 0.5523428099144133 + ], + [ + 542, + 0.6531582027124223 + ], + [ + 547, + 0.7983102541719761 + ], + [ + 550, + 0.39558883178044 + ], + [ + 554, + 0.9861022579930279 + ], + [ + 568, + 0.9406136276935716 + ], + [ + 579, + 0.4244955950165664 + ], + [ + 582, + 0.6157272092699928 + ], + [ + 591, + 0.3937550269095694 + ], + [ + 599, + 0.722229502617793 + ], + [ + 601, + 0.881210133285057 + ], + [ + 612, + 0.7265169211888043 + ], + [ + 615, + 0.9694385367670116 + ], + [ + 625, + 0.9618134685335965 + ], + [ + 634, + 0.6584265047651547 + ], + [ + 637, + 0.41732014148128505 + ] + ], + [ + [ + 38, + 0.37337568183968184 + ], + [ + 46, + 0.8314870810772514 + ], + [ + 55, + 0.05254929349011084 + ], + [ + 64, + 0.8931412132216846 + ], + [ + 73, + 0.3438805848004296 + ], + [ + 76, + 0.325455535440039 + ], + [ + 91, + 0.6612366019397925 + ], + [ + 101, + 0.9646760686493935 + ], + [ + 105, + 0.4013030864463163 + ], + [ + 112, + 0.3810221830796735 + ], + [ + 114, + 0.1899734738239206 + ], + [ + 119, + 0.7543039345621384 + ], + [ + 125, + 0.6154861874091271 + ], + [ + 132, + 0.09433024895285769 + ], + [ + 135, + 0.4402954805293283 + ], + [ + 138, + 0.5547284697978927 + ], + [ + 144, + 0.7652857103752206 + ], + [ + 147, + 0.8045429048235416 + ], + [ + 150, + 0.9331447889123536 + ], + [ + 166, + 0.9083081770991418 + ], + [ + 169, + 0.5486083097635432 + ], + [ + 172, + 0.594021409043046 + ], + [ + 176, + 0.29803204428617036 + ], + [ + 179, + 0.6255290106071374 + ], + [ + 183, + 0.9062204720991638 + ], + [ + 187, + 0.25591591253921864 + ], + [ + 195, + 0.627893900291424 + ], + [ + 198, + 0.6906029109094404 + ], + [ + 202, + 0.35636376280206417 + ], + [ + 207, + 0.9757564796139993 + ], + [ + 216, + 0.35607004012983967 + ], + [ + 221, + 0.6040415643515366 + ], + [ + 225, + 0.7488232724437635 + ], + [ + 231, + 0.6368898494859185 + ], + [ + 234, + 0.5110845019182115 + ], + [ + 238, + 0.37116597774046267 + ], + [ + 247, + 0.22748217089908573 + ], + [ + 251, + 0.805998427608663 + ], + [ + 254, + 0.23173631596470667 + ], + [ + 264, + 0.19175689045028455 + ], + [ + 267, + 0.15298854565753706 + ], + [ + 276, + 0.5586044487053198 + ], + [ + 280, + 0.5133560616929543 + ], + [ + 287, + 0.23664609090776278 + ], + [ + 290, + 0.46826034256235316 + ], + [ + 296, + 0.8755526037106054 + ], + [ + 308, + 0.6751339530532486 + ], + [ + 315, + 0.7162263142696316 + ], + [ + 319, + 0.29652719055298427 + ], + [ + 324, + 0.44478614247345616 + ], + [ + 329, + 0.02767132616083723 + ], + [ + 335, + 0.9850910812093614 + ], + [ + 340, + 0.8732422463766257 + ], + [ + 343, + 0.5799334528947521 + ], + [ + 347, + 0.4617683080635544 + ], + [ + 356, + 0.45855096447403676 + ], + [ + 363, + 0.31454849503865956 + ], + [ + 370, + 0.23166025243343524 + ], + [ + 374, + 0.6875508786426817 + ], + [ + 378, + 0.31693498832889744 + ], + [ + 383, + 0.5768973780541075 + ], + [ + 385, + 0.723892367667149 + ], + [ + 392, + 0.42372564000241364 + ], + [ + 397, + 0.2930607999709818 + ], + [ + 405, + 0.8419877790687116 + ], + [ + 412, + 0.356055241523914 + ], + [ + 416, + 0.814010032134231 + ], + [ + 420, + 0.007488205565631456 + ], + [ + 425, + 0.7931562431982624 + ], + [ + 434, + 0.954157423183999 + ], + [ + 438, + 0.40241743695007015 + ], + [ + 443, + 0.49687395657512334 + ], + [ + 445, + 0.3014258730442745 + ], + [ + 456, + 0.6626990737054786 + ], + [ + 461, + 0.16653401631344333 + ], + [ + 466, + 0.7495890265110241 + ], + [ + 469, + 0.6567296454080804 + ], + [ + 479, + 0.39571085419662744 + ], + [ + 489, + 0.6058225802300987 + ], + [ + 492, + 0.20558756429341563 + ], + [ + 496, + 0.804752410021165 + ], + [ + 504, + 0.5932590293790729 + ], + [ + 507, + 0.07832012104082742 + ], + [ + 511, + 0.0184356271123437 + ], + [ + 514, + 0.3763932717420686 + ], + [ + 525, + 0.48065937055791097 + ], + [ + 530, + 0.10962123056014506 + ], + [ + 536, + 0.20805348121933198 + ], + [ + 539, + 0.3788384156449759 + ], + [ + 543, + 0.979341740526004 + ], + [ + 552, + 0.24851894311155054 + ], + [ + 557, + 0.32309924777884214 + ], + [ + 565, + 0.8400798678401796 + ], + [ + 568, + 0.9607015985768218 + ], + [ + 572, + 0.9561824914145388 + ], + [ + 585, + 0.7200919244422491 + ], + [ + 588, + 0.5558987531211155 + ], + [ + 592, + 0.7111444885980085 + ], + [ + 598, + 0.30982850116103355 + ], + [ + 601, + 0.2397346210284481 + ] + ], + [ + [ + 38, + 0.603520932099422 + ], + [ + 46, + 0.0762506323997365 + ], + [ + 55, + 0.3895020523873157 + ], + [ + 64, + 0.3774000540845184 + ], + [ + 73, + 0.23786129696635538 + ], + [ + 76, + 0.06397026040024434 + ], + [ + 91, + 0.46988913479038774 + ], + [ + 101, + 0.29774207292067034 + ], + [ + 105, + 0.7553350135556048 + ], + [ + 112, + 0.008667484071393616 + ], + [ + 114, + 0.5581177493331327 + ], + [ + 119, + 0.928783519881658 + ], + [ + 125, + 0.08848841314129707 + ], + [ + 132, + 0.46735517313459585 + ], + [ + 135, + 0.1260308021231742 + ], + [ + 138, + 0.2213730211046392 + ], + [ + 144, + 0.36889343697425736 + ], + [ + 147, + 0.10286089729768033 + ], + [ + 150, + 0.29502606272089016 + ], + [ + 166, + 0.9032753818813676 + ], + [ + 169, + 0.35120871710418944 + ], + [ + 172, + 0.034553103188077916 + ], + [ + 176, + 0.9331594419574355 + ], + [ + 179, + 0.7761010727771397 + ], + [ + 183, + 0.7032093753887285 + ], + [ + 187, + 0.842220907531042 + ], + [ + 195, + 0.7679164629227986 + ], + [ + 198, + 0.7854420134779525 + ], + [ + 202, + 0.3890550948944199 + ], + [ + 207, + 0.05497971573457883 + ], + [ + 216, + 0.699347240333242 + ], + [ + 221, + 0.8064976132197964 + ], + [ + 225, + 0.07827542815389377 + ], + [ + 231, + 0.8864627626323892 + ], + [ + 234, + 0.6884719495222267 + ], + [ + 238, + 0.4460731766723919 + ], + [ + 247, + 0.46685959119707465 + ], + [ + 251, + 0.6274688724044505 + ], + [ + 254, + 0.47704076925583927 + ], + [ + 264, + 0.5003266927168084 + ], + [ + 267, + 0.0493680420069289 + ], + [ + 276, + 0.5961877642003286 + ], + [ + 280, + 0.02315228204035391 + ], + [ + 287, + 0.789709489557732 + ], + [ + 290, + 0.857801698219925 + ], + [ + 296, + 0.986191019845217 + ], + [ + 308, + 0.28588448093164864 + ], + [ + 315, + 0.25811670784794016 + ], + [ + 319, + 0.08171091550879261 + ], + [ + 324, + 0.09058421446563814 + ], + [ + 329, + 0.989225467521274 + ], + [ + 335, + 0.5330819293308688 + ], + [ + 340, + 0.37801674557217657 + ], + [ + 343, + 0.1909601666581524 + ], + [ + 347, + 0.09155486214150232 + ], + [ + 356, + 0.9653414857273553 + ], + [ + 363, + 0.5105232513239396 + ], + [ + 370, + 0.1569412479821508 + ], + [ + 374, + 0.5197053247230333 + ], + [ + 378, + 0.6022079090558656 + ], + [ + 383, + 0.3673918675622929 + ], + [ + 385, + 0.18852514793262654 + ], + [ + 392, + 0.2619917524911628 + ], + [ + 397, + 0.15612141106529442 + ], + [ + 405, + 0.41649548724658314 + ], + [ + 412, + 0.4583296490297408 + ], + [ + 416, + 0.4923381309697026 + ], + [ + 420, + 0.7006963800507172 + ], + [ + 425, + 0.9142920213196485 + ], + [ + 434, + 0.642733061750795 + ], + [ + 438, + 0.3386881949764107 + ], + [ + 443, + 0.13763953426555908 + ], + [ + 445, + 0.030906060874842156 + ], + [ + 456, + 0.9085789533860049 + ], + [ + 461, + 0.48964690344281636 + ], + [ + 466, + 0.8128647950067759 + ], + [ + 469, + 0.8128075655993304 + ], + [ + 479, + 0.6596808382378957 + ], + [ + 489, + 0.8874590133837322 + ], + [ + 492, + 0.8058382737281328 + ], + [ + 496, + 0.13196842623613136 + ], + [ + 504, + 0.10458029266690638 + ], + [ + 507, + 0.03337089129019444 + ], + [ + 511, + 0.17988278161434657 + ], + [ + 514, + 0.8714081949531116 + ], + [ + 525, + 0.6040266454814409 + ], + [ + 530, + 0.44794720375829244 + ], + [ + 536, + 0.7738269536236897 + ], + [ + 539, + 0.9241519508165542 + ], + [ + 543, + 0.2699325055235672 + ], + [ + 552, + 0.3183689112299497 + ], + [ + 557, + 0.8197654056781183 + ], + [ + 565, + 0.2988091874180716 + ], + [ + 568, + 0.470408023018947 + ], + [ + 572, + 0.8710772597697137 + ], + [ + 585, + 0.2701983545581935 + ], + [ + 588, + 0.2851079770173752 + ], + [ + 592, + 0.8031574029814366 + ], + [ + 598, + 0.5845960315336762 + ], + [ + 601, + 0.6167920310742061 + ] + ], + [ + [ + 37, + 0.513381576424309 + ], + [ + 43, + 0.7196252974902934 + ], + [ + 54, + 0.27152458216832853 + ], + [ + 60, + 0.9517817802988677 + ], + [ + 71, + 0.7012610207305694 + ], + [ + 77, + 0.9807058079093713 + ], + [ + 80, + 0.5853994338351054 + ], + [ + 89, + 0.25923597037414403 + ], + [ + 95, + 0.40453955884423454 + ], + [ + 98, + 0.7233399470036999 + ], + [ + 106, + 0.1329166207630713 + ], + [ + 113, + 0.7968998516981478 + ], + [ + 127, + 0.9781083560721823 + ], + [ + 138, + 0.9382776221768705 + ], + [ + 142, + 0.5463890956272981 + ], + [ + 150, + 0.007525934776732135 + ], + [ + 159, + 0.1573516356700635 + ], + [ + 166, + 0.4547596827198923 + ], + [ + 177, + 0.6177591388392978 + ], + [ + 184, + 0.9320078503175888 + ], + [ + 187, + 0.6423439045965761 + ], + [ + 189, + 0.7779544198033739 + ], + [ + 197, + 0.1373071527631161 + ], + [ + 201, + 0.5041130654355112 + ], + [ + 204, + 0.6392083454948945 + ], + [ + 212, + 0.31661098786369746 + ], + [ + 216, + 0.6145036202717641 + ], + [ + 220, + 0.45545015152780244 + ], + [ + 225, + 0.5907322078131989 + ], + [ + 233, + 0.1960679598009586 + ], + [ + 241, + 0.8218116249629631 + ], + [ + 248, + 0.06903378487202261 + ], + [ + 252, + 0.3843788284367947 + ], + [ + 263, + 0.8491551400337625 + ], + [ + 270, + 0.4043516537281038 + ], + [ + 273, + 0.5676297409375225 + ], + [ + 277, + 0.6091694547446456 + ], + [ + 282, + 0.973806291567238 + ], + [ + 287, + 0.5816717469870128 + ], + [ + 293, + 0.8523751480647667 + ], + [ + 298, + 0.5245575617903098 + ], + [ + 302, + 0.76909151274186 + ], + [ + 307, + 0.40638914568314044 + ], + [ + 315, + 0.19876974692321514 + ], + [ + 318, + 0.07205646858901127 + ], + [ + 327, + 0.13672622547294522 + ], + [ + 334, + 0.03133491470414873 + ], + [ + 345, + 0.39658702653530686 + ], + [ + 352, + 0.7412706147496134 + ], + [ + 357, + 0.7759801227659198 + ], + [ + 361, + 0.6184357176220328 + ], + [ + 369, + 0.7244427454635112 + ], + [ + 380, + 0.7388456175494494 + ], + [ + 384, + 0.9034878004117507 + ], + [ + 387, + 0.043907737150515436 + ], + [ + 392, + 0.6028143984730646 + ], + [ + 399, + 0.7917719162871802 + ], + [ + 409, + 0.881552810705541 + ], + [ + 415, + 0.6610696017457188 + ], + [ + 426, + 0.9415431525095521 + ], + [ + 432, + 0.34315324766170097 + ], + [ + 435, + 0.13315142929398438 + ], + [ + 439, + 0.20709278706178702 + ], + [ + 449, + 0.007907989183935094 + ], + [ + 460, + 0.5482012323181518 + ], + [ + 466, + 0.5770931649103702 + ], + [ + 470, + 0.712568956463001 + ], + [ + 477, + 0.23287230009546867 + ], + [ + 481, + 0.4170334087408497 + ], + [ + 486, + 0.1573756513358605 + ], + [ + 492, + 0.7184536629668724 + ], + [ + 495, + 0.224261706891709 + ], + [ + 504, + 0.7129364971898476 + ], + [ + 517, + 0.8488846583073799 + ], + [ + 521, + 0.5712011747062626 + ], + [ + 530, + 0.1932415004748217 + ], + [ + 535, + 0.08034917320423784 + ], + [ + 539, + 0.7834463191735928 + ], + [ + 550, + 0.43830879484326224 + ], + [ + 554, + 0.09854436673413458 + ], + [ + 559, + 0.14718573381567213 + ], + [ + 562, + 0.8889656029902756 + ], + [ + 571, + 0.16723352115676948 + ], + [ + 575, + 0.20758100991968553 + ], + [ + 581, + 0.295960451933183 + ], + [ + 585, + 0.4609993073816194 + ], + [ + 590, + 0.6466873188537672 + ], + [ + 597, + 0.2752990969138407 + ], + [ + 602, + 0.7096580147375906 + ], + [ + 608, + 0.17623647475444648 + ], + [ + 613, + 0.4032886556915962 + ], + [ + 619, + 0.16378243960649597 + ], + [ + 631, + 0.7137679097181779 + ], + [ + 634, + 0.7446590318965053 + ], + [ + 639, + 0.4193413455109938 + ], + [ + 643, + 0.8484369907411853 + ], + [ + 649, + 0.18308250365803003 + ], + [ + 652, + 0.3101096938589071 + ], + [ + 656, + 0.29121296726999224 + ], + [ + 660, + 0.3148352585114369 + ] + ], + [ + [ + 50, + 0.6422797038955337 + ], + [ + 58, + 0.6934207009221011 + ], + [ + 61, + 0.49640817594807274 + ], + [ + 66, + 0.7724288478172139 + ], + [ + 75, + 0.39600934739417526 + ], + [ + 78, + 0.4627477344382034 + ], + [ + 85, + 0.5078349795403009 + ], + [ + 88, + 0.9988360525390081 + ], + [ + 100, + 0.026690485422848798 + ], + [ + 105, + 0.3680321661937489 + ], + [ + 110, + 0.32299484974977777 + ], + [ + 115, + 0.5562527883665164 + ], + [ + 125, + 0.04049288412519525 + ], + [ + 128, + 0.6193364695002048 + ], + [ + 133, + 0.9210080736098937 + ], + [ + 142, + 0.7229171472534107 + ], + [ + 149, + 0.5854843122371021 + ], + [ + 159, + 0.757081240692507 + ], + [ + 163, + 0.40044288922191484 + ], + [ + 168, + 0.8571777767200383 + ], + [ + 173, + 0.40817403884935577 + ], + [ + 178, + 0.9296736979696516 + ], + [ + 189, + 0.5758837190053285 + ], + [ + 199, + 0.7724813401379371 + ], + [ + 206, + 0.5348802760263863 + ], + [ + 217, + 0.661786398032506 + ], + [ + 221, + 0.6764058091985736 + ], + [ + 225, + 0.4277899112484257 + ], + [ + 232, + 0.7592384634389471 + ], + [ + 240, + 0.9084338231062205 + ], + [ + 249, + 0.8889180265551503 + ], + [ + 255, + 0.5150905967731343 + ], + [ + 259, + 0.26532560890249324 + ], + [ + 266, + 0.8720295978297656 + ], + [ + 270, + 0.5219125895996651 + ], + [ + 274, + 0.06817657401098287 + ], + [ + 279, + 0.9211726073482962 + ], + [ + 285, + 0.28134914474280515 + ], + [ + 292, + 0.9966773891768785 + ], + [ + 295, + 0.17237526261080038 + ], + [ + 297, + 0.4294110943124677 + ], + [ + 304, + 0.7698193266691236 + ], + [ + 316, + 0.6020476537585204 + ], + [ + 321, + 0.3087112650492133 + ], + [ + 324, + 0.8544656843796182 + ], + [ + 330, + 0.984413479694586 + ], + [ + 333, + 0.7608912570377389 + ], + [ + 339, + 0.6230218432568632 + ], + [ + 343, + 0.03128879918304184 + ], + [ + 351, + 0.7864839878887987 + ], + [ + 356, + 0.43362327535793066 + ], + [ + 361, + 0.34633875318052953 + ], + [ + 369, + 0.3824050813418347 + ], + [ + 372, + 0.5998279984099412 + ], + [ + 377, + 0.5395407954605627 + ], + [ + 384, + 0.4058277215525202 + ], + [ + 389, + 0.33279814672823527 + ], + [ + 396, + 0.4331350363340575 + ], + [ + 399, + 0.3270907526113195 + ], + [ + 405, + 0.23779136688787872 + ], + [ + 412, + 0.5110142910127471 + ], + [ + 415, + 0.9080171931515583 + ], + [ + 420, + 0.9090684879887588 + ], + [ + 424, + 0.9262803655785424 + ], + [ + 435, + 0.07808216224986386 + ], + [ + 443, + 0.40020958703753307 + ], + [ + 446, + 0.3576986542943096 + ], + [ + 452, + 0.6620478949162486 + ], + [ + 457, + 0.2640896063996898 + ], + [ + 461, + 0.14195512419501688 + ], + [ + 464, + 0.755335186847766 + ], + [ + 468, + 0.2379075172530839 + ], + [ + 471, + 0.07456230436999944 + ], + [ + 485, + 0.39026385980981926 + ], + [ + 489, + 0.5334206937448264 + ], + [ + 496, + 0.7474824316722386 + ], + [ + 504, + 0.5780347241545671 + ], + [ + 517, + 0.23569914983131757 + ], + [ + 521, + 0.3052867400892182 + ], + [ + 528, + 0.18774685935019297 + ], + [ + 532, + 0.36185279363568945 + ], + [ + 535, + 0.3027767000168563 + ], + [ + 541, + 0.691005938287552 + ], + [ + 546, + 0.39123646976170234 + ], + [ + 550, + 0.08447491808751129 + ], + [ + 558, + 0.5976829086366219 + ], + [ + 563, + 0.11133670969954812 + ], + [ + 566, + 0.38060381864252524 + ], + [ + 571, + 0.4342609151753515 + ], + [ + 576, + 0.9449167106926449 + ], + [ + 582, + 0.637668352901396 + ], + [ + 588, + 0.9355492263865708 + ], + [ + 595, + 0.8333817180813899 + ], + [ + 599, + 0.10710285476346681 + ], + [ + 602, + 0.5560599527094183 + ], + [ + 611, + 0.7510470414101504 + ], + [ + 614, + 0.19949914188533013 + ], + [ + 618, + 0.5617190695639953 + ], + [ + 624, + 0.511777037362084 + ], + [ + 629, + 0.7440889421311377 + ] + ], + [ + [ + 53, + 0.5449624260433773 + ], + [ + 63, + 0.3765899795869678 + ], + [ + 70, + 0.9894650921091898 + ], + [ + 73, + 0.21208951658098563 + ], + [ + 77, + 0.7988046750801162 + ], + [ + 87, + 0.7362408203124704 + ], + [ + 90, + 0.7880420117534773 + ], + [ + 96, + 0.6267066545352198 + ], + [ + 110, + 0.06973596538653448 + ], + [ + 121, + 0.4499489016308217 + ], + [ + 124, + 0.8692652745798012 + ], + [ + 133, + 0.4335345004696497 + ], + [ + 141, + 0.7970260857666713 + ], + [ + 145, + 0.36931068328741345 + ], + [ + 157, + 0.6656417505025083 + ], + [ + 163, + 0.4760550429168492 + ], + [ + 170, + 0.845879980813278 + ], + [ + 178, + 0.4163075030886403 + ], + [ + 187, + 0.394140696366314 + ], + [ + 194, + 0.4990235338336304 + ], + [ + 197, + 0.3186398977144621 + ], + [ + 201, + 0.9268107880336539 + ], + [ + 211, + 0.9285920800333733 + ], + [ + 214, + 0.9371175521971177 + ], + [ + 220, + 0.9504154950059747 + ], + [ + 234, + 0.38836663068375155 + ], + [ + 245, + 0.7407235912999218 + ], + [ + 248, + 0.4429384066471006 + ], + [ + 257, + 0.47503447260579623 + ], + [ + 265, + 0.31296255810123486 + ], + [ + 269, + 0.2569574266281244 + ], + [ + 281, + 0.1343001849015718 + ], + [ + 288, + 0.904361189951702 + ], + [ + 293, + 0.769661648060284 + ], + [ + 299, + 0.06778685858270572 + ], + [ + 303, + 0.37879914474455345 + ], + [ + 314, + 0.3671181123078415 + ], + [ + 317, + 0.6382772496205779 + ], + [ + 322, + 0.21326068941873488 + ], + [ + 325, + 0.22880616662619158 + ], + [ + 329, + 0.23236038914936785 + ], + [ + 337, + 0.7009211752966925 + ], + [ + 343, + 0.6729974598215379 + ], + [ + 352, + 0.7468788838550607 + ], + [ + 355, + 0.47342424222242363 + ], + [ + 357, + 0.5831257014475185 + ], + [ + 370, + 0.8235894169982207 + ], + [ + 382, + 0.114833787125735 + ], + [ + 391, + 0.22653270705518247 + ], + [ + 395, + 0.9502530663570846 + ], + [ + 402, + 0.6600004432522415 + ], + [ + 412, + 0.26400979822619697 + ], + [ + 423, + 0.9797193378686831 + ], + [ + 430, + 0.9073558289720868 + ], + [ + 441, + 0.6843969856808978 + ], + [ + 446, + 0.30827463822792434 + ], + [ + 450, + 0.9395315829849773 + ], + [ + 456, + 0.4938581290689249 + ], + [ + 459, + 0.7187865698692597 + ], + [ + 468, + 0.7529256135103046 + ], + [ + 474, + 0.051473127322253065 + ], + [ + 484, + 0.0020468019208339516 + ], + [ + 488, + 0.3663418144648368 + ], + [ + 494, + 0.8940924356980108 + ], + [ + 500, + 0.9571762829809473 + ], + [ + 506, + 0.5208577422252744 + ], + [ + 512, + 0.17510654793232383 + ], + [ + 518, + 0.31643147558924023 + ], + [ + 524, + 0.6776197197956842 + ], + [ + 528, + 0.5454542459910157 + ], + [ + 536, + 0.006961026610380561 + ], + [ + 543, + 0.009102923719640943 + ], + [ + 547, + 0.6953258152041047 + ], + [ + 551, + 0.5569946048070685 + ], + [ + 557, + 0.2658730726667128 + ], + [ + 566, + 0.8613494947193191 + ], + [ + 570, + 0.1367043545813934 + ], + [ + 575, + 0.6962549432190652 + ], + [ + 581, + 0.6914560236469384 + ], + [ + 585, + 0.19544187852677808 + ], + [ + 590, + 0.029484497685531297 + ], + [ + 597, + 0.42438554903360093 + ], + [ + 604, + 0.06279993607190881 + ], + [ + 607, + 0.7686180212441796 + ], + [ + 613, + 0.02457933056854411 + ], + [ + 617, + 0.7570114120642094 + ], + [ + 625, + 0.1500356730479605 + ], + [ + 630, + 0.5316503764867768 + ], + [ + 633, + 0.5543794565666095 + ], + [ + 637, + 0.5641157196899806 + ], + [ + 643, + 0.6289393340677137 + ], + [ + 647, + 0.22098575481071536 + ], + [ + 660, + 0.197185396514996 + ], + [ + 672, + 0.11303499480480139 + ], + [ + 677, + 0.896460292565938 + ], + [ + 682, + 0.0016286011921740062 + ], + [ + 691, + 0.7675419533115697 + ], + [ + 695, + 0.26034016975070273 + ], + [ + 698, + 0.9159669970884728 + ], + [ + 702, + 0.20315086228728496 + ] + ] + ] + ] + }, + { + "data": { + "question": "Who is the original artist of sound of silence?", + "answer": "There are several songs with the title \"Sound of Silence\". Sounds of Silence is the second studio album by Simon & Garfunkel, released on January 17, 1966. The album's title is a slight modification of the title of the duo's first major hit, \"The Sound of Silence\", which was recorded in March 1964 and originally was released as \"The Sounds of Silence\". Another \"Sound of Silence\" is a song performed by Australian recording artist Dami Im, and is best known as Australia's entry at the Eurovision Song Contest 2016.", + "qa_pairs": [ + { + "context": "Sounds of Silence is the second studio album by Simon & Garfunkel, released on January 17, 1966. The album's title is a slight modification of the title of the duo's first major hit, \"The Sound of Silence\", which originally was released as \"The Sounds of Silence\". The song had earlier been released in an acoustic version on the album \"Wednesday Morning, 3 A.M.\", and later on the soundtrack to the movie \"The Graduate\". Without the knowledge of Paul Simon or Art Garfunkel, electric guitars, bass and drums were overdubbed by Columbia Records staff producer Tom Wilson on June 15, 1965. This new version was released as a single in September 1965, and opens the album.", + "question": "Who is the original artist of sound of silence, the song, released in 1964?", + "short_answers": [ + "Simon & Garfunkel", + "Paul Simon and Art Garfunkel", + "Art Garfunkel", + "Paul Simon" + ], + "wikipage": "Sounds of Silence" + }, + { + "context": "Sounds of Silence is the second studio album by Simon & Garfunkel, released on January 17, 1966. The album's title is a slight modification of the title of the duo's first major hit, \"The Sound of Silence\", which originally was released as \"The Sounds of Silence\". The song had earlier been released in an acoustic version on the album \"Wednesday Morning, 3 A.M.\", and later on the soundtrack to the movie \"The Graduate\". Without the knowledge of Paul Simon or Art Garfunkel, electric guitars, bass and drums were overdubbed by Columbia Records staff producer Tom Wilson on June 15, 1965. This new version was released as a single in September 1965, and opens the album.", + "question": "Who is the original artist of sound of silence, the album?", + "short_answers": [ + "Simon & Garfunkel", + "Paul Simon and Art Garfunkel", + "Art Garfunkel", + "Paul Simon" + ], + "wikipage": "Sounds of Silence" + }, + { + "context": "\"Sound of Silence\" is a song performed by Australian recording artist Dami Im. Written by Anthony Egizii and David Musumeci of DNA Songs, it is best known as Australia's entry at the Eurovision Song Contest 2016 which was held in Stockholm, Sweden, where it finished 2nd, receiving a total of 511 points. The song also won the Marcel Bezen\u00e7on Award in the composer category. The song was leaked on 10 March 2016, one day before its initial release date. It is Dami Im's fourth Australian top 20 hit and worldwide, it reached the top 40 in more than six countries after the Eurovision Song Contest 2016 Final.", + "question": "Who is the original artist of sound of silence, the song, released in 2016?", + "short_answers": [ + "Dami Im" + ], + "wikipage": "Sound of Silence (Dami Im song)" + } + ], + "docs": [ + "Document [1](Title:The Sound of Silence): The Sound of Silence \"The Sound of Silence\", originally \"The Sounds of Silence\", is a song by the American music duo Simon & Garfunkel. The song was written by Paul Simon over a period of several months in 1963 and 1964. A studio audition led to the duo signing a record deal with Columbia Records, and the song was recorded in March 1964 at Columbia Studios in New York City for inclusion on their debut album, \"Wednesday Morning, 3 A.M.\". Released in October 1964, the album was a commercial failure and led to the duo breaking apart, with Paul Simon\n", + "Document [2](Title:Sounds of Silence): Sounds of Silence Sounds of Silence is the second studio album by Simon & Garfunkel, released on January 17, 1966. The album's title is a slight modification of the title of the duo's first major hit, \"The Sound of Silence\", which originally was released as \"The Sounds of Silence\". The song had earlier been released in an acoustic version on the album \"Wednesday Morning, 3 A.M.\", and later on the soundtrack to the movie \"The Graduate\". Without the knowledge of Paul Simon or Art Garfunkel, electric guitars, bass and drums were overdubbed by Columbia Records staff producer Tom Wilson on\n", + "Document [3](Title:The Sound of Silence): downloadable content for the video game, \"Rock Band 4\". The Disturbed version was used in the episode \"Ian Garvey\" of \"The Blacklist\" in November 2017. Notes Bibliography The Sound of Silence \"The Sound of Silence\", originally \"The Sounds of Silence\", is a song by the American music duo Simon & Garfunkel. The song was written by Paul Simon over a period of several months in 1963 and 1964. A studio audition led to the duo signing a record deal with Columbia Records, and the song was recorded in March 1964 at Columbia Studios in New York City for inclusion on\n", + "Document [4](Title:Immortalized (Disturbed album)): 1964 song \"The Sound of Silence\" by Simon & Garfunkel, which marked Disturbed's highest ranked single on the \"Billboard\" Hot 100 at number 42 and has since become Disturbed's biggest song since \"Down with the Sickness\". In 2011, following the tour of their fifth studio album, \"Asylum\", Disturbed announced that they would go on a hiatus. During the hiatus, the band released a compilation album of previously recorded B-sides, \"The Lost Children\" (2011), and a box set of their five studio albums, \"The Collection\" (2012). In January 2014, band members David Draiman (vocals), Dan Donegan (guitar), Mike Wengren (drums) met\n", + "Document [5](Title:Sounds of Silence): Angeles, California. \"The Sound of Silence\" (electric overdubs) personnel \"The Sound of Silence\" overdubs were recorded at Columbia's \"Studio A\" at 799 Seventh Avenue near 52nd Street by Columbia Records staff producer Tom Wilson on June 15, 1965. Sounds of Silence Sounds of Silence is the second studio album by Simon & Garfunkel, released on January 17, 1966. The album's title is a slight modification of the title of the duo's first major hit, \"The Sound of Silence\", which originally was released as \"The Sounds of Silence\". The song had earlier been released in an acoustic version on the album\n" + ] + }, + "doc_cache": [ + "Document [1](Title:The Sound of Silence)attention of Columbia Records producer Tom Wilson, who worked with Bob Dylan. Simon convinced Wilson to let him and his partner have a studio audition, where a performance of \"The Sound of Silence\" got the duo signed to Columbia. The song's origin and basis remain unclear, with multiple answers coming forward over the years. Many believe that the song commented on the John F. Kennedy assassination, as the song was released three months after the assassination. Simon stated unambiguously in interviews, however, \"I wrote The Sound of Silence when I was 21 years old\", which places the timeframe firmly prior\n", + "Document [2](Title:Live at Benaroya Hall with the Seattle Symphony)proud of, believing it truly represents their live act. Overall, critical reception of the album was positive. Nick Vissey of \"The Seattle Times\" wrote that \"standout\" tracks included \"Dreams\" and \"Pride and Joy\", which highlighted Carlile's \"unique folk-rock style\" and \"enthralling, emotional and fun\" voice. The Hanseroth brothers' cover of Paul Simon's \"The Sound of Silence\" was deemed \"spot-on\" by Allmusic's Andrew Leahey and \"breathtaking\" by Gene Stout of the \"Seattle Post-Intelligencer\". \"The Source Weekly\" contributor Mike Bookey complimented the album for capturing Carlile's live act and recommended this album for first-time listeners of her music. \"Glide\" magazine's reviewer complimented\n", + "Document [3](Title:Sounds of Silence)Angeles, California. \"The Sound of Silence\" (electric overdubs) personnel \"The Sound of Silence\" overdubs were recorded at Columbia's \"Studio A\" at 799 Seventh Avenue near 52nd Street by Columbia Records staff producer Tom Wilson on June 15, 1965. Sounds of Silence Sounds of Silence is the second studio album by Simon & Garfunkel, released on January 17, 1966. The album's title is a slight modification of the title of the duo's first major hit, \"The Sound of Silence\", which originally was released as \"The Sounds of Silence\". The song had earlier been released in an acoustic version on the album\n", + "Document [4](Title:The Sound of Silence)downloadable content for the video game, \"Rock Band 4\". The Disturbed version was used in the episode \"Ian Garvey\" of \"The Blacklist\" in November 2017. Notes Bibliography The Sound of Silence \"The Sound of Silence\", originally \"The Sounds of Silence\", is a song by the American music duo Simon & Garfunkel. The song was written by Paul Simon over a period of several months in 1963 and 1964. A studio audition led to the duo signing a record deal with Columbia Records, and the song was recorded in March 1964 at Columbia Studios in New York City for inclusion on\n", + "Document [5](Title:Paul Simon in Concert: Live Rhymin')UK. Further, two of the live performances were released as a single as part of the promotion for the album: the breakthrough Simon & Garfunkel \"The Sound of Silence\" as the A-side, along with Simon's debut single \"Mother and Child Reunion\" on the B-side. All tracks composed by Paul Simon, except where indicated. Paul Simon in Concert: Live Rhymin' Paul Simon in Concert: Live Rhymin' is a live album by Paul Simon, released in March 1974 by Columbia Records. It was recorded in the wake of the release of \"There Goes Rhymin' Simon\" \u2013 which produced a number of hit\n", + "Document [6](Title:The Sound of Silence)attention of Columbia Records producer Tom Wilson, who worked with Bob Dylan. Simon convinced Wilson to let him and his partner have a studio audition, where a performance of \"The Sound of Silence\" got the duo signed to Columbia. The song's origin and basis remain unclear, with multiple answers coming forward over the years. Many believe that the song commented on the John F. Kennedy assassination, as the song was released three months after the assassination. Simon stated unambiguously in interviews, however, \"I wrote The Sound of Silence when I was 21 years old\", which places the timeframe firmly prior\n" + ], + "log": [ + "gpt-3.5-turbo-[1] -> retriever-[4]\n: Query: \"Who wrote the song 'The Sound of Silence'?\"", + "retriever-[4] -> gpt-3.5-turbo-[3]\n: Document [1](Title:The Sound of Silence)attention of Columbia Records producer Tom Wilson, who worked with Bob Dylan. Simon convinced Wilson to let him and his partner have a studio audition, where a performance of \"The Sound of Silence\" got the duo signed to Columbia. The song's origin and basis remain unclear, with multiple answers coming forward over the years. Many believe that the song commented on the John F. Kennedy assassination, as the song was released three months after the assassination. Simon stated unambiguously in interviews, however, \"I wrote The Sound of Silence when I was 21 years old\", which places the timeframe firmly prior\n\nDocument [2](Title:Live at Benaroya Hall with the Seattle Symphony)proud of, believing it truly represents their live act. Overall, critical reception of the album was positive. Nick Vissey of \"The Seattle Times\" wrote that \"standout\" tracks included \"Dreams\" and \"Pride and Joy\", which highlighted Carlile's \"unique folk-rock style\" and \"enthralling, emotional and fun\" voice. The Hanseroth brothers' cover of Paul Simon's \"The Sound of Silence\" was deemed \"spot-on\" by Allmusic's Andrew Leahey and \"breathtaking\" by Gene Stout of the \"Seattle Post-Intelligencer\". \"The Source Weekly\" contributor Mike Bookey complimented the album for capturing Carlile's live act and recommended this album for first-time listeners of her music. \"Glide\" magazine's reviewer complimented\n", + "gpt-3.5-turbo-[3] -> evaluator-[2]\n: ['The original artist of \"The Sound of Silence\" is Simon & Garfunkel[1]. ', 'The original artist of \"Sound of Silence\" is Paul Simon, covered by the Hanseroth brothers[2]. ']", + "evaluator-[2] -> gpt-3.5-turbo-[1]\n: {'answer': 'The original artist of \"Sound of Silence\" is Paul Simon, covered by the Hanseroth brothers[2]. ', 'ans': 'The original artist of \"The Sound of Silence\" is Simon & Garfunkel[1]. '}", + "gpt-3.5-turbo-[1] -> retriever-[4]\n: \"Who recorded the original version of 'The Sound of Silence'?\"", + "retriever-[4] -> gpt-3.5-turbo-[3]\n: Document [3](Title:Sounds of Silence)Angeles, California. \"The Sound of Silence\" (electric overdubs) personnel \"The Sound of Silence\" overdubs were recorded at Columbia's \"Studio A\" at 799 Seventh Avenue near 52nd Street by Columbia Records staff producer Tom Wilson on June 15, 1965. Sounds of Silence Sounds of Silence is the second studio album by Simon & Garfunkel, released on January 17, 1966. The album's title is a slight modification of the title of the duo's first major hit, \"The Sound of Silence\", which originally was released as \"The Sounds of Silence\". The song had earlier been released in an acoustic version on the album\n\nDocument [4](Title:The Sound of Silence)downloadable content for the video game, \"Rock Band 4\". The Disturbed version was used in the episode \"Ian Garvey\" of \"The Blacklist\" in November 2017. Notes Bibliography The Sound of Silence \"The Sound of Silence\", originally \"The Sounds of Silence\", is a song by the American music duo Simon & Garfunkel. The song was written by Paul Simon over a period of several months in 1963 and 1964. A studio audition led to the duo signing a record deal with Columbia Records, and the song was recorded in March 1964 at Columbia Studios in New York City for inclusion on\n", + "gpt-3.5-turbo-[3] -> evaluator-[2]\n: ['The original artist of \"The Sound of Silence\" is Simon & Garfunkel[3]. ', '\"The Sound of Silence\" was originally performed by the American music duo Simon & Garfunkel[4]. ']", + "evaluator-[2] -> gpt-3.5-turbo-[1]\n: {'answer': '\"The Sound of Silence\" was originally performed by the American music duo Simon & Garfunkel[4]. ', 'ans': 'The original artist of \"The Sound of Silence\" is Simon & Garfunkel[3]. '}", + "gpt-3.5-turbo-[1] -> retriever-[4]\n: \"Who composed the song 'The Sound of Silence'?\"", + "retriever-[4] -> gpt-3.5-turbo-[3]\n: Document [5](Title:Paul Simon in Concert: Live Rhymin')UK. Further, two of the live performances were released as a single as part of the promotion for the album: the breakthrough Simon & Garfunkel \"The Sound of Silence\" as the A-side, along with Simon's debut single \"Mother and Child Reunion\" on the B-side. All tracks composed by Paul Simon, except where indicated. Paul Simon in Concert: Live Rhymin' Paul Simon in Concert: Live Rhymin' is a live album by Paul Simon, released in March 1974 by Columbia Records. It was recorded in the wake of the release of \"There Goes Rhymin' Simon\" \u2013 which produced a number of hit\n\nDocument [6](Title:The Sound of Silence)attention of Columbia Records producer Tom Wilson, who worked with Bob Dylan. Simon convinced Wilson to let him and his partner have a studio audition, where a performance of \"The Sound of Silence\" got the duo signed to Columbia. The song's origin and basis remain unclear, with multiple answers coming forward over the years. Many believe that the song commented on the John F. Kennedy assassination, as the song was released three months after the assassination. Simon stated unambiguously in interviews, however, \"I wrote The Sound of Silence when I was 21 years old\", which places the timeframe firmly prior\n", + "gpt-3.5-turbo-[3] -> evaluator-[2]\n: ['The original artist of \"The Sound of Silence\" is Paul Simon, from the duo Simon & Garfunkel[5]. ', 'The original artist of \"The Sound of Silence\" is Simon & Garfunkel, with Paul Simon specifically writing the song[6]. ']", + "evaluator-[2] -> gpt-3.5-turbo-[1]\n: {'answer': 'The original artist of \"The Sound of Silence\" is Paul Simon, from the duo Simon & Garfunkel[5]. '}" + ], + "output": [ + "The original artist of \"Sound of Silence\" is Paul Simon, covered by the Hanseroth brothers [2].", + "\"The Sound of Silence\" was originally performed by the American music duo Simon & Garfunkel [4].", + "The original artist of \"The Sound of Silence\" is Paul Simon, from the duo Simon & Garfunkel [5]." + ], + "result": { + "length": 50 + }, + "token_used": 4669, + "doc_level": [ + -141.40997314453125, + 3.88739013671875, + 11.92987060546875, + 13.794189453125, + 13.39117431640625, + -124.92514038085938 + ], + "span_level": [ + [ + [ + [ + 40, + 0.7681294423101986 + ], + [ + 117, + 0.12539611062153633 + ], + [ + 269, + 0.1812376754362931 + ], + [ + 366, + 0.6756486027645137 + ], + [ + 418, + 0.008627386193367226 + ], + [ + 504, + 0.8095821619308646 + ] + ], + [ + [ + 67, + 0.5113659608324188 + ], + [ + 122, + 0.050920739615043065 + ], + [ + 177, + 0.37493604377783263 + ], + [ + 377, + 0.8771259814975895 + ], + [ + 565, + 0.3151337108264409 + ], + [ + 731, + 0.11020622289044668 + ] + ], + [ + [ + 37, + 0.042786309412687795 + ], + [ + 57, + 0.02282435115060466 + ], + [ + 284, + 0.28643454965628234 + ], + [ + 399, + 0.5465521783962286 + ], + [ + 567, + 0.3610175450258071 + ] + ], + [ + [ + 40, + 0.9661758136948437 + ], + [ + 95, + 0.9580554182775121 + ], + [ + 191, + 0.064585752545268 + ], + [ + 346, + 0.5256548100811523 + ], + [ + 431, + 0.17145880160760307 + ] + ], + [ + [ + 55, + 0.12835290041490688 + ], + [ + 58, + 0.31822335200632035 + ], + [ + 309, + 0.7485066444246173 + ], + [ + 368, + 0.04701415586484314 + ], + [ + 515, + 0.8795424013398948 + ] + ], + [ + [ + 40, + 0.29015652611589926 + ], + [ + 117, + 0.7828885219938004 + ], + [ + 269, + 0.5296553255279034 + ], + [ + 366, + 0.7337078792285606 + ], + [ + 418, + 0.4742902991252723 + ], + [ + 504, + 0.013367543131954429 + ] + ] + ], + [ + [ + [ + 40, + 0.8542689341321462 + ], + [ + 117, + 0.06472050476192825 + ], + [ + 269, + 0.03228351667331575 + ], + [ + 366, + 0.36884822170236686 + ], + [ + 418, + 0.8512291151093191 + ], + [ + 504, + 0.3875923195672669 + ] + ], + [ + [ + 67, + 0.6691845910527903 + ], + [ + 122, + 0.2113431394428288 + ], + [ + 177, + 0.9564582036280341 + ], + [ + 377, + 0.9936133086370688 + ], + [ + 565, + 0.5605392344202653 + ], + [ + 731, + 0.9459072130699254 + ] + ], + [ + [ + 37, + 0.9670972272651837 + ], + [ + 57, + 0.5329611856099494 + ], + [ + 284, + 0.45434707903470006 + ], + [ + 399, + 0.07948614645292662 + ], + [ + 567, + 0.15368079281705493 + ] + ], + [ + [ + 40, + 0.7899463298554238 + ], + [ + 95, + 0.7980724953414075 + ], + [ + 191, + 0.8119536506563895 + ], + [ + 346, + 0.19096097234177056 + ], + [ + 431, + 0.5898819006629646 + ] + ], + [ + [ + 55, + 0.491814913530217 + ], + [ + 58, + 0.6756438574225181 + ], + [ + 309, + 0.6487175118485787 + ], + [ + 368, + 0.9748902149264159 + ], + [ + 515, + 0.2704056330241854 + ] + ], + [ + [ + 40, + 0.6010134450823624 + ], + [ + 117, + 0.4418386006098185 + ], + [ + 269, + 0.6848638927644026 + ], + [ + 366, + 0.06805763117331531 + ], + [ + 418, + 0.26491571112624057 + ], + [ + 504, + 0.34119306073493927 + ] + ] + ], + [ + [ + [ + 40, + 0.8110803508609925 + ], + [ + 117, + 0.8542440349387599 + ], + [ + 269, + 0.5595809823194299 + ], + [ + 366, + 0.6584133157791371 + ], + [ + 418, + 0.42118306154514007 + ], + [ + 504, + 0.5821060684606612 + ] + ], + [ + [ + 67, + 0.719542272342304 + ], + [ + 122, + 0.858851210740909 + ], + [ + 177, + 0.5856103163397098 + ], + [ + 377, + 0.3279682387174606 + ], + [ + 565, + 0.18723590082444175 + ], + [ + 731, + 0.5340128817408214 + ] + ], + [ + [ + 37, + 0.018378128676158734 + ], + [ + 57, + 0.9492933770079399 + ], + [ + 284, + 0.5096428633421242 + ], + [ + 399, + 0.6108366486601625 + ], + [ + 567, + 0.1530182831550987 + ] + ], + [ + [ + 40, + 0.5076920613009369 + ], + [ + 95, + 0.2096825929171856 + ], + [ + 191, + 0.7100441077756393 + ], + [ + 346, + 0.2648455898484564 + ], + [ + 431, + 0.8763368288649335 + ] + ], + [ + [ + 55, + 0.6239206469473701 + ], + [ + 58, + 0.6916514177455892 + ], + [ + 309, + 0.10918969881231355 + ], + [ + 368, + 0.22688107777037947 + ], + [ + 515, + 0.18222589934755773 + ] + ], + [ + [ + 40, + 0.8899701029974798 + ], + [ + 117, + 0.8637501705295068 + ], + [ + 269, + 0.9281316157742715 + ], + [ + 366, + 0.49150581198655263 + ], + [ + 418, + 0.8083819346925372 + ], + [ + 504, + 0.1370956775652542 + ] + ] + ] + ], + "word_level": [ + [ + [ + [ + 40, + 0.9605412385330322 + ], + [ + 50, + 0.18211029022678038 + ], + [ + 53, + 0.46197340337128523 + ], + [ + 62, + 0.8514623812183474 + ], + [ + 70, + 0.8881739745783069 + ], + [ + 79, + 0.30383146896422575 + ], + [ + 83, + 0.7571557717529283 + ], + [ + 91, + 0.9215257366265682 + ], + [ + 95, + 0.11617371511809083 + ], + [ + 102, + 0.5247025069285346 + ], + [ + 107, + 0.3559602977428513 + ], + [ + 111, + 0.8114831912650965 + ], + [ + 118, + 0.016239430249439413 + ], + [ + 124, + 0.48342791830559817 + ], + [ + 134, + 0.13475242715650082 + ], + [ + 141, + 0.06535730633766124 + ], + [ + 144, + 0.18839253353705665 + ], + [ + 148, + 0.6600727744395591 + ], + [ + 152, + 0.3220369146028077 + ], + [ + 156, + 0.7647090737890311 + ], + [ + 160, + 0.6876408933682095 + ], + [ + 168, + 0.5678902367891606 + ], + [ + 173, + 0.6037522617426855 + ], + [ + 175, + 0.2182360968442838 + ], + [ + 182, + 0.5284087318497207 + ], + [ + 192, + 0.597713278621435 + ], + [ + 198, + 0.9558549662886522 + ], + [ + 200, + 0.4381647787249878 + ], + [ + 212, + 0.5293378462992459 + ], + [ + 215, + 0.5331695042785122 + ], + [ + 220, + 0.2750618516215776 + ], + [ + 226, + 0.7054789485657814 + ], + [ + 229, + 0.6225825831961702 + ], + [ + 238, + 0.8975866572552488 + ], + [ + 242, + 0.7641915341944463 + ], + [ + 246, + 0.5450490327746135 + ], + [ + 250, + 0.896763409211327 + ], + [ + 257, + 0.213054129448756 + ], + [ + 260, + 0.667587863746069 + ], + [ + 270, + 0.6373021763524115 + ], + [ + 274, + 0.2780378797144065 + ], + [ + 281, + 0.3734096849949671 + ], + [ + 288, + 0.49562839665170966 + ], + [ + 292, + 0.21584792429524058 + ], + [ + 298, + 0.6524179484556836 + ], + [ + 305, + 0.6574816826181559 + ], + [ + 314, + 0.9194656623289584 + ], + [ + 319, + 0.594105795142457 + ], + [ + 328, + 0.18567840438358074 + ], + [ + 336, + 0.7074095450966162 + ], + [ + 343, + 0.844959070975232 + ], + [ + 351, + 0.9766694470097628 + ], + [ + 356, + 0.11406399949468116 + ], + [ + 360, + 0.3032478892954886 + ], + [ + 367, + 0.5733373316818415 + ], + [ + 372, + 0.17678433727198206 + ], + [ + 380, + 0.03956964408199637 + ], + [ + 385, + 0.14078951322823285 + ], + [ + 389, + 0.6914753312133811 + ], + [ + 394, + 0.9300676750399136 + ], + [ + 404, + 0.8457465934902453 + ], + [ + 407, + 0.4768645463461696 + ], + [ + 411, + 0.6212189384868129 + ], + [ + 416, + 0.47644551243288635 + ], + [ + 419, + 0.6287030465102317 + ], + [ + 427, + 0.6390892371837799 + ], + [ + 442, + 0.39655972755260016 + ], + [ + 445, + 0.8424572459303744 + ], + [ + 449, + 0.9212255977129526 + ], + [ + 454, + 0.3786964583395893 + ], + [ + 458, + 0.30148178485846167 + ], + [ + 467, + 0.15432637051399467 + ], + [ + 473, + 0.6661763734903027 + ], + [ + 480, + 0.6014527967581909 + ], + [ + 486, + 0.4287098459493146 + ], + [ + 490, + 0.0015093473522774792 + ], + [ + 505, + 0.9057086297382925 + ], + [ + 511, + 0.7848634058398484 + ], + [ + 518, + 0.00838119481094346 + ], + [ + 532, + 0.9020393679217024 + ], + [ + 535, + 0.28478690343067814 + ], + [ + 547, + 0.7361323528897495 + ], + [ + 556, + 0.2107945294300878 + ], + [ + 559, + 0.832506451625992 + ], + [ + 565, + 0.05046214850223707 + ], + [ + 569, + 0.9739063304944957 + ], + [ + 575, + 0.7635916938793046 + ], + [ + 578, + 0.1724092507497892 + ], + [ + 586, + 0.7135465919973957 + ], + [ + 591, + 0.3124908012166563 + ], + [ + 593, + 0.020435709951401315 + ], + [ + 597, + 0.5034254995290416 + ], + [ + 600, + 0.5557184666767141 + ], + [ + 606, + 0.6254173249035448 + ], + [ + 612, + 0.6792884009606209 + ], + [ + 618, + 0.2368562884407095 + ], + [ + 625, + 0.17982137109468177 + ], + [ + 629, + 0.5565610663936242 + ], + [ + 639, + 0.5629250845363122 + ], + [ + 646, + 0.2988619231883396 + ] + ], + [ + [ + 67, + 0.7910301666793349 + ], + [ + 73, + 0.5828173040914035 + ], + [ + 77, + 0.5788630316530053 + ], + [ + 87, + 0.8174110204260391 + ], + [ + 90, + 0.3436803282909502 + ], + [ + 96, + 0.5338258589528199 + ], + [ + 107, + 0.819228048246379 + ], + [ + 113, + 0.9406141037396134 + ], + [ + 118, + 0.2883209489751204 + ], + [ + 123, + 0.10905768392063098 + ], + [ + 132, + 0.22474635195828407 + ], + [ + 141, + 0.3001923077294254 + ], + [ + 151, + 0.16704265810603358 + ], + [ + 154, + 0.22595833122660391 + ], + [ + 158, + 0.295876677149536 + ], + [ + 164, + 0.9556763772199932 + ], + [ + 168, + 0.5172947961277051 + ], + [ + 178, + 0.4819115725117782 + ], + [ + 183, + 0.6668203747401273 + ], + [ + 190, + 0.19166864261253913 + ], + [ + 193, + 0.15987236912271408 + ], + [ + 198, + 0.12885449615863098 + ], + [ + 206, + 0.9085684950333187 + ], + [ + 213, + 0.4208488731661424 + ], + [ + 219, + 0.6239560568416535 + ], + [ + 224, + 0.3997041390158017 + ], + [ + 235, + 0.7678612706644352 + ], + [ + 242, + 0.5855043320097695 + ], + [ + 251, + 0.9515547884986183 + ], + [ + 260, + 0.7257598421388842 + ], + [ + 264, + 0.11846465317411037 + ], + [ + 271, + 0.6141767866912936 + ], + [ + 275, + 0.8349010986352566 + ], + [ + 281, + 0.29651052159571667 + ], + [ + 287, + 0.4597744575617866 + ], + [ + 299, + 0.7373314518791655 + ], + [ + 309, + 0.2622008372539516 + ], + [ + 317, + 0.1504487937488398 + ], + [ + 327, + 0.052192422356992396 + ], + [ + 334, + 0.8796708844554361 + ], + [ + 338, + 0.38814745323583 + ], + [ + 352, + 0.47298730664084954 + ], + [ + 362, + 0.09980488721659309 + ], + [ + 366, + 0.5439061984705799 + ], + [ + 371, + 0.41355870423765273 + ], + [ + 378, + 0.6488231269614887 + ], + [ + 382, + 0.6999486344752295 + ], + [ + 392, + 0.2760171955275086 + ], + [ + 402, + 0.8512497011841459 + ], + [ + 408, + 0.7625237822894922 + ], + [ + 411, + 0.9875244500528946 + ], + [ + 416, + 0.5317066145751385 + ], + [ + 424, + 0.8750858130886776 + ], + [ + 429, + 0.25178902097489564 + ], + [ + 435, + 0.8050722862824983 + ], + [ + 438, + 0.9264675808461766 + ], + [ + 447, + 0.9821943780834063 + ], + [ + 451, + 0.09386482722919065 + ], + [ + 458, + 0.510145236378245 + ], + [ + 468, + 0.12665088551489345 + ], + [ + 471, + 0.28452757816831764 + ], + [ + 482, + 0.8861424709357505 + ], + [ + 489, + 0.6962254682695743 + ], + [ + 496, + 0.7153338353131422 + ], + [ + 500, + 0.6847134624277891 + ], + [ + 515, + 0.33875827548474735 + ], + [ + 518, + 0.6232745923900574 + ], + [ + 523, + 0.00548181507509149 + ], + [ + 529, + 0.7383013123751838 + ], + [ + 532, + 0.34497758761720787 + ], + [ + 536, + 0.0504738093671363 + ], + [ + 545, + 0.8093244296828489 + ], + [ + 566, + 0.6883814297208313 + ], + [ + 571, + 0.4040019784033618 + ], + [ + 578, + 0.5499571324350097 + ], + [ + 586, + 0.1963431049937291 + ], + [ + 598, + 0.8759314118158871 + ], + [ + 603, + 0.14273888895445552 + ], + [ + 610, + 0.8026488721682715 + ], + [ + 623, + 0.37509103467380966 + ], + [ + 627, + 0.41136257561182865 + ], + [ + 633, + 0.924691965506206 + ], + [ + 637, + 0.3593135216895569 + ], + [ + 647, + 0.3494180576235135 + ], + [ + 657, + 0.15684870701962939 + ], + [ + 662, + 0.5363543836015993 + ], + [ + 666, + 0.6709684788287471 + ], + [ + 670, + 0.8857130098438557 + ], + [ + 682, + 0.7991505627786832 + ], + [ + 687, + 0.573911464432348 + ], + [ + 693, + 0.18897229448088215 + ], + [ + 697, + 0.5315569893565631 + ], + [ + 708, + 0.5167899116278792 + ], + [ + 718, + 0.9298917351552454 + ], + [ + 721, + 0.7721478868437628 + ], + [ + 725, + 0.8737601696081212 + ], + [ + 732, + 0.6335984168082618 + ], + [ + 740, + 0.156463113808762 + ], + [ + 751, + 0.5315651393570353 + ], + [ + 760, + 0.14940858180382088 + ] + ], + [ + [ + 37, + 0.5473416468869303 + ], + [ + 46, + 0.9198465840747733 + ], + [ + 58, + 0.8399769097076705 + ], + [ + 63, + 0.47083068076091894 + ], + [ + 69, + 0.7175407978863261 + ], + [ + 72, + 0.3428675001334993 + ], + [ + 81, + 0.699590696133906 + ], + [ + 91, + 0.19512205692796047 + ], + [ + 101, + 0.6710554418759244 + ], + [ + 111, + 0.6245834834532189 + ], + [ + 116, + 0.5062594316573675 + ], + [ + 122, + 0.677142153906883 + ], + [ + 125, + 0.8608142472949045 + ], + [ + 134, + 0.10253269366675766 + ], + [ + 143, + 0.6188694672722853 + ], + [ + 148, + 0.4257171081385801 + ], + [ + 157, + 0.22570900921732795 + ], + [ + 160, + 0.9317800466740576 + ], + [ + 171, + 0.17091416030212359 + ], + [ + 179, + 0.20793976793312607 + ], + [ + 182, + 0.35709482969283346 + ], + [ + 185, + 0.14436560576805535 + ], + [ + 189, + 0.13018217113975428 + ], + [ + 197, + 0.779686124921897 + ], + [ + 204, + 0.4557996832810477 + ], + [ + 209, + 0.4343245823343663 + ], + [ + 214, + 0.39135371406492725 + ], + [ + 221, + 0.8223504732280361 + ], + [ + 224, + 0.9026689440474712 + ], + [ + 233, + 0.9999723638434582 + ], + [ + 241, + 0.8411749835652765 + ], + [ + 247, + 0.8924764827199527 + ], + [ + 256, + 0.1768370898769478 + ], + [ + 260, + 0.635363699272674 + ], + [ + 267, + 0.16031170546606277 + ], + [ + 270, + 0.6162658621346826 + ], + [ + 275, + 0.34969508638890134 + ], + [ + 279, + 0.13960252003876605 + ], + [ + 285, + 0.07888149856681503 + ], + [ + 292, + 0.04276062925959845 + ], + [ + 295, + 0.5575547661857926 + ], + [ + 303, + 0.6231652431404555 + ], + [ + 310, + 0.9068274462249533 + ], + [ + 313, + 0.7367754234006907 + ], + [ + 321, + 0.5375368264775117 + ], + [ + 324, + 0.3009237329396848 + ], + [ + 328, + 0.062482412690374134 + ], + [ + 335, + 0.45228579383122247 + ], + [ + 342, + 0.7073229240545774 + ], + [ + 348, + 0.39570367041146715 + ], + [ + 351, + 0.3668825596748191 + ], + [ + 357, + 0.3368359440772296 + ], + [ + 359, + 0.6003006175236284 + ], + [ + 370, + 0.09448586781305313 + ], + [ + 379, + 0.16375945203755782 + ], + [ + 382, + 0.8184858233336592 + ], + [ + 390, + 0.7932732664220652 + ], + [ + 394, + 0.6149599405647693 + ], + [ + 400, + 0.6620313127160088 + ], + [ + 404, + 0.6033955840919228 + ], + [ + 412, + 0.019130548072651243 + ], + [ + 418, + 0.7860380477563449 + ], + [ + 421, + 0.5322939986484965 + ], + [ + 423, + 0.5569504672778084 + ], + [ + 430, + 0.14092412798720755 + ], + [ + 443, + 0.8331361816964848 + ], + [ + 446, + 0.6214486172972558 + ], + [ + 450, + 0.6064458922645708 + ], + [ + 456, + 0.2778222153388009 + ], + [ + 459, + 0.2171492966191474 + ], + [ + 463, + 0.8907740205667246 + ], + [ + 469, + 0.835568512437555 + ], + [ + 475, + 0.9171495117951879 + ], + [ + 481, + 0.4852568030023173 + ], + [ + 486, + 0.6306989115121965 + ], + [ + 491, + 0.5880025316922137 + ], + [ + 497, + 0.6958261533099146 + ], + [ + 500, + 0.6805757785978933 + ], + [ + 510, + 0.38341461036079816 + ], + [ + 516, + 0.7516372895807764 + ], + [ + 527, + 0.8371585529564474 + ], + [ + 531, + 0.671134400891974 + ], + [ + 540, + 0.6838291898585097 + ], + [ + 543, + 0.4904029268321288 + ], + [ + 548, + 0.6938093471181668 + ], + [ + 555, + 0.6672875761382534 + ], + [ + 558, + 0.11829677038117481 + ], + [ + 568, + 0.23812036859743524 + ], + [ + 572, + 0.10953744802041421 + ], + [ + 577, + 0.8390410859764094 + ], + [ + 581, + 0.8790268666619699 + ], + [ + 589, + 0.07279638189211346 + ], + [ + 594, + 0.3597021157027117 + ], + [ + 603, + 0.5961460806683614 + ], + [ + 606, + 0.024878525552953112 + ], + [ + 609, + 0.21767344995993543 + ], + [ + 618, + 0.10773438942212421 + ], + [ + 626, + 0.747527975999817 + ], + [ + 629, + 0.022802851775995392 + ], + [ + 633, + 0.4091772526404194 + ] + ], + [ + [ + 40, + 0.9401963027067148 + ], + [ + 53, + 0.9154527320220854 + ], + [ + 61, + 0.8569996552770855 + ], + [ + 65, + 0.20442932262906421 + ], + [ + 69, + 0.13274032124105728 + ], + [ + 75, + 0.6950676932881548 + ], + [ + 81, + 0.3443167026980536 + ], + [ + 87, + 0.7406920215155557 + ], + [ + 92, + 0.2756459289843035 + ], + [ + 96, + 0.33375330178817564 + ], + [ + 100, + 0.9999776246754618 + ], + [ + 110, + 0.12510901127980356 + ], + [ + 118, + 0.6083822017335674 + ], + [ + 122, + 0.4004968648095719 + ], + [ + 127, + 0.5395696219791905 + ], + [ + 130, + 0.1881752208256413 + ], + [ + 134, + 0.4120454769152827 + ], + [ + 142, + 0.41379395586650847 + ], + [ + 147, + 0.19540695350638038 + ], + [ + 155, + 0.7882483254760861 + ], + [ + 158, + 0.7444574006475361 + ], + [ + 163, + 0.05905829592193246 + ], + [ + 174, + 0.361986002141347 + ], + [ + 177, + 0.5090400185465178 + ], + [ + 186, + 0.9661223391833355 + ], + [ + 192, + 0.5882189262096941 + ], + [ + 198, + 0.2798115538926724 + ], + [ + 211, + 0.39409296319829923 + ], + [ + 215, + 0.9116642182058046 + ], + [ + 221, + 0.6617136836992278 + ], + [ + 224, + 0.385873654797675 + ], + [ + 232, + 0.42035743932890957 + ], + [ + 237, + 0.6009549488894071 + ], + [ + 243, + 0.48560167522890585 + ], + [ + 246, + 0.37991280299744146 + ], + [ + 256, + 0.5127160829194604 + ], + [ + 267, + 0.3548191125451735 + ], + [ + 272, + 0.6454887326632138 + ], + [ + 279, + 0.26194279854174884 + ], + [ + 282, + 0.5853488766287414 + ], + [ + 292, + 0.12007498633963876 + ], + [ + 295, + 0.32966325632955384 + ], + [ + 297, + 0.6857495217791901 + ], + [ + 302, + 0.34797793603451077 + ], + [ + 305, + 0.618360673602469 + ], + [ + 309, + 0.4175555532252542 + ], + [ + 318, + 0.4925807960410661 + ], + [ + 324, + 0.9431792795032607 + ], + [ + 328, + 0.3687808003109294 + ], + [ + 334, + 0.9419066794691522 + ], + [ + 336, + 0.42315407546279593 + ], + [ + 347, + 0.11007087231440105 + ], + [ + 351, + 0.856511341975634 + ], + [ + 356, + 0.9584162887455749 + ], + [ + 360, + 0.5399662507169477 + ], + [ + 368, + 0.9666946922905914 + ], + [ + 371, + 0.017680851868121072 + ], + [ + 376, + 0.2888767459698799 + ], + [ + 382, + 0.5396417507021747 + ], + [ + 387, + 0.8918578417240931 + ], + [ + 389, + 0.27878192484343634 + ], + [ + 396, + 0.48767988317120115 + ], + [ + 399, + 0.7665422972633305 + ], + [ + 407, + 0.4669208817358491 + ], + [ + 414, + 0.4654201848694455 + ], + [ + 417, + 0.053848394393243626 + ], + [ + 422, + 0.4671061478554861 + ], + [ + 426, + 0.08268469027876046 + ], + [ + 432, + 0.6064589045331252 + ], + [ + 434, + 0.379530142593958 + ], + [ + 441, + 0.7027683891996319 + ], + [ + 450, + 0.43506948690835634 + ], + [ + 454, + 0.19611675819469754 + ], + [ + 457, + 0.7073624135744019 + ], + [ + 461, + 0.17508401288347142 + ], + [ + 465, + 0.33715691102100764 + ], + [ + 473, + 0.5814652674499443 + ], + [ + 475, + 0.21457563601114282 + ], + [ + 482, + 0.6835139262585697 + ], + [ + 487, + 0.6123135002603773 + ], + [ + 492, + 0.4666166818608569 + ], + [ + 501, + 0.15985935151306108 + ], + [ + 510, + 0.4544848249686918 + ], + [ + 514, + 0.8644697216196301 + ], + [ + 518, + 0.5858595832365663 + ], + [ + 523, + 0.9411041763538076 + ], + [ + 527, + 0.46209921591985637 + ], + [ + 536, + 0.8310933493148706 + ], + [ + 539, + 0.7661912750016969 + ], + [ + 545, + 0.9221242764040725 + ], + [ + 550, + 0.9556166071620322 + ], + [ + 553, + 0.5898327318623012 + ], + [ + 562, + 0.7486929371152691 + ], + [ + 570, + 0.4198879201594182 + ], + [ + 573, + 0.9878137110983661 + ], + [ + 577, + 0.8694050225870468 + ], + [ + 582, + 0.7345086832838233 + ], + [ + 587, + 0.07812579182177926 + ], + [ + 591, + 0.37266908380365316 + ], + [ + 601, + 0.06915245726919417 + ] + ], + [ + [ + 55, + 0.007538735117148532 + ], + [ + 59, + 0.47706456244514317 + ], + [ + 68, + 0.4432275023911466 + ], + [ + 72, + 0.05089198669364514 + ], + [ + 75, + 0.35230871043576373 + ], + [ + 79, + 0.15987263858563183 + ], + [ + 84, + 0.5650785919616889 + ], + [ + 97, + 0.6635616405529976 + ], + [ + 102, + 0.97286453776442 + ], + [ + 111, + 0.39492403348597616 + ], + [ + 114, + 0.11262841399195389 + ], + [ + 116, + 0.4502905599116007 + ], + [ + 123, + 0.6118436064682393 + ], + [ + 126, + 0.19586648442588517 + ], + [ + 131, + 0.4528947187515795 + ], + [ + 134, + 0.06336583113215932 + ], + [ + 138, + 0.4049021292320646 + ], + [ + 148, + 0.41852420330274454 + ], + [ + 152, + 0.3256952869317644 + ], + [ + 156, + 0.9325653361130937 + ], + [ + 163, + 0.9531469663883744 + ], + [ + 167, + 0.665220224928726 + ], + [ + 180, + 0.026622250904397515 + ], + [ + 186, + 0.5635309117230937 + ], + [ + 188, + 0.23223831837317943 + ], + [ + 198, + 0.8182089160183806 + ], + [ + 203, + 0.3934966278041334 + ], + [ + 209, + 0.38852929899101574 + ], + [ + 212, + 0.08425495156436569 + ], + [ + 221, + 0.9791231194218332 + ], + [ + 224, + 0.0068715788203177475 + ], + [ + 228, + 0.4233146885999365 + ], + [ + 236, + 0.7575801369219444 + ], + [ + 242, + 0.038526498838275525 + ], + [ + 247, + 0.313155596858233 + ], + [ + 255, + 0.031032417663927836 + ], + [ + 261, + 0.49034837992763713 + ], + [ + 268, + 0.15450805389729083 + ], + [ + 276, + 0.9063450117660965 + ], + [ + 280, + 0.44009863465304444 + ], + [ + 286, + 0.5152508970914618 + ], + [ + 295, + 0.7200633336790611 + ], + [ + 298, + 0.3218199117974092 + ], + [ + 302, + 0.8718719431667709 + ], + [ + 310, + 0.51612001737524 + ], + [ + 314, + 0.6248237531726877 + ], + [ + 321, + 0.19932383986574087 + ], + [ + 330, + 0.9974527070030615 + ], + [ + 333, + 0.23375380200632934 + ], + [ + 338, + 0.6804718837238033 + ], + [ + 345, + 0.3405330655921949 + ], + [ + 352, + 0.25119594284625213 + ], + [ + 358, + 0.49611000398481875 + ], + [ + 369, + 0.2343273736930337 + ], + [ + 374, + 0.8889785212880931 + ], + [ + 380, + 0.8632456225071109 + ], + [ + 383, + 0.24172531003072506 + ], + [ + 392, + 0.05146965314832175 + ], + [ + 397, + 0.8017028233622493 + ], + [ + 405, + 0.17773336920393235 + ], + [ + 410, + 0.13659925695374397 + ], + [ + 416, + 0.006236365854376591 + ], + [ + 419, + 0.4368471859679611 + ], + [ + 428, + 0.7323631301424366 + ], + [ + 433, + 0.5593610349888041 + ], + [ + 441, + 0.1408868637743319 + ], + [ + 444, + 0.9783093897951937 + ], + [ + 446, + 0.3513605569940006 + ], + [ + 451, + 0.9230139835960286 + ], + [ + 457, + 0.005511076348878596 + ], + [ + 460, + 0.4302843003101212 + ], + [ + 465, + 0.5698986630841959 + ], + [ + 472, + 0.7628370830250785 + ], + [ + 481, + 0.021974738441010166 + ], + [ + 484, + 0.6384374304219848 + ], + [ + 490, + 0.6518161156435068 + ], + [ + 495, + 0.6945374450863386 + ], + [ + 498, + 0.4480394467939247 + ], + [ + 507, + 0.5704296663261712 + ], + [ + 516, + 0.7397896966571698 + ], + [ + 519, + 0.6105997132733016 + ], + [ + 523, + 0.44661522458116965 + ], + [ + 532, + 0.2915108722239006 + ], + [ + 535, + 0.005804380065666415 + ], + [ + 539, + 0.7540144264801911 + ], + [ + 544, + 0.6283706696054371 + ], + [ + 547, + 0.6708396095867315 + ], + [ + 551, + 0.8323307903859924 + ], + [ + 559, + 0.9769735282438531 + ], + [ + 562, + 0.1011401550509925 + ], + [ + 569, + 0.7204914573394465 + ], + [ + 574, + 0.9304036806323365 + ], + [ + 582, + 0.5450761866482776 + ], + [ + 589, + 0.044387492306080745 + ], + [ + 591, + 0.8837529085320627 + ], + [ + 597, + 0.08082490051734481 + ], + [ + 606, + 0.3160376519326964 + ], + [ + 608, + 0.41993823561267396 + ], + [ + 615, + 0.5493911405723058 + ], + [ + 618, + 0.11191977019583066 + ] + ], + [ + [ + 40, + 0.15465263971594911 + ], + [ + 50, + 0.8276002943776449 + ], + [ + 53, + 0.5989023701418245 + ], + [ + 62, + 0.5200962525904713 + ], + [ + 70, + 0.11286095555529851 + ], + [ + 79, + 0.053246374033044 + ], + [ + 83, + 0.11217406862709534 + ], + [ + 91, + 0.06569128938814606 + ], + [ + 95, + 0.9305942468265554 + ], + [ + 102, + 0.7842577276500082 + ], + [ + 107, + 0.33317591808821523 + ], + [ + 111, + 0.6930292348493342 + ], + [ + 118, + 0.028824759735432526 + ], + [ + 124, + 0.6084070778451751 + ], + [ + 134, + 0.44781429273906526 + ], + [ + 141, + 0.5249298376456489 + ], + [ + 144, + 0.9141801317749775 + ], + [ + 148, + 0.25738897587459475 + ], + [ + 152, + 0.6051593679539709 + ], + [ + 156, + 0.493899788043362 + ], + [ + 160, + 0.36517000984772563 + ], + [ + 168, + 0.5845664083866664 + ], + [ + 173, + 0.7500207538539965 + ], + [ + 175, + 0.4160800125921108 + ], + [ + 182, + 0.6994382445065684 + ], + [ + 192, + 0.6010115381019006 + ], + [ + 198, + 0.14479774536703915 + ], + [ + 200, + 0.09646301599243512 + ], + [ + 212, + 0.9691734589876208 + ], + [ + 215, + 0.12224289401769839 + ], + [ + 220, + 0.9801323818050047 + ], + [ + 226, + 0.5493423833761154 + ], + [ + 229, + 0.36456654893523466 + ], + [ + 238, + 0.6701895571928304 + ], + [ + 242, + 0.6737419882236179 + ], + [ + 246, + 0.9209337068815381 + ], + [ + 250, + 0.43510452898369345 + ], + [ + 257, + 0.9449079742605331 + ], + [ + 260, + 0.2879453082576502 + ], + [ + 270, + 0.24989978925229572 + ], + [ + 274, + 0.6629434300757326 + ], + [ + 281, + 0.48279295001990097 + ], + [ + 288, + 0.6365353618220866 + ], + [ + 292, + 0.4909690926841205 + ], + [ + 298, + 0.35027267160329856 + ], + [ + 305, + 0.0751663768287244 + ], + [ + 314, + 0.19061859429676264 + ], + [ + 319, + 0.4040294944706423 + ], + [ + 328, + 0.05147395385206632 + ], + [ + 336, + 0.5081667483785475 + ], + [ + 343, + 0.7613945824935878 + ], + [ + 351, + 0.3987661497300964 + ], + [ + 356, + 0.5611170988941767 + ], + [ + 360, + 0.7090155842223429 + ], + [ + 367, + 0.4820144879295295 + ], + [ + 372, + 0.035632958822508165 + ], + [ + 380, + 0.03775756763638294 + ], + [ + 385, + 0.9320701071022266 + ], + [ + 389, + 0.688047300092107 + ], + [ + 394, + 0.5045210149997502 + ], + [ + 404, + 0.5684377391199208 + ], + [ + 407, + 0.028460703196245274 + ], + [ + 411, + 0.4343245101692782 + ], + [ + 416, + 0.35811774672755636 + ], + [ + 419, + 0.7696885634409204 + ], + [ + 427, + 0.20015285493259993 + ], + [ + 442, + 0.09050428253973952 + ], + [ + 445, + 0.1604493471860432 + ], + [ + 449, + 0.9074488272011187 + ], + [ + 454, + 0.7637357178368924 + ], + [ + 458, + 0.5990546496010695 + ], + [ + 467, + 0.17659122396180849 + ], + [ + 473, + 0.5436636742106011 + ], + [ + 480, + 0.07557988868979593 + ], + [ + 486, + 0.4046814545287021 + ], + [ + 490, + 0.11396315290986658 + ], + [ + 505, + 0.2594935413184333 + ], + [ + 511, + 0.15620759070248436 + ], + [ + 518, + 0.29416198005454774 + ], + [ + 532, + 0.5378646338166136 + ], + [ + 535, + 0.2150510964333826 + ], + [ + 547, + 0.5223601803281631 + ], + [ + 556, + 0.23865728356578297 + ], + [ + 559, + 0.7475067668977392 + ], + [ + 565, + 0.6654923287792893 + ], + [ + 569, + 0.031631757522018855 + ], + [ + 575, + 0.0766968543487857 + ], + [ + 578, + 0.8222146377097642 + ], + [ + 586, + 0.41977582202778696 + ], + [ + 591, + 0.7442859877726449 + ], + [ + 593, + 0.029922708674847365 + ], + [ + 597, + 0.9807746274522833 + ], + [ + 600, + 0.25088077545916254 + ], + [ + 606, + 0.15049728862272882 + ], + [ + 612, + 0.138034621994498 + ], + [ + 618, + 0.30626249416520057 + ], + [ + 625, + 0.2359332587758003 + ], + [ + 629, + 0.6485725187589244 + ], + [ + 639, + 0.1443879414318614 + ], + [ + 646, + 0.5968773876541607 + ] + ] + ], + [ + [ + [ + 40, + 0.25446610348375787 + ], + [ + 50, + 0.6726782163636316 + ], + [ + 53, + 0.6777009741729352 + ], + [ + 62, + 0.5560729927203558 + ], + [ + 70, + 0.9926375475494202 + ], + [ + 79, + 0.3843660170614027 + ], + [ + 83, + 0.5166321564575784 + ], + [ + 91, + 0.011573090685839338 + ], + [ + 95, + 0.18398175206865786 + ], + [ + 102, + 0.48131608214527166 + ], + [ + 107, + 0.19520944850110533 + ], + [ + 111, + 0.3045877516136145 + ], + [ + 118, + 0.949415229979485 + ], + [ + 124, + 0.2823140253322305 + ], + [ + 134, + 0.3756343627417513 + ], + [ + 141, + 0.9848694940720513 + ], + [ + 144, + 0.9947970215376665 + ], + [ + 148, + 0.2824346772427464 + ], + [ + 152, + 0.7186267556120499 + ], + [ + 156, + 0.9438532658938501 + ], + [ + 160, + 0.9572828582559778 + ], + [ + 168, + 0.08978256480483104 + ], + [ + 173, + 0.0819863476278413 + ], + [ + 175, + 0.3344912070813042 + ], + [ + 182, + 0.4893797030491317 + ], + [ + 192, + 0.6180102030720818 + ], + [ + 198, + 0.8639236827022585 + ], + [ + 200, + 0.21189889080770563 + ], + [ + 212, + 0.1908690920719166 + ], + [ + 215, + 0.42485255353888507 + ], + [ + 220, + 0.13636924920803362 + ], + [ + 226, + 0.678038837388657 + ], + [ + 229, + 0.7427329934255535 + ], + [ + 238, + 0.6382901296746666 + ], + [ + 242, + 0.973527981344339 + ], + [ + 246, + 0.5775892206126588 + ], + [ + 250, + 0.8409564764989682 + ], + [ + 257, + 0.7526912687719403 + ], + [ + 260, + 0.24068688465422616 + ], + [ + 270, + 0.6963848852530731 + ], + [ + 274, + 0.6913157083195889 + ], + [ + 281, + 0.4626524336061675 + ], + [ + 288, + 0.9328217245177592 + ], + [ + 292, + 0.6573917192404184 + ], + [ + 298, + 0.6774422127160985 + ], + [ + 305, + 0.0123737141128335 + ], + [ + 314, + 0.48000332927915046 + ], + [ + 319, + 0.3549794071382155 + ], + [ + 328, + 0.19271879593328023 + ], + [ + 336, + 0.7618064658117225 + ], + [ + 343, + 0.5356474595385792 + ], + [ + 351, + 0.4616632667524837 + ], + [ + 356, + 0.6710479148564595 + ], + [ + 360, + 0.12678351719639447 + ], + [ + 367, + 0.9804391079369325 + ], + [ + 372, + 0.9113349620807667 + ], + [ + 380, + 0.49313991251688627 + ], + [ + 385, + 0.6614671738173012 + ], + [ + 389, + 0.22007777721672972 + ], + [ + 394, + 0.8159914204970277 + ], + [ + 404, + 0.3038135837030488 + ], + [ + 407, + 0.03761518149890708 + ], + [ + 411, + 0.9876027556839924 + ], + [ + 416, + 0.054233342555295994 + ], + [ + 419, + 0.7352418910242043 + ], + [ + 427, + 0.3286269665771887 + ], + [ + 442, + 0.8431321359745975 + ], + [ + 445, + 0.8314708675188835 + ], + [ + 449, + 0.42546244284691137 + ], + [ + 454, + 0.44868133564687607 + ], + [ + 458, + 0.6689109337740411 + ], + [ + 467, + 0.14441932351629805 + ], + [ + 473, + 0.11526729447422535 + ], + [ + 480, + 0.24261745004059931 + ], + [ + 486, + 0.948228214080521 + ], + [ + 490, + 0.188614805420382 + ], + [ + 505, + 0.48567804805787296 + ], + [ + 511, + 0.38351805676994066 + ], + [ + 518, + 0.42188702063128347 + ], + [ + 532, + 0.7072976398136354 + ], + [ + 535, + 0.21752980212238338 + ], + [ + 547, + 0.03215065509658488 + ], + [ + 556, + 0.5921417179150742 + ], + [ + 559, + 0.2471288556062523 + ], + [ + 565, + 0.2755387601066205 + ], + [ + 569, + 0.4285930429116339 + ], + [ + 575, + 0.5267311070755851 + ], + [ + 578, + 0.04735947759820158 + ], + [ + 586, + 0.9134983440039472 + ], + [ + 591, + 0.9733375829715871 + ], + [ + 593, + 0.8405057756844512 + ], + [ + 597, + 0.16818116376699255 + ], + [ + 600, + 0.1491165074652463 + ], + [ + 606, + 0.264854335996626 + ], + [ + 612, + 0.25140740641639714 + ], + [ + 618, + 0.7234734559366994 + ], + [ + 625, + 0.15196278555010145 + ], + [ + 629, + 0.7498676986096013 + ], + [ + 639, + 0.8947505686152156 + ], + [ + 646, + 0.9702555148845312 + ] + ], + [ + [ + 67, + 0.8379237193289492 + ], + [ + 73, + 0.7508085059885482 + ], + [ + 77, + 0.04477391085185056 + ], + [ + 87, + 0.2282735732608313 + ], + [ + 90, + 0.2183875543989645 + ], + [ + 96, + 0.289089156924083 + ], + [ + 107, + 0.7974409975319744 + ], + [ + 113, + 0.6998070448194156 + ], + [ + 118, + 0.6185297019970959 + ], + [ + 123, + 0.9928807168781009 + ], + [ + 132, + 0.8350604199547895 + ], + [ + 141, + 0.6036983439236547 + ], + [ + 151, + 0.1637574002402553 + ], + [ + 154, + 0.9920713528404963 + ], + [ + 158, + 0.07704270499527 + ], + [ + 164, + 0.682930015863718 + ], + [ + 168, + 0.45962912897962716 + ], + [ + 178, + 0.9403763139064625 + ], + [ + 183, + 0.5177280117198294 + ], + [ + 190, + 0.5580517871052273 + ], + [ + 193, + 0.25100753396539066 + ], + [ + 198, + 0.8225916715291338 + ], + [ + 206, + 0.44849218802291835 + ], + [ + 213, + 0.04811841596578359 + ], + [ + 219, + 0.2083467570148807 + ], + [ + 224, + 0.14096488084232994 + ], + [ + 235, + 0.6463098579451981 + ], + [ + 242, + 0.9941890934347782 + ], + [ + 251, + 0.15931101837379125 + ], + [ + 260, + 0.3001364595789593 + ], + [ + 264, + 0.3668056892875412 + ], + [ + 271, + 0.867401241036848 + ], + [ + 275, + 0.7473099348145247 + ], + [ + 281, + 0.4225786435460156 + ], + [ + 287, + 0.3637839799229824 + ], + [ + 299, + 0.7192445459814863 + ], + [ + 309, + 0.8524407869019279 + ], + [ + 317, + 0.2499148072137013 + ], + [ + 327, + 0.6562126806648912 + ], + [ + 334, + 0.6992441528125181 + ], + [ + 338, + 0.8941749094413923 + ], + [ + 352, + 0.013646047790266125 + ], + [ + 362, + 0.5508233440350615 + ], + [ + 366, + 0.6974518874004897 + ], + [ + 371, + 0.25517023957977214 + ], + [ + 378, + 0.4828063144306548 + ], + [ + 382, + 0.19179566920367475 + ], + [ + 392, + 0.7779591968372589 + ], + [ + 402, + 0.44966127697858205 + ], + [ + 408, + 0.8488827802649651 + ], + [ + 411, + 0.6056319934091198 + ], + [ + 416, + 0.8134103746691269 + ], + [ + 424, + 0.5329230730249622 + ], + [ + 429, + 0.9384526252869863 + ], + [ + 435, + 0.06033899974202739 + ], + [ + 438, + 0.42174840346838494 + ], + [ + 447, + 0.45197451679566303 + ], + [ + 451, + 0.3714776026073172 + ], + [ + 458, + 0.5846611478648899 + ], + [ + 468, + 0.288818895698427 + ], + [ + 471, + 0.7417707698641056 + ], + [ + 482, + 0.4631345828406559 + ], + [ + 489, + 0.19336224362773302 + ], + [ + 496, + 0.8829460879851629 + ], + [ + 500, + 0.4041213479331378 + ], + [ + 515, + 0.6679362196390256 + ], + [ + 518, + 0.5841157346925171 + ], + [ + 523, + 0.1060014136024583 + ], + [ + 529, + 0.3196808690482381 + ], + [ + 532, + 0.261356128670329 + ], + [ + 536, + 0.4838476181841962 + ], + [ + 545, + 0.012523790693381387 + ], + [ + 566, + 0.861355673192335 + ], + [ + 571, + 0.96720309684749 + ], + [ + 578, + 0.7469924943578418 + ], + [ + 586, + 0.3882486634402985 + ], + [ + 598, + 0.16960148612637116 + ], + [ + 603, + 0.5894441331705881 + ], + [ + 610, + 0.43091130854245385 + ], + [ + 623, + 0.8616227381409876 + ], + [ + 627, + 0.007909576832289322 + ], + [ + 633, + 0.40507838864061274 + ], + [ + 637, + 0.7586035194590686 + ], + [ + 647, + 0.9699660647972698 + ], + [ + 657, + 0.9011485550885442 + ], + [ + 662, + 0.2828032424025262 + ], + [ + 666, + 0.3133493874916523 + ], + [ + 670, + 0.16181613607417777 + ], + [ + 682, + 0.49740269054501574 + ], + [ + 687, + 0.816375320286971 + ], + [ + 693, + 0.43373294788269345 + ], + [ + 697, + 0.1777418749372861 + ], + [ + 708, + 0.8241355712392736 + ], + [ + 718, + 0.08610032299474801 + ], + [ + 721, + 0.37426941000917024 + ], + [ + 725, + 0.5670773243026386 + ], + [ + 732, + 0.4938839126293161 + ], + [ + 740, + 0.49987707900869727 + ], + [ + 751, + 0.7616425043483379 + ], + [ + 760, + 0.6658135447547918 + ] + ], + [ + [ + 37, + 0.49394698602351883 + ], + [ + 46, + 0.7838024981034764 + ], + [ + 58, + 0.6460631854138505 + ], + [ + 63, + 0.5243770230726671 + ], + [ + 69, + 0.40008931356233446 + ], + [ + 72, + 0.4123165881147668 + ], + [ + 81, + 0.6696912745326993 + ], + [ + 91, + 0.664191478351343 + ], + [ + 101, + 0.05914247498027991 + ], + [ + 111, + 0.2197473949393658 + ], + [ + 116, + 0.6400773189986326 + ], + [ + 122, + 0.9857246767000574 + ], + [ + 125, + 0.017267287182480606 + ], + [ + 134, + 0.5267122123275636 + ], + [ + 143, + 0.34177428116247466 + ], + [ + 148, + 0.9542786069996256 + ], + [ + 157, + 0.02354674163459025 + ], + [ + 160, + 0.9576947060489708 + ], + [ + 171, + 0.36846614994554994 + ], + [ + 179, + 0.6557680055459466 + ], + [ + 182, + 0.3420453638175034 + ], + [ + 185, + 0.2792513893330373 + ], + [ + 189, + 0.9280644877821799 + ], + [ + 197, + 0.7137150384609056 + ], + [ + 204, + 0.4248836398277933 + ], + [ + 209, + 0.9722646232247286 + ], + [ + 214, + 0.15673476330590352 + ], + [ + 221, + 0.7532829196804539 + ], + [ + 224, + 0.8862977953943214 + ], + [ + 233, + 0.4157589884732672 + ], + [ + 241, + 0.5003650929540384 + ], + [ + 247, + 0.37634813771734976 + ], + [ + 256, + 0.8327259753622251 + ], + [ + 260, + 0.5808036457538774 + ], + [ + 267, + 0.45266239344326464 + ], + [ + 270, + 0.9143426712131865 + ], + [ + 275, + 0.7871071604332133 + ], + [ + 279, + 0.9298028782389396 + ], + [ + 285, + 0.8352546420299467 + ], + [ + 292, + 0.29800152380182354 + ], + [ + 295, + 0.2962492487885455 + ], + [ + 303, + 0.1540772046009099 + ], + [ + 310, + 0.33991592499745926 + ], + [ + 313, + 0.07466174811769599 + ], + [ + 321, + 0.14848951234002505 + ], + [ + 324, + 0.893636654228004 + ], + [ + 328, + 0.6723090335954937 + ], + [ + 335, + 0.29566630220769263 + ], + [ + 342, + 0.6872692168717919 + ], + [ + 348, + 0.5834999174222631 + ], + [ + 351, + 0.5666070524181754 + ], + [ + 357, + 0.5970853277766227 + ], + [ + 359, + 0.10999862137443495 + ], + [ + 370, + 0.4801092778751461 + ], + [ + 379, + 0.2754445090925882 + ], + [ + 382, + 0.6505833995356314 + ], + [ + 390, + 0.025396967721890817 + ], + [ + 394, + 0.19918863080689742 + ], + [ + 400, + 0.8475043345325207 + ], + [ + 404, + 0.20748126479874407 + ], + [ + 412, + 0.5281387845405536 + ], + [ + 418, + 0.1533804618642215 + ], + [ + 421, + 0.47138744879323147 + ], + [ + 423, + 0.23917695019469176 + ], + [ + 430, + 0.6148055354371349 + ], + [ + 443, + 0.319791072731469 + ], + [ + 446, + 0.5053229987178848 + ], + [ + 450, + 0.569695185987864 + ], + [ + 456, + 0.3197881981952123 + ], + [ + 459, + 0.2995142530016489 + ], + [ + 463, + 0.12655920007218124 + ], + [ + 469, + 0.8466076662457465 + ], + [ + 475, + 0.6861147452126901 + ], + [ + 481, + 0.6337548538352584 + ], + [ + 486, + 0.6833515605445383 + ], + [ + 491, + 0.9239754560769365 + ], + [ + 497, + 0.3684738156962407 + ], + [ + 500, + 0.030445106127881094 + ], + [ + 510, + 0.47355048123800203 + ], + [ + 516, + 0.8309699752697971 + ], + [ + 527, + 0.15610383272310835 + ], + [ + 531, + 0.3077664659374405 + ], + [ + 540, + 0.8373245084187128 + ], + [ + 543, + 0.9256471540272363 + ], + [ + 548, + 0.58593892153662 + ], + [ + 555, + 0.8461057272641072 + ], + [ + 558, + 0.009371391072989232 + ], + [ + 568, + 0.5449174490549501 + ], + [ + 572, + 0.9849213018352765 + ], + [ + 577, + 0.6000605081993177 + ], + [ + 581, + 0.07414116086138645 + ], + [ + 589, + 0.26469023995494734 + ], + [ + 594, + 0.23099349500850597 + ], + [ + 603, + 0.30556107769731766 + ], + [ + 606, + 0.8278839091950699 + ], + [ + 609, + 0.032002171412608416 + ], + [ + 618, + 0.45627434473569106 + ], + [ + 626, + 0.3120922864469339 + ], + [ + 629, + 0.39057080046649784 + ], + [ + 633, + 0.4320909138625836 + ] + ], + [ + [ + 40, + 0.36102902282636684 + ], + [ + 53, + 0.8760266226169617 + ], + [ + 61, + 0.8451868353071897 + ], + [ + 65, + 0.8608183238698289 + ], + [ + 69, + 0.31628649771948236 + ], + [ + 75, + 0.9789619059743778 + ], + [ + 81, + 0.45124071366801743 + ], + [ + 87, + 0.09700524109145747 + ], + [ + 92, + 0.45159641683081375 + ], + [ + 96, + 0.6803349764162029 + ], + [ + 100, + 0.9241438272031793 + ], + [ + 110, + 0.19190746187538343 + ], + [ + 118, + 0.7485337344473404 + ], + [ + 122, + 0.6984464803769818 + ], + [ + 127, + 0.8423511784274816 + ], + [ + 130, + 0.3361380355052799 + ], + [ + 134, + 0.8313316653773362 + ], + [ + 142, + 0.23032048120072868 + ], + [ + 147, + 0.33702704726195865 + ], + [ + 155, + 0.9212346100652604 + ], + [ + 158, + 0.569239621643856 + ], + [ + 163, + 0.5557951677922369 + ], + [ + 174, + 0.9573925785654934 + ], + [ + 177, + 0.3662141905911467 + ], + [ + 186, + 0.6384775072835821 + ], + [ + 192, + 0.7109499523266655 + ], + [ + 198, + 0.7235977447884105 + ], + [ + 211, + 0.7950964401605625 + ], + [ + 215, + 0.45184241258390545 + ], + [ + 221, + 0.4993983139777717 + ], + [ + 224, + 0.9688063610053294 + ], + [ + 232, + 0.925146240466334 + ], + [ + 237, + 0.6473912949163064 + ], + [ + 243, + 0.5325537863473729 + ], + [ + 246, + 0.5901574971687233 + ], + [ + 256, + 0.16699128728192203 + ], + [ + 267, + 0.6233074902640243 + ], + [ + 272, + 0.1266695029570878 + ], + [ + 279, + 0.4271965062730252 + ], + [ + 282, + 0.04385243948330941 + ], + [ + 292, + 0.15739622645852913 + ], + [ + 295, + 0.35544659957418645 + ], + [ + 297, + 0.289414431987271 + ], + [ + 302, + 0.9111751734347087 + ], + [ + 305, + 0.5406697337870487 + ], + [ + 309, + 0.9030428043458507 + ], + [ + 318, + 0.4261601596454815 + ], + [ + 324, + 0.7059107640838715 + ], + [ + 328, + 0.6893151266681549 + ], + [ + 334, + 0.0898219753918249 + ], + [ + 336, + 0.6647351414277577 + ], + [ + 347, + 0.5453221645296704 + ], + [ + 351, + 0.6766736864497213 + ], + [ + 356, + 0.23322311208610957 + ], + [ + 360, + 0.9499293463826262 + ], + [ + 368, + 0.23896212828089292 + ], + [ + 371, + 0.07155572341462846 + ], + [ + 376, + 0.8268312405486353 + ], + [ + 382, + 0.10566648158346881 + ], + [ + 387, + 0.3008196014663186 + ], + [ + 389, + 0.6219141723561963 + ], + [ + 396, + 0.47617846535084185 + ], + [ + 399, + 0.2700052435868384 + ], + [ + 407, + 0.5700039055811453 + ], + [ + 414, + 0.7470397387825413 + ], + [ + 417, + 0.0141625759729469 + ], + [ + 422, + 0.5399231267135159 + ], + [ + 426, + 0.3088911792808491 + ], + [ + 432, + 0.12099671150381208 + ], + [ + 434, + 0.6558524054532608 + ], + [ + 441, + 0.6274216130672894 + ], + [ + 450, + 0.4671482150064562 + ], + [ + 454, + 0.9452778581045195 + ], + [ + 457, + 0.16679983669798937 + ], + [ + 461, + 0.04921485123777802 + ], + [ + 465, + 0.21941810401963147 + ], + [ + 473, + 0.8778475067212018 + ], + [ + 475, + 0.28875089376345264 + ], + [ + 482, + 0.09861060503238406 + ], + [ + 487, + 0.5332684560342577 + ], + [ + 492, + 0.29457088416605826 + ], + [ + 501, + 0.2047768532372769 + ], + [ + 510, + 0.9997441757344713 + ], + [ + 514, + 0.024765448651724475 + ], + [ + 518, + 0.8397178077381557 + ], + [ + 523, + 0.2908577677351494 + ], + [ + 527, + 0.4879882225729083 + ], + [ + 536, + 0.37392290160433106 + ], + [ + 539, + 0.5994342819461455 + ], + [ + 545, + 0.7089009837373103 + ], + [ + 550, + 0.18196660880505489 + ], + [ + 553, + 0.9545851708544396 + ], + [ + 562, + 0.8416312588565468 + ], + [ + 570, + 0.20303520879112036 + ], + [ + 573, + 0.0315250170813649 + ], + [ + 577, + 0.9649355941680752 + ], + [ + 582, + 0.6586972989148768 + ], + [ + 587, + 0.2572017750093274 + ], + [ + 591, + 0.7969819759285943 + ], + [ + 601, + 0.7812215370955592 + ] + ], + [ + [ + 55, + 0.23823362137760595 + ], + [ + 59, + 0.7088437331732331 + ], + [ + 68, + 0.6190011707345612 + ], + [ + 72, + 0.17577479455825096 + ], + [ + 75, + 0.47569026806205916 + ], + [ + 79, + 0.5538795660700832 + ], + [ + 84, + 0.34248120612933164 + ], + [ + 97, + 0.3341670160725526 + ], + [ + 102, + 0.7599658252871266 + ], + [ + 111, + 0.2036464100956088 + ], + [ + 114, + 0.4012711202934577 + ], + [ + 116, + 0.20791632769467738 + ], + [ + 123, + 0.6425957871435805 + ], + [ + 126, + 0.05153995913224807 + ], + [ + 131, + 0.6429137023472733 + ], + [ + 134, + 0.28864061101537464 + ], + [ + 138, + 0.7219288946363471 + ], + [ + 148, + 0.1315011430636558 + ], + [ + 152, + 0.6598451904982364 + ], + [ + 156, + 0.8156083581964401 + ], + [ + 163, + 0.5743947511723883 + ], + [ + 167, + 0.3733498263249273 + ], + [ + 180, + 0.3060751189920864 + ], + [ + 186, + 0.20506963284870483 + ], + [ + 188, + 0.34155705046713314 + ], + [ + 198, + 0.9610872628015364 + ], + [ + 203, + 0.5045129151603424 + ], + [ + 209, + 0.13821784382790325 + ], + [ + 212, + 0.11261739613185995 + ], + [ + 221, + 0.11407077127028431 + ], + [ + 224, + 0.49504351697913795 + ], + [ + 228, + 0.7172206516718271 + ], + [ + 236, + 0.806622840034087 + ], + [ + 242, + 0.7026468628647973 + ], + [ + 247, + 0.0635467525491814 + ], + [ + 255, + 0.7810927252454664 + ], + [ + 261, + 0.4847696494034943 + ], + [ + 268, + 0.2928626128599805 + ], + [ + 276, + 0.7919087438329595 + ], + [ + 280, + 0.8551744216714955 + ], + [ + 286, + 0.09852597615236447 + ], + [ + 295, + 0.5716111235997449 + ], + [ + 298, + 0.3091707392264271 + ], + [ + 302, + 0.9704291668786966 + ], + [ + 310, + 0.5165489043952509 + ], + [ + 314, + 0.8622629142725603 + ], + [ + 321, + 0.23578904640723708 + ], + [ + 330, + 0.20361685548616082 + ], + [ + 333, + 0.14741906081477962 + ], + [ + 338, + 0.3358698015678223 + ], + [ + 345, + 0.2541021972477222 + ], + [ + 352, + 0.9717436086920719 + ], + [ + 358, + 0.06886310502514192 + ], + [ + 369, + 0.03455500195446526 + ], + [ + 374, + 0.2190825794427429 + ], + [ + 380, + 0.8699731669657218 + ], + [ + 383, + 0.3043522628378277 + ], + [ + 392, + 0.16799072515627866 + ], + [ + 397, + 0.5451209604150219 + ], + [ + 405, + 0.6528015601036944 + ], + [ + 410, + 0.946769007873055 + ], + [ + 416, + 0.802922785110998 + ], + [ + 419, + 0.14624717435024537 + ], + [ + 428, + 0.9939393589743016 + ], + [ + 433, + 0.5438767770124041 + ], + [ + 441, + 0.9539277335982252 + ], + [ + 444, + 0.199141619975017 + ], + [ + 446, + 0.9199158730765585 + ], + [ + 451, + 0.5712582013637526 + ], + [ + 457, + 0.1972596226478589 + ], + [ + 460, + 0.36518101067484243 + ], + [ + 465, + 0.25629366029468537 + ], + [ + 472, + 0.3019447175391856 + ], + [ + 481, + 0.8419526294137317 + ], + [ + 484, + 0.2509895071463887 + ], + [ + 490, + 0.1037026745630002 + ], + [ + 495, + 0.17974485193097733 + ], + [ + 498, + 0.3698549897426032 + ], + [ + 507, + 0.06198659986959043 + ], + [ + 516, + 0.9528624331528326 + ], + [ + 519, + 0.7256244653914851 + ], + [ + 523, + 0.03260954802375304 + ], + [ + 532, + 0.3412627977929068 + ], + [ + 535, + 0.5710531550998826 + ], + [ + 539, + 0.23900524401272627 + ], + [ + 544, + 0.5145189665895855 + ], + [ + 547, + 0.5298327676088075 + ], + [ + 551, + 0.9082192922979959 + ], + [ + 559, + 0.657283974764944 + ], + [ + 562, + 0.6111169216808569 + ], + [ + 569, + 0.8999656846508128 + ], + [ + 574, + 0.2377204724818196 + ], + [ + 582, + 0.5688315109961426 + ], + [ + 589, + 0.812692269883015 + ], + [ + 591, + 0.5785809576228357 + ], + [ + 597, + 0.9632126334578439 + ], + [ + 606, + 0.2306508403947446 + ], + [ + 608, + 0.8381525257821419 + ], + [ + 615, + 0.43975755845575426 + ], + [ + 618, + 0.2880151623184569 + ] + ], + [ + [ + 40, + 0.11270183745931939 + ], + [ + 50, + 0.16735047842895456 + ], + [ + 53, + 0.6638378040881124 + ], + [ + 62, + 0.9663719199942588 + ], + [ + 70, + 0.4737144352085344 + ], + [ + 79, + 0.2515490054821109 + ], + [ + 83, + 0.18127523933550305 + ], + [ + 91, + 0.770364170772674 + ], + [ + 95, + 0.07781189454636062 + ], + [ + 102, + 0.32388479900849554 + ], + [ + 107, + 0.9354636128516608 + ], + [ + 111, + 0.06876837514770884 + ], + [ + 118, + 0.611803723488172 + ], + [ + 124, + 0.4433575158252292 + ], + [ + 134, + 0.6320887681586572 + ], + [ + 141, + 0.43935283086072596 + ], + [ + 144, + 0.7529713835409011 + ], + [ + 148, + 0.50350437452824 + ], + [ + 152, + 0.9870942861249873 + ], + [ + 156, + 0.8520727552075322 + ], + [ + 160, + 0.32708854954559274 + ], + [ + 168, + 0.13951404229289788 + ], + [ + 173, + 0.48120354251977127 + ], + [ + 175, + 0.3788052239386335 + ], + [ + 182, + 0.18837263224051837 + ], + [ + 192, + 0.0008931050043605726 + ], + [ + 198, + 0.4576049258520205 + ], + [ + 200, + 0.12308795026333441 + ], + [ + 212, + 0.16941205845120066 + ], + [ + 215, + 0.4217614226331424 + ], + [ + 220, + 0.4605022789324754 + ], + [ + 226, + 0.3883769274661114 + ], + [ + 229, + 0.6476092656994826 + ], + [ + 238, + 0.7444528208644631 + ], + [ + 242, + 0.8382830327376373 + ], + [ + 246, + 0.3255388035327307 + ], + [ + 250, + 0.37823777418750104 + ], + [ + 257, + 0.5337917136695883 + ], + [ + 260, + 0.35638761478146375 + ], + [ + 270, + 0.643013841847097 + ], + [ + 274, + 0.7539173713347824 + ], + [ + 281, + 0.39747392447259533 + ], + [ + 288, + 0.31619836437311044 + ], + [ + 292, + 0.96479929818097 + ], + [ + 298, + 0.34350320637777165 + ], + [ + 305, + 0.34759392109673715 + ], + [ + 314, + 0.6782617587907382 + ], + [ + 319, + 0.014351986862333743 + ], + [ + 328, + 0.7414548713892026 + ], + [ + 336, + 0.7550689659932851 + ], + [ + 343, + 0.18525052292160726 + ], + [ + 351, + 0.044682469470754826 + ], + [ + 356, + 0.95819919974473 + ], + [ + 360, + 0.9588856351198082 + ], + [ + 367, + 0.6415762654545767 + ], + [ + 372, + 0.3642104945755331 + ], + [ + 380, + 0.34162011763836 + ], + [ + 385, + 0.2434627982095635 + ], + [ + 389, + 0.24680042371409217 + ], + [ + 394, + 0.36417017562720533 + ], + [ + 404, + 0.2345021560495697 + ], + [ + 407, + 0.8092412650300804 + ], + [ + 411, + 0.44232105674852007 + ], + [ + 416, + 0.13702169118416818 + ], + [ + 419, + 0.7197497319430902 + ], + [ + 427, + 0.9622769916663304 + ], + [ + 442, + 0.8722310899613255 + ], + [ + 445, + 0.9214162819259044 + ], + [ + 449, + 0.15157844184554103 + ], + [ + 454, + 0.432781404410706 + ], + [ + 458, + 0.747383436121126 + ], + [ + 467, + 0.24822552902861983 + ], + [ + 473, + 0.9009134687487332 + ], + [ + 480, + 0.968273300785165 + ], + [ + 486, + 0.015824272513084603 + ], + [ + 490, + 0.9127836401642463 + ], + [ + 505, + 0.3504947096329971 + ], + [ + 511, + 0.02251349443082218 + ], + [ + 518, + 0.03360383611982509 + ], + [ + 532, + 0.7875699252039838 + ], + [ + 535, + 0.8368935316633649 + ], + [ + 547, + 0.3357167540992335 + ], + [ + 556, + 0.861053457929378 + ], + [ + 559, + 0.7040841739238658 + ], + [ + 565, + 0.21166386174452112 + ], + [ + 569, + 0.2850460833063009 + ], + [ + 575, + 0.22725529972672243 + ], + [ + 578, + 0.9435052257086358 + ], + [ + 586, + 0.5259384431352176 + ], + [ + 591, + 0.768476666492965 + ], + [ + 593, + 0.3391260567589758 + ], + [ + 597, + 0.419062436402027 + ], + [ + 600, + 0.2565828965330561 + ], + [ + 606, + 0.6000259174655532 + ], + [ + 612, + 0.1118540785078942 + ], + [ + 618, + 0.6994202688473531 + ], + [ + 625, + 0.980766192013755 + ], + [ + 629, + 0.8405326264934084 + ], + [ + 639, + 0.20794253986093048 + ], + [ + 646, + 0.7930554679628796 + ] + ] + ], + [ + [ + [ + 40, + 0.00468037300073465 + ], + [ + 50, + 0.7091589305756109 + ], + [ + 53, + 0.9855450714336487 + ], + [ + 62, + 0.659484088601776 + ], + [ + 70, + 0.5532704759083182 + ], + [ + 79, + 0.8502547418420897 + ], + [ + 83, + 0.9998272652249254 + ], + [ + 91, + 0.3765466575981966 + ], + [ + 95, + 0.5119397333962138 + ], + [ + 102, + 0.24561789284597402 + ], + [ + 107, + 0.036317858624365984 + ], + [ + 111, + 0.07580616170262722 + ], + [ + 118, + 0.19221948989142512 + ], + [ + 124, + 0.9637946198694191 + ], + [ + 134, + 0.15745065828019078 + ], + [ + 141, + 0.34241147161645147 + ], + [ + 144, + 0.8630937558859838 + ], + [ + 148, + 0.7005749347150823 + ], + [ + 152, + 0.5333854093601684 + ], + [ + 156, + 0.41777964717116745 + ], + [ + 160, + 0.15402452403111266 + ], + [ + 168, + 0.8842699478840672 + ], + [ + 173, + 0.9582200679557799 + ], + [ + 175, + 0.7470818600466819 + ], + [ + 182, + 0.18768352231006835 + ], + [ + 192, + 0.2966546592609577 + ], + [ + 198, + 0.332938347647986 + ], + [ + 200, + 0.1675778896835809 + ], + [ + 212, + 0.5044890402387665 + ], + [ + 215, + 0.4458683404774584 + ], + [ + 220, + 0.8776198032207227 + ], + [ + 226, + 0.9211677274232412 + ], + [ + 229, + 0.6848635414563774 + ], + [ + 238, + 0.17986851369146017 + ], + [ + 242, + 0.7163132871253787 + ], + [ + 246, + 0.014894167673424441 + ], + [ + 250, + 0.7104715485933737 + ], + [ + 257, + 0.900923679144467 + ], + [ + 260, + 0.9964659455121411 + ], + [ + 270, + 0.8754115164584669 + ], + [ + 274, + 0.8761082690805428 + ], + [ + 281, + 0.5178151660661322 + ], + [ + 288, + 0.9792882331250669 + ], + [ + 292, + 0.8437735070547404 + ], + [ + 298, + 0.654019513252547 + ], + [ + 305, + 0.18273520950569744 + ], + [ + 314, + 0.2914322522100662 + ], + [ + 319, + 0.28137248463310105 + ], + [ + 328, + 0.9978500699374595 + ], + [ + 336, + 0.4245710747348871 + ], + [ + 343, + 0.785588780157941 + ], + [ + 351, + 0.05272192008868892 + ], + [ + 356, + 0.681840890870945 + ], + [ + 360, + 0.7228517996243333 + ], + [ + 367, + 0.6054900550214497 + ], + [ + 372, + 0.23501992652197945 + ], + [ + 380, + 0.04311524803467137 + ], + [ + 385, + 0.91321573771126 + ], + [ + 389, + 0.7481939455497512 + ], + [ + 394, + 0.6194452021817266 + ], + [ + 404, + 0.6757805421784167 + ], + [ + 407, + 0.5621276372370088 + ], + [ + 411, + 0.07392525913253589 + ], + [ + 416, + 0.6164229487347065 + ], + [ + 419, + 0.5957333196165489 + ], + [ + 427, + 0.588762239426099 + ], + [ + 442, + 0.7488673303391555 + ], + [ + 445, + 0.9980808100820333 + ], + [ + 449, + 0.80781121629929 + ], + [ + 454, + 0.9009274129029268 + ], + [ + 458, + 0.8221117509083123 + ], + [ + 467, + 0.5846232018136769 + ], + [ + 473, + 0.46394007325919717 + ], + [ + 480, + 0.5609069579369708 + ], + [ + 486, + 0.2518581519498554 + ], + [ + 490, + 0.46589932880988905 + ], + [ + 505, + 0.5981228995249036 + ], + [ + 511, + 0.5219414075703898 + ], + [ + 518, + 0.15814267073358745 + ], + [ + 532, + 0.5101270655437143 + ], + [ + 535, + 0.8717357640271157 + ], + [ + 547, + 0.9917065579813802 + ], + [ + 556, + 0.2632147801947341 + ], + [ + 559, + 0.3648909665861758 + ], + [ + 565, + 0.010164530862904808 + ], + [ + 569, + 0.7739326668419476 + ], + [ + 575, + 0.8701288781852516 + ], + [ + 578, + 0.9984674742590662 + ], + [ + 586, + 0.7457917711254775 + ], + [ + 591, + 0.7486189552312057 + ], + [ + 593, + 0.8955777559386559 + ], + [ + 597, + 0.07463333507526826 + ], + [ + 600, + 0.3434757954807912 + ], + [ + 606, + 0.32579509928106865 + ], + [ + 612, + 0.73445890541621 + ], + [ + 618, + 0.6842180952913891 + ], + [ + 625, + 0.42772014880740905 + ], + [ + 629, + 0.11001325145106278 + ], + [ + 639, + 0.6894574568230319 + ], + [ + 646, + 0.29588650285394436 + ] + ], + [ + [ + 67, + 0.8378256633339197 + ], + [ + 73, + 0.11238752726254153 + ], + [ + 77, + 0.6688869724488592 + ], + [ + 87, + 0.21110261253826512 + ], + [ + 90, + 0.7272902724695384 + ], + [ + 96, + 0.445938021152243 + ], + [ + 107, + 0.8905477445297064 + ], + [ + 113, + 0.3678681272534291 + ], + [ + 118, + 0.055757154839916945 + ], + [ + 123, + 0.010537705702711286 + ], + [ + 132, + 0.5873928176490126 + ], + [ + 141, + 0.15966008527509035 + ], + [ + 151, + 0.49492722023904456 + ], + [ + 154, + 0.15440331570785137 + ], + [ + 158, + 0.6549071325924316 + ], + [ + 164, + 0.6309564617718901 + ], + [ + 168, + 0.9043675902927752 + ], + [ + 178, + 0.8839817787287991 + ], + [ + 183, + 0.35791922902116446 + ], + [ + 190, + 0.2653018562745991 + ], + [ + 193, + 0.5962843386678608 + ], + [ + 198, + 0.6201126631422961 + ], + [ + 206, + 0.8335102447350686 + ], + [ + 213, + 0.821133736777388 + ], + [ + 219, + 0.043197328705780635 + ], + [ + 224, + 0.45297573639983313 + ], + [ + 235, + 0.19321919322744208 + ], + [ + 242, + 0.9771521307046775 + ], + [ + 251, + 0.04518219769627141 + ], + [ + 260, + 0.3213682623447581 + ], + [ + 264, + 0.3947868328662738 + ], + [ + 271, + 0.8643078229124727 + ], + [ + 275, + 0.9708665647870902 + ], + [ + 281, + 0.8269046745997526 + ], + [ + 287, + 0.3037695632698202 + ], + [ + 299, + 0.9407593732330037 + ], + [ + 309, + 0.4092571085390768 + ], + [ + 317, + 0.05688818842179255 + ], + [ + 327, + 0.8229539092083631 + ], + [ + 334, + 0.18216364621300085 + ], + [ + 338, + 0.6353512870160297 + ], + [ + 352, + 0.38989365269983767 + ], + [ + 362, + 0.5028360285374778 + ], + [ + 366, + 0.10834687609008009 + ], + [ + 371, + 0.059006402166790095 + ], + [ + 378, + 0.9225750117244444 + ], + [ + 382, + 0.677432981942718 + ], + [ + 392, + 0.4468017267520701 + ], + [ + 402, + 0.2754201761144789 + ], + [ + 408, + 0.2767002419320891 + ], + [ + 411, + 0.2201497670365835 + ], + [ + 416, + 0.009922046703021326 + ], + [ + 424, + 0.8004344362654916 + ], + [ + 429, + 0.5156489750814924 + ], + [ + 435, + 0.32303067734039725 + ], + [ + 438, + 0.773350721027946 + ], + [ + 447, + 0.560125445286094 + ], + [ + 451, + 0.27995420608471133 + ], + [ + 458, + 0.4042124068946066 + ], + [ + 468, + 0.402964344569103 + ], + [ + 471, + 0.7438425580852883 + ], + [ + 482, + 0.4198682506835377 + ], + [ + 489, + 0.23580873040393124 + ], + [ + 496, + 0.8601224417363401 + ], + [ + 500, + 0.5967565704810796 + ], + [ + 515, + 0.2387173297570796 + ], + [ + 518, + 0.9725550533527945 + ], + [ + 523, + 0.7100412419976694 + ], + [ + 529, + 0.847385309452072 + ], + [ + 532, + 0.35340509077481663 + ], + [ + 536, + 0.41466010236285644 + ], + [ + 545, + 0.3540969812621818 + ], + [ + 566, + 0.7793778304677912 + ], + [ + 571, + 0.23006938521840836 + ], + [ + 578, + 0.7240602625920394 + ], + [ + 586, + 0.5062068776738957 + ], + [ + 598, + 0.6415320320257208 + ], + [ + 603, + 0.5004493284665397 + ], + [ + 610, + 0.6617599237167048 + ], + [ + 623, + 0.8624219023457487 + ], + [ + 627, + 0.7724244934048127 + ], + [ + 633, + 0.7290816628122504 + ], + [ + 637, + 0.5956693969052627 + ], + [ + 647, + 0.7465014939359419 + ], + [ + 657, + 0.5930914890425232 + ], + [ + 662, + 0.8877778186321397 + ], + [ + 666, + 0.16126464682403385 + ], + [ + 670, + 0.5113139520976122 + ], + [ + 682, + 0.5275626506640148 + ], + [ + 687, + 0.9757650740757693 + ], + [ + 693, + 0.6799290920834802 + ], + [ + 697, + 0.16523900315923012 + ], + [ + 708, + 0.40970134756920307 + ], + [ + 718, + 0.4280458760248915 + ], + [ + 721, + 0.7950303542421573 + ], + [ + 725, + 0.5000165292176463 + ], + [ + 732, + 0.7234324462656435 + ], + [ + 740, + 0.38210386460776014 + ], + [ + 751, + 0.8908020807180658 + ], + [ + 760, + 0.4975839062930545 + ] + ], + [ + [ + 37, + 0.8749076011525886 + ], + [ + 46, + 0.9238608553951507 + ], + [ + 58, + 0.8173596144243834 + ], + [ + 63, + 0.13584912437996188 + ], + [ + 69, + 0.582020921931331 + ], + [ + 72, + 0.7834757741568337 + ], + [ + 81, + 0.4362509444837007 + ], + [ + 91, + 0.035612825282180016 + ], + [ + 101, + 0.0572687943274095 + ], + [ + 111, + 0.38120510044206446 + ], + [ + 116, + 0.7594862807778495 + ], + [ + 122, + 0.20985001759193123 + ], + [ + 125, + 0.4033772974636246 + ], + [ + 134, + 0.9620782174638446 + ], + [ + 143, + 0.2364452909770428 + ], + [ + 148, + 0.7933701272954701 + ], + [ + 157, + 0.9215215575504396 + ], + [ + 160, + 0.5643546267471512 + ], + [ + 171, + 0.42191057810395116 + ], + [ + 179, + 0.6193705118444447 + ], + [ + 182, + 0.5761067154659435 + ], + [ + 185, + 0.2847817223710878 + ], + [ + 189, + 0.5906833498143659 + ], + [ + 197, + 0.9047121360782789 + ], + [ + 204, + 0.8433754744976968 + ], + [ + 209, + 0.2741338889921223 + ], + [ + 214, + 0.7989580502468088 + ], + [ + 221, + 0.8805703658885133 + ], + [ + 224, + 0.9554036390607906 + ], + [ + 233, + 0.29226666314388106 + ], + [ + 241, + 0.6598386905915258 + ], + [ + 247, + 0.8996789551777957 + ], + [ + 256, + 0.26127214844827196 + ], + [ + 260, + 0.45845571136845886 + ], + [ + 267, + 0.885118581494726 + ], + [ + 270, + 0.495391189751641 + ], + [ + 275, + 0.8576018182254923 + ], + [ + 279, + 0.9288817863888063 + ], + [ + 285, + 0.02378828507763775 + ], + [ + 292, + 0.018330391663468282 + ], + [ + 295, + 0.5875923915453738 + ], + [ + 303, + 0.7796320182135565 + ], + [ + 310, + 0.544589753590219 + ], + [ + 313, + 0.3145912113950323 + ], + [ + 321, + 0.35060672541332993 + ], + [ + 324, + 0.8810896749680897 + ], + [ + 328, + 0.7777661648615947 + ], + [ + 335, + 0.9912738149943338 + ], + [ + 342, + 0.09989003936140128 + ], + [ + 348, + 0.8777215941261566 + ], + [ + 351, + 0.5654004331505517 + ], + [ + 357, + 0.3531840170245151 + ], + [ + 359, + 0.4206508971258035 + ], + [ + 370, + 0.4074249978402136 + ], + [ + 379, + 0.6260882427102118 + ], + [ + 382, + 0.217012876769885 + ], + [ + 390, + 0.1548315692192982 + ], + [ + 394, + 0.41908554749487237 + ], + [ + 400, + 0.9826427362567685 + ], + [ + 404, + 0.5394801537630239 + ], + [ + 412, + 0.38199590646750514 + ], + [ + 418, + 0.5630344098234448 + ], + [ + 421, + 0.7296567723496572 + ], + [ + 423, + 0.39895151611226987 + ], + [ + 430, + 0.25618713082368727 + ], + [ + 443, + 0.9355016083240484 + ], + [ + 446, + 0.6670065119887693 + ], + [ + 450, + 0.38956955520819525 + ], + [ + 456, + 0.771827764683902 + ], + [ + 459, + 0.8554937968169967 + ], + [ + 463, + 0.31632276905408696 + ], + [ + 469, + 0.12160571754003568 + ], + [ + 475, + 0.4519912152556457 + ], + [ + 481, + 0.36509492128891563 + ], + [ + 486, + 0.15520855333460581 + ], + [ + 491, + 0.42171525906338936 + ], + [ + 497, + 0.5725791531143966 + ], + [ + 500, + 0.6181448865718943 + ], + [ + 510, + 0.6001536124461999 + ], + [ + 516, + 0.36399772072990577 + ], + [ + 527, + 0.2938833155991196 + ], + [ + 531, + 0.6351646951523332 + ], + [ + 540, + 0.10386387891436111 + ], + [ + 543, + 0.0648999284744588 + ], + [ + 548, + 0.1853270724319741 + ], + [ + 555, + 0.8178912273103575 + ], + [ + 558, + 0.9888581469836909 + ], + [ + 568, + 0.3425276089985204 + ], + [ + 572, + 0.9097010616120178 + ], + [ + 577, + 0.9881644316646269 + ], + [ + 581, + 0.22631373470898974 + ], + [ + 589, + 0.12326702082936758 + ], + [ + 594, + 0.5065990893081265 + ], + [ + 603, + 0.08143529712464836 + ], + [ + 606, + 0.23927838484825048 + ], + [ + 609, + 0.7197068948340273 + ], + [ + 618, + 0.2053853818698017 + ], + [ + 626, + 0.08945629128495713 + ], + [ + 629, + 0.8889921175798989 + ], + [ + 633, + 0.8804989892429534 + ] + ], + [ + [ + 40, + 0.9449196777294301 + ], + [ + 53, + 0.7530536301465847 + ], + [ + 61, + 0.9076707145271552 + ], + [ + 65, + 0.8349538347890376 + ], + [ + 69, + 0.11383373092515581 + ], + [ + 75, + 0.12048643181236729 + ], + [ + 81, + 0.28299768793806224 + ], + [ + 87, + 0.3525680434382047 + ], + [ + 92, + 0.43554872072793616 + ], + [ + 96, + 0.4813208383682711 + ], + [ + 100, + 0.583369357691057 + ], + [ + 110, + 0.08696101167907078 + ], + [ + 118, + 0.04245316822236178 + ], + [ + 122, + 0.8876632788392032 + ], + [ + 127, + 0.10202028492728177 + ], + [ + 130, + 0.5011845469175062 + ], + [ + 134, + 0.5367957860316404 + ], + [ + 142, + 0.4070049095745485 + ], + [ + 147, + 0.5100916991594104 + ], + [ + 155, + 0.4253060086632018 + ], + [ + 158, + 0.6115790498947735 + ], + [ + 163, + 0.8709600907234496 + ], + [ + 174, + 0.848619444980096 + ], + [ + 177, + 0.7544012444352146 + ], + [ + 186, + 0.8175680220703477 + ], + [ + 192, + 0.6616179227551636 + ], + [ + 198, + 0.8684651728156997 + ], + [ + 211, + 0.669055681153056 + ], + [ + 215, + 0.6728020695628941 + ], + [ + 221, + 0.5537810008200017 + ], + [ + 224, + 0.7844651897430557 + ], + [ + 232, + 0.14417874287006227 + ], + [ + 237, + 0.6720215874302555 + ], + [ + 243, + 0.38642391083563443 + ], + [ + 246, + 0.5685703450797933 + ], + [ + 256, + 0.1290730988046246 + ], + [ + 267, + 0.5655476645228389 + ], + [ + 272, + 0.9641834653902639 + ], + [ + 279, + 0.007467592277920931 + ], + [ + 282, + 0.3263406023906875 + ], + [ + 292, + 0.35855768426045787 + ], + [ + 295, + 0.26676263940895095 + ], + [ + 297, + 0.41136524610382574 + ], + [ + 302, + 0.12439233897913105 + ], + [ + 305, + 0.4014355805309098 + ], + [ + 309, + 0.10278893070367312 + ], + [ + 318, + 0.3316058604089369 + ], + [ + 324, + 0.13923073752323023 + ], + [ + 328, + 0.7172775401373558 + ], + [ + 334, + 0.8298382908508004 + ], + [ + 336, + 0.045587700215413696 + ], + [ + 347, + 0.7618166985032617 + ], + [ + 351, + 0.06304557480714046 + ], + [ + 356, + 0.9867981801705887 + ], + [ + 360, + 0.8104590927884868 + ], + [ + 368, + 0.6158428600941366 + ], + [ + 371, + 0.026999104280058805 + ], + [ + 376, + 0.05293042758469979 + ], + [ + 382, + 0.16679833912794995 + ], + [ + 387, + 0.334305361760751 + ], + [ + 389, + 0.1915422680624359 + ], + [ + 396, + 0.8596628850196515 + ], + [ + 399, + 0.4296923972519836 + ], + [ + 407, + 0.16489830059409571 + ], + [ + 414, + 0.39669106674100674 + ], + [ + 417, + 0.6839031336156742 + ], + [ + 422, + 0.39815003063523324 + ], + [ + 426, + 0.19644618408994574 + ], + [ + 432, + 0.05206904090695752 + ], + [ + 434, + 0.04762740192299997 + ], + [ + 441, + 0.32010606322671853 + ], + [ + 450, + 0.955478716962234 + ], + [ + 454, + 0.29837003384604077 + ], + [ + 457, + 0.35767442742461153 + ], + [ + 461, + 0.09201990313320785 + ], + [ + 465, + 0.5735409748172002 + ], + [ + 473, + 0.5693747905536197 + ], + [ + 475, + 0.9777018586879608 + ], + [ + 482, + 0.73741589871055 + ], + [ + 487, + 0.16135673617765423 + ], + [ + 492, + 0.9517441272510851 + ], + [ + 501, + 0.9045470733683137 + ], + [ + 510, + 0.014465779557026215 + ], + [ + 514, + 0.9689450265108979 + ], + [ + 518, + 0.9012337566153539 + ], + [ + 523, + 0.1479337792347597 + ], + [ + 527, + 0.8994241315974092 + ], + [ + 536, + 0.20804147953076413 + ], + [ + 539, + 0.00010275359517497584 + ], + [ + 545, + 0.15590563122715273 + ], + [ + 550, + 0.9390702566890677 + ], + [ + 553, + 0.4607240754375751 + ], + [ + 562, + 0.689910967406846 + ], + [ + 570, + 0.8142494969476447 + ], + [ + 573, + 0.33423919794793 + ], + [ + 577, + 0.2653500563129778 + ], + [ + 582, + 0.015618220679541439 + ], + [ + 587, + 0.689365739195002 + ], + [ + 591, + 0.16284459959526443 + ], + [ + 601, + 0.4480517931243433 + ] + ], + [ + [ + 55, + 0.5009249344049626 + ], + [ + 59, + 0.27453710321841585 + ], + [ + 68, + 0.577514755929023 + ], + [ + 72, + 0.8211335821763152 + ], + [ + 75, + 0.10522516775824642 + ], + [ + 79, + 0.5443350288995569 + ], + [ + 84, + 0.6415238004543551 + ], + [ + 97, + 0.2819371960865855 + ], + [ + 102, + 0.3458520118750954 + ], + [ + 111, + 0.025154312036510418 + ], + [ + 114, + 0.5889537997786137 + ], + [ + 116, + 0.7093074912594917 + ], + [ + 123, + 0.5735217148305929 + ], + [ + 126, + 0.3626353283561987 + ], + [ + 131, + 0.6564418188153487 + ], + [ + 134, + 0.8184444222421968 + ], + [ + 138, + 0.6274472367181377 + ], + [ + 148, + 0.5439662595161793 + ], + [ + 152, + 0.47587665655981737 + ], + [ + 156, + 0.25156620276828656 + ], + [ + 163, + 0.05656884341487145 + ], + [ + 167, + 0.60162106771304 + ], + [ + 180, + 0.9296650464517605 + ], + [ + 186, + 0.10165410856756163 + ], + [ + 188, + 0.6117594661395453 + ], + [ + 198, + 0.7440284454600358 + ], + [ + 203, + 0.6746015007741764 + ], + [ + 209, + 0.5902445716646115 + ], + [ + 212, + 0.28823206518135247 + ], + [ + 221, + 0.6613667590472555 + ], + [ + 224, + 0.5246512053241804 + ], + [ + 228, + 0.1798893335220607 + ], + [ + 236, + 0.46432387893529004 + ], + [ + 242, + 0.7109630775914935 + ], + [ + 247, + 0.3620599972983827 + ], + [ + 255, + 0.5994875301948253 + ], + [ + 261, + 0.46086894721839766 + ], + [ + 268, + 0.13558877241390344 + ], + [ + 276, + 0.9904569589730559 + ], + [ + 280, + 0.7408691004871412 + ], + [ + 286, + 0.7267885705727423 + ], + [ + 295, + 0.5453447273376516 + ], + [ + 298, + 0.4551361174793479 + ], + [ + 302, + 0.10273762827036037 + ], + [ + 310, + 0.9588809956569723 + ], + [ + 314, + 0.024286375785273262 + ], + [ + 321, + 0.8080341990628193 + ], + [ + 330, + 0.20536067400855407 + ], + [ + 333, + 0.9358431308862746 + ], + [ + 338, + 0.14009586188858902 + ], + [ + 345, + 0.277977629543663 + ], + [ + 352, + 0.6179920280839059 + ], + [ + 358, + 0.3976908014305722 + ], + [ + 369, + 0.3615981233770672 + ], + [ + 374, + 0.934687393038061 + ], + [ + 380, + 0.3831576931332187 + ], + [ + 383, + 0.38853962233051087 + ], + [ + 392, + 0.16068540976000234 + ], + [ + 397, + 0.8052935899014265 + ], + [ + 405, + 0.9312130351401108 + ], + [ + 410, + 0.37988043965450313 + ], + [ + 416, + 0.5193941459595316 + ], + [ + 419, + 0.3286442448676834 + ], + [ + 428, + 0.04844797645100607 + ], + [ + 433, + 0.7467909394231091 + ], + [ + 441, + 0.8595004309567624 + ], + [ + 444, + 0.8060904853737619 + ], + [ + 446, + 0.1102432179044267 + ], + [ + 451, + 0.7649293395177283 + ], + [ + 457, + 0.5388301133864662 + ], + [ + 460, + 0.31665422549720734 + ], + [ + 465, + 0.39951706068523574 + ], + [ + 472, + 0.7091281506958467 + ], + [ + 481, + 0.7877470122433153 + ], + [ + 484, + 0.6062854435744452 + ], + [ + 490, + 0.40091589452679044 + ], + [ + 495, + 0.019897473081658013 + ], + [ + 498, + 0.540408033192753 + ], + [ + 507, + 0.6938173810029878 + ], + [ + 516, + 0.1735655597618886 + ], + [ + 519, + 0.8990667519858062 + ], + [ + 523, + 0.2483749343864362 + ], + [ + 532, + 0.08885863809564676 + ], + [ + 535, + 0.5082993240453448 + ], + [ + 539, + 0.3348426426619061 + ], + [ + 544, + 0.5713034172888001 + ], + [ + 547, + 0.9864182959325798 + ], + [ + 551, + 0.025121479182103834 + ], + [ + 559, + 0.5499442670188757 + ], + [ + 562, + 0.8374195493494969 + ], + [ + 569, + 0.5169861095603084 + ], + [ + 574, + 0.2433744264572073 + ], + [ + 582, + 0.24961167770684567 + ], + [ + 589, + 0.1237140227285306 + ], + [ + 591, + 0.632137150494751 + ], + [ + 597, + 0.41136875353689095 + ], + [ + 606, + 0.5386833442843523 + ], + [ + 608, + 0.4223653554757354 + ], + [ + 615, + 0.23260885562168088 + ], + [ + 618, + 0.5807452569282614 + ] + ], + [ + [ + 40, + 0.026488972633580632 + ], + [ + 50, + 0.31589463077417623 + ], + [ + 53, + 0.3752812685550959 + ], + [ + 62, + 0.5802840227140457 + ], + [ + 70, + 0.48952959503077786 + ], + [ + 79, + 0.6724515819815962 + ], + [ + 83, + 0.6390682287810072 + ], + [ + 91, + 0.8506554881822808 + ], + [ + 95, + 0.5668921847377932 + ], + [ + 102, + 0.2714961804515672 + ], + [ + 107, + 0.2937632015368574 + ], + [ + 111, + 0.4126502667255627 + ], + [ + 118, + 0.9997839441614715 + ], + [ + 124, + 0.2773504634622319 + ], + [ + 134, + 0.8163124355286742 + ], + [ + 141, + 0.25768103716838064 + ], + [ + 144, + 0.27071178611618585 + ], + [ + 148, + 0.7739141925655263 + ], + [ + 152, + 0.8432961693395837 + ], + [ + 156, + 0.12813933401687871 + ], + [ + 160, + 0.8231219001106251 + ], + [ + 168, + 0.8951416362885661 + ], + [ + 173, + 0.08023585468047012 + ], + [ + 175, + 0.9631479902450203 + ], + [ + 182, + 0.3574881382863949 + ], + [ + 192, + 0.047127450447726926 + ], + [ + 198, + 0.5764301789481726 + ], + [ + 200, + 0.1735296450028755 + ], + [ + 212, + 0.045856881409146566 + ], + [ + 215, + 0.41559030657017915 + ], + [ + 220, + 0.011983373421425969 + ], + [ + 226, + 0.9821552197620144 + ], + [ + 229, + 0.20950633030862575 + ], + [ + 238, + 0.2306130643632197 + ], + [ + 242, + 0.8909108541506392 + ], + [ + 246, + 0.47716143115444876 + ], + [ + 250, + 0.1938206566589753 + ], + [ + 257, + 0.29667952792319485 + ], + [ + 260, + 0.9451320710903021 + ], + [ + 270, + 0.8355970044569841 + ], + [ + 274, + 0.09029732856372219 + ], + [ + 281, + 0.48943041240827023 + ], + [ + 288, + 0.9464562378506813 + ], + [ + 292, + 0.28176617810705884 + ], + [ + 298, + 0.20928133251979542 + ], + [ + 305, + 0.0549761066518033 + ], + [ + 314, + 0.3674665237301846 + ], + [ + 319, + 0.03238435137015938 + ], + [ + 328, + 0.5827419989823187 + ], + [ + 336, + 0.573606520911795 + ], + [ + 343, + 0.5418523620253239 + ], + [ + 351, + 0.7693611920215081 + ], + [ + 356, + 0.5416746916046151 + ], + [ + 360, + 0.6866821351719342 + ], + [ + 367, + 0.5050211720934898 + ], + [ + 372, + 0.045620007056940026 + ], + [ + 380, + 0.33169499660173074 + ], + [ + 385, + 0.004950721188543117 + ], + [ + 389, + 0.039483199496972876 + ], + [ + 394, + 0.6330534837805182 + ], + [ + 404, + 0.7817103172047252 + ], + [ + 407, + 0.6823999699851321 + ], + [ + 411, + 0.6433131381760853 + ], + [ + 416, + 0.38836166710308684 + ], + [ + 419, + 0.9424404531268578 + ], + [ + 427, + 0.9416208996288798 + ], + [ + 442, + 0.464631561467023 + ], + [ + 445, + 0.05811426540646991 + ], + [ + 449, + 0.5236142720162741 + ], + [ + 454, + 0.6959606686441475 + ], + [ + 458, + 0.7114773875585536 + ], + [ + 467, + 0.7370182907002597 + ], + [ + 473, + 0.712551829757311 + ], + [ + 480, + 0.2703085545685765 + ], + [ + 486, + 0.8409356030547432 + ], + [ + 490, + 0.3339936044373294 + ], + [ + 505, + 0.6731480730017201 + ], + [ + 511, + 0.5510372198280904 + ], + [ + 518, + 0.9099507706921075 + ], + [ + 532, + 0.7444376522606013 + ], + [ + 535, + 0.7873342440786003 + ], + [ + 547, + 0.38720446700816824 + ], + [ + 556, + 0.3797918005287988 + ], + [ + 559, + 0.9643028358284021 + ], + [ + 565, + 0.34065266945391315 + ], + [ + 569, + 0.7580761034949406 + ], + [ + 575, + 0.18638052073179645 + ], + [ + 578, + 0.5601071719554319 + ], + [ + 586, + 0.18460257671331792 + ], + [ + 591, + 0.9108698803491804 + ], + [ + 593, + 0.03680235791375863 + ], + [ + 597, + 0.13957554349728152 + ], + [ + 600, + 0.8352311796002897 + ], + [ + 606, + 0.39928574672731754 + ], + [ + 612, + 0.6878967967005524 + ], + [ + 618, + 0.005342657606916434 + ], + [ + 625, + 0.12972991033918113 + ], + [ + 629, + 0.4445868231476443 + ], + [ + 639, + 0.381442154995869 + ], + [ + 646, + 0.6365175693696874 + ] + ] + ] + ] + } +] \ No newline at end of file diff --git a/htmls/html_templates/pipeline_template.txt b/htmls/html_templates/pipeline_template.txt new file mode 100644 index 0000000000000000000000000000000000000000..f291cd3073c8a177a0084165cb4e96a863569e09 --- /dev/null +++ b/htmls/html_templates/pipeline_template.txt @@ -0,0 +1,1896 @@ +<!DOCTYPE html> +<html> + +<head> + <meta charset="UTF-8"> + <title>Pipeline Graph | Sliding Toys + + + + + + + +
+ +
+
+ + +
+ CONFIGS +
+ +
+ INPUT + + +
+ +
+ +
+ + +
+ OUTPUT + +
+ +
+ + +
+
+ + +
+
+

📚 Documents

+
+
+ +
+
+

📝 Question

+
+

📖 Answer

+
+

🔍 Output

+
+
+ + +
+ + + 📤 Send to Assistant + New + +
+
+
Click to show value
+
+
+ + +
+ + +
+ +
+
+ + +
+
+

Assistant

+ +
+ 🔑 API Key: +
+ Results: +
+ Your Specific Requirement + +
+ Send +
+ + +
+
+

Response

+
+
+
+ + + + + + + \ No newline at end of file diff --git a/htmls/html_templates/pipeline_template_emb.txt b/htmls/html_templates/pipeline_template_emb.txt new file mode 100644 index 0000000000000000000000000000000000000000..6816a1dc06b764d1ce4f8eaecce8f53b2a074d52 --- /dev/null +++ b/htmls/html_templates/pipeline_template_emb.txt @@ -0,0 +1,1888 @@ + + + + + + Pipeline Graph | Sliding Toys + + + + + + + +
+ +
+
+ + +
+ CONFIGS +
+ +
+ INPUT + + +
+ +
+ +
+ + +
+ OUTPUT + +
+ +
+ + +
+
+ + +
+
+

📚 Documents

+
+
+ +
+
+

📝 Question

+
+

📖 Answer

+
+

🔍 Output

+
+
+ + +
+ + + 📤 Send to Assistant + New + +
+
+
Click to show value
+
+
+ + +
+ + +
+ +
+
+ + +
+
+

Assistant

+ +
+ 🔑 API Key: +
+ Results: +
+ Your Specific Requirement + +
+ Send +
+ + +
+
+

Response

+
+
+
+ + + + + + + \ No newline at end of file diff --git a/htmls/pipeline copy.html b/htmls/pipeline copy.html new file mode 100644 index 0000000000000000000000000000000000000000..09931d99665bcf7fc0efb5e73cb2e54deed543ee --- /dev/null +++ b/htmls/pipeline copy.html @@ -0,0 +1,454 @@ + + + + + + Pipeline Graph + + + + +
+
+ + + + +
+ +
+
Documents
+
+
+ + +
+ +
+
+
+
+ + +
+ + +
+
+
Click document to show value
+
+
+ + +
+
+
+ + + + \ No newline at end of file diff --git a/htmls/pipeline.html b/htmls/pipeline.html new file mode 100644 index 0000000000000000000000000000000000000000..720ea1729c2a57b6454617e57753ee559684ed3c --- /dev/null +++ b/htmls/pipeline.html @@ -0,0 +1,645 @@ + + + + + Pipeline Graph + + + + +
+
+ + + + +
+ +
+
Documents
+
+
+ + +
+ +
+
+
+
+ + +
+ + +
+
+
+
Click document to show value
+
+ + +
+ + +
+
+
+ + + + \ No newline at end of file diff --git a/htmls/pipeline_backup.html b/htmls/pipeline_backup.html new file mode 100644 index 0000000000000000000000000000000000000000..80e4849bbf25f115fa58bbae98ebd9df44835616 --- /dev/null +++ b/htmls/pipeline_backup.html @@ -0,0 +1,646 @@ + + + + + + Pipeline Graph + + + + +
+
+ + + + +
+ +
+
Documents
+
+
+ + +
+ +
+
+
+
+ + +
+ + +
+
+
+
Click document to show value
+
+ + +
+ + +
+
+
+ + + + \ No newline at end of file diff --git a/htmls/pipeline_embed.html b/htmls/pipeline_embed.html new file mode 100644 index 0000000000000000000000000000000000000000..c676f2cf977875eeff2e27f169293c4f60f9adec --- /dev/null +++ b/htmls/pipeline_embed.html @@ -0,0 +1,636 @@ + + + + + Pipeline Graph + + + + +
+
+ + + + +
+ +
+
Documents
+
+
+ + +
+ +
+
+
+
+ + +
+ + +
+
+
+
Click document to show value
+
+ + +
+ + +
+
+
+ + + + \ No newline at end of file diff --git a/htmls/pipeline_graph.pdf b/htmls/pipeline_graph.pdf new file mode 100644 index 0000000000000000000000000000000000000000..5de61dc8e3fda824c9eb29771ea5cb74cdac7236 Binary files /dev/null and b/htmls/pipeline_graph.pdf differ diff --git a/htmls/pipeline_template.txt b/htmls/pipeline_template.txt new file mode 100644 index 0000000000000000000000000000000000000000..fcf0085e3dc1108712024cc1998b6bd700de6af1 --- /dev/null +++ b/htmls/pipeline_template.txt @@ -0,0 +1,645 @@ + + + + + Pipeline Graph + + + + +
+
+ + + + +
+ +
+
Documents
+
+
+ + +
+ +
+
+
+
+ + +
+ + +
+
+
+
Click document to show value
+
+ + +
+ + +
+
+
+ + + + \ No newline at end of file diff --git a/htmls/pipeline_template_EMBED.txt b/htmls/pipeline_template_EMBED.txt new file mode 100644 index 0000000000000000000000000000000000000000..ce2d4a00f005d47b8a9e18be2bfe4fe6080128c7 --- /dev/null +++ b/htmls/pipeline_template_EMBED.txt @@ -0,0 +1,636 @@ + + + + + Pipeline Graph + + + + +
+
+ + + + +
+ +
+
Documents
+
+
+ + +
+ +
+
+
+
+ + +
+ + +
+
+
+
Click document to show value
+
+ + +
+ + +
+
+
+ + + + \ No newline at end of file diff --git a/htmls/res_attr_span.json b/htmls/res_attr_span.json new file mode 100644 index 0000000000000000000000000000000000000000..c8966f30370b1506d0c7bd6dfae60532d13a348c --- /dev/null +++ b/htmls/res_attr_span.json @@ -0,0 +1,7785 @@ +[ + { + "data": { + "question": "Who has the highest goals in world football?", + "answer": "The players with the highest all-time goals and highest men's and women's international football goals differ. The player with the highest all-time men's football goals is Josef Bican, who in 2020 was recognized by FIFA, the international governing body of football, as the record scorer with an estimated 805 goals. Christine Sinclair has the highest goals in women's international football with 187 and is the all-time leader for international goals scored for men or women. Cristiano Ronaldo and Ali Daei are currently tied for leading goalscorer in the history of men's international football with 109.", + "qa_pairs": [ + { + "context": "No context provided", + "question": "Who has the highest goals in men's world international football?", + "short_answers": [ + "Daei", + "Ali Daei" + ], + "wikipage": null + }, + { + "context": "No context provided", + "question": "Who has the highest goals all-time in men's football?", + "short_answers": [ + "Bican", + "Josef Bican" + ], + "wikipage": null + }, + { + "context": "The first player to reach 100 international goals was Italian Elisabetta Vignotto. Abby Wambach scored 100 goals in 9 years, while Christine Sinclair reached the milestone in just under 10 years while Mia Hamm is the youngest player to score 100 international goals at the age of 26 years 185 days. Most played exclusively in the forward position, with Kristine Lilly and Michelle Akers having also played as midfielder. All players scored at a high average rate of more than one goal every three matches. International goals in this list should not include goals scored in penalty-shoot-out; see Penalty shootout (association football). Players who are currently active at international level are indicated in bold type background.", + "question": "Who has the highest goals in women's world international football?", + "short_answers": [ + "Sinclair", + "Christine Sinclair" + ], + "wikipage": "List of women's footballers with 100 or more international goals ..." + } + ], + "docs": [ + "Document [1](Title:Argentina\u2013Brazil football rivalry): \"Football Player of the Century\", by IFFHS International Federation of Football History and Statistics, 1999, \"South America Football Player of the Century\", by IFFHS International Federation of Football History and Statistics. Pel\u00e9's 1281 goals are recognized by FIFA as the highest total achieved by a professional footballer, although the Soccer Statistic Foundation (rssf) recognizes only 767 goals in official mode, occupying the third place after Josef Bican (805) and Romario (772). For his part, Maradona has been named the best soccer player in World Cup history both by The Times and FourFourTwo, publication that also rewarded him as the \"Best\n", + "Document [2](Title:Godfrey Chitalu): have beaten Gerd M\u00fcller's record of 85 goals in a year, the Football Association of Zambia claimed that the world record actually pertained to Godfrey Chitalu who had scored 116 goals (possibly 117) during the 1972 calendar year and 107 during the 1972 season. The difference of goals is due to first 9 goals being scored before the season officially started. The Football Association of Zambia presented the evidence to FIFA but a spokesperson responded that they would ratify neither Lionel Messi's nor Chitalu's records as they do not keep statistical track of domestic competitions. Nonetheless, it could constitute the\n", + "Document [3](Title:Godfrey Chitalu): highest official tally claimed by a national football association. Chitalu made his international debut on 29 June 1968 in a friendly match against Uganda in Lusaka which Zambia won 2\u20131. He scored his first goal in a 2\u20132 draw against the same team five days later. Chitalu played a prominent role during the World Cup qualification matches against Sudan with Zambia being eliminated on a strange rule which was peculiar to Africa and favoured the team that won the second leg. Despite the aggregate score being tied at 6\u20136 after Zambia won the first leg 4\u20132 and lost the return\n", + "Document [4](Title:Wartan Ghazarian): goals (4 in World Cup qualifiers, 3 in Asian Cup qualifiers, 12 in friendlies). His record was later broken by Roda Antar, after Roda scored his 20th goal in 2018 FIFA World Cup qualification match against Laos. On 16 November 2008, during Round 6 of the Lebanese Football League, at the age of 39 years, Vartan scored his 130th goal in the Lebanese first division against Tadamon Tyre, becoming officially the highest all-time scorer in the history of Lebanese football. Some officials do not recognize the 12 goals he scored in the 2000\u20132001 season which was canceled. However, his remaining\n", + "Document [5](Title:Josef Bican): for Christmas, but died less than 2 weeks before that, at the age of 88. Josef Bican Josef \"Pepi\" Bican (25 September 1913 \u2013 12 December 2001) was a Czech-Austrian professional footballer who played as a striker. Rec.Sport.Soccer Statistics Foundation (RSSSF) estimates that he scored at least 805 goals in all competitive matches, which would make him the most prolific scorer of all time. Having scored a record 600 league goals and at least 1468 goals overall, the International Federation of Football History & Statistics (IFFHS) awarded Bican the \"Golden Ball\" as the greatest goalscorer of the last century. He\n" + ] + }, + "doc_cache": [ + "Document [1](Title:Humberto Suazo)announced his retirement from football on January 14, 2016. Suazo had also become a fixture with his international side. In 2006, he scored 17 goals in national and international matches, surpassing Peter Crouch by one goal for the \"World's Top Goal Scorer Award\". His four international goals all came in friendlies. He scored goals against New Zealand and Sweden. His other two goals came in the form of penalties versus Ivory Coast and Colombia. In January 2007, he was awarded world's top goal scorer of 2006 by the International Federation of Football History & Statistics in Salzburg, Austria. He was\n", + "Document [2](Title:FIFA Puska\u0301s Award)forward European football produced in first-division football, and scored a once world record of 84 goals in 85 internationals. He is one of the top scorers of the 20th century, with 512 goals in 528 matches. \"It is important to preserve the memory of those footballing greats who have left their mark on our history. Ferenc Pusk\u00e1s was not only a player with immense talent who won many honours, but also a remarkable man. FIFA is therefore delighted to pay tribute to him by dedicating this award to his memory,\" said Blatter at the inauguration of the award in Budapest.\n", + "Document [3](Title:FIFA Puska\u0301s Award)forward European football produced in first-division football, and scored a once world record of 84 goals in 85 internationals. He is one of the top scorers of the 20th century, with 512 goals in 528 matches. \"It is important to preserve the memory of those footballing greats who have left their mark on our history. Ferenc Pusk\u00e1s was not only a player with immense talent who won many honours, but also a remarkable man. FIFA is therefore delighted to pay tribute to him by dedicating this award to his memory,\" said Blatter at the inauguration of the award in Budapest.\n", + "Document [4](Title:Oktay Dereliog\u0306lu)Oktay Derelio\u011flu Oktay Derelio\u011flu (born 17 December 1975) is Turkish former international footballer and current football coach. Derelio\u011flu played as a forward and is notable for his goal against Belgium during the qualifying rounds of the 1998 FIFA World Cup. The goal started at midfield before Derelio\u011flu jinked past six Belgian defenders, one of them twice, mimicking Diego Maradona's \"Goal of the Century\". Derelio\u011flu holds the record for most goals in European competitions for Be\u015fikta\u015f with 14. Derelio\u011flu was born in Istanbul and began his club career with local club Fatih Karag\u00fcmr\u00fck. He made his debut at the age of\n", + "Document [5](Title:FIFA World Cup top goalscorers)Germany or West Germany. In total, 64 footballers came from UEFA (Europe), 29 from CONMEBOL (South America), and only four from elsewhere: Cameroon, Ghana, Australia, and the United States. Fontaine holds the record for the most goals scored at a single tournament, with 13 goals in 1958. The players that came closest to this record were Kocsis in 1954, M\u00fcller in 1970 and Portugal's Eus\u00e9bio in 1966, with 11, 10 and 9, respectively. The lowest scoring tournament's top scorer was in 1962, when six players tied at only four goals each. Across the 21 editions of the World Cup, 30\n", + "Document [6](Title:Argentina\u2013Brazil football rivalry)\"Football Player of the Century\", by IFFHS International Federation of Football History and Statistics, 1999, \"South America Football Player of the Century\", by IFFHS International Federation of Football History and Statistics. Pel\u00e9's 1281 goals are recognized by FIFA as the highest total achieved by a professional footballer, although the Soccer Statistic Foundation (rssf) recognizes only 767 goals in official mode, occupying the third place after Josef Bican (805) and Romario (772). For his part, Maradona has been named the best soccer player in World Cup history both by The Times and FourFourTwo, publication that also rewarded him as the \"Best\n" + ], + "log": [ + "gpt-3.5-turbo-[1] -> retriever-[4]\n: Query: \"Top goal scorers in international football history\"", + "retriever-[4] -> gpt-3.5-turbo-[3]\n: Document [1](Title:Humberto Suazo)announced his retirement from football on January 14, 2016. Suazo had also become a fixture with his international side. In 2006, he scored 17 goals in national and international matches, surpassing Peter Crouch by one goal for the \"World's Top Goal Scorer Award\". His four international goals all came in friendlies. He scored goals against New Zealand and Sweden. His other two goals came in the form of penalties versus Ivory Coast and Colombia. In January 2007, he was awarded world's top goal scorer of 2006 by the International Federation of Football History & Statistics in Salzburg, Austria. He was\n\nDocument [2](Title:FIFA Puska\u0301s Award)forward European football produced in first-division football, and scored a once world record of 84 goals in 85 internationals. He is one of the top scorers of the 20th century, with 512 goals in 528 matches. \"It is important to preserve the memory of those footballing greats who have left their mark on our history. Ferenc Pusk\u00e1s was not only a player with immense talent who won many honours, but also a remarkable man. FIFA is therefore delighted to pay tribute to him by dedicating this award to his memory,\" said Blatter at the inauguration of the award in Budapest.\n", + "gpt-3.5-turbo-[3] -> evaluator-[2]\n: ['Humberto Suazo was awarded the title of the world\\'s top goal scorer of 2006, surpassing Peter Crouch by one goal for the \"World\\'s Top Goal Scorer Award\"[1]. ', 'Ferenc Pusk\u00e1s holds the record for the highest number of goals in world football, scoring 84 goals in 85 internationals and a total of 512 goals in 528 matches[2]. ']", + "evaluator-[2] -> gpt-3.5-turbo-[1]\n: {'answer': 'Humberto Suazo was awarded the title of the world\\'s top goal scorer of 2006, surpassing Peter Crouch by one goal for the \"World\\'s Top Goal Scorer Award\"[1]. '}", + "gpt-3.5-turbo-[1] -> retriever-[4]\n: Query: \"Current leading goal scorers in top European football leagues\"", + "retriever-[4] -> gpt-3.5-turbo-[3]\n: Document [3](Title:FIFA Puska\u0301s Award)forward European football produced in first-division football, and scored a once world record of 84 goals in 85 internationals. He is one of the top scorers of the 20th century, with 512 goals in 528 matches. \"It is important to preserve the memory of those footballing greats who have left their mark on our history. Ferenc Pusk\u00e1s was not only a player with immense talent who won many honours, but also a remarkable man. FIFA is therefore delighted to pay tribute to him by dedicating this award to his memory,\" said Blatter at the inauguration of the award in Budapest.\n\nDocument [4](Title:Oktay Dereliog\u0306lu)Oktay Derelio\u011flu Oktay Derelio\u011flu (born 17 December 1975) is Turkish former international footballer and current football coach. Derelio\u011flu played as a forward and is notable for his goal against Belgium during the qualifying rounds of the 1998 FIFA World Cup. The goal started at midfield before Derelio\u011flu jinked past six Belgian defenders, one of them twice, mimicking Diego Maradona's \"Goal of the Century\". Derelio\u011flu holds the record for most goals in European competitions for Be\u015fikta\u015f with 14. Derelio\u011flu was born in Istanbul and began his club career with local club Fatih Karag\u00fcmr\u00fck. He made his debut at the age of\n", + "gpt-3.5-turbo-[3] -> evaluator-[2]\n: ['Ferenc Pusk\u00e1s holds the once world record of scoring 84 goals in 85 internationals in European football, making him one of the top scorers of the 20th century[3]. ', 'Oktay Derelio\u011flu holds the record for most goals in European competitions for Be\u015fikta\u015f with 14[4]. ']", + "evaluator-[2] -> gpt-3.5-turbo-[1]\n: {'answer': 'Ferenc Pusk\u00e1s holds the once world record of scoring 84 goals in 85 internationals in European football, making him one of the top scorers of the 20th century[3]. '}", + "gpt-3.5-turbo-[1] -> retriever-[4]\n: Query: \"Players with most goals in FIFA World Cup history\"", + "retriever-[4] -> gpt-3.5-turbo-[3]\n: Document [5](Title:FIFA World Cup top goalscorers)Germany or West Germany. In total, 64 footballers came from UEFA (Europe), 29 from CONMEBOL (South America), and only four from elsewhere: Cameroon, Ghana, Australia, and the United States. Fontaine holds the record for the most goals scored at a single tournament, with 13 goals in 1958. The players that came closest to this record were Kocsis in 1954, M\u00fcller in 1970 and Portugal's Eus\u00e9bio in 1966, with 11, 10 and 9, respectively. The lowest scoring tournament's top scorer was in 1962, when six players tied at only four goals each. Across the 21 editions of the World Cup, 30\n\nDocument [6](Title:Argentina\u2013Brazil football rivalry)\"Football Player of the Century\", by IFFHS International Federation of Football History and Statistics, 1999, \"South America Football Player of the Century\", by IFFHS International Federation of Football History and Statistics. Pel\u00e9's 1281 goals are recognized by FIFA as the highest total achieved by a professional footballer, although the Soccer Statistic Foundation (rssf) recognizes only 767 goals in official mode, occupying the third place after Josef Bican (805) and Romario (772). For his part, Maradona has been named the best soccer player in World Cup history both by The Times and FourFourTwo, publication that also rewarded him as the \"Best\n", + "gpt-3.5-turbo-[3] -> evaluator-[2]\n: ['At the FIFA World Cup, Just Fontaine holds the record for the most goals scored at a single tournament with 13 goals in 1958[5]. ', 'Pel\u00e9 holds the record for the highest number of goals in world football, with 1281 goals recognized by FIFA[6]. ']", + "evaluator-[2] -> gpt-3.5-turbo-[1]\n: {'answer': 'At the FIFA World Cup, Just Fontaine holds the record for the most goals scored at a single tournament with 13 goals in 1958[5]. '}" + ], + "output": [ + "Humberto Suazo was awarded the title of the world's top goal scorer of 2006, surpassing Peter Crouch by one goal for the \"World's Top Goal Scorer Award\" [1].", + "Ferenc Pusk\u00e1s holds the once world record of scoring 84 goals in 85 internationals in European football, making him one of the top scorers of the 20th century [3]. ", + "At the FIFA World Cup, Just Fontaine holds the record for the most goals scored at a single tournament with 13 goals in 1958 [5]. " + ], + "result": { + "length": 82 + }, + "token_used": 4774, + "doc_level": [ + [ + 0.1525312689055882, + 0.13723357149030624, + 0.8945967055108693, + 0.8918892159503512, + 0.9125552169410704, + 0.5720154682715246 + ], + [ + 0.07981304257462607, + 0.9104953104479965, + 0.9505872882718894, + 0.588912583235743, + 0.9384378755311519, + 0.9662720416898128 + ], + [ + 0.06625038357332969, + 0.08866438991752135, + 0.4683328250838106, + 0.9047185162464918, + 0.5219900685430617, + 0.3008872490613551 + ] + ], + "span_level": [ + [ + [ + [ + 34, + 0.31635037610603406 + ], + [ + 94, + 0.10924533968104348 + ], + [ + 155, + 1.0 + ], + [ + 299, + 0.03795645050167794 + ], + [ + 352, + 0.06997578794840735 + ], + [ + 400, + 0.06945329053658097 + ], + [ + 483, + 0.36921570465361125 + ], + [ + 634, + 0.06945329053658097 + ] + ], + [ + [ + 38, + 0.0888711380112385 + ], + [ + 166, + 0.04813612205845545 + ], + [ + 247, + 0.06945329053658097 + ], + [ + 356, + 0.07075119657827829 + ], + [ + 461, + 0.0486374789745556 + ] + ], + [ + [ + 38, + 0.05986623308934264 + ], + [ + 166, + 0.04493846287017659 + ], + [ + 247, + 0.05648863131193116 + ], + [ + 356, + 0.055667330737615275 + ], + [ + 461, + 0.017075403445374317 + ] + ], + [ + [ + 37, + 0.07691232071717985 + ], + [ + 166, + 0.10717279220393511 + ], + [ + 298, + 0.06945329053658097 + ], + [ + 449, + 0.06945329053658097 + ], + [ + 539, + 0.02466750309982005 + ], + [ + 631, + 0.07808994066595093 + ] + ], + [ + [ + 50, + 0.00407066465173121 + ], + [ + 75, + 0.06945329053658097 + ], + [ + 240, + 0.06945329053658097 + ], + [ + 339, + 0.08606646563369867 + ], + [ + 485, + 0.07858350684942052 + ], + [ + 588, + 0.06540941290401069 + ] + ], + [ + [ + 53, + 0.04926487893947416 + ], + [ + 281, + 0.09495672544988305 + ], + [ + 543, + 0.02142143113571107 + ] + ] + ], + [ + [ + [ + 34, + 0.06945329053658097 + ], + [ + 94, + 0.014892285888584392 + ], + [ + 155, + 0.11873730965045484 + ], + [ + 299, + 0.016837276891539257 + ], + [ + 352, + 0.06945329053658097 + ], + [ + 400, + 0.13866681916497617 + ], + [ + 483, + 0.0 + ], + [ + 634, + 0.06945329053658097 + ] + ], + [ + [ + 38, + 0.35364032532577566 + ], + [ + 166, + 0.21181791821962617 + ], + [ + 247, + 0.06945329053658097 + ], + [ + 356, + 0.12972420926835188 + ], + [ + 461, + 0.10658348160850646 + ] + ], + [ + [ + 38, + 0.40054953958438266 + ], + [ + 166, + 0.1892697527829389 + ], + [ + 247, + 0.06945329053658097 + ], + [ + 356, + 0.08801543291255308 + ], + [ + 461, + 0.06945329053658097 + ] + ], + [ + [ + 37, + 0.06945329053658097 + ], + [ + 166, + 0.06945329053658097 + ], + [ + 298, + 0.06945329053658097 + ], + [ + 449, + 0.06664175852342942 + ], + [ + 539, + 0.06945329053658097 + ], + [ + 631, + 0.06945329053658097 + ] + ], + [ + [ + 50, + 0.06945329053658097 + ], + [ + 75, + 0.06945329053658097 + ], + [ + 240, + 0.06945329053658097 + ], + [ + 339, + 0.06576248490502087 + ], + [ + 485, + 0.14325220329686497 + ], + [ + 588, + 0.09072133578534769 + ] + ], + [ + [ + 53, + 0.10273279677330707 + ], + [ + 281, + 0.06945329053658097 + ], + [ + 543, + 0.06945329053658097 + ] + ] + ], + [ + [ + [ + 34, + 0.08982520282075931 + ], + [ + 94, + 0.07234878208608048 + ], + [ + 155, + 0.07356061147826867 + ], + [ + 299, + 0.066006091346853 + ], + [ + 352, + 0.06945329053658097 + ], + [ + 400, + 0.06945329053658097 + ], + [ + 483, + 0.06945329053658097 + ], + [ + 634, + 0.06945329053658097 + ] + ], + [ + [ + 38, + 0.04346522132532467 + ], + [ + 166, + 0.06945329053658097 + ], + [ + 247, + 0.06945329053658097 + ], + [ + 356, + 0.09051516577363684 + ], + [ + 461, + 0.06945329053658097 + ] + ], + [ + [ + 38, + 0.06140389118968235 + ], + [ + 166, + 0.05038320508075013 + ], + [ + 247, + 0.05651735052559898 + ], + [ + 356, + 0.06945329053658097 + ], + [ + 461, + 0.06945329053658097 + ] + ], + [ + [ + 37, + 0.06945329053658097 + ], + [ + 166, + 0.07011971520735087 + ], + [ + 298, + 0.06958718873230818 + ], + [ + 449, + 0.06945329053658097 + ], + [ + 539, + 0.06945329053658097 + ], + [ + 631, + 0.06945329053658097 + ] + ], + [ + [ + 50, + 0.1306698503144333 + ], + [ + 75, + 0.06945329053658097 + ], + [ + 240, + 0.2960196172592031 + ], + [ + 339, + 0.06945329053658097 + ], + [ + 485, + 0.06945329053658097 + ], + [ + 588, + 0.06945329053658097 + ] + ], + [ + [ + 53, + 0.09464446248792138 + ], + [ + 281, + 0.03402347259595398 + ], + [ + 543, + 0.06945329053658097 + ] + ] + ] + ], + "word_level": [ + [ + [ + [ + 34, + 0.9726499810603795 + ], + [ + 44, + 0.08605671020733052 + ], + [ + 48, + 0.7596409861809017 + ], + [ + 59, + 0.05763882612999671 + ], + [ + 64, + 0.5145239051825758 + ], + [ + 73, + 0.45700032932647927 + ], + [ + 76, + 0.16541894551748604 + ], + [ + 84, + 0.5532297599051686 + ], + [ + 88, + 0.43057656720854987 + ], + [ + 94, + 0.14214088498206523 + ], + [ + 100, + 0.8758168922684791 + ], + [ + 104, + 0.20081190189567377 + ], + [ + 109, + 0.9356991116217385 + ], + [ + 116, + 0.7219574313352661 + ], + [ + 118, + 0.8708936103353166 + ], + [ + 126, + 0.09109601532609823 + ], + [ + 131, + 0.42478689455399443 + ], + [ + 135, + 0.9532934971578741 + ], + [ + 149, + 0.9217453358083582 + ], + [ + 155, + 0.7332662377274012 + ], + [ + 158, + 0.8796839520635138 + ], + [ + 164, + 0.6275774828812761 + ], + [ + 167, + 0.6734740068382253 + ], + [ + 174, + 0.6371931070574532 + ], + [ + 177, + 0.13655041255713818 + ], + [ + 183, + 0.28302491354819126 + ], + [ + 186, + 0.8787614548337237 + ], + [ + 195, + 0.9899222403661093 + ], + [ + 199, + 0.6945864751528461 + ], + [ + 213, + 0.9552826041096931 + ], + [ + 222, + 0.583279356201901 + ], + [ + 233, + 0.9941956623758189 + ], + [ + 239, + 0.48195696524188913 + ], + [ + 246, + 0.6677641280707589 + ], + [ + 249, + 0.7327067274655747 + ], + [ + 253, + 0.640038487613746 + ], + [ + 258, + 0.6762132799133805 + ], + [ + 262, + 0.5008711689992287 + ], + [ + 266, + 0.8340187102575166 + ], + [ + 275, + 0.8529102970249229 + ], + [ + 279, + 0.8697714238530159 + ], + [ + 284, + 0.2753275960783972 + ], + [ + 291, + 0.19085553946188516 + ], + [ + 299, + 0.4304598693894228 + ], + [ + 303, + 0.7311499877171561 + ], + [ + 308, + 0.1983888594019283 + ], + [ + 322, + 0.6189171026425311 + ], + [ + 328, + 0.2633704071272944 + ], + [ + 332, + 0.7764581833090596 + ], + [ + 337, + 0.0047831781014120445 + ], + [ + 340, + 0.9517417934051873 + ], + [ + 352, + 0.17670655740992414 + ], + [ + 355, + 0.5730771523543212 + ], + [ + 362, + 0.27374706562798024 + ], + [ + 368, + 0.30172016092127685 + ], + [ + 376, + 0.9076941653477242 + ], + [ + 380, + 0.7007656495300092 + ], + [ + 388, + 0.9118009978077177 + ], + [ + 392, + 0.6101883977635021 + ], + [ + 400, + 0.9372889641903638 + ], + [ + 404, + 0.8401677216195338 + ], + [ + 410, + 0.6423756646375247 + ], + [ + 414, + 0.051317570730764706 + ], + [ + 420, + 0.04104655183006123 + ], + [ + 425, + 0.6447578663016221 + ], + [ + 428, + 0.9253858884115785 + ], + [ + 432, + 0.06469230538917103 + ], + [ + 437, + 0.7059482176331049 + ], + [ + 440, + 0.1979618935792936 + ], + [ + 450, + 0.7946809861922456 + ], + [ + 457, + 0.3861910212203763 + ], + [ + 463, + 0.596413727876745 + ], + [ + 469, + 0.678685363839173 + ], + [ + 473, + 0.49727031812918976 + ], + [ + 483, + 0.8449105859428789 + ], + [ + 486, + 0.4863451116791887 + ], + [ + 494, + 0.4022791995796927 + ], + [ + 500, + 0.738801668038707 + ], + [ + 503, + 0.3752619227763957 + ], + [ + 507, + 0.48756856950671845 + ], + [ + 515, + 0.23666096661853975 + ], + [ + 523, + 0.9717143834132929 + ], + [ + 527, + 0.3015668769856851 + ], + [ + 532, + 0.4106303810200229 + ], + [ + 539, + 0.8240988498744837 + ], + [ + 542, + 0.5130940235821316 + ], + [ + 547, + 0.8835523732699343 + ], + [ + 550, + 0.5091943652363315 + ], + [ + 554, + 0.8363470240292314 + ], + [ + 568, + 0.8422641425363049 + ], + [ + 579, + 0.16864565697441403 + ], + [ + 582, + 0.4769481421525644 + ], + [ + 591, + 0.97665055636654 + ], + [ + 599, + 0.2481313029375718 + ], + [ + 601, + 0.435512214938918 + ], + [ + 612, + 0.3262569046900523 + ], + [ + 615, + 0.3887858238150872 + ], + [ + 625, + 0.4116203910375821 + ], + [ + 634, + 0.20804166314868577 + ], + [ + 637, + 0.998009136720274 + ] + ], + [ + [ + 38, + 0.01895191794983786 + ], + [ + 46, + 0.8091549259035867 + ], + [ + 55, + 0.22371531154224544 + ], + [ + 64, + 0.04170305585498946 + ], + [ + 73, + 0.7577407985746973 + ], + [ + 76, + 0.5606045725204206 + ], + [ + 91, + 0.2021533517584695 + ], + [ + 101, + 0.9593754788838803 + ], + [ + 105, + 0.5116245004131195 + ], + [ + 112, + 0.3831201417660244 + ], + [ + 114, + 0.3527125414730288 + ], + [ + 119, + 0.7386062631099651 + ], + [ + 125, + 0.7990031572361751 + ], + [ + 132, + 0.6375414755678795 + ], + [ + 135, + 0.1986568214797113 + ], + [ + 138, + 0.2990939923338106 + ], + [ + 144, + 0.042350902993752104 + ], + [ + 147, + 0.7923706775051015 + ], + [ + 150, + 0.5650679729269246 + ], + [ + 166, + 0.1876496745408287 + ], + [ + 169, + 0.9566224077010641 + ], + [ + 172, + 0.09018415352300413 + ], + [ + 176, + 0.565513881421995 + ], + [ + 179, + 0.1958777987756306 + ], + [ + 183, + 0.2778699853420834 + ], + [ + 187, + 0.42248651482377897 + ], + [ + 195, + 0.4950370421279131 + ], + [ + 198, + 0.09944192654106454 + ], + [ + 202, + 0.8355734164643888 + ], + [ + 207, + 0.849422416524635 + ], + [ + 216, + 0.9025131530244342 + ], + [ + 221, + 0.2246783253758975 + ], + [ + 225, + 0.6567175368851491 + ], + [ + 231, + 0.7061986337567662 + ], + [ + 234, + 0.9172364160432521 + ], + [ + 238, + 0.9291620251673874 + ], + [ + 247, + 0.35835364150186755 + ], + [ + 251, + 0.19806664327112145 + ], + [ + 254, + 0.817528899263466 + ], + [ + 264, + 0.6818162944034358 + ], + [ + 267, + 0.3471461344822452 + ], + [ + 276, + 0.24402369364302656 + ], + [ + 280, + 0.15965250543770604 + ], + [ + 287, + 0.11910002417525267 + ], + [ + 290, + 0.8414386419662918 + ], + [ + 296, + 0.3693408189901478 + ], + [ + 308, + 0.03007754373349747 + ], + [ + 315, + 0.7652163147590383 + ], + [ + 319, + 0.9630074508590972 + ], + [ + 324, + 0.06632505255957166 + ], + [ + 329, + 0.6930792074511971 + ], + [ + 335, + 0.41169847826129946 + ], + [ + 340, + 0.089983276967049 + ], + [ + 343, + 0.9221129577424041 + ], + [ + 347, + 0.024352377304205364 + ], + [ + 356, + 0.8363505282373193 + ], + [ + 363, + 0.7775150374476623 + ], + [ + 370, + 0.0069677471496412124 + ], + [ + 374, + 0.19400039836301552 + ], + [ + 378, + 0.13912023964891063 + ], + [ + 383, + 0.4061422221890222 + ], + [ + 385, + 0.6396258935475556 + ], + [ + 392, + 0.15742931839217533 + ], + [ + 397, + 0.2671888605480571 + ], + [ + 405, + 0.5029253150782247 + ], + [ + 412, + 0.7820817314198254 + ], + [ + 416, + 0.9298850123676912 + ], + [ + 420, + 0.730881373753405 + ], + [ + 425, + 0.15083904029380713 + ], + [ + 434, + 0.11458460764916611 + ], + [ + 438, + 0.8203304115935969 + ], + [ + 443, + 0.05505432706003832 + ], + [ + 445, + 0.18991814324255174 + ], + [ + 456, + 0.789323812722833 + ], + [ + 461, + 0.08249958198216845 + ], + [ + 466, + 0.7474939904482418 + ], + [ + 469, + 0.2321942352038885 + ], + [ + 479, + 0.6523289419822869 + ], + [ + 489, + 0.042554905862500436 + ], + [ + 492, + 0.6484626644520137 + ], + [ + 496, + 0.09481440010070763 + ], + [ + 504, + 0.7484695642032869 + ], + [ + 507, + 0.7946473942871619 + ], + [ + 511, + 0.7196758040157935 + ], + [ + 514, + 0.20146651780318037 + ], + [ + 525, + 0.2566481277219588 + ], + [ + 530, + 0.516785834040892 + ], + [ + 536, + 0.9657973341980828 + ], + [ + 539, + 0.16493605065462802 + ], + [ + 543, + 0.24896831106607964 + ], + [ + 552, + 0.133105467995994 + ], + [ + 557, + 0.4495983478008049 + ], + [ + 565, + 0.08028646005213202 + ], + [ + 568, + 0.7061667025527271 + ], + [ + 572, + 0.46326981753103535 + ], + [ + 585, + 0.40632475728473716 + ], + [ + 588, + 0.22393808944445026 + ], + [ + 592, + 0.2877759699302114 + ], + [ + 598, + 0.14091002413323384 + ], + [ + 601, + 0.4754504181304122 + ] + ], + [ + [ + 38, + 0.9794652317785165 + ], + [ + 46, + 0.7592533367173389 + ], + [ + 55, + 0.15029712452277344 + ], + [ + 64, + 0.7262454247566498 + ], + [ + 73, + 0.5179997577367508 + ], + [ + 76, + 0.08462317188116808 + ], + [ + 91, + 0.5267665429059917 + ], + [ + 101, + 0.2992592195180812 + ], + [ + 105, + 0.4133018610033953 + ], + [ + 112, + 0.942428206399702 + ], + [ + 114, + 0.7781863816223291 + ], + [ + 119, + 0.38582786134814384 + ], + [ + 125, + 0.5156602790786449 + ], + [ + 132, + 0.8926193689742762 + ], + [ + 135, + 0.5488150895385591 + ], + [ + 138, + 0.6737410553077283 + ], + [ + 144, + 0.5398186070880724 + ], + [ + 147, + 0.5152492017639415 + ], + [ + 150, + 0.3901711046270677 + ], + [ + 166, + 0.029766350051515733 + ], + [ + 169, + 0.20663417157776365 + ], + [ + 172, + 0.7288940548079337 + ], + [ + 176, + 0.7344787705705658 + ], + [ + 179, + 0.6229041369512515 + ], + [ + 183, + 0.9260335313847975 + ], + [ + 187, + 0.6477851651074203 + ], + [ + 195, + 0.4076175883589801 + ], + [ + 198, + 0.5767604149545458 + ], + [ + 202, + 0.20272054804405792 + ], + [ + 207, + 0.42307064574956366 + ], + [ + 216, + 0.6422707404844868 + ], + [ + 221, + 0.5261544740747952 + ], + [ + 225, + 0.48358792850477894 + ], + [ + 231, + 0.2615922892969095 + ], + [ + 234, + 0.1089786668351792 + ], + [ + 238, + 0.6399359365157113 + ], + [ + 247, + 0.7151429307962669 + ], + [ + 251, + 0.6894405827717566 + ], + [ + 254, + 0.05739548760055557 + ], + [ + 264, + 0.3883109323975241 + ], + [ + 267, + 0.9803369286933918 + ], + [ + 276, + 0.20182822729204963 + ], + [ + 280, + 0.5334362762069331 + ], + [ + 287, + 0.2603792884306666 + ], + [ + 290, + 0.02265491900456451 + ], + [ + 296, + 0.5723929074083922 + ], + [ + 308, + 0.8290600915093783 + ], + [ + 315, + 0.6556228427425396 + ], + [ + 319, + 0.22863902437526407 + ], + [ + 324, + 0.41073426672828317 + ], + [ + 329, + 0.693002119124838 + ], + [ + 335, + 0.30202850028750283 + ], + [ + 340, + 0.8445374136110521 + ], + [ + 343, + 0.08917522423474677 + ], + [ + 347, + 0.5714662784385183 + ], + [ + 356, + 0.2768355861289127 + ], + [ + 363, + 0.08349113497294103 + ], + [ + 370, + 0.05071155820628903 + ], + [ + 374, + 0.753990553339965 + ], + [ + 378, + 0.8917792065894531 + ], + [ + 383, + 0.3229700214611694 + ], + [ + 385, + 0.6017303429942582 + ], + [ + 392, + 0.9433062412100356 + ], + [ + 397, + 0.7093642077812163 + ], + [ + 405, + 0.4387203468807386 + ], + [ + 412, + 0.9137111697964068 + ], + [ + 416, + 0.5185792052653937 + ], + [ + 420, + 0.23895882737279361 + ], + [ + 425, + 0.3789222896282899 + ], + [ + 434, + 0.3383545103692106 + ], + [ + 438, + 0.6237546870972888 + ], + [ + 443, + 0.4478093437008268 + ], + [ + 445, + 0.8730950451877334 + ], + [ + 456, + 0.9843639356651239 + ], + [ + 461, + 0.6164389340883191 + ], + [ + 466, + 0.4512979468735716 + ], + [ + 469, + 0.35787370575737076 + ], + [ + 479, + 0.1813561722378061 + ], + [ + 489, + 0.7023033967532876 + ], + [ + 492, + 0.23272337126755438 + ], + [ + 496, + 0.6075361189595367 + ], + [ + 504, + 0.9358718504466712 + ], + [ + 507, + 0.6826051938380512 + ], + [ + 511, + 0.01681792279553651 + ], + [ + 514, + 0.6038252851495565 + ], + [ + 525, + 0.8422884785841935 + ], + [ + 530, + 0.5711223845833082 + ], + [ + 536, + 0.34628068509472 + ], + [ + 539, + 0.6634623105289472 + ], + [ + 543, + 0.6520574099131884 + ], + [ + 552, + 0.023118082819996943 + ], + [ + 557, + 0.08254306442643256 + ], + [ + 565, + 0.7991224776739679 + ], + [ + 568, + 0.06812939578506794 + ], + [ + 572, + 0.6933946560110053 + ], + [ + 585, + 0.7751816211920642 + ], + [ + 588, + 0.4719612811355067 + ], + [ + 592, + 0.6385257568754354 + ], + [ + 598, + 0.2027583817092793 + ], + [ + 601, + 0.5310095810091856 + ] + ], + [ + [ + 37, + 0.6786180923777138 + ], + [ + 43, + 0.7356796981712884 + ], + [ + 54, + 0.3122624183076076 + ], + [ + 60, + 0.6960494822400738 + ], + [ + 71, + 0.15055360315844535 + ], + [ + 77, + 0.4650389443155072 + ], + [ + 80, + 0.3238184922255305 + ], + [ + 89, + 0.7516940222095565 + ], + [ + 95, + 0.04539187097069175 + ], + [ + 98, + 0.9652899114059557 + ], + [ + 106, + 0.5557162320991196 + ], + [ + 113, + 0.93032680725156 + ], + [ + 127, + 0.567558913809352 + ], + [ + 138, + 0.5018757460910768 + ], + [ + 142, + 0.39583999956108173 + ], + [ + 150, + 0.18811390405119 + ], + [ + 159, + 0.5526108608245548 + ], + [ + 166, + 0.5607599981278294 + ], + [ + 177, + 0.639715257789935 + ], + [ + 184, + 0.2866673539359168 + ], + [ + 187, + 0.12031800704191531 + ], + [ + 189, + 0.9801948217129368 + ], + [ + 197, + 0.5936832049036366 + ], + [ + 201, + 0.5597963379805618 + ], + [ + 204, + 0.06429644050515182 + ], + [ + 212, + 0.7830986671175035 + ], + [ + 216, + 0.10297368806592178 + ], + [ + 220, + 0.009071092071019304 + ], + [ + 225, + 0.395915278184511 + ], + [ + 233, + 0.9618035411308056 + ], + [ + 241, + 0.5271786312897606 + ], + [ + 248, + 0.8895340283488209 + ], + [ + 252, + 0.2352986752761781 + ], + [ + 263, + 0.6123150016530776 + ], + [ + 270, + 0.3841032651074002 + ], + [ + 273, + 0.7839371360987384 + ], + [ + 277, + 0.17580201465906675 + ], + [ + 282, + 0.535494650333246 + ], + [ + 287, + 0.8418923054436044 + ], + [ + 293, + 0.9786186844928716 + ], + [ + 298, + 0.14803181622976813 + ], + [ + 302, + 0.9855370806350663 + ], + [ + 307, + 0.7433888801183093 + ], + [ + 315, + 0.34361312415503853 + ], + [ + 318, + 0.042822262077160556 + ], + [ + 327, + 0.352962714409533 + ], + [ + 334, + 0.8531925329491449 + ], + [ + 345, + 0.4332054261440911 + ], + [ + 352, + 0.11664314069391213 + ], + [ + 357, + 0.9806071645369893 + ], + [ + 361, + 0.8410292477122081 + ], + [ + 369, + 0.24147037549446226 + ], + [ + 380, + 0.7431827362279382 + ], + [ + 384, + 0.7586627544696628 + ], + [ + 387, + 0.31432726090688146 + ], + [ + 392, + 0.9215318718781551 + ], + [ + 399, + 0.6066644289228255 + ], + [ + 409, + 0.3426301344417091 + ], + [ + 415, + 0.15318413972607248 + ], + [ + 426, + 0.9936992737784514 + ], + [ + 432, + 0.9168614614913766 + ], + [ + 435, + 0.410322332376718 + ], + [ + 439, + 0.29025666713439335 + ], + [ + 449, + 0.2656483776458468 + ], + [ + 460, + 0.7939536573715431 + ], + [ + 466, + 0.23925041705171124 + ], + [ + 470, + 0.7643591618713277 + ], + [ + 477, + 0.5153722370211955 + ], + [ + 481, + 0.6415445619502043 + ], + [ + 486, + 0.5384996847469026 + ], + [ + 492, + 0.29896405132460746 + ], + [ + 495, + 0.7307102179473692 + ], + [ + 504, + 0.14217340812378376 + ], + [ + 517, + 0.003339393350253017 + ], + [ + 521, + 0.45156981921232253 + ], + [ + 530, + 0.7355754723301862 + ], + [ + 535, + 0.19204499683207 + ], + [ + 539, + 0.5058963573199146 + ], + [ + 550, + 0.07530362469826835 + ], + [ + 554, + 0.13473159760964692 + ], + [ + 559, + 0.8842680623215865 + ], + [ + 562, + 0.4003265968065788 + ], + [ + 571, + 0.2500603578159315 + ], + [ + 575, + 0.8857690385941768 + ], + [ + 581, + 0.6726743051429795 + ], + [ + 585, + 0.8563962875675292 + ], + [ + 590, + 0.6757159081925681 + ], + [ + 597, + 0.7585621678845341 + ], + [ + 602, + 0.42496674443286775 + ], + [ + 608, + 0.36906409880961966 + ], + [ + 613, + 0.45559683377485727 + ], + [ + 619, + 0.3097425746420145 + ], + [ + 631, + 0.007828954910206853 + ], + [ + 634, + 0.7712049790122333 + ], + [ + 639, + 0.9756689855037052 + ], + [ + 643, + 0.6565913655130314 + ], + [ + 649, + 0.3273860558570758 + ], + [ + 652, + 0.08695398500152751 + ], + [ + 656, + 0.11115207992931708 + ], + [ + 660, + 0.21888978587399288 + ] + ], + [ + [ + 50, + 0.4181276173985091 + ], + [ + 58, + 0.22358256875618876 + ], + [ + 61, + 0.8428357644094672 + ], + [ + 66, + 0.07895283485215976 + ], + [ + 75, + 0.832648782274282 + ], + [ + 78, + 0.8449929241209239 + ], + [ + 85, + 0.6584257332091348 + ], + [ + 88, + 0.17492752108361387 + ], + [ + 100, + 0.09675999690622195 + ], + [ + 105, + 0.7716554892649223 + ], + [ + 110, + 0.7115584342294312 + ], + [ + 115, + 0.11527010768059043 + ], + [ + 125, + 0.4732062209830158 + ], + [ + 128, + 0.15341365678925434 + ], + [ + 133, + 0.14633899700124475 + ], + [ + 142, + 0.5347904100729775 + ], + [ + 149, + 0.7684896594208711 + ], + [ + 159, + 0.079850026024186 + ], + [ + 163, + 0.38816043411066536 + ], + [ + 168, + 0.6818105453267002 + ], + [ + 173, + 0.8252105470104025 + ], + [ + 178, + 0.4908887946599344 + ], + [ + 189, + 0.5065778347092146 + ], + [ + 199, + 0.6348193683334751 + ], + [ + 206, + 0.21549879074603184 + ], + [ + 217, + 0.7687890049662857 + ], + [ + 221, + 0.5725686535030459 + ], + [ + 225, + 0.3836186927038474 + ], + [ + 232, + 0.6006526539962808 + ], + [ + 240, + 0.045687993584656694 + ], + [ + 249, + 0.4398687287162385 + ], + [ + 255, + 0.7340205160386013 + ], + [ + 259, + 0.3331977478453423 + ], + [ + 266, + 0.8646282116020737 + ], + [ + 270, + 0.4868806585463368 + ], + [ + 274, + 0.5922812386454627 + ], + [ + 279, + 0.021955731053276528 + ], + [ + 285, + 0.5937530491539885 + ], + [ + 292, + 0.2073257803631705 + ], + [ + 295, + 0.11047876430716452 + ], + [ + 297, + 0.6780533483992277 + ], + [ + 304, + 0.5338115459008669 + ], + [ + 316, + 0.6673120036770543 + ], + [ + 321, + 0.6753018690987888 + ], + [ + 324, + 0.40200274252480783 + ], + [ + 330, + 0.8498680485854615 + ], + [ + 333, + 0.7631680250486181 + ], + [ + 339, + 0.21364832955951873 + ], + [ + 343, + 0.7119872680988123 + ], + [ + 351, + 0.9161671862600897 + ], + [ + 356, + 0.9298029467253388 + ], + [ + 361, + 0.010687990855487883 + ], + [ + 369, + 0.9283025539274666 + ], + [ + 372, + 0.4340117187969056 + ], + [ + 377, + 0.8900909042273508 + ], + [ + 384, + 0.2138711682285921 + ], + [ + 389, + 0.2579528792821434 + ], + [ + 396, + 0.17704807663823663 + ], + [ + 399, + 0.39858301459241097 + ], + [ + 405, + 0.3025286136075479 + ], + [ + 412, + 0.3443583794015198 + ], + [ + 415, + 0.17733489469957564 + ], + [ + 420, + 0.8249570400727799 + ], + [ + 424, + 0.39554238549450427 + ], + [ + 435, + 0.018804878657817103 + ], + [ + 443, + 0.1666952078023889 + ], + [ + 446, + 0.5287648550754626 + ], + [ + 452, + 0.2517074888144806 + ], + [ + 457, + 0.18380530314453425 + ], + [ + 461, + 0.8352231114666506 + ], + [ + 464, + 0.07630747586006958 + ], + [ + 468, + 0.3774003546970661 + ], + [ + 471, + 0.8875923306116066 + ], + [ + 485, + 0.15362065249011259 + ], + [ + 489, + 0.016613189077949952 + ], + [ + 496, + 0.2957986416693388 + ], + [ + 504, + 0.8646000106947709 + ], + [ + 517, + 0.07017434826811253 + ], + [ + 521, + 0.5167609162738179 + ], + [ + 528, + 0.7761046244708356 + ], + [ + 532, + 0.9736986047136892 + ], + [ + 535, + 0.24459705279663302 + ], + [ + 541, + 0.9699143521402658 + ], + [ + 546, + 0.08631147057753963 + ], + [ + 550, + 0.9329536461710488 + ], + [ + 558, + 0.4029094662189612 + ], + [ + 563, + 0.48950213814290533 + ], + [ + 566, + 0.5720925927347997 + ], + [ + 571, + 0.06653739284208882 + ], + [ + 576, + 0.9369613636588959 + ], + [ + 582, + 0.6335837590905603 + ], + [ + 588, + 0.4927572074779326 + ], + [ + 595, + 0.743473566610875 + ], + [ + 599, + 0.08764534745472508 + ], + [ + 602, + 0.33164384004392455 + ], + [ + 611, + 0.7018519136829592 + ], + [ + 614, + 0.3332136079757716 + ], + [ + 618, + 0.9181048304139274 + ], + [ + 624, + 0.029015313742146498 + ], + [ + 629, + 0.5887503014423027 + ] + ], + [ + [ + 53, + 0.7098624436551949 + ], + [ + 63, + 0.27350597663329 + ], + [ + 70, + 0.3778543171535603 + ], + [ + 73, + 0.6788789518251402 + ], + [ + 77, + 0.10726250942979199 + ], + [ + 87, + 0.8394934840325272 + ], + [ + 90, + 0.13144659671593306 + ], + [ + 96, + 0.14916667941771888 + ], + [ + 110, + 0.6081108713582365 + ], + [ + 121, + 0.7754722094408412 + ], + [ + 124, + 0.048850174508976885 + ], + [ + 133, + 0.9249341678422579 + ], + [ + 141, + 0.0806650159535468 + ], + [ + 145, + 0.8178204511915984 + ], + [ + 157, + 0.23806659840467403 + ], + [ + 163, + 0.9636346881028524 + ], + [ + 170, + 0.7348125410146231 + ], + [ + 178, + 0.695970320773431 + ], + [ + 187, + 0.2683538153629337 + ], + [ + 194, + 0.027531170222428236 + ], + [ + 197, + 0.9897871581832495 + ], + [ + 201, + 0.9349219621561988 + ], + [ + 211, + 0.20826974781683527 + ], + [ + 214, + 0.40830153079211073 + ], + [ + 220, + 0.9522543601443701 + ], + [ + 234, + 0.48722825670149117 + ], + [ + 245, + 0.2189698220892068 + ], + [ + 248, + 0.7894018204012684 + ], + [ + 257, + 0.11658783085727709 + ], + [ + 265, + 0.7612179589301148 + ], + [ + 269, + 0.25687250457509336 + ], + [ + 281, + 0.7915486959101194 + ], + [ + 288, + 0.6620038717870498 + ], + [ + 293, + 0.17793247221924535 + ], + [ + 299, + 0.8874161227734442 + ], + [ + 303, + 0.1845146677702384 + ], + [ + 314, + 0.42922325616576695 + ], + [ + 317, + 0.7297133293653615 + ], + [ + 322, + 0.22172246028971176 + ], + [ + 325, + 0.405231052472529 + ], + [ + 329, + 0.6403644213120393 + ], + [ + 337, + 0.8296638150344615 + ], + [ + 343, + 0.9234366987033985 + ], + [ + 352, + 0.24389357853446325 + ], + [ + 355, + 0.8054120495364607 + ], + [ + 357, + 0.2261120490782631 + ], + [ + 370, + 0.2090667153511956 + ], + [ + 382, + 0.6598685218022208 + ], + [ + 391, + 0.7674729035384641 + ], + [ + 395, + 0.9908798421289623 + ], + [ + 402, + 0.24050918953720957 + ], + [ + 412, + 0.5033565750721927 + ], + [ + 423, + 0.10195103581721132 + ], + [ + 430, + 0.48872813200214615 + ], + [ + 441, + 0.549337304909546 + ], + [ + 446, + 0.3665262549052506 + ], + [ + 450, + 0.5454383881404011 + ], + [ + 456, + 0.30445706687802043 + ], + [ + 459, + 0.2857412394338058 + ], + [ + 468, + 0.8965825738674744 + ], + [ + 474, + 0.5961864023812716 + ], + [ + 484, + 0.3506319466997945 + ], + [ + 488, + 0.9566836717788472 + ], + [ + 494, + 0.614234466994785 + ], + [ + 500, + 0.42844585050155337 + ], + [ + 506, + 0.14717073025662597 + ], + [ + 512, + 0.3449488647931255 + ], + [ + 518, + 0.3568748346660552 + ], + [ + 524, + 0.9628157595798801 + ], + [ + 528, + 0.8971519829989567 + ], + [ + 536, + 0.21282978468896085 + ], + [ + 543, + 0.5048568894709257 + ], + [ + 547, + 0.809505242754497 + ], + [ + 551, + 0.404914214074809 + ], + [ + 557, + 0.9863217837460115 + ], + [ + 566, + 0.009213726267557787 + ], + [ + 570, + 0.7146586576269167 + ], + [ + 575, + 0.3220464582074578 + ], + [ + 581, + 0.7102890944830219 + ], + [ + 585, + 0.6449417276264502 + ], + [ + 590, + 0.5784561547249889 + ], + [ + 597, + 0.6175460902863847 + ], + [ + 604, + 0.6964329578800111 + ], + [ + 607, + 0.6507577711812726 + ], + [ + 613, + 0.18064408949669108 + ], + [ + 617, + 0.4927123476219185 + ], + [ + 625, + 0.07186669634357168 + ], + [ + 630, + 0.940838267869826 + ], + [ + 633, + 0.15705888252056666 + ], + [ + 637, + 0.635591687954609 + ], + [ + 643, + 0.6464760716695656 + ], + [ + 647, + 0.10630347815627617 + ], + [ + 660, + 0.5190274367083727 + ], + [ + 672, + 0.6925411729540314 + ], + [ + 677, + 0.06048738781455276 + ], + [ + 682, + 0.5395952027074449 + ], + [ + 691, + 0.550134321974646 + ], + [ + 695, + 0.841719121163114 + ], + [ + 698, + 0.7938554667851336 + ], + [ + 702, + 0.9392368087223598 + ] + ] + ], + [ + [ + [ + 34, + 0.7890569315564715 + ], + [ + 44, + 0.34833360797833524 + ], + [ + 48, + 0.23310670203749118 + ], + [ + 59, + 0.10275006708192691 + ], + [ + 64, + 0.97349460252762 + ], + [ + 73, + 0.11341489632136548 + ], + [ + 76, + 0.5005021251608037 + ], + [ + 84, + 0.10080262327645861 + ], + [ + 88, + 0.7057930822155167 + ], + [ + 94, + 0.6671977752330589 + ], + [ + 100, + 0.27742499712242763 + ], + [ + 104, + 0.05879260321763102 + ], + [ + 109, + 0.6867775661796186 + ], + [ + 116, + 0.8961247400426459 + ], + [ + 118, + 0.7451806366818057 + ], + [ + 126, + 0.5649536420386371 + ], + [ + 131, + 0.5478797376490352 + ], + [ + 135, + 0.5520351101773397 + ], + [ + 149, + 0.359549602214737 + ], + [ + 155, + 0.7706253301123137 + ], + [ + 158, + 0.17622559182261788 + ], + [ + 164, + 0.1723096796991308 + ], + [ + 167, + 0.9194426318991031 + ], + [ + 174, + 0.29398725467038445 + ], + [ + 177, + 0.8543548849533317 + ], + [ + 183, + 0.17475980276352232 + ], + [ + 186, + 0.31541291279916905 + ], + [ + 195, + 0.08876444151372109 + ], + [ + 199, + 0.9299259304403733 + ], + [ + 213, + 0.1443561245915812 + ], + [ + 222, + 0.7763669289734871 + ], + [ + 233, + 0.8092179054280617 + ], + [ + 239, + 0.4243292377445288 + ], + [ + 246, + 0.5753206683531717 + ], + [ + 249, + 0.32035625650257094 + ], + [ + 253, + 0.12439978464843215 + ], + [ + 258, + 0.4234276559626138 + ], + [ + 262, + 0.6973360806652288 + ], + [ + 266, + 0.17190614283098815 + ], + [ + 275, + 0.7470478621483679 + ], + [ + 279, + 0.8906130893055474 + ], + [ + 284, + 0.04323681805940083 + ], + [ + 291, + 0.27735355451116883 + ], + [ + 299, + 0.5517245544053182 + ], + [ + 303, + 0.9299521135290709 + ], + [ + 308, + 0.6967870645268078 + ], + [ + 322, + 0.16348400211186676 + ], + [ + 328, + 0.146858094033943 + ], + [ + 332, + 0.7683391053402875 + ], + [ + 337, + 0.49708211060653185 + ], + [ + 340, + 0.13032249800068885 + ], + [ + 352, + 0.7034000833108022 + ], + [ + 355, + 0.3348032001824297 + ], + [ + 362, + 0.7335572085931431 + ], + [ + 368, + 0.8297282542996905 + ], + [ + 376, + 0.9091750363509475 + ], + [ + 380, + 0.9480042082107611 + ], + [ + 388, + 0.5157719166272101 + ], + [ + 392, + 0.4498340428048001 + ], + [ + 400, + 0.32065773904952244 + ], + [ + 404, + 0.7356036622976159 + ], + [ + 410, + 0.6743563799767319 + ], + [ + 414, + 0.6007528393800347 + ], + [ + 420, + 0.3314474275334316 + ], + [ + 425, + 0.7761390531361201 + ], + [ + 428, + 0.7443898732023506 + ], + [ + 432, + 0.07624013605948221 + ], + [ + 437, + 0.311582912457708 + ], + [ + 440, + 0.42557166487562614 + ], + [ + 450, + 0.9877386816445454 + ], + [ + 457, + 0.5272789637644623 + ], + [ + 463, + 0.811916139737507 + ], + [ + 469, + 0.6214926563240035 + ], + [ + 473, + 0.38155171117913533 + ], + [ + 483, + 0.016273613123231057 + ], + [ + 486, + 0.47113637116926343 + ], + [ + 494, + 0.9897673645970705 + ], + [ + 500, + 0.2487503589994795 + ], + [ + 503, + 0.5540829249848721 + ], + [ + 507, + 0.08958239496129661 + ], + [ + 515, + 0.19572452991574718 + ], + [ + 523, + 0.9795019261831417 + ], + [ + 527, + 0.14527644667934325 + ], + [ + 532, + 0.3499402780039973 + ], + [ + 539, + 0.021265322037981704 + ], + [ + 542, + 0.9099254878829618 + ], + [ + 547, + 0.11797028477615568 + ], + [ + 550, + 0.6968430370568808 + ], + [ + 554, + 0.1580095061675706 + ], + [ + 568, + 0.7693688223587518 + ], + [ + 579, + 0.09026817384677954 + ], + [ + 582, + 0.7929368137849006 + ], + [ + 591, + 0.3378386266204926 + ], + [ + 599, + 0.21694987999357185 + ], + [ + 601, + 0.6018431410757022 + ], + [ + 612, + 0.6672460618027466 + ], + [ + 615, + 0.5971250824814032 + ], + [ + 625, + 0.10007799042948928 + ], + [ + 634, + 0.3404581184176648 + ], + [ + 637, + 0.34453347465320106 + ] + ], + [ + [ + 38, + 0.4474508735131496 + ], + [ + 46, + 0.6173468090368438 + ], + [ + 55, + 0.43569101970742796 + ], + [ + 64, + 0.2660048812850917 + ], + [ + 73, + 0.9105305294237704 + ], + [ + 76, + 0.20133702563552835 + ], + [ + 91, + 0.8431824808304823 + ], + [ + 101, + 0.39242147668620475 + ], + [ + 105, + 0.4523651702455115 + ], + [ + 112, + 0.5025991007172359 + ], + [ + 114, + 0.9869319972850592 + ], + [ + 119, + 0.6505647072855929 + ], + [ + 125, + 0.27903360528920096 + ], + [ + 132, + 0.08731932969559009 + ], + [ + 135, + 0.3884063737134462 + ], + [ + 138, + 0.026617118148956953 + ], + [ + 144, + 0.8998392229481966 + ], + [ + 147, + 0.2141054275903428 + ], + [ + 150, + 0.9539375465280628 + ], + [ + 166, + 0.650296476547029 + ], + [ + 169, + 0.9467686652257072 + ], + [ + 172, + 0.8860464587053568 + ], + [ + 176, + 0.7204309611438177 + ], + [ + 179, + 0.5220835420329334 + ], + [ + 183, + 0.8668190208141641 + ], + [ + 187, + 0.7461309322619611 + ], + [ + 195, + 0.2408011610688514 + ], + [ + 198, + 0.19083236798586178 + ], + [ + 202, + 0.34742758197134427 + ], + [ + 207, + 0.6383805012025395 + ], + [ + 216, + 0.10391665776135739 + ], + [ + 221, + 0.8347054339269112 + ], + [ + 225, + 0.43627018017125474 + ], + [ + 231, + 0.8223928300403988 + ], + [ + 234, + 0.1569636235773324 + ], + [ + 238, + 0.7553043446621545 + ], + [ + 247, + 0.9991102860142057 + ], + [ + 251, + 0.8370294246230476 + ], + [ + 254, + 0.5633964291714767 + ], + [ + 264, + 0.1711744688368977 + ], + [ + 267, + 0.1197465030716971 + ], + [ + 276, + 0.7214710914747487 + ], + [ + 280, + 0.8730046136761482 + ], + [ + 287, + 0.004680736782659345 + ], + [ + 290, + 0.6514818895313449 + ], + [ + 296, + 0.7265939042782648 + ], + [ + 308, + 0.35072817332768425 + ], + [ + 315, + 0.509532497256609 + ], + [ + 319, + 0.23702464477799312 + ], + [ + 324, + 0.26775784493550436 + ], + [ + 329, + 0.5862771208752532 + ], + [ + 335, + 0.9226897967936665 + ], + [ + 340, + 0.9924423589735745 + ], + [ + 343, + 0.9705702072722224 + ], + [ + 347, + 0.17385858335942872 + ], + [ + 356, + 0.7375280113008328 + ], + [ + 363, + 0.5370579250712801 + ], + [ + 370, + 0.5096066515774006 + ], + [ + 374, + 0.3026193963951479 + ], + [ + 378, + 0.744546771426147 + ], + [ + 383, + 0.17813370150353847 + ], + [ + 385, + 0.40484831905046326 + ], + [ + 392, + 0.7863284993585248 + ], + [ + 397, + 0.5357557855679704 + ], + [ + 405, + 0.3656328425624815 + ], + [ + 412, + 0.5791704518952997 + ], + [ + 416, + 0.1302520449085095 + ], + [ + 420, + 0.45361529060189 + ], + [ + 425, + 0.13919444569045403 + ], + [ + 434, + 0.5866746485538024 + ], + [ + 438, + 0.8081423761008428 + ], + [ + 443, + 0.16992182969890557 + ], + [ + 445, + 0.5465276495143276 + ], + [ + 456, + 0.8998816614613986 + ], + [ + 461, + 0.10777714316124698 + ], + [ + 466, + 0.1681655477447167 + ], + [ + 469, + 0.34987661138864024 + ], + [ + 479, + 0.15535949595187803 + ], + [ + 489, + 0.3757914607234336 + ], + [ + 492, + 0.2894894181396652 + ], + [ + 496, + 0.22934716715178394 + ], + [ + 504, + 0.2245356214959584 + ], + [ + 507, + 0.33532189175623917 + ], + [ + 511, + 0.7363271827278677 + ], + [ + 514, + 0.6972419906319351 + ], + [ + 525, + 0.21701061469009597 + ], + [ + 530, + 0.22648002551906332 + ], + [ + 536, + 0.9655385419126402 + ], + [ + 539, + 0.6011661386585093 + ], + [ + 543, + 0.8753815007412826 + ], + [ + 552, + 0.06014784364721837 + ], + [ + 557, + 0.14280920818668497 + ], + [ + 565, + 0.41597597793906715 + ], + [ + 568, + 0.9082852794108708 + ], + [ + 572, + 0.3661377260303065 + ], + [ + 585, + 0.20771362920034397 + ], + [ + 588, + 0.7614214032350628 + ], + [ + 592, + 0.605107440558353 + ], + [ + 598, + 0.16347158872749246 + ], + [ + 601, + 0.1704727549155275 + ] + ], + [ + [ + 38, + 0.44223362740057326 + ], + [ + 46, + 0.9280547524038262 + ], + [ + 55, + 0.9034089510910916 + ], + [ + 64, + 0.3095912719501891 + ], + [ + 73, + 0.24585514816742682 + ], + [ + 76, + 0.5397430397763092 + ], + [ + 91, + 0.655343282168449 + ], + [ + 101, + 0.08860800579137951 + ], + [ + 105, + 0.654978832506852 + ], + [ + 112, + 0.8400418661223789 + ], + [ + 114, + 0.8957113239762413 + ], + [ + 119, + 0.8427544709335894 + ], + [ + 125, + 0.28985461809442115 + ], + [ + 132, + 0.7558537845057792 + ], + [ + 135, + 0.8684569811462889 + ], + [ + 138, + 0.9484454664108761 + ], + [ + 144, + 0.5379735850620495 + ], + [ + 147, + 0.5014763132418425 + ], + [ + 150, + 0.5678434404591313 + ], + [ + 166, + 0.6520788962194506 + ], + [ + 169, + 0.23184137819664719 + ], + [ + 172, + 0.975243465645978 + ], + [ + 176, + 0.4765016425093468 + ], + [ + 179, + 0.685695923774693 + ], + [ + 183, + 0.04411807915427313 + ], + [ + 187, + 0.17154494744554327 + ], + [ + 195, + 0.9870644223493374 + ], + [ + 198, + 0.48672308554197696 + ], + [ + 202, + 0.11182799710902858 + ], + [ + 207, + 0.849294217698516 + ], + [ + 216, + 0.26463403807142627 + ], + [ + 221, + 0.6572949966504793 + ], + [ + 225, + 0.8705604820148202 + ], + [ + 231, + 0.9998271654633966 + ], + [ + 234, + 0.055561167557820035 + ], + [ + 238, + 0.5265122837185042 + ], + [ + 247, + 0.8795068353399654 + ], + [ + 251, + 0.8286267641217387 + ], + [ + 254, + 0.16680918130025335 + ], + [ + 264, + 0.029303041082990866 + ], + [ + 267, + 0.01631659330665891 + ], + [ + 276, + 0.7342031455937434 + ], + [ + 280, + 0.916753860836387 + ], + [ + 287, + 0.14993438226629008 + ], + [ + 290, + 0.2324555461946166 + ], + [ + 296, + 0.14412313518608755 + ], + [ + 308, + 0.4148133054311651 + ], + [ + 315, + 0.38313557809137877 + ], + [ + 319, + 0.5912985712885397 + ], + [ + 324, + 0.3084532351825161 + ], + [ + 329, + 0.2626734465106423 + ], + [ + 335, + 0.6496635758964893 + ], + [ + 340, + 0.5735613446979964 + ], + [ + 343, + 0.6145812190368586 + ], + [ + 347, + 0.8280822248061036 + ], + [ + 356, + 0.0802347596520121 + ], + [ + 363, + 0.5489584675860743 + ], + [ + 370, + 0.1496634456274345 + ], + [ + 374, + 0.790379971748317 + ], + [ + 378, + 0.8563851669925344 + ], + [ + 383, + 0.07260133819233983 + ], + [ + 385, + 0.359146273632164 + ], + [ + 392, + 0.9875119736672606 + ], + [ + 397, + 0.0010048374462110488 + ], + [ + 405, + 0.8768785631742823 + ], + [ + 412, + 0.6390138897009717 + ], + [ + 416, + 0.6702662976304804 + ], + [ + 420, + 0.37123299858531567 + ], + [ + 425, + 0.5985473170947941 + ], + [ + 434, + 0.7274824270781993 + ], + [ + 438, + 0.17930792251885608 + ], + [ + 443, + 0.11585138168612297 + ], + [ + 445, + 0.3542090975184625 + ], + [ + 456, + 0.6169097550976957 + ], + [ + 461, + 0.40886719041661623 + ], + [ + 466, + 0.17305083058611526 + ], + [ + 469, + 0.7275878633454099 + ], + [ + 479, + 0.37219549597331436 + ], + [ + 489, + 0.033353248620686005 + ], + [ + 492, + 0.9121686998529818 + ], + [ + 496, + 0.14788561593654248 + ], + [ + 504, + 0.7956804487463186 + ], + [ + 507, + 0.22629374156939908 + ], + [ + 511, + 0.491929807631321 + ], + [ + 514, + 0.7633822197033119 + ], + [ + 525, + 0.8634766748901725 + ], + [ + 530, + 0.018129181905868874 + ], + [ + 536, + 0.7635377465854861 + ], + [ + 539, + 0.026559657817842974 + ], + [ + 543, + 0.8052449135417911 + ], + [ + 552, + 0.9629753671776455 + ], + [ + 557, + 0.17177489540973312 + ], + [ + 565, + 0.5860837122340373 + ], + [ + 568, + 0.9099135384449513 + ], + [ + 572, + 0.9792669981821831 + ], + [ + 585, + 0.8429095476588548 + ], + [ + 588, + 0.5597289537548925 + ], + [ + 592, + 0.5898246302489015 + ], + [ + 598, + 0.3978855787411818 + ], + [ + 601, + 0.8666192781340598 + ] + ], + [ + [ + 37, + 0.48732145806003135 + ], + [ + 43, + 0.7315389182587093 + ], + [ + 54, + 0.7346138137733782 + ], + [ + 60, + 0.791522838015047 + ], + [ + 71, + 0.18195736644326188 + ], + [ + 77, + 0.35697200873315327 + ], + [ + 80, + 0.9233267381378623 + ], + [ + 89, + 0.5194318225323206 + ], + [ + 95, + 0.22660113007996685 + ], + [ + 98, + 0.657942742848426 + ], + [ + 106, + 0.5520156569255555 + ], + [ + 113, + 0.7283308804898909 + ], + [ + 127, + 0.855074350464641 + ], + [ + 138, + 0.024644201734533078 + ], + [ + 142, + 0.49746366194427527 + ], + [ + 150, + 0.6559447356188574 + ], + [ + 159, + 0.11427698263984032 + ], + [ + 166, + 0.1736951553985978 + ], + [ + 177, + 0.22407257720246776 + ], + [ + 184, + 0.03509654016049246 + ], + [ + 187, + 0.794032855225741 + ], + [ + 189, + 0.8890970130262511 + ], + [ + 197, + 0.19735847534869033 + ], + [ + 201, + 0.12462554183011332 + ], + [ + 204, + 0.7848573846433101 + ], + [ + 212, + 0.23957492131038105 + ], + [ + 216, + 0.9274059868185438 + ], + [ + 220, + 0.5978834990800633 + ], + [ + 225, + 0.6398212705297761 + ], + [ + 233, + 0.4687133261214791 + ], + [ + 241, + 0.9608768853171389 + ], + [ + 248, + 0.8114157081430573 + ], + [ + 252, + 0.3590144858714962 + ], + [ + 263, + 0.6817101427091221 + ], + [ + 270, + 0.7095178203646221 + ], + [ + 273, + 0.1673861870523642 + ], + [ + 277, + 0.11653279908183722 + ], + [ + 282, + 0.7298884293063613 + ], + [ + 287, + 0.07169441501234297 + ], + [ + 293, + 0.9629308785122198 + ], + [ + 298, + 0.2936764705129362 + ], + [ + 302, + 0.08099748615118574 + ], + [ + 307, + 0.13470815750882104 + ], + [ + 315, + 0.1861520657337956 + ], + [ + 318, + 0.09642572354107981 + ], + [ + 327, + 0.0868199658530866 + ], + [ + 334, + 0.7496740459061344 + ], + [ + 345, + 0.36588958478232625 + ], + [ + 352, + 0.5833815986780084 + ], + [ + 357, + 0.5164128122826452 + ], + [ + 361, + 0.8681044387306317 + ], + [ + 369, + 0.5556029430440954 + ], + [ + 380, + 0.06985286216672626 + ], + [ + 384, + 0.2601032901187196 + ], + [ + 387, + 0.8064223748398224 + ], + [ + 392, + 0.523700877153728 + ], + [ + 399, + 0.21393520864874005 + ], + [ + 409, + 0.8933881149332326 + ], + [ + 415, + 0.7599273305849825 + ], + [ + 426, + 0.9183367161617594 + ], + [ + 432, + 0.0945655190082928 + ], + [ + 435, + 0.02184027158112578 + ], + [ + 439, + 0.265318312570446 + ], + [ + 449, + 0.8485020415964083 + ], + [ + 460, + 0.4426380762786035 + ], + [ + 466, + 0.5299663098990308 + ], + [ + 470, + 0.25011117822767226 + ], + [ + 477, + 0.541226019879165 + ], + [ + 481, + 0.9960845696119888 + ], + [ + 486, + 0.5733265068360727 + ], + [ + 492, + 0.393271503311305 + ], + [ + 495, + 0.1229780997077572 + ], + [ + 504, + 0.4108992856472402 + ], + [ + 517, + 0.8491335702308893 + ], + [ + 521, + 0.20381313802331047 + ], + [ + 530, + 0.2158719856177248 + ], + [ + 535, + 0.2734639885830148 + ], + [ + 539, + 0.49026614281676295 + ], + [ + 550, + 0.15607270191293132 + ], + [ + 554, + 0.1486443303981022 + ], + [ + 559, + 0.2100724327675879 + ], + [ + 562, + 0.3090396245764352 + ], + [ + 571, + 0.21905180265059887 + ], + [ + 575, + 0.8517598840808955 + ], + [ + 581, + 0.7026969072298469 + ], + [ + 585, + 0.05112447583666746 + ], + [ + 590, + 0.5786512058922415 + ], + [ + 597, + 0.18198441080102112 + ], + [ + 602, + 0.07706488776076181 + ], + [ + 608, + 0.6787417286256527 + ], + [ + 613, + 0.16325673110718686 + ], + [ + 619, + 0.9972789429762873 + ], + [ + 631, + 0.4537956751197344 + ], + [ + 634, + 0.5125121509447853 + ], + [ + 639, + 0.46469297697024536 + ], + [ + 643, + 0.41655212576143974 + ], + [ + 649, + 0.0037973950714607074 + ], + [ + 652, + 0.6633751282998047 + ], + [ + 656, + 0.10757093043061772 + ], + [ + 660, + 0.338893725753324 + ] + ], + [ + [ + 50, + 0.46329562499023536 + ], + [ + 58, + 0.16645052615652 + ], + [ + 61, + 0.23279229373333654 + ], + [ + 66, + 0.8874775658699657 + ], + [ + 75, + 0.35852094977349924 + ], + [ + 78, + 0.7123989022131302 + ], + [ + 85, + 0.03191142584116824 + ], + [ + 88, + 0.11294809300439423 + ], + [ + 100, + 0.8674428435670353 + ], + [ + 105, + 0.2618953090979549 + ], + [ + 110, + 0.9654296486707095 + ], + [ + 115, + 0.740593506110645 + ], + [ + 125, + 0.8671278333134325 + ], + [ + 128, + 0.9371048574358949 + ], + [ + 133, + 0.5822008087842866 + ], + [ + 142, + 0.5426908406935381 + ], + [ + 149, + 0.4579294006700333 + ], + [ + 159, + 0.9872539446091603 + ], + [ + 163, + 0.6522262964951245 + ], + [ + 168, + 0.701208762229819 + ], + [ + 173, + 0.539651122434288 + ], + [ + 178, + 0.5205971563303006 + ], + [ + 189, + 0.7910740688808839 + ], + [ + 199, + 0.880707545901061 + ], + [ + 206, + 0.42411548104058716 + ], + [ + 217, + 0.7288803035308135 + ], + [ + 221, + 0.935122773844572 + ], + [ + 225, + 0.18607254650758875 + ], + [ + 232, + 0.14851879159265857 + ], + [ + 240, + 0.6439653702786083 + ], + [ + 249, + 0.25214938115435515 + ], + [ + 255, + 0.5787773104041685 + ], + [ + 259, + 0.21170691366237548 + ], + [ + 266, + 0.2281767295730729 + ], + [ + 270, + 0.7282859114876636 + ], + [ + 274, + 0.6927018853358593 + ], + [ + 279, + 0.3149043477308541 + ], + [ + 285, + 0.37786853627529815 + ], + [ + 292, + 0.9621863241354417 + ], + [ + 295, + 0.22654863189104157 + ], + [ + 297, + 0.1950885421191273 + ], + [ + 304, + 0.7035432388336782 + ], + [ + 316, + 0.9868707975181374 + ], + [ + 321, + 0.44489616163056334 + ], + [ + 324, + 0.9962101293432382 + ], + [ + 330, + 0.4057357576448518 + ], + [ + 333, + 0.3816366350967332 + ], + [ + 339, + 0.09632930174882737 + ], + [ + 343, + 0.21041083337242017 + ], + [ + 351, + 0.44440712708069463 + ], + [ + 356, + 0.17164252340076802 + ], + [ + 361, + 0.27224777073991924 + ], + [ + 369, + 0.2746708244330234 + ], + [ + 372, + 0.5042090949750971 + ], + [ + 377, + 0.26893772188424614 + ], + [ + 384, + 0.13703162466306829 + ], + [ + 389, + 0.6709946993478709 + ], + [ + 396, + 0.7425031025858074 + ], + [ + 399, + 0.07367990763711374 + ], + [ + 405, + 0.5737329087841551 + ], + [ + 412, + 0.42283768445053327 + ], + [ + 415, + 0.14538208626702298 + ], + [ + 420, + 0.7385864330759113 + ], + [ + 424, + 0.5352434565723423 + ], + [ + 435, + 0.9562950262489279 + ], + [ + 443, + 0.37689879630749135 + ], + [ + 446, + 0.9800283875284765 + ], + [ + 452, + 0.27194862024284394 + ], + [ + 457, + 0.9238213995157135 + ], + [ + 461, + 0.7697259871148568 + ], + [ + 464, + 0.8196762336433535 + ], + [ + 468, + 0.08948938931763672 + ], + [ + 471, + 0.8910718142313844 + ], + [ + 485, + 0.4414001787442354 + ], + [ + 489, + 0.04360567041065011 + ], + [ + 496, + 0.9270145494284555 + ], + [ + 504, + 0.042545759396008465 + ], + [ + 517, + 0.9521675973975832 + ], + [ + 521, + 0.008791207278368351 + ], + [ + 528, + 0.2685486830216608 + ], + [ + 532, + 0.9200116043998585 + ], + [ + 535, + 0.17764129517234362 + ], + [ + 541, + 0.20861002583786192 + ], + [ + 546, + 0.741317459447591 + ], + [ + 550, + 0.3459646744396313 + ], + [ + 558, + 0.8155329648321876 + ], + [ + 563, + 0.6424834691936478 + ], + [ + 566, + 0.37024342286240786 + ], + [ + 571, + 0.6222972219125422 + ], + [ + 576, + 0.5601816460629756 + ], + [ + 582, + 0.9843033745506632 + ], + [ + 588, + 0.33742712567610544 + ], + [ + 595, + 0.8322894387460966 + ], + [ + 599, + 0.05355159325555381 + ], + [ + 602, + 0.8699304334693828 + ], + [ + 611, + 0.6706494662230841 + ], + [ + 614, + 0.4756513274513827 + ], + [ + 618, + 0.31775158476108134 + ], + [ + 624, + 0.6611687751408006 + ], + [ + 629, + 0.8608485985316154 + ] + ], + [ + [ + 53, + 0.699102494724101 + ], + [ + 63, + 0.800372721585696 + ], + [ + 70, + 0.5323197752654771 + ], + [ + 73, + 0.2962854028122266 + ], + [ + 77, + 0.3228392073801596 + ], + [ + 87, + 0.5540922604007493 + ], + [ + 90, + 0.6351743082939401 + ], + [ + 96, + 0.8311140720562941 + ], + [ + 110, + 0.2556625214416398 + ], + [ + 121, + 0.4513850797741661 + ], + [ + 124, + 0.22907572689690436 + ], + [ + 133, + 0.5380537917758845 + ], + [ + 141, + 0.6408602129882965 + ], + [ + 145, + 0.9445972392341314 + ], + [ + 157, + 0.6199553571763167 + ], + [ + 163, + 0.554232805563841 + ], + [ + 170, + 0.9013474997291948 + ], + [ + 178, + 0.8918302072501937 + ], + [ + 187, + 0.8909589946359359 + ], + [ + 194, + 0.46741390005375927 + ], + [ + 197, + 0.23394093920211934 + ], + [ + 201, + 0.7554857786207828 + ], + [ + 211, + 0.6716484338360507 + ], + [ + 214, + 0.6982625810142954 + ], + [ + 220, + 0.07756829617786964 + ], + [ + 234, + 0.10543133801207416 + ], + [ + 245, + 0.8212706147019501 + ], + [ + 248, + 0.7819417976845624 + ], + [ + 257, + 0.4273771639622532 + ], + [ + 265, + 0.6528465272724773 + ], + [ + 269, + 0.9638939836964517 + ], + [ + 281, + 0.7450668218566974 + ], + [ + 288, + 0.05525705408212278 + ], + [ + 293, + 0.21389849085183343 + ], + [ + 299, + 0.4636204567619403 + ], + [ + 303, + 0.9966123272587805 + ], + [ + 314, + 0.22428482939894145 + ], + [ + 317, + 0.7921600916608913 + ], + [ + 322, + 0.3038124297513308 + ], + [ + 325, + 0.7563636611097031 + ], + [ + 329, + 0.11444958049492016 + ], + [ + 337, + 0.994068284057543 + ], + [ + 343, + 0.7317072006838083 + ], + [ + 352, + 0.36369310428714974 + ], + [ + 355, + 0.11292761408657626 + ], + [ + 357, + 0.5286592496178362 + ], + [ + 370, + 0.06028885848596577 + ], + [ + 382, + 0.8470438298307269 + ], + [ + 391, + 0.5776859639730917 + ], + [ + 395, + 0.5251835536712407 + ], + [ + 402, + 0.45990716537290444 + ], + [ + 412, + 0.7428870061519649 + ], + [ + 423, + 0.7510285213638478 + ], + [ + 430, + 0.4352733619724859 + ], + [ + 441, + 0.2147695920996574 + ], + [ + 446, + 0.501240001459065 + ], + [ + 450, + 0.667244416534853 + ], + [ + 456, + 0.297215479510085 + ], + [ + 459, + 0.8939244300696457 + ], + [ + 468, + 0.4683704890372028 + ], + [ + 474, + 0.2948296859509618 + ], + [ + 484, + 0.5558978008927696 + ], + [ + 488, + 0.3187845041977845 + ], + [ + 494, + 0.8358701346915884 + ], + [ + 500, + 0.7021760106841318 + ], + [ + 506, + 0.09997668241321533 + ], + [ + 512, + 0.5363629667239129 + ], + [ + 518, + 0.6095471050954253 + ], + [ + 524, + 0.06645269858275804 + ], + [ + 528, + 0.0004996794999616583 + ], + [ + 536, + 0.18583913985147682 + ], + [ + 543, + 0.2422029115331914 + ], + [ + 547, + 0.8878040824945956 + ], + [ + 551, + 0.0005814220018177663 + ], + [ + 557, + 0.06492637518782718 + ], + [ + 566, + 0.9356451130645658 + ], + [ + 570, + 0.27496408425900665 + ], + [ + 575, + 0.8149729817986036 + ], + [ + 581, + 0.7210780924167814 + ], + [ + 585, + 0.1510244300524528 + ], + [ + 590, + 0.20294576451397206 + ], + [ + 597, + 0.1553282518502751 + ], + [ + 604, + 0.4912031029658829 + ], + [ + 607, + 0.8394845881297451 + ], + [ + 613, + 0.7167414521575795 + ], + [ + 617, + 0.6258405704852249 + ], + [ + 625, + 0.1432368900797274 + ], + [ + 630, + 0.002903796951071125 + ], + [ + 633, + 0.20218923597005156 + ], + [ + 637, + 0.8332197633190558 + ], + [ + 643, + 0.8234395593505321 + ], + [ + 647, + 0.6349695464514649 + ], + [ + 660, + 0.6012635910462582 + ], + [ + 672, + 0.9362407256861722 + ], + [ + 677, + 0.22284153273302432 + ], + [ + 682, + 0.151976768826095 + ], + [ + 691, + 0.8250511870651384 + ], + [ + 695, + 0.5892820741377013 + ], + [ + 698, + 0.25155675749236817 + ], + [ + 702, + 0.742731328132861 + ] + ] + ], + [ + [ + [ + 34, + 0.9636600537618455 + ], + [ + 44, + 0.7437945944807257 + ], + [ + 48, + 0.8167755204855559 + ], + [ + 59, + 0.4921773678349116 + ], + [ + 64, + 0.8053567616220777 + ], + [ + 73, + 0.5700045151611166 + ], + [ + 76, + 0.7630111189110091 + ], + [ + 84, + 0.88365565859156 + ], + [ + 88, + 0.2233792197327109 + ], + [ + 94, + 0.7386905488629014 + ], + [ + 100, + 0.37523879679703187 + ], + [ + 104, + 0.1784340102949673 + ], + [ + 109, + 0.33578723681849654 + ], + [ + 116, + 0.545012726220743 + ], + [ + 118, + 0.3277119915329856 + ], + [ + 126, + 0.35126075314636085 + ], + [ + 131, + 0.24592953074169677 + ], + [ + 135, + 0.006141596161829521 + ], + [ + 149, + 0.078418659674236 + ], + [ + 155, + 0.05163370682889579 + ], + [ + 158, + 0.9078188180121576 + ], + [ + 164, + 0.40364899609502813 + ], + [ + 167, + 0.875723422854441 + ], + [ + 174, + 0.1902459583262993 + ], + [ + 177, + 0.47800210901623597 + ], + [ + 183, + 0.78738848908616 + ], + [ + 186, + 0.6462206526354649 + ], + [ + 195, + 0.5599955063440741 + ], + [ + 199, + 0.4119511106798204 + ], + [ + 213, + 0.06903002557771654 + ], + [ + 222, + 0.7847289018575392 + ], + [ + 233, + 0.3075813157578189 + ], + [ + 239, + 0.7629280971179497 + ], + [ + 246, + 0.540157297334651 + ], + [ + 249, + 0.530198079020022 + ], + [ + 253, + 0.35168666422334294 + ], + [ + 258, + 0.2208951506629111 + ], + [ + 262, + 0.3314450632464927 + ], + [ + 266, + 0.9619402242871128 + ], + [ + 275, + 0.24937888882457504 + ], + [ + 279, + 0.46324533269749 + ], + [ + 284, + 0.9040467515838522 + ], + [ + 291, + 0.5615296680496175 + ], + [ + 299, + 0.8539580258622507 + ], + [ + 303, + 0.07360629092607218 + ], + [ + 308, + 0.9217870208950464 + ], + [ + 322, + 0.45025230717064535 + ], + [ + 328, + 0.7413576004604062 + ], + [ + 332, + 0.3676868225662393 + ], + [ + 337, + 0.9568505036804705 + ], + [ + 340, + 0.18032789101920532 + ], + [ + 352, + 0.5209139443916063 + ], + [ + 355, + 0.332870464442223 + ], + [ + 362, + 0.509244063757699 + ], + [ + 368, + 0.825667793299011 + ], + [ + 376, + 0.9055345060630526 + ], + [ + 380, + 0.4413353736406316 + ], + [ + 388, + 0.7144565042653581 + ], + [ + 392, + 0.2677189870911154 + ], + [ + 400, + 0.16986184341191557 + ], + [ + 404, + 0.0162948263290561 + ], + [ + 410, + 0.11925579193360836 + ], + [ + 414, + 0.006545359502425185 + ], + [ + 420, + 0.24793733793349504 + ], + [ + 425, + 0.035782021601621694 + ], + [ + 428, + 0.8058232222224312 + ], + [ + 432, + 0.11160633573976575 + ], + [ + 437, + 0.8785918499614999 + ], + [ + 440, + 0.47396294743805045 + ], + [ + 450, + 0.17272158928731962 + ], + [ + 457, + 0.48890922599812725 + ], + [ + 463, + 0.40641396640679905 + ], + [ + 469, + 0.7317979087049525 + ], + [ + 473, + 0.022496394301480538 + ], + [ + 483, + 0.6959406470731367 + ], + [ + 486, + 0.563754028973515 + ], + [ + 494, + 0.6773912152907232 + ], + [ + 500, + 0.013363086227264676 + ], + [ + 503, + 0.35886327121950623 + ], + [ + 507, + 0.045743575352879096 + ], + [ + 515, + 0.7127070952580636 + ], + [ + 523, + 0.36466298249342854 + ], + [ + 527, + 0.3221549319243515 + ], + [ + 532, + 0.6765505496499103 + ], + [ + 539, + 0.5523428099144133 + ], + [ + 542, + 0.6531582027124223 + ], + [ + 547, + 0.7983102541719761 + ], + [ + 550, + 0.39558883178044 + ], + [ + 554, + 0.9861022579930279 + ], + [ + 568, + 0.9406136276935716 + ], + [ + 579, + 0.4244955950165664 + ], + [ + 582, + 0.6157272092699928 + ], + [ + 591, + 0.3937550269095694 + ], + [ + 599, + 0.722229502617793 + ], + [ + 601, + 0.881210133285057 + ], + [ + 612, + 0.7265169211888043 + ], + [ + 615, + 0.9694385367670116 + ], + [ + 625, + 0.9618134685335965 + ], + [ + 634, + 0.6584265047651547 + ], + [ + 637, + 0.41732014148128505 + ] + ], + [ + [ + 38, + 0.37337568183968184 + ], + [ + 46, + 0.8314870810772514 + ], + [ + 55, + 0.05254929349011084 + ], + [ + 64, + 0.8931412132216846 + ], + [ + 73, + 0.3438805848004296 + ], + [ + 76, + 0.325455535440039 + ], + [ + 91, + 0.6612366019397925 + ], + [ + 101, + 0.9646760686493935 + ], + [ + 105, + 0.4013030864463163 + ], + [ + 112, + 0.3810221830796735 + ], + [ + 114, + 0.1899734738239206 + ], + [ + 119, + 0.7543039345621384 + ], + [ + 125, + 0.6154861874091271 + ], + [ + 132, + 0.09433024895285769 + ], + [ + 135, + 0.4402954805293283 + ], + [ + 138, + 0.5547284697978927 + ], + [ + 144, + 0.7652857103752206 + ], + [ + 147, + 0.8045429048235416 + ], + [ + 150, + 0.9331447889123536 + ], + [ + 166, + 0.9083081770991418 + ], + [ + 169, + 0.5486083097635432 + ], + [ + 172, + 0.594021409043046 + ], + [ + 176, + 0.29803204428617036 + ], + [ + 179, + 0.6255290106071374 + ], + [ + 183, + 0.9062204720991638 + ], + [ + 187, + 0.25591591253921864 + ], + [ + 195, + 0.627893900291424 + ], + [ + 198, + 0.6906029109094404 + ], + [ + 202, + 0.35636376280206417 + ], + [ + 207, + 0.9757564796139993 + ], + [ + 216, + 0.35607004012983967 + ], + [ + 221, + 0.6040415643515366 + ], + [ + 225, + 0.7488232724437635 + ], + [ + 231, + 0.6368898494859185 + ], + [ + 234, + 0.5110845019182115 + ], + [ + 238, + 0.37116597774046267 + ], + [ + 247, + 0.22748217089908573 + ], + [ + 251, + 0.805998427608663 + ], + [ + 254, + 0.23173631596470667 + ], + [ + 264, + 0.19175689045028455 + ], + [ + 267, + 0.15298854565753706 + ], + [ + 276, + 0.5586044487053198 + ], + [ + 280, + 0.5133560616929543 + ], + [ + 287, + 0.23664609090776278 + ], + [ + 290, + 0.46826034256235316 + ], + [ + 296, + 0.8755526037106054 + ], + [ + 308, + 0.6751339530532486 + ], + [ + 315, + 0.7162263142696316 + ], + [ + 319, + 0.29652719055298427 + ], + [ + 324, + 0.44478614247345616 + ], + [ + 329, + 0.02767132616083723 + ], + [ + 335, + 0.9850910812093614 + ], + [ + 340, + 0.8732422463766257 + ], + [ + 343, + 0.5799334528947521 + ], + [ + 347, + 0.4617683080635544 + ], + [ + 356, + 0.45855096447403676 + ], + [ + 363, + 0.31454849503865956 + ], + [ + 370, + 0.23166025243343524 + ], + [ + 374, + 0.6875508786426817 + ], + [ + 378, + 0.31693498832889744 + ], + [ + 383, + 0.5768973780541075 + ], + [ + 385, + 0.723892367667149 + ], + [ + 392, + 0.42372564000241364 + ], + [ + 397, + 0.2930607999709818 + ], + [ + 405, + 0.8419877790687116 + ], + [ + 412, + 0.356055241523914 + ], + [ + 416, + 0.814010032134231 + ], + [ + 420, + 0.007488205565631456 + ], + [ + 425, + 0.7931562431982624 + ], + [ + 434, + 0.954157423183999 + ], + [ + 438, + 0.40241743695007015 + ], + [ + 443, + 0.49687395657512334 + ], + [ + 445, + 0.3014258730442745 + ], + [ + 456, + 0.6626990737054786 + ], + [ + 461, + 0.16653401631344333 + ], + [ + 466, + 0.7495890265110241 + ], + [ + 469, + 0.6567296454080804 + ], + [ + 479, + 0.39571085419662744 + ], + [ + 489, + 0.6058225802300987 + ], + [ + 492, + 0.20558756429341563 + ], + [ + 496, + 0.804752410021165 + ], + [ + 504, + 0.5932590293790729 + ], + [ + 507, + 0.07832012104082742 + ], + [ + 511, + 0.0184356271123437 + ], + [ + 514, + 0.3763932717420686 + ], + [ + 525, + 0.48065937055791097 + ], + [ + 530, + 0.10962123056014506 + ], + [ + 536, + 0.20805348121933198 + ], + [ + 539, + 0.3788384156449759 + ], + [ + 543, + 0.979341740526004 + ], + [ + 552, + 0.24851894311155054 + ], + [ + 557, + 0.32309924777884214 + ], + [ + 565, + 0.8400798678401796 + ], + [ + 568, + 0.9607015985768218 + ], + [ + 572, + 0.9561824914145388 + ], + [ + 585, + 0.7200919244422491 + ], + [ + 588, + 0.5558987531211155 + ], + [ + 592, + 0.7111444885980085 + ], + [ + 598, + 0.30982850116103355 + ], + [ + 601, + 0.2397346210284481 + ] + ], + [ + [ + 38, + 0.603520932099422 + ], + [ + 46, + 0.0762506323997365 + ], + [ + 55, + 0.3895020523873157 + ], + [ + 64, + 0.3774000540845184 + ], + [ + 73, + 0.23786129696635538 + ], + [ + 76, + 0.06397026040024434 + ], + [ + 91, + 0.46988913479038774 + ], + [ + 101, + 0.29774207292067034 + ], + [ + 105, + 0.7553350135556048 + ], + [ + 112, + 0.008667484071393616 + ], + [ + 114, + 0.5581177493331327 + ], + [ + 119, + 0.928783519881658 + ], + [ + 125, + 0.08848841314129707 + ], + [ + 132, + 0.46735517313459585 + ], + [ + 135, + 0.1260308021231742 + ], + [ + 138, + 0.2213730211046392 + ], + [ + 144, + 0.36889343697425736 + ], + [ + 147, + 0.10286089729768033 + ], + [ + 150, + 0.29502606272089016 + ], + [ + 166, + 0.9032753818813676 + ], + [ + 169, + 0.35120871710418944 + ], + [ + 172, + 0.034553103188077916 + ], + [ + 176, + 0.9331594419574355 + ], + [ + 179, + 0.7761010727771397 + ], + [ + 183, + 0.7032093753887285 + ], + [ + 187, + 0.842220907531042 + ], + [ + 195, + 0.7679164629227986 + ], + [ + 198, + 0.7854420134779525 + ], + [ + 202, + 0.3890550948944199 + ], + [ + 207, + 0.05497971573457883 + ], + [ + 216, + 0.699347240333242 + ], + [ + 221, + 0.8064976132197964 + ], + [ + 225, + 0.07827542815389377 + ], + [ + 231, + 0.8864627626323892 + ], + [ + 234, + 0.6884719495222267 + ], + [ + 238, + 0.4460731766723919 + ], + [ + 247, + 0.46685959119707465 + ], + [ + 251, + 0.6274688724044505 + ], + [ + 254, + 0.47704076925583927 + ], + [ + 264, + 0.5003266927168084 + ], + [ + 267, + 0.0493680420069289 + ], + [ + 276, + 0.5961877642003286 + ], + [ + 280, + 0.02315228204035391 + ], + [ + 287, + 0.789709489557732 + ], + [ + 290, + 0.857801698219925 + ], + [ + 296, + 0.986191019845217 + ], + [ + 308, + 0.28588448093164864 + ], + [ + 315, + 0.25811670784794016 + ], + [ + 319, + 0.08171091550879261 + ], + [ + 324, + 0.09058421446563814 + ], + [ + 329, + 0.989225467521274 + ], + [ + 335, + 0.5330819293308688 + ], + [ + 340, + 0.37801674557217657 + ], + [ + 343, + 0.1909601666581524 + ], + [ + 347, + 0.09155486214150232 + ], + [ + 356, + 0.9653414857273553 + ], + [ + 363, + 0.5105232513239396 + ], + [ + 370, + 0.1569412479821508 + ], + [ + 374, + 0.5197053247230333 + ], + [ + 378, + 0.6022079090558656 + ], + [ + 383, + 0.3673918675622929 + ], + [ + 385, + 0.18852514793262654 + ], + [ + 392, + 0.2619917524911628 + ], + [ + 397, + 0.15612141106529442 + ], + [ + 405, + 0.41649548724658314 + ], + [ + 412, + 0.4583296490297408 + ], + [ + 416, + 0.4923381309697026 + ], + [ + 420, + 0.7006963800507172 + ], + [ + 425, + 0.9142920213196485 + ], + [ + 434, + 0.642733061750795 + ], + [ + 438, + 0.3386881949764107 + ], + [ + 443, + 0.13763953426555908 + ], + [ + 445, + 0.030906060874842156 + ], + [ + 456, + 0.9085789533860049 + ], + [ + 461, + 0.48964690344281636 + ], + [ + 466, + 0.8128647950067759 + ], + [ + 469, + 0.8128075655993304 + ], + [ + 479, + 0.6596808382378957 + ], + [ + 489, + 0.8874590133837322 + ], + [ + 492, + 0.8058382737281328 + ], + [ + 496, + 0.13196842623613136 + ], + [ + 504, + 0.10458029266690638 + ], + [ + 507, + 0.03337089129019444 + ], + [ + 511, + 0.17988278161434657 + ], + [ + 514, + 0.8714081949531116 + ], + [ + 525, + 0.6040266454814409 + ], + [ + 530, + 0.44794720375829244 + ], + [ + 536, + 0.7738269536236897 + ], + [ + 539, + 0.9241519508165542 + ], + [ + 543, + 0.2699325055235672 + ], + [ + 552, + 0.3183689112299497 + ], + [ + 557, + 0.8197654056781183 + ], + [ + 565, + 0.2988091874180716 + ], + [ + 568, + 0.470408023018947 + ], + [ + 572, + 0.8710772597697137 + ], + [ + 585, + 0.2701983545581935 + ], + [ + 588, + 0.2851079770173752 + ], + [ + 592, + 0.8031574029814366 + ], + [ + 598, + 0.5845960315336762 + ], + [ + 601, + 0.6167920310742061 + ] + ], + [ + [ + 37, + 0.513381576424309 + ], + [ + 43, + 0.7196252974902934 + ], + [ + 54, + 0.27152458216832853 + ], + [ + 60, + 0.9517817802988677 + ], + [ + 71, + 0.7012610207305694 + ], + [ + 77, + 0.9807058079093713 + ], + [ + 80, + 0.5853994338351054 + ], + [ + 89, + 0.25923597037414403 + ], + [ + 95, + 0.40453955884423454 + ], + [ + 98, + 0.7233399470036999 + ], + [ + 106, + 0.1329166207630713 + ], + [ + 113, + 0.7968998516981478 + ], + [ + 127, + 0.9781083560721823 + ], + [ + 138, + 0.9382776221768705 + ], + [ + 142, + 0.5463890956272981 + ], + [ + 150, + 0.007525934776732135 + ], + [ + 159, + 0.1573516356700635 + ], + [ + 166, + 0.4547596827198923 + ], + [ + 177, + 0.6177591388392978 + ], + [ + 184, + 0.9320078503175888 + ], + [ + 187, + 0.6423439045965761 + ], + [ + 189, + 0.7779544198033739 + ], + [ + 197, + 0.1373071527631161 + ], + [ + 201, + 0.5041130654355112 + ], + [ + 204, + 0.6392083454948945 + ], + [ + 212, + 0.31661098786369746 + ], + [ + 216, + 0.6145036202717641 + ], + [ + 220, + 0.45545015152780244 + ], + [ + 225, + 0.5907322078131989 + ], + [ + 233, + 0.1960679598009586 + ], + [ + 241, + 0.8218116249629631 + ], + [ + 248, + 0.06903378487202261 + ], + [ + 252, + 0.3843788284367947 + ], + [ + 263, + 0.8491551400337625 + ], + [ + 270, + 0.4043516537281038 + ], + [ + 273, + 0.5676297409375225 + ], + [ + 277, + 0.6091694547446456 + ], + [ + 282, + 0.973806291567238 + ], + [ + 287, + 0.5816717469870128 + ], + [ + 293, + 0.8523751480647667 + ], + [ + 298, + 0.5245575617903098 + ], + [ + 302, + 0.76909151274186 + ], + [ + 307, + 0.40638914568314044 + ], + [ + 315, + 0.19876974692321514 + ], + [ + 318, + 0.07205646858901127 + ], + [ + 327, + 0.13672622547294522 + ], + [ + 334, + 0.03133491470414873 + ], + [ + 345, + 0.39658702653530686 + ], + [ + 352, + 0.7412706147496134 + ], + [ + 357, + 0.7759801227659198 + ], + [ + 361, + 0.6184357176220328 + ], + [ + 369, + 0.7244427454635112 + ], + [ + 380, + 0.7388456175494494 + ], + [ + 384, + 0.9034878004117507 + ], + [ + 387, + 0.043907737150515436 + ], + [ + 392, + 0.6028143984730646 + ], + [ + 399, + 0.7917719162871802 + ], + [ + 409, + 0.881552810705541 + ], + [ + 415, + 0.6610696017457188 + ], + [ + 426, + 0.9415431525095521 + ], + [ + 432, + 0.34315324766170097 + ], + [ + 435, + 0.13315142929398438 + ], + [ + 439, + 0.20709278706178702 + ], + [ + 449, + 0.007907989183935094 + ], + [ + 460, + 0.5482012323181518 + ], + [ + 466, + 0.5770931649103702 + ], + [ + 470, + 0.712568956463001 + ], + [ + 477, + 0.23287230009546867 + ], + [ + 481, + 0.4170334087408497 + ], + [ + 486, + 0.1573756513358605 + ], + [ + 492, + 0.7184536629668724 + ], + [ + 495, + 0.224261706891709 + ], + [ + 504, + 0.7129364971898476 + ], + [ + 517, + 0.8488846583073799 + ], + [ + 521, + 0.5712011747062626 + ], + [ + 530, + 0.1932415004748217 + ], + [ + 535, + 0.08034917320423784 + ], + [ + 539, + 0.7834463191735928 + ], + [ + 550, + 0.43830879484326224 + ], + [ + 554, + 0.09854436673413458 + ], + [ + 559, + 0.14718573381567213 + ], + [ + 562, + 0.8889656029902756 + ], + [ + 571, + 0.16723352115676948 + ], + [ + 575, + 0.20758100991968553 + ], + [ + 581, + 0.295960451933183 + ], + [ + 585, + 0.4609993073816194 + ], + [ + 590, + 0.6466873188537672 + ], + [ + 597, + 0.2752990969138407 + ], + [ + 602, + 0.7096580147375906 + ], + [ + 608, + 0.17623647475444648 + ], + [ + 613, + 0.4032886556915962 + ], + [ + 619, + 0.16378243960649597 + ], + [ + 631, + 0.7137679097181779 + ], + [ + 634, + 0.7446590318965053 + ], + [ + 639, + 0.4193413455109938 + ], + [ + 643, + 0.8484369907411853 + ], + [ + 649, + 0.18308250365803003 + ], + [ + 652, + 0.3101096938589071 + ], + [ + 656, + 0.29121296726999224 + ], + [ + 660, + 0.3148352585114369 + ] + ], + [ + [ + 50, + 0.6422797038955337 + ], + [ + 58, + 0.6934207009221011 + ], + [ + 61, + 0.49640817594807274 + ], + [ + 66, + 0.7724288478172139 + ], + [ + 75, + 0.39600934739417526 + ], + [ + 78, + 0.4627477344382034 + ], + [ + 85, + 0.5078349795403009 + ], + [ + 88, + 0.9988360525390081 + ], + [ + 100, + 0.026690485422848798 + ], + [ + 105, + 0.3680321661937489 + ], + [ + 110, + 0.32299484974977777 + ], + [ + 115, + 0.5562527883665164 + ], + [ + 125, + 0.04049288412519525 + ], + [ + 128, + 0.6193364695002048 + ], + [ + 133, + 0.9210080736098937 + ], + [ + 142, + 0.7229171472534107 + ], + [ + 149, + 0.5854843122371021 + ], + [ + 159, + 0.757081240692507 + ], + [ + 163, + 0.40044288922191484 + ], + [ + 168, + 0.8571777767200383 + ], + [ + 173, + 0.40817403884935577 + ], + [ + 178, + 0.9296736979696516 + ], + [ + 189, + 0.5758837190053285 + ], + [ + 199, + 0.7724813401379371 + ], + [ + 206, + 0.5348802760263863 + ], + [ + 217, + 0.661786398032506 + ], + [ + 221, + 0.6764058091985736 + ], + [ + 225, + 0.4277899112484257 + ], + [ + 232, + 0.7592384634389471 + ], + [ + 240, + 0.9084338231062205 + ], + [ + 249, + 0.8889180265551503 + ], + [ + 255, + 0.5150905967731343 + ], + [ + 259, + 0.26532560890249324 + ], + [ + 266, + 0.8720295978297656 + ], + [ + 270, + 0.5219125895996651 + ], + [ + 274, + 0.06817657401098287 + ], + [ + 279, + 0.9211726073482962 + ], + [ + 285, + 0.28134914474280515 + ], + [ + 292, + 0.9966773891768785 + ], + [ + 295, + 0.17237526261080038 + ], + [ + 297, + 0.4294110943124677 + ], + [ + 304, + 0.7698193266691236 + ], + [ + 316, + 0.6020476537585204 + ], + [ + 321, + 0.3087112650492133 + ], + [ + 324, + 0.8544656843796182 + ], + [ + 330, + 0.984413479694586 + ], + [ + 333, + 0.7608912570377389 + ], + [ + 339, + 0.6230218432568632 + ], + [ + 343, + 0.03128879918304184 + ], + [ + 351, + 0.7864839878887987 + ], + [ + 356, + 0.43362327535793066 + ], + [ + 361, + 0.34633875318052953 + ], + [ + 369, + 0.3824050813418347 + ], + [ + 372, + 0.5998279984099412 + ], + [ + 377, + 0.5395407954605627 + ], + [ + 384, + 0.4058277215525202 + ], + [ + 389, + 0.33279814672823527 + ], + [ + 396, + 0.4331350363340575 + ], + [ + 399, + 0.3270907526113195 + ], + [ + 405, + 0.23779136688787872 + ], + [ + 412, + 0.5110142910127471 + ], + [ + 415, + 0.9080171931515583 + ], + [ + 420, + 0.9090684879887588 + ], + [ + 424, + 0.9262803655785424 + ], + [ + 435, + 0.07808216224986386 + ], + [ + 443, + 0.40020958703753307 + ], + [ + 446, + 0.3576986542943096 + ], + [ + 452, + 0.6620478949162486 + ], + [ + 457, + 0.2640896063996898 + ], + [ + 461, + 0.14195512419501688 + ], + [ + 464, + 0.755335186847766 + ], + [ + 468, + 0.2379075172530839 + ], + [ + 471, + 0.07456230436999944 + ], + [ + 485, + 0.39026385980981926 + ], + [ + 489, + 0.5334206937448264 + ], + [ + 496, + 0.7474824316722386 + ], + [ + 504, + 0.5780347241545671 + ], + [ + 517, + 0.23569914983131757 + ], + [ + 521, + 0.3052867400892182 + ], + [ + 528, + 0.18774685935019297 + ], + [ + 532, + 0.36185279363568945 + ], + [ + 535, + 0.3027767000168563 + ], + [ + 541, + 0.691005938287552 + ], + [ + 546, + 0.39123646976170234 + ], + [ + 550, + 0.08447491808751129 + ], + [ + 558, + 0.5976829086366219 + ], + [ + 563, + 0.11133670969954812 + ], + [ + 566, + 0.38060381864252524 + ], + [ + 571, + 0.4342609151753515 + ], + [ + 576, + 0.9449167106926449 + ], + [ + 582, + 0.637668352901396 + ], + [ + 588, + 0.9355492263865708 + ], + [ + 595, + 0.8333817180813899 + ], + [ + 599, + 0.10710285476346681 + ], + [ + 602, + 0.5560599527094183 + ], + [ + 611, + 0.7510470414101504 + ], + [ + 614, + 0.19949914188533013 + ], + [ + 618, + 0.5617190695639953 + ], + [ + 624, + 0.511777037362084 + ], + [ + 629, + 0.7440889421311377 + ] + ], + [ + [ + 53, + 0.5449624260433773 + ], + [ + 63, + 0.3765899795869678 + ], + [ + 70, + 0.9894650921091898 + ], + [ + 73, + 0.21208951658098563 + ], + [ + 77, + 0.7988046750801162 + ], + [ + 87, + 0.7362408203124704 + ], + [ + 90, + 0.7880420117534773 + ], + [ + 96, + 0.6267066545352198 + ], + [ + 110, + 0.06973596538653448 + ], + [ + 121, + 0.4499489016308217 + ], + [ + 124, + 0.8692652745798012 + ], + [ + 133, + 0.4335345004696497 + ], + [ + 141, + 0.7970260857666713 + ], + [ + 145, + 0.36931068328741345 + ], + [ + 157, + 0.6656417505025083 + ], + [ + 163, + 0.4760550429168492 + ], + [ + 170, + 0.845879980813278 + ], + [ + 178, + 0.4163075030886403 + ], + [ + 187, + 0.394140696366314 + ], + [ + 194, + 0.4990235338336304 + ], + [ + 197, + 0.3186398977144621 + ], + [ + 201, + 0.9268107880336539 + ], + [ + 211, + 0.9285920800333733 + ], + [ + 214, + 0.9371175521971177 + ], + [ + 220, + 0.9504154950059747 + ], + [ + 234, + 0.38836663068375155 + ], + [ + 245, + 0.7407235912999218 + ], + [ + 248, + 0.4429384066471006 + ], + [ + 257, + 0.47503447260579623 + ], + [ + 265, + 0.31296255810123486 + ], + [ + 269, + 0.2569574266281244 + ], + [ + 281, + 0.1343001849015718 + ], + [ + 288, + 0.904361189951702 + ], + [ + 293, + 0.769661648060284 + ], + [ + 299, + 0.06778685858270572 + ], + [ + 303, + 0.37879914474455345 + ], + [ + 314, + 0.3671181123078415 + ], + [ + 317, + 0.6382772496205779 + ], + [ + 322, + 0.21326068941873488 + ], + [ + 325, + 0.22880616662619158 + ], + [ + 329, + 0.23236038914936785 + ], + [ + 337, + 0.7009211752966925 + ], + [ + 343, + 0.6729974598215379 + ], + [ + 352, + 0.7468788838550607 + ], + [ + 355, + 0.47342424222242363 + ], + [ + 357, + 0.5831257014475185 + ], + [ + 370, + 0.8235894169982207 + ], + [ + 382, + 0.114833787125735 + ], + [ + 391, + 0.22653270705518247 + ], + [ + 395, + 0.9502530663570846 + ], + [ + 402, + 0.6600004432522415 + ], + [ + 412, + 0.26400979822619697 + ], + [ + 423, + 0.9797193378686831 + ], + [ + 430, + 0.9073558289720868 + ], + [ + 441, + 0.6843969856808978 + ], + [ + 446, + 0.30827463822792434 + ], + [ + 450, + 0.9395315829849773 + ], + [ + 456, + 0.4938581290689249 + ], + [ + 459, + 0.7187865698692597 + ], + [ + 468, + 0.7529256135103046 + ], + [ + 474, + 0.051473127322253065 + ], + [ + 484, + 0.0020468019208339516 + ], + [ + 488, + 0.3663418144648368 + ], + [ + 494, + 0.8940924356980108 + ], + [ + 500, + 0.9571762829809473 + ], + [ + 506, + 0.5208577422252744 + ], + [ + 512, + 0.17510654793232383 + ], + [ + 518, + 0.31643147558924023 + ], + [ + 524, + 0.6776197197956842 + ], + [ + 528, + 0.5454542459910157 + ], + [ + 536, + 0.006961026610380561 + ], + [ + 543, + 0.009102923719640943 + ], + [ + 547, + 0.6953258152041047 + ], + [ + 551, + 0.5569946048070685 + ], + [ + 557, + 0.2658730726667128 + ], + [ + 566, + 0.8613494947193191 + ], + [ + 570, + 0.1367043545813934 + ], + [ + 575, + 0.6962549432190652 + ], + [ + 581, + 0.6914560236469384 + ], + [ + 585, + 0.19544187852677808 + ], + [ + 590, + 0.029484497685531297 + ], + [ + 597, + 0.42438554903360093 + ], + [ + 604, + 0.06279993607190881 + ], + [ + 607, + 0.7686180212441796 + ], + [ + 613, + 0.02457933056854411 + ], + [ + 617, + 0.7570114120642094 + ], + [ + 625, + 0.1500356730479605 + ], + [ + 630, + 0.5316503764867768 + ], + [ + 633, + 0.5543794565666095 + ], + [ + 637, + 0.5641157196899806 + ], + [ + 643, + 0.6289393340677137 + ], + [ + 647, + 0.22098575481071536 + ], + [ + 660, + 0.197185396514996 + ], + [ + 672, + 0.11303499480480139 + ], + [ + 677, + 0.896460292565938 + ], + [ + 682, + 0.0016286011921740062 + ], + [ + 691, + 0.7675419533115697 + ], + [ + 695, + 0.26034016975070273 + ], + [ + 698, + 0.9159669970884728 + ], + [ + 702, + 0.20315086228728496 + ] + ] + ] + ] + } +] \ No newline at end of file diff --git a/htmls/res_attr_span_word.json b/htmls/res_attr_span_word.json new file mode 100644 index 0000000000000000000000000000000000000000..e3e1ab9d6f836d2b9ab256d006d3494cc1ac40f9 --- /dev/null +++ b/htmls/res_attr_span_word.json @@ -0,0 +1,23551 @@ +[ + { + "data": { + "question": "Who has the highest goals in world football?", + "answer": "The players with the highest all-time goals and highest men's and women's international football goals differ. The player with the highest all-time men's football goals is Josef Bican, who in 2020 was recognized by FIFA, the international governing body of football, as the record scorer with an estimated 805 goals. Christine Sinclair has the highest goals in women's international football with 187 and is the all-time leader for international goals scored for men or women. Cristiano Ronaldo and Ali Daei are currently tied for leading goalscorer in the history of men's international football with 109.", + "qa_pairs": [ + { + "context": "No context provided", + "question": "Who has the highest goals in men's world international football?", + "short_answers": [ + "Daei", + "Ali Daei" + ], + "wikipage": null + }, + { + "context": "No context provided", + "question": "Who has the highest goals all-time in men's football?", + "short_answers": [ + "Bican", + "Josef Bican" + ], + "wikipage": null + }, + { + "context": "The first player to reach 100 international goals was Italian Elisabetta Vignotto. Abby Wambach scored 100 goals in 9 years, while Christine Sinclair reached the milestone in just under 10 years while Mia Hamm is the youngest player to score 100 international goals at the age of 26 years 185 days. Most played exclusively in the forward position, with Kristine Lilly and Michelle Akers having also played as midfielder. All players scored at a high average rate of more than one goal every three matches. International goals in this list should not include goals scored in penalty-shoot-out; see Penalty shootout (association football). Players who are currently active at international level are indicated in bold type background.", + "question": "Who has the highest goals in women's world international football?", + "short_answers": [ + "Sinclair", + "Christine Sinclair" + ], + "wikipage": "List of women's footballers with 100 or more international goals ..." + } + ], + "docs": [ + "Document [1](Title:Argentina\u2013Brazil football rivalry): \"Football Player of the Century\", by IFFHS International Federation of Football History and Statistics, 1999, \"South America Football Player of the Century\", by IFFHS International Federation of Football History and Statistics. Pel\u00e9's 1281 goals are recognized by FIFA as the highest total achieved by a professional footballer, although the Soccer Statistic Foundation (rssf) recognizes only 767 goals in official mode, occupying the third place after Josef Bican (805) and Romario (772). For his part, Maradona has been named the best soccer player in World Cup history both by The Times and FourFourTwo, publication that also rewarded him as the \"Best\n", + "Document [2](Title:Godfrey Chitalu): have beaten Gerd M\u00fcller's record of 85 goals in a year, the Football Association of Zambia claimed that the world record actually pertained to Godfrey Chitalu who had scored 116 goals (possibly 117) during the 1972 calendar year and 107 during the 1972 season. The difference of goals is due to first 9 goals being scored before the season officially started. The Football Association of Zambia presented the evidence to FIFA but a spokesperson responded that they would ratify neither Lionel Messi's nor Chitalu's records as they do not keep statistical track of domestic competitions. Nonetheless, it could constitute the\n", + "Document [3](Title:Godfrey Chitalu): highest official tally claimed by a national football association. Chitalu made his international debut on 29 June 1968 in a friendly match against Uganda in Lusaka which Zambia won 2\u20131. He scored his first goal in a 2\u20132 draw against the same team five days later. Chitalu played a prominent role during the World Cup qualification matches against Sudan with Zambia being eliminated on a strange rule which was peculiar to Africa and favoured the team that won the second leg. Despite the aggregate score being tied at 6\u20136 after Zambia won the first leg 4\u20132 and lost the return\n", + "Document [4](Title:Wartan Ghazarian): goals (4 in World Cup qualifiers, 3 in Asian Cup qualifiers, 12 in friendlies). His record was later broken by Roda Antar, after Roda scored his 20th goal in 2018 FIFA World Cup qualification match against Laos. On 16 November 2008, during Round 6 of the Lebanese Football League, at the age of 39 years, Vartan scored his 130th goal in the Lebanese first division against Tadamon Tyre, becoming officially the highest all-time scorer in the history of Lebanese football. Some officials do not recognize the 12 goals he scored in the 2000\u20132001 season which was canceled. However, his remaining\n", + "Document [5](Title:Josef Bican): for Christmas, but died less than 2 weeks before that, at the age of 88. Josef Bican Josef \"Pepi\" Bican (25 September 1913 \u2013 12 December 2001) was a Czech-Austrian professional footballer who played as a striker. Rec.Sport.Soccer Statistics Foundation (RSSSF) estimates that he scored at least 805 goals in all competitive matches, which would make him the most prolific scorer of all time. Having scored a record 600 league goals and at least 1468 goals overall, the International Federation of Football History & Statistics (IFFHS) awarded Bican the \"Golden Ball\" as the greatest goalscorer of the last century. He\n" + ] + }, + "doc_cache": [ + "Document [1](Title:Humberto Suazo)announced his retirement from football on January 14, 2016. Suazo had also become a fixture with his international side. In 2006, he scored 17 goals in national and international matches, surpassing Peter Crouch by one goal for the \"World's Top Goal Scorer Award\". His four international goals all came in friendlies. He scored goals against New Zealand and Sweden. His other two goals came in the form of penalties versus Ivory Coast and Colombia. In January 2007, he was awarded world's top goal scorer of 2006 by the International Federation of Football History & Statistics in Salzburg, Austria. He was\n", + "Document [2](Title:FIFA Puska\u0301s Award)forward European football produced in first-division football, and scored a once world record of 84 goals in 85 internationals. He is one of the top scorers of the 20th century, with 512 goals in 528 matches. \"It is important to preserve the memory of those footballing greats who have left their mark on our history. Ferenc Pusk\u00e1s was not only a player with immense talent who won many honours, but also a remarkable man. FIFA is therefore delighted to pay tribute to him by dedicating this award to his memory,\" said Blatter at the inauguration of the award in Budapest.\n", + "Document [3](Title:FIFA Puska\u0301s Award)forward European football produced in first-division football, and scored a once world record of 84 goals in 85 internationals. He is one of the top scorers of the 20th century, with 512 goals in 528 matches. \"It is important to preserve the memory of those footballing greats who have left their mark on our history. Ferenc Pusk\u00e1s was not only a player with immense talent who won many honours, but also a remarkable man. FIFA is therefore delighted to pay tribute to him by dedicating this award to his memory,\" said Blatter at the inauguration of the award in Budapest.\n", + "Document [4](Title:Oktay Dereliog\u0306lu)Oktay Derelio\u011flu Oktay Derelio\u011flu (born 17 December 1975) is Turkish former international footballer and current football coach. Derelio\u011flu played as a forward and is notable for his goal against Belgium during the qualifying rounds of the 1998 FIFA World Cup. The goal started at midfield before Derelio\u011flu jinked past six Belgian defenders, one of them twice, mimicking Diego Maradona's \"Goal of the Century\". Derelio\u011flu holds the record for most goals in European competitions for Be\u015fikta\u015f with 14. Derelio\u011flu was born in Istanbul and began his club career with local club Fatih Karag\u00fcmr\u00fck. He made his debut at the age of\n", + "Document [5](Title:FIFA World Cup top goalscorers)Germany or West Germany. In total, 64 footballers came from UEFA (Europe), 29 from CONMEBOL (South America), and only four from elsewhere: Cameroon, Ghana, Australia, and the United States. Fontaine holds the record for the most goals scored at a single tournament, with 13 goals in 1958. The players that came closest to this record were Kocsis in 1954, M\u00fcller in 1970 and Portugal's Eus\u00e9bio in 1966, with 11, 10 and 9, respectively. The lowest scoring tournament's top scorer was in 1962, when six players tied at only four goals each. Across the 21 editions of the World Cup, 30\n", + "Document [6](Title:Argentina\u2013Brazil football rivalry)\"Football Player of the Century\", by IFFHS International Federation of Football History and Statistics, 1999, \"South America Football Player of the Century\", by IFFHS International Federation of Football History and Statistics. Pel\u00e9's 1281 goals are recognized by FIFA as the highest total achieved by a professional footballer, although the Soccer Statistic Foundation (rssf) recognizes only 767 goals in official mode, occupying the third place after Josef Bican (805) and Romario (772). For his part, Maradona has been named the best soccer player in World Cup history both by The Times and FourFourTwo, publication that also rewarded him as the \"Best\n" + ], + "log": [ + "gpt-3.5-turbo-[1] -> retriever-[4]\n: Query: \"Top goal scorers in international football history\"", + "retriever-[4] -> gpt-3.5-turbo-[3]\n: Document [1](Title:Humberto Suazo)announced his retirement from football on January 14, 2016. Suazo had also become a fixture with his international side. In 2006, he scored 17 goals in national and international matches, surpassing Peter Crouch by one goal for the \"World's Top Goal Scorer Award\". His four international goals all came in friendlies. He scored goals against New Zealand and Sweden. His other two goals came in the form of penalties versus Ivory Coast and Colombia. In January 2007, he was awarded world's top goal scorer of 2006 by the International Federation of Football History & Statistics in Salzburg, Austria. He was\n\nDocument [2](Title:FIFA Puska\u0301s Award)forward European football produced in first-division football, and scored a once world record of 84 goals in 85 internationals. He is one of the top scorers of the 20th century, with 512 goals in 528 matches. \"It is important to preserve the memory of those footballing greats who have left their mark on our history. Ferenc Pusk\u00e1s was not only a player with immense talent who won many honours, but also a remarkable man. FIFA is therefore delighted to pay tribute to him by dedicating this award to his memory,\" said Blatter at the inauguration of the award in Budapest.\n", + "gpt-3.5-turbo-[3] -> evaluator-[2]\n: ['Humberto Suazo was awarded the title of the world\\'s top goal scorer of 2006, surpassing Peter Crouch by one goal for the \"World\\'s Top Goal Scorer Award\"[1]. ', 'Ferenc Pusk\u00e1s holds the record for the highest number of goals in world football, scoring 84 goals in 85 internationals and a total of 512 goals in 528 matches[2]. ']", + "evaluator-[2] -> gpt-3.5-turbo-[1]\n: {'answer': 'Humberto Suazo was awarded the title of the world\\'s top goal scorer of 2006, surpassing Peter Crouch by one goal for the \"World\\'s Top Goal Scorer Award\"[1]. '}", + "gpt-3.5-turbo-[1] -> retriever-[4]\n: Query: \"Current leading goal scorers in top European football leagues\"", + "retriever-[4] -> gpt-3.5-turbo-[3]\n: Document [3](Title:FIFA Puska\u0301s Award)forward European football produced in first-division football, and scored a once world record of 84 goals in 85 internationals. He is one of the top scorers of the 20th century, with 512 goals in 528 matches. \"It is important to preserve the memory of those footballing greats who have left their mark on our history. Ferenc Pusk\u00e1s was not only a player with immense talent who won many honours, but also a remarkable man. FIFA is therefore delighted to pay tribute to him by dedicating this award to his memory,\" said Blatter at the inauguration of the award in Budapest.\n\nDocument [4](Title:Oktay Dereliog\u0306lu)Oktay Derelio\u011flu Oktay Derelio\u011flu (born 17 December 1975) is Turkish former international footballer and current football coach. Derelio\u011flu played as a forward and is notable for his goal against Belgium during the qualifying rounds of the 1998 FIFA World Cup. The goal started at midfield before Derelio\u011flu jinked past six Belgian defenders, one of them twice, mimicking Diego Maradona's \"Goal of the Century\". Derelio\u011flu holds the record for most goals in European competitions for Be\u015fikta\u015f with 14. Derelio\u011flu was born in Istanbul and began his club career with local club Fatih Karag\u00fcmr\u00fck. He made his debut at the age of\n", + "gpt-3.5-turbo-[3] -> evaluator-[2]\n: ['Ferenc Pusk\u00e1s holds the once world record of scoring 84 goals in 85 internationals in European football, making him one of the top scorers of the 20th century[3]. ', 'Oktay Derelio\u011flu holds the record for most goals in European competitions for Be\u015fikta\u015f with 14[4]. ']", + "evaluator-[2] -> gpt-3.5-turbo-[1]\n: {'answer': 'Ferenc Pusk\u00e1s holds the once world record of scoring 84 goals in 85 internationals in European football, making him one of the top scorers of the 20th century[3]. '}", + "gpt-3.5-turbo-[1] -> retriever-[4]\n: Query: \"Players with most goals in FIFA World Cup history\"", + "retriever-[4] -> gpt-3.5-turbo-[3]\n: Document [5](Title:FIFA World Cup top goalscorers)Germany or West Germany. In total, 64 footballers came from UEFA (Europe), 29 from CONMEBOL (South America), and only four from elsewhere: Cameroon, Ghana, Australia, and the United States. Fontaine holds the record for the most goals scored at a single tournament, with 13 goals in 1958. The players that came closest to this record were Kocsis in 1954, M\u00fcller in 1970 and Portugal's Eus\u00e9bio in 1966, with 11, 10 and 9, respectively. The lowest scoring tournament's top scorer was in 1962, when six players tied at only four goals each. Across the 21 editions of the World Cup, 30\n\nDocument [6](Title:Argentina\u2013Brazil football rivalry)\"Football Player of the Century\", by IFFHS International Federation of Football History and Statistics, 1999, \"South America Football Player of the Century\", by IFFHS International Federation of Football History and Statistics. Pel\u00e9's 1281 goals are recognized by FIFA as the highest total achieved by a professional footballer, although the Soccer Statistic Foundation (rssf) recognizes only 767 goals in official mode, occupying the third place after Josef Bican (805) and Romario (772). For his part, Maradona has been named the best soccer player in World Cup history both by The Times and FourFourTwo, publication that also rewarded him as the \"Best\n", + "gpt-3.5-turbo-[3] -> evaluator-[2]\n: ['At the FIFA World Cup, Just Fontaine holds the record for the most goals scored at a single tournament with 13 goals in 1958[5]. ', 'Pel\u00e9 holds the record for the highest number of goals in world football, with 1281 goals recognized by FIFA[6]. ']", + "evaluator-[2] -> gpt-3.5-turbo-[1]\n: {'answer': 'At the FIFA World Cup, Just Fontaine holds the record for the most goals scored at a single tournament with 13 goals in 1958[5]. '}" + ], + "output": [ + "Humberto Suazo was awarded the title of the world's top goal scorer of 2006, surpassing Peter Crouch by one goal for the \"World's Top Goal Scorer Award\" [1].", + "Ferenc Pusk\u00e1s holds the once world record of scoring 84 goals in 85 internationals in European football, making him one of the top scorers of the 20th century [3]. ", + "At the FIFA World Cup, Just Fontaine holds the record for the most goals scored at a single tournament with 13 goals in 1958 [5]. " + ], + "result": { + "length": 82 + }, + "token_used": 4774, + "doc_level": [ + [ + 37.058350018914005, + 29.856228780671866, + 14.709315979190839, + 29.14141611185829, + 29.656878958271196, + 36.184975376222475 + ], + [ + 34.671008236972256, + 21.70285875316371, + 18.20363647696773, + 29.734538824443682, + 31.52580952179077, + 41.700312892419205 + ], + [ + 26.12779277501497, + 27.722579037265284, + 13.329065133636627, + 35.02061584447701, + 12.331497496314029, + 23.653628335594913 + ] + ], + "span_level": [ + [ + [ + [ + 34, + 0.310100532820032 + ], + [ + 94, + 0.10706984868310372 + ], + [ + 155, + 1.0 + ], + [ + 299, + 0.03546586090029468 + ], + [ + 352, + 0.06782192653370361 + ], + [ + 400, + 0.06589344200218025 + ], + [ + 483, + 0.3675333851063783 + ], + [ + 634, + 0.06589344200218025 + ] + ], + [ + [ + 38, + 0.08257354591334747 + ], + [ + 166, + 0.04493381980072218 + ], + [ + 247, + 0.06589344200218025 + ], + [ + 356, + 0.06668646512164718 + ], + [ + 461, + 0.04491493083171277 + ] + ], + [ + [ + 38, + 0.05495716646558315 + ], + [ + 166, + 0.041740006091755204 + ], + [ + 247, + 0.051047455940630745 + ], + [ + 356, + 0.05093126673037013 + ], + [ + 461, + 0.01406617729908423 + ] + ], + [ + [ + 37, + 0.07185622091401767 + ], + [ + 166, + 0.10302305068952847 + ], + [ + 298, + 0.06589344200218025 + ], + [ + 449, + 0.06589344200218025 + ], + [ + 539, + 0.019307101596016153 + ], + [ + 631, + 0.07349275810118132 + ] + ], + [ + [ + 50, + 0.0 + ], + [ + 75, + 0.06589344200218025 + ], + [ + 240, + 0.06589344200218025 + ], + [ + 339, + 0.08224379518090785 + ], + [ + 485, + 0.0752296227182542 + ], + [ + 588, + 0.062239812790974944 + ] + ], + [ + [ + 53, + 0.043231425091250976 + ], + [ + 281, + 0.09100532378324357 + ], + [ + 543, + 0.017929320149162466 + ] + ] + ], + [ + [ + [ + 34, + 0.1734130270251284 + ], + [ + 94, + 0.03484000788049954 + ], + [ + 155, + 0.29529677219349004 + ], + [ + 299, + 0.04558746553646283 + ], + [ + 352, + 0.1734130270251284 + ], + [ + 400, + 0.34318206077358787 + ], + [ + 483, + 0.0 + ], + [ + 634, + 0.1734130270251284 + ] + ], + [ + [ + 38, + 0.8819636197735287 + ], + [ + 166, + 0.5297819878957586 + ], + [ + 247, + 0.1734130270251284 + ], + [ + 356, + 0.32397273025519646 + ], + [ + 461, + 0.269664988140138 + ] + ], + [ + [ + 38, + 1.0 + ], + [ + 166, + 0.47517529927283775 + ], + [ + 247, + 0.1734130270251284 + ], + [ + 356, + 0.2198477642656767 + ], + [ + 461, + 0.1734130270251284 + ] + ], + [ + [ + 37, + 0.1734130270251284 + ], + [ + 166, + 0.1734130270251284 + ], + [ + 298, + 0.1734130270251284 + ], + [ + 449, + 0.16808944315079566 + ], + [ + 539, + 0.1734130270251284 + ], + [ + 631, + 0.1734130270251284 + ] + ], + [ + [ + 50, + 0.1734130270251284 + ], + [ + 75, + 0.1734130270251284 + ], + [ + 240, + 0.1734130270251284 + ], + [ + 339, + 0.15904147641491576 + ], + [ + 485, + 0.35354139382652855 + ], + [ + 588, + 0.22558150709651104 + ] + ], + [ + [ + 53, + 0.2533070705115283 + ], + [ + 281, + 0.1734130270251284 + ], + [ + 543, + 0.1734130270251284 + ] + ] + ], + [ + [ + [ + 34, + 0.1996714230673918 + ], + [ + 94, + 0.1412741804239454 + ], + [ + 155, + 0.14753661922979144 + ], + [ + 299, + 0.11907617442220175 + ], + [ + 352, + 0.129636582219201 + ], + [ + 400, + 0.129636582219201 + ], + [ + 483, + 0.129636582219201 + ], + [ + 634, + 0.129636582219201 + ] + ], + [ + [ + 38, + 0.03171152928410699 + ], + [ + 166, + 0.129636582219201 + ], + [ + 247, + 0.129636582219201 + ], + [ + 356, + 0.20729877762759913 + ], + [ + 461, + 0.129636582219201 + ] + ], + [ + [ + 38, + 0.10033127618370548 + ], + [ + 166, + 0.06296244210866726 + ], + [ + 247, + 0.08337349874433016 + ], + [ + 356, + 0.129636582219201 + ], + [ + 461, + 0.129636582219201 + ] + ], + [ + [ + 37, + 0.129636582219201 + ], + [ + 166, + 0.129636582219201 + ], + [ + 298, + 0.129636582219201 + ], + [ + 449, + 0.129636582219201 + ], + [ + 539, + 0.129636582219201 + ], + [ + 631, + 0.129636582219201 + ] + ], + [ + [ + 50, + 0.3728266338850687 + ], + [ + 75, + 0.129636582219201 + ], + [ + 240, + 1.0 + ], + [ + 339, + 0.129636582219201 + ], + [ + 485, + 0.129636582219201 + ], + [ + 588, + 0.129636582219201 + ] + ], + [ + [ + 53, + 0.22465722077639674 + ], + [ + 281, + 0.0 + ], + [ + 543, + 0.129636582219201 + ] + ] + ] + ], + "word_level": [ + [ + [ + [ + 34, + 0.6886811754848322 + ], + [ + 44, + 0.43188307593793707 + ], + [ + 48, + 0.28279309756568477 + ], + [ + 52, + 0.15399390951193234 + ], + [ + 59, + 0.26198388577142445 + ], + [ + 64, + 0.19325911283881456 + ], + [ + 73, + 0.26105673423014447 + ], + [ + 76, + 0.5113993143706349 + ], + [ + 84, + 0.6791902391365991 + ], + [ + 85, + 0.6593228411441239 + ], + [ + 86, + 0.32667539603392287 + ], + [ + 87, + 0.306943351170503 + ], + [ + 88, + 0.7087274880731874 + ], + [ + 89, + 0.5622900756496044 + ], + [ + 90, + 0.9510562930384433 + ], + [ + 91, + 0.1032207956200768 + ], + [ + 92, + 0.10081866348609013 + ], + [ + 93, + 0.11960681840440883 + ], + [ + 94, + 0.37665943982786115 + ], + [ + 97, + 0.2305939928007548 + ], + [ + 99, + 0.2865692300029994 + ], + [ + 100, + 0.4765569927301579 + ], + [ + 104, + 0.13662868738170558 + ], + [ + 109, + 0.0957547716543951 + ], + [ + 116, + 0.1089702144181944 + ], + [ + 118, + 0.10355988117742052 + ], + [ + 121, + 0.22275045380987846 + ], + [ + 126, + 0.2748719720420597 + ], + [ + 131, + 0.17213069755985266 + ], + [ + 135, + 0.131497112807453 + ], + [ + 149, + 0.1829362268161883 + ], + [ + 154, + 0.24880719041357402 + ], + [ + 155, + 0.29942827398620375 + ], + [ + 158, + 0.17158110114675787 + ], + [ + 159, + 0.14791500589625275 + ], + [ + 160, + 0.18009058594388175 + ], + [ + 161, + 0.17488257994196363 + ], + [ + 162, + 0.21776639930301123 + ], + [ + 163, + 0.2683203390872335 + ], + [ + 164, + 0.2150922904692515 + ], + [ + 167, + 0.09711048267429735 + ], + [ + 174, + 0.20801215549417848 + ], + [ + 175, + 0.10776647349356118 + ], + [ + 176, + 0.07327193452951962 + ], + [ + 177, + 0.07790429020639654 + ], + [ + 183, + 0.23797056436123368 + ], + [ + 186, + 0.1280066489587521 + ], + [ + 195, + 0.13620648829791718 + ], + [ + 199, + 0.14639458541693015 + ], + [ + 213, + 0.14200024971539899 + ], + [ + 221, + 0.16340977506817844 + ], + [ + 222, + 0.2990987052527213 + ], + [ + 226, + 0.25109511628199166 + ], + [ + 230, + 0.34816611296482597 + ], + [ + 233, + 0.32184755702201256 + ], + [ + 239, + 0.24777109977774803 + ], + [ + 242, + 0.21274516902703333 + ], + [ + 246, + 0.5427612682247499 + ], + [ + 249, + 0.4953544994056835 + ], + [ + 253, + 0.36902269158481754 + ], + [ + 258, + 0.3582252475253672 + ], + [ + 262, + 0.5809758682312599 + ], + [ + 266, + 0.4071170701705696 + ], + [ + 268, + 0.6273945729298004 + ], + [ + 273, + 0.7801137984979606 + ], + [ + 274, + 0.5958465230796481 + ], + [ + 275, + 1.0 + ], + [ + 279, + 0.7452547795506018 + ], + [ + 282, + 0.2754984543142292 + ], + [ + 284, + 0.44723779037240546 + ], + [ + 287, + 0.3287879813301859 + ], + [ + 291, + 0.7357023360797357 + ], + [ + 297, + 0.6694921965608875 + ], + [ + 299, + 0.25328942531144516 + ], + [ + 303, + 0.05402595639466222 + ], + [ + 308, + 0.04879983457957716 + ], + [ + 322, + 0.08492293223568326 + ], + [ + 328, + 0.06467728038178899 + ], + [ + 332, + 0.07019394644539784 + ], + [ + 337, + 0.05381680286491313 + ], + [ + 340, + 0.06972214165993723 + ], + [ + 347, + 0.10451636880465744 + ], + [ + 351, + 0.15301183451247247 + ], + [ + 352, + 0.13973653478977716 + ], + [ + 355, + 0.0402558652023625 + ], + [ + 362, + 0.04529232269826182 + ], + [ + 368, + 0.03807887483439929 + ], + [ + 376, + 0.036503571743301426 + ], + [ + 380, + 0.0 + ], + [ + 388, + 0.03370463128566821 + ], + [ + 392, + 0.053863885759437465 + ], + [ + 399, + 0.014083658076912309 + ], + [ + 400, + 0.13258508004065767 + ], + [ + 404, + 0.02063263818815522 + ], + [ + 410, + 0.030958432811735666 + ], + [ + 414, + 0.029404580631017075 + ], + [ + 420, + 0.04045498013846941 + ], + [ + 425, + 0.028287936372727533 + ], + [ + 428, + 0.08805211547006259 + ], + [ + 432, + 0.08861669725569035 + ], + [ + 437, + 0.048494409272693 + ], + [ + 440, + 0.09180792625950791 + ], + [ + 444, + 0.04475572074975048 + ], + [ + 447, + 0.07757244936897845 + ], + [ + 450, + 0.06029668722329708 + ], + [ + 457, + 0.10717502930513105 + ], + [ + 459, + 0.04748122023752116 + ], + [ + 460, + 0.05008208019216451 + ], + [ + 463, + 0.03923521016455543 + ], + [ + 469, + 0.05069731656658084 + ], + [ + 473, + 0.08578276796612065 + ], + [ + 482, + 0.057062770728942114 + ], + [ + 483, + 0.2910964074713516 + ], + [ + 486, + 0.14519221231780843 + ], + [ + 494, + 0.16100683742277175 + ], + [ + 495, + 0.24388495226387455 + ], + [ + 496, + 0.17965680947786855 + ], + [ + 497, + 0.14290097827063875 + ], + [ + 498, + 0.1400553173058887 + ], + [ + 499, + 0.22330987970369923 + ], + [ + 500, + 0.33926916100889 + ], + [ + 503, + 0.18495757061475282 + ], + [ + 507, + 0.2605522335078347 + ], + [ + 515, + 0.608214216676032 + ], + [ + 521, + 0.625511447461259 + ], + [ + 522, + 0.5089149303884267 + ], + [ + 523, + 0.6809335386518509 + ], + [ + 527, + 0.4848321623212235 + ], + [ + 532, + 0.32106334560447625 + ], + [ + 535, + 0.25175842433420836 + ], + [ + 539, + 0.4927139179264597 + ], + [ + 542, + 0.6640690578888995 + ], + [ + 543, + 0.3951073267195846 + ], + [ + 544, + 0.20994133905654686 + ], + [ + 545, + 0.17870488456176273 + ], + [ + 546, + 0.185055875493207 + ], + [ + 547, + 0.27138704529393026 + ], + [ + 550, + 0.26268449462986837 + ], + [ + 554, + 0.10482033359564251 + ], + [ + 568, + 0.11676466253442051 + ], + [ + 579, + 0.17626862984613934 + ], + [ + 582, + 0.12968116477388722 + ], + [ + 591, + 0.0887359333633737 + ], + [ + 599, + 0.1350712307303947 + ], + [ + 601, + 0.09160119717046022 + ], + [ + 612, + 0.1890302181984035 + ], + [ + 615, + 0.12628905250883543 + ], + [ + 619, + 0.08262213846482606 + ], + [ + 624, + 0.06648259970901213 + ], + [ + 625, + 0.13805755396105462 + ], + [ + 633, + 0.03028203514655606 + ], + [ + 634, + 0.301613402498496 + ], + [ + 637, + 0.06649539042894838 + ] + ], + [ + [ + 38, + 1.0 + ], + [ + 46, + 0.7499535355790204 + ], + [ + 55, + 0.5389683848228134 + ], + [ + 64, + 0.3582171956304144 + ], + [ + 73, + 0.5571716121575203 + ], + [ + 76, + 0.3412373007660908 + ], + [ + 82, + 0.2626846952896361 + ], + [ + 83, + 0.21114686408130656 + ], + [ + 86, + 0.3275188248124052 + ], + [ + 91, + 0.3106235791931256 + ], + [ + 100, + 0.19932201126210825 + ], + [ + 101, + 0.25836654046615565 + ], + [ + 105, + 0.21442168661283725 + ], + [ + 112, + 0.3162193817414216 + ], + [ + 114, + 0.29008405972016926 + ], + [ + 119, + 0.5746196858568806 + ], + [ + 125, + 0.3829629624048218 + ], + [ + 132, + 0.3520216687553073 + ], + [ + 135, + 0.22925803932387856 + ], + [ + 136, + 0.12608226743230008 + ], + [ + 137, + 0.09412374810019947 + ], + [ + 138, + 0.1656749664388821 + ], + [ + 144, + 0.229128222454826 + ], + [ + 147, + 0.1461045220957741 + ], + [ + 148, + 0.1341439218600021 + ], + [ + 149, + 0.07118617830549331 + ], + [ + 150, + 0.11271117932304231 + ], + [ + 164, + 0.302519444818475 + ], + [ + 165, + 0.3344148163757913 + ], + [ + 166, + 0.5629181287564319 + ], + [ + 169, + 0.27436875349844575 + ], + [ + 172, + 0.19645959153995196 + ], + [ + 176, + 0.19073445357800928 + ], + [ + 179, + 0.23322416875674404 + ], + [ + 183, + 0.3010632643942215 + ], + [ + 187, + 0.4251073958352688 + ], + [ + 190, + 0.3829584033715571 + ], + [ + 192, + 0.3374531648350928 + ], + [ + 195, + 0.5025795157975191 + ], + [ + 198, + 0.4274766040284954 + ], + [ + 202, + 0.28123527326371295 + ], + [ + 203, + 0.20367134048642493 + ], + [ + 204, + 0.35279964734340175 + ], + [ + 205, + 0.2738990591707143 + ], + [ + 207, + 0.31539104714915867 + ], + [ + 215, + 0.2818540041700153 + ], + [ + 216, + 0.11783360620304766 + ], + [ + 221, + 0.06528940843309815 + ], + [ + 222, + 0.10786257208369547 + ], + [ + 223, + 0.034707747200410206 + ], + [ + 224, + 0.04215338607487985 + ], + [ + 225, + 0.05535708795225108 + ], + [ + 231, + 0.06018193386602017 + ], + [ + 234, + 0.048891274471201056 + ], + [ + 235, + 0.043672904300919496 + ], + [ + 236, + 0.0 + ], + [ + 237, + 0.010799412781033082 + ], + [ + 238, + 0.008751792069814723 + ], + [ + 246, + 0.05348267696785837 + ], + [ + 247, + 0.3821837963192422 + ], + [ + 249, + 0.397688714641818 + ], + [ + 251, + 0.07644238687754068 + ], + [ + 254, + 0.04693562371433164 + ], + [ + 264, + 0.061739751432100094 + ], + [ + 267, + 0.096730384913754 + ], + [ + 276, + 0.10627766653653323 + ], + [ + 280, + 0.08119211140484463 + ], + [ + 287, + 0.140537421812593 + ], + [ + 290, + 0.10184416252292743 + ], + [ + 296, + 0.08802641454537592 + ], + [ + 305, + 0.10223068653245883 + ], + [ + 308, + 0.10086355487528834 + ], + [ + 312, + 0.09684770933284771 + ], + [ + 315, + 0.22664885543435018 + ], + [ + 319, + 0.06512189810054625 + ], + [ + 324, + 0.020815235363609727 + ], + [ + 329, + 0.0499540917686572 + ], + [ + 335, + 0.02419037553057483 + ], + [ + 340, + 0.053798880412924126 + ], + [ + 343, + 0.04040351839416185 + ], + [ + 347, + 0.07034915083219552 + ], + [ + 355, + 0.0784372976463596 + ], + [ + 356, + 0.3898768524972945 + ], + [ + 360, + 0.3469981316013002 + ], + [ + 363, + 0.6150638964098196 + ], + [ + 365, + 0.4041466607900225 + ], + [ + 368, + 0.8664708720908871 + ], + [ + 370, + 0.8786350552131902 + ], + [ + 374, + 0.3575574791956388 + ], + [ + 378, + 0.07973180476350299 + ], + [ + 383, + 0.06063736028128904 + ], + [ + 385, + 0.0377151453823855 + ], + [ + 392, + 0.04436690839078041 + ], + [ + 397, + 0.0363141941126763 + ], + [ + 405, + 0.04526003948316098 + ], + [ + 412, + 0.04953798407789967 + ], + [ + 416, + 0.08436230217031579 + ], + [ + 420, + 0.050443120417630286 + ], + [ + 425, + 0.07510032910035723 + ], + [ + 429, + 0.1850855375750034 + ], + [ + 433, + 0.2425350564624895 + ], + [ + 434, + 0.19121441414532991 + ], + [ + 438, + 0.06033815843252041 + ], + [ + 443, + 0.016368804945320996 + ], + [ + 445, + 0.050268378852433425 + ], + [ + 456, + 0.029883134919717124 + ], + [ + 460, + 0.05766990171273518 + ], + [ + 461, + 0.3987626741695756 + ], + [ + 466, + 0.2554196547244924 + ], + [ + 469, + 0.16223646823406082 + ], + [ + 479, + 0.15137476690465193 + ], + [ + 487, + 0.43662167956902626 + ], + [ + 489, + 0.14292637750115342 + ], + [ + 492, + 0.21680160819006428 + ], + [ + 496, + 0.32155757309697564 + ], + [ + 498, + 0.6055675017119072 + ], + [ + 504, + 0.16726493081377292 + ], + [ + 507, + 0.10110827045577188 + ], + [ + 511, + 0.09090435870922756 + ], + [ + 514, + 0.1310008790918704 + ], + [ + 520, + 0.1868391445876358 + ], + [ + 525, + 0.15964701827412434 + ], + [ + 530, + 0.17373080559333878 + ], + [ + 536, + 0.44474882679776734 + ], + [ + 539, + 0.24123578246424376 + ], + [ + 543, + 0.044709457742771984 + ], + [ + 550, + 0.07370967655805367 + ], + [ + 552, + 0.4580460816480895 + ], + [ + 557, + 0.2351794242258358 + ], + [ + 560, + 0.07260429922505299 + ], + [ + 565, + 0.2243392938511465 + ], + [ + 568, + 0.15103612783922685 + ], + [ + 572, + 0.07988202133680672 + ], + [ + 578, + 0.123549359389128 + ], + [ + 585, + 0.1471814564599008 + ], + [ + 588, + 0.11413885216526609 + ], + [ + 592, + 0.12793057023038187 + ], + [ + 598, + 0.2554939154585528 + ], + [ + 601, + 0.10505185150515997 + ], + [ + 610, + 0.10772194611800301 + ] + ], + [ + [ + 38, + 0.37956306542208995 + ], + [ + 46, + 0.21985606678322506 + ], + [ + 55, + 0.10953041391220077 + ], + [ + 64, + 0.09391019147789043 + ], + [ + 73, + 0.16173717005626254 + ], + [ + 76, + 0.22965159187485176 + ], + [ + 82, + 0.36363465216822444 + ], + [ + 83, + 0.39517657001992995 + ], + [ + 86, + 1.0 + ], + [ + 91, + 0.03471804543217248 + ], + [ + 100, + 0.02548961484662742 + ], + [ + 101, + 0.11642140740661513 + ], + [ + 105, + 0.08375584352843637 + ], + [ + 112, + 0.06946930482372043 + ], + [ + 114, + 0.08424229473323215 + ], + [ + 119, + 0.08186273829987839 + ], + [ + 125, + 0.08478664713569943 + ], + [ + 132, + 0.0727080324157578 + ], + [ + 135, + 0.11350618853484187 + ], + [ + 136, + 0.07262144968974796 + ], + [ + 137, + 0.035337808573495194 + ], + [ + 138, + 0.0402541088165897 + ], + [ + 144, + 0.03912081869201968 + ], + [ + 147, + 0.08275945690903495 + ], + [ + 148, + 0.1479052081342235 + ], + [ + 149, + 0.013462730571316086 + ], + [ + 150, + 0.013049008593652461 + ], + [ + 164, + 0.04060335073308406 + ], + [ + 165, + 0.06911256089012278 + ], + [ + 166, + 0.27658318044164143 + ], + [ + 169, + 0.0822539429302149 + ], + [ + 172, + 0.11834759248023852 + ], + [ + 176, + 0.15129511435883092 + ], + [ + 179, + 0.10809197733035467 + ], + [ + 183, + 0.1276129954971841 + ], + [ + 187, + 0.13253039025377902 + ], + [ + 190, + 0.09596093106455018 + ], + [ + 192, + 0.11407902190599455 + ], + [ + 195, + 0.19723056261956662 + ], + [ + 198, + 0.2280495608228348 + ], + [ + 202, + 0.1724975918774963 + ], + [ + 203, + 0.17625750024178094 + ], + [ + 204, + 0.28914193651610615 + ], + [ + 205, + 0.19663129954489547 + ], + [ + 207, + 0.06712495883404539 + ], + [ + 215, + 0.17197665637279339 + ], + [ + 216, + 0.1426577629829346 + ], + [ + 221, + 0.04695705427019746 + ], + [ + 222, + 0.0964150104630435 + ], + [ + 223, + 0.01972818579645454 + ], + [ + 224, + 0.05393000749095496 + ], + [ + 225, + 0.07392719088216397 + ], + [ + 231, + 0.03418089089838429 + ], + [ + 234, + 0.06998524645509067 + ], + [ + 235, + 0.0734491477157695 + ], + [ + 236, + 0.011780935984915227 + ], + [ + 237, + 0.04721976573792368 + ], + [ + 238, + 0.0 + ], + [ + 246, + 0.015234750803816036 + ], + [ + 247, + 0.3353501165151575 + ], + [ + 249, + 0.18698483707019745 + ], + [ + 251, + 0.027027323608221507 + ], + [ + 254, + 0.05970797521577971 + ], + [ + 264, + 0.023539255727158623 + ], + [ + 267, + 0.09212904608569417 + ], + [ + 276, + 0.03606504973924523 + ], + [ + 280, + 0.103147485142419 + ], + [ + 287, + 0.030702045501820183 + ], + [ + 290, + 0.1350651836662819 + ], + [ + 296, + 0.025256051010167015 + ], + [ + 305, + 0.05036254957712337 + ], + [ + 308, + 0.054859885938243966 + ], + [ + 312, + 0.10211778405758107 + ], + [ + 315, + 0.05128766285376034 + ], + [ + 319, + 0.02284928347609927 + ], + [ + 324, + 0.021665384727557406 + ], + [ + 329, + 0.008659588483424048 + ], + [ + 335, + 0.026791707303788558 + ], + [ + 340, + 0.030001731081919957 + ], + [ + 343, + 0.03420527842303123 + ], + [ + 347, + 0.03222138669296058 + ], + [ + 355, + 0.0308895435556794 + ], + [ + 356, + 0.3116437324186193 + ], + [ + 360, + 0.14276769984219884 + ], + [ + 363, + 0.08115785881486752 + ], + [ + 365, + 0.08037828459025241 + ], + [ + 368, + 0.09575141778383729 + ], + [ + 370, + 0.14625716045307752 + ], + [ + 374, + 0.29373024578539425 + ], + [ + 378, + 0.05096228437677094 + ], + [ + 383, + 0.052223455093837474 + ], + [ + 385, + 0.1360557902232637 + ], + [ + 392, + 0.04216085314895159 + ], + [ + 397, + 0.0682235136350179 + ], + [ + 405, + 0.055431224704390235 + ], + [ + 412, + 0.05167311487899015 + ], + [ + 416, + 0.1252512820272161 + ], + [ + 420, + 0.1099952316709463 + ], + [ + 425, + 0.14382516963378836 + ], + [ + 429, + 0.575943995033395 + ], + [ + 433, + 0.013509756720363994 + ], + [ + 434, + 0.11933205042522291 + ], + [ + 438, + 0.02596603273694193 + ], + [ + 443, + 0.006891715910249068 + ], + [ + 445, + 0.06569699316486295 + ], + [ + 456, + 0.017741862848891198 + ], + [ + 460, + 0.014599894719309342 + ], + [ + 461, + 0.13293273403994738 + ], + [ + 466, + 0.05646024553942845 + ], + [ + 469, + 0.07684263994145779 + ], + [ + 479, + 0.07473640922285461 + ], + [ + 487, + 0.20490886248381623 + ], + [ + 489, + 0.035350804813479714 + ], + [ + 492, + 0.1280172997446571 + ], + [ + 496, + 0.09031358556041831 + ], + [ + 498, + 0.1982535481902995 + ], + [ + 504, + 0.020248127780146268 + ], + [ + 507, + 0.09503363052086826 + ], + [ + 511, + 0.028983500924659802 + ], + [ + 514, + 0.061300227277440236 + ], + [ + 520, + 0.04161604045127463 + ], + [ + 525, + 0.03195367406210043 + ], + [ + 530, + 0.05516451338244651 + ], + [ + 536, + 0.12857461233975767 + ], + [ + 539, + 0.11927396377756537 + ], + [ + 543, + 0.033983357866922616 + ], + [ + 550, + 0.02142291463340836 + ], + [ + 552, + 0.19976920264473694 + ], + [ + 557, + 0.08760765893238256 + ], + [ + 560, + 0.027541289602066997 + ], + [ + 565, + 0.08843100756173794 + ], + [ + 568, + 0.03824438347205041 + ], + [ + 572, + 0.08784444960871933 + ], + [ + 578, + 0.03297338335874238 + ], + [ + 585, + 0.03012292307664893 + ], + [ + 588, + 0.08620783733735793 + ], + [ + 592, + 0.08976133265799682 + ], + [ + 598, + 0.11631436038462252 + ], + [ + 601, + 0.09124736885202472 + ], + [ + 610, + 0.08444581000593976 + ] + ], + [ + [ + 38, + 0.6050884824090624 + ], + [ + 44, + 0.5031188502655918 + ], + [ + 46, + 0.6955713897443164 + ], + [ + 48, + 1.0 + ], + [ + 50, + 0.2395898670506257 + ], + [ + 52, + 0.2569688313515887 + ], + [ + 54, + 0.19400325231057933 + ], + [ + 58, + 0.2113569094199485 + ], + [ + 60, + 0.21364462868071973 + ], + [ + 62, + 0.3175318186603211 + ], + [ + 63, + 0.4580517256399761 + ], + [ + 65, + 0.3255045712322015 + ], + [ + 67, + 0.28433800144565563 + ], + [ + 69, + 0.16275114594711015 + ], + [ + 71, + 0.1433703522137823 + ], + [ + 75, + 0.10134983815236216 + ], + [ + 77, + 0.03295969319178015 + ], + [ + 79, + 0.054397680638758245 + ], + [ + 80, + 0.10072249586373949 + ], + [ + 82, + 0.08258320639796063 + ], + [ + 84, + 0.3451619572934288 + ], + [ + 88, + 0.4768781982825378 + ], + [ + 89, + 0.131115038688081 + ], + [ + 90, + 0.15906021947861135 + ], + [ + 91, + 0.0466803534322778 + ], + [ + 100, + 0.14562894390507114 + ], + [ + 101, + 0.2953310419349031 + ], + [ + 102, + 0.3224293212955428 + ], + [ + 103, + 0.09051556959355447 + ], + [ + 104, + 0.14402110021611877 + ], + [ + 105, + 0.057383724253729104 + ], + [ + 106, + 0.3200237785379248 + ], + [ + 109, + 0.4542284287972133 + ], + [ + 117, + 0.2685319726450099 + ], + [ + 124, + 0.2083767956781875 + ], + [ + 138, + 0.1467198596856815 + ], + [ + 149, + 0.45659449385313794 + ], + [ + 153, + 0.12325350276461237 + ], + [ + 161, + 0.055010933112565566 + ], + [ + 170, + 0.0637793305299049 + ], + [ + 176, + 0.09283487853099115 + ], + [ + 177, + 0.3481928204701873 + ], + [ + 181, + 0.0843202585109378 + ], + [ + 183, + 0.08410506612783106 + ], + [ + 185, + 0.08692483380567657 + ], + [ + 186, + 0.11229239442457892 + ], + [ + 188, + 0.15416416761537324 + ], + [ + 195, + 0.11289186555047323 + ], + [ + 198, + 0.11917195854629051 + ], + [ + 200, + 0.08977257111940552 + ], + [ + 208, + 0.25432482741041396 + ], + [ + 212, + 0.07787676068699571 + ], + [ + 215, + 0.12806403944924818 + ], + [ + 223, + 0.18871357425566945 + ], + [ + 227, + 0.10362113691687319 + ], + [ + 231, + 0.08273412435407076 + ], + [ + 236, + 0.12392840374377045 + ], + [ + 244, + 0.149782567558058 + ], + [ + 249, + 0.07905615244020514 + ], + [ + 252, + 0.06619407282531826 + ], + [ + 259, + 0.10084802587021355 + ], + [ + 263, + 0.18109421587268057 + ], + [ + 268, + 0.15145266898623003 + ], + [ + 274, + 0.28256630510675596 + ], + [ + 276, + 0.46772881213938083 + ], + [ + 281, + 0.13720189414857564 + ], + [ + 284, + 0.09939037748916628 + ], + [ + 288, + 0.1300724831982365 + ], + [ + 289, + 0.10870763868135735 + ], + [ + 290, + 0.12146782585372724 + ], + [ + 291, + 0.03874084137104769 + ], + [ + 292, + 0.07594473207581842 + ], + [ + 293, + 0.08840908422073536 + ], + [ + 298, + 0.17843817868215947 + ], + [ + 304, + 0.15801452435304017 + ], + [ + 308, + 0.16596114405586906 + ], + [ + 309, + 0.3519315170063402 + ], + [ + 313, + 0.09335454547181281 + ], + [ + 318, + 0.0830761972968243 + ], + [ + 326, + 0.12247233050377389 + ], + [ + 329, + 0.06729683339503333 + ], + [ + 333, + 0.02758228570454427 + ], + [ + 338, + 0.07591068904087672 + ], + [ + 345, + 0.08060705832667005 + ], + [ + 349, + 0.05295636893928884 + ], + [ + 351, + 0.0363981461758428 + ], + [ + 353, + 0.04712526010382135 + ], + [ + 354, + 0.06444641795073798 + ], + [ + 356, + 0.03073182530123564 + ], + [ + 358, + 0.10303193336006078 + ], + [ + 361, + 0.20134646034950146 + ], + [ + 363, + 0.1011461364038478 + ], + [ + 368, + 0.09701628789676041 + ], + [ + 372, + 0.0709822018639264 + ], + [ + 377, + 0.025845640242904235 + ], + [ + 380, + 0.0 + ], + [ + 384, + 0.030543346790880943 + ], + [ + 390, + 0.051359155381979146 + ], + [ + 391, + 0.25241941350997293 + ], + [ + 395, + 0.05212744593626973 + ], + [ + 398, + 0.09832187990256622 + ], + [ + 403, + 0.07051710178806686 + ], + [ + 409, + 0.06606372753492169 + ], + [ + 410, + 0.3009417080451016 + ], + [ + 412, + 0.06797693251593309 + ], + [ + 414, + 0.08135507325387692 + ], + [ + 417, + 0.12566920952794408 + ], + [ + 420, + 0.13901338990100437 + ], + [ + 426, + 0.10936590159858586 + ], + [ + 430, + 0.09404793139518404 + ], + [ + 432, + 0.1292195901265978 + ], + [ + 435, + 0.2306492873326798 + ], + [ + 436, + 0.2456088606939956 + ], + [ + 437, + 0.18283241944223616 + ], + [ + 439, + 0.29815605212554297 + ], + [ + 441, + 0.23097296046830185 + ], + [ + 443, + 0.4158399456489749 + ], + [ + 446, + 0.41516305775402584 + ], + [ + 450, + 0.3155420648825082 + ], + [ + 458, + 0.6965463976886392 + ], + [ + 460, + 0.4247554966615132 + ], + [ + 464, + 0.0979022203945911 + ], + [ + 466, + 0.09911331989305076 + ], + [ + 468, + 0.12217050447087957 + ], + [ + 469, + 0.114005776148106 + ], + [ + 471, + 0.06286182377182029 + ], + [ + 477, + 0.19031719670160294 + ], + [ + 481, + 0.1316064309247923 + ], + [ + 488, + 0.161229353039865 + ], + [ + 492, + 0.09874635796539713 + ], + [ + 497, + 0.062391863388126925 + ], + [ + 503, + 0.09316307760723061 + ], + [ + 506, + 0.10362436365720752 + ], + [ + 515, + 0.08963354626744069 + ], + [ + 522, + 0.04883027706603688 + ], + [ + 528, + 0.07936210868452874 + ], + [ + 532, + 0.12529239557724262 + ], + [ + 535, + 0.08060181547396937 + ], + [ + 536, + 0.1561455469065408 + ], + [ + 538, + 0.09255343115979921 + ], + [ + 540, + 0.0856002919729339 + ], + [ + 541, + 0.10308551195529264 + ], + [ + 546, + 0.14531846345964985 + ], + [ + 547, + 0.19049510174663598 + ], + [ + 548, + 0.0997818194950966 + ], + [ + 549, + 0.07021968084231237 + ], + [ + 550, + 0.3480993463182657 + ], + [ + 554, + 0.10324058823398233 + ], + [ + 556, + 0.07492963383685787 + ], + [ + 558, + 0.12937755837077303 + ], + [ + 559, + 0.040252967088305 + ], + [ + 561, + 0.03549400960183353 + ], + [ + 565, + 0.26994104972662497 + ], + [ + 570, + 0.12661951056503007 + ], + [ + 573, + 0.08638627953482893 + ], + [ + 577, + 0.019656842348908278 + ], + [ + 579, + 0.019575931438258184 + ], + [ + 582, + 0.013202653529650298 + ], + [ + 586, + 0.09822085523363226 + ], + [ + 592, + 0.07645649024159129 + ], + [ + 596, + 0.05663167273636014 + ], + [ + 601, + 0.07052859539067023 + ], + [ + 608, + 0.05882734040967696 + ], + [ + 613, + 0.05640739628083418 + ], + [ + 619, + 0.028767304589525 + ], + [ + 624, + 0.03612904892360795 + ], + [ + 626, + 0.06621096385604384 + ], + [ + 629, + 0.049984655517983166 + ], + [ + 630, + 0.08177726076637439 + ], + [ + 634, + 0.06016766928579051 + ], + [ + 636, + 0.03761669267519468 + ], + [ + 637, + 0.07507392961543762 + ], + [ + 638, + 0.05188972688793157 + ], + [ + 639, + 0.05061636211864464 + ], + [ + 641, + 0.1358798517662996 + ], + [ + 642, + 0.5794157560895158 + ], + [ + 645, + 0.1863456865335671 + ], + [ + 650, + 0.09450931054448405 + ], + [ + 654, + 0.09838868542731329 + ], + [ + 660, + 0.17604903643540157 + ], + [ + 663, + 0.19952519098178378 + ], + [ + 667, + 0.26667386760334494 + ], + [ + 671, + 0.3015950965011766 + ] + ], + [ + [ + 50, + 1.0 + ], + [ + 58, + 0.28250185801061556 + ], + [ + 61, + 0.39000549708149734 + ], + [ + 66, + 0.12596986576866023 + ], + [ + 74, + 0.2816123838070215 + ], + [ + 75, + 0.5603435175132419 + ], + [ + 78, + 0.11512106214147061 + ], + [ + 84, + 0.1817564501609516 + ], + [ + 85, + 0.21175530315352192 + ], + [ + 86, + 0.16516032217936194 + ], + [ + 87, + 0.06886520596587029 + ], + [ + 88, + 0.0612483656531206 + ], + [ + 97, + 0.25037626203962254 + ], + [ + 100, + 0.3196328584318351 + ], + [ + 105, + 0.12123215970269022 + ], + [ + 110, + 0.1308997806986437 + ], + [ + 115, + 0.24064951322108963 + ], + [ + 117, + 0.15556794924753145 + ], + [ + 123, + 0.06276201239296542 + ], + [ + 125, + 0.12500403113804623 + ], + [ + 126, + 0.09407859025250401 + ], + [ + 127, + 0.03891256276510562 + ], + [ + 128, + 0.01735585732879202 + ], + [ + 133, + 0.08125850221548134 + ], + [ + 137, + 0.07355438387833406 + ], + [ + 139, + 0.08887554082071152 + ], + [ + 140, + 0.09865531703474457 + ], + [ + 142, + 0.15690094486291623 + ], + [ + 144, + 0.13216565971379668 + ], + [ + 145, + 0.049832729074244735 + ], + [ + 149, + 0.0258866954749916 + ], + [ + 157, + 0.0595632306112861 + ], + [ + 159, + 0.08176769068504516 + ], + [ + 163, + 0.09776444403954189 + ], + [ + 168, + 0.08481505750269486 + ], + [ + 173, + 0.07513491592072181 + ], + [ + 178, + 0.07640369220763349 + ], + [ + 188, + 0.15930500557871682 + ], + [ + 189, + 0.21306351366118403 + ], + [ + 193, + 0.02508232915837283 + ], + [ + 196, + 0.04600690114579576 + ], + [ + 198, + 0.05509029290729457 + ], + [ + 199, + 0.1920370190341621 + ], + [ + 202, + 0.015101295207889956 + ], + [ + 205, + 0.03164984231127289 + ], + [ + 206, + 0.24492401961043253 + ], + [ + 216, + 0.013981437031164633 + ], + [ + 217, + 0.3277319562159159 + ], + [ + 221, + 0.17824565972712345 + ], + [ + 225, + 0.17716677278803028 + ], + [ + 232, + 0.03461181410755645 + ], + [ + 239, + 0.09669755152217359 + ], + [ + 240, + 0.5261129869849375 + ], + [ + 245, + 0.39286560696379397 + ], + [ + 249, + 0.6808426517463242 + ], + [ + 255, + 0.2687309794671408 + ], + [ + 259, + 0.1985326306687671 + ], + [ + 266, + 0.15367256950213778 + ], + [ + 270, + 0.13468743704658634 + ], + [ + 274, + 0.2520144678913616 + ], + [ + 279, + 0.07530144560453535 + ], + [ + 285, + 0.09968395611538867 + ], + [ + 292, + 0.08072091945248738 + ], + [ + 295, + 0.09803765858998674 + ], + [ + 297, + 0.1561584026459154 + ], + [ + 304, + 0.10157184466707318 + ], + [ + 315, + 0.1627427302189796 + ], + [ + 316, + 0.2896436634406995 + ], + [ + 321, + 0.0921313176102594 + ], + [ + 322, + 0.16996444844707218 + ], + [ + 323, + 0.12540481136919873 + ], + [ + 324, + 0.04835401870937436 + ], + [ + 330, + 0.06212036926800556 + ], + [ + 333, + 0.1829411736323766 + ], + [ + 334, + 0.20325409016787047 + ], + [ + 335, + 0.08171180012399622 + ], + [ + 336, + 0.07982507115920222 + ], + [ + 337, + 0.09920151569406116 + ], + [ + 338, + 0.10883574994074252 + ], + [ + 339, + 0.37710249915914884 + ], + [ + 343, + 0.12479595082294871 + ], + [ + 351, + 0.11712226897694868 + ], + [ + 356, + 0.14650591941827226 + ], + [ + 361, + 0.08493485076409878 + ], + [ + 369, + 0.15823989321671117 + ], + [ + 372, + 0.1693385698841575 + ], + [ + 377, + 0.10972488432442236 + ], + [ + 384, + 0.12706282173063715 + ], + [ + 389, + 0.131396914528399 + ], + [ + 391, + 0.11742271046806124 + ], + [ + 394, + 0.21471625128822155 + ], + [ + 396, + 0.1841233219654596 + ], + [ + 399, + 0.2289733368990379 + ], + [ + 400, + 0.2493479208543213 + ], + [ + 401, + 0.2740535685352059 + ], + [ + 402, + 0.05621773734956242 + ], + [ + 403, + 0.07493224761562894 + ], + [ + 404, + 0.09253986676263193 + ], + [ + 405, + 0.3879120020655353 + ], + [ + 412, + 0.20051127775867933 + ], + [ + 415, + 0.25731072201594724 + ], + [ + 416, + 0.3298195245935574 + ], + [ + 417, + 0.3458362848149927 + ], + [ + 418, + 0.03556555365035201 + ], + [ + 419, + 0.04913791164563911 + ], + [ + 420, + 0.03585411899782092 + ], + [ + 424, + 0.15105385682310773 + ], + [ + 433, + 0.1332686824999411 + ], + [ + 434, + 0.28465385900688783 + ], + [ + 435, + 0.12639932049968194 + ], + [ + 437, + 0.07296766395936324 + ], + [ + 439, + 0.06749605651900441 + ], + [ + 441, + 0.1110175352250781 + ], + [ + 443, + 0.09668843742301757 + ], + [ + 446, + 0.16026001132699513 + ], + [ + 447, + 0.17959077235645476 + ], + [ + 448, + 0.2452598636656962 + ], + [ + 449, + 0.0 + ], + [ + 450, + 0.013077701694317554 + ], + [ + 451, + 0.019453661207735368 + ], + [ + 452, + 0.2047994650479455 + ], + [ + 457, + 0.0669624017061973 + ], + [ + 458, + 0.13299933848490098 + ], + [ + 459, + 0.08476951838056286 + ], + [ + 460, + 0.07979520513828404 + ], + [ + 461, + 0.22939197636332054 + ], + [ + 462, + 0.14197035137568634 + ], + [ + 463, + 0.06642294364845465 + ], + [ + 464, + 0.01528501756333145 + ], + [ + 468, + 0.10786104301435015 + ], + [ + 469, + 0.13672633047320845 + ], + [ + 470, + 0.001872787280851116 + ], + [ + 471, + 0.23154514407756205 + ], + [ + 484, + 0.09373185010277509 + ], + [ + 485, + 0.40188786703604207 + ], + [ + 489, + 0.1508457688279392 + ], + [ + 496, + 0.22563126132486758 + ], + [ + 504, + 0.23211120702768737 + ], + [ + 515, + 0.2700265385091903 + ], + [ + 516, + 0.3745122679108169 + ], + [ + 517, + 0.35682891986508447 + ], + [ + 521, + 0.2725242215992069 + ], + [ + 524, + 0.2790526749549601 + ], + [ + 528, + 0.37088986658877476 + ], + [ + 532, + 0.14046856146114414 + ], + [ + 535, + 0.16122081637237465 + ], + [ + 536, + 0.11180776570546348 + ], + [ + 537, + 0.07842953121581941 + ], + [ + 538, + 0.010444740717107024 + ], + [ + 539, + 0.037199917114003805 + ], + [ + 540, + 0.08919849801525932 + ], + [ + 541, + 0.2420346837822343 + ], + [ + 546, + 0.0691553252002561 + ], + [ + 550, + 0.036375894665453025 + ], + [ + 558, + 0.060940871752604764 + ], + [ + 563, + 0.07463457810351791 + ], + [ + 566, + 0.09597159526952906 + ], + [ + 571, + 0.06494966159186374 + ], + [ + 576, + 0.05735064892178563 + ], + [ + 582, + 0.09055604304608467 + ], + [ + 587, + 0.06524926887776274 + ], + [ + 588, + 0.4457461388351405 + ], + [ + 591, + 0.10348623279610285 + ], + [ + 595, + 0.18045118321209436 + ], + [ + 599, + 0.2053461519148146 + ], + [ + 600, + 0.2603835147272986 + ], + [ + 601, + 0.16984113715476062 + ], + [ + 602, + 0.16814580976788157 + ], + [ + 605, + 0.19144165720760042 + ], + [ + 611, + 0.2324665524827523 + ], + [ + 614, + 0.2502655206940119 + ], + [ + 618, + 0.23672916472856936 + ], + [ + 624, + 0.38327235555354816 + ], + [ + 628, + 0.26563520396874124 + ], + [ + 629, + 0.5669040453538274 + ], + [ + 630, + 0.5002345635607698 + ], + [ + 631, + 0.22126106605366108 + ] + ], + [ + [ + 53, + 0.6712737942124958 + ], + [ + 55, + 0.457854047997657 + ], + [ + 59, + 0.15928209239074154 + ], + [ + 63, + 0.2737103765276284 + ], + [ + 70, + 0.5025454390909344 + ], + [ + 73, + 0.5657331432583151 + ], + [ + 77, + 0.42813426504297253 + ], + [ + 85, + 0.9305502037410829 + ], + [ + 87, + 0.4854936962085817 + ], + [ + 90, + 0.3683676379119444 + ], + [ + 92, + 0.4414451326929575 + ], + [ + 94, + 0.6339448065809459 + ], + [ + 96, + 0.9229272722088816 + ], + [ + 110, + 0.08647991136493914 + ], + [ + 121, + 0.13338368363107492 + ], + [ + 124, + 0.07537951069352994 + ], + [ + 133, + 0.06140880982138963 + ], + [ + 141, + 0.11321061049554774 + ], + [ + 145, + 0.1073660290021352 + ], + [ + 156, + 0.10443504945915374 + ], + [ + 157, + 0.24038119933932514 + ], + [ + 158, + 0.16684652412910006 + ], + [ + 159, + 0.08321088684877573 + ], + [ + 160, + 0.02271059736371058 + ], + [ + 161, + 0.032690341931380235 + ], + [ + 162, + 0.03971764571983559 + ], + [ + 163, + 0.24244578691268842 + ], + [ + 165, + 0.1972614063100767 + ], + [ + 166, + 0.034631562178948624 + ], + [ + 170, + 0.03849604514722453 + ], + [ + 178, + 0.10073654173596354 + ], + [ + 187, + 0.06090298962121467 + ], + [ + 194, + 0.10827623756773148 + ], + [ + 197, + 0.12315201985059518 + ], + [ + 201, + 0.09447394604926132 + ], + [ + 209, + 0.17536789504587377 + ], + [ + 211, + 0.12927731920833144 + ], + [ + 214, + 0.10500040066880884 + ], + [ + 216, + 0.03403383164237537 + ], + [ + 218, + 0.01857355434486245 + ], + [ + 220, + 0.03126944745437756 + ], + [ + 234, + 0.0 + ], + [ + 245, + 0.0028340240531626953 + ], + [ + 248, + 0.07163648532864678 + ], + [ + 257, + 0.0056916711124823886 + ], + [ + 265, + 0.04103182152293015 + ], + [ + 269, + 0.08145151000823708 + ], + [ + 280, + 0.04980723339324394 + ], + [ + 281, + 0.43044957006702356 + ], + [ + 285, + 0.278194160846185 + ], + [ + 286, + 0.5051935188368262 + ], + [ + 287, + 0.17117391906336102 + ], + [ + 288, + 0.2217747444500541 + ], + [ + 289, + 0.18589215731546432 + ], + [ + 290, + 0.10472613776286309 + ], + [ + 291, + 0.13597008254698145 + ], + [ + 292, + 0.10617152704551236 + ], + [ + 293, + 0.1404536966816317 + ], + [ + 299, + 0.2831284259527185 + ], + [ + 303, + 0.20904065610157183 + ], + [ + 314, + 0.2472598093647421 + ], + [ + 317, + 0.1611238726087893 + ], + [ + 322, + 0.18654585780325694 + ], + [ + 325, + 0.24024548400435827 + ], + [ + 329, + 0.34650916266085646 + ], + [ + 337, + 0.5994556069689077 + ], + [ + 343, + 0.08473209372385021 + ], + [ + 352, + 0.06339866830874824 + ], + [ + 355, + 0.10550756467317807 + ], + [ + 357, + 0.148898107642446 + ], + [ + 370, + 0.06268836720274427 + ], + [ + 381, + 0.22560708341949273 + ], + [ + 382, + 0.3485092990911218 + ], + [ + 391, + 0.07580937730471453 + ], + [ + 395, + 0.16835526020462124 + ], + [ + 402, + 0.13565715688246244 + ], + [ + 407, + 0.10161476380911103 + ], + [ + 412, + 0.1384126671691917 + ], + [ + 423, + 0.1797662603781806 + ], + [ + 425, + 0.32712822501726485 + ], + [ + 426, + 0.10866333078796107 + ], + [ + 428, + 0.11747268813703063 + ], + [ + 429, + 0.14735160191021351 + ], + [ + 430, + 0.125999600017705 + ], + [ + 437, + 0.06554268188961544 + ], + [ + 441, + 0.055680932430039534 + ], + [ + 446, + 0.04649153700443899 + ], + [ + 447, + 0.30863873286849663 + ], + [ + 448, + 0.00847739082691284 + ], + [ + 449, + 0.050123145523283355 + ], + [ + 450, + 0.08096353665922432 + ], + [ + 456, + 0.09301963056268615 + ], + [ + 459, + 0.2770550093417015 + ], + [ + 468, + 0.05959126108841027 + ], + [ + 473, + 0.16242414911547595 + ], + [ + 474, + 0.509663513876152 + ], + [ + 480, + 0.09020314321500905 + ], + [ + 484, + 0.09389075728285362 + ], + [ + 488, + 0.36117612036896307 + ], + [ + 494, + 0.0554174708368838 + ], + [ + 500, + 0.1383830316249619 + ], + [ + 506, + 0.09796381224734935 + ], + [ + 512, + 0.13509835180676674 + ], + [ + 514, + 0.26854146754392244 + ], + [ + 518, + 0.18853806687919 + ], + [ + 520, + 0.3482050092882702 + ], + [ + 521, + 0.031360885289712065 + ], + [ + 522, + 0.06509755667502179 + ], + [ + 523, + 0.10075244008736414 + ], + [ + 524, + 0.1717951392827852 + ], + [ + 528, + 0.18775567090532444 + ], + [ + 532, + 0.045766269901772565 + ], + [ + 536, + 0.06914143218524398 + ], + [ + 538, + 0.3966605403563724 + ], + [ + 539, + 0.09644338171114035 + ], + [ + 540, + 0.10748237270593902 + ], + [ + 541, + 0.27545338136541314 + ], + [ + 543, + 0.6616240377425999 + ], + [ + 547, + 0.1778430892710657 + ], + [ + 551, + 0.24074953747270342 + ], + [ + 556, + 0.3084867605923712 + ], + [ + 557, + 0.6012420312273985 + ], + [ + 561, + 0.15113112529561235 + ], + [ + 563, + 0.18445857862509327 + ], + [ + 566, + 0.2526249535796497 + ], + [ + 570, + 0.2886332184121985 + ], + [ + 575, + 0.13335691407529907 + ], + [ + 581, + 0.29007842214053275 + ], + [ + 585, + 0.7375668253150317 + ], + [ + 590, + 0.23994004379451894 + ], + [ + 592, + 0.2687626792492082 + ], + [ + 597, + 0.21649898662426986 + ], + [ + 604, + 0.26168318776819055 + ], + [ + 607, + 0.5264751380141772 + ], + [ + 613, + 0.3351403362475093 + ], + [ + 617, + 0.2913176669149428 + ], + [ + 625, + 0.17077053987823052 + ], + [ + 630, + 0.15714216724390562 + ], + [ + 633, + 0.42001967096104026 + ], + [ + 637, + 0.39551057624845204 + ], + [ + 643, + 0.26527948874672314 + ], + [ + 647, + 0.26501824053991757 + ], + [ + 652, + 0.13987017171056446 + ], + [ + 653, + 0.21908204775649212 + ], + [ + 656, + 0.16953523886894364 + ], + [ + 659, + 0.28458289885936644 + ], + [ + 660, + 0.8940268004094555 + ], + [ + 672, + 0.42471142525463995 + ], + [ + 677, + 0.5742887355027214 + ], + [ + 682, + 0.23557017734616723 + ], + [ + 689, + 0.381459013332661 + ], + [ + 691, + 0.30386240024092853 + ], + [ + 695, + 0.2786121339818317 + ], + [ + 698, + 0.5296515202128992 + ], + [ + 702, + 0.7918558314352621 + ], + [ + 704, + 1.0 + ] + ] + ], + [ + [ + [ + 34, + 1.0 + ], + [ + 44, + 0.6926284511132109 + ], + [ + 48, + 0.3578133800390095 + ], + [ + 52, + 0.14112404905785492 + ], + [ + 59, + 0.27206440832317513 + ], + [ + 64, + 0.20517615572139772 + ], + [ + 73, + 0.28289728881330745 + ], + [ + 76, + 0.4899738184809815 + ], + [ + 84, + 0.6511530828119431 + ], + [ + 85, + 0.6212546772963489 + ], + [ + 86, + 0.3109953927702756 + ], + [ + 87, + 0.28996170209975974 + ], + [ + 88, + 0.7555110543449992 + ], + [ + 89, + 0.544327422261243 + ], + [ + 90, + 0.9136114933155891 + ], + [ + 91, + 0.09337503524673675 + ], + [ + 92, + 0.0999189362141588 + ], + [ + 93, + 0.11846785663069882 + ], + [ + 94, + 0.4467317507252544 + ], + [ + 97, + 0.18477275491663922 + ], + [ + 99, + 0.1919113940791941 + ], + [ + 100, + 0.3256689146734668 + ], + [ + 104, + 0.15675091752214304 + ], + [ + 109, + 0.09531433306075864 + ], + [ + 116, + 0.1010161267600662 + ], + [ + 118, + 0.12250509462022374 + ], + [ + 121, + 0.14620549132789756 + ], + [ + 126, + 0.2721382924605474 + ], + [ + 131, + 0.19736103365112875 + ], + [ + 135, + 0.13695688901337755 + ], + [ + 149, + 0.22850845556002064 + ], + [ + 154, + 0.2592685345248994 + ], + [ + 155, + 0.34895378620554657 + ], + [ + 158, + 0.20247814227113864 + ], + [ + 159, + 0.15305866481082603 + ], + [ + 160, + 0.1588438467882212 + ], + [ + 161, + 0.11254837337463126 + ], + [ + 162, + 0.1364566623666695 + ], + [ + 163, + 0.2065051998261819 + ], + [ + 164, + 0.24905355315008937 + ], + [ + 167, + 0.10333917885352771 + ], + [ + 174, + 0.2154658326326447 + ], + [ + 175, + 0.16208050213940273 + ], + [ + 176, + 0.10181596120002852 + ], + [ + 177, + 0.10624876459106539 + ], + [ + 183, + 0.23712737180899318 + ], + [ + 186, + 0.21170415825526676 + ], + [ + 195, + 0.1997819566877897 + ], + [ + 199, + 0.17436182454072616 + ], + [ + 213, + 0.19713645584221476 + ], + [ + 221, + 0.17865023856922493 + ], + [ + 222, + 0.32164275749380494 + ], + [ + 226, + 0.22361926331701926 + ], + [ + 230, + 0.22779972748196245 + ], + [ + 233, + 0.19466328305025143 + ], + [ + 239, + 0.15336179402447686 + ], + [ + 242, + 0.12160791130635061 + ], + [ + 246, + 0.3093449210241144 + ], + [ + 249, + 0.2927388815353542 + ], + [ + 253, + 0.18766380192815668 + ], + [ + 258, + 0.20135097498393864 + ], + [ + 262, + 0.3591290822350072 + ], + [ + 266, + 0.26586372646895484 + ], + [ + 268, + 0.3972962550512752 + ], + [ + 273, + 0.533173289859945 + ], + [ + 274, + 0.42431333535226723 + ], + [ + 275, + 0.7408880897930485 + ], + [ + 279, + 0.5121166527759659 + ], + [ + 282, + 0.20069138430783215 + ], + [ + 284, + 0.3322726890858245 + ], + [ + 287, + 0.1974986603755897 + ], + [ + 291, + 0.4295065638359288 + ], + [ + 297, + 0.4228486966267275 + ], + [ + 299, + 0.4007912915522553 + ], + [ + 303, + 0.0914980906240878 + ], + [ + 308, + 0.10196793215091161 + ], + [ + 322, + 0.1750723773048557 + ], + [ + 328, + 0.10292906018660776 + ], + [ + 332, + 0.10136208444011019 + ], + [ + 337, + 0.08226039599146036 + ], + [ + 340, + 0.1766091989924531 + ], + [ + 347, + 0.19028327390109118 + ], + [ + 351, + 0.22943837528703045 + ], + [ + 352, + 0.2823630326955566 + ], + [ + 355, + 0.053447655718710414 + ], + [ + 362, + 0.09640550794405187 + ], + [ + 368, + 0.061881072272188115 + ], + [ + 376, + 0.05466651812065412 + ], + [ + 380, + 0.0 + ], + [ + 388, + 0.049204850471914835 + ], + [ + 392, + 0.07549942151019491 + ], + [ + 399, + 0.01586585381635042 + ], + [ + 400, + 0.23681782243984295 + ], + [ + 404, + 0.026907025849727907 + ], + [ + 410, + 0.04578832380999506 + ], + [ + 414, + 0.03422875285378108 + ], + [ + 420, + 0.03905673673068704 + ], + [ + 425, + 0.03872413243930608 + ], + [ + 428, + 0.12848682628003075 + ], + [ + 432, + 0.13923077298251035 + ], + [ + 437, + 0.0760625305262842 + ], + [ + 440, + 0.13239875239436902 + ], + [ + 444, + 0.04339058729450429 + ], + [ + 447, + 0.07567181426181506 + ], + [ + 450, + 0.07571267703422693 + ], + [ + 457, + 0.15824237897550514 + ], + [ + 459, + 0.06520157850619596 + ], + [ + 460, + 0.04721047815928455 + ], + [ + 463, + 0.044878389868668475 + ], + [ + 469, + 0.05451248021193407 + ], + [ + 473, + 0.09306201342521937 + ], + [ + 482, + 0.07437525599978702 + ], + [ + 483, + 0.37466195410337777 + ], + [ + 486, + 0.18675381169359512 + ], + [ + 494, + 0.13858136680047567 + ], + [ + 495, + 0.19622920032015398 + ], + [ + 496, + 0.14293204535130807 + ], + [ + 497, + 0.08757616116089921 + ], + [ + 498, + 0.08816347846033432 + ], + [ + 499, + 0.16850793779935686 + ], + [ + 500, + 0.26015891721817724 + ], + [ + 503, + 0.13968428971007163 + ], + [ + 507, + 0.19564579605853058 + ], + [ + 515, + 0.4561305970688828 + ], + [ + 521, + 0.5039777661035755 + ], + [ + 522, + 0.2750747103639688 + ], + [ + 523, + 0.4880174330852478 + ], + [ + 527, + 0.3502724037044357 + ], + [ + 532, + 0.2743812449155516 + ], + [ + 535, + 0.22091769592733976 + ], + [ + 539, + 0.4384101749260289 + ], + [ + 542, + 0.5573361194571178 + ], + [ + 543, + 0.32413831781446634 + ], + [ + 544, + 0.18420006459411525 + ], + [ + 545, + 0.19296886145538322 + ], + [ + 546, + 0.10001079111541594 + ], + [ + 547, + 0.20316997739591655 + ], + [ + 550, + 0.21260744477321722 + ], + [ + 554, + 0.15974990423570334 + ], + [ + 568, + 0.12568057175018102 + ], + [ + 579, + 0.16113214940284876 + ], + [ + 582, + 0.15749033607802682 + ], + [ + 591, + 0.16778382805434752 + ], + [ + 599, + 0.18380975932040097 + ], + [ + 601, + 0.11354616909836382 + ], + [ + 612, + 0.15989526743885316 + ], + [ + 615, + 0.1509573890711531 + ], + [ + 619, + 0.06345220870778545 + ], + [ + 624, + 0.13739548990533304 + ], + [ + 625, + 0.26589034318958427 + ], + [ + 633, + 0.0747279172554797 + ], + [ + 634, + 0.6078776816296789 + ], + [ + 637, + 0.19923893416911978 + ] + ], + [ + [ + 38, + 0.5172134813087379 + ], + [ + 46, + 0.5678837943545396 + ], + [ + 55, + 0.47687563162673485 + ], + [ + 64, + 0.43621217203745516 + ], + [ + 73, + 0.6038949082786957 + ], + [ + 76, + 0.35154469245698233 + ], + [ + 82, + 0.26967989728194663 + ], + [ + 83, + 0.21315423992859 + ], + [ + 86, + 0.24478054928748513 + ], + [ + 91, + 0.3082436244843466 + ], + [ + 100, + 0.2384367836563298 + ], + [ + 101, + 0.31227329798254283 + ], + [ + 105, + 0.2582885285393537 + ], + [ + 112, + 0.37309842267817495 + ], + [ + 114, + 0.38770359309970615 + ], + [ + 119, + 0.7326441078224074 + ], + [ + 125, + 0.480801499263002 + ], + [ + 132, + 0.5894392692320899 + ], + [ + 135, + 0.3810651966384728 + ], + [ + 136, + 0.1834275530296149 + ], + [ + 137, + 0.13646729891349937 + ], + [ + 138, + 0.18194245599290856 + ], + [ + 144, + 0.2816769684371483 + ], + [ + 147, + 0.19470922979573196 + ], + [ + 148, + 0.20830389725283419 + ], + [ + 149, + 0.15633076866856535 + ], + [ + 150, + 0.20688703360549943 + ], + [ + 164, + 0.5074610733367977 + ], + [ + 165, + 0.5202799204971602 + ], + [ + 166, + 0.3545244374349224 + ], + [ + 169, + 0.157064786302609 + ], + [ + 172, + 0.13975999495923944 + ], + [ + 176, + 0.15262579557223363 + ], + [ + 179, + 0.1791938333288192 + ], + [ + 183, + 0.16616435749084776 + ], + [ + 187, + 0.27183107481772095 + ], + [ + 190, + 0.20518783707673974 + ], + [ + 192, + 0.25631907925250713 + ], + [ + 195, + 0.3570325989894132 + ], + [ + 198, + 0.18442966898365487 + ], + [ + 202, + 0.1343473521668515 + ], + [ + 203, + 0.14146563610940094 + ], + [ + 204, + 0.19501244053745026 + ], + [ + 205, + 0.20586221331092627 + ], + [ + 207, + 0.2710269127852475 + ], + [ + 215, + 0.2963587326603684 + ], + [ + 216, + 0.12477384394779144 + ], + [ + 221, + 0.08475965787814073 + ], + [ + 222, + 0.07596173347021863 + ], + [ + 223, + 0.06816651260304592 + ], + [ + 224, + 0.08526638079929953 + ], + [ + 225, + 0.09616405758618975 + ], + [ + 231, + 0.09219946614367053 + ], + [ + 234, + 0.06615915971750434 + ], + [ + 235, + 0.04462980612657327 + ], + [ + 236, + 0.02290836669642587 + ], + [ + 237, + 0.03283350914416332 + ], + [ + 238, + 0.03170603846075874 + ], + [ + 246, + 0.08721072689294194 + ], + [ + 247, + 0.2164627003855016 + ], + [ + 249, + 0.15857447429250474 + ], + [ + 251, + 0.048135820789428435 + ], + [ + 254, + 0.02322602533631539 + ], + [ + 264, + 0.03357824734944493 + ], + [ + 267, + 0.045333022715801495 + ], + [ + 276, + 0.05455044321813932 + ], + [ + 280, + 0.0451699638284426 + ], + [ + 287, + 0.07710213188808829 + ], + [ + 290, + 0.05239835994457135 + ], + [ + 296, + 0.043067905558239604 + ], + [ + 305, + 0.072614396155751 + ], + [ + 308, + 0.06574002070910617 + ], + [ + 312, + 0.07677700678192839 + ], + [ + 315, + 0.1073280850307967 + ], + [ + 319, + 0.03227848527024348 + ], + [ + 324, + 0.00960061585828795 + ], + [ + 329, + 0.025618461546332173 + ], + [ + 335, + 0.023900799492380373 + ], + [ + 340, + 0.03501941322362487 + ], + [ + 343, + 0.03231377820165339 + ], + [ + 347, + 0.04330680968245242 + ], + [ + 355, + 0.059628308124624066 + ], + [ + 356, + 0.27149485429445097 + ], + [ + 360, + 0.3396303654541254 + ], + [ + 363, + 0.5366013792336005 + ], + [ + 365, + 0.4767654047129948 + ], + [ + 368, + 0.9738833255277315 + ], + [ + 370, + 1.0 + ], + [ + 374, + 0.11261231323091295 + ], + [ + 378, + 0.01856182689997216 + ], + [ + 383, + 0.02139054716517702 + ], + [ + 385, + 0.013271585578858342 + ], + [ + 392, + 0.0175618208667225 + ], + [ + 397, + 0.011134551277158941 + ], + [ + 405, + 0.026429507795878167 + ], + [ + 412, + 0.012153073586179312 + ], + [ + 416, + 0.011721961147668603 + ], + [ + 420, + 0.007185669457009916 + ], + [ + 425, + 0.007525724987000935 + ], + [ + 429, + 0.022244795754479393 + ], + [ + 433, + 0.031886235312052015 + ], + [ + 434, + 0.03954194560350877 + ], + [ + 438, + 0.008349873567947244 + ], + [ + 443, + 0.0 + ], + [ + 445, + 0.013516876306223674 + ], + [ + 456, + 0.014027348752953353 + ], + [ + 460, + 0.019562619360857472 + ], + [ + 461, + 0.08127958175122825 + ], + [ + 466, + 0.06089455767859192 + ], + [ + 469, + 0.04486430857591081 + ], + [ + 479, + 0.03575616115724626 + ], + [ + 487, + 0.09922982415780371 + ], + [ + 489, + 0.02427872945082873 + ], + [ + 492, + 0.0298422435622692 + ], + [ + 496, + 0.022844296866067627 + ], + [ + 498, + 0.04594661500180438 + ], + [ + 504, + 0.029640263438288437 + ], + [ + 507, + 0.035682225651870514 + ], + [ + 511, + 0.04370067329398424 + ], + [ + 514, + 0.02896058153150867 + ], + [ + 520, + 0.02268411245953504 + ], + [ + 525, + 0.01773880631686011 + ], + [ + 530, + 0.023817636136668305 + ], + [ + 536, + 0.0563483014713474 + ], + [ + 539, + 0.042582197528107636 + ], + [ + 543, + 0.013413256906637328 + ], + [ + 550, + 0.027732085763029102 + ], + [ + 552, + 0.11748089403293832 + ], + [ + 557, + 0.05880899731011285 + ], + [ + 560, + 0.02462449359148104 + ], + [ + 565, + 0.0834000340062221 + ], + [ + 568, + 0.037189713656716535 + ], + [ + 572, + 0.0329026189456305 + ], + [ + 578, + 0.03346362673941117 + ], + [ + 585, + 0.03746494305104519 + ], + [ + 588, + 0.03658233943763533 + ], + [ + 592, + 0.03255186900269806 + ], + [ + 598, + 0.042495117369651426 + ], + [ + 601, + 0.04077791301596535 + ], + [ + 610, + 0.04743495231446207 + ] + ], + [ + [ + 38, + 0.4972872095070381 + ], + [ + 46, + 0.49960368537551364 + ], + [ + 55, + 0.3509970038257997 + ], + [ + 64, + 0.2797026835489436 + ], + [ + 73, + 0.38444597945156916 + ], + [ + 76, + 0.2908706514694423 + ], + [ + 82, + 0.45972628746453936 + ], + [ + 83, + 0.3504584670017485 + ], + [ + 86, + 0.6022168671698347 + ], + [ + 91, + 0.17241475278127324 + ], + [ + 100, + 0.16317979937999227 + ], + [ + 101, + 0.31840026219656936 + ], + [ + 105, + 0.25655081804098484 + ], + [ + 112, + 0.39810536400637453 + ], + [ + 114, + 0.48916310148393516 + ], + [ + 119, + 1.0 + ], + [ + 125, + 0.4026884921230187 + ], + [ + 132, + 0.5051258233222035 + ], + [ + 135, + 0.32581517565842816 + ], + [ + 136, + 0.22432401141922673 + ], + [ + 137, + 0.14753167522827482 + ], + [ + 138, + 0.19701160410425952 + ], + [ + 144, + 0.19655242306255516 + ], + [ + 147, + 0.19817386949909893 + ], + [ + 148, + 0.2022030768333207 + ], + [ + 149, + 0.16151329292890282 + ], + [ + 150, + 0.2032244881322951 + ], + [ + 164, + 0.4347571674843206 + ], + [ + 165, + 0.40898152045539254 + ], + [ + 166, + 0.2881677344545708 + ], + [ + 169, + 0.18709599659214665 + ], + [ + 172, + 0.23823139286069694 + ], + [ + 176, + 0.25466394041561374 + ], + [ + 179, + 0.25365436912909184 + ], + [ + 183, + 0.23812621929754238 + ], + [ + 187, + 0.29226325514123164 + ], + [ + 190, + 0.163858821368744 + ], + [ + 192, + 0.20399817591471225 + ], + [ + 195, + 0.2682411010638471 + ], + [ + 198, + 0.1832063043897359 + ], + [ + 202, + 0.15684798162244998 + ], + [ + 203, + 0.16758142280634306 + ], + [ + 204, + 0.18220203094909942 + ], + [ + 205, + 0.23275481838012244 + ], + [ + 207, + 0.2217509975977875 + ], + [ + 215, + 0.16599379987826662 + ], + [ + 216, + 0.10767698880853928 + ], + [ + 221, + 0.08866030489516845 + ], + [ + 222, + 0.07909158242669562 + ], + [ + 223, + 0.06819595720822108 + ], + [ + 224, + 0.07244421613546533 + ], + [ + 225, + 0.11775288671191159 + ], + [ + 231, + 0.03276346383634167 + ], + [ + 234, + 0.04972643357363754 + ], + [ + 235, + 0.035152586514984874 + ], + [ + 236, + 0.014002922587679753 + ], + [ + 237, + 0.02699141417260383 + ], + [ + 238, + 0.024735344896638478 + ], + [ + 246, + 0.05846708815757151 + ], + [ + 247, + 0.09850962187534876 + ], + [ + 249, + 0.11212951544344711 + ], + [ + 251, + 0.02267335660139052 + ], + [ + 254, + 0.032536581371757134 + ], + [ + 264, + 0.03220218971902799 + ], + [ + 267, + 0.03054952593006797 + ], + [ + 276, + 0.029853677350965693 + ], + [ + 280, + 0.04029132250337674 + ], + [ + 287, + 0.03336140859146029 + ], + [ + 290, + 0.0444007971265016 + ], + [ + 296, + 0.022016357257292646 + ], + [ + 305, + 0.034928935287496445 + ], + [ + 308, + 0.0305454961260769 + ], + [ + 312, + 0.06735903996266238 + ], + [ + 315, + 0.02637838651232712 + ], + [ + 319, + 0.021524437650416933 + ], + [ + 324, + 0.006667420389192227 + ], + [ + 329, + 0.007294194928302301 + ], + [ + 335, + 0.020734501062763956 + ], + [ + 340, + 0.01343492928155909 + ], + [ + 343, + 0.021203123022854646 + ], + [ + 347, + 0.03300230220247316 + ], + [ + 355, + 0.030017362532219316 + ], + [ + 356, + 0.1761269722522835 + ], + [ + 360, + 0.27190024215942504 + ], + [ + 363, + 0.21031661183381403 + ], + [ + 365, + 0.13967365022768422 + ], + [ + 368, + 0.3052046099751957 + ], + [ + 370, + 0.30401503595026746 + ], + [ + 374, + 0.08221952148462015 + ], + [ + 378, + 0.02142315205643365 + ], + [ + 383, + 0.010968201156439405 + ], + [ + 385, + 0.03436586674721461 + ], + [ + 392, + 0.007778189049007493 + ], + [ + 397, + 0.01048322124595546 + ], + [ + 405, + 0.008899183241915411 + ], + [ + 412, + 0.005805274769734936 + ], + [ + 416, + 0.01985531646149863 + ], + [ + 420, + 0.01420843439266741 + ], + [ + 425, + 0.02003745146032747 + ], + [ + 429, + 0.09746946708309155 + ], + [ + 433, + 0.005262721141440739 + ], + [ + 434, + 0.0645674542649444 + ], + [ + 438, + 0.015923416257822526 + ], + [ + 443, + 0.0 + ], + [ + 445, + 0.03128258490042238 + ], + [ + 456, + 0.005135777847234632 + ], + [ + 460, + 0.008330456967134107 + ], + [ + 461, + 0.07478896949998094 + ], + [ + 466, + 0.030768765144485122 + ], + [ + 469, + 0.027162623592202312 + ], + [ + 479, + 0.01489526851846411 + ], + [ + 487, + 0.043395228473492795 + ], + [ + 489, + 0.009077525631022586 + ], + [ + 492, + 0.0411482190772974 + ], + [ + 496, + 0.01346342924289858 + ], + [ + 498, + 0.04298641020004345 + ], + [ + 504, + 0.004471469960438473 + ], + [ + 507, + 0.03366650822914363 + ], + [ + 511, + 0.0076463741169784894 + ], + [ + 514, + 0.023404463618293618 + ], + [ + 520, + 0.00808118026623506 + ], + [ + 525, + 0.005038655098256497 + ], + [ + 530, + 0.01609929692242857 + ], + [ + 536, + 0.02725641901235337 + ], + [ + 539, + 0.040313350552441736 + ], + [ + 543, + 0.018115171885327463 + ], + [ + 550, + 0.006858794490609952 + ], + [ + 552, + 0.08115573449873184 + ], + [ + 557, + 0.04053364746086193 + ], + [ + 560, + 0.013712893000184022 + ], + [ + 565, + 0.03302846196106695 + ], + [ + 568, + 0.017805057495495897 + ], + [ + 572, + 0.0503307321830525 + ], + [ + 578, + 0.009026566286326148 + ], + [ + 585, + 0.00835223971056996 + ], + [ + 588, + 0.03442917610395951 + ], + [ + 592, + 0.04064310131636564 + ], + [ + 598, + 0.025766139318408987 + ], + [ + 601, + 0.03366976126744276 + ], + [ + 610, + 0.048352442434978465 + ] + ], + [ + [ + 38, + 0.5217588577322185 + ], + [ + 44, + 0.5283265552903518 + ], + [ + 46, + 1.0 + ], + [ + 48, + 0.8574076259601887 + ], + [ + 50, + 0.1619475125078302 + ], + [ + 52, + 0.21833635571240717 + ], + [ + 54, + 0.16200865536935707 + ], + [ + 58, + 0.18912929989908883 + ], + [ + 60, + 0.2077204611981338 + ], + [ + 62, + 0.25216053733108845 + ], + [ + 63, + 0.33404923757136945 + ], + [ + 65, + 0.30542525523537994 + ], + [ + 67, + 0.17462554131623384 + ], + [ + 69, + 0.13147351218245948 + ], + [ + 71, + 0.11005689265797086 + ], + [ + 75, + 0.0605529811164477 + ], + [ + 77, + 0.018136240532831097 + ], + [ + 79, + 0.03373388652589914 + ], + [ + 80, + 0.0652870731024817 + ], + [ + 82, + 0.06543646970699087 + ], + [ + 84, + 0.26840809389028897 + ], + [ + 88, + 0.29877407522044913 + ], + [ + 89, + 0.1005138966814576 + ], + [ + 90, + 0.09764898066918795 + ], + [ + 91, + 0.02766962996114017 + ], + [ + 100, + 0.09272265560802537 + ], + [ + 101, + 0.1686498552053605 + ], + [ + 102, + 0.20370067372265596 + ], + [ + 103, + 0.054071371255892106 + ], + [ + 104, + 0.10249931618894627 + ], + [ + 105, + 0.04966665742474567 + ], + [ + 106, + 0.2860020555430413 + ], + [ + 109, + 0.29093767020899974 + ], + [ + 117, + 0.2403256903517799 + ], + [ + 124, + 0.20999938512739458 + ], + [ + 138, + 0.2573262704011772 + ], + [ + 149, + 0.468164855154214 + ], + [ + 153, + 0.07660345877837708 + ], + [ + 161, + 0.058803938853243004 + ], + [ + 170, + 0.08724018500678803 + ], + [ + 176, + 0.09875892836924467 + ], + [ + 177, + 0.2835447297563595 + ], + [ + 181, + 0.08802909886275663 + ], + [ + 183, + 0.07931911959470712 + ], + [ + 185, + 0.1014404627730406 + ], + [ + 186, + 0.10426230087312412 + ], + [ + 188, + 0.17753867710540194 + ], + [ + 195, + 0.1036427017002839 + ], + [ + 198, + 0.11147473102023599 + ], + [ + 200, + 0.07588738366315971 + ], + [ + 208, + 0.17248976809508285 + ], + [ + 212, + 0.09648664981670552 + ], + [ + 215, + 0.12692586399259612 + ], + [ + 223, + 0.2755769823592437 + ], + [ + 227, + 0.11686204562013419 + ], + [ + 231, + 0.09181820019267649 + ], + [ + 236, + 0.19656203164662614 + ], + [ + 244, + 0.24530319445073095 + ], + [ + 249, + 0.09289883019961846 + ], + [ + 252, + 0.08173702407564952 + ], + [ + 259, + 0.0924801420336879 + ], + [ + 263, + 0.15042370153463466 + ], + [ + 268, + 0.12022716490526282 + ], + [ + 274, + 0.18572001516522108 + ], + [ + 276, + 0.28397668966001455 + ], + [ + 281, + 0.1311114471286637 + ], + [ + 284, + 0.06815981853642593 + ], + [ + 288, + 0.08028163761450303 + ], + [ + 289, + 0.08098581488665704 + ], + [ + 290, + 0.09369049509405959 + ], + [ + 291, + 0.034303503586350204 + ], + [ + 292, + 0.05989039670103213 + ], + [ + 293, + 0.0861915460464369 + ], + [ + 298, + 0.1566991540513329 + ], + [ + 304, + 0.10987457621634232 + ], + [ + 308, + 0.1470780293159859 + ], + [ + 309, + 0.31091678335443323 + ], + [ + 313, + 0.08938316712347975 + ], + [ + 318, + 0.10779401685367536 + ], + [ + 326, + 0.1606240590584185 + ], + [ + 329, + 0.07756619058313845 + ], + [ + 333, + 0.046239186332258625 + ], + [ + 338, + 0.11464346751226089 + ], + [ + 345, + 0.09305408576426569 + ], + [ + 349, + 0.054379768357705596 + ], + [ + 351, + 0.03357587125033473 + ], + [ + 353, + 0.03910994182624389 + ], + [ + 354, + 0.05908598508660837 + ], + [ + 356, + 0.0343246466043907 + ], + [ + 358, + 0.10256305647908723 + ], + [ + 361, + 0.23356170945971044 + ], + [ + 363, + 0.09859263695016801 + ], + [ + 368, + 0.13268750380294012 + ], + [ + 372, + 0.09311604871125999 + ], + [ + 377, + 0.06639913208568132 + ], + [ + 380, + 0.019901519080310136 + ], + [ + 384, + 0.04980367567389953 + ], + [ + 390, + 0.10892745096360376 + ], + [ + 391, + 0.20785582951755066 + ], + [ + 395, + 0.07092314236518642 + ], + [ + 398, + 0.1285720489861118 + ], + [ + 403, + 0.11205421976728196 + ], + [ + 409, + 0.10364637259761071 + ], + [ + 410, + 0.23726168150982835 + ], + [ + 412, + 0.08298055110472174 + ], + [ + 414, + 0.10865522695131184 + ], + [ + 417, + 0.14483812416434683 + ], + [ + 420, + 0.16984694914865361 + ], + [ + 426, + 0.15996719507766274 + ], + [ + 430, + 0.11579507954498877 + ], + [ + 432, + 0.13567684321500223 + ], + [ + 435, + 0.27046122515833915 + ], + [ + 436, + 0.1933235434671238 + ], + [ + 437, + 0.0839210884772467 + ], + [ + 439, + 0.27979165212190577 + ], + [ + 441, + 0.11450502443969637 + ], + [ + 443, + 0.20570238286991271 + ], + [ + 446, + 0.229667268403192 + ], + [ + 450, + 0.19088596649560508 + ], + [ + 458, + 0.30337376169594277 + ], + [ + 460, + 0.4515849788231198 + ], + [ + 464, + 0.18405580947874856 + ], + [ + 466, + 0.0903660941276219 + ], + [ + 468, + 0.1494150207788959 + ], + [ + 469, + 0.15644282908409873 + ], + [ + 471, + 0.19880041445277802 + ], + [ + 477, + 0.934305208261893 + ], + [ + 481, + 0.6291497486550109 + ], + [ + 488, + 0.9707942776664611 + ], + [ + 492, + 0.23051447926437196 + ], + [ + 497, + 0.1975920684065076 + ], + [ + 503, + 0.20313449479947102 + ], + [ + 506, + 0.3723931315942733 + ], + [ + 515, + 0.6678603093430352 + ], + [ + 522, + 0.5044194701274494 + ], + [ + 528, + 0.2716449599513489 + ], + [ + 532, + 0.2391021503802544 + ], + [ + 535, + 0.15315367301441696 + ], + [ + 536, + 0.1635866489369988 + ], + [ + 538, + 0.10254432481905304 + ], + [ + 540, + 0.11005959448669825 + ], + [ + 541, + 0.1687957823806362 + ], + [ + 546, + 0.20317780134849914 + ], + [ + 547, + 0.11964738646707483 + ], + [ + 548, + 0.06435110369579417 + ], + [ + 549, + 0.04362447550868415 + ], + [ + 550, + 0.2629961002241736 + ], + [ + 554, + 0.046184012360621716 + ], + [ + 556, + 0.0018421648648672645 + ], + [ + 558, + 0.013845861159503363 + ], + [ + 559, + 0.01213697363667092 + ], + [ + 561, + 0.0053549783790686496 + ], + [ + 565, + 0.13080376968140378 + ], + [ + 570, + 0.07197237575197767 + ], + [ + 573, + 0.049894318966934394 + ], + [ + 577, + 0.001584117077709169 + ], + [ + 579, + 0.0059285826097431405 + ], + [ + 582, + 0.0 + ], + [ + 586, + 0.0419423938198527 + ], + [ + 592, + 0.04376599715422027 + ], + [ + 596, + 0.042742229993473144 + ], + [ + 601, + 0.05341196250674997 + ], + [ + 608, + 0.036835546859593137 + ], + [ + 613, + 0.03060175101261719 + ], + [ + 619, + 0.01581988425139138 + ], + [ + 624, + 0.06772430273413783 + ], + [ + 626, + 0.21746049075573567 + ], + [ + 629, + 0.24542995254427485 + ], + [ + 630, + 0.057226301121268856 + ], + [ + 634, + 0.04376378543885112 + ], + [ + 636, + 0.04630305646666422 + ], + [ + 637, + 0.06846840776074295 + ], + [ + 638, + 0.021969684485478657 + ], + [ + 639, + 0.024581730921369897 + ], + [ + 641, + 0.09901426613010614 + ], + [ + 642, + 0.42352907620726266 + ], + [ + 645, + 0.06629900619726362 + ], + [ + 650, + 0.10465277415329129 + ], + [ + 654, + 0.06833308111222654 + ], + [ + 660, + 0.1013485278790409 + ], + [ + 663, + 0.08889563146024275 + ], + [ + 667, + 0.15891087705776108 + ], + [ + 671, + 0.16753901301521343 + ] + ], + [ + [ + 50, + 0.6837552413834016 + ], + [ + 58, + 0.26909414777209356 + ], + [ + 61, + 0.3830580415420201 + ], + [ + 66, + 0.16375592458383317 + ], + [ + 74, + 0.3443821416873057 + ], + [ + 75, + 0.5607540801554302 + ], + [ + 78, + 0.1520359461621958 + ], + [ + 84, + 0.22361121005087295 + ], + [ + 85, + 0.29309387202695114 + ], + [ + 86, + 0.22922747136471722 + ], + [ + 87, + 0.11776613391581606 + ], + [ + 88, + 0.12422881309221552 + ], + [ + 97, + 0.29228009337760735 + ], + [ + 100, + 0.33928684270301895 + ], + [ + 105, + 0.1804650036996808 + ], + [ + 110, + 0.1704634959496637 + ], + [ + 115, + 0.3577085901386311 + ], + [ + 117, + 0.2697063996205903 + ], + [ + 123, + 0.23149283928818076 + ], + [ + 125, + 0.2196567798632376 + ], + [ + 126, + 0.1399773759229084 + ], + [ + 127, + 0.07143768347568931 + ], + [ + 128, + 0.0403405564287424 + ], + [ + 133, + 0.11080060934031616 + ], + [ + 137, + 0.1232128633699099 + ], + [ + 139, + 0.14476571666430887 + ], + [ + 140, + 0.16912411474219072 + ], + [ + 142, + 0.3005343036665297 + ], + [ + 144, + 0.1671519038176669 + ], + [ + 145, + 0.051586007770406124 + ], + [ + 149, + 0.06721963084578568 + ], + [ + 157, + 0.12061300279008058 + ], + [ + 159, + 0.1408769756357225 + ], + [ + 163, + 0.12919986463807553 + ], + [ + 168, + 0.11033514740506573 + ], + [ + 173, + 0.10449412818747991 + ], + [ + 178, + 0.07646171331816494 + ], + [ + 188, + 0.1872230029849286 + ], + [ + 189, + 0.21227718043994306 + ], + [ + 193, + 0.04183210218825661 + ], + [ + 196, + 0.052206993367940074 + ], + [ + 198, + 0.052212944130414295 + ], + [ + 199, + 0.18215111024326155 + ], + [ + 202, + 0.0074228226058799165 + ], + [ + 205, + 0.020389674669569237 + ], + [ + 206, + 0.20094594936445234 + ], + [ + 216, + 0.03505136357330504 + ], + [ + 217, + 0.19681212430074932 + ], + [ + 221, + 0.13234293538991532 + ], + [ + 225, + 0.2034473302109531 + ], + [ + 232, + 0.08365545276297746 + ], + [ + 239, + 0.1185197102259373 + ], + [ + 240, + 0.7356070745500628 + ], + [ + 245, + 0.715770761753138 + ], + [ + 249, + 1.0 + ], + [ + 255, + 0.9602549287270391 + ], + [ + 259, + 0.6116598704755667 + ], + [ + 266, + 0.5637365983986102 + ], + [ + 270, + 0.18366001164864712 + ], + [ + 274, + 0.16945072817598944 + ], + [ + 279, + 0.10926289639822105 + ], + [ + 285, + 0.09904888864902497 + ], + [ + 292, + 0.11446228928999506 + ], + [ + 295, + 0.11739092480353913 + ], + [ + 297, + 0.13847047535842003 + ], + [ + 304, + 0.12544608151321554 + ], + [ + 315, + 0.21626846993123883 + ], + [ + 316, + 0.23607693472256205 + ], + [ + 321, + 0.09882546813421017 + ], + [ + 322, + 0.16037851644227835 + ], + [ + 323, + 0.11210140255030993 + ], + [ + 324, + 0.06164487197244974 + ], + [ + 330, + 0.10011783044412914 + ], + [ + 333, + 0.1855450087490631 + ], + [ + 334, + 0.17718475651837254 + ], + [ + 335, + 0.0898339395777479 + ], + [ + 336, + 0.053639191341723146 + ], + [ + 337, + 0.07875886001846907 + ], + [ + 338, + 0.07199089205017022 + ], + [ + 339, + 0.31344655476414746 + ], + [ + 343, + 0.1310180546384617 + ], + [ + 351, + 0.11103531698944566 + ], + [ + 356, + 0.12726479194397344 + ], + [ + 361, + 0.08648329509279223 + ], + [ + 369, + 0.17143312610533562 + ], + [ + 372, + 0.16344144854905257 + ], + [ + 377, + 0.14043091814931005 + ], + [ + 384, + 0.27063693847377396 + ], + [ + 389, + 0.13651651767069453 + ], + [ + 391, + 0.14843482736579156 + ], + [ + 394, + 0.33768422931630226 + ], + [ + 396, + 0.2965795423928562 + ], + [ + 399, + 0.27827682697587436 + ], + [ + 400, + 0.31979715492189703 + ], + [ + 401, + 0.3304562688061125 + ], + [ + 402, + 0.06613279213124662 + ], + [ + 403, + 0.09303295869627494 + ], + [ + 404, + 0.0852011138365282 + ], + [ + 405, + 0.33092331250989276 + ], + [ + 412, + 0.2717532510104603 + ], + [ + 415, + 0.2980820897981542 + ], + [ + 416, + 0.3527411857219166 + ], + [ + 417, + 0.4705301854204476 + ], + [ + 418, + 0.031940122422553674 + ], + [ + 419, + 0.06096105353963828 + ], + [ + 420, + 0.06435539080412245 + ], + [ + 424, + 0.1732269098122254 + ], + [ + 433, + 0.14221545989188303 + ], + [ + 434, + 0.31768390612763087 + ], + [ + 435, + 0.15315217906170459 + ], + [ + 437, + 0.11040275881632235 + ], + [ + 439, + 0.0845843075845013 + ], + [ + 441, + 0.12381054203063033 + ], + [ + 443, + 0.11239217178279656 + ], + [ + 446, + 0.17359181708167776 + ], + [ + 447, + 0.18688257912960402 + ], + [ + 448, + 0.31298453658933 + ], + [ + 449, + 0.004066016787700343 + ], + [ + 450, + 0.005289652793317763 + ], + [ + 451, + 0.009893483200417592 + ], + [ + 452, + 0.1995721166837755 + ], + [ + 457, + 0.04988405080784992 + ], + [ + 458, + 0.08478118738826178 + ], + [ + 459, + 0.04941780032973001 + ], + [ + 460, + 0.05952491220976437 + ], + [ + 461, + 0.17746346831739157 + ], + [ + 462, + 0.09785940081772034 + ], + [ + 463, + 0.03505206635797319 + ], + [ + 464, + 0.008502925378091249 + ], + [ + 468, + 0.07539110695276285 + ], + [ + 469, + 0.06101073986363295 + ], + [ + 470, + 0.0014058797935025227 + ], + [ + 471, + 0.20001532845623538 + ], + [ + 484, + 0.08277143262108913 + ], + [ + 485, + 0.3631162124828134 + ], + [ + 489, + 0.14933322342196242 + ], + [ + 496, + 0.1918321791128444 + ], + [ + 504, + 0.17397359966425419 + ], + [ + 515, + 0.21640842533050814 + ], + [ + 516, + 0.36046573428154827 + ], + [ + 517, + 0.25370964671591967 + ], + [ + 521, + 0.11432492184425526 + ], + [ + 524, + 0.11558070033062713 + ], + [ + 528, + 0.19272174240741724 + ], + [ + 532, + 0.09489412782229065 + ], + [ + 535, + 0.09125997062568734 + ], + [ + 536, + 0.08265898821124165 + ], + [ + 537, + 0.04834229292211284 + ], + [ + 538, + 0.0 + ], + [ + 539, + 0.010395866837518204 + ], + [ + 540, + 0.05882176887523165 + ], + [ + 541, + 0.1756851206657896 + ], + [ + 546, + 0.05353678694013724 + ], + [ + 550, + 0.028096977487279386 + ], + [ + 558, + 0.034477610122856245 + ], + [ + 563, + 0.05627951003533696 + ], + [ + 566, + 0.06315999030287228 + ], + [ + 571, + 0.05879084837189241 + ], + [ + 576, + 0.06621366388445923 + ], + [ + 582, + 0.08723784012909745 + ], + [ + 587, + 0.055519181219399256 + ], + [ + 588, + 0.3732834464321924 + ], + [ + 591, + 0.08375619210234707 + ], + [ + 595, + 0.12680364516246181 + ], + [ + 599, + 0.17863158971096035 + ], + [ + 600, + 0.15777310646578713 + ], + [ + 601, + 0.13852177596872178 + ], + [ + 602, + 0.11827061278745217 + ], + [ + 605, + 0.1302702503321446 + ], + [ + 611, + 0.15522111757688403 + ], + [ + 614, + 0.1603850034463322 + ], + [ + 618, + 0.19173818137854368 + ], + [ + 624, + 0.17708033329995682 + ], + [ + 628, + 0.14859817689612578 + ], + [ + 629, + 0.4601115447131871 + ], + [ + 630, + 0.3302086911503199 + ], + [ + 631, + 0.12986788455823722 + ] + ], + [ + [ + 53, + 0.6582585399168678 + ], + [ + 55, + 0.5963133136153707 + ], + [ + 59, + 0.10063110706394497 + ], + [ + 63, + 0.21081088436278086 + ], + [ + 70, + 0.37885702138656235 + ], + [ + 73, + 0.4885758704243729 + ], + [ + 77, + 0.3437925368796317 + ], + [ + 85, + 0.7100699210271766 + ], + [ + 87, + 0.5168668723706196 + ], + [ + 90, + 0.42920339351065984 + ], + [ + 92, + 0.3842881402769266 + ], + [ + 94, + 0.4831088696194465 + ], + [ + 96, + 0.6886146921029977 + ], + [ + 110, + 0.1550616668926261 + ], + [ + 121, + 0.15176091141145556 + ], + [ + 124, + 0.1569918422129763 + ], + [ + 133, + 0.059731530378007935 + ], + [ + 141, + 0.08689747628545026 + ], + [ + 145, + 0.12058331924239866 + ], + [ + 156, + 0.05988184428138947 + ], + [ + 157, + 0.2994867527058008 + ], + [ + 158, + 0.25938188536281104 + ], + [ + 159, + 0.0712112758345618 + ], + [ + 160, + 0.0 + ], + [ + 161, + 0.012922008031960025 + ], + [ + 162, + 0.005607978031348435 + ], + [ + 163, + 0.2993339213069111 + ], + [ + 165, + 0.3072924197847063 + ], + [ + 166, + 0.08294557771246869 + ], + [ + 170, + 0.0747565520305279 + ], + [ + 178, + 0.19576345269200846 + ], + [ + 187, + 0.04637987563868802 + ], + [ + 194, + 0.07037975243248006 + ], + [ + 197, + 0.2115399679883897 + ], + [ + 201, + 0.20759786983471307 + ], + [ + 209, + 0.12640069315226446 + ], + [ + 211, + 0.23970565769285093 + ], + [ + 214, + 0.1913838107560497 + ], + [ + 216, + 0.09854230013477022 + ], + [ + 218, + 0.017855383667826893 + ], + [ + 220, + 0.03932157684769817 + ], + [ + 234, + 0.06200393937665901 + ], + [ + 245, + 0.01841566679304679 + ], + [ + 248, + 0.1736418256352882 + ], + [ + 257, + 0.016497894446059064 + ], + [ + 265, + 0.06924161090095 + ], + [ + 269, + 0.1427942869801057 + ], + [ + 280, + 0.0735527132157195 + ], + [ + 281, + 0.7378571829029843 + ], + [ + 285, + 0.5617913349453808 + ], + [ + 286, + 1.0 + ], + [ + 287, + 0.26323339432108295 + ], + [ + 288, + 0.34354963088506296 + ], + [ + 289, + 0.3699402408332985 + ], + [ + 290, + 0.13815250596865308 + ], + [ + 291, + 0.21157437203509583 + ], + [ + 292, + 0.21496687657473093 + ], + [ + 293, + 0.20536453919342718 + ], + [ + 299, + 0.4489539411275406 + ], + [ + 303, + 0.33640282012299133 + ], + [ + 314, + 0.35356494429682683 + ], + [ + 317, + 0.28140125436854535 + ], + [ + 322, + 0.24944046598689446 + ], + [ + 325, + 0.34118556684200957 + ], + [ + 329, + 0.4554346256152296 + ], + [ + 337, + 0.979377615903178 + ], + [ + 343, + 0.0767401687841315 + ], + [ + 352, + 0.061985072924645244 + ], + [ + 355, + 0.155596511147482 + ], + [ + 357, + 0.1901495863957176 + ], + [ + 370, + 0.05441598864112429 + ], + [ + 381, + 0.3510135962150655 + ], + [ + 382, + 0.44069393857697864 + ], + [ + 391, + 0.1264651506318244 + ], + [ + 395, + 0.2892539830297463 + ], + [ + 402, + 0.17256084077670533 + ], + [ + 407, + 0.08813105054584931 + ], + [ + 412, + 0.14702971123807537 + ], + [ + 423, + 0.18027387317756502 + ], + [ + 425, + 0.4176124312021726 + ], + [ + 426, + 0.12640012987948496 + ], + [ + 428, + 0.17010202711110683 + ], + [ + 429, + 0.17884919055527826 + ], + [ + 430, + 0.17029250452226563 + ], + [ + 437, + 0.051881079701400185 + ], + [ + 441, + 0.057502446767087115 + ], + [ + 446, + 0.05604810178940015 + ], + [ + 447, + 0.2506723264162943 + ], + [ + 448, + 0.022829095573347807 + ], + [ + 449, + 0.04073613963525571 + ], + [ + 450, + 0.0630492878253677 + ], + [ + 456, + 0.11451685775480841 + ], + [ + 459, + 0.32284395004020083 + ], + [ + 468, + 0.14133538891353212 + ], + [ + 473, + 0.3197353026850082 + ], + [ + 474, + 0.5782850005806471 + ], + [ + 480, + 0.10644622625830114 + ], + [ + 484, + 0.16043029421081478 + ], + [ + 488, + 0.39052507653614155 + ], + [ + 494, + 0.10874733942536662 + ], + [ + 500, + 0.19883730630757135 + ], + [ + 506, + 0.07195295960789329 + ], + [ + 512, + 0.1485671657727433 + ], + [ + 514, + 0.2502864555836198 + ], + [ + 518, + 0.45054862738119916 + ], + [ + 520, + 0.39700476607513674 + ], + [ + 521, + 0.07091728433666063 + ], + [ + 522, + 0.05281117462764966 + ], + [ + 523, + 0.06404463492553296 + ], + [ + 524, + 0.19343266208508317 + ], + [ + 528, + 0.17696214262741136 + ], + [ + 532, + 0.10791415477860113 + ], + [ + 536, + 0.2328700101449916 + ], + [ + 538, + 0.37176082750116185 + ], + [ + 539, + 0.03657779519409084 + ], + [ + 540, + 0.04402123897040517 + ], + [ + 541, + 0.23400359873646412 + ], + [ + 543, + 0.6470990002652671 + ], + [ + 547, + 0.2575699877592206 + ], + [ + 551, + 0.26714498743284476 + ], + [ + 556, + 0.4087300614197009 + ], + [ + 557, + 0.5230386488304619 + ], + [ + 561, + 0.2067257512983222 + ], + [ + 563, + 0.32321175457441137 + ], + [ + 566, + 0.514935904826391 + ], + [ + 570, + 0.27063376230489683 + ], + [ + 575, + 0.07233386165546149 + ], + [ + 581, + 0.23939391523070327 + ], + [ + 585, + 0.6179592420371278 + ], + [ + 590, + 0.2364625143110678 + ], + [ + 592, + 0.23999625708954422 + ], + [ + 597, + 0.2837854492537854 + ], + [ + 604, + 0.202811014100928 + ], + [ + 607, + 0.3940827933327381 + ], + [ + 613, + 0.2837320901998689 + ], + [ + 617, + 0.24365232071721973 + ], + [ + 625, + 0.1643023613628157 + ], + [ + 630, + 0.17438179099210213 + ], + [ + 633, + 0.43045252710838344 + ], + [ + 637, + 0.40052249015621694 + ], + [ + 643, + 0.24873014997029866 + ], + [ + 647, + 0.29157941609912763 + ], + [ + 652, + 0.17198890576498677 + ], + [ + 653, + 0.6039028485542983 + ], + [ + 656, + 0.14271139932094562 + ], + [ + 659, + 0.30227098529044666 + ], + [ + 660, + 0.9338607495979158 + ], + [ + 672, + 0.4363628820120393 + ], + [ + 677, + 0.6926862288244954 + ], + [ + 682, + 0.221145440632094 + ], + [ + 689, + 0.3628782521523585 + ], + [ + 691, + 0.2893765166672598 + ], + [ + 695, + 0.2579901384588144 + ], + [ + 698, + 0.6109322878408743 + ], + [ + 702, + 0.7675569855405753 + ], + [ + 704, + 0.9912710311375318 + ] + ] + ], + [ + [ + [ + 34, + 1.0 + ], + [ + 44, + 0.6669788340336903 + ], + [ + 48, + 0.28786009337004664 + ], + [ + 52, + 0.104160082769842 + ], + [ + 59, + 0.19408870998798064 + ], + [ + 64, + 0.14203846816555107 + ], + [ + 73, + 0.1613940034577686 + ], + [ + 76, + 0.3627676305132625 + ], + [ + 84, + 0.4770081718578338 + ], + [ + 85, + 0.510301639010639 + ], + [ + 86, + 0.24327278248667317 + ], + [ + 87, + 0.20319602751085417 + ], + [ + 88, + 0.5089089576403126 + ], + [ + 89, + 0.4113280844560498 + ], + [ + 90, + 0.6845555563307751 + ], + [ + 91, + 0.07407713197651139 + ], + [ + 92, + 0.09133393071988796 + ], + [ + 93, + 0.11288144759948268 + ], + [ + 94, + 0.3936034725022763 + ], + [ + 97, + 0.1531288951632902 + ], + [ + 99, + 0.12891655374649755 + ], + [ + 100, + 0.21714369487569782 + ], + [ + 104, + 0.13515500577214956 + ], + [ + 109, + 0.06026464673777316 + ], + [ + 116, + 0.07460995495868007 + ], + [ + 118, + 0.10682121548779161 + ], + [ + 121, + 0.13707782615104008 + ], + [ + 126, + 0.17028993230399087 + ], + [ + 131, + 0.11598594700158214 + ], + [ + 135, + 0.07226818798881336 + ], + [ + 149, + 0.11423294217486092 + ], + [ + 154, + 0.12015519771015085 + ], + [ + 155, + 0.2642313820135377 + ], + [ + 158, + 0.1323084863884983 + ], + [ + 159, + 0.12333962259288894 + ], + [ + 160, + 0.154816040971036 + ], + [ + 161, + 0.07173384476603994 + ], + [ + 162, + 0.1128219799620956 + ], + [ + 163, + 0.21409478354913902 + ], + [ + 164, + 0.19149815493148978 + ], + [ + 167, + 0.07920005611727922 + ], + [ + 174, + 0.13545373271209393 + ], + [ + 175, + 0.10265915660849478 + ], + [ + 176, + 0.12560661141482884 + ], + [ + 177, + 0.10333279568913932 + ], + [ + 183, + 0.14790731533719723 + ], + [ + 186, + 0.10475178440630124 + ], + [ + 195, + 0.07783661582604792 + ], + [ + 199, + 0.08434668801378177 + ], + [ + 213, + 0.07022318513374999 + ], + [ + 221, + 0.0925808574125291 + ], + [ + 222, + 0.26043356071880897 + ], + [ + 226, + 0.23231161050666302 + ], + [ + 230, + 0.19700553321356132 + ], + [ + 233, + 0.1998778444991174 + ], + [ + 239, + 0.09984905861110144 + ], + [ + 242, + 0.0754966829048109 + ], + [ + 246, + 0.2022928446376606 + ], + [ + 249, + 0.22354755965237408 + ], + [ + 253, + 0.10572567439523599 + ], + [ + 258, + 0.1079180579008827 + ], + [ + 262, + 0.2225970764888002 + ], + [ + 266, + 0.18975489238673576 + ], + [ + 268, + 0.23779800770618434 + ], + [ + 273, + 0.20767935599728812 + ], + [ + 274, + 0.20857678188265438 + ], + [ + 275, + 0.20510146170517374 + ], + [ + 279, + 0.2371894749796951 + ], + [ + 282, + 0.1648939701039744 + ], + [ + 284, + 0.23981407312361105 + ], + [ + 287, + 0.1266420338708441 + ], + [ + 291, + 0.25860439610774866 + ], + [ + 297, + 0.2772858907399377 + ], + [ + 299, + 0.3612540239321766 + ], + [ + 303, + 0.10532742490071395 + ], + [ + 308, + 0.0836477961981522 + ], + [ + 322, + 0.09682133237619762 + ], + [ + 328, + 0.06989069841470544 + ], + [ + 332, + 0.07111396682890307 + ], + [ + 337, + 0.0731508052531485 + ], + [ + 340, + 0.136571270866097 + ], + [ + 347, + 0.1165893205343651 + ], + [ + 351, + 0.1648687936182395 + ], + [ + 352, + 0.28058879543526255 + ], + [ + 355, + 0.05083953521802601 + ], + [ + 362, + 0.059693866540006246 + ], + [ + 368, + 0.052004241497042504 + ], + [ + 376, + 0.05971715678873581 + ], + [ + 380, + 0.0 + ], + [ + 388, + 0.04149110401083215 + ], + [ + 392, + 0.071402738666751 + ], + [ + 399, + 0.023981544764342666 + ], + [ + 400, + 0.26177123554120885 + ], + [ + 404, + 0.034633201043843966 + ], + [ + 410, + 0.0422574326965864 + ], + [ + 414, + 0.04331047821817629 + ], + [ + 420, + 0.04064211842685697 + ], + [ + 425, + 0.057073479388010585 + ], + [ + 428, + 0.1434229078391526 + ], + [ + 432, + 0.1482792429128047 + ], + [ + 437, + 0.055407903645911506 + ], + [ + 440, + 0.13106790566678098 + ], + [ + 444, + 0.03555517105449743 + ], + [ + 447, + 0.0739374419947385 + ], + [ + 450, + 0.05542317789569673 + ], + [ + 457, + 0.11433728503758878 + ], + [ + 459, + 0.08005935941089588 + ], + [ + 460, + 0.05681584028843184 + ], + [ + 463, + 0.026487417403643684 + ], + [ + 469, + 0.03683565080604953 + ], + [ + 473, + 0.07771994701787305 + ], + [ + 482, + 0.05120977446579291 + ], + [ + 483, + 0.31190637408663174 + ], + [ + 486, + 0.11110618883606249 + ], + [ + 494, + 0.11566554561923977 + ], + [ + 495, + 0.11555375947415741 + ], + [ + 496, + 0.11884787618924109 + ], + [ + 497, + 0.10378789086350076 + ], + [ + 498, + 0.09525332653010186 + ], + [ + 499, + 0.13627408449989073 + ], + [ + 500, + 0.23378323187843073 + ], + [ + 503, + 0.11469036176525177 + ], + [ + 507, + 0.13958194776867264 + ], + [ + 515, + 0.3400221986034654 + ], + [ + 521, + 0.23533403047152918 + ], + [ + 522, + 0.1494612520105453 + ], + [ + 523, + 0.19918129099382503 + ], + [ + 527, + 0.207560644386981 + ], + [ + 532, + 0.1599734155396674 + ], + [ + 535, + 0.12928706003608267 + ], + [ + 539, + 0.3195826551719379 + ], + [ + 542, + 0.41053208837317096 + ], + [ + 543, + 0.23650764266296037 + ], + [ + 544, + 0.28563012542291344 + ], + [ + 545, + 0.24712924589357427 + ], + [ + 546, + 0.12789389744826674 + ], + [ + 547, + 0.26095049539056475 + ], + [ + 550, + 0.24250944819369274 + ], + [ + 554, + 0.14566566058277583 + ], + [ + 568, + 0.07637304908617795 + ], + [ + 579, + 0.1001223101682504 + ], + [ + 582, + 0.1720332897367006 + ], + [ + 591, + 0.1011930177272539 + ], + [ + 599, + 0.12480014885154946 + ], + [ + 601, + 0.1300274463547452 + ], + [ + 612, + 0.09558484539199816 + ], + [ + 615, + 0.16822346918220096 + ], + [ + 619, + 0.03843047402174289 + ], + [ + 624, + 0.09573027202846496 + ], + [ + 625, + 0.33963471956968777 + ], + [ + 633, + 0.06585864213467439 + ], + [ + 634, + 0.6741000341335021 + ], + [ + 637, + 0.2618639708958686 + ] + ], + [ + [ + 38, + 1.0 + ], + [ + 46, + 0.7294386402001508 + ], + [ + 55, + 0.45116729028009095 + ], + [ + 64, + 0.4028159526548645 + ], + [ + 73, + 0.5844387540462723 + ], + [ + 76, + 0.3713750337183371 + ], + [ + 82, + 0.2276713665084433 + ], + [ + 83, + 0.24491892217785713 + ], + [ + 86, + 0.3757236936155045 + ], + [ + 91, + 0.27866092968526324 + ], + [ + 100, + 0.2147207659521844 + ], + [ + 101, + 0.3883132070888218 + ], + [ + 105, + 0.27969089441460016 + ], + [ + 112, + 0.3495415754958155 + ], + [ + 114, + 0.3588464058629745 + ], + [ + 119, + 0.6931829058539902 + ], + [ + 125, + 0.4718359175618501 + ], + [ + 132, + 0.493777859153893 + ], + [ + 135, + 0.3187956130515734 + ], + [ + 136, + 0.1344205204772161 + ], + [ + 137, + 0.09410441861643984 + ], + [ + 138, + 0.17762382834086704 + ], + [ + 144, + 0.24780983708641804 + ], + [ + 147, + 0.2175932952743929 + ], + [ + 148, + 0.17728995353057903 + ], + [ + 149, + 0.0759412902204005 + ], + [ + 150, + 0.12458721464311272 + ], + [ + 164, + 0.34317040388288184 + ], + [ + 165, + 0.34839558922168073 + ], + [ + 166, + 0.5069453894141471 + ], + [ + 169, + 0.1996138182775535 + ], + [ + 172, + 0.13186858195672496 + ], + [ + 176, + 0.13987525907023524 + ], + [ + 179, + 0.18256964751510538 + ], + [ + 183, + 0.1284535810059022 + ], + [ + 187, + 0.2152857680062079 + ], + [ + 190, + 0.17309359023559154 + ], + [ + 192, + 0.21772266023814987 + ], + [ + 195, + 0.3796813421114168 + ], + [ + 198, + 0.24178843372005898 + ], + [ + 202, + 0.13599855838608924 + ], + [ + 203, + 0.15547492255883819 + ], + [ + 204, + 0.27108734581719995 + ], + [ + 205, + 0.21531409210184757 + ], + [ + 207, + 0.24811988524463718 + ], + [ + 215, + 0.25196299977015624 + ], + [ + 216, + 0.20286996125029547 + ], + [ + 221, + 0.09272980805052422 + ], + [ + 222, + 0.07540261236945608 + ], + [ + 223, + 0.05131688085480627 + ], + [ + 224, + 0.07753116237005059 + ], + [ + 225, + 0.109825071747139 + ], + [ + 231, + 0.07821602372566058 + ], + [ + 234, + 0.07244258080365187 + ], + [ + 235, + 0.03899380368511985 + ], + [ + 236, + 0.004686924132137934 + ], + [ + 237, + 0.039745529504152034 + ], + [ + 238, + 0.018410122071339387 + ], + [ + 246, + 0.08338714483930103 + ], + [ + 247, + 0.4192459557947151 + ], + [ + 249, + 0.4689982582799123 + ], + [ + 251, + 0.08268673339315653 + ], + [ + 254, + 0.042521531604647216 + ], + [ + 264, + 0.043371815400492786 + ], + [ + 267, + 0.0712009900977353 + ], + [ + 276, + 0.09844576794955971 + ], + [ + 280, + 0.07789029585424097 + ], + [ + 287, + 0.11273770153850249 + ], + [ + 290, + 0.08325311114487893 + ], + [ + 296, + 0.05434305238591756 + ], + [ + 305, + 0.07016235530948516 + ], + [ + 308, + 0.07882975618645092 + ], + [ + 312, + 0.06367488284577107 + ], + [ + 315, + 0.15545268609911783 + ], + [ + 319, + 0.05037904774999223 + ], + [ + 324, + 0.007659257506192716 + ], + [ + 329, + 0.02705823694736239 + ], + [ + 335, + 0.020243706822557356 + ], + [ + 340, + 0.03719886462010051 + ], + [ + 343, + 0.03198738302040964 + ], + [ + 347, + 0.062284378843568834 + ], + [ + 355, + 0.06426219928742724 + ], + [ + 356, + 0.5970341267727165 + ], + [ + 360, + 0.46770915469441277 + ], + [ + 363, + 0.6137176409846491 + ], + [ + 365, + 0.4812311906010571 + ], + [ + 368, + 0.9609858613733756 + ], + [ + 370, + 0.9083385786115998 + ], + [ + 374, + 0.22487270050205546 + ], + [ + 378, + 0.042479316279971965 + ], + [ + 383, + 0.03131377414670673 + ], + [ + 385, + 0.021911846372416853 + ], + [ + 392, + 0.024049799417238022 + ], + [ + 397, + 0.031149392159834543 + ], + [ + 405, + 0.020202199407382222 + ], + [ + 412, + 0.01799350980777499 + ], + [ + 416, + 0.02406849825836652 + ], + [ + 420, + 0.008145896486134337 + ], + [ + 425, + 0.010305670503737664 + ], + [ + 429, + 0.046473943499167315 + ], + [ + 433, + 0.08263187522916907 + ], + [ + 434, + 0.09640989121105661 + ], + [ + 438, + 0.02182357488002374 + ], + [ + 443, + 0.0 + ], + [ + 445, + 0.0339664410521026 + ], + [ + 456, + 0.019514446617601443 + ], + [ + 460, + 0.051055194296006745 + ], + [ + 461, + 0.3305291802246076 + ], + [ + 466, + 0.20178690678048522 + ], + [ + 469, + 0.09774502550267893 + ], + [ + 479, + 0.09656114080158354 + ], + [ + 487, + 0.1866392004930442 + ], + [ + 489, + 0.09216050993100024 + ], + [ + 492, + 0.09577825847734772 + ], + [ + 496, + 0.0874828461038437 + ], + [ + 498, + 0.16805993936014815 + ], + [ + 504, + 0.08272492655736953 + ], + [ + 507, + 0.08120548040820164 + ], + [ + 511, + 0.07761566786105942 + ], + [ + 514, + 0.07537816460877947 + ], + [ + 520, + 0.0625815712109531 + ], + [ + 525, + 0.05842036574941734 + ], + [ + 530, + 0.07048029501905821 + ], + [ + 536, + 0.1803503932824405 + ], + [ + 539, + 0.11408891752790246 + ], + [ + 543, + 0.06837254637009665 + ], + [ + 550, + 0.06767352605292078 + ], + [ + 552, + 0.5092427227855304 + ], + [ + 557, + 0.3400718697269338 + ], + [ + 560, + 0.15008426385661353 + ], + [ + 565, + 0.3900387208712027 + ], + [ + 568, + 0.33933438928457815 + ], + [ + 572, + 0.3161788346502522 + ], + [ + 578, + 0.35915662145047056 + ], + [ + 585, + 0.26597521767627846 + ], + [ + 588, + 0.16314901337308518 + ], + [ + 592, + 0.11699944558032499 + ], + [ + 598, + 0.14853388418409 + ], + [ + 601, + 0.14774791329121223 + ], + [ + 610, + 0.202893206915058 + ] + ], + [ + [ + 38, + 0.619607851400233 + ], + [ + 46, + 0.3252972317483403 + ], + [ + 55, + 0.13907565976743214 + ], + [ + 64, + 0.12419704617713935 + ], + [ + 73, + 0.19516214822900124 + ], + [ + 76, + 0.20900991086566306 + ], + [ + 82, + 0.31801889907562164 + ], + [ + 83, + 0.3902067326744198 + ], + [ + 86, + 1.0 + ], + [ + 91, + 0.05255772181572169 + ], + [ + 100, + 0.052395797537058614 + ], + [ + 101, + 0.14640479849540175 + ], + [ + 105, + 0.09987731194071657 + ], + [ + 112, + 0.12953622100709755 + ], + [ + 114, + 0.1618676507772953 + ], + [ + 119, + 0.34650537177448715 + ], + [ + 125, + 0.21506440468931937 + ], + [ + 132, + 0.24439837731195657 + ], + [ + 135, + 0.179338207454113 + ], + [ + 136, + 0.08156171506769937 + ], + [ + 137, + 0.046331135949550496 + ], + [ + 138, + 0.09676191103809033 + ], + [ + 144, + 0.0771724052374606 + ], + [ + 147, + 0.09029194653537666 + ], + [ + 148, + 0.058276749814211354 + ], + [ + 149, + 0.03486014666374716 + ], + [ + 150, + 0.058365058220566216 + ], + [ + 164, + 0.1329077405881256 + ], + [ + 165, + 0.1297593815202988 + ], + [ + 166, + 0.18617686048622234 + ], + [ + 169, + 0.09799187554318849 + ], + [ + 172, + 0.10848964637237676 + ], + [ + 176, + 0.12416163617199816 + ], + [ + 179, + 0.08264458798532692 + ], + [ + 183, + 0.06134990607892945 + ], + [ + 187, + 0.0935923099716687 + ], + [ + 190, + 0.06378627898077256 + ], + [ + 192, + 0.09128954323425224 + ], + [ + 195, + 0.14124368998214193 + ], + [ + 198, + 0.11357061533526805 + ], + [ + 202, + 0.13590404068080694 + ], + [ + 203, + 0.14705861883912633 + ], + [ + 204, + 0.18172491740054383 + ], + [ + 205, + 0.13890815195571937 + ], + [ + 207, + 0.11221116929032043 + ], + [ + 215, + 0.09814878965894391 + ], + [ + 216, + 0.1119066430034099 + ], + [ + 221, + 0.04641702619008417 + ], + [ + 222, + 0.039850235549016975 + ], + [ + 223, + 0.03027719335454425 + ], + [ + 224, + 0.04355447848638421 + ], + [ + 225, + 0.07052960573801098 + ], + [ + 231, + 0.03177151911304879 + ], + [ + 234, + 0.0324565226014972 + ], + [ + 235, + 0.02723912279196108 + ], + [ + 236, + 0.002950304127529213 + ], + [ + 237, + 0.019711608612818187 + ], + [ + 238, + 0.018547606550428843 + ], + [ + 246, + 0.04691839378382502 + ], + [ + 247, + 0.18698774177699795 + ], + [ + 249, + 0.18516335079399782 + ], + [ + 251, + 0.018838417762868178 + ], + [ + 254, + 0.023540571994263005 + ], + [ + 264, + 0.013497922057990207 + ], + [ + 267, + 0.04311129309690655 + ], + [ + 276, + 0.0377121046153056 + ], + [ + 280, + 0.04394061809728759 + ], + [ + 287, + 0.014033747528728992 + ], + [ + 290, + 0.04836341122696808 + ], + [ + 296, + 0.01142523176982136 + ], + [ + 305, + 0.030780862923225648 + ], + [ + 308, + 0.0296560463111719 + ], + [ + 312, + 0.0738370125259365 + ], + [ + 315, + 0.014018627394039395 + ], + [ + 319, + 0.009545974724897927 + ], + [ + 324, + 0.0035339291007187167 + ], + [ + 329, + 0.0 + ], + [ + 335, + 0.01405256271544494 + ], + [ + 340, + 0.007039059093359649 + ], + [ + 343, + 0.017379944626984827 + ], + [ + 347, + 0.023143460384456756 + ], + [ + 355, + 0.01612321132051548 + ], + [ + 356, + 0.1509663323342992 + ], + [ + 360, + 0.1835548030290531 + ], + [ + 363, + 0.12504455702224712 + ], + [ + 365, + 0.08756398427590147 + ], + [ + 368, + 0.1664969029670402 + ], + [ + 370, + 0.16616588697956836 + ], + [ + 374, + 0.08375164559487078 + ], + [ + 378, + 0.020930842601888097 + ], + [ + 383, + 0.014320784964511368 + ], + [ + 385, + 0.04004726464215305 + ], + [ + 392, + 0.010266532016078696 + ], + [ + 397, + 0.02099866045824167 + ], + [ + 405, + 0.011112640379616855 + ], + [ + 412, + 0.00727765615662516 + ], + [ + 416, + 0.02276717424562365 + ], + [ + 420, + 0.015223938196512481 + ], + [ + 425, + 0.01600409958923455 + ], + [ + 429, + 0.1107896487982937 + ], + [ + 433, + 0.003093654398980529 + ], + [ + 434, + 0.07711117355347913 + ], + [ + 438, + 0.019588152616757507 + ], + [ + 443, + 0.007335394074870888 + ], + [ + 445, + 0.06194931814673301 + ], + [ + 456, + 0.008405130748837274 + ], + [ + 460, + 0.01174836376615522 + ], + [ + 461, + 0.1483882429625391 + ], + [ + 466, + 0.07312640699885034 + ], + [ + 469, + 0.05119871960173586 + ], + [ + 479, + 0.03893952778618169 + ], + [ + 487, + 0.0729486422323748 + ], + [ + 489, + 0.02567814169265018 + ], + [ + 492, + 0.05102083773426835 + ], + [ + 496, + 0.028775603694092343 + ], + [ + 498, + 0.055175068233242544 + ], + [ + 504, + 0.015066488935291165 + ], + [ + 507, + 0.06290635967771151 + ], + [ + 511, + 0.014865731419379729 + ], + [ + 514, + 0.05042922835432338 + ], + [ + 520, + 0.022542551183504587 + ], + [ + 525, + 0.013394873791799196 + ], + [ + 530, + 0.036198949563359395 + ], + [ + 536, + 0.05873797391374967 + ], + [ + 539, + 0.06482983872910696 + ], + [ + 543, + 0.037220685448584465 + ], + [ + 550, + 0.034302162863982875 + ], + [ + 552, + 0.27329816372601184 + ], + [ + 557, + 0.1361473959578251 + ], + [ + 560, + 0.06655543287933952 + ], + [ + 565, + 0.12605568525647437 + ], + [ + 568, + 0.13419812435115214 + ], + [ + 572, + 0.15277424787514443 + ], + [ + 578, + 0.24096464707051937 + ], + [ + 585, + 0.16921261573466648 + ], + [ + 588, + 0.0999845370112014 + ], + [ + 592, + 0.09822303350807435 + ], + [ + 598, + 0.07080848448071694 + ], + [ + 601, + 0.1139992444040291 + ], + [ + 610, + 0.12766541191897032 + ] + ], + [ + [ + 38, + 0.28234998847742726 + ], + [ + 44, + 0.3267910030609417 + ], + [ + 46, + 0.6318171683370721 + ], + [ + 48, + 0.7720884777716852 + ], + [ + 50, + 0.20860329441790174 + ], + [ + 52, + 0.23393998279534164 + ], + [ + 54, + 0.10664665019972279 + ], + [ + 58, + 0.14611105038902242 + ], + [ + 60, + 0.1359784312824213 + ], + [ + 62, + 0.17264979188712246 + ], + [ + 63, + 0.21538462123794974 + ], + [ + 65, + 0.2513807089994393 + ], + [ + 67, + 0.12334794851976671 + ], + [ + 69, + 0.09022632326613293 + ], + [ + 71, + 0.08174566265697453 + ], + [ + 75, + 0.05977569989234098 + ], + [ + 77, + 0.010318400285564976 + ], + [ + 79, + 0.026713589371379784 + ], + [ + 80, + 0.04689022531660964 + ], + [ + 82, + 0.04000322914336584 + ], + [ + 84, + 0.2790386237321527 + ], + [ + 88, + 0.3218416267719986 + ], + [ + 89, + 0.10585074286580072 + ], + [ + 90, + 0.08559639299864198 + ], + [ + 91, + 0.016784907532591445 + ], + [ + 100, + 0.10355348709785867 + ], + [ + 101, + 0.18613227403696495 + ], + [ + 102, + 0.14863874763997573 + ], + [ + 103, + 0.07739989117782665 + ], + [ + 104, + 0.11333179651703888 + ], + [ + 105, + 0.05812743045172448 + ], + [ + 106, + 0.27060366104822087 + ], + [ + 109, + 0.2857014396439133 + ], + [ + 117, + 0.24473538543025458 + ], + [ + 124, + 0.2224957577836723 + ], + [ + 138, + 0.17718955407419537 + ], + [ + 149, + 0.35001015862234297 + ], + [ + 153, + 0.0971190303278604 + ], + [ + 161, + 0.09602663208041633 + ], + [ + 170, + 0.08013917525830087 + ], + [ + 176, + 0.16043758522506582 + ], + [ + 177, + 0.4302131395546546 + ], + [ + 181, + 0.1141325770703513 + ], + [ + 183, + 0.08463219023650492 + ], + [ + 185, + 0.1170549398497383 + ], + [ + 186, + 0.12488968355665885 + ], + [ + 188, + 0.21596055896665428 + ], + [ + 195, + 0.13856457065747854 + ], + [ + 198, + 0.16713749520661988 + ], + [ + 200, + 0.14797280544337985 + ], + [ + 208, + 0.17684606309608142 + ], + [ + 212, + 0.10218270855903731 + ], + [ + 215, + 0.17911769829168617 + ], + [ + 223, + 0.28355597573566127 + ], + [ + 227, + 0.18614806737838543 + ], + [ + 231, + 0.17126204970405534 + ], + [ + 236, + 0.2397341994093023 + ], + [ + 244, + 0.33075344019325725 + ], + [ + 249, + 0.15879121220237247 + ], + [ + 252, + 0.16377672285870556 + ], + [ + 259, + 0.2194409503258778 + ], + [ + 263, + 0.29795719969014406 + ], + [ + 268, + 0.35068570392773746 + ], + [ + 274, + 0.254532281773336 + ], + [ + 276, + 0.22171794721215612 + ], + [ + 281, + 0.2449382913306728 + ], + [ + 284, + 0.2654727404348664 + ], + [ + 288, + 0.24713826491566698 + ], + [ + 289, + 0.1596314041062026 + ], + [ + 290, + 0.204968461284532 + ], + [ + 291, + 0.13532733286611467 + ], + [ + 292, + 0.19083334871017835 + ], + [ + 293, + 0.293571100089461 + ], + [ + 298, + 0.5613210333061828 + ], + [ + 304, + 0.45486879011361353 + ], + [ + 308, + 0.7242093244259256 + ], + [ + 309, + 0.9701379608595333 + ], + [ + 313, + 0.16984488295740513 + ], + [ + 318, + 0.13823321100735164 + ], + [ + 326, + 0.21299123905406145 + ], + [ + 329, + 0.2437362240122314 + ], + [ + 333, + 0.20815761051474196 + ], + [ + 338, + 0.16585813685730522 + ], + [ + 345, + 0.060841230190952854 + ], + [ + 349, + 0.0632307279295016 + ], + [ + 351, + 0.037437252476906505 + ], + [ + 353, + 0.06060769494380033 + ], + [ + 354, + 0.06922294274416424 + ], + [ + 356, + 0.02523073658981012 + ], + [ + 358, + 0.08829576069323714 + ], + [ + 361, + 0.17449880071434803 + ], + [ + 363, + 0.07933297656123231 + ], + [ + 368, + 0.09611484627570144 + ], + [ + 372, + 0.06296831777572551 + ], + [ + 377, + 0.03601069913762158 + ], + [ + 380, + 0.0 + ], + [ + 384, + 0.015142548256459635 + ], + [ + 390, + 0.06770549504034223 + ], + [ + 391, + 0.28500743519464883 + ], + [ + 395, + 0.043425469094958466 + ], + [ + 398, + 0.1935030717755811 + ], + [ + 403, + 0.05135511888472319 + ], + [ + 409, + 0.048885545659965515 + ], + [ + 410, + 0.37368047910654845 + ], + [ + 412, + 0.11403587192579823 + ], + [ + 414, + 0.056131765305109854 + ], + [ + 417, + 0.0761254662194514 + ], + [ + 420, + 0.10262194903580975 + ], + [ + 426, + 0.07800881407082455 + ], + [ + 430, + 0.038086038852146224 + ], + [ + 432, + 0.06506381360702107 + ], + [ + 435, + 0.14383524070768988 + ], + [ + 436, + 0.19408876946476278 + ], + [ + 437, + 0.052376375768826526 + ], + [ + 439, + 0.2195367623623822 + ], + [ + 441, + 0.08026467519342623 + ], + [ + 443, + 0.13535839727343132 + ], + [ + 446, + 0.3201744915224922 + ], + [ + 450, + 0.2235470919912535 + ], + [ + 458, + 0.21734514417164935 + ], + [ + 460, + 0.3625486019017947 + ], + [ + 464, + 0.17857527104518095 + ], + [ + 466, + 0.10872751386527808 + ], + [ + 468, + 0.15748419927953627 + ], + [ + 469, + 0.15313950480350838 + ], + [ + 471, + 0.11389857059207849 + ], + [ + 477, + 0.48776733036317427 + ], + [ + 481, + 0.4693091276032504 + ], + [ + 488, + 0.45930300058624957 + ], + [ + 492, + 0.213335793513599 + ], + [ + 497, + 0.19550171101932368 + ], + [ + 503, + 0.1990713523357782 + ], + [ + 506, + 0.3813483302612577 + ], + [ + 515, + 0.2697264504580548 + ], + [ + 522, + 0.1858018713849023 + ], + [ + 528, + 0.24906778428968385 + ], + [ + 532, + 0.21065568713751404 + ], + [ + 535, + 0.19075562870348606 + ], + [ + 536, + 0.25162342345020766 + ], + [ + 538, + 0.17269697652287058 + ], + [ + 540, + 0.1520003299375057 + ], + [ + 541, + 0.2469163167482206 + ], + [ + 546, + 0.277963651570972 + ], + [ + 547, + 0.24935316915784997 + ], + [ + 548, + 0.09283610397521376 + ], + [ + 549, + 0.13665401918115952 + ], + [ + 550, + 0.48506451226969477 + ], + [ + 554, + 0.12104898428548179 + ], + [ + 556, + 0.07199219191803605 + ], + [ + 558, + 0.11106931922837808 + ], + [ + 559, + 0.042371639796345055 + ], + [ + 561, + 0.042210815105403005 + ], + [ + 565, + 0.1802400710977153 + ], + [ + 570, + 0.1567259647440145 + ], + [ + 573, + 0.2992614244391781 + ], + [ + 577, + 0.11202392733277677 + ], + [ + 579, + 0.043121375945665806 + ], + [ + 582, + 0.04541817163788712 + ], + [ + 586, + 0.12276171469532192 + ], + [ + 592, + 0.1388622724389151 + ], + [ + 596, + 0.271645197434575 + ], + [ + 601, + 0.3700834178180053 + ], + [ + 608, + 0.11406719493555895 + ], + [ + 613, + 0.06480277894781346 + ], + [ + 619, + 0.0467503680442238 + ], + [ + 624, + 0.049885137067934876 + ], + [ + 626, + 0.1506916082102842 + ], + [ + 629, + 0.08020124018291859 + ], + [ + 630, + 0.149825534089291 + ], + [ + 634, + 0.04744071048005492 + ], + [ + 636, + 0.06700975305308754 + ], + [ + 637, + 0.06567283115189392 + ], + [ + 638, + 0.0529043317577789 + ], + [ + 639, + 0.1191946935455915 + ], + [ + 641, + 0.2063408029314597 + ], + [ + 642, + 1.0 + ], + [ + 645, + 0.2106864469686188 + ], + [ + 650, + 0.11904335500077405 + ], + [ + 654, + 0.22516240804356324 + ], + [ + 660, + 0.2673202294234142 + ], + [ + 663, + 0.30481158558812776 + ], + [ + 667, + 0.6352936100087709 + ], + [ + 671, + 0.3026346032816017 + ] + ], + [ + [ + 50, + 0.2675865024176666 + ], + [ + 58, + 0.1762063342931574 + ], + [ + 61, + 0.17116682994083054 + ], + [ + 66, + 0.06949181209044343 + ], + [ + 74, + 0.15699094704809421 + ], + [ + 75, + 0.16064482590429596 + ], + [ + 78, + 0.0512957555678645 + ], + [ + 84, + 0.09372889437397289 + ], + [ + 85, + 0.10168042522632056 + ], + [ + 86, + 0.060737872080519104 + ], + [ + 87, + 0.04144939469550792 + ], + [ + 88, + 0.051020845509843975 + ], + [ + 97, + 0.11998989937514071 + ], + [ + 100, + 0.14781797319521667 + ], + [ + 105, + 0.10106011276399046 + ], + [ + 110, + 0.10042226735819065 + ], + [ + 115, + 0.10151155763648634 + ], + [ + 117, + 0.07648714011716017 + ], + [ + 123, + 0.04626795632307752 + ], + [ + 125, + 0.07055716068342234 + ], + [ + 126, + 0.04168294234520327 + ], + [ + 127, + 0.018297683409000128 + ], + [ + 128, + 0.014233760595648706 + ], + [ + 133, + 0.03897650343688624 + ], + [ + 137, + 0.03547300322607352 + ], + [ + 139, + 0.04480755752677844 + ], + [ + 140, + 0.05217056511295904 + ], + [ + 142, + 0.07519462459789103 + ], + [ + 144, + 0.05116643974955315 + ], + [ + 145, + 0.017538453824407847 + ], + [ + 149, + 0.01498560930114294 + ], + [ + 157, + 0.029639230907229974 + ], + [ + 159, + 0.058558736729438066 + ], + [ + 163, + 0.04842314984214837 + ], + [ + 168, + 0.05432948288441072 + ], + [ + 173, + 0.04155009723005758 + ], + [ + 178, + 0.043803960993128055 + ], + [ + 188, + 0.07142131559251072 + ], + [ + 189, + 0.08710317125777164 + ], + [ + 193, + 0.038995683894082464 + ], + [ + 196, + 0.04149155540938913 + ], + [ + 198, + 0.037286914958818765 + ], + [ + 199, + 0.05959662270179042 + ], + [ + 202, + 0.015758801751601638 + ], + [ + 205, + 0.02598579286473565 + ], + [ + 206, + 0.07496446938007527 + ], + [ + 216, + 0.034802203951754554 + ], + [ + 217, + 0.11970522467005348 + ], + [ + 221, + 0.05996816161826891 + ], + [ + 225, + 0.06276370768026894 + ], + [ + 232, + 0.05937470534599412 + ], + [ + 239, + 0.06432827097567524 + ], + [ + 240, + 0.2978034778891031 + ], + [ + 245, + 0.5594202497818335 + ], + [ + 249, + 1.0 + ], + [ + 255, + 0.21858893956542477 + ], + [ + 259, + 0.08814583547468051 + ], + [ + 266, + 0.12442808982250103 + ], + [ + 270, + 0.06944346296822217 + ], + [ + 274, + 0.09799555562620522 + ], + [ + 279, + 0.10430026485155104 + ], + [ + 285, + 0.09280203580538289 + ], + [ + 292, + 0.12109622618639494 + ], + [ + 295, + 0.173449545511153 + ], + [ + 297, + 0.13003987275908227 + ], + [ + 304, + 0.1901438955457509 + ], + [ + 315, + 0.21969578713829677 + ], + [ + 316, + 0.14694739238115184 + ], + [ + 321, + 0.05574221042875211 + ], + [ + 322, + 0.034637188310917555 + ], + [ + 323, + 0.05055492317098729 + ], + [ + 324, + 0.04354690323013833 + ], + [ + 330, + 0.054756410299942 + ], + [ + 333, + 0.06005319040167101 + ], + [ + 334, + 0.043990479321242996 + ], + [ + 335, + 0.03685426033903933 + ], + [ + 336, + 0.052894852253119344 + ], + [ + 337, + 0.06210670484813883 + ], + [ + 338, + 0.07207986090588334 + ], + [ + 339, + 0.13031144392831912 + ], + [ + 343, + 0.02395768337038382 + ], + [ + 351, + 0.03131152379177072 + ], + [ + 356, + 0.03782822406970449 + ], + [ + 361, + 0.03715846624527436 + ], + [ + 369, + 0.06348407971991753 + ], + [ + 372, + 0.03822936234306993 + ], + [ + 377, + 0.03562022048170917 + ], + [ + 384, + 0.05277631634698924 + ], + [ + 389, + 0.05090171593546961 + ], + [ + 391, + 0.061171965322604306 + ], + [ + 394, + 0.12504413321311272 + ], + [ + 396, + 0.11923366578347815 + ], + [ + 399, + 0.10837895111007281 + ], + [ + 400, + 0.14524453680407798 + ], + [ + 401, + 0.14219853344922667 + ], + [ + 402, + 0.03232718160456888 + ], + [ + 403, + 0.03709513605820387 + ], + [ + 404, + 0.032354732681323745 + ], + [ + 405, + 0.07185854029889298 + ], + [ + 412, + 0.1161744251083834 + ], + [ + 415, + 0.12504105235804333 + ], + [ + 416, + 0.12436068141184306 + ], + [ + 417, + 0.18549450384314067 + ], + [ + 418, + 0.007807248455161697 + ], + [ + 419, + 0.014186493117311556 + ], + [ + 420, + 0.015560307079825307 + ], + [ + 424, + 0.04290837687351326 + ], + [ + 433, + 0.031961472137647724 + ], + [ + 434, + 0.04917444001247087 + ], + [ + 435, + 0.04088431068971653 + ], + [ + 437, + 0.012330535531627837 + ], + [ + 439, + 0.01895480176219334 + ], + [ + 441, + 0.03085032674114828 + ], + [ + 443, + 0.03346368515798543 + ], + [ + 446, + 0.058769980672471286 + ], + [ + 447, + 0.08357387713725714 + ], + [ + 448, + 0.10888847601237157 + ], + [ + 449, + 0.0 + ], + [ + 450, + 0.002305296270822538 + ], + [ + 451, + 0.006016756105834608 + ], + [ + 452, + 0.03615056440663245 + ], + [ + 457, + 0.00950893136508839 + ], + [ + 458, + 0.01105450329110841 + ], + [ + 459, + 0.003955702938155857 + ], + [ + 460, + 0.005800749834480238 + ], + [ + 461, + 0.01750808696758169 + ], + [ + 462, + 0.014944077388578515 + ], + [ + 463, + 0.0022234318843888123 + ], + [ + 464, + 0.0031858252454143227 + ], + [ + 468, + 0.012257093694221111 + ], + [ + 469, + 0.008121909934440816 + ], + [ + 470, + 0.0007905757666466704 + ], + [ + 471, + 0.01911177090256873 + ], + [ + 484, + 0.015773916307457097 + ], + [ + 485, + 0.053021275570704164 + ], + [ + 489, + 0.028646772951156738 + ], + [ + 496, + 0.08146910823216785 + ], + [ + 504, + 0.090035733662129 + ], + [ + 515, + 0.09988283738993131 + ], + [ + 516, + 0.04942891259743526 + ], + [ + 517, + 0.050341076593412526 + ], + [ + 521, + 0.027999457752051267 + ], + [ + 524, + 0.03404784634880998 + ], + [ + 528, + 0.02881738105834163 + ], + [ + 532, + 0.022873995166854625 + ], + [ + 535, + 0.017958650139315982 + ], + [ + 536, + 0.02043841831181379 + ], + [ + 537, + 0.02315199106752061 + ], + [ + 538, + 0.003445337262557878 + ], + [ + 539, + 0.007275812471121937 + ], + [ + 540, + 0.015241317704768989 + ], + [ + 541, + 0.03047997926857388 + ], + [ + 546, + 0.014002750301925914 + ], + [ + 550, + 0.011621963363464263 + ], + [ + 558, + 0.014518770201798856 + ], + [ + 563, + 0.028624294165905655 + ], + [ + 566, + 0.02714345210216061 + ], + [ + 571, + 0.02321482029658414 + ], + [ + 576, + 0.012585662882643187 + ], + [ + 582, + 0.014349905472486183 + ], + [ + 587, + 0.0112891972031365 + ], + [ + 588, + 0.07757250722680636 + ], + [ + 591, + 0.03470413947820579 + ], + [ + 595, + 0.06112358568815304 + ], + [ + 599, + 0.050457846457538036 + ], + [ + 600, + 0.057830377380120215 + ], + [ + 601, + 0.050067980399449395 + ], + [ + 602, + 0.05131176774071689 + ], + [ + 605, + 0.07084583703813524 + ], + [ + 611, + 0.061375253348358735 + ], + [ + 614, + 0.04268474775227535 + ], + [ + 618, + 0.046277085200890314 + ], + [ + 624, + 0.07080445629313338 + ], + [ + 628, + 0.08069680819108992 + ], + [ + 629, + 0.10156684843772576 + ], + [ + 630, + 0.05108112931479721 + ], + [ + 631, + 0.0335992826816478 + ] + ], + [ + [ + 53, + 1.0 + ], + [ + 55, + 0.31102738520915757 + ], + [ + 59, + 0.09100878882295611 + ], + [ + 63, + 0.15754924703465212 + ], + [ + 70, + 0.26421523943006975 + ], + [ + 73, + 0.32525072466992294 + ], + [ + 77, + 0.1727151748570381 + ], + [ + 85, + 0.4666846825835882 + ], + [ + 87, + 0.28290678717244977 + ], + [ + 90, + 0.23698615566980819 + ], + [ + 92, + 0.19429024179308133 + ], + [ + 94, + 0.27924172736426917 + ], + [ + 96, + 0.3818209371154489 + ], + [ + 110, + 0.04948558567967704 + ], + [ + 121, + 0.0642069234533485 + ], + [ + 124, + 0.052278119084293515 + ], + [ + 133, + 0.020541000709472176 + ], + [ + 141, + 0.04054188448415822 + ], + [ + 145, + 0.06514939197751846 + ], + [ + 156, + 0.05072517737619629 + ], + [ + 157, + 0.16177505490828706 + ], + [ + 158, + 0.1199221571977675 + ], + [ + 159, + 0.04943747571551727 + ], + [ + 160, + 0.03102926347560799 + ], + [ + 161, + 0.028700009989724124 + ], + [ + 162, + 0.030199387538893053 + ], + [ + 163, + 0.24763306625250844 + ], + [ + 165, + 0.15595449372186046 + ], + [ + 166, + 0.01897361752805972 + ], + [ + 170, + 0.03703302430887045 + ], + [ + 178, + 0.09444440233342287 + ], + [ + 187, + 0.033274597340001516 + ], + [ + 194, + 0.05108696504903368 + ], + [ + 197, + 0.07062907500523434 + ], + [ + 201, + 0.05127545398157981 + ], + [ + 209, + 0.10715326472623374 + ], + [ + 211, + 0.12865468147400477 + ], + [ + 214, + 0.10189735215520483 + ], + [ + 216, + 0.049458385229643936 + ], + [ + 218, + 0.038922533208979176 + ], + [ + 220, + 0.04713605792420596 + ], + [ + 234, + 0.017264438539021022 + ], + [ + 245, + 0.019561900633951513 + ], + [ + 248, + 0.05426207156112987 + ], + [ + 257, + 0.017569355387698108 + ], + [ + 265, + 0.03506830097058022 + ], + [ + 269, + 0.08149759897050933 + ], + [ + 280, + 0.07239015421550118 + ], + [ + 281, + 0.5318188186173302 + ], + [ + 285, + 0.38038382625724604 + ], + [ + 286, + 0.6124198611241433 + ], + [ + 287, + 0.1582232701247253 + ], + [ + 288, + 0.20780297910758921 + ], + [ + 289, + 0.20609002046656627 + ], + [ + 290, + 0.1344015770947872 + ], + [ + 291, + 0.20009141052266619 + ], + [ + 292, + 0.16621344169332566 + ], + [ + 293, + 0.17343456013397276 + ], + [ + 299, + 0.351591407806439 + ], + [ + 303, + 0.19556503971764408 + ], + [ + 314, + 0.26501821185910723 + ], + [ + 317, + 0.15893191245143537 + ], + [ + 322, + 0.216167856932261 + ], + [ + 325, + 0.25239963266851945 + ], + [ + 329, + 0.2580693238070832 + ], + [ + 337, + 0.7904769775249342 + ], + [ + 343, + 0.07334939255920732 + ], + [ + 352, + 0.04643567059498337 + ], + [ + 355, + 0.06317331165463246 + ], + [ + 357, + 0.058926040151225 + ], + [ + 370, + 0.06476750873076145 + ], + [ + 381, + 0.1638092882100048 + ], + [ + 382, + 0.14533678529882302 + ], + [ + 391, + 0.08643527841897221 + ], + [ + 395, + 0.08618148921114434 + ], + [ + 402, + 0.08916068217129436 + ], + [ + 407, + 0.059804799692987055 + ], + [ + 412, + 0.08712977674497692 + ], + [ + 423, + 0.08313797429472367 + ], + [ + 425, + 0.10049731879214041 + ], + [ + 426, + 0.06753891680778919 + ], + [ + 428, + 0.09717261006998923 + ], + [ + 429, + 0.10416598008689186 + ], + [ + 430, + 0.09912636138823765 + ], + [ + 437, + 0.029186354905932996 + ], + [ + 441, + 0.04445646071177035 + ], + [ + 446, + 0.03936991716710799 + ], + [ + 447, + 0.06956228520081481 + ], + [ + 448, + 0.0 + ], + [ + 449, + 0.02188859882481404 + ], + [ + 450, + 0.008597902012644267 + ], + [ + 456, + 0.04208163877013868 + ], + [ + 459, + 0.08262015136779749 + ], + [ + 468, + 0.060231742881707795 + ], + [ + 473, + 0.16138280678255473 + ], + [ + 474, + 0.17722262513146228 + ], + [ + 480, + 0.06122970728088047 + ], + [ + 484, + 0.09123379583245275 + ], + [ + 488, + 0.1093127080142512 + ], + [ + 494, + 0.059069822821363166 + ], + [ + 500, + 0.17569928963673326 + ], + [ + 506, + 0.06078447933635547 + ], + [ + 512, + 0.1516981972918576 + ], + [ + 514, + 0.08674464675171324 + ], + [ + 518, + 0.19180352348776783 + ], + [ + 520, + 0.12390521575615485 + ], + [ + 521, + 0.04095400706780604 + ], + [ + 522, + 0.03749780802853281 + ], + [ + 523, + 0.03566420648069109 + ], + [ + 524, + 0.10242964784044443 + ], + [ + 528, + 0.10291787515908654 + ], + [ + 532, + 0.0381807351472423 + ], + [ + 536, + 0.060874137899123236 + ], + [ + 538, + 0.15592896940537423 + ], + [ + 539, + 0.0166106695688752 + ], + [ + 540, + 0.015570802626411655 + ], + [ + 541, + 0.11881325990680629 + ], + [ + 543, + 0.3784739004491685 + ], + [ + 547, + 0.1360980336946483 + ], + [ + 551, + 0.13757707705550465 + ], + [ + 556, + 0.2289552499730611 + ], + [ + 557, + 0.2765788077071378 + ], + [ + 561, + 0.1135934788574113 + ], + [ + 563, + 0.14852563865122387 + ], + [ + 566, + 0.2676204927056866 + ], + [ + 570, + 0.12999560421914047 + ], + [ + 575, + 0.04694050812664548 + ], + [ + 581, + 0.10945997021249272 + ], + [ + 585, + 0.1968862857896833 + ], + [ + 590, + 0.10132248129921337 + ], + [ + 592, + 0.10123057445860986 + ], + [ + 597, + 0.11068670271424906 + ], + [ + 604, + 0.09708151663933856 + ], + [ + 607, + 0.24950924350135623 + ], + [ + 613, + 0.18291005115221814 + ], + [ + 617, + 0.32104862340135415 + ], + [ + 625, + 0.14978546325750225 + ], + [ + 630, + 0.07534054058562648 + ], + [ + 633, + 0.19730668405073268 + ], + [ + 637, + 0.20305052063499754 + ], + [ + 643, + 0.12546930022614838 + ], + [ + 647, + 0.14790179245620755 + ], + [ + 652, + 0.09119485926606619 + ], + [ + 653, + 0.18925219900992896 + ], + [ + 656, + 0.09260264070359664 + ], + [ + 659, + 0.22557977471317536 + ], + [ + 660, + 0.47185036815933995 + ], + [ + 672, + 0.20991547368815924 + ], + [ + 677, + 0.28925227987369545 + ], + [ + 682, + 0.1542141092727666 + ], + [ + 689, + 0.17186002435356823 + ], + [ + 691, + 0.15215971994814972 + ], + [ + 695, + 0.15656924711774295 + ], + [ + 698, + 0.29318584344393556 + ], + [ + 702, + 0.47204877231529424 + ], + [ + 704, + 0.5829918661947627 + ] + ] + ] + ] + }, + { + "data": { + "question": "Who is the original artist of sound of silence?", + "answer": "There are several songs with the title \"Sound of Silence\". Sounds of Silence is the second studio album by Simon & Garfunkel, released on January 17, 1966. The album's title is a slight modification of the title of the duo's first major hit, \"The Sound of Silence\", which was recorded in March 1964 and originally was released as \"The Sounds of Silence\". Another \"Sound of Silence\" is a song performed by Australian recording artist Dami Im, and is best known as Australia's entry at the Eurovision Song Contest 2016.", + "qa_pairs": [ + { + "context": "Sounds of Silence is the second studio album by Simon & Garfunkel, released on January 17, 1966. The album's title is a slight modification of the title of the duo's first major hit, \"The Sound of Silence\", which originally was released as \"The Sounds of Silence\". The song had earlier been released in an acoustic version on the album \"Wednesday Morning, 3 A.M.\", and later on the soundtrack to the movie \"The Graduate\". Without the knowledge of Paul Simon or Art Garfunkel, electric guitars, bass and drums were overdubbed by Columbia Records staff producer Tom Wilson on June 15, 1965. This new version was released as a single in September 1965, and opens the album.", + "question": "Who is the original artist of sound of silence, the song, released in 1964?", + "short_answers": [ + "Simon & Garfunkel", + "Paul Simon and Art Garfunkel", + "Art Garfunkel", + "Paul Simon" + ], + "wikipage": "Sounds of Silence" + }, + { + "context": "Sounds of Silence is the second studio album by Simon & Garfunkel, released on January 17, 1966. The album's title is a slight modification of the title of the duo's first major hit, \"The Sound of Silence\", which originally was released as \"The Sounds of Silence\". The song had earlier been released in an acoustic version on the album \"Wednesday Morning, 3 A.M.\", and later on the soundtrack to the movie \"The Graduate\". Without the knowledge of Paul Simon or Art Garfunkel, electric guitars, bass and drums were overdubbed by Columbia Records staff producer Tom Wilson on June 15, 1965. This new version was released as a single in September 1965, and opens the album.", + "question": "Who is the original artist of sound of silence, the album?", + "short_answers": [ + "Simon & Garfunkel", + "Paul Simon and Art Garfunkel", + "Art Garfunkel", + "Paul Simon" + ], + "wikipage": "Sounds of Silence" + }, + { + "context": "\"Sound of Silence\" is a song performed by Australian recording artist Dami Im. Written by Anthony Egizii and David Musumeci of DNA Songs, it is best known as Australia's entry at the Eurovision Song Contest 2016 which was held in Stockholm, Sweden, where it finished 2nd, receiving a total of 511 points. The song also won the Marcel Bezen\u00e7on Award in the composer category. The song was leaked on 10 March 2016, one day before its initial release date. It is Dami Im's fourth Australian top 20 hit and worldwide, it reached the top 40 in more than six countries after the Eurovision Song Contest 2016 Final.", + "question": "Who is the original artist of sound of silence, the song, released in 2016?", + "short_answers": [ + "Dami Im" + ], + "wikipage": "Sound of Silence (Dami Im song)" + } + ], + "docs": [ + "Document [1](Title:The Sound of Silence): The Sound of Silence \"The Sound of Silence\", originally \"The Sounds of Silence\", is a song by the American music duo Simon & Garfunkel. The song was written by Paul Simon over a period of several months in 1963 and 1964. A studio audition led to the duo signing a record deal with Columbia Records, and the song was recorded in March 1964 at Columbia Studios in New York City for inclusion on their debut album, \"Wednesday Morning, 3 A.M.\". Released in October 1964, the album was a commercial failure and led to the duo breaking apart, with Paul Simon\n", + "Document [2](Title:Sounds of Silence): Sounds of Silence Sounds of Silence is the second studio album by Simon & Garfunkel, released on January 17, 1966. The album's title is a slight modification of the title of the duo's first major hit, \"The Sound of Silence\", which originally was released as \"The Sounds of Silence\". The song had earlier been released in an acoustic version on the album \"Wednesday Morning, 3 A.M.\", and later on the soundtrack to the movie \"The Graduate\". Without the knowledge of Paul Simon or Art Garfunkel, electric guitars, bass and drums were overdubbed by Columbia Records staff producer Tom Wilson on\n", + "Document [3](Title:The Sound of Silence): downloadable content for the video game, \"Rock Band 4\". The Disturbed version was used in the episode \"Ian Garvey\" of \"The Blacklist\" in November 2017. Notes Bibliography The Sound of Silence \"The Sound of Silence\", originally \"The Sounds of Silence\", is a song by the American music duo Simon & Garfunkel. The song was written by Paul Simon over a period of several months in 1963 and 1964. A studio audition led to the duo signing a record deal with Columbia Records, and the song was recorded in March 1964 at Columbia Studios in New York City for inclusion on\n", + "Document [4](Title:Immortalized (Disturbed album)): 1964 song \"The Sound of Silence\" by Simon & Garfunkel, which marked Disturbed's highest ranked single on the \"Billboard\" Hot 100 at number 42 and has since become Disturbed's biggest song since \"Down with the Sickness\". In 2011, following the tour of their fifth studio album, \"Asylum\", Disturbed announced that they would go on a hiatus. During the hiatus, the band released a compilation album of previously recorded B-sides, \"The Lost Children\" (2011), and a box set of their five studio albums, \"The Collection\" (2012). In January 2014, band members David Draiman (vocals), Dan Donegan (guitar), Mike Wengren (drums) met\n", + "Document [5](Title:Sounds of Silence): Angeles, California. \"The Sound of Silence\" (electric overdubs) personnel \"The Sound of Silence\" overdubs were recorded at Columbia's \"Studio A\" at 799 Seventh Avenue near 52nd Street by Columbia Records staff producer Tom Wilson on June 15, 1965. Sounds of Silence Sounds of Silence is the second studio album by Simon & Garfunkel, released on January 17, 1966. The album's title is a slight modification of the title of the duo's first major hit, \"The Sound of Silence\", which originally was released as \"The Sounds of Silence\". The song had earlier been released in an acoustic version on the album\n" + ] + }, + "doc_cache": [ + "Document [1](Title:The Sound of Silence)attention of Columbia Records producer Tom Wilson, who worked with Bob Dylan. Simon convinced Wilson to let him and his partner have a studio audition, where a performance of \"The Sound of Silence\" got the duo signed to Columbia. The song's origin and basis remain unclear, with multiple answers coming forward over the years. Many believe that the song commented on the John F. Kennedy assassination, as the song was released three months after the assassination. Simon stated unambiguously in interviews, however, \"I wrote The Sound of Silence when I was 21 years old\", which places the timeframe firmly prior\n", + "Document [2](Title:Live at Benaroya Hall with the Seattle Symphony)proud of, believing it truly represents their live act. Overall, critical reception of the album was positive. Nick Vissey of \"The Seattle Times\" wrote that \"standout\" tracks included \"Dreams\" and \"Pride and Joy\", which highlighted Carlile's \"unique folk-rock style\" and \"enthralling, emotional and fun\" voice. The Hanseroth brothers' cover of Paul Simon's \"The Sound of Silence\" was deemed \"spot-on\" by Allmusic's Andrew Leahey and \"breathtaking\" by Gene Stout of the \"Seattle Post-Intelligencer\". \"The Source Weekly\" contributor Mike Bookey complimented the album for capturing Carlile's live act and recommended this album for first-time listeners of her music. \"Glide\" magazine's reviewer complimented\n", + "Document [3](Title:Sounds of Silence)Angeles, California. \"The Sound of Silence\" (electric overdubs) personnel \"The Sound of Silence\" overdubs were recorded at Columbia's \"Studio A\" at 799 Seventh Avenue near 52nd Street by Columbia Records staff producer Tom Wilson on June 15, 1965. Sounds of Silence Sounds of Silence is the second studio album by Simon & Garfunkel, released on January 17, 1966. The album's title is a slight modification of the title of the duo's first major hit, \"The Sound of Silence\", which originally was released as \"The Sounds of Silence\". The song had earlier been released in an acoustic version on the album\n", + "Document [4](Title:The Sound of Silence)downloadable content for the video game, \"Rock Band 4\". The Disturbed version was used in the episode \"Ian Garvey\" of \"The Blacklist\" in November 2017. Notes Bibliography The Sound of Silence \"The Sound of Silence\", originally \"The Sounds of Silence\", is a song by the American music duo Simon & Garfunkel. The song was written by Paul Simon over a period of several months in 1963 and 1964. A studio audition led to the duo signing a record deal with Columbia Records, and the song was recorded in March 1964 at Columbia Studios in New York City for inclusion on\n", + "Document [5](Title:Paul Simon in Concert: Live Rhymin')UK. Further, two of the live performances were released as a single as part of the promotion for the album: the breakthrough Simon & Garfunkel \"The Sound of Silence\" as the A-side, along with Simon's debut single \"Mother and Child Reunion\" on the B-side. All tracks composed by Paul Simon, except where indicated. Paul Simon in Concert: Live Rhymin' Paul Simon in Concert: Live Rhymin' is a live album by Paul Simon, released in March 1974 by Columbia Records. It was recorded in the wake of the release of \"There Goes Rhymin' Simon\" \u2013 which produced a number of hit\n", + "Document [6](Title:The Sound of Silence)attention of Columbia Records producer Tom Wilson, who worked with Bob Dylan. Simon convinced Wilson to let him and his partner have a studio audition, where a performance of \"The Sound of Silence\" got the duo signed to Columbia. The song's origin and basis remain unclear, with multiple answers coming forward over the years. Many believe that the song commented on the John F. Kennedy assassination, as the song was released three months after the assassination. Simon stated unambiguously in interviews, however, \"I wrote The Sound of Silence when I was 21 years old\", which places the timeframe firmly prior\n" + ], + "log": [ + "gpt-3.5-turbo-[1] -> retriever-[4]\n: Query: \"Who wrote the song 'The Sound of Silence'?\"", + "retriever-[4] -> gpt-3.5-turbo-[3]\n: Document [1](Title:The Sound of Silence)attention of Columbia Records producer Tom Wilson, who worked with Bob Dylan. Simon convinced Wilson to let him and his partner have a studio audition, where a performance of \"The Sound of Silence\" got the duo signed to Columbia. The song's origin and basis remain unclear, with multiple answers coming forward over the years. Many believe that the song commented on the John F. Kennedy assassination, as the song was released three months after the assassination. Simon stated unambiguously in interviews, however, \"I wrote The Sound of Silence when I was 21 years old\", which places the timeframe firmly prior\n\nDocument [2](Title:Live at Benaroya Hall with the Seattle Symphony)proud of, believing it truly represents their live act. Overall, critical reception of the album was positive. Nick Vissey of \"The Seattle Times\" wrote that \"standout\" tracks included \"Dreams\" and \"Pride and Joy\", which highlighted Carlile's \"unique folk-rock style\" and \"enthralling, emotional and fun\" voice. The Hanseroth brothers' cover of Paul Simon's \"The Sound of Silence\" was deemed \"spot-on\" by Allmusic's Andrew Leahey and \"breathtaking\" by Gene Stout of the \"Seattle Post-Intelligencer\". \"The Source Weekly\" contributor Mike Bookey complimented the album for capturing Carlile's live act and recommended this album for first-time listeners of her music. \"Glide\" magazine's reviewer complimented\n", + "gpt-3.5-turbo-[3] -> evaluator-[2]\n: ['The original artist of \"The Sound of Silence\" is Simon & Garfunkel[1]. ', 'The original artist of \"Sound of Silence\" is Paul Simon, covered by the Hanseroth brothers[2]. ']", + "evaluator-[2] -> gpt-3.5-turbo-[1]\n: {'answer': 'The original artist of \"Sound of Silence\" is Paul Simon, covered by the Hanseroth brothers[2]. ', 'ans': 'The original artist of \"The Sound of Silence\" is Simon & Garfunkel[1]. '}", + "gpt-3.5-turbo-[1] -> retriever-[4]\n: \"Who recorded the original version of 'The Sound of Silence'?\"", + "retriever-[4] -> gpt-3.5-turbo-[3]\n: Document [3](Title:Sounds of Silence)Angeles, California. \"The Sound of Silence\" (electric overdubs) personnel \"The Sound of Silence\" overdubs were recorded at Columbia's \"Studio A\" at 799 Seventh Avenue near 52nd Street by Columbia Records staff producer Tom Wilson on June 15, 1965. Sounds of Silence Sounds of Silence is the second studio album by Simon & Garfunkel, released on January 17, 1966. The album's title is a slight modification of the title of the duo's first major hit, \"The Sound of Silence\", which originally was released as \"The Sounds of Silence\". The song had earlier been released in an acoustic version on the album\n\nDocument [4](Title:The Sound of Silence)downloadable content for the video game, \"Rock Band 4\". The Disturbed version was used in the episode \"Ian Garvey\" of \"The Blacklist\" in November 2017. Notes Bibliography The Sound of Silence \"The Sound of Silence\", originally \"The Sounds of Silence\", is a song by the American music duo Simon & Garfunkel. The song was written by Paul Simon over a period of several months in 1963 and 1964. A studio audition led to the duo signing a record deal with Columbia Records, and the song was recorded in March 1964 at Columbia Studios in New York City for inclusion on\n", + "gpt-3.5-turbo-[3] -> evaluator-[2]\n: ['The original artist of \"The Sound of Silence\" is Simon & Garfunkel[3]. ', '\"The Sound of Silence\" was originally performed by the American music duo Simon & Garfunkel[4]. ']", + "evaluator-[2] -> gpt-3.5-turbo-[1]\n: {'answer': '\"The Sound of Silence\" was originally performed by the American music duo Simon & Garfunkel[4]. ', 'ans': 'The original artist of \"The Sound of Silence\" is Simon & Garfunkel[3]. '}", + "gpt-3.5-turbo-[1] -> retriever-[4]\n: \"Who composed the song 'The Sound of Silence'?\"", + "retriever-[4] -> gpt-3.5-turbo-[3]\n: Document [5](Title:Paul Simon in Concert: Live Rhymin')UK. Further, two of the live performances were released as a single as part of the promotion for the album: the breakthrough Simon & Garfunkel \"The Sound of Silence\" as the A-side, along with Simon's debut single \"Mother and Child Reunion\" on the B-side. All tracks composed by Paul Simon, except where indicated. Paul Simon in Concert: Live Rhymin' Paul Simon in Concert: Live Rhymin' is a live album by Paul Simon, released in March 1974 by Columbia Records. It was recorded in the wake of the release of \"There Goes Rhymin' Simon\" \u2013 which produced a number of hit\n\nDocument [6](Title:The Sound of Silence)attention of Columbia Records producer Tom Wilson, who worked with Bob Dylan. Simon convinced Wilson to let him and his partner have a studio audition, where a performance of \"The Sound of Silence\" got the duo signed to Columbia. The song's origin and basis remain unclear, with multiple answers coming forward over the years. Many believe that the song commented on the John F. Kennedy assassination, as the song was released three months after the assassination. Simon stated unambiguously in interviews, however, \"I wrote The Sound of Silence when I was 21 years old\", which places the timeframe firmly prior\n", + "gpt-3.5-turbo-[3] -> evaluator-[2]\n: ['The original artist of \"The Sound of Silence\" is Paul Simon, from the duo Simon & Garfunkel[5]. ', 'The original artist of \"The Sound of Silence\" is Simon & Garfunkel, with Paul Simon specifically writing the song[6]. ']", + "evaluator-[2] -> gpt-3.5-turbo-[1]\n: {'answer': 'The original artist of \"The Sound of Silence\" is Paul Simon, from the duo Simon & Garfunkel[5]. '}" + ], + "output": [ + "The original artist of \"Sound of Silence\" is Paul Simon, covered by the Hanseroth brothers [2].", + "\"The Sound of Silence\" was originally performed by the American music duo Simon & Garfunkel [4].", + "The original artist of \"The Sound of Silence\" is Paul Simon, from the duo Simon & Garfunkel [5]." + ], + "result": { + "length": 50 + }, + "token_used": 4669, + "doc_level": [ + [ + 31.345982351292164, + 9.814030783070152, + 20.28832355977496, + 21.572409578388246, + 25.20364455602536, + 30.66743305850985 + ], + [ + 26.12250113458684, + 13.940475269637894, + 35.73394230760885, + 33.16552474906199, + 28.933107501205303, + 22.90627032104506 + ], + [ + 32.52873740806488, + 21.183938606366905, + 21.680280064519902, + 28.300516517130653, + 31.426902113423424, + 29.118466399323072 + ] + ], + "span_level": [ + [ + [ + [ + 40, + 0.031000953369108266 + ], + [ + 118, + 0.06542317999698267 + ], + [ + 270, + 0.06542317999698267 + ], + [ + 367, + 0.06542317999698267 + ], + [ + 505, + 0.06542317999698267 + ] + ], + [ + [ + 67, + 0.04353641338263766 + ], + [ + 123, + 0.07570848979384534 + ], + [ + 178, + 0.0 + ], + [ + 378, + 1.0 + ], + [ + 566, + 0.020709522745889254 + ], + [ + 732, + 0.06542317999698267 + ] + ], + [ + [ + 37, + 0.06519448453622809 + ], + [ + 58, + 0.008892985874469022 + ], + [ + 285, + 0.06542317999698267 + ], + [ + 400, + 0.047967124258308194 + ], + [ + 568, + 0.06542317999698267 + ] + ], + [ + [ + 40, + 0.06542317999698267 + ], + [ + 96, + 0.04966344778422236 + ], + [ + 192, + 0.03581467576098593 + ], + [ + 347, + 0.06542317999698267 + ], + [ + 432, + 0.06542317999698267 + ] + ], + [ + [ + 55, + 0.02825097206201647 + ], + [ + 59, + 0.047919885204251916 + ], + [ + 310, + 0.047071907555204295 + ], + [ + 369, + 0.06542317999698267 + ], + [ + 516, + 0.08653700174069713 + ] + ], + [ + [ + 40, + 0.06542317999698267 + ], + [ + 118, + 0.028427837801681895 + ], + [ + 270, + 0.05971952133444869 + ], + [ + 367, + 0.06542317999698267 + ], + [ + 505, + 0.06542317999698267 + ] + ] + ], + [ + [ + [ + 40, + 0.08450978976638523 + ], + [ + 118, + 0.11957626372490102 + ], + [ + 270, + 0.08450978976638523 + ], + [ + 367, + 0.12849268616454287 + ], + [ + 505, + 0.08450978976638523 + ] + ], + [ + [ + 67, + 0.14237234755766012 + ], + [ + 123, + 0.08450978976638523 + ], + [ + 178, + 0.08450978976638523 + ], + [ + 378, + 0.18703871319873405 + ], + [ + 566, + 0.08450978976638523 + ], + [ + 732, + 0.10202343369474869 + ] + ], + [ + [ + 37, + 0.0832069520975214 + ], + [ + 58, + 0.08450978976638523 + ], + [ + 285, + 0.13088314244940977 + ], + [ + 400, + 0.05906399611785642 + ], + [ + 568, + 0.10618799058788225 + ] + ], + [ + [ + 40, + 0.08450978976638523 + ], + [ + 96, + 0.08450978976638523 + ], + [ + 192, + 1.0 + ], + [ + 347, + 0.08450978976638523 + ], + [ + 432, + 0.08450978976638523 + ] + ], + [ + [ + 55, + 0.08450978976638523 + ], + [ + 59, + 0.10881207395974353 + ], + [ + 310, + 0.08450978976638523 + ], + [ + 369, + 0.08450978976638523 + ], + [ + 516, + 0.08450978976638523 + ] + ], + [ + [ + 40, + 0.08450978976638523 + ], + [ + 118, + 0.08450978976638523 + ], + [ + 270, + 0.0 + ], + [ + 367, + 0.10724035575992644 + ], + [ + 505, + 0.05831909332546213 + ] + ] + ], + [ + [ + [ + 40, + 1.0 + ], + [ + 118, + 0.2333871100088499 + ], + [ + 270, + 0.2333871100088499 + ], + [ + 367, + 0.2333871100088499 + ], + [ + 505, + 0.2333871100088499 + ] + ], + [ + [ + 67, + 0.2333871100088499 + ], + [ + 123, + 0.2333871100088499 + ], + [ + 178, + 0.2707555163782165 + ], + [ + 378, + 0.7470623769593163 + ], + [ + 566, + 0.2333871100088499 + ], + [ + 732, + 0.2333871100088499 + ] + ], + [ + [ + 37, + 0.2333871100088499 + ], + [ + 58, + 0.2333871100088499 + ], + [ + 285, + 0.2333871100088499 + ], + [ + 400, + 0.32359201765287404 + ], + [ + 568, + 0.24270611289371452 + ] + ], + [ + [ + 40, + 0.2333871100088499 + ], + [ + 96, + 0.2333871100088499 + ], + [ + 192, + 0.2333871100088499 + ], + [ + 347, + 0.2333871100088499 + ], + [ + 432, + 0.2333871100088499 + ] + ], + [ + [ + 55, + 0.2333871100088499 + ], + [ + 59, + 0.2766348955746465 + ], + [ + 310, + 0.2333871100088499 + ], + [ + 369, + 0.0 + ], + [ + 516, + 0.2333871100088499 + ] + ], + [ + [ + 40, + 0.2333871100088499 + ], + [ + 118, + 0.2333871100088499 + ], + [ + 270, + 0.2333871100088499 + ], + [ + 367, + 0.2333871100088499 + ], + [ + 505, + 0.2333871100088499 + ] + ] + ] + ], + "word_level": [ + [ + [ + [ + 40, + 1.0 + ], + [ + 50, + 0.6673350080770191 + ], + [ + 53, + 0.5501252855936065 + ], + [ + 62, + 0.3023019185922069 + ], + [ + 70, + 0.37497279006975565 + ], + [ + 79, + 0.287190888962328 + ], + [ + 83, + 0.3656519585099589 + ], + [ + 90, + 0.8224433181589654 + ], + [ + 91, + 0.5458418314042807 + ], + [ + 95, + 0.2527383051354567 + ], + [ + 102, + 0.19425259960687252 + ], + [ + 107, + 0.26412656654951316 + ], + [ + 111, + 0.3050758850044445 + ], + [ + 113, + 0.44574945857900256 + ], + [ + 115, + 0.41993779551280747 + ], + [ + 117, + 0.46913077200706144 + ], + [ + 118, + 0.6903558606696457 + ], + [ + 124, + 0.8095609802644569 + ], + [ + 134, + 0.5778014155662778 + ], + [ + 141, + 0.0528279103540944 + ], + [ + 144, + 0.17569301829847556 + ], + [ + 148, + 0.12303196261616305 + ], + [ + 152, + 0.13408385655151353 + ], + [ + 156, + 0.1802224715393769 + ], + [ + 160, + 0.07256120591877202 + ], + [ + 168, + 0.25963881072724254 + ], + [ + 173, + 0.12519219783439725 + ], + [ + 175, + 0.08715886952658043 + ], + [ + 182, + 0.10070511542553796 + ], + [ + 186, + 0.07499585393831797 + ], + [ + 191, + 0.16534331834784446 + ], + [ + 192, + 0.3280409001545978 + ], + [ + 198, + 0.12495586400388888 + ], + [ + 200, + 0.08718377582742771 + ], + [ + 212, + 0.11009567026869922 + ], + [ + 215, + 0.16283174658057906 + ], + [ + 217, + 0.21905321099813652 + ], + [ + 220, + 0.14312615019803301 + ], + [ + 226, + 0.35854703799944126 + ], + [ + 229, + 0.31156772796287724 + ], + [ + 233, + 0.21334995022591166 + ], + [ + 237, + 0.6494967213055153 + ], + [ + 238, + 0.1562732858672031 + ], + [ + 242, + 0.21758365116463282 + ], + [ + 246, + 0.22180012998501186 + ], + [ + 249, + 0.5955590614164147 + ], + [ + 250, + 0.19515673543472206 + ], + [ + 257, + 0.1939671777880346 + ], + [ + 260, + 0.13441997570083647 + ], + [ + 269, + 0.0962245436940045 + ], + [ + 270, + 0.41758132708327717 + ], + [ + 274, + 0.1724423642837319 + ], + [ + 279, + 0.292210040138366 + ], + [ + 280, + 0.16060691471275226 + ], + [ + 281, + 0.19111119741905586 + ], + [ + 288, + 0.34586500068499376 + ], + [ + 292, + 0.20842819666403836 + ], + [ + 298, + 0.39658184837946536 + ], + [ + 305, + 0.17580137026793496 + ], + [ + 313, + 0.2697485866041862 + ], + [ + 314, + 0.40245038777270215 + ], + [ + 319, + 0.23042264009262628 + ], + [ + 328, + 0.2745373840976133 + ], + [ + 336, + 0.686310601922405 + ], + [ + 343, + 0.1038747825842126 + ], + [ + 351, + 0.1843237403850703 + ], + [ + 356, + 0.04780697292168929 + ], + [ + 360, + 0.13771456499078513 + ], + [ + 366, + 0.14370507041578556 + ], + [ + 367, + 0.41106429462481175 + ], + [ + 372, + 0.07914415111312408 + ], + [ + 380, + 0.17004586492983942 + ], + [ + 385, + 0.06260971552942095 + ], + [ + 389, + 0.13650137236898538 + ], + [ + 394, + 0.15737987585285654 + ], + [ + 404, + 0.2124160398325827 + ], + [ + 407, + 0.11697951136862085 + ], + [ + 411, + 0.088965401678595 + ], + [ + 416, + 0.03976475222322191 + ], + [ + 418, + 0.07265929941114826 + ], + [ + 419, + 0.1903972306803229 + ], + [ + 427, + 0.1400295723897883 + ], + [ + 434, + 0.09403427276855186 + ], + [ + 441, + 0.09561594514616545 + ], + [ + 442, + 0.2193981001529797 + ], + [ + 445, + 0.05958169726821103 + ], + [ + 449, + 0.07284891601019089 + ], + [ + 454, + 0.056114344223359196 + ], + [ + 458, + 0.01982976574765662 + ], + [ + 467, + 0.02881915343402458 + ], + [ + 473, + 0.0 + ], + [ + 480, + 0.02613442430372832 + ], + [ + 486, + 0.048561738678291576 + ], + [ + 490, + 0.09237919714679328 + ], + [ + 497, + 0.036965116533931294 + ], + [ + 504, + 0.008035833940425341 + ], + [ + 505, + 0.43286723947072503 + ], + [ + 511, + 0.6185850061954502 + ], + [ + 518, + 0.2487652377281224 + ], + [ + 521, + 0.07205101865339957 + ], + [ + 527, + 0.20779769841109977 + ], + [ + 532, + 0.12020902426743268 + ], + [ + 535, + 0.18347201775334404 + ], + [ + 541, + 0.09111446161067192 + ], + [ + 546, + 0.11190162470489842 + ], + [ + 547, + 0.34098457528064735 + ], + [ + 555, + 0.18144335112665105 + ], + [ + 556, + 0.3285475666833446 + ], + [ + 558, + 0.3294471757227279 + ], + [ + 559, + 0.14225157866238816 + ], + [ + 565, + 0.2862078247674584 + ], + [ + 569, + 0.17259243450015546 + ], + [ + 575, + 0.22466031937528955 + ], + [ + 578, + 0.24589698389553089 + ], + [ + 582, + 0.11136804767680572 + ], + [ + 586, + 0.3792024714981594 + ], + [ + 591, + 0.2567733162662349 + ], + [ + 593, + 0.10757064388596567 + ], + [ + 597, + 0.14877045695153748 + ], + [ + 598, + 0.18949918384454095 + ], + [ + 599, + 0.10481652426444972 + ], + [ + 600, + 0.17359263306387454 + ], + [ + 606, + 0.0710241851414031 + ], + [ + 610, + 0.10233980197272273 + ], + [ + 612, + 0.5367157674723516 + ], + [ + 618, + 0.12905457476051962 + ], + [ + 625, + 0.25457963978710013 + ], + [ + 629, + 0.23496587403614405 + ], + [ + 634, + 0.21672977275836997 + ], + [ + 639, + 0.2538421482939006 + ], + [ + 644, + 0.49638013049402474 + ], + [ + 646, + 0.24688291561846262 + ] + ], + [ + [ + 67, + 0.12734938171340532 + ], + [ + 73, + 0.06856975184821909 + ], + [ + 76, + 0.07046680795079475 + ], + [ + 77, + 0.07286720657497607 + ], + [ + 81, + 0.04382881905107714 + ], + [ + 87, + 0.036904743783589496 + ], + [ + 90, + 0.022221612293471187 + ], + [ + 96, + 0.025020566213296434 + ], + [ + 107, + 0.041946483435496174 + ], + [ + 113, + 0.035583552778043266 + ], + [ + 118, + 0.037495550238701394 + ], + [ + 122, + 0.050798193302644276 + ], + [ + 123, + 0.06975198681255333 + ], + [ + 128, + 0.02034793492711768 + ], + [ + 131, + 0.02647338417296204 + ], + [ + 132, + 0.05335801939626078 + ], + [ + 141, + 0.0375097321152414 + ], + [ + 151, + 0.06276723510085101 + ], + [ + 154, + 0.03388178357561394 + ], + [ + 158, + 0.033615525074932665 + ], + [ + 164, + 0.07354083263098112 + ], + [ + 168, + 0.025542309341805212 + ], + [ + 177, + 0.019357812110952978 + ], + [ + 178, + 0.06868066105146832 + ], + [ + 183, + 0.0337533727134262 + ], + [ + 185, + 0.04721782151810469 + ], + [ + 189, + 0.12948065723764357 + ], + [ + 190, + 0.06638836437434842 + ], + [ + 193, + 0.041961967430127686 + ], + [ + 195, + 0.028138386110402733 + ], + [ + 198, + 0.015986753506446953 + ], + [ + 206, + 0.032426144629925956 + ], + [ + 212, + 0.031565693058208825 + ], + [ + 213, + 0.026245694749057896 + ], + [ + 219, + 0.03581549405131284 + ], + [ + 224, + 0.04150026511180111 + ], + [ + 226, + 0.029457927057486514 + ], + [ + 231, + 0.025150880361008775 + ], + [ + 234, + 0.03223526117304314 + ], + [ + 235, + 0.027827980238900662 + ], + [ + 242, + 0.03243604024202799 + ], + [ + 251, + 0.03019321212383555 + ], + [ + 253, + 0.02595817511534449 + ], + [ + 254, + 0.0153994118313002 + ], + [ + 258, + 0.018905748858584603 + ], + [ + 259, + 0.028286617528664865 + ], + [ + 260, + 0.02408076153127012 + ], + [ + 264, + 0.025039283530466236 + ], + [ + 266, + 0.03321203579266074 + ], + [ + 267, + 0.016746852888053266 + ], + [ + 271, + 0.03603194147145067 + ], + [ + 275, + 0.04009236987853154 + ], + [ + 279, + 0.0574464030020107 + ], + [ + 281, + 0.04392050615702573 + ], + [ + 287, + 0.02612808794618552 + ], + [ + 297, + 0.03582488296368533 + ], + [ + 299, + 0.04018375691414672 + ], + [ + 304, + 0.17939327605248503 + ], + [ + 307, + 0.36732473096500085 + ], + [ + 308, + 0.09132073335538295 + ], + [ + 309, + 0.0300436209813877 + ], + [ + 311, + 0.03785728782387551 + ], + [ + 317, + 0.015591057719681176 + ], + [ + 322, + 0.028193528757668784 + ], + [ + 323, + 0.03324603226414198 + ], + [ + 327, + 0.024508314925855854 + ], + [ + 333, + 0.016626304161208624 + ], + [ + 334, + 0.028781219742507396 + ], + [ + 338, + 0.03164645939119423 + ], + [ + 340, + 0.054199017052742174 + ], + [ + 343, + 0.018324960167906006 + ], + [ + 345, + 0.011404181803469672 + ], + [ + 351, + 0.023407219016947478 + ], + [ + 352, + 0.05877781365373689 + ], + [ + 357, + 0.019269605046425546 + ], + [ + 362, + 0.016038937630023526 + ], + [ + 366, + 0.038714956295252866 + ], + [ + 370, + 0.02924551479199231 + ], + [ + 371, + 0.04395801474546731 + ], + [ + 377, + 0.04652197666511727 + ], + [ + 378, + 0.196005937035379 + ], + [ + 382, + 0.14789652327573807 + ], + [ + 387, + 0.5004849342896411 + ], + [ + 389, + 0.7836212289445034 + ], + [ + 392, + 1.0 + ], + [ + 401, + 0.18711668527353098 + ], + [ + 402, + 0.10734063441248641 + ], + [ + 408, + 0.23797196961287015 + ], + [ + 411, + 0.1226991703468775 + ], + [ + 416, + 0.10367707250949461 + ], + [ + 422, + 0.18818492022935476 + ], + [ + 423, + 0.13919272440378133 + ], + [ + 424, + 0.03873577725300972 + ], + [ + 426, + 0.03546679076036186 + ], + [ + 429, + 0.026244732337009344 + ], + [ + 435, + 0.058266128456348626 + ], + [ + 438, + 0.03685269504953203 + ], + [ + 442, + 0.024913014273813275 + ], + [ + 446, + 0.06075580275757557 + ], + [ + 447, + 0.03733034551326952 + ], + [ + 451, + 0.02987804766616434 + ], + [ + 454, + 0.01580296363616938 + ], + [ + 458, + 0.0237334131058124 + ], + [ + 460, + 0.026518286812064685 + ], + [ + 464, + 0.04566952369098982 + ], + [ + 465, + 0.0355435810405121 + ], + [ + 467, + 0.04668085882237869 + ], + [ + 468, + 0.019057401086744686 + ], + [ + 471, + 0.03261039992021254 + ], + [ + 475, + 0.037615191316459906 + ], + [ + 480, + 0.08253837964179224 + ], + [ + 481, + 0.12883961717680545 + ], + [ + 482, + 0.009533410603445822 + ], + [ + 489, + 0.011366522245915216 + ], + [ + 492, + 0.013274508943688009 + ], + [ + 493, + 0.02085292685439296 + ], + [ + 495, + 0.026420225348188827 + ], + [ + 496, + 0.036888662027501976 + ], + [ + 500, + 0.033745379838276586 + ], + [ + 502, + 0.03988540773015062 + ], + [ + 505, + 0.05040144430629067 + ], + [ + 507, + 0.06598786176108301 + ], + [ + 509, + 0.136402612640506 + ], + [ + 514, + 0.005862421210408692 + ], + [ + 515, + 0.003678167351611359 + ], + [ + 518, + 0.008064832488070639 + ], + [ + 523, + 0.012724421191298867 + ], + [ + 526, + 0.00802024452868274 + ], + [ + 529, + 0.0322216254494439 + ], + [ + 532, + 0.014672244979375231 + ], + [ + 536, + 0.010581811716365682 + ], + [ + 538, + 0.00856988875718127 + ], + [ + 540, + 0.0 + ], + [ + 545, + 0.007896618996972157 + ], + [ + 550, + 0.0072689671310533464 + ], + [ + 551, + 0.006787102177817009 + ], + [ + 554, + 0.005110981178675171 + ], + [ + 557, + 0.004810528289818129 + ], + [ + 561, + 0.011383471712322584 + ], + [ + 564, + 0.027038523378936384 + ], + [ + 566, + 0.047243708598188314 + ], + [ + 568, + 0.016303163598394888 + ], + [ + 571, + 0.014550204210237548 + ], + [ + 578, + 0.04337763604620566 + ], + [ + 583, + 0.02710194850653316 + ], + [ + 585, + 0.03663520601172648 + ], + [ + 586, + 0.009355746040848179 + ], + [ + 594, + 0.018074287234965535 + ], + [ + 598, + 0.01590348524612625 + ], + [ + 603, + 0.00901162382525181 + ], + [ + 606, + 0.010720337398611666 + ], + [ + 607, + 0.02447060313984841 + ], + [ + 610, + 0.02561638142494979 + ], + [ + 616, + 0.01749898569653109 + ], + [ + 621, + 0.011224645023240916 + ], + [ + 623, + 0.015436691115400171 + ], + [ + 627, + 0.015086379987104187 + ], + [ + 633, + 0.030074305237552928 + ], + [ + 637, + 0.023652544004008073 + ], + [ + 642, + 0.030035946578470883 + ], + [ + 647, + 0.0127717788024351 + ], + [ + 652, + 0.04126680169370997 + ], + [ + 655, + 0.0788739416850845 + ], + [ + 656, + 0.043447688614085864 + ], + [ + 657, + 0.007957335905813996 + ], + [ + 662, + 0.0037638057141658418 + ], + [ + 666, + 0.007871403020228825 + ], + [ + 670, + 0.015614473612759602 + ], + [ + 682, + 0.011485452399342755 + ], + [ + 687, + 0.008705577908506443 + ], + [ + 693, + 0.01278487109378182 + ], + [ + 697, + 0.013385273553416326 + ], + [ + 703, + 0.005495350293044893 + ], + [ + 704, + 0.008233416909481416 + ], + [ + 708, + 0.005070945188313617 + ], + [ + 715, + 0.007427594585736153 + ], + [ + 718, + 0.01190424781318206 + ], + [ + 721, + 0.020545999834168032 + ], + [ + 725, + 0.022026667269957823 + ], + [ + 731, + 0.005441224769218102 + ], + [ + 732, + 0.051184409977263326 + ], + [ + 734, + 0.024565302668342226 + ], + [ + 735, + 0.013821918364946173 + ], + [ + 739, + 0.036575108442321286 + ], + [ + 740, + 0.01174315955594493 + ], + [ + 749, + 0.02044380062026139 + ], + [ + 750, + 0.04107858730758203 + ], + [ + 751, + 0.014549202069479808 + ], + [ + 755, + 0.005797983092156271 + ], + [ + 760, + 0.01510842626365414 + ], + [ + 766, + 0.01888448848418175 + ], + [ + 771, + 0.020821753544820406 + ] + ], + [ + [ + 37, + 0.3205496180904802 + ], + [ + 45, + 0.3289666157692288 + ], + [ + 46, + 0.13645539102186802 + ], + [ + 57, + 0.084949990029442 + ], + [ + 58, + 0.2760514890434911 + ], + [ + 60, + 0.11986514599176608 + ], + [ + 63, + 0.051607250524489015 + ], + [ + 69, + 0.10383758614452453 + ], + [ + 72, + 0.07460136876344708 + ], + [ + 76, + 0.0345439839417111 + ], + [ + 80, + 0.09686036216521496 + ], + [ + 81, + 0.09552900172800081 + ], + [ + 83, + 0.11550977596403823 + ], + [ + 88, + 0.05273804838028889 + ], + [ + 91, + 0.09370955820244828 + ], + [ + 96, + 0.12686113919604014 + ], + [ + 97, + 0.10016317397050499 + ], + [ + 100, + 0.23482882551523743 + ], + [ + 101, + 0.1228797649355469 + ], + [ + 111, + 0.3027475807313155 + ], + [ + 113, + 0.12906768471194333 + ], + [ + 116, + 0.061946812895951026 + ], + [ + 122, + 0.04679993742624592 + ], + [ + 125, + 0.06947204367484361 + ], + [ + 129, + 0.046950346558696066 + ], + [ + 133, + 0.04770546467650245 + ], + [ + 134, + 0.05823990759799542 + ], + [ + 139, + 0.10923154070832687 + ], + [ + 140, + 0.0664264754101538 + ], + [ + 143, + 0.12867644157569505 + ], + [ + 148, + 0.10507851454329285 + ], + [ + 157, + 0.0821574810285029 + ], + [ + 160, + 0.06670605874839035 + ], + [ + 169, + 0.09640910429795178 + ], + [ + 170, + 0.2548344149177881 + ], + [ + 171, + 0.022066421399018585 + ], + [ + 173, + 0.03560830219123276 + ], + [ + 179, + 0.022115933579525925 + ], + [ + 181, + 0.021725411493972693 + ], + [ + 182, + 0.012366552353955281 + ], + [ + 185, + 0.025785292453255056 + ], + [ + 186, + 0.04072503278184203 + ], + [ + 187, + 0.0 + ], + [ + 188, + 0.0012835731105617831 + ], + [ + 189, + 0.0023957854206237243 + ], + [ + 192, + 0.0024084514575994516 + ], + [ + 196, + 0.003460529056491646 + ], + [ + 197, + 0.023246375588007517 + ], + [ + 204, + 0.023087545097751884 + ], + [ + 209, + 0.028509404867628367 + ], + [ + 210, + 0.08079672416717261 + ], + [ + 211, + 0.005675510789211404 + ], + [ + 212, + 0.04217499072272457 + ], + [ + 214, + 0.007499123849358274 + ], + [ + 221, + 0.025159891430322497 + ], + [ + 224, + 0.10948855826125826 + ], + [ + 233, + 0.03508409796201198 + ], + [ + 241, + 0.06851032761244912 + ], + [ + 247, + 0.050120931140721144 + ], + [ + 256, + 0.06568319388588219 + ], + [ + 260, + 0.10065734670475598 + ], + [ + 267, + 0.18068089180460936 + ], + [ + 270, + 0.2255075109322255 + ], + [ + 275, + 0.17398359885320508 + ], + [ + 276, + 0.2968806083867036 + ], + [ + 277, + 0.18386796486288975 + ], + [ + 278, + 0.2079826869511897 + ], + [ + 279, + 0.4216342316206541 + ], + [ + 280, + 0.30119915364061023 + ], + [ + 281, + 0.5481634571896805 + ], + [ + 282, + 0.06669457169795254 + ], + [ + 283, + 0.10336649647983574 + ], + [ + 284, + 0.07919719453470427 + ], + [ + 285, + 0.3464251991070779 + ], + [ + 287, + 0.12220383491021508 + ], + [ + 292, + 0.1954909548616731 + ], + [ + 295, + 0.194367836547719 + ], + [ + 299, + 0.06761798866675785 + ], + [ + 303, + 0.19221215429435679 + ], + [ + 305, + 0.2076843527002988 + ], + [ + 310, + 0.1135684757507726 + ], + [ + 313, + 0.1672895472216169 + ], + [ + 317, + 0.052029576595645256 + ], + [ + 321, + 0.09059924097862591 + ], + [ + 324, + 0.27849801953942477 + ], + [ + 328, + 0.18036246263670513 + ], + [ + 335, + 0.1391765893256503 + ], + [ + 342, + 0.1591564499428232 + ], + [ + 348, + 0.21825947555532776 + ], + [ + 351, + 0.3168339951513043 + ], + [ + 357, + 0.6496010945373956 + ], + [ + 359, + 0.6595908405731141 + ], + [ + 363, + 0.6930338277442681 + ], + [ + 364, + 0.49275204288151386 + ], + [ + 367, + 0.7083245561828443 + ], + [ + 369, + 1.0 + ], + [ + 370, + 0.29780298050953247 + ], + [ + 379, + 0.11640046043249223 + ], + [ + 382, + 0.08244463029931684 + ], + [ + 390, + 0.1238056904886413 + ], + [ + 391, + 0.0822061991658962 + ], + [ + 392, + 0.08976985047371738 + ], + [ + 393, + 0.11297146124611046 + ], + [ + 394, + 0.5057388595507619 + ], + [ + 395, + 0.18972491197615673 + ], + [ + 396, + 0.4540918071225432 + ], + [ + 397, + 0.030658102969148716 + ], + [ + 398, + 0.04649062321350309 + ], + [ + 399, + 0.05285793286251479 + ], + [ + 400, + 0.12928324931849466 + ], + [ + 404, + 0.022772870570518626 + ], + [ + 410, + 0.044540032842756004 + ], + [ + 411, + 0.07879795896089166 + ], + [ + 412, + 0.018527037437023604 + ], + [ + 418, + 0.04272672342203713 + ], + [ + 421, + 0.029910612389568818 + ], + [ + 423, + 0.025066912807872776 + ], + [ + 430, + 0.026689636501738275 + ], + [ + 443, + 0.05563832375839093 + ], + [ + 446, + 0.04992411352259348 + ], + [ + 450, + 0.034583856355710534 + ], + [ + 456, + 0.030784138259222486 + ], + [ + 459, + 0.07145972964245283 + ], + [ + 463, + 0.06614884791300306 + ], + [ + 466, + 0.12421073456547345 + ], + [ + 467, + 0.08720891415143656 + ], + [ + 468, + 0.07901881556696896 + ], + [ + 469, + 0.03003280351484477 + ], + [ + 475, + 0.019848086806591544 + ], + [ + 481, + 0.01764803348688011 + ], + [ + 485, + 0.04984350873241446 + ], + [ + 486, + 0.10467000123120448 + ], + [ + 488, + 0.05756178172759093 + ], + [ + 491, + 0.0629957225860571 + ], + [ + 497, + 0.1709829065828109 + ], + [ + 500, + 0.07351769572904886 + ], + [ + 504, + 0.02655530558236307 + ], + [ + 508, + 0.09153308040504107 + ], + [ + 510, + 0.09415315773564878 + ], + [ + 516, + 0.01643238474272946 + ], + [ + 527, + 0.0513850496857362 + ], + [ + 531, + 0.03883913623596829 + ], + [ + 540, + 0.01895291689573372 + ], + [ + 543, + 0.029692284799121987 + ], + [ + 545, + 0.04139436320556956 + ], + [ + 548, + 0.04240991604532501 + ], + [ + 550, + 0.07629095371943588 + ], + [ + 555, + 0.10233755625577272 + ], + [ + 558, + 0.057421155365544464 + ], + [ + 562, + 0.011616819892027544 + ], + [ + 566, + 0.07855818598439515 + ], + [ + 568, + 0.08745988559473115 + ], + [ + 572, + 0.027532619861191866 + ], + [ + 577, + 0.06936197889016339 + ], + [ + 581, + 0.03194448178661335 + ], + [ + 589, + 0.026149544860710534 + ], + [ + 594, + 0.0054053999909773704 + ], + [ + 603, + 0.016690520984335377 + ], + [ + 606, + 0.0354932253668667 + ], + [ + 609, + 0.02238384911892655 + ], + [ + 612, + 0.011317925683986034 + ], + [ + 616, + 0.046579309671173234 + ], + [ + 618, + 0.019707093929106845 + ], + [ + 626, + 0.06920685356991206 + ], + [ + 629, + 0.05137102958941672 + ], + [ + 633, + 0.06796092671440666 + ] + ], + [ + [ + 40, + 0.336630039240243 + ], + [ + 49, + 0.2871812190105031 + ], + [ + 53, + 0.2066030012303554 + ], + [ + 61, + 0.22741189204409687 + ], + [ + 65, + 0.11862260384184717 + ], + [ + 69, + 0.1055596730937151 + ], + [ + 75, + 0.10710403230654428 + ], + [ + 80, + 0.18446487855577537 + ], + [ + 81, + 0.1958413021950977 + ], + [ + 83, + 0.16037702689986919 + ], + [ + 84, + 0.06712411174206487 + ], + [ + 87, + 0.17042555858397548 + ], + [ + 92, + 0.3053279109608649 + ], + [ + 93, + 0.19007030642756934 + ], + [ + 94, + 0.15331813017073717 + ], + [ + 96, + 0.3239578041645922 + ], + [ + 100, + 0.268597658161371 + ], + [ + 105, + 0.3773031300280363 + ], + [ + 110, + 1.0 + ], + [ + 118, + 0.7467706572967469 + ], + [ + 122, + 0.0521032438830854 + ], + [ + 127, + 0.04836577424995874 + ], + [ + 130, + 0.055422817007087524 + ], + [ + 134, + 0.055165182060907654 + ], + [ + 142, + 0.08202546682514628 + ], + [ + 144, + 0.05125602445048435 + ], + [ + 145, + 0.06672119152754473 + ], + [ + 147, + 0.11386528247244591 + ], + [ + 151, + 0.07360323053513063 + ], + [ + 154, + 0.1389073786528606 + ], + [ + 155, + 0.03835751043707684 + ], + [ + 158, + 0.05419809846915578 + ], + [ + 160, + 0.053934732762504614 + ], + [ + 163, + 0.04195496110162641 + ], + [ + 169, + 0.04766632688426731 + ], + [ + 173, + 0.08813481487781218 + ], + [ + 174, + 0.05669296120501824 + ], + [ + 177, + 0.14612049397447535 + ], + [ + 186, + 0.07091093125396894 + ], + [ + 187, + 0.13305624752969153 + ], + [ + 188, + 0.19942396594621517 + ], + [ + 189, + 0.03625356178302539 + ], + [ + 190, + 0.050159022326608384 + ], + [ + 191, + 0.04780170099983619 + ], + [ + 192, + 0.2726178611058135 + ], + [ + 198, + 0.23808523319140537 + ], + [ + 204, + 0.29185445468611826 + ], + [ + 211, + 0.24247341544187725 + ], + [ + 215, + 0.2198245889689277 + ], + [ + 221, + 0.10312285330339181 + ], + [ + 224, + 0.1222665359100164 + ], + [ + 228, + 0.05999496303412062 + ], + [ + 232, + 0.1320403557770154 + ], + [ + 234, + 0.17457589464898385 + ], + [ + 237, + 0.058070436028888846 + ], + [ + 243, + 0.07443250878701753 + ], + [ + 246, + 0.07573819299273052 + ], + [ + 250, + 0.03957491443787352 + ], + [ + 254, + 0.0919449062219277 + ], + [ + 256, + 0.14768740694286162 + ], + [ + 267, + 0.2008852221792316 + ], + [ + 269, + 0.13788255037645372 + ], + [ + 272, + 0.06341593874562712 + ], + [ + 274, + 0.09565463735584451 + ], + [ + 279, + 0.13392317035136825 + ], + [ + 282, + 0.09292913842644951 + ], + [ + 286, + 0.058236822573282505 + ], + [ + 290, + 0.13728316668840748 + ], + [ + 292, + 0.1253645188352349 + ], + [ + 295, + 0.20170134461458888 + ], + [ + 297, + 0.15817947105616306 + ], + [ + 302, + 0.20056228977401386 + ], + [ + 305, + 0.2563793094725985 + ], + [ + 309, + 0.13637979425144778 + ], + [ + 318, + 0.18886069046996076 + ], + [ + 324, + 0.18366724062801193 + ], + [ + 327, + 0.3755215890855737 + ], + [ + 328, + 0.3977571665653181 + ], + [ + 334, + 0.5292838478086294 + ], + [ + 336, + 0.6427420047069092 + ], + [ + 340, + 0.23042064234132167 + ], + [ + 341, + 0.22337582221765337 + ], + [ + 344, + 0.3506078016194171 + ], + [ + 346, + 0.48049096311336803 + ], + [ + 347, + 0.2340935102799047 + ], + [ + 351, + 0.08746271037783843 + ], + [ + 356, + 0.1331419294021216 + ], + [ + 360, + 0.12804544327920042 + ], + [ + 368, + 0.2999610359376052 + ], + [ + 371, + 0.35700680941898527 + ], + [ + 376, + 0.6411348925841559 + ], + [ + 382, + 0.5767080633639071 + ], + [ + 387, + 0.34831358585607075 + ], + [ + 389, + 0.05141947653925097 + ], + [ + 396, + 0.04990627467720594 + ], + [ + 399, + 0.07589098772730513 + ], + [ + 407, + 0.050135124767763335 + ], + [ + 414, + 0.08416445326648091 + ], + [ + 417, + 0.1953206793032988 + ], + [ + 418, + 0.16826444892764073 + ], + [ + 419, + 0.4101064478522876 + ], + [ + 420, + 0.06599757056746113 + ], + [ + 421, + 0.05538326890525786 + ], + [ + 422, + 0.05203313233178356 + ], + [ + 426, + 0.139204307299593 + ], + [ + 427, + 0.08274408490788354 + ], + [ + 428, + 0.18473172658958661 + ], + [ + 429, + 0.0 + ], + [ + 430, + 0.0018766639866066924 + ], + [ + 431, + 0.007360429454931333 + ], + [ + 432, + 0.08271035479311954 + ], + [ + 434, + 0.017531483658903346 + ], + [ + 441, + 0.030277900777853653 + ], + [ + 445, + 0.02605891537872888 + ], + [ + 450, + 0.0370293112978538 + ], + [ + 454, + 0.02844255445081381 + ], + [ + 457, + 0.04391573155999294 + ], + [ + 461, + 0.04833155713180263 + ], + [ + 464, + 0.04634542254141923 + ], + [ + 465, + 0.05142380207123799 + ], + [ + 473, + 0.04184421799733743 + ], + [ + 475, + 0.027701573947475546 + ], + [ + 482, + 0.009696439476188711 + ], + [ + 487, + 0.018966023578827537 + ], + [ + 492, + 0.01912661211222309 + ], + [ + 501, + 0.021288777572643785 + ], + [ + 509, + 0.023553115620858427 + ], + [ + 510, + 0.07547034738235482 + ], + [ + 514, + 0.027292512851438532 + ], + [ + 518, + 0.03976276531720503 + ], + [ + 523, + 0.032420953298848275 + ], + [ + 527, + 0.024537534722980123 + ], + [ + 536, + 0.025317411057599118 + ], + [ + 539, + 0.053882821642852546 + ], + [ + 545, + 0.042625427813766714 + ], + [ + 546, + 0.05748813143319872 + ], + [ + 547, + 0.10582402544525138 + ], + [ + 548, + 0.03208587930146468 + ], + [ + 549, + 0.018711499914342303 + ], + [ + 550, + 0.018097065502852325 + ], + [ + 553, + 0.05853326539186848 + ], + [ + 562, + 0.06625992229424059 + ], + [ + 570, + 0.10644202154352984 + ], + [ + 573, + 0.14176391791146806 + ], + [ + 577, + 0.20731943522611004 + ], + [ + 582, + 0.01129464521286328 + ], + [ + 587, + 0.008450054234528985 + ], + [ + 591, + 0.032686696318173496 + ], + [ + 601, + 0.05089087109411624 + ] + ], + [ + [ + 55, + 0.9030357526920276 + ], + [ + 58, + 0.23979743705947504 + ], + [ + 59, + 0.42859624042620187 + ], + [ + 67, + 0.18434445363428917 + ], + [ + 68, + 0.33415745412637304 + ], + [ + 72, + 0.0947743459783671 + ], + [ + 75, + 0.12973115531531346 + ], + [ + 79, + 0.12290036801150614 + ], + [ + 84, + 0.051156256401959796 + ], + [ + 97, + 0.10907845250679962 + ], + [ + 102, + 0.13398553479272154 + ], + [ + 111, + 0.07611843981368649 + ], + [ + 114, + 0.0788262056673687 + ], + [ + 116, + 0.13749923558292426 + ], + [ + 123, + 0.10011326090217712 + ], + [ + 126, + 0.08464477867237832 + ], + [ + 131, + 0.023636309947903635 + ], + [ + 134, + 0.13870331996540253 + ], + [ + 138, + 0.09175294394812306 + ], + [ + 148, + 0.07996226932129907 + ], + [ + 152, + 0.0529779833081306 + ], + [ + 156, + 0.07995802180565639 + ], + [ + 162, + 0.17521429263750862 + ], + [ + 163, + 0.2352358597663052 + ], + [ + 167, + 0.1518411256292329 + ], + [ + 173, + 0.16798906456352025 + ], + [ + 180, + 0.3113421176351655 + ], + [ + 186, + 0.37781006251639776 + ], + [ + 188, + 0.35064968357553894 + ], + [ + 192, + 0.31700797820785437 + ], + [ + 193, + 0.31492107698235816 + ], + [ + 196, + 0.4137291536532775 + ], + [ + 198, + 0.6512788164191702 + ], + [ + 200, + 0.24450751905594068 + ], + [ + 203, + 0.0623764510148816 + ], + [ + 209, + 0.16108724018678877 + ], + [ + 212, + 0.12723154128894124 + ], + [ + 216, + 0.06305007893848898 + ], + [ + 220, + 0.1496974786133251 + ], + [ + 221, + 0.08958214948724409 + ], + [ + 224, + 0.1333328622006317 + ], + [ + 228, + 0.15126589852743325 + ], + [ + 230, + 0.03440479471167786 + ], + [ + 231, + 0.07651967814061028 + ], + [ + 235, + 0.08448293800319827 + ], + [ + 236, + 0.26295016261151966 + ], + [ + 242, + 0.03194524378074914 + ], + [ + 247, + 0.13673029168287695 + ], + [ + 253, + 0.11164026097263637 + ], + [ + 254, + 0.5609849657855067 + ], + [ + 255, + 0.05258804583441797 + ], + [ + 261, + 0.0534548411516816 + ], + [ + 268, + 0.022877267007675832 + ], + [ + 270, + 0.06386934790429757 + ], + [ + 271, + 0.008745579737186438 + ], + [ + 276, + 0.02204844265918385 + ], + [ + 280, + 0.06628466790360031 + ], + [ + 286, + 0.06735929911908828 + ], + [ + 289, + 0.015989714589670415 + ], + [ + 294, + 0.21028211778435657 + ], + [ + 295, + 0.033507515826709036 + ], + [ + 298, + 0.04915901972369317 + ], + [ + 302, + 0.09184231208031395 + ], + [ + 304, + 0.09653861111440964 + ], + [ + 305, + 0.05386521711061567 + ], + [ + 309, + 0.06438693493095023 + ], + [ + 310, + 0.5736235145594477 + ], + [ + 314, + 0.1307182354341252 + ], + [ + 321, + 0.3128801721782993 + ], + [ + 330, + 0.21320812782400697 + ], + [ + 333, + 0.2991948323341173 + ], + [ + 338, + 0.2416555565948461 + ], + [ + 344, + 0.2111443378786858 + ], + [ + 345, + 0.4067616727032481 + ], + [ + 352, + 0.21357213515782258 + ], + [ + 358, + 0.12456994020994618 + ], + [ + 368, + 0.21594951270082166 + ], + [ + 369, + 0.5073048214915472 + ], + [ + 374, + 0.19325275648030277 + ], + [ + 380, + 0.17277872629980306 + ], + [ + 383, + 0.35494684098405216 + ], + [ + 387, + 0.12630485118949417 + ], + [ + 391, + 0.11073376677007594 + ], + [ + 392, + 0.30526381720457757 + ], + [ + 397, + 0.09998556684497921 + ], + [ + 400, + 0.17007865378576217 + ], + [ + 402, + 0.29942635868365824 + ], + [ + 404, + 0.5877453516801945 + ], + [ + 405, + 0.6571158867238457 + ], + [ + 410, + 0.16481718046614563 + ], + [ + 416, + 0.10478820298058085 + ], + [ + 419, + 0.10602482474639899 + ], + [ + 423, + 0.06264947404815836 + ], + [ + 427, + 0.05812242263955829 + ], + [ + 428, + 0.2316501594554823 + ], + [ + 433, + 0.0893132742046921 + ], + [ + 436, + 0.1176810893784861 + ], + [ + 438, + 0.019688165246220267 + ], + [ + 440, + 0.1428111159033476 + ], + [ + 441, + 0.24202253975968527 + ], + [ + 444, + 0.35923997168634036 + ], + [ + 446, + 0.12393407880509916 + ], + [ + 451, + 0.10843768970891739 + ], + [ + 457, + 0.24181697599440075 + ], + [ + 460, + 0.24017956856661465 + ], + [ + 465, + 0.30910644388201575 + ], + [ + 471, + 0.1758961789024544 + ], + [ + 472, + 0.41000060212645506 + ], + [ + 481, + 0.18106744453625434 + ], + [ + 484, + 0.34723749688456357 + ], + [ + 490, + 0.27277889513074977 + ], + [ + 491, + 0.3080072393169877 + ], + [ + 492, + 1.0 + ], + [ + 493, + 0.004842152945922096 + ], + [ + 494, + 0.07276667577748247 + ], + [ + 495, + 0.025026533953599286 + ], + [ + 498, + 0.08063438160173125 + ], + [ + 507, + 0.0035249078793317653 + ], + [ + 515, + 0.05308323896624179 + ], + [ + 516, + 0.26816690610213695 + ], + [ + 519, + 0.0 + ], + [ + 523, + 0.01954158107192633 + ], + [ + 532, + 0.020585394732407065 + ], + [ + 535, + 0.0563047091242155 + ], + [ + 539, + 0.07576361802713545 + ], + [ + 541, + 0.046423471890003454 + ], + [ + 544, + 0.023976980931331663 + ], + [ + 547, + 0.0936009285294771 + ], + [ + 551, + 0.0746292086458846 + ], + [ + 559, + 0.00643072666136539 + ], + [ + 562, + 0.13656527670627283 + ], + [ + 564, + 0.1253722814506191 + ], + [ + 569, + 0.07130566582468154 + ], + [ + 572, + 0.011836236120959031 + ], + [ + 574, + 0.06917743993825473 + ], + [ + 577, + 0.029523874249767885 + ], + [ + 579, + 0.059767413835551404 + ], + [ + 581, + 0.14792777100185778 + ], + [ + 582, + 0.08614902913712878 + ], + [ + 588, + 0.13338039481840586 + ], + [ + 589, + 0.09857680142469338 + ], + [ + 591, + 0.2863720873818767 + ], + [ + 597, + 0.04953291781345246 + ], + [ + 606, + 0.04645033500155151 + ], + [ + 608, + 0.138377779209248 + ], + [ + 615, + 0.04774929216100929 + ], + [ + 618, + 0.19743413207985466 + ] + ], + [ + [ + 40, + 0.6617518812535067 + ], + [ + 50, + 0.41720602133760465 + ], + [ + 53, + 0.396853076722832 + ], + [ + 62, + 0.07237010183278336 + ], + [ + 70, + 0.11140044316327405 + ], + [ + 79, + 0.1558162907824022 + ], + [ + 83, + 0.13596648028880348 + ], + [ + 90, + 0.33814589918229065 + ], + [ + 91, + 0.3461462654951254 + ], + [ + 95, + 0.16567908044483623 + ], + [ + 102, + 0.1316556742706282 + ], + [ + 107, + 0.20296578768863974 + ], + [ + 111, + 0.24993679482756942 + ], + [ + 113, + 0.4236239442900675 + ], + [ + 115, + 0.1483906917295677 + ], + [ + 117, + 0.15132013455552495 + ], + [ + 118, + 0.5346597946030986 + ], + [ + 124, + 0.17405440997532573 + ], + [ + 134, + 0.187567376123927 + ], + [ + 141, + 0.0 + ], + [ + 144, + 0.12391235544686258 + ], + [ + 148, + 0.031051463564672775 + ], + [ + 152, + 0.06877214834589893 + ], + [ + 156, + 0.14494172112697737 + ], + [ + 160, + 0.04328820455408761 + ], + [ + 168, + 0.09013880747052236 + ], + [ + 173, + 0.0545114485634049 + ], + [ + 175, + 0.09562367781585321 + ], + [ + 182, + 0.028848598896494233 + ], + [ + 186, + 0.003869843870811641 + ], + [ + 191, + 0.05749849723401999 + ], + [ + 192, + 0.14768078814912744 + ], + [ + 198, + 0.032339266293052865 + ], + [ + 200, + 0.11836181555094459 + ], + [ + 212, + 0.038235005083354714 + ], + [ + 215, + 0.1639302913751657 + ], + [ + 217, + 0.16113140506260049 + ], + [ + 220, + 0.1418780562893044 + ], + [ + 226, + 0.17366904410207246 + ], + [ + 229, + 0.27326983857439874 + ], + [ + 233, + 0.039601193061374176 + ], + [ + 237, + 0.15416095984264988 + ], + [ + 238, + 0.1341708154605019 + ], + [ + 242, + 0.10123600426837008 + ], + [ + 246, + 0.1576783321488926 + ], + [ + 249, + 0.3126705095202829 + ], + [ + 250, + 0.07171474549859076 + ], + [ + 257, + 0.07793703002444827 + ], + [ + 260, + 0.15833295605510236 + ], + [ + 269, + 0.014830561856782857 + ], + [ + 270, + 0.42261837649619316 + ], + [ + 274, + 0.19762902655262699 + ], + [ + 279, + 0.35496513664242096 + ], + [ + 280, + 0.12745523697359876 + ], + [ + 281, + 0.14184147429940056 + ], + [ + 288, + 0.1525449046684155 + ], + [ + 292, + 0.17976326600627449 + ], + [ + 298, + 0.145926024612623 + ], + [ + 305, + 0.06482031972220538 + ], + [ + 313, + 0.12654916582702472 + ], + [ + 314, + 0.29147678907716834 + ], + [ + 319, + 0.08029730204425768 + ], + [ + 328, + 0.0810483646451642 + ], + [ + 336, + 0.31211380290221935 + ], + [ + 343, + 0.03266259850933581 + ], + [ + 351, + 0.019211195037147816 + ], + [ + 356, + 0.030878914863700088 + ], + [ + 360, + 0.2044190774456798 + ], + [ + 366, + 0.041468632715159146 + ], + [ + 367, + 0.426832092306488 + ], + [ + 372, + 0.12112554157728472 + ], + [ + 380, + 0.06952658877610012 + ], + [ + 385, + 0.1115212520746462 + ], + [ + 389, + 0.18663142467903956 + ], + [ + 394, + 0.2805596284878942 + ], + [ + 404, + 0.11709923711280217 + ], + [ + 407, + 0.13062658989679524 + ], + [ + 411, + 0.29487620875526777 + ], + [ + 416, + 0.06095228191794946 + ], + [ + 418, + 0.13831321365335847 + ], + [ + 419, + 0.2531303283451327 + ], + [ + 427, + 0.10873614757884979 + ], + [ + 434, + 0.17509104421629268 + ], + [ + 441, + 0.07275134642763598 + ], + [ + 442, + 0.4996604446983905 + ], + [ + 445, + 0.12051644429150693 + ], + [ + 449, + 0.19083319973856866 + ], + [ + 454, + 0.25598107259529523 + ], + [ + 458, + 0.17424824438096667 + ], + [ + 467, + 0.12088582658553726 + ], + [ + 473, + 0.09928498792649744 + ], + [ + 480, + 0.00802860987182899 + ], + [ + 486, + 0.016092976198293307 + ], + [ + 490, + 0.3208956701578117 + ], + [ + 497, + 0.1570710842497169 + ], + [ + 504, + 0.10044533668977804 + ], + [ + 505, + 0.9123670036799327 + ], + [ + 511, + 0.20294424929399024 + ], + [ + 518, + 0.2565924389625657 + ], + [ + 521, + 0.20145606000657948 + ], + [ + 527, + 0.2679676382375167 + ], + [ + 532, + 0.1748254801830466 + ], + [ + 535, + 0.5519631891352729 + ], + [ + 541, + 0.16887247500122918 + ], + [ + 546, + 0.196696115859768 + ], + [ + 547, + 0.6519119041928995 + ], + [ + 555, + 0.24877615814080228 + ], + [ + 556, + 0.9211526203710017 + ], + [ + 558, + 0.6391137656290311 + ], + [ + 559, + 0.4756973516225311 + ], + [ + 565, + 0.43511313371019533 + ], + [ + 569, + 0.40041355176337523 + ], + [ + 575, + 0.33811426495785585 + ], + [ + 578, + 0.5039495803636953 + ], + [ + 582, + 0.20257006164667393 + ], + [ + 586, + 0.23215239215309846 + ], + [ + 591, + 0.3071373252000151 + ], + [ + 593, + 0.44443772550527644 + ], + [ + 597, + 0.3424429396593139 + ], + [ + 598, + 0.6641969352111369 + ], + [ + 599, + 0.4096182072795361 + ], + [ + 600, + 0.2560193984844068 + ], + [ + 606, + 0.15047571793754091 + ], + [ + 610, + 0.19773436697304364 + ], + [ + 612, + 0.870188566228186 + ], + [ + 618, + 0.30855937683167817 + ], + [ + 625, + 0.38009342445114824 + ], + [ + 629, + 1.0 + ], + [ + 634, + 0.30033134957694 + ], + [ + 639, + 0.35654304201642456 + ], + [ + 644, + 0.499240239894302 + ], + [ + 646, + 0.3575630424450564 + ] + ] + ], + [ + [ + [ + 40, + 1.0 + ], + [ + 50, + 0.5735565468816659 + ], + [ + 53, + 0.4694586786479066 + ], + [ + 62, + 0.22415323534495435 + ], + [ + 70, + 0.28964431193907675 + ], + [ + 79, + 0.22768011205835728 + ], + [ + 83, + 0.2886864061363834 + ], + [ + 90, + 0.6699584629717696 + ], + [ + 91, + 0.3744178688016451 + ], + [ + 95, + 0.18970706466530557 + ], + [ + 102, + 0.12597406436119088 + ], + [ + 107, + 0.1645166129543124 + ], + [ + 111, + 0.178987339542025 + ], + [ + 113, + 0.34382211439283833 + ], + [ + 115, + 0.28290515880101336 + ], + [ + 117, + 0.2778158782279602 + ], + [ + 118, + 0.5890719481227842 + ], + [ + 124, + 0.4552053443676982 + ], + [ + 134, + 0.381139853640995 + ], + [ + 141, + 0.03567925574031676 + ], + [ + 144, + 0.1576636475493666 + ], + [ + 148, + 0.10502284768398933 + ], + [ + 152, + 0.08603634955935555 + ], + [ + 156, + 0.13164000420184496 + ], + [ + 160, + 0.03738936427396035 + ], + [ + 168, + 0.13151445057888153 + ], + [ + 173, + 0.1471059121376345 + ], + [ + 175, + 0.08958963636001896 + ], + [ + 182, + 0.1448895314484913 + ], + [ + 186, + 0.14012001049963135 + ], + [ + 191, + 0.2600542226019825 + ], + [ + 192, + 0.28250874711492774 + ], + [ + 198, + 0.13516542011484775 + ], + [ + 200, + 0.10724783859174737 + ], + [ + 212, + 0.18832848117353385 + ], + [ + 215, + 0.19300062057254466 + ], + [ + 217, + 0.1922455107675213 + ], + [ + 220, + 0.16367295205415563 + ], + [ + 226, + 0.2852971406297681 + ], + [ + 229, + 0.2878813672606105 + ], + [ + 233, + 0.17264676427677572 + ], + [ + 237, + 0.5255867702895528 + ], + [ + 238, + 0.15824188174467352 + ], + [ + 242, + 0.23375328872017964 + ], + [ + 246, + 0.16476305296600957 + ], + [ + 249, + 0.37435881881442284 + ], + [ + 250, + 0.20948381598073226 + ], + [ + 257, + 0.22526977426935132 + ], + [ + 260, + 0.12368843070611313 + ], + [ + 269, + 0.0712962688948487 + ], + [ + 270, + 0.32756847242515885 + ], + [ + 274, + 0.1590226867885437 + ], + [ + 279, + 0.22447683795870038 + ], + [ + 280, + 0.15088053123386025 + ], + [ + 281, + 0.14641320795133878 + ], + [ + 288, + 0.32083768110170935 + ], + [ + 292, + 0.16576914664592451 + ], + [ + 298, + 0.3239449111536672 + ], + [ + 305, + 0.125062766437306 + ], + [ + 313, + 0.2108221360431995 + ], + [ + 314, + 0.28570657064100063 + ], + [ + 319, + 0.1398753790078383 + ], + [ + 328, + 0.19217402461601577 + ], + [ + 336, + 0.48345320606797476 + ], + [ + 343, + 0.09888571609614896 + ], + [ + 351, + 0.1496548893224358 + ], + [ + 356, + 0.04864794842863193 + ], + [ + 360, + 0.07206870250431825 + ], + [ + 366, + 0.12005579727214986 + ], + [ + 367, + 0.31255570860684556 + ], + [ + 372, + 0.08369304795166098 + ], + [ + 380, + 0.13436992634811018 + ], + [ + 385, + 0.05299054986614245 + ], + [ + 389, + 0.07959768696974065 + ], + [ + 394, + 0.12776093986647438 + ], + [ + 404, + 0.18909969974207266 + ], + [ + 407, + 0.09626503766723377 + ], + [ + 411, + 0.04531836275658627 + ], + [ + 416, + 0.02763394272415393 + ], + [ + 418, + 0.04598908531776788 + ], + [ + 419, + 0.13729841826377598 + ], + [ + 427, + 0.11178983170207576 + ], + [ + 434, + 0.08045035720654886 + ], + [ + 441, + 0.09020823131066931 + ], + [ + 442, + 0.17757498227706947 + ], + [ + 445, + 0.06327807029688834 + ], + [ + 449, + 0.0656577356931315 + ], + [ + 454, + 0.07223438644445539 + ], + [ + 458, + 0.05719820337060598 + ], + [ + 467, + 0.07534877498852519 + ], + [ + 473, + 0.04451141668834278 + ], + [ + 480, + 0.02872248275585935 + ], + [ + 486, + 0.03084863003356047 + ], + [ + 490, + 0.07889875348024059 + ], + [ + 497, + 0.01668688762910254 + ], + [ + 504, + 0.0 + ], + [ + 505, + 0.3269108768090111 + ], + [ + 511, + 0.32586219214811607 + ], + [ + 518, + 0.18731876038518144 + ], + [ + 521, + 0.05419545818937326 + ], + [ + 527, + 0.17305631173917482 + ], + [ + 532, + 0.0957569486639819 + ], + [ + 535, + 0.1655347015741945 + ], + [ + 541, + 0.037471236550117734 + ], + [ + 546, + 0.09079007817866812 + ], + [ + 547, + 0.22002207640718652 + ], + [ + 555, + 0.16357394960057073 + ], + [ + 556, + 0.2996384124252706 + ], + [ + 558, + 0.3009608177454946 + ], + [ + 559, + 0.08935806654913298 + ], + [ + 565, + 0.22855754122427901 + ], + [ + 569, + 0.14192576580076546 + ], + [ + 575, + 0.17085014355148362 + ], + [ + 578, + 0.197851837990804 + ], + [ + 582, + 0.08104281155740664 + ], + [ + 586, + 0.2898971997068399 + ], + [ + 591, + 0.2162263601603357 + ], + [ + 593, + 0.06382322931204783 + ], + [ + 597, + 0.08525065021547014 + ], + [ + 598, + 0.12893487122697356 + ], + [ + 599, + 0.08606254312256405 + ], + [ + 600, + 0.13247542321325362 + ], + [ + 606, + 0.03942146339741731 + ], + [ + 610, + 0.072616606264199 + ], + [ + 612, + 0.5241002970684452 + ], + [ + 618, + 0.1262545413618755 + ], + [ + 625, + 0.3094265768601326 + ], + [ + 629, + 0.20812456357775658 + ], + [ + 634, + 0.3009671166799421 + ], + [ + 639, + 0.34395886704882445 + ], + [ + 644, + 0.8656912131526224 + ], + [ + 646, + 0.23944668403868782 + ] + ], + [ + [ + 67, + 0.27646251861610893 + ], + [ + 73, + 0.12576835229997424 + ], + [ + 76, + 0.10601507662821774 + ], + [ + 77, + 0.10780695365276212 + ], + [ + 81, + 0.060597021532946396 + ], + [ + 87, + 0.05969575690371849 + ], + [ + 90, + 0.031682892886309016 + ], + [ + 96, + 0.04253010458267892 + ], + [ + 107, + 0.06250851231088793 + ], + [ + 113, + 0.05452770720385907 + ], + [ + 118, + 0.06053439919793856 + ], + [ + 122, + 0.08045495699866227 + ], + [ + 123, + 0.12211375349205417 + ], + [ + 128, + 0.030392636211589948 + ], + [ + 131, + 0.039435383920427294 + ], + [ + 132, + 0.07930006662118537 + ], + [ + 141, + 0.044308707879429886 + ], + [ + 151, + 0.07967662395088766 + ], + [ + 154, + 0.04969666879954052 + ], + [ + 158, + 0.045437848576113445 + ], + [ + 164, + 0.11704058446535474 + ], + [ + 168, + 0.04279867418125431 + ], + [ + 177, + 0.028759589798046915 + ], + [ + 178, + 0.1084219113825578 + ], + [ + 183, + 0.03681178855547492 + ], + [ + 185, + 0.042410865897223024 + ], + [ + 189, + 0.11447144351673001 + ], + [ + 190, + 0.06777281523578758 + ], + [ + 193, + 0.07677074442267554 + ], + [ + 195, + 0.05647095512038531 + ], + [ + 198, + 0.035744854229304 + ], + [ + 206, + 0.04739118936309414 + ], + [ + 212, + 0.0504255798041532 + ], + [ + 213, + 0.05277199189533896 + ], + [ + 219, + 0.054722727004662276 + ], + [ + 224, + 0.061525793195850456 + ], + [ + 226, + 0.05523891904351875 + ], + [ + 231, + 0.05204167734346992 + ], + [ + 234, + 0.05570037073525134 + ], + [ + 235, + 0.044400467482792014 + ], + [ + 242, + 0.051710470407699924 + ], + [ + 251, + 0.04372731041303452 + ], + [ + 253, + 0.04149889626761911 + ], + [ + 254, + 0.01769108433767331 + ], + [ + 258, + 0.021623330759381918 + ], + [ + 259, + 0.03202564563507592 + ], + [ + 260, + 0.04033902842471456 + ], + [ + 264, + 0.025110351867383737 + ], + [ + 266, + 0.04114903869821648 + ], + [ + 267, + 0.012580699995488134 + ], + [ + 271, + 0.02071530255949567 + ], + [ + 275, + 0.029669462006030364 + ], + [ + 279, + 0.03713602395161133 + ], + [ + 281, + 0.06693197535347394 + ], + [ + 287, + 0.02764505653727419 + ], + [ + 297, + 0.03999616536054607 + ], + [ + 299, + 0.04808308995085788 + ], + [ + 304, + 0.17836065328474138 + ], + [ + 307, + 0.3507156811950463 + ], + [ + 308, + 0.1540372636999492 + ], + [ + 309, + 0.04961194020481651 + ], + [ + 311, + 0.05101033640660168 + ], + [ + 317, + 0.016253464551411353 + ], + [ + 322, + 0.03427627993029706 + ], + [ + 323, + 0.024562533152781046 + ], + [ + 327, + 0.030432885074531894 + ], + [ + 333, + 0.021863484933246018 + ], + [ + 334, + 0.04364113314725474 + ], + [ + 338, + 0.03361308557784387 + ], + [ + 340, + 0.04593523693337248 + ], + [ + 343, + 0.019434736832888846 + ], + [ + 345, + 0.009486490177382902 + ], + [ + 351, + 0.029723324541628064 + ], + [ + 352, + 0.09418401588360359 + ], + [ + 357, + 0.015448073343263273 + ], + [ + 362, + 0.010801675978314284 + ], + [ + 366, + 0.04983060099656208 + ], + [ + 370, + 0.039765443660079065 + ], + [ + 371, + 0.05974742962439928 + ], + [ + 377, + 0.05645649395911344 + ], + [ + 378, + 0.24129661502795152 + ], + [ + 382, + 0.16880699232920476 + ], + [ + 387, + 0.48540468004277054 + ], + [ + 389, + 0.6965495960135365 + ], + [ + 392, + 1.0 + ], + [ + 401, + 0.2980434796696436 + ], + [ + 402, + 0.21352931635504527 + ], + [ + 408, + 0.4503886127569406 + ], + [ + 411, + 0.21429491580683913 + ], + [ + 416, + 0.11701900091527151 + ], + [ + 422, + 0.23754546081883082 + ], + [ + 423, + 0.28399547885110094 + ], + [ + 424, + 0.05916421743757298 + ], + [ + 426, + 0.07066629266723856 + ], + [ + 429, + 0.0651323697208193 + ], + [ + 435, + 0.11745999724260073 + ], + [ + 438, + 0.07756478653405585 + ], + [ + 442, + 0.0535000061974383 + ], + [ + 446, + 0.11998507372935197 + ], + [ + 447, + 0.08015733588442675 + ], + [ + 451, + 0.08468237085860975 + ], + [ + 454, + 0.0710583796713786 + ], + [ + 458, + 0.043718728140238354 + ], + [ + 460, + 0.04562480512403836 + ], + [ + 464, + 0.05892319734863808 + ], + [ + 465, + 0.04346400563082513 + ], + [ + 467, + 0.07037231888500121 + ], + [ + 468, + 0.040726277579767184 + ], + [ + 471, + 0.05535011499073509 + ], + [ + 475, + 0.06578255021379165 + ], + [ + 480, + 0.1449190678260951 + ], + [ + 481, + 0.20511259157043357 + ], + [ + 482, + 0.019958816000455286 + ], + [ + 489, + 0.015534107148143633 + ], + [ + 492, + 0.017343969224196083 + ], + [ + 493, + 0.022722521718207208 + ], + [ + 495, + 0.03239226649134649 + ], + [ + 496, + 0.052224398262202924 + ], + [ + 500, + 0.05308805570077659 + ], + [ + 502, + 0.0665738605203125 + ], + [ + 505, + 0.07411717592733688 + ], + [ + 507, + 0.09826343802445625 + ], + [ + 509, + 0.195962832334397 + ], + [ + 514, + 0.01284147163346742 + ], + [ + 515, + 0.019821225809309315 + ], + [ + 518, + 0.025281632897583534 + ], + [ + 523, + 0.016565218734270606 + ], + [ + 526, + 0.014823469859940666 + ], + [ + 529, + 0.04164501300093707 + ], + [ + 532, + 0.03287857000445202 + ], + [ + 536, + 0.027792134673630484 + ], + [ + 538, + 0.027595972403879968 + ], + [ + 540, + 0.0 + ], + [ + 545, + 0.009707605715761873 + ], + [ + 550, + 0.009848775264533146 + ], + [ + 551, + 0.018574855107603948 + ], + [ + 554, + 0.0014949911479714283 + ], + [ + 557, + 0.0019511438425963783 + ], + [ + 561, + 0.011008364000844645 + ], + [ + 564, + 0.025913637785616107 + ], + [ + 566, + 0.10796489248087021 + ], + [ + 568, + 0.08718914544731468 + ], + [ + 571, + 0.06629423170429158 + ], + [ + 578, + 0.1058599298880115 + ], + [ + 583, + 0.03770508853658373 + ], + [ + 585, + 0.05031887285541085 + ], + [ + 586, + 0.03409478530207769 + ], + [ + 594, + 0.026482662995931435 + ], + [ + 598, + 0.026215448549177147 + ], + [ + 603, + 0.013371644369872985 + ], + [ + 606, + 0.013306989167773736 + ], + [ + 607, + 0.040467325426994744 + ], + [ + 610, + 0.04106163611301713 + ], + [ + 616, + 0.03640151423354067 + ], + [ + 621, + 0.02749520321978594 + ], + [ + 623, + 0.02890692473901352 + ], + [ + 627, + 0.03651950593687943 + ], + [ + 633, + 0.052858047880538786 + ], + [ + 637, + 0.04444127117446919 + ], + [ + 642, + 0.048855592125117815 + ], + [ + 647, + 0.016466952225500705 + ], + [ + 652, + 0.0443418039819851 + ], + [ + 655, + 0.08507178572155609 + ], + [ + 656, + 0.12111779567054748 + ], + [ + 657, + 0.020407205198369357 + ], + [ + 662, + 0.00834023770378366 + ], + [ + 666, + 0.016450301267008395 + ], + [ + 670, + 0.04733144881499625 + ], + [ + 682, + 0.029551479659702812 + ], + [ + 687, + 0.019922388403741043 + ], + [ + 693, + 0.03367403233095617 + ], + [ + 697, + 0.04383208350425153 + ], + [ + 703, + 0.010934810544326221 + ], + [ + 704, + 0.01731871588086367 + ], + [ + 708, + 0.008628672272536402 + ], + [ + 715, + 0.012971133642293279 + ], + [ + 718, + 0.024963564981826517 + ], + [ + 721, + 0.042070171133481854 + ], + [ + 725, + 0.030261873094903174 + ], + [ + 731, + 0.018585727257823356 + ], + [ + 732, + 0.10090497405815717 + ], + [ + 734, + 0.08347790639870518 + ], + [ + 735, + 0.04079152026723437 + ], + [ + 739, + 0.07060913392867728 + ], + [ + 740, + 0.03875675010131814 + ], + [ + 749, + 0.04391978883189846 + ], + [ + 750, + 0.0835433312227181 + ], + [ + 751, + 0.035642544412209334 + ], + [ + 755, + 0.017820343709661766 + ], + [ + 760, + 0.04057396375765494 + ], + [ + 766, + 0.050922151893802295 + ], + [ + 771, + 0.05183172599139051 + ] + ], + [ + [ + 37, + 0.5430437434846188 + ], + [ + 45, + 0.5267164474950109 + ], + [ + 46, + 0.20407498694526563 + ], + [ + 57, + 0.1345965791125409 + ], + [ + 58, + 0.5012650047349656 + ], + [ + 60, + 0.49511891074861997 + ], + [ + 63, + 0.2124436944490688 + ], + [ + 69, + 0.17294061729081708 + ], + [ + 72, + 0.18997678661986736 + ], + [ + 76, + 0.06416749415995175 + ], + [ + 80, + 0.17431593809903273 + ], + [ + 81, + 0.24088666227342911 + ], + [ + 83, + 0.2911757285313726 + ], + [ + 88, + 0.13837268414792678 + ], + [ + 91, + 0.2448548781870112 + ], + [ + 96, + 0.21373358403245948 + ], + [ + 97, + 0.17950791312036937 + ], + [ + 100, + 0.39231804028854245 + ], + [ + 101, + 0.20727661415495935 + ], + [ + 111, + 0.6006306982782765 + ], + [ + 113, + 0.24173879975138227 + ], + [ + 116, + 0.1489381156842521 + ], + [ + 122, + 0.06940776220986059 + ], + [ + 125, + 0.10198912727044865 + ], + [ + 129, + 0.048372370426819085 + ], + [ + 133, + 0.0775702214554726 + ], + [ + 134, + 0.1418013517603487 + ], + [ + 139, + 0.1915410718531151 + ], + [ + 140, + 0.10599477232498047 + ], + [ + 143, + 0.2124142037579923 + ], + [ + 148, + 0.15310252448699596 + ], + [ + 157, + 0.13216744455801227 + ], + [ + 160, + 0.08861470082321808 + ], + [ + 169, + 0.12036627339358884 + ], + [ + 170, + 0.35093179730850077 + ], + [ + 171, + 0.023952196425173846 + ], + [ + 173, + 0.07726552990103104 + ], + [ + 179, + 0.05023392502014254 + ], + [ + 181, + 0.04591107414388443 + ], + [ + 182, + 0.03944568326193574 + ], + [ + 185, + 0.03880180077562521 + ], + [ + 186, + 0.04556065132989969 + ], + [ + 187, + 0.0 + ], + [ + 188, + 0.004746701064711805 + ], + [ + 189, + 0.014820900719216887 + ], + [ + 192, + 0.01628442445494544 + ], + [ + 196, + 0.008217865898757759 + ], + [ + 197, + 0.030026258232550593 + ], + [ + 204, + 0.03709385153717411 + ], + [ + 209, + 0.04225678967262962 + ], + [ + 210, + 0.07718017102354678 + ], + [ + 211, + 0.007343928814638561 + ], + [ + 212, + 0.06158573415360254 + ], + [ + 214, + 0.00946454891271085 + ], + [ + 221, + 0.03348226294502958 + ], + [ + 224, + 0.1482939772847164 + ], + [ + 233, + 0.045150026087084065 + ], + [ + 241, + 0.08297411081735848 + ], + [ + 247, + 0.06700518109784047 + ], + [ + 256, + 0.07271858029359424 + ], + [ + 260, + 0.0969857843308261 + ], + [ + 267, + 0.18773700588286 + ], + [ + 270, + 0.25413021075308406 + ], + [ + 275, + 0.22582477084902644 + ], + [ + 276, + 0.3630205177296157 + ], + [ + 277, + 0.2496705921139447 + ], + [ + 278, + 0.30413310896397966 + ], + [ + 279, + 0.5260137266200821 + ], + [ + 280, + 0.4419755001426142 + ], + [ + 281, + 0.7716773144632485 + ], + [ + 282, + 0.12183740435376901 + ], + [ + 283, + 0.18492424203274485 + ], + [ + 284, + 0.18302579878316905 + ], + [ + 285, + 0.6506346328763712 + ], + [ + 287, + 0.14831276805441426 + ], + [ + 292, + 0.2969018620498931 + ], + [ + 295, + 0.2906845680248092 + ], + [ + 299, + 0.11660829058938245 + ], + [ + 303, + 0.30094784975079464 + ], + [ + 305, + 0.3149214714306988 + ], + [ + 310, + 0.18177152964505944 + ], + [ + 313, + 0.2869941193610195 + ], + [ + 317, + 0.06102227034173275 + ], + [ + 321, + 0.11776509393446137 + ], + [ + 324, + 0.4192136061778138 + ], + [ + 328, + 0.23249713475058442 + ], + [ + 335, + 0.18103831554467512 + ], + [ + 342, + 0.21839516444098608 + ], + [ + 348, + 0.3159214199936838 + ], + [ + 351, + 0.3335152879725991 + ], + [ + 357, + 0.5357559589099754 + ], + [ + 359, + 0.48401912420835996 + ], + [ + 363, + 0.5992067665214648 + ], + [ + 364, + 0.5628760257908745 + ], + [ + 367, + 0.8442801502028303 + ], + [ + 369, + 1.0 + ], + [ + 370, + 0.5310668710315456 + ], + [ + 379, + 0.25856826813494244 + ], + [ + 382, + 0.1969222833249989 + ], + [ + 390, + 0.2827226816351763 + ], + [ + 391, + 0.18244536915537252 + ], + [ + 392, + 0.17875673477673362 + ], + [ + 393, + 0.2534327976868585 + ], + [ + 394, + 0.9550270186457516 + ], + [ + 395, + 0.41967156421262125 + ], + [ + 396, + 0.996899375145902 + ], + [ + 397, + 0.03407342679661371 + ], + [ + 398, + 0.038710179782568854 + ], + [ + 399, + 0.07106304869625574 + ], + [ + 400, + 0.3083777515508156 + ], + [ + 404, + 0.053885445747645914 + ], + [ + 410, + 0.14886045526128902 + ], + [ + 411, + 0.12597385802984465 + ], + [ + 412, + 0.05272972937457955 + ], + [ + 418, + 0.16542831285396817 + ], + [ + 421, + 0.096080222533874 + ], + [ + 423, + 0.11213373472750345 + ], + [ + 430, + 0.1167416621221538 + ], + [ + 443, + 0.21368069030112918 + ], + [ + 446, + 0.14464775872782998 + ], + [ + 450, + 0.12508126045349038 + ], + [ + 456, + 0.11637843946695065 + ], + [ + 459, + 0.17859384043766552 + ], + [ + 463, + 0.13273639157896544 + ], + [ + 466, + 0.2312380665681577 + ], + [ + 467, + 0.21705288240560563 + ], + [ + 468, + 0.1315107642718594 + ], + [ + 469, + 0.0839111072665238 + ], + [ + 475, + 0.08433646081910816 + ], + [ + 481, + 0.11534849376935323 + ], + [ + 485, + 0.21521531062157204 + ], + [ + 486, + 0.2025670794404789 + ], + [ + 488, + 0.14389447702560787 + ], + [ + 491, + 0.13822592768881736 + ], + [ + 497, + 0.3521471454364215 + ], + [ + 500, + 0.2658423960493579 + ], + [ + 504, + 0.17375216236975682 + ], + [ + 508, + 0.4484381565162333 + ], + [ + 510, + 0.3046985168706477 + ], + [ + 516, + 0.1601848822629075 + ], + [ + 527, + 0.4578954026429285 + ], + [ + 531, + 0.21979363545044553 + ], + [ + 540, + 0.278581572868251 + ], + [ + 543, + 0.1351320486470653 + ], + [ + 545, + 0.1930792785311183 + ], + [ + 548, + 0.1442174165988794 + ], + [ + 550, + 0.23650830175174112 + ], + [ + 555, + 0.3577394370064925 + ], + [ + 558, + 0.24202773442197997 + ], + [ + 562, + 0.11523251445709042 + ], + [ + 566, + 0.35482998682878364 + ], + [ + 568, + 0.22660525014512176 + ], + [ + 572, + 0.0758275159542851 + ], + [ + 577, + 0.19353833973528556 + ], + [ + 581, + 0.11943085463651573 + ], + [ + 589, + 0.13553723600420717 + ], + [ + 594, + 0.05800972632749134 + ], + [ + 603, + 0.13731388323476917 + ], + [ + 606, + 0.10210694732069411 + ], + [ + 609, + 0.08262658205496556 + ], + [ + 612, + 0.13088209353691943 + ], + [ + 616, + 0.1086479379415006 + ], + [ + 618, + 0.07170541640860138 + ], + [ + 626, + 0.16193535020603575 + ], + [ + 629, + 0.0808780989977042 + ], + [ + 633, + 0.15838363964618957 + ] + ], + [ + [ + 40, + 0.5872945268833754 + ], + [ + 49, + 0.544488749124687 + ], + [ + 53, + 0.39205413780441756 + ], + [ + 61, + 0.42002746023679594 + ], + [ + 65, + 0.2052929440809521 + ], + [ + 69, + 0.22770454933686168 + ], + [ + 75, + 0.17415507834293037 + ], + [ + 80, + 0.3093239424469011 + ], + [ + 81, + 0.22290564003851712 + ], + [ + 83, + 0.2347060143257882 + ], + [ + 84, + 0.05752794859927595 + ], + [ + 87, + 0.17434782512882896 + ], + [ + 92, + 0.3248700502728597 + ], + [ + 93, + 0.22794305259899544 + ], + [ + 94, + 0.12657119962362634 + ], + [ + 96, + 0.3429771860731126 + ], + [ + 100, + 0.2530888143416525 + ], + [ + 105, + 0.2861991734305518 + ], + [ + 110, + 0.8601970960867834 + ], + [ + 118, + 0.6101432346997964 + ], + [ + 122, + 0.1033373891379086 + ], + [ + 127, + 0.061721274359904606 + ], + [ + 130, + 0.0805265225219635 + ], + [ + 134, + 0.06838209971449292 + ], + [ + 142, + 0.11568222145891682 + ], + [ + 144, + 0.09535165531356063 + ], + [ + 145, + 0.06652077827382809 + ], + [ + 147, + 0.11356439110822691 + ], + [ + 151, + 0.06475548145199321 + ], + [ + 154, + 0.14536261336128956 + ], + [ + 155, + 0.08047013589156286 + ], + [ + 158, + 0.10948712381677536 + ], + [ + 160, + 0.10246518565652231 + ], + [ + 163, + 0.07040778651999591 + ], + [ + 169, + 0.04595322991751067 + ], + [ + 173, + 0.1076041252828459 + ], + [ + 174, + 0.09822539259695301 + ], + [ + 177, + 0.1945851794121262 + ], + [ + 186, + 0.07511947094910727 + ], + [ + 187, + 0.10220968760547829 + ], + [ + 188, + 0.2230755584349759 + ], + [ + 189, + 0.045942567854637274 + ], + [ + 190, + 0.08670189697238397 + ], + [ + 191, + 0.06381901554071194 + ], + [ + 192, + 0.4408358626161388 + ], + [ + 198, + 0.5396255681626395 + ], + [ + 204, + 0.5990334472763531 + ], + [ + 211, + 0.5616437566285293 + ], + [ + 215, + 0.4089337265314307 + ], + [ + 221, + 0.204343449266583 + ], + [ + 224, + 0.2441191100796784 + ], + [ + 228, + 0.13182814203972704 + ], + [ + 232, + 0.28604274441954886 + ], + [ + 234, + 0.5482937233041456 + ], + [ + 237, + 0.20810578655282608 + ], + [ + 243, + 0.16201985224160262 + ], + [ + 246, + 0.23787805789959152 + ], + [ + 250, + 0.09464654740492853 + ], + [ + 254, + 0.23621763254554692 + ], + [ + 256, + 0.588789926842763 + ], + [ + 267, + 0.7804614441032327 + ], + [ + 269, + 0.46357285718514607 + ], + [ + 272, + 0.18359905522773273 + ], + [ + 274, + 0.26082535956902003 + ], + [ + 279, + 0.32794725330749797 + ], + [ + 282, + 0.21698652883717265 + ], + [ + 286, + 0.13146553337105174 + ], + [ + 290, + 0.3560154777081641 + ], + [ + 292, + 0.24343207183074947 + ], + [ + 295, + 0.32664056472386066 + ], + [ + 297, + 0.24882297033530767 + ], + [ + 302, + 0.4822699566135023 + ], + [ + 305, + 0.6408923358763209 + ], + [ + 309, + 0.48221967204444904 + ], + [ + 318, + 0.5684787016233621 + ], + [ + 324, + 0.6307259124908393 + ], + [ + 327, + 1.0 + ], + [ + 328, + 0.706316553885136 + ], + [ + 334, + 0.3400795376425658 + ], + [ + 336, + 0.3363291983915313 + ], + [ + 340, + 0.2651332203177038 + ], + [ + 341, + 0.28039286345525566 + ], + [ + 344, + 0.4337244183639721 + ], + [ + 346, + 0.5450080744826121 + ], + [ + 347, + 0.39450613813922725 + ], + [ + 351, + 0.10248695683743056 + ], + [ + 356, + 0.19720947854966994 + ], + [ + 360, + 0.20660684550444025 + ], + [ + 368, + 0.4637284524874273 + ], + [ + 371, + 0.1917647894338745 + ], + [ + 376, + 0.27084662502113743 + ], + [ + 382, + 0.3217653424318476 + ], + [ + 387, + 0.328091277495865 + ], + [ + 389, + 0.10183168161056215 + ], + [ + 396, + 0.08142282771408466 + ], + [ + 399, + 0.11385829935671565 + ], + [ + 407, + 0.04843318424526842 + ], + [ + 414, + 0.11356206268898497 + ], + [ + 417, + 0.253819306793831 + ], + [ + 418, + 0.17583733570993262 + ], + [ + 419, + 0.35944531434391946 + ], + [ + 420, + 0.08713678897314572 + ], + [ + 421, + 0.09228295951814418 + ], + [ + 422, + 0.12173423663613843 + ], + [ + 426, + 0.19409660387064373 + ], + [ + 427, + 0.13109296664446457 + ], + [ + 428, + 0.2105127316253916 + ], + [ + 429, + 0.006592412428666714 + ], + [ + 430, + 0.009242590219154218 + ], + [ + 431, + 0.034452600520166105 + ], + [ + 432, + 0.2783201375212165 + ], + [ + 434, + 0.08148755820778997 + ], + [ + 441, + 0.1284452499866452 + ], + [ + 445, + 0.13940077827453845 + ], + [ + 450, + 0.16200419963181276 + ], + [ + 454, + 0.10744814874232039 + ], + [ + 457, + 0.1597236053363004 + ], + [ + 461, + 0.12265395750177363 + ], + [ + 464, + 0.12683930258628867 + ], + [ + 465, + 0.15195407557210003 + ], + [ + 473, + 0.1082171652246624 + ], + [ + 475, + 0.06868441275553372 + ], + [ + 482, + 0.02849538943286741 + ], + [ + 487, + 0.06080250089685563 + ], + [ + 492, + 0.023817179328147472 + ], + [ + 501, + 0.02767887095827073 + ], + [ + 509, + 0.03660545405198466 + ], + [ + 510, + 0.1759354519737278 + ], + [ + 514, + 0.042709721772681945 + ], + [ + 518, + 0.10409000261429245 + ], + [ + 523, + 0.04693176426657026 + ], + [ + 527, + 0.046706883926029986 + ], + [ + 536, + 0.08463981635539493 + ], + [ + 539, + 0.1132382157550734 + ], + [ + 545, + 0.06523540495009333 + ], + [ + 546, + 0.08936162216279367 + ], + [ + 547, + 0.11757785685482879 + ], + [ + 548, + 0.07426270427022671 + ], + [ + 549, + 0.04539383306817019 + ], + [ + 550, + 0.10260291138808275 + ], + [ + 553, + 0.0761635198601975 + ], + [ + 562, + 0.09360783339930875 + ], + [ + 570, + 0.16702465043680823 + ], + [ + 573, + 0.20032124413032923 + ], + [ + 577, + 0.3036683648201452 + ], + [ + 582, + 0.0 + ], + [ + 587, + 0.004667202343348996 + ], + [ + 591, + 0.047952718715696 + ], + [ + 601, + 0.09170892936033365 + ] + ], + [ + [ + 55, + 0.9152244385405898 + ], + [ + 58, + 0.27645290561883074 + ], + [ + 59, + 0.5148566301227929 + ], + [ + 67, + 0.2399637426199175 + ], + [ + 68, + 0.34166172362684444 + ], + [ + 72, + 0.1369900615572421 + ], + [ + 75, + 0.18779428249071645 + ], + [ + 79, + 0.23266217474081632 + ], + [ + 84, + 0.10384169895610362 + ], + [ + 97, + 0.19218068136027971 + ], + [ + 102, + 0.16018169139238148 + ], + [ + 111, + 0.12615586843633655 + ], + [ + 114, + 0.15893097052531302 + ], + [ + 116, + 0.13974259021673802 + ], + [ + 123, + 0.14482245216146086 + ], + [ + 126, + 0.22433686181097315 + ], + [ + 131, + 0.10858699441774627 + ], + [ + 134, + 0.21248370459088042 + ], + [ + 138, + 0.239503097811128 + ], + [ + 148, + 0.11085728049316217 + ], + [ + 152, + 0.11819991326571457 + ], + [ + 156, + 0.1555052751912138 + ], + [ + 162, + 0.1753002551686658 + ], + [ + 163, + 0.3512832000705076 + ], + [ + 167, + 0.30259014886338154 + ], + [ + 173, + 0.26357428737934896 + ], + [ + 180, + 0.44906920266543077 + ], + [ + 186, + 0.42034326382548376 + ], + [ + 188, + 0.40719577713058774 + ], + [ + 192, + 0.3663746513766708 + ], + [ + 193, + 0.3786418506003904 + ], + [ + 196, + 0.4864311626888587 + ], + [ + 198, + 0.7166043510083331 + ], + [ + 200, + 0.39615093185933425 + ], + [ + 203, + 0.11999138952183022 + ], + [ + 209, + 0.21993525990367463 + ], + [ + 212, + 0.19441173303489437 + ], + [ + 216, + 0.08880273121772853 + ], + [ + 220, + 0.21032443126061293 + ], + [ + 221, + 0.20200798916521642 + ], + [ + 224, + 0.22033800713119678 + ], + [ + 228, + 0.1838255145006231 + ], + [ + 230, + 0.05395146299952069 + ], + [ + 231, + 0.12592771054505256 + ], + [ + 235, + 0.12328464470294816 + ], + [ + 236, + 0.2226918664357004 + ], + [ + 242, + 0.04274694757857151 + ], + [ + 247, + 0.11643423659111514 + ], + [ + 253, + 0.11609389045232486 + ], + [ + 254, + 0.5858170591727823 + ], + [ + 255, + 0.03791331458056755 + ], + [ + 261, + 0.06271924026548371 + ], + [ + 268, + 0.03008137234349412 + ], + [ + 270, + 0.08198903192616831 + ], + [ + 271, + 0.008027556066599337 + ], + [ + 276, + 0.0 + ], + [ + 280, + 0.04168214815526631 + ], + [ + 286, + 0.03733366518281877 + ], + [ + 289, + 0.027404714748330616 + ], + [ + 294, + 0.12897427185077365 + ], + [ + 295, + 0.041102439545014406 + ], + [ + 298, + 0.03427116336058029 + ], + [ + 302, + 0.1038097348671601 + ], + [ + 304, + 0.051497490577971386 + ], + [ + 305, + 0.10164008091956041 + ], + [ + 309, + 0.08780395394818694 + ], + [ + 310, + 0.5410789625736376 + ], + [ + 314, + 0.10485365065886393 + ], + [ + 321, + 0.24445562038941823 + ], + [ + 330, + 0.23849430360416207 + ], + [ + 333, + 0.20495713425939688 + ], + [ + 338, + 0.14086784689424453 + ], + [ + 344, + 0.11738237435628189 + ], + [ + 345, + 0.36416165215088697 + ], + [ + 352, + 0.15078005519148027 + ], + [ + 358, + 0.15260852184938686 + ], + [ + 368, + 0.22319039835692905 + ], + [ + 369, + 0.5184011548671563 + ], + [ + 374, + 0.15457704924677415 + ], + [ + 380, + 0.10732890856264969 + ], + [ + 383, + 0.2844044528211079 + ], + [ + 387, + 0.11273620590895275 + ], + [ + 391, + 0.10900779355032297 + ], + [ + 392, + 0.3066776472255376 + ], + [ + 397, + 0.11894425313264673 + ], + [ + 400, + 0.18908168426815156 + ], + [ + 402, + 0.35067353943793855 + ], + [ + 404, + 0.6590966924079741 + ], + [ + 405, + 0.6807624676963884 + ], + [ + 410, + 0.21906836635035362 + ], + [ + 416, + 0.08131589272408588 + ], + [ + 419, + 0.20180938941311838 + ], + [ + 423, + 0.0669226558679385 + ], + [ + 427, + 0.047323573015607794 + ], + [ + 428, + 0.21364827718833806 + ], + [ + 433, + 0.08658576218903022 + ], + [ + 436, + 0.10936251941417031 + ], + [ + 438, + 0.03802526771578102 + ], + [ + 440, + 0.1443157445882339 + ], + [ + 441, + 0.20147491832730313 + ], + [ + 444, + 0.29729159189228277 + ], + [ + 446, + 0.2158253475344622 + ], + [ + 451, + 0.1312897207646063 + ], + [ + 457, + 0.2324625143886424 + ], + [ + 460, + 0.23128124038986483 + ], + [ + 465, + 0.2060255657532919 + ], + [ + 471, + 0.14947069987906586 + ], + [ + 472, + 0.5223463603947923 + ], + [ + 481, + 0.1984253544783712 + ], + [ + 484, + 0.45250217348908317 + ], + [ + 490, + 0.2779549048677765 + ], + [ + 491, + 0.33674266461227736 + ], + [ + 492, + 1.0 + ], + [ + 493, + 0.01604122268152043 + ], + [ + 494, + 0.06486120192382362 + ], + [ + 495, + 0.03390761432545513 + ], + [ + 498, + 0.16870979963894298 + ], + [ + 507, + 0.027116056727271548 + ], + [ + 515, + 0.04533655224961452 + ], + [ + 516, + 0.39425016424083714 + ], + [ + 519, + 0.046287854804269876 + ], + [ + 523, + 0.0668243674156801 + ], + [ + 532, + 0.039973576815428624 + ], + [ + 535, + 0.14100584782118147 + ], + [ + 539, + 0.13973916260601124 + ], + [ + 541, + 0.07975908904675338 + ], + [ + 544, + 0.05124191899999768 + ], + [ + 547, + 0.15851888788930296 + ], + [ + 551, + 0.16330501378227125 + ], + [ + 559, + 0.01698868244047504 + ], + [ + 562, + 0.16292556072274234 + ], + [ + 564, + 0.18114820614716515 + ], + [ + 569, + 0.08436310314495149 + ], + [ + 572, + 0.033090964017312076 + ], + [ + 574, + 0.07755350599489079 + ], + [ + 577, + 0.011554400399644057 + ], + [ + 579, + 0.0447080216597567 + ], + [ + 581, + 0.1169690739452226 + ], + [ + 582, + 0.10818438736627964 + ], + [ + 588, + 0.08832109549974414 + ], + [ + 589, + 0.11212541740173099 + ], + [ + 591, + 0.36443768672590454 + ], + [ + 597, + 0.07045941869126719 + ], + [ + 606, + 0.08315028888351797 + ], + [ + 608, + 0.22158760547862627 + ], + [ + 615, + 0.0720751498975132 + ], + [ + 618, + 0.2532645520354862 + ] + ], + [ + [ + 40, + 0.47318005640138266 + ], + [ + 50, + 0.22419108799515777 + ], + [ + 53, + 0.24729616292526752 + ], + [ + 62, + 0.04733588641044824 + ], + [ + 70, + 0.06269176615217224 + ], + [ + 79, + 0.08321472318076424 + ], + [ + 83, + 0.05652376806784328 + ], + [ + 90, + 0.14822201526364465 + ], + [ + 91, + 0.2358122391500841 + ], + [ + 95, + 0.07348257781650913 + ], + [ + 102, + 0.06053358711470451 + ], + [ + 107, + 0.08126103663296216 + ], + [ + 111, + 0.07503997218563964 + ], + [ + 113, + 0.1264931690778358 + ], + [ + 115, + 0.078541456166721 + ], + [ + 117, + 0.07425381306834984 + ], + [ + 118, + 0.3213351234251724 + ], + [ + 124, + 0.08978804138852622 + ], + [ + 134, + 0.12207837882539192 + ], + [ + 141, + 0.0 + ], + [ + 144, + 0.12339673796019603 + ], + [ + 148, + 0.040913224590359475 + ], + [ + 152, + 0.05092433978242276 + ], + [ + 156, + 0.10237866533108426 + ], + [ + 160, + 0.026034901470439572 + ], + [ + 168, + 0.055044550329824964 + ], + [ + 173, + 0.06802929320569286 + ], + [ + 175, + 0.07731375459980078 + ], + [ + 182, + 0.06476591158801262 + ], + [ + 186, + 0.042375412709870365 + ], + [ + 191, + 0.07590256781024114 + ], + [ + 192, + 0.1975938641646564 + ], + [ + 198, + 0.06302399748184044 + ], + [ + 200, + 0.11114436664098863 + ], + [ + 212, + 0.19122869301013737 + ], + [ + 215, + 0.19345297589509847 + ], + [ + 217, + 0.1419809682206451 + ], + [ + 220, + 0.12543501972182897 + ], + [ + 226, + 0.12869490105183345 + ], + [ + 229, + 0.18264873371846424 + ], + [ + 233, + 0.04742119379571272 + ], + [ + 237, + 0.1395815401468434 + ], + [ + 238, + 0.1015122482058154 + ], + [ + 242, + 0.0869309021189146 + ], + [ + 246, + 0.08806044640587354 + ], + [ + 249, + 0.16885767319356568 + ], + [ + 250, + 0.06513186238557507 + ], + [ + 257, + 0.08625722534088859 + ], + [ + 260, + 0.1722750046939893 + ], + [ + 269, + 0.023489330804361035 + ], + [ + 270, + 0.20670386069778746 + ], + [ + 274, + 0.10152410120209268 + ], + [ + 279, + 0.21910303630612843 + ], + [ + 280, + 0.11318108556268887 + ], + [ + 281, + 0.14452654609543258 + ], + [ + 288, + 0.14571194476632798 + ], + [ + 292, + 0.1262168906034768 + ], + [ + 298, + 0.09210333320977421 + ], + [ + 305, + 0.051962111854024925 + ], + [ + 313, + 0.1077098427168715 + ], + [ + 314, + 0.1943846414071698 + ], + [ + 319, + 0.08926530910828342 + ], + [ + 328, + 0.07692096249097807 + ], + [ + 336, + 0.21221029085167525 + ], + [ + 343, + 0.01969585847552614 + ], + [ + 351, + 0.025486209669453884 + ], + [ + 356, + 0.058726484673883216 + ], + [ + 360, + 0.13927157347636437 + ], + [ + 366, + 0.04535227822716524 + ], + [ + 367, + 0.24199877680856627 + ], + [ + 372, + 0.09155331787893323 + ], + [ + 380, + 0.05645072714153961 + ], + [ + 385, + 0.16330431674048368 + ], + [ + 389, + 0.17134634383171235 + ], + [ + 394, + 0.1753323511855161 + ], + [ + 404, + 0.13141598686532005 + ], + [ + 407, + 0.14427500797827436 + ], + [ + 411, + 0.15026324111919395 + ], + [ + 416, + 0.0990182302304132 + ], + [ + 418, + 0.16368798796052975 + ], + [ + 419, + 0.37155229826661373 + ], + [ + 427, + 0.06950045962585341 + ], + [ + 434, + 0.09633972762894269 + ], + [ + 441, + 0.04665989713440794 + ], + [ + 442, + 0.28998290324724213 + ], + [ + 445, + 0.1815095880710452 + ], + [ + 449, + 0.18527001835322188 + ], + [ + 454, + 0.16841912496526684 + ], + [ + 458, + 0.08454075183052098 + ], + [ + 467, + 0.11680437769246632 + ], + [ + 473, + 0.08832870596412609 + ], + [ + 480, + 0.0649056188317066 + ], + [ + 486, + 0.052969664105542075 + ], + [ + 490, + 0.19176745334761097 + ], + [ + 497, + 0.0931096028556648 + ], + [ + 504, + 0.07666981887194894 + ], + [ + 505, + 0.4121905797227717 + ], + [ + 511, + 0.13300925623565676 + ], + [ + 518, + 0.18001134927974077 + ], + [ + 521, + 0.11780346132424308 + ], + [ + 527, + 0.24404820304364344 + ], + [ + 532, + 0.1370429102217899 + ], + [ + 535, + 0.2448675309351973 + ], + [ + 541, + 0.10924970234733601 + ], + [ + 546, + 0.16370169485431993 + ], + [ + 547, + 0.46354882626430544 + ], + [ + 555, + 0.18666177913567514 + ], + [ + 556, + 0.5231299890042629 + ], + [ + 558, + 0.5090713376088505 + ], + [ + 559, + 0.20145013659443603 + ], + [ + 565, + 0.38381339895797434 + ], + [ + 569, + 0.36468891138552767 + ], + [ + 575, + 0.1745025783579914 + ], + [ + 578, + 0.3947111049119814 + ], + [ + 582, + 0.12533975384890272 + ], + [ + 586, + 0.14086238230127318 + ], + [ + 591, + 0.2143565307150986 + ], + [ + 593, + 0.34811575922040405 + ], + [ + 597, + 0.3378818519348871 + ], + [ + 598, + 0.6105229952204546 + ], + [ + 599, + 0.3309534630019935 + ], + [ + 600, + 0.26313305058584024 + ], + [ + 606, + 0.15624613236853047 + ], + [ + 610, + 0.14924428275689794 + ], + [ + 612, + 0.9095976319034667 + ], + [ + 618, + 0.2581660864741236 + ], + [ + 625, + 0.25480868442242893 + ], + [ + 629, + 1.0 + ], + [ + 634, + 0.2528254967880108 + ], + [ + 639, + 0.29124835413415334 + ], + [ + 644, + 0.37441035279040014 + ], + [ + 646, + 0.2849469348411808 + ] + ] + ], + [ + [ + [ + 40, + 0.8447524902095843 + ], + [ + 50, + 0.7301799675270848 + ], + [ + 53, + 0.5688742462943729 + ], + [ + 62, + 0.36987629318182746 + ], + [ + 70, + 0.38483216146611904 + ], + [ + 79, + 0.2818132225536531 + ], + [ + 83, + 0.2754312127905414 + ], + [ + 90, + 0.6792912753017426 + ], + [ + 91, + 0.39573440889042855 + ], + [ + 95, + 0.23565296304876182 + ], + [ + 102, + 0.16963323698317853 + ], + [ + 107, + 0.23746076876387315 + ], + [ + 111, + 0.2300789477466297 + ], + [ + 113, + 0.3284862582730546 + ], + [ + 115, + 0.2811601664899884 + ], + [ + 117, + 0.31541364491637935 + ], + [ + 118, + 1.0 + ], + [ + 124, + 0.7723837565870827 + ], + [ + 134, + 0.69097853302027 + ], + [ + 141, + 0.0734339078069312 + ], + [ + 144, + 0.10724187009826645 + ], + [ + 148, + 0.04884218693036476 + ], + [ + 152, + 0.03251631763239498 + ], + [ + 156, + 0.08825791740055516 + ], + [ + 160, + 0.026447678694014532 + ], + [ + 168, + 0.14878809907142193 + ], + [ + 173, + 0.08814736410437383 + ], + [ + 175, + 0.1321702189374275 + ], + [ + 182, + 0.05385786714111592 + ], + [ + 186, + 0.028063067079550016 + ], + [ + 191, + 0.10874978976106962 + ], + [ + 192, + 0.17627598874261718 + ], + [ + 198, + 0.1183074361327125 + ], + [ + 200, + 0.24830290795159224 + ], + [ + 212, + 0.1155129763705705 + ], + [ + 215, + 0.28917975718920236 + ], + [ + 217, + 0.2677590870066588 + ], + [ + 220, + 0.19683218674324307 + ], + [ + 226, + 0.22003246472609417 + ], + [ + 229, + 0.37065655377887613 + ], + [ + 233, + 0.09752542278821992 + ], + [ + 237, + 0.38043547052554266 + ], + [ + 238, + 0.21014676168444954 + ], + [ + 242, + 0.17910287747163903 + ], + [ + 246, + 0.29521932470525364 + ], + [ + 249, + 0.28001065788369406 + ], + [ + 250, + 0.1290979837822136 + ], + [ + 257, + 0.1816048759954386 + ], + [ + 260, + 0.14460637287342035 + ], + [ + 269, + 0.05908307996951841 + ], + [ + 270, + 0.48198257655484783 + ], + [ + 274, + 0.2509897898025612 + ], + [ + 279, + 0.1868345382527985 + ], + [ + 280, + 0.14881826237815782 + ], + [ + 281, + 0.10559839298636786 + ], + [ + 288, + 0.23208007159595243 + ], + [ + 292, + 0.1498505298962796 + ], + [ + 298, + 0.30933266768420287 + ], + [ + 305, + 0.13000280109295992 + ], + [ + 313, + 0.19542201803195233 + ], + [ + 314, + 0.20451648703826067 + ], + [ + 319, + 0.14216322691914346 + ], + [ + 328, + 0.1246572353058257 + ], + [ + 336, + 0.422046257164246 + ], + [ + 343, + 0.07047004217908517 + ], + [ + 351, + 0.13455145504584284 + ], + [ + 356, + 0.07837378394274519 + ], + [ + 360, + 0.08182741432122959 + ], + [ + 366, + 0.10979783500886862 + ], + [ + 367, + 0.26371605742542203 + ], + [ + 372, + 0.08241348327654147 + ], + [ + 380, + 0.1614669516139205 + ], + [ + 385, + 0.044501045608980716 + ], + [ + 389, + 0.15184193597438242 + ], + [ + 394, + 0.09472562644104003 + ], + [ + 404, + 0.24770777635570448 + ], + [ + 407, + 0.13171806960460095 + ], + [ + 411, + 0.11166089007496045 + ], + [ + 416, + 0.07771039592288179 + ], + [ + 418, + 0.07238643579844573 + ], + [ + 419, + 0.23754427062387778 + ], + [ + 427, + 0.17446376089565857 + ], + [ + 434, + 0.11371835515967042 + ], + [ + 441, + 0.16703193157258692 + ], + [ + 442, + 0.19489753135717997 + ], + [ + 445, + 0.08085215681769416 + ], + [ + 449, + 0.12742781009657048 + ], + [ + 454, + 0.029657109124933946 + ], + [ + 458, + 0.06501930062647278 + ], + [ + 467, + 0.033744783129850894 + ], + [ + 473, + 0.030413976098568444 + ], + [ + 480, + 0.018619372494304422 + ], + [ + 486, + 0.0290149855538916 + ], + [ + 490, + 0.1536959283998984 + ], + [ + 497, + 0.0 + ], + [ + 504, + 0.00011276436210495605 + ], + [ + 505, + 0.7357284327340995 + ], + [ + 511, + 0.4984202957617319 + ], + [ + 518, + 0.3820438708508751 + ], + [ + 521, + 0.12263438295680905 + ], + [ + 527, + 0.3208447598467808 + ], + [ + 532, + 0.24419646718343463 + ], + [ + 535, + 0.3332354913914537 + ], + [ + 541, + 0.2106007126265472 + ], + [ + 546, + 0.31309759430704576 + ], + [ + 547, + 0.2270951279312507 + ], + [ + 555, + 0.32009048609861224 + ], + [ + 556, + 0.37434745063894276 + ], + [ + 558, + 0.5215903012494065 + ], + [ + 559, + 0.17091333496451766 + ], + [ + 565, + 0.48853743161954477 + ], + [ + 569, + 0.48158534900767974 + ], + [ + 575, + 0.29436230463441193 + ], + [ + 578, + 0.3212821687401165 + ], + [ + 582, + 0.16763781822988094 + ], + [ + 586, + 0.4235899087442594 + ], + [ + 591, + 0.601589505265674 + ], + [ + 593, + 0.1334378694365299 + ], + [ + 597, + 0.33918942787290723 + ], + [ + 598, + 0.3588104814733899 + ], + [ + 599, + 0.555094719675671 + ], + [ + 600, + 0.8052094596661244 + ], + [ + 606, + 0.15808456669297516 + ], + [ + 610, + 0.2639499390653434 + ], + [ + 612, + 0.7490847449567531 + ], + [ + 618, + 0.14467603509856966 + ], + [ + 625, + 0.2158196198225008 + ], + [ + 629, + 0.07645111198804594 + ], + [ + 634, + 0.22779647258280303 + ], + [ + 639, + 0.25705381552932327 + ], + [ + 644, + 0.28026525803023317 + ], + [ + 646, + 0.26677057675613675 + ] + ], + [ + [ + 67, + 0.44644755297497596 + ], + [ + 73, + 0.20707004047655658 + ], + [ + 76, + 0.19466984149951144 + ], + [ + 77, + 0.21587081920227552 + ], + [ + 81, + 0.14659129751967687 + ], + [ + 87, + 0.11369916862232132 + ], + [ + 90, + 0.05730891345663202 + ], + [ + 96, + 0.07830738715278746 + ], + [ + 107, + 0.11670481414342264 + ], + [ + 113, + 0.06822542673573585 + ], + [ + 118, + 0.08220825995598902 + ], + [ + 122, + 0.10570783840438995 + ], + [ + 123, + 0.2850752948896786 + ], + [ + 128, + 0.051590528803150786 + ], + [ + 131, + 0.05212509798511977 + ], + [ + 132, + 0.11523085137524144 + ], + [ + 141, + 0.0724070209684745 + ], + [ + 151, + 0.1150868351727176 + ], + [ + 154, + 0.10716339019999536 + ], + [ + 158, + 0.10328262257944795 + ], + [ + 164, + 0.18975735001641192 + ], + [ + 168, + 0.06296800910323277 + ], + [ + 177, + 0.04271995362972053 + ], + [ + 178, + 0.2939726185522772 + ], + [ + 183, + 0.045294869189518236 + ], + [ + 185, + 0.07509904567024105 + ], + [ + 189, + 0.13258964437958765 + ], + [ + 190, + 0.07913001377739147 + ], + [ + 193, + 0.15703339024752505 + ], + [ + 195, + 0.18476797644837675 + ], + [ + 198, + 0.16592640744509685 + ], + [ + 206, + 0.059753925667317326 + ], + [ + 212, + 0.06823818755942958 + ], + [ + 213, + 0.08280684190298801 + ], + [ + 219, + 0.13056243438926313 + ], + [ + 224, + 0.09354656477770713 + ], + [ + 226, + 0.1230582521628313 + ], + [ + 231, + 0.06646057727754737 + ], + [ + 234, + 0.0742930690412348 + ], + [ + 235, + 0.09677057978994408 + ], + [ + 242, + 0.0748342564921306 + ], + [ + 251, + 0.07401571099426708 + ], + [ + 253, + 0.10485653539639048 + ], + [ + 254, + 0.04123492424792398 + ], + [ + 258, + 0.017994029979247636 + ], + [ + 259, + 0.035347302195579834 + ], + [ + 260, + 0.03685184420379623 + ], + [ + 264, + 0.04649193293316675 + ], + [ + 266, + 0.07928722478845669 + ], + [ + 267, + 0.03709152068177951 + ], + [ + 271, + 0.024782930063971913 + ], + [ + 275, + 0.03926931506837997 + ], + [ + 279, + 0.040924207788723556 + ], + [ + 281, + 0.11857360297262595 + ], + [ + 287, + 0.04591944015220216 + ], + [ + 297, + 0.07254428536827222 + ], + [ + 299, + 0.056282823051237045 + ], + [ + 304, + 0.12651590959449655 + ], + [ + 307, + 0.24189664014063647 + ], + [ + 308, + 0.2790421376878438 + ], + [ + 309, + 0.07663596127892046 + ], + [ + 311, + 0.07752433660794514 + ], + [ + 317, + 0.03741983021903512 + ], + [ + 322, + 0.04770719900282968 + ], + [ + 323, + 0.05986121172917661 + ], + [ + 327, + 0.035767612347608356 + ], + [ + 333, + 0.020620744134372426 + ], + [ + 334, + 0.07240869848682795 + ], + [ + 338, + 0.074983271866265 + ], + [ + 340, + 0.07556614690012364 + ], + [ + 343, + 0.021625122778590657 + ], + [ + 345, + 0.0033757344458067535 + ], + [ + 351, + 0.03724828936213462 + ], + [ + 352, + 0.20541046572737573 + ], + [ + 357, + 0.011497472937529054 + ], + [ + 362, + 0.0028129061735650707 + ], + [ + 366, + 0.11689720296473577 + ], + [ + 370, + 0.01820576450973151 + ], + [ + 371, + 0.11279120555411472 + ], + [ + 377, + 0.07277985568192763 + ], + [ + 378, + 0.30398808748314965 + ], + [ + 382, + 0.19667718833719303 + ], + [ + 387, + 0.5720207681946593 + ], + [ + 389, + 0.7649626865044145 + ], + [ + 392, + 1.0 + ], + [ + 401, + 0.2992519774571745 + ], + [ + 402, + 0.21376909735195257 + ], + [ + 408, + 0.6339628245992276 + ], + [ + 411, + 0.3678865475809741 + ], + [ + 416, + 0.47092353056371467 + ], + [ + 422, + 0.7256595498284356 + ], + [ + 423, + 0.4669024940328857 + ], + [ + 424, + 0.1622398063145655 + ], + [ + 426, + 0.08826419301646356 + ], + [ + 429, + 0.13830913068691292 + ], + [ + 435, + 0.16447714842978253 + ], + [ + 438, + 0.11573734507591173 + ], + [ + 442, + 0.04769903674934867 + ], + [ + 446, + 0.1432283489351512 + ], + [ + 447, + 0.10445679362238235 + ], + [ + 451, + 0.10945736733783216 + ], + [ + 454, + 0.06676196306252308 + ], + [ + 458, + 0.05956538428859595 + ], + [ + 460, + 0.03072033935879148 + ], + [ + 464, + 0.06766483515463212 + ], + [ + 465, + 0.057515849116204255 + ], + [ + 467, + 0.06578268459963832 + ], + [ + 468, + 0.042344314706482505 + ], + [ + 471, + 0.06041935797258103 + ], + [ + 475, + 0.09399688247819268 + ], + [ + 480, + 0.21568093413969888 + ], + [ + 481, + 0.34288952110744353 + ], + [ + 482, + 0.03705001670779069 + ], + [ + 489, + 0.01003302445254658 + ], + [ + 492, + 0.0057541425011697666 + ], + [ + 493, + 0.017443399184983967 + ], + [ + 495, + 0.02431770248021047 + ], + [ + 496, + 0.03381816075777727 + ], + [ + 500, + 0.07056715055267783 + ], + [ + 502, + 0.10037312973969092 + ], + [ + 505, + 0.09464822696763232 + ], + [ + 507, + 0.13236347150705657 + ], + [ + 509, + 0.25619443786407964 + ], + [ + 514, + 0.01733032109458089 + ], + [ + 515, + 0.03157495150125436 + ], + [ + 518, + 0.04112556841217584 + ], + [ + 523, + 0.01852511476364165 + ], + [ + 526, + 0.01905571548801327 + ], + [ + 529, + 0.05757265940193558 + ], + [ + 532, + 0.07451621236569468 + ], + [ + 536, + 0.06401456356336746 + ], + [ + 538, + 0.030876285971694766 + ], + [ + 540, + 0.0039054463358269144 + ], + [ + 545, + 0.020816212578931462 + ], + [ + 550, + 0.029915497767079104 + ], + [ + 551, + 0.030248539408346705 + ], + [ + 554, + 0.0 + ], + [ + 557, + 0.0023494061833318164 + ], + [ + 561, + 0.015688965363139348 + ], + [ + 564, + 0.03520382847411558 + ], + [ + 566, + 0.21196367031050778 + ], + [ + 568, + 0.1868799804012443 + ], + [ + 571, + 0.06932934230852389 + ], + [ + 578, + 0.12315182597267699 + ], + [ + 583, + 0.06861665573003986 + ], + [ + 585, + 0.057989677111225595 + ], + [ + 586, + 0.04813689407718887 + ], + [ + 594, + 0.046205569700610086 + ], + [ + 598, + 0.04881604698032863 + ], + [ + 603, + 0.030910374646892298 + ], + [ + 606, + 0.017569642873409837 + ], + [ + 607, + 0.05186717745180981 + ], + [ + 610, + 0.05829334131665152 + ], + [ + 616, + 0.14275782590166206 + ], + [ + 621, + 0.05429324455472222 + ], + [ + 623, + 0.054063933637908075 + ], + [ + 627, + 0.07128670577596234 + ], + [ + 633, + 0.08061052804759986 + ], + [ + 637, + 0.04922336679744591 + ], + [ + 642, + 0.035525031509490726 + ], + [ + 647, + 0.04657867982752424 + ], + [ + 652, + 0.0974341342761554 + ], + [ + 655, + 0.16641694385675315 + ], + [ + 656, + 0.3896834365547515 + ], + [ + 657, + 0.06010666354397877 + ], + [ + 662, + 0.020177720717672563 + ], + [ + 666, + 0.04488521253091354 + ], + [ + 670, + 0.07526239923607317 + ], + [ + 682, + 0.08061723812101369 + ], + [ + 687, + 0.06688559866893175 + ], + [ + 693, + 0.05437041039898134 + ], + [ + 697, + 0.08254563309987077 + ], + [ + 703, + 0.02211061828932412 + ], + [ + 704, + 0.028179249579523974 + ], + [ + 708, + 0.029066802841096827 + ], + [ + 715, + 0.028771931001767264 + ], + [ + 718, + 0.05530260985175733 + ], + [ + 721, + 0.055955281333327306 + ], + [ + 725, + 0.05592357540186069 + ], + [ + 731, + 0.019543097163661566 + ], + [ + 732, + 0.21112567895312406 + ], + [ + 734, + 0.07709246326348518 + ], + [ + 735, + 0.038146204011945385 + ], + [ + 739, + 0.06750729865493114 + ], + [ + 740, + 0.049905490827619664 + ], + [ + 749, + 0.06672083465234473 + ], + [ + 750, + 0.10931535414229752 + ], + [ + 751, + 0.06253394079694716 + ], + [ + 755, + 0.03863818425780639 + ], + [ + 760, + 0.08453087174790898 + ], + [ + 766, + 0.08872630342059805 + ], + [ + 771, + 0.09947145110591295 + ] + ], + [ + [ + 37, + 0.36155376013667856 + ], + [ + 45, + 0.2888163814820227 + ], + [ + 46, + 0.1263143263321095 + ], + [ + 57, + 0.05985704325992295 + ], + [ + 58, + 0.4242285501277373 + ], + [ + 60, + 0.23060372032812806 + ], + [ + 63, + 0.07375337157233827 + ], + [ + 69, + 0.10055457140899665 + ], + [ + 72, + 0.09517482838228859 + ], + [ + 76, + 0.031408255785000776 + ], + [ + 80, + 0.07814588058398253 + ], + [ + 81, + 0.13848014964025732 + ], + [ + 83, + 0.13610009963402436 + ], + [ + 88, + 0.04817013415823575 + ], + [ + 91, + 0.08000831928250443 + ], + [ + 96, + 0.1310593390194628 + ], + [ + 97, + 0.09883602859808767 + ], + [ + 100, + 0.2107620840376663 + ], + [ + 101, + 0.1342679754126519 + ], + [ + 111, + 0.31121911568006927 + ], + [ + 113, + 0.14419509975216063 + ], + [ + 116, + 0.06474381248295007 + ], + [ + 122, + 0.09009472047127938 + ], + [ + 125, + 0.09103796493601472 + ], + [ + 129, + 0.035595212226840985 + ], + [ + 133, + 0.11986510309356967 + ], + [ + 134, + 0.16883756763808475 + ], + [ + 139, + 0.11726116221643419 + ], + [ + 140, + 0.06296155753406536 + ], + [ + 143, + 0.12032247696236942 + ], + [ + 148, + 0.09824549754944752 + ], + [ + 157, + 0.06994486369197774 + ], + [ + 160, + 0.045903225107655286 + ], + [ + 169, + 0.06844823320099495 + ], + [ + 170, + 0.2303575969762903 + ], + [ + 171, + 0.009670255288552277 + ], + [ + 173, + 0.01864621746374731 + ], + [ + 179, + 0.014583004354081077 + ], + [ + 181, + 0.015962047393224307 + ], + [ + 182, + 0.022003161284005337 + ], + [ + 185, + 0.028129768851867038 + ], + [ + 186, + 0.01919932383614649 + ], + [ + 187, + 0.0 + ], + [ + 188, + 0.013970156928351206 + ], + [ + 189, + 0.02355983345620566 + ], + [ + 192, + 0.009748750628080696 + ], + [ + 196, + 0.005664322278352206 + ], + [ + 197, + 0.03172882998258676 + ], + [ + 204, + 0.03491735227274106 + ], + [ + 209, + 0.054320575225313875 + ], + [ + 210, + 0.06373215769558853 + ], + [ + 211, + 0.003256422244507449 + ], + [ + 212, + 0.07144769550704527 + ], + [ + 214, + 0.0065678978447144334 + ], + [ + 221, + 0.02276993018780759 + ], + [ + 224, + 0.10982829851478043 + ], + [ + 233, + 0.03174116970864044 + ], + [ + 241, + 0.06832453531257132 + ], + [ + 247, + 0.05858392610714758 + ], + [ + 256, + 0.05857696971767255 + ], + [ + 260, + 0.05517365928937601 + ], + [ + 267, + 0.11509528510987735 + ], + [ + 270, + 0.17491570420269992 + ], + [ + 275, + 0.18154576560220542 + ], + [ + 276, + 0.24308458767901286 + ], + [ + 277, + 0.17005189456061345 + ], + [ + 278, + 0.2593486961850695 + ], + [ + 279, + 0.4091031361609523 + ], + [ + 280, + 0.4230338688880476 + ], + [ + 281, + 0.733670539011364 + ], + [ + 282, + 0.07161167054096194 + ], + [ + 283, + 0.09916894252172732 + ], + [ + 284, + 0.09658208849471943 + ], + [ + 285, + 0.6293627298094387 + ], + [ + 287, + 0.11741327992718541 + ], + [ + 292, + 0.19587745203421764 + ], + [ + 295, + 0.19354698465527798 + ], + [ + 299, + 0.05124128572828289 + ], + [ + 303, + 0.12477751657448058 + ], + [ + 305, + 0.1614553352656552 + ], + [ + 310, + 0.13658019537799376 + ], + [ + 313, + 0.16721666596828944 + ], + [ + 317, + 0.0489483409044311 + ], + [ + 321, + 0.06688192695372269 + ], + [ + 324, + 0.31991637832577946 + ], + [ + 328, + 0.136976884361725 + ], + [ + 335, + 0.10261233433782316 + ], + [ + 342, + 0.1232294507494599 + ], + [ + 348, + 0.21005634958624286 + ], + [ + 351, + 0.21811278543870657 + ], + [ + 357, + 0.28978403972770955 + ], + [ + 359, + 0.2751141259489411 + ], + [ + 363, + 0.30402913134561366 + ], + [ + 364, + 0.32945291013544864 + ], + [ + 367, + 0.36763734995133446 + ], + [ + 369, + 0.5031045544743831 + ], + [ + 370, + 0.5318011222501938 + ], + [ + 379, + 0.20617380334956026 + ], + [ + 382, + 0.18501067678192348 + ], + [ + 390, + 0.26809157037342796 + ], + [ + 391, + 0.1635387704177503 + ], + [ + 392, + 0.18247299988412893 + ], + [ + 393, + 0.24831858910270962 + ], + [ + 394, + 0.8185902265685215 + ], + [ + 395, + 0.375562096806661 + ], + [ + 396, + 1.0 + ], + [ + 397, + 0.006671911597894932 + ], + [ + 398, + 0.010703551788094645 + ], + [ + 399, + 0.014355299703823154 + ], + [ + 400, + 0.21036237479237405 + ], + [ + 404, + 0.045062438821569586 + ], + [ + 410, + 0.048151607090789715 + ], + [ + 411, + 0.1324765128608138 + ], + [ + 412, + 0.013794845426561415 + ], + [ + 418, + 0.047281918579488214 + ], + [ + 421, + 0.037081083036012645 + ], + [ + 423, + 0.04249311909084646 + ], + [ + 430, + 0.04382060794306628 + ], + [ + 443, + 0.08336561966141896 + ], + [ + 446, + 0.04813875000511679 + ], + [ + 450, + 0.05782767917938542 + ], + [ + 456, + 0.05374733377584127 + ], + [ + 459, + 0.1615259338796439 + ], + [ + 463, + 0.15664338742972522 + ], + [ + 466, + 0.0820831690614585 + ], + [ + 467, + 0.10816592417929086 + ], + [ + 468, + 0.18746040254444593 + ], + [ + 469, + 0.0380143927977892 + ], + [ + 475, + 0.03738374540461872 + ], + [ + 481, + 0.029190537846326078 + ], + [ + 485, + 0.0664643618102202 + ], + [ + 486, + 0.11456033526357527 + ], + [ + 488, + 0.09230479686855086 + ], + [ + 491, + 0.1271451499498516 + ], + [ + 497, + 0.11689199117136996 + ], + [ + 500, + 0.11596154087019168 + ], + [ + 504, + 0.032180484930189955 + ], + [ + 508, + 0.11035016056098478 + ], + [ + 510, + 0.14315104007513405 + ], + [ + 516, + 0.04304712734690622 + ], + [ + 527, + 0.12002005929088055 + ], + [ + 531, + 0.05549558980812597 + ], + [ + 540, + 0.03721677807452586 + ], + [ + 543, + 0.0274763925921807 + ], + [ + 545, + 0.04791101389380094 + ], + [ + 548, + 0.06637581641052058 + ], + [ + 550, + 0.07538301941623735 + ], + [ + 555, + 0.10733944917888893 + ], + [ + 558, + 0.08659216788358796 + ], + [ + 562, + 0.02201055113694513 + ], + [ + 566, + 0.10780187079964945 + ], + [ + 568, + 0.11565009037940459 + ], + [ + 572, + 0.03240610266360538 + ], + [ + 577, + 0.06766441240379614 + ], + [ + 581, + 0.036749735173923145 + ], + [ + 589, + 0.04285899022050143 + ], + [ + 594, + 0.009445315715718845 + ], + [ + 603, + 0.01950770512594912 + ], + [ + 606, + 0.0295560034779011 + ], + [ + 609, + 0.034288809274778606 + ], + [ + 612, + 0.013542726995351967 + ], + [ + 616, + 0.050320287727175186 + ], + [ + 618, + 0.008155108217142782 + ], + [ + 626, + 0.04599352533525262 + ], + [ + 629, + 0.06530142128230595 + ], + [ + 633, + 0.12802365454871895 + ] + ], + [ + [ + 40, + 0.619983295486363 + ], + [ + 49, + 0.33547405921125756 + ], + [ + 53, + 0.18763031385852622 + ], + [ + 61, + 0.1498985222288151 + ], + [ + 65, + 0.1345612920650741 + ], + [ + 69, + 0.08793296374624465 + ], + [ + 75, + 0.09527413499974972 + ], + [ + 80, + 0.1952484918363269 + ], + [ + 81, + 0.3665114114571533 + ], + [ + 83, + 0.23971033916026033 + ], + [ + 84, + 0.03459760271951297 + ], + [ + 87, + 0.09991980123407267 + ], + [ + 92, + 0.1801691956886375 + ], + [ + 93, + 0.08780209990865809 + ], + [ + 94, + 0.11719924565118614 + ], + [ + 96, + 0.3632392586500953 + ], + [ + 100, + 0.1825856057307966 + ], + [ + 105, + 0.18174457354622212 + ], + [ + 110, + 0.47687013864553734 + ], + [ + 118, + 0.38440434972137805 + ], + [ + 122, + 0.12666335783022004 + ], + [ + 127, + 0.07283755324834179 + ], + [ + 130, + 0.07308574208499866 + ], + [ + 134, + 0.12751787084629057 + ], + [ + 142, + 0.15727201499397941 + ], + [ + 144, + 0.0783631163707778 + ], + [ + 145, + 0.05347913142631426 + ], + [ + 147, + 0.11707426372544855 + ], + [ + 151, + 0.08412933901864779 + ], + [ + 154, + 0.10237748907181093 + ], + [ + 155, + 0.10306467503739292 + ], + [ + 158, + 0.08804264342005168 + ], + [ + 160, + 0.12480728400185467 + ], + [ + 163, + 0.1563417216067107 + ], + [ + 169, + 0.0687648223526015 + ], + [ + 173, + 0.10927378128454192 + ], + [ + 174, + 0.11768173644386234 + ], + [ + 177, + 0.07706393313920588 + ], + [ + 186, + 0.10247405915905122 + ], + [ + 187, + 0.20666083939743074 + ], + [ + 188, + 0.3270677282485029 + ], + [ + 189, + 0.06156457335398685 + ], + [ + 190, + 0.07739047785875809 + ], + [ + 191, + 0.10429018935330218 + ], + [ + 192, + 1.0 + ], + [ + 198, + 0.6530544434691338 + ], + [ + 204, + 0.7495268004817854 + ], + [ + 211, + 0.5647444630950549 + ], + [ + 215, + 0.33924934170878135 + ], + [ + 221, + 0.17539083259488947 + ], + [ + 224, + 0.19392073434660112 + ], + [ + 228, + 0.1273964853558058 + ], + [ + 232, + 0.21596305398306742 + ], + [ + 234, + 0.41338335765944734 + ], + [ + 237, + 0.2003942713960649 + ], + [ + 243, + 0.12636820651000374 + ], + [ + 246, + 0.09534624932727083 + ], + [ + 250, + 0.06137315018730438 + ], + [ + 254, + 0.16407143548296502 + ], + [ + 256, + 0.3873899014429983 + ], + [ + 267, + 0.4576677731099722 + ], + [ + 269, + 0.29340625442249146 + ], + [ + 272, + 0.14264574207270117 + ], + [ + 274, + 0.15687587345158308 + ], + [ + 279, + 0.2176672538311273 + ], + [ + 282, + 0.1246729397430557 + ], + [ + 286, + 0.049239908781176386 + ], + [ + 290, + 0.18270496097557412 + ], + [ + 292, + 0.1929881670841403 + ], + [ + 295, + 0.2661164573184606 + ], + [ + 297, + 0.20124820434193336 + ], + [ + 302, + 0.2429790811545302 + ], + [ + 305, + 0.24915891705928284 + ], + [ + 309, + 0.24296061171929634 + ], + [ + 318, + 0.222674118178874 + ], + [ + 324, + 0.20750119870057082 + ], + [ + 327, + 0.3303683044954623 + ], + [ + 328, + 0.35180008864284784 + ], + [ + 334, + 0.2990196633763287 + ], + [ + 336, + 0.2607776768010906 + ], + [ + 340, + 0.2323409648935307 + ], + [ + 341, + 0.25239981568385345 + ], + [ + 344, + 0.32302336278564375 + ], + [ + 346, + 0.42561569245819464 + ], + [ + 347, + 0.46661227000339756 + ], + [ + 351, + 0.18940860027279424 + ], + [ + 356, + 0.30057831200928564 + ], + [ + 360, + 0.3704458652181246 + ], + [ + 368, + 0.7135993868240313 + ], + [ + 371, + 0.4545306606459556 + ], + [ + 376, + 0.6907527882509936 + ], + [ + 382, + 0.6385997901178393 + ], + [ + 387, + 0.7487150734438194 + ], + [ + 389, + 0.05119820258055019 + ], + [ + 396, + 0.09625612424342676 + ], + [ + 399, + 0.09445117780267169 + ], + [ + 407, + 0.06523801872683559 + ], + [ + 414, + 0.11731137322124591 + ], + [ + 417, + 0.25909340816517373 + ], + [ + 418, + 0.19860129760428047 + ], + [ + 419, + 0.5427382702307735 + ], + [ + 420, + 0.08695359641988096 + ], + [ + 421, + 0.08143797570070842 + ], + [ + 422, + 0.09142818835020998 + ], + [ + 426, + 0.28963285135242234 + ], + [ + 427, + 0.13005146667067147 + ], + [ + 428, + 0.2827831735811296 + ], + [ + 429, + 0.0 + ], + [ + 430, + 0.00511971120485703 + ], + [ + 431, + 0.03142361519275092 + ], + [ + 432, + 0.1396159193932591 + ], + [ + 434, + 0.041719385621893874 + ], + [ + 441, + 0.0617985968763092 + ], + [ + 445, + 0.05690895311499642 + ], + [ + 450, + 0.09172199390755748 + ], + [ + 454, + 0.06475081776411859 + ], + [ + 457, + 0.059810997930326924 + ], + [ + 461, + 0.03429115163176039 + ], + [ + 464, + 0.04287792503227235 + ], + [ + 465, + 0.05865198286238275 + ], + [ + 473, + 0.08487655064873978 + ], + [ + 475, + 0.02477344096605541 + ], + [ + 482, + 0.02156866331296759 + ], + [ + 487, + 0.025789364316553428 + ], + [ + 492, + 0.012707029447907687 + ], + [ + 501, + 0.018267640411952633 + ], + [ + 509, + 0.03147022963418911 + ], + [ + 510, + 0.1560124041515926 + ], + [ + 514, + 0.039530160074374744 + ], + [ + 518, + 0.03864859838478176 + ], + [ + 523, + 0.051659393195406446 + ], + [ + 527, + 0.03902437366238349 + ], + [ + 536, + 0.049346444474488216 + ], + [ + 539, + 0.049482905989524734 + ], + [ + 545, + 0.07494328929167726 + ], + [ + 546, + 0.05777763144540147 + ], + [ + 547, + 0.13561114952239312 + ], + [ + 548, + 0.038142794570684445 + ], + [ + 549, + 0.013876503181595147 + ], + [ + 550, + 0.02088538121984552 + ], + [ + 553, + 0.09000270063283977 + ], + [ + 562, + 0.08035088933782068 + ], + [ + 570, + 0.13766223135128972 + ], + [ + 573, + 0.16450593126712212 + ], + [ + 577, + 0.27683000590853707 + ], + [ + 582, + 0.03108585771619273 + ], + [ + 587, + 0.033091659265115904 + ], + [ + 591, + 0.09691605851091559 + ], + [ + 601, + 0.12509751632675742 + ] + ], + [ + [ + 55, + 0.7326064968532933 + ], + [ + 58, + 0.23004873216605912 + ], + [ + 59, + 0.38146805363868547 + ], + [ + 67, + 0.2605302206682815 + ], + [ + 68, + 0.4185288954593056 + ], + [ + 72, + 0.09028401103943177 + ], + [ + 75, + 0.07022154623754615 + ], + [ + 79, + 0.053093022533937106 + ], + [ + 84, + 0.0716980242712402 + ], + [ + 97, + 0.1528716803665475 + ], + [ + 102, + 0.09828905202664721 + ], + [ + 111, + 0.10807266448158538 + ], + [ + 114, + 0.13703786628575118 + ], + [ + 116, + 0.07500718958205514 + ], + [ + 123, + 0.0775401342727495 + ], + [ + 126, + 0.0975522920592532 + ], + [ + 131, + 0.04853986062926671 + ], + [ + 134, + 0.12256268963190296 + ], + [ + 138, + 0.16625384914161528 + ], + [ + 148, + 0.09793218542790791 + ], + [ + 152, + 0.11973200165262993 + ], + [ + 156, + 0.05913442719899635 + ], + [ + 162, + 0.21330682038907758 + ], + [ + 163, + 0.23158282545949113 + ], + [ + 167, + 0.24666006224362116 + ], + [ + 173, + 0.1682251417197635 + ], + [ + 180, + 0.26089779829181775 + ], + [ + 186, + 0.2658706912304751 + ], + [ + 188, + 0.26748469186232887 + ], + [ + 192, + 0.2082070385411383 + ], + [ + 193, + 0.1862366966147758 + ], + [ + 196, + 0.28037987635475253 + ], + [ + 198, + 0.44362656661667255 + ], + [ + 200, + 0.37275999047878833 + ], + [ + 203, + 0.408795358011992 + ], + [ + 209, + 0.2713231828715014 + ], + [ + 212, + 0.1644631746025516 + ], + [ + 216, + 0.05795007593275553 + ], + [ + 220, + 0.18680128749224817 + ], + [ + 221, + 0.2288213917957823 + ], + [ + 224, + 0.15152396663817386 + ], + [ + 228, + 0.1721851854597993 + ], + [ + 230, + 0.12734761761916016 + ], + [ + 231, + 0.07387035157111835 + ], + [ + 235, + 0.10850307753939958 + ], + [ + 236, + 0.20694849312750005 + ], + [ + 242, + 0.1089208268404593 + ], + [ + 247, + 0.10092531750527434 + ], + [ + 253, + 0.13994904162283936 + ], + [ + 254, + 0.2855880638273061 + ], + [ + 255, + 0.08342145358524826 + ], + [ + 261, + 0.11828771469526599 + ], + [ + 268, + 0.04505764653245024 + ], + [ + 270, + 0.04149176463814518 + ], + [ + 271, + 0.009862214749724967 + ], + [ + 276, + 0.023449536384215364 + ], + [ + 280, + 0.12789621929535627 + ], + [ + 286, + 0.06177634786670849 + ], + [ + 289, + 0.0 + ], + [ + 294, + 0.08247743335463584 + ], + [ + 295, + 0.09951109488646492 + ], + [ + 298, + 0.05268126676660852 + ], + [ + 302, + 0.2562028506464466 + ], + [ + 304, + 0.1713957210635309 + ], + [ + 305, + 0.09272078851471117 + ], + [ + 309, + 0.1153295191193581 + ], + [ + 310, + 0.6582288313197944 + ], + [ + 314, + 0.3976250351914683 + ], + [ + 321, + 0.7185757503925982 + ], + [ + 330, + 1.0 + ], + [ + 333, + 0.4242579016791146 + ], + [ + 338, + 0.31946022969908683 + ], + [ + 344, + 0.3684504463863086 + ], + [ + 345, + 0.7993800308042499 + ], + [ + 352, + 0.8331760594636656 + ], + [ + 358, + 0.2537630184023623 + ], + [ + 368, + 0.2922780075100331 + ], + [ + 369, + 0.38045711850406194 + ], + [ + 374, + 0.3225097396614854 + ], + [ + 380, + 0.21152641219327595 + ], + [ + 383, + 0.3082383238205158 + ], + [ + 387, + 0.22351927984946796 + ], + [ + 391, + 0.09860614862898351 + ], + [ + 392, + 0.32564977131720385 + ], + [ + 397, + 0.08515542156162352 + ], + [ + 400, + 0.1289328879570641 + ], + [ + 402, + 0.19828323539883105 + ], + [ + 404, + 0.29565074294635757 + ], + [ + 405, + 0.32251971599505075 + ], + [ + 410, + 0.18071599469309324 + ], + [ + 416, + 0.1046444598572783 + ], + [ + 419, + 0.3370736786164937 + ], + [ + 423, + 0.0235235661927847 + ], + [ + 427, + 0.08029783647836121 + ], + [ + 428, + 0.3190699539832914 + ], + [ + 433, + 0.15903436708675311 + ], + [ + 436, + 0.2591881717959828 + ], + [ + 438, + 0.08611400027900866 + ], + [ + 440, + 0.11370784289753251 + ], + [ + 441, + 0.24274640072107237 + ], + [ + 444, + 0.32761139690546315 + ], + [ + 446, + 0.20981943445267054 + ], + [ + 451, + 0.12346193236534993 + ], + [ + 457, + 0.24039241134720166 + ], + [ + 460, + 0.233778411537042 + ], + [ + 465, + 0.242800303856693 + ], + [ + 471, + 0.20228558188662077 + ], + [ + 472, + 0.4702916048466691 + ], + [ + 481, + 0.2935255132212182 + ], + [ + 484, + 0.247007223184512 + ], + [ + 490, + 0.11370065065705515 + ], + [ + 491, + 0.33325235618136384 + ], + [ + 492, + 0.4281950733210732 + ], + [ + 493, + 0.02367851837371417 + ], + [ + 494, + 0.07885341418313882 + ], + [ + 495, + 0.029092158382376953 + ], + [ + 498, + 0.25231227389575256 + ], + [ + 507, + 0.07105277203013467 + ], + [ + 515, + 0.055595313199693706 + ], + [ + 516, + 0.40896990518142634 + ], + [ + 519, + 0.0858468626804183 + ], + [ + 523, + 0.12612437605259624 + ], + [ + 532, + 0.06040738609455899 + ], + [ + 535, + 0.25395496615359664 + ], + [ + 539, + 0.22145062134915855 + ], + [ + 541, + 0.15245029627728957 + ], + [ + 544, + 0.03643466255056549 + ], + [ + 547, + 0.3154362484879257 + ], + [ + 551, + 0.1804344146116088 + ], + [ + 559, + 0.12606428604344672 + ], + [ + 562, + 0.3593155276208764 + ], + [ + 564, + 0.28329489915381645 + ], + [ + 569, + 0.11363155101332917 + ], + [ + 572, + 0.06551545255283307 + ], + [ + 574, + 0.06931265584819006 + ], + [ + 577, + 0.0543565174463985 + ], + [ + 579, + 0.12225765743294804 + ], + [ + 581, + 0.16562274937370713 + ], + [ + 582, + 0.4244501201060658 + ], + [ + 588, + 0.15905049162588783 + ], + [ + 589, + 0.2166528169313693 + ], + [ + 591, + 0.548870969533447 + ], + [ + 597, + 0.08890466692014158 + ], + [ + 606, + 0.09044949057235029 + ], + [ + 608, + 0.3623572653234043 + ], + [ + 615, + 0.15151385496675004 + ], + [ + 618, + 0.34498359642320214 + ] + ], + [ + [ + 40, + 0.5766464394616311 + ], + [ + 50, + 0.36240615151822486 + ], + [ + 53, + 0.33924956748525525 + ], + [ + 62, + 0.08312085761177831 + ], + [ + 70, + 0.09255455756166064 + ], + [ + 79, + 0.1540878028991258 + ], + [ + 83, + 0.14502534384644597 + ], + [ + 90, + 0.32195783157057817 + ], + [ + 91, + 0.2484536737287916 + ], + [ + 95, + 0.11665834333342463 + ], + [ + 102, + 0.11692062826977076 + ], + [ + 107, + 0.11746654812147575 + ], + [ + 111, + 0.10961277623015929 + ], + [ + 113, + 0.23778389586777227 + ], + [ + 115, + 0.12176211355908739 + ], + [ + 117, + 0.10820629824380877 + ], + [ + 118, + 0.894660091899667 + ], + [ + 124, + 0.17545406830364646 + ], + [ + 134, + 0.5267677952889391 + ], + [ + 141, + 0.0 + ], + [ + 144, + 0.12524791959852563 + ], + [ + 148, + 0.05480037695594013 + ], + [ + 152, + 0.061062287305275825 + ], + [ + 156, + 0.07708101961695887 + ], + [ + 160, + 0.09069435550676218 + ], + [ + 168, + 0.037452615853617914 + ], + [ + 173, + 0.06676603339746717 + ], + [ + 175, + 0.10105161231576945 + ], + [ + 182, + 0.0794762888643277 + ], + [ + 186, + 0.05747150806805639 + ], + [ + 191, + 0.07215126208431762 + ], + [ + 192, + 0.26221428301751304 + ], + [ + 198, + 0.08009850305851034 + ], + [ + 200, + 0.09217733511694 + ], + [ + 212, + 0.2621622553987538 + ], + [ + 215, + 0.2588313625565489 + ], + [ + 217, + 0.36864312888236117 + ], + [ + 220, + 0.5976538712383062 + ], + [ + 226, + 0.14819941354183863 + ], + [ + 229, + 0.23455590340985222 + ], + [ + 233, + 0.04069853814828168 + ], + [ + 237, + 0.127934300695651 + ], + [ + 238, + 0.2337591435134633 + ], + [ + 242, + 0.09425888404163098 + ], + [ + 246, + 0.14949530692815702 + ], + [ + 249, + 0.08472530409284428 + ], + [ + 250, + 0.08963958955331609 + ], + [ + 257, + 0.1421475976461722 + ], + [ + 260, + 0.3576778862632429 + ], + [ + 269, + 0.024794787762405235 + ], + [ + 270, + 0.3315078755808505 + ], + [ + 274, + 0.07486997907198659 + ], + [ + 279, + 0.22285830617083982 + ], + [ + 280, + 0.16293356410656926 + ], + [ + 281, + 0.23412893344121835 + ], + [ + 288, + 0.18717508833516802 + ], + [ + 292, + 0.15188401720139086 + ], + [ + 298, + 0.11813400887312235 + ], + [ + 305, + 0.07257974224564172 + ], + [ + 313, + 0.13972276119485855 + ], + [ + 314, + 0.3589344850279941 + ], + [ + 319, + 0.1328248820532106 + ], + [ + 328, + 0.06455657707423855 + ], + [ + 336, + 0.17810552249335773 + ], + [ + 343, + 0.019764942949509964 + ], + [ + 351, + 0.05503892817775264 + ], + [ + 356, + 0.05925662985695366 + ], + [ + 360, + 0.2710855990542048 + ], + [ + 366, + 0.09110429879310435 + ], + [ + 367, + 0.37177479473594205 + ], + [ + 372, + 0.040617668810301294 + ], + [ + 380, + 0.08218422722181547 + ], + [ + 385, + 0.258224324318322 + ], + [ + 389, + 0.3851527626532309 + ], + [ + 394, + 0.1648279379725367 + ], + [ + 404, + 0.09635008635489084 + ], + [ + 407, + 0.08805863989381273 + ], + [ + 411, + 0.08773646545323177 + ], + [ + 416, + 0.10387511002568067 + ], + [ + 418, + 0.277645372702971 + ], + [ + 419, + 0.540557986587416 + ], + [ + 427, + 0.040678298605070544 + ], + [ + 434, + 0.045172084236179334 + ], + [ + 441, + 0.021231043935502927 + ], + [ + 442, + 0.19841876903009356 + ], + [ + 445, + 0.20324641088641648 + ], + [ + 449, + 0.22316703693531564 + ], + [ + 454, + 0.1982970652662344 + ], + [ + 458, + 0.16684206123535242 + ], + [ + 467, + 0.051156267188620846 + ], + [ + 473, + 0.005238577131981217 + ], + [ + 480, + 0.09435950433106977 + ], + [ + 486, + 0.11194808194423983 + ], + [ + 490, + 0.14407484041186805 + ], + [ + 497, + 0.035573654836577165 + ], + [ + 504, + 0.05202026027134609 + ], + [ + 505, + 0.3412275053467484 + ], + [ + 511, + 0.20312106489312795 + ], + [ + 518, + 0.3342340695029171 + ], + [ + 521, + 0.09367785961278363 + ], + [ + 527, + 0.22658495833274728 + ], + [ + 532, + 0.19538899676567065 + ], + [ + 535, + 0.3165245136106116 + ], + [ + 541, + 0.4705885087661504 + ], + [ + 546, + 0.29102129741838484 + ], + [ + 547, + 0.32417594929305954 + ], + [ + 555, + 0.1601783804071746 + ], + [ + 556, + 0.7100552626883947 + ], + [ + 558, + 0.45547361197375524 + ], + [ + 559, + 0.11679048519381917 + ], + [ + 565, + 0.3583514618147038 + ], + [ + 569, + 0.514886783667118 + ], + [ + 575, + 0.20516367453570827 + ], + [ + 578, + 0.44325556087742196 + ], + [ + 582, + 0.14647960018388817 + ], + [ + 586, + 0.1863974279345672 + ], + [ + 591, + 0.434183744552387 + ], + [ + 593, + 0.4341251727653035 + ], + [ + 597, + 0.3707163865530666 + ], + [ + 598, + 0.5727027637266522 + ], + [ + 599, + 0.5308010165669619 + ], + [ + 600, + 0.2873803116771208 + ], + [ + 606, + 0.10121933253924835 + ], + [ + 610, + 0.12141050516685778 + ], + [ + 612, + 0.7062264800443582 + ], + [ + 618, + 0.1725157367260355 + ], + [ + 625, + 0.33632769996912987 + ], + [ + 629, + 1.0 + ], + [ + 634, + 0.34842985117939385 + ], + [ + 639, + 0.3703937087149223 + ], + [ + 644, + 0.5057244150039124 + ], + [ + 646, + 0.39611936740693254 + ] + ] + ] + ] + } +] \ No newline at end of file diff --git a/index.html b/index.html new file mode 100644 index 0000000000000000000000000000000000000000..a27c56b5c8767771515e579b2b2644e226fdd0e7 --- /dev/null +++ b/index.html @@ -0,0 +1,1892 @@ + + + + + + Pipeline Graph | Sliding Toys + + + + + + + +
+ +
+
+ + +
+ CONFIGS +
+ +
+ INPUT + + +
+ +
+ +
+ + +
+ OUTPUT + +
+ +
+ + +
+
+ + +
+
+

📚 Documents

+
+
+ +
+
+

📝 Question

+
+

📖 Answer

+
+

🔍 Output

+
+
+ + +
+ + + 📤 Send to Assistant + New + +
+
+
Click to show value
+
+
+ + +
+ + +
+ +
+
+ + +
+
+

Assistant

+ +
+ 🔑 API Key: +
+ Results: +
+ Your Specific Requirement + +
+ Send +
+ + +
+
+

Response

+
+
+
+ + + + + + + \ No newline at end of file diff --git a/methods/AAR.py b/methods/AAR.py new file mode 100644 index 0000000000000000000000000000000000000000..0c1fa7e182014a80ce0d2bdf81fd8d458b733b8f --- /dev/null +++ b/methods/AAR.py @@ -0,0 +1,90 @@ +from citekit.cite_modules.LLM import LLM +from citekit.cite_modules.augment_model import EvalModule +from citekit.pipeline.pipeline import Pipeline, PIPELINE_OUTPUT,PIPELINE_DOC_CACHE +from citekit.prompt.prompt import Prompt, ALCEVanillaPrompt, DocPrompt,ALCEDocPrompt,NewALCEVanillaPrompt +from citekit.Dataset.Dataset import PromptDataset +from citekit.evaluator.evaluator import Evaluator,DefaultEvaluator +import argparse +import json +from citekit.utils.utils import cut_and_make_as,one_paragraph,make_as + +test = False +def convert_result(result): + key = 'cite_recall_precision' + if test: + key = 'test_pr' + p = result[key]['citation_prec'] + r = result[key]['citation_rec'] + text = f'citation_precision_score: {p}\ncitation_recall_score: {r}\n' + return {'score':text, 'shots':Prompt.UNABLE} + +if __name__ == '__main__': + + # SETTING ARGS + parser = argparse.ArgumentParser() + parser.add_argument("--save_path", type=str, default='res.json', help="Path to the config file") + parser.add_argument("--model", type=str, default='gpt-3.5-turbo', help="model name or path") + parser.add_argument("--shots", type=int, default=1, help="number of shots") + parser.add_argument("--ndoc", type=int, default=3, help="number of docs") + parser.add_argument("--pr", action='store_true', help="use cite PR") + parser.add_argument("--rouge", action='store_true', help="use rouge") + parser.add_argument("--temp", type=float, default=0.5, help="temperature") + parser.add_argument("--qa", action='store_true', help="eval qa") + parser.add_argument("--mauve", action='store_true', help="eval mauve") + parser.add_argument("--length", type=bool, default=True, help="eval length") + parser.add_argument("--claims", action='store_true', help="eval claims") + parser.add_argument("--qampari", type=str, default=False, help="eval qampari") + parser.add_argument("--turns", type=int, default=1, help="k") + parser.add_argument("--use_fast_pr", type=str, default=False, help="test") + parser.add_argument("--dataset", type=str, default='data/asqa_eval_gtr_top100.json', help="dataset") + parser.add_argument("--demo", type=str, default='prompts/asqa_default.json', help="demo") + args = parser.parse_args() + + # DATA LOADING + file_path = args.dataset + demo_path = args.demo + with open(file_path,'r',encoding='utf-8') as file: + dataset = json.load(file) + with open(demo_path,'r',encoding='utf-8') as file: + demo = json.load(file) + + dataset =PromptDataset(dataset,'question','answer','answers','qa_pairs','claims', docs = lambda data: ALCEDocPrompt().default_load_data(data['docs'][:args.ndoc]))[:200] + revise_instruction = '''You will be provided with a set of questions, search results, and corresponding answers. Your task is to evaluate each answer and provide feedback to enhance its quality. Following , offer specific feedback according to the reward scores for the following aspects: Citation Recall, and Citation Precision.\n\n \n\n1) Citation Recall: If the reward score is below {citation_recall_score}, provide feedback to offer citations from credible sources for each factual statement you make. If the score is above {citation_recall_score}, affirm that performance on citation recall is satisfactory. \n\n2) Citation Precision: If the reward score is below {citation_score}, provide feedback to cite properly, ensuring all factual statements refer to an appropriate search result. If the score is above {citation_precision_score}, affirm that performance on citation precision is satisfactory.''' + llm_instruction = demo['instruction'] + re_ans_instruction = '''You will be provided with a set of questions, search results, corresponding answers, the score of answer and the feedback. Your task is to refine the answer, only when the citation score is low. Please still generate an answer with accurate citation.''' + + shots = '\n'.join(NewALCEVanillaPrompt().load_data(demo['demos'][:args.shots],'question','answer',INST = lambda _: llm_instruction, docs = lambda data: ''.join(ALCEDocPrompt().default_load_data(data['docs'][:args.ndoc])))) + prompt = Prompt(template='\n\nAnswer:', + components={'INST':'{INST}\n\n', + 'question':'Question:{question}\n\n', + 'docs':'{docs}\n', + 'shots':'{shots}\n\n', + 'score':'Here is the score of the initial answer:\n{score}\n', + 'previousAnswer':'Initial Answer:\n{previousAnswer}\n\n', + 'feedback':'feedback:\n{feedback}\n', + 'answer':'Here is answer and you have to give feedback on:\n{answer}'}) + + + # PIPELINE + llm = LLM(model=args.model, prompt_maker=prompt, self_prompt={'INST':llm_instruction,'shots':shots}) + revise = LLM(model=args.model, prompt_maker=prompt, self_prompt={'INST':revise_instruction,'shots':Prompt.UNABLE}, share_model_with=llm) + re_ans = LLM(model=args.model, prompt_maker=prompt, self_prompt={'INST':re_ans_instruction}, share_model_with=llm) + auto_eval_model = EvalModule() + eval = DefaultEvaluator(args) + + if args.use_fast_pr: + auto_eval_model.set_eval('test_pr', output = 'previousAnswer', docs = PIPELINE_DOC_CACHE) + test = True + else: + auto_eval_model.set_eval('cite_recall_precision', output = 'previousAnswer', docs = PIPELINE_DOC_CACHE, question = 'question') + pipeline = Pipeline(save_path=args.save_path, llm = llm, module = [auto_eval_model,revise,re_ans],head_prompt_maker=prompt,evaluator = eval,dataset = dataset) + llm.set_target(auto_eval_model,lambda self: self.turns < args.turns + 1 , post_processing=cut_and_make_as('previousAnswer')) + re_ans.set_output(lambda self: True, post_processing=one_paragraph, end=True) + auto_eval_model.set_target(revise, lambda self: self.turns < args.turns + 1, post_processing = convert_result) + revise.set_target(re_ans, post_processing=make_as('feedback')) + llm.add_to_head('previousAnswer', sub = True) + + pipeline.run_on_dataset(datakeys=['question','docs'],init_docs='docs') + + + diff --git a/methods/ALCE_Vani_Summ_VTG.py b/methods/ALCE_Vani_Summ_VTG.py new file mode 100644 index 0000000000000000000000000000000000000000..42d24517cbf91b7da36dab0f47c6034eb60dc17b --- /dev/null +++ b/methods/ALCE_Vani_Summ_VTG.py @@ -0,0 +1,95 @@ +from citekit.cite_modules.LLM import LLM +from citekit.cite_modules.augment_model import Retriever,CitationSimplyfier,Verifier +from citekit.pipeline.pipeline import Pipeline, PIPELINE_OUTPUT, PIPELINE_DOC_CACHE +from citekit.prompt.prompt import Prompt, ALCEDocPrompt,DocPrompt,NewALCEVanillaPrompt +from citekit.Dataset.Dataset import PromptDataset +from citekit.evaluator.evaluator import DefaultEvaluator +from citekit.utils.utils import sentence, one_paragraph, each_make_as, each_make_as, three_sentences +import json +import argparse +from parser import * + + +if __name__ == '__main__': + + # SETTING ARGS + parser = argparse.ArgumentParser() + parser.add_argument("--save_path", type=str, default='res.jsonl', help="Path to the config file") + parser.add_argument("--model", type=str, default='gpt-3.5-turbo', help="model name or path") + parser.add_argument("--shots", type=int, default=2, help="number of shots") + parser.add_argument("--ndoc", type=int, default=5, help="number of docs") + parser.add_argument("--pr", action='store_true', help="use cite PR") + parser.add_argument("--rouge", action='store_true', help="use rouge") + parser.add_argument("--temp", type=float, default=0.5, help="temperature") + parser.add_argument("--qa", action='store_true', help="eval qa") + parser.add_argument("--mauve", action='store_true', help="eval mauve") + parser.add_argument("--length", default=True, help="eval length") + parser.add_argument("--claims", action='store_true', help="eval length") + parser.add_argument("--qampari", action='store_true', help="eval qampari") + parser.add_argument("--dataset", type=str, default='data/asqa_eval_gtr_top100.json', help="dataset") + parser.add_argument("--demo", type=str, default='prompts/asqa_default.json', help="demo") + parser.add_argument("--doctype", type=str, default='text', help="demo") + parser.add_argument("--mode", type=str, default='VTG', help="mode: text, summary, extraction or VTG") + parser.add_argument("--data_num", type=int, default=1000, help="num of data") + args = parser.parse_args() + + # DATA LOADING + file_path = args.dataset + demo_path = args.demo + + + with open(file_path,'r',encoding='utf-8') as file: + dataset = json.load(file) + with open(demo_path,'r',encoding='utf-8') as file: + demo = json.load(file) + data_num = min(args.data_num,len(dataset)) + + llm_instruction = demo['instruction'] + shots = '\n\n'.join(NewALCEVanillaPrompt().load_data([demo['demos'][1],demo['demos'][3]],'question','answer', INST = lambda _: llm_instruction, docs = lambda data: ''.join(ALCEDocPrompt().default_load_data(data['docs'][:args.ndoc])))) + documents = [DocPrompt().load_data(list(enumerate(data['docs'])),Title = lambda data: data[1]['title'], Passage = lambda data: data[1]['text']) for data in dataset] + + if args.doctype == 'text': + dataset = PromptDataset(dataset,'question','answer','answers','qa_pairs','claims', docs = lambda data: ALCEDocPrompt().default_load_data(data['docs'][:args.ndoc]))[:data_num] + elif args.doctype == 'summary': + dataset = PromptDataset(dataset,'question','answer','answers','qa_pairs','claims', docs = lambda data: ALCEDocPrompt().default_load_data_summary(data['docs'][:args.ndoc]))[:data_num] + elif args.doctype == 'extraction': + dataset = PromptDataset(dataset,'question','answer','answers','qa_pairs','claims', docs = lambda data: ALCEDocPrompt().default_load_data_extraction(data['docs'][:args.ndoc]))[:data_num] + + prompt = Prompt(template='\nAnswer: \n', components= {'INST':'{INST}\n\n','shots':'{shots}\n','question':'Question:{question}\n\n', 'docs':'{docs}\n'}) + queryprompt = Prompt(template='',components={'q':'Given the original question: {q}\n','answer':'The claim is: {answer}\n','qg_num':'Please generate up to {qg_num} questions that can help verify the claim with the following constraints: \n1. You should output no more than {qg_num} questions. \n2. The generated questions should be diverse and focus on different aspects of the given claim. \nGenerated questions:'}) + retriever_prompt = Prompt(template='',components={'query':'{query}'}) + eval = DefaultEvaluator(args) + + # PIPELINE CONSTRUCTING + llm = LLM(model=args.model,prompt_maker=prompt, self_prompt={'INST':llm_instruction, 'shots':shots}, max_turn = 3) + regen_llm = LLM(model=args.model,prompt_maker=prompt, self_prompt={'INST':llm_instruction, 'shots':shots}, max_turn = 3,share_model_with=llm) + simplifier = CitationSimplyfier() + verifier = Verifier() + query_generator = LLM(model=args.model,prompt_maker=queryprompt, self_prompt={'qg_num':'2'}) + + + + pipeline = Pipeline(save_path=args.save_path , llm = llm, module = [simplifier,verifier,query_generator],head_prompt_maker=prompt, evaluator=eval,dataset = dataset, train_data=True) + retriever = Retriever(prompt_maker=retriever_prompt,pipeline=pipeline, retrieve_by='bm25',documents=documents, topk=1, merge = True) + if args.mode == 'vanilla': + llm.set_output(post_processing = one_paragraph, cond = lambda self: True, end=True) + elif args.mode == 'simplify': + llm.set_target(simplifier, post_processing = each_make_as('answer')) + simplifier.set_output() + elif args.mode == 'VTG': + llm.set_target(verifier, post_processing = three_sentences('answer')) + verifier.set_target(simplifier, condition = lambda self: self.last_message or self.turns == 3) + verifier.set_target(query_generator, condition = lambda self: not self.last_message) + query_generator.set_target(retriever,post_processing=each_make_as('query')) + retriever.set_target(regen_llm, post_processing = lambda i,o: {'docs': o}) + regen_llm.set_target(verifier, post_processing = sentence('answer')) + simplifier.set_output() + + graph = PipelineGraph(pipeline=pipeline) + #html = graph.generate_html_embed(results='result_.json') + #graph.visualize() + #print(html) + #with open('pipeline_.html','w') as file: + # file.write(html) + # RUN PIPELINE + pipeline.run_on_dataset(datakeys=['question','docs'], init_docs='docs') \ No newline at end of file diff --git a/methods/ALCE_interact.py b/methods/ALCE_interact.py new file mode 100644 index 0000000000000000000000000000000000000000..167bef4ee8e2bf21d2836939a280aa81088385a8 --- /dev/null +++ b/methods/ALCE_interact.py @@ -0,0 +1,101 @@ +from citekit.cite_modules.LLM import LLM +from citekit.cite_modules.augment_model import Retriever +from citekit.pipeline.pipeline import Pipeline, PIPELINE_OUTPUT,PIPELINE_DOC_CACHE +from citekit.prompt.prompt import Prompt, DocPrompt,ALCEDocPrompt,NewALCEVanillaPrompt +from citekit.Dataset.Dataset import PromptDataset +from citekit.evaluator.evaluator import DefaultEvaluator +from citekit.utils.utils import output_begin_with,output_end_with +import json +from parser import * +import argparse + +def one_paragraph(text): + paras = text.lstrip('\n').split('\n') + if not paras: + return '' + else: + return paras[0].rstrip('\n') + +def cut_and_make_as(datakey): + def f(passage): + return {datakey:one_paragraph(passage)} + return f + +def if_output(x): + return not (output_begin_with('check')(x)) + +def drop_end_and_output(x): + x = one_paragraph(x) + if x[-len('End.'):] == 'End.': + x = x[:-len('End.')] + if x[:len('output')] =='output' : + x = x[len('output'):] + if x[-len('End'):] == 'End': + x = x[:-len('End')] + return x + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("--save_path", type=str, default='result.json', help="Path to the config file") + parser.add_argument("--model", type=str, default='gpt-3.5-turbo-0301', help="model name or path") + parser.add_argument("--shots", type=int, default=2, help="number of shots") + parser.add_argument("--ndoc", type=int, default=5, help="number of docs") + parser.add_argument("--pr", action='store_true', help="use cite PR") + parser.add_argument("--rouge", action='store_true', help="use rouge") + parser.add_argument("--mauve", action='store_true', help="eval mauve") + parser.add_argument("--qa", action='store_true', help="eval qa") + parser.add_argument("--length", type=bool, default=True, help="eval length") + parser.add_argument("--temp", type=float, default=0.5, help="temperature") + parser.add_argument("--claims", action='store_true', help="eval claims") + parser.add_argument("--qampari", type=str, default=False, help="eval qampari") + parser.add_argument("--turns", type=int, default=6, help="turns") + parser.add_argument("--dataset", type=str, default='data/asqa_eval_gtr_top100.json', help="dataset") + parser.add_argument("--demo", type=str, default='prompts/asqa_default.json', help="demo") + args = parser.parse_args() + + with open('data/asqa_eval_gtr_top100.json','r',encoding='utf-8') as file: + dataset = json.load(file) + with open('prompts/asqa_interact_doc_id.json','r',encoding='utf-8') as file: + demo = json.load(file) + documents = [DocPrompt().load_data(list(enumerate(data['docs'][:10])),Title = lambda data: data[1]['title'], Passage = lambda data: data[1]['text']) for data in dataset] + + llm_instruction = demo['instruction'] + dataset = PromptDataset(dataset,'question','answer','qa_pairs', extract = lambda data: ''.join(ALCEDocPrompt().default_load_data_extraction(data['docs'][:10])), docs = lambda data: ALCEDocPrompt().default_load_data(data['docs'][:args.ndoc]))[:100] + shots = '\n'.join(NewALCEVanillaPrompt().load_data(demo['demos'][:args.shots], INST = lambda _:llm_instruction, + question = lambda data: data['question'], + docs = lambda data:''.join(ALCEDocPrompt().default_load_data_extraction(demo['demos'][0]['docs'][:args.ndoc])), + answer = lambda data: '\n'.join(data['answer']))) + + # llm + llm_prompt = Prompt(template='',components={'INST':'{INST}\n\n', + 'shots':'{shots}\n', + 'question':'Question:{question}\n\n', + 'extract':'{extract}\n', + 'docs':'{docs}', + 'record':'Answer:\n{record}', + 'forceAnswer':'\n{forceAnswer}'}) + retriever_prompt = Prompt(template='',components={'IDs':'{IDs}'}) + + llm = LLM(model=args.model, prompt_maker=llm_prompt, self_prompt={'INST':llm_instruction, 'shots': shots+'\n','forceAnswer': 'Answer: \n'},stop=['\n\n'],max_turn=args.turns) + + eval = DefaultEvaluator(args) + + + pipeline = Pipeline(llm = llm, head_prompt_maker=llm_prompt,evaluator = eval,dataset = dataset,save_path=args.save_path) + retriever = Retriever(prompt_maker=retriever_prompt,pipeline=pipeline,topk=3, documents=documents) + + llm.set_target(retriever, output_begin_with('check'), post_processing=cut_and_make_as('IDs')) + llm.set_target(llm,if_output, post_processing=lambda x: {'forceAnswer': Prompt.UNABLE}) + llm.add_to_head('record') + llm.set_output(if_output,post_processing= drop_end_and_output, end=False) + llm.set_output(output_end_with('End.'), post_processing=drop_end_and_output , end = True) + llm.set_output(output_end_with('End'), post_processing=drop_end_and_output , end = True) + + retriever.set_target(llm ,post_processing=lambda input, output: {'docs': output, 'forceAnswer': 'Output:'}) + + graph = PipelineGraph(pipeline) + graph.visualize() + #pipeline.run_on_dataset(datakeys=['question','extract'],init_docs='docs') + + diff --git a/methods/AnG_Blueprint.py b/methods/AnG_Blueprint.py new file mode 100644 index 0000000000000000000000000000000000000000..433163d962e1935205fb45bf205da7c8df54281e --- /dev/null +++ b/methods/AnG_Blueprint.py @@ -0,0 +1,89 @@ +from citekit.cite_modules.LLM import LLM +from citekit.cite_modules.augment_model import AttributingModule +from citekit.pipeline.pipeline import Pipeline, PIPELINE_OUTPUT,PIPELINE_DOC_CACHE +from citekit.prompt.prompt import Prompt, ALCEDocPrompt +from citekit.Dataset.Dataset import PromptDataset +from citekit.evaluator.evaluator import DefaultEvaluator +import argparse +import json +from citekit.utils.utils import cut_and_make_as,one_paragraph,make_as + +PARA_SEP = '\n\n' +if __name__ == '__main__': + + # SETTING ARGS + parser = argparse.ArgumentParser() + parser.add_argument("--save_path", type=str, default='res.json', help="Path to the config file") + parser.add_argument("--model", type=str, default='gpt-3.5-turbo', help="model name or path") + parser.add_argument("--shots", type=int, default=1, help="number of shots") + parser.add_argument("--ndoc", type=int, default=3, help="number of docs") + parser.add_argument("--pr", action='store_true', help="use cite PR") + parser.add_argument("--rouge", action='store_true', help="use rouge") + parser.add_argument("--temp", type=float, default=0.5, help="temperature") + parser.add_argument("--qa", action='store_true', help="eval qa") + parser.add_argument("--mauve", action='store_true', help="eval mauve") + parser.add_argument("--length", type=bool, default=True, help="eval length") + parser.add_argument("--claims", action='store_true', help="eval claims") + parser.add_argument("--qampari", type=str, default=False, help="eval qampari") + parser.add_argument("--turns", type=int, default=1, help="k") + parser.add_argument("--use_fast_pr", type=str, default=False, help="test") + parser.add_argument("--dataset", type=str, default='data/asqa_eval_gtr_top100.json', help="dataset") + parser.add_argument("--demo", type=str, default='prompts/AnG.json', help="demo") + parser.add_argument("--mode", type=str, default='AnG', help="mode: AnG or plan") + args = parser.parse_args() + + # DATA LOADING + file_path = args.dataset + demo_path = args.demo + with open(file_path,'r',encoding='utf-8') as file: + dataset = json.load(file) + with open(demo_path,'r',encoding='utf-8') as file: + demo = json.load(file)[args.mode] + + + dataset = PromptDataset(dataset,'question','answer','answers','qa_pairs','claims', docs = lambda data: ALCEDocPrompt().default_load_data_wo_title(data['docs'][:args.ndoc]))[:200] + if args.mode == 'AnG': + gen_shot = demo['gen_instruction'] + PARA_SEP + demo['gen_shot'] + PARA_SEP + answer_ppt = {'INST':demo['gen_instruction'],'shot':gen_shot, 'add':'The next sentence is:'} + elif args.mode == 'plan': + shot = demo['shot1'] + demo['shot2'] + self_ppt = {'INST':demo['INST'],'shot':shot, 'add':'subquestions: \n'} + answer_shot = demo['answer_shot_1'] + demo['answer_shot_2'] + answer_ppt = {'INST':demo['answer_inst'],'shot':answer_shot,'add':''} + + prompt = Prompt(template='', + components={'INST':'{INST}\n\n', + 'shot':'{shot}', + 'question':'Question:{question}\n\n', + 'docs':'{docs}\n', + 'span':'The highlighted spans are: \n{span}\n\n', + 'prefix':'Prefix: {prefix}\n\n', + 'sub':'subquestions: \n{sub}\n\n', + 'add':'Answer: \n{add}' + }) + + plan_prompt = Prompt(template='', + components={'INST':'{INST}\n\n', + 'shot':'{shot}', + 'question':'Question:{question}\n\n', + 'docs':'{docs}\n', + 'sub':'subquestions: \n{sub}\n\n', + 'add':'{add}'}) + + # PIPELINE + evaluator = DefaultEvaluator(args) + if args.mode == 'AnG': + attribute = AttributingModule(model = args.model) + elif args.mode == 'plan': + attribute = LLM(model = args.model, prompt_maker = plan_prompt,self_prompt=self_ppt,post_processing=cut_and_make_as('sub')) + + answer = LLM(model = args.model, prompt_maker = prompt, self_prompt=answer_ppt, share_model_with=attribute.get_first_module(), post_processing=cut_and_make_as('prefix'), iterative=True) + if args.mode == 'AnG': + attribute.set_target(answer) + elif args.mode == 'plan': + attribute.set_target(answer,post_processing=cut_and_make_as('sub')) + pipeline = Pipeline(save_path=args.save_path, llm = answer, module = attribute, evaluator = evaluator, dataset = dataset) + + answer.set_output(post_processing=lambda ls: ''.join(map(one_paragraph,ls))) + + pipeline.run_on_dataset(datakeys=['question','docs'],init_docs='docs',initial_module = attribute) diff --git a/methods/Decomp.py b/methods/Decomp.py new file mode 100644 index 0000000000000000000000000000000000000000..017f9fe4fd5367b83abc8897dff06da1a56af4b2 --- /dev/null +++ b/methods/Decomp.py @@ -0,0 +1,104 @@ +from citekit.cite_modules.LLM import LLM +from citekit.cite_modules.augment_model import CitationSimplyfier +from citekit.pipeline.pipeline import Pipeline, PIPELINE_OUTPUT,PIPELINE_DOC_CACHE +from citekit.prompt.prompt import Prompt, ALCEDocPrompt +from citekit.Dataset.Dataset import PromptDataset +from citekit.evaluator.evaluator import DefaultEvaluator +from parser import * +import argparse +import json +from citekit.utils.utils import cut_and_make_as,one_paragraph,make_as +from nltk import sent_tokenize + + +def sentences_as(datakey): + def f(passage): + print( sent_tokenize(passage)) + return [{datakey:one_paragraph(s)} for s in sent_tokenize(passage)] + return f + +PARA_SEP = '\n\n' +if __name__ == '__main__': + + # SETTING ARGS + parser = argparse.ArgumentParser() + parser.add_argument("--save_path", type=str, default='res.json', help="Path to the config file") + parser.add_argument("--model", type=str, default='gpt-3.5-turbo', help="model name or path") + parser.add_argument("--shots", type=int, default=1, help="number of shots") + parser.add_argument("--ndoc", type=int, default=3, help="number of docs") + parser.add_argument("--pr", action='store_true', help="use cite PR") + parser.add_argument("--rouge", action='store_true', help="use rouge") + parser.add_argument("--temp", type=float, default=0.5, help="temperature") + parser.add_argument("--qa", action='store_true', help="eval qa") + parser.add_argument("--str_em", action='store_true', help="eval str_em") + parser.add_argument("--mauve", action='store_true', help="eval mauve") + parser.add_argument("--length", type=bool, default=True, help="eval length") + parser.add_argument("--claims", action='store_true', help="eval claims") + parser.add_argument("--qampari", type=str, default=False, help="eval qampari") + parser.add_argument("--turns", type=int, default=1, help="k") + parser.add_argument("--use_fast_pr", type=str, default=False, help="test") + parser.add_argument("--dataset", type=str, default='data/asqa_eval_gtr_top100.json', help="dataset") + parser.add_argument("--demo", type=str, default='prompts/AnG.json', help="demo") + parser.add_argument("--mode", type=str, default='plan', help="mode: AnG or plan") + parser.add_argument("--data_num", type=int, default=200, help="k") + args = parser.parse_args() + + # DATA LOADING + file_path = args.dataset + demo_path = args.demo + with open(file_path,'r',encoding='utf-8') as file: + dataset = json.load(file) + with open(demo_path,'r',encoding='utf-8') as file: + demo = json.load(file)[args.mode] + + answer_inst = 'Instruction: Write an accurate, engaging, and concise answer for the given question using only the provided search results (some of which might be irrelevant) and cite them properly, by answering the subquestion. Use an unbiased and journalistic tone. Always cite for any factual claim. When citing several search results, use [1][2][3]. Cite at least one document and at most three documents in each sentence. If multiple documents support the sentence, only cite a minimum sufficient subset of the documents.' + revise_instruction = '''You will be provided with a question, some documents and an answer. Your task is to correct possible errors in the answer and made it more coherent and fluent. Only make changes that are necessary. Keep the citations.''' + dataset = PromptDataset(dataset,'question','answer','answers','qa_pairs','claims', docs = lambda data: ALCEDocPrompt().default_load_data_wo_title(data['docs'][:args.ndoc]))[:args.data_num] + if args.mode == 'AnG': + gen_shot = demo['gen_instruction'] + PARA_SEP + demo['gen_shot'] + PARA_SEP + answer_ppt = {'INST':demo['gen_instruction'],'shot':gen_shot, 'add':'The next sentence is:'} + elif args.mode == 'plan': + shot = demo['shot1'] + demo['shot2'] + self_ppt = {'INST':demo['INST'],'shot':shot, 'add':'subquestions: \n'} + answer_shot = demo['answer_shot_1'] + demo['answer_shot_2'] + answer_ppt = {'INST':answer_inst,'shot':answer_shot,'add':''} + + prompt = Prompt(template='', + components={'INST':'{INST}\n\n', + 'shot':'{shot}', + 'question':'Question:{question}\n\n', + 'docs':'{docs}\n', + 'span':'The highlighted spans are: \n{span}\n\n', + 'prefix':'Prefix: {prefix}\n\n', + 'sub':'subquestions: \n{sub}\n\n', + 'add':'Answer: \n{add}', + 'new_ans': "\nNew Answer: {new_ans}" + }) + + plan_prompt = Prompt(template='', + components={'INST':'{INST}\n\n', + 'shot':'{shot}', + 'question':'Question:{question}\n\n', + 'docs':'{docs}\n', + 'sub':'subquestions: \n{sub}\n\n', + 'add':'{add}'}) + + # PIPELINE + evaluator = DefaultEvaluator(args) + + attribute = LLM(model = args.model, prompt_maker = plan_prompt, self_prompt=self_ppt, post_processing=sentences_as('sub')) + reviser = LLM(model=args.model, prompt_maker=prompt, self_prompt={'INST':revise_instruction, 'new_ans': ''}, max_turn=args.turns) + answer = LLM(model = args.model, prompt_maker = prompt, self_prompt=answer_ppt, share_model_with=attribute.get_first_module(), merge=True, parallel=True) + + # No citation -> simply combine + # simplifier + answer.set_target(reviser, post_processing=lambda output: {'add': output}) + attribute.set_target(answer,post_processing=sentences_as('sub')) + pipeline = Pipeline(save_path=args.save_path, llm = answer, module = [attribute, reviser], evaluator = evaluator, dataset = dataset) + pipeline.set_initial_module(module=attribute) + answer.set_output(post_processing=lambda ls: ''.join(map(one_paragraph,ls))) + + + + + #pipeline.run_on_dataset(datakeys=['question','docs'], init_docs='docs', initial_module = attribute) diff --git a/methods/__init__.py b/methods/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/methods/__pycache__/__init__.cpython-312.pyc b/methods/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2e2d8a775750332a7d12acc7d0f448b6063ff946 Binary files /dev/null and b/methods/__pycache__/__init__.cpython-312.pyc differ diff --git a/methods/__pycache__/self_RAG_demo.cpython-312.pyc b/methods/__pycache__/self_RAG_demo.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5bc6f6f3408381a7d38c480b9d62cabbae77a128 Binary files /dev/null and b/methods/__pycache__/self_RAG_demo.cpython-312.pyc differ diff --git a/methods/__pycache__/self_RAG_para.cpython-312.pyc b/methods/__pycache__/self_RAG_para.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b1dfdb4ac720e106931564b62144024902a99627 Binary files /dev/null and b/methods/__pycache__/self_RAG_para.cpython-312.pyc differ diff --git a/methods/citation_augmented.py b/methods/citation_augmented.py new file mode 100644 index 0000000000000000000000000000000000000000..25895c996b4d8e941ec8411b8899c4d8030532db --- /dev/null +++ b/methods/citation_augmented.py @@ -0,0 +1,94 @@ +from citekit.cite_modules.LLM import LLM +from citekit.cite_modules.augment_model import Retriever +from citekit.pipeline.pipeline import Pipeline, PIPELINE_OUTPUT,PIPELINE_DOC_CACHE +from citekit.prompt.prompt import Prompt, DocPrompt +from citekit.Dataset.Dataset import PromptDataset +from citekit.evaluator.evaluator import DefaultEvaluator +from citekit.utils.utils import output_begin_with, make_as,output_end_with,one_paragraph,remove_citations +import json +import argparse +import nltk +import re + +def each_make_as(key): + def function(output): + sents = nltk.sent_tokenize(one_paragraph(output)) + if len(sents)>3: + sents = sents[:3] + return [make_as(key)(sent) for sent in sents] + return function + +def add_citation(ls): + output = '' + pattern = r'([.!?])\s*$' + for i, answer in enumerate(ls): + cite = f'[{i+1}]' + answer = one_paragraph(answer) + if not answer: + return cite + else: + answer = re.sub(pattern, rf'{cite}\1 ', answer) + if cite not in answer: + answer += cite + output += answer + return output + +if __name__ == '__main__': + # SETTING ARGS + parser = argparse.ArgumentParser() + parser.add_argument("--save_path", type=str, default='res.json', help="Path to the config file") + parser.add_argument("--model", type=str, default='gpt-3.5-turbo', help="model name or path") + parser.add_argument("--shots", type=int, default=2, help="number of shots") + parser.add_argument("--ndoc", type=int, default=3, help="number of docs") + parser.add_argument("--pr", action='store_true', help="use cite PR") + parser.add_argument("--rouge", action='store_true', help="use rouge") + parser.add_argument("--temp", type=float, default=0.5, help="temperature") + parser.add_argument("--qa", action='store_true', help="eval qa") + parser.add_argument("--mauve", action='store_true', help="eval mauve") + parser.add_argument("--length", type=bool, default=True, help="eval length") + parser.add_argument("--claims", action='store_true', help="eval length") + parser.add_argument("--qampari", type=str, default=False, help="eval qampari") + parser.add_argument("--dataset", type=str, default='data/asqa_eval_gtr_top100.json', help="dataset") + parser.add_argument("--demo", type=str, default='prompts/asqa_default.json', help="demo") + parser.add_argument("--add_cite", action='store_true', help="manuel add cite") + parser.add_argument("--top_k", type=int, default=1, help="retrieve docs") + args = parser.parse_args() + + # DATA LOADING + file_path = args.dataset + demo_path = args.demo + with open(file_path,'r',encoding='utf-8') as file: + dataset = json.load(file) + with open(demo_path,'r',encoding='utf-8') as file: + demo = json.load(file) + + # DATA + documents = [DocPrompt().load_data(list(enumerate(data['docs'])),Title = lambda data: data[1]['title'], Passage = lambda data: data[1]['text']) for data in dataset] + + dataset =PromptDataset(dataset, 'question','answer','qa_pairs','answers','claims')[:200] + + llm_instruction = 'Instruction: Write an accurate, engaging, and concise answer for the given question. Use an unbiased and journalistic tone.' + if args.add_cite: + llm_instruction_after = 'Instruction: Revise and correct the answer to an accurate, engaging, and concise answer for the given question using only the provided document using only one sentence. Use an unbiased and journalistic tone. Your revised answer must contain only one short sentence.' + else: + llm_instruction_after = 'Instruction: Revise and correct the answer to an accurate, engaging, and concise answer for the given question using only the provided search results (some of which might be irrelevant) and cite them properly. Use an unbiased and journalistic tone. Always cite for any factual claim. When citing several search results, use [1][2][3]. Cite at least one document and at most three documents in each sentence. If multiple documents support the sentence, only cite a minimum sufficient subset of the documents. Your revised answer must contain only one short sentence.' + shots = '\n\n'.join(llm_instruction + '\n\nQuestion: '+ d['question']+'\n\nAnswer: '+remove_citations(d['answer']) for d in demo['demos'][:args.shots]) + llm_prompt = Prompt(template='\n\nAnswer: ',components={'INST':'{INST}\n\n', 'shots':'{shots}\n\n', 'question':'Question: {question}\n\n','docs':'{docs}', 'answer':'\nThis is the answer you should revise based on the provided document: \n{answer}'}) + retriever_prompt = Prompt(template='',components={'query':'{query}'}) + + # PIPELINE + llm = LLM(model=args.model, prompt_maker=llm_prompt, self_prompt={'INST':llm_instruction,'shots':shots},stop=['\n','\n\n']) + eval = DefaultEvaluator(args) + pipeline = Pipeline(llm = llm, head_prompt_maker=llm_prompt,evaluator = eval,dataset = dataset,save_path=args.save_path) + retriever = Retriever(prompt_maker=retriever_prompt,pipeline=pipeline,retrieve_by='bm25',documents=documents,topk=args.top_k) + llm.set_target(retriever,lambda self: self.turns == 1, post_processing=each_make_as('query')) + if args.add_cite: + llm.set_output(lambda self: self.turns > 1, post_processing = add_citation, end=True) + else: + llm.set_output(lambda self: self.turns > 1, post_processing = lambda ls: ''.join(map(one_paragraph,ls)), end=True) + retriever.set_target(llm ,post_processing=lambda input, output: {'docs': output,'answer': input,'INST':llm_instruction_after,'shots':Prompt.UNABLE}) + + # RUN PIPELINE + pipeline.run_on_dataset(datakeys=['question']) + + diff --git a/methods/demonstration.ipynb b/methods/demonstration.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..8e032256ff552e7450697fadb5908f899f6c39b9 --- /dev/null +++ b/methods/demonstration.ipynb @@ -0,0 +1,212 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "5e21c7be", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[nltk_data] Downloading package punkt to\n", + "[nltk_data] /Users/shenjiajun/nltk_data...\n", + "[nltk_data] Package punkt is already up-to-date!\n", + "/Users/shenjiajun/miniconda3/envs/torch_gpu/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "from citekit.cite_modules.LLM import LLM\n", + "from citekit.cite_modules.augment_model import AttributingModule\n", + "from citekit.pipeline.pipeline import Sequence\n", + "from citekit.prompt.prompt import Prompt\n", + "from citekit.Dataset.Dataset import FileDataset\n", + "from citekit.evaluator.evaluator import DefaultEvaluator\n", + "import json" + ] + }, + { + "cell_type": "markdown", + "id": "0260d8c5", + "metadata": {}, + "source": [ + "# Data Loading" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "c374b212", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Instruction: Write an accurate, engaging, and concise answer for the given question using only the provided search results (some of which might be irrelevant) and cite them properly. Use an unbiased and journalistic tone. Always cite for any factual claim. When citing several search results, use [1][2][3]. Cite at least one document and at most three documents in each sentence. If multiple documents support the sentence, only cite a minimum sufficient subset of the documents.\n" + ] + } + ], + "source": [ + "dataset = FileDataset('data/asqa.json')\n", + "\n", + "with open('prompts/asqa.json','r',encoding='utf-8') as file:\n", + " demo = json.load(file)\n", + " instruction = demo['INST']\n", + "print(instruction)" + ] + }, + { + "cell_type": "markdown", + "id": "fb237785", + "metadata": {}, + "source": [ + "# Make a Request Wrapper\n", + "\n", + "The request wrapper contains is a template that consist of different components. Components with no data passing in will be automatically dropped." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "62abd5d2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{INST}\n", + "\n", + "Question:{question}\n", + "\n", + "{docs}\n", + "Prefix: {prefix}\n", + "\n", + "The highlighted spans are: \n", + "{span}\n", + "\n", + "Answer: \n" + ] + } + ], + "source": [ + "prompt = Prompt(template='Answer: ',\n", + " components={'INST':'{INST}\\n\\n', \n", + " 'question':'Question:{question}\\n\\n',\n", + " 'docs':'{docs}\\n',\n", + " 'span':'The highlighted spans are: \\n{span}\\n\\n',\n", + " 'prefix':'Prefix: {prefix}\\n\\n',\n", + " })\n", + "\n", + "print(prompt)" + ] + }, + { + "cell_type": "markdown", + "id": "72d33ff4", + "metadata": {}, + "source": [ + "# Define an Evaluater\n", + "An Evaluator will automatically evaluate the output, once some metrics are defined. You can use some pre-defined ones or new ones." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "37788167", + "metadata": {}, + "outputs": [], + "source": [ + "evaluator = DefaultEvaluator(criteria = ['str_em','length','rouge'])" + ] + }, + { + "cell_type": "markdown", + "id": "fd3541a3", + "metadata": {}, + "source": [ + "# Define Generation Modules and Enhancing Modules" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "9ad59c1a", + "metadata": {}, + "outputs": [], + "source": [ + "# attributer = AttributingModule(model='gpt-3.5-turbo')\n", + "llm = LLM(model='gpt-3.5-turbo',prompt_maker=prompt, self_prompt={'INST':instruction})" + ] + }, + { + "cell_type": "markdown", + "id": "1e4a38c0", + "metadata": {}, + "source": [ + "# Construct a Pipeline" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14b5c262", + "metadata": {}, + "outputs": [], + "source": [ + "pipeline = Sequence(sequence = [llm], head_prompt_maker = prompt, evaluator = evaluator, dataset = dataset)" + ] + }, + { + "cell_type": "markdown", + "id": "6788a72b", + "metadata": {}, + "source": [ + "# Run" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f12bb819", + "metadata": {}, + "outputs": [], + "source": [ + "pipeline.run_on_dataset(datakeys=['question','docs'], init_docs='docs')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18a787c6", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "torch", + "language": "python", + "name": "torch" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/methods/recitation_aug.py b/methods/recitation_aug.py new file mode 100644 index 0000000000000000000000000000000000000000..0c5112948f07cd802c197651aee4638a3e4242e1 --- /dev/null +++ b/methods/recitation_aug.py @@ -0,0 +1,62 @@ +from citekit.cite_modules.LLM import LLM,Module +from citekit.cite_modules.augment_model import Retriever +from citekit.pipeline.pipeline import Pipeline, PIPELINE_OUTPUT, PIPELINE_DOC_CACHE +from citekit.prompt.prompt import Prompt, ALCEVanillaPrompt, DocPrompt,ALCEDocPrompt,NewALCEVanillaPrompt +from citekit.Dataset.Dataset import PromptDataset +from citekit.evaluator.evaluator import Evaluator,DefaultEvaluator +from citekit.utils.utils import output_begin_with, make_as,output_end_with,one_paragraph +import json +import argparse + +RECITE_PROMPT = "The answer to the above question can be found in the following Wikipedia page, section, and paragraph:\n" +class R_LLM(LLM): + def generate_content(self, prompt): + output=one_paragraph(super().generate_content(prompt)) + self.pipeline.doc_cache.add_doc(doc = output, add_id = True) + return f'Document [{len(self.pipeline.doc_cache)}]\n'+output + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("--save_path", type=str, default='result.json', help="Path to the config file") + parser.add_argument("--recite_model", type=str, default='gpt-3.5-turbo', help="model name or path") + parser.add_argument("--answer_model", type=str, default='gpt-3.5-turbo', help="model name or path") + parser.add_argument("--shots", type=int, default=4, help="number of shots") + parser.add_argument("--ndoc", type=int, default=3, help="number of docs") + parser.add_argument("--pr", action='store_true', help="use cite PR") + parser.add_argument("--rouge", action='store_true', help="use rouge") + parser.add_argument("--temp", type=float, default=0.5, help="temperature") + parser.add_argument("--qa", action='store_true', help="eval qa") + parser.add_argument("--mauve", action='store_true', help="eval mauve") + parser.add_argument("--length", type=bool, default=True, help="eval claims") + parser.add_argument("--claims", action='store_true', help="eval length") + parser.add_argument("--qampari", type=str, default=False, help="eval qampari") + parser.add_argument("--dataset", type=str, default='data/asqa_eval_gtr_top100.json', help="dataset") + parser.add_argument("--demo", type=str, default='prompts/asqa_default.json', help="demo") + args = parser.parse_args() + + file_path = args.dataset + demo_path = args.demo + with open(file_path,'r',encoding='utf-8') as file: + dataset = json.load(file) + with open(demo_path,'r',encoding='utf-8') as file: + demo = json.load(file) + + llm_inst = demo['instruction'] + prompt = Prompt(template='',components= {'passage':'{passage}\n\n', 'shots':'{shots}\n','question':'Question:{question}\n\n','RP':'\n{RP}\nAnswer:'}) + llm_prompt = Prompt(template='',components= {'INST':'{INST}\n\n','passage':'{passage}\n\nAnswer:\n','question':'Question:{question}\n\n','shots':'{shots}\n'}) + recite_shots = ''.join(NewALCEVanillaPrompt().load_data(demo['demos'][:args.shots],'question', docs = lambda data: RECITE_PROMPT, answer = lambda data: ''.join(ALCEDocPrompt().default_load_data_wo_ID(data['docs'][:1])))) + llm_shots = '\n'.join(NewALCEVanillaPrompt().load_data(demo['demos'][:args.shots],'question','answer', docs = lambda data: ''.join(ALCEDocPrompt().default_load_data(data['docs'][:args.ndoc])))) + dataset = PromptDataset(dataset,'question', 'answer', 'answers' ,'qa_pairs','claims')[:1] + + # PIPELINE + eval = DefaultEvaluator(args) + recite = R_LLM(model=args.recite_model,prompt_maker=prompt, self_prompt={'shots':recite_shots,'RP':RECITE_PROMPT}) + llm = LLM(model=args.answer_model,prompt_maker=llm_prompt, self_prompt={'INST':llm_inst},share_model_with=recite) + recite.set_target(llm, condition=lambda self:self.turns==args.ndoc,post_processing= lambda x:{'shots':llm_shots}) + recite.add_to_head('passage') + recite.set_target(recite, condition=lambda self:self.turns", components={"query": "{query}"}) + + query_generator = LLM( + model=args.model, prompt_maker=queryprompt, self_prompt={"INST": query_inst} + ) + retriever_prompt = Prompt(template="", components={"query": "{query}"}) + eval = DefaultEvaluator(args) + ranker = Ranker(max_turn=3, iterative=True) + # ranker.set_eval('length', output = 'answer') + # ranker.new_eval('score', score , output = 'answer', docs = 'doc_cache', qa_pairs = 'qa_pairs') + ranker.new_eval("score", score, output="answer", docs="doc_cache") + # PIPELINE CONSTRUCTING + llm = LLM( + model=args.model, + prompt_maker=prompt, + self_prompt={"INST": llm_instruction, "shots": shots}, + max_turn=30, + auto_cite=True, + share_model_with=query_generator, + parallel=True, + ) + pipeline = Pipeline( + save_path=args.save_path, + llm=llm, + module=[ranker, query_generator], + head_prompt_maker=prompt, + evaluator=eval, + dataset=dataset, + ) + + retriever = Retriever( + prompt_maker=retriever_prompt, + pipeline=pipeline, + retrieve_by="bm25", + documents=documents, + topk=args.topk, + ) + query_generator.set_target(retriever, post_processing=make_as("query")) + query_generator.add_to_head("prev", sub=False) + retriever.set_target(llm, post_processing=segment) + llm.set_target(ranker, post_processing=make_as("answer")) + ranker.set_output(post_processing=lambda x: x["answer"], end=False) + + ranker.add_to_head( + "ans", sub=True, process=lambda text: one_paragraph(text["answer"]) + ) + ranker.set_target(query_generator, post_processing=lambda x: {"ans": x["answer"]}) + pipeline.set_initial_module(query_generator) + pipeline.set_data_keys(["question"]) + + # graph = PipelineGraph(pipeline=pipeline) + # html = graph.generate_html(results='results.json') + # graph.visualize() + # print(html) + # with open('pipeline_.html','w') as file: + # file.write(html) + # RUN PIPELINE + pipeline.run_on_dataset(datakeys=['question'], initial_module=query_generator) diff --git a/methods/self_RAG_attr.py b/methods/self_RAG_attr.py new file mode 100644 index 0000000000000000000000000000000000000000..b575de333f714bc0616c1b2f2b0480d8f4ad2138 --- /dev/null +++ b/methods/self_RAG_attr.py @@ -0,0 +1,210 @@ +from citekit.cite_modules.LLM import LLM +from citekit.cite_modules.augment_model import ( + Retriever, + CitationSimplyfier, + Verifier, + Ranker, + AttributingModule, +) +from citekit.pipeline.pipeline import Pipeline, PIPELINE_OUTPUT, PIPELINE_DOC_CACHE +from citekit.prompt.prompt import Prompt, ALCEDocPrompt, DocPrompt, NewALCEVanillaPrompt +from citekit.Dataset.Dataset import PromptDataset +from citekit.evaluator.evaluator import ( + DefaultEvaluator, + compute_autoais, + test_compute_autoais, +) + +from citekit.utils.utils import ( + sentence, + one_paragraph, + each_make_as, + each_make_as, + make_as, + remove_citations, + compute_str_em, +) +import json +import argparse + + +def segment(i, text): + return [make_as("docs")(doc) for doc in text.split("\n") if doc] + + +if __name__ == "__main__": + + # SETTING ARGS + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", type=str, default="res.json", help="Path to the config file" + ) + parser.add_argument( + "--model", type=str, default="gpt-3.5-turbo", help="model name or path" + ) + parser.add_argument("--shots", type=int, default=2, help="number of shots") + parser.add_argument("--ndoc", type=int, default=5, help="number of docs") + parser.add_argument("--pr", action="store_true", help="use cite PR") + parser.add_argument("--rouge", action="store_true", help="use rouge") + parser.add_argument("--temp", type=float, default=0.5, help="temperature") + parser.add_argument("--qa", action="store_true", help="eval qa") + parser.add_argument("--mauve", action="store_true", help="eval mauve") + parser.add_argument("--length", type=bool, default=True, help="eval length") + parser.add_argument("--claims", action="store_true", help="eval claims") + parser.add_argument("--qampari", type=str, default=False, help="eval qampari") + parser.add_argument( + "--dataset", type=str, default="data/asqa_eval_gtr_top100.json", help="dataset" + ) + parser.add_argument( + "--demo", type=str, default="prompts/asqa_default.json", help="demo" + ) + parser.add_argument("--doctype", type=str, default="text", help="demo") + parser.add_argument("--data_num", type=int, default=1000, help="num of data") + parser.add_argument( + "--mode", + type=str, + default="text", + help="mode-granularity: text, extraction or summary", + ) + parser.add_argument("--k", type=float, default=1.5, help="coefficient of em") + parser.add_argument("--topk", type=int, default=2, help="topk") + args = parser.parse_args() + + def score(data): + # pr = compute_autoais(data) + # p = pr["citation_prec"] + # r = pr["citation_rec"] + # em = compute_str_em(data) + # return p + r + args.k * em + return 1 + + # DATA LOADING + file_path = args.dataset + demo_path = args.demo + + with open(file_path, "r", encoding="utf-8") as file: + dataset = json.load(file) + with open(demo_path, "r", encoding="utf-8") as file: + demo = json.load(file) + data_num = min(args.data_num, len(dataset)) + + llm_instruction = demo["one_sentence_instruction"] + query_inst = demo["query_instruction"] + shots = "\n\n".join( + NewALCEVanillaPrompt().load_data( + [demo["demos"][1], demo["demos"][3]], + "question", + answer=lambda data: remove_citations( + sentence("first")(data["answer"])["first"] + ), + INST=lambda _: llm_instruction, + docs=lambda data: "".join( + ALCEDocPrompt().default_load_data(data["docs"][1:2]) + ), + ) + ) + + documents = [ + DocPrompt().load_data( + list(enumerate(data["docs"])), + Title=lambda data: data[1]["title"], + Passage=lambda data: data[1][args.mode], + ) + for data in dataset + ] + + dataset = PromptDataset( + dataset, + "question", + "answer", + "answers", + "qa_pairs", + "claims", + docs=lambda data: ALCEDocPrompt().default_load_data(data["docs"][: args.ndoc]), + )[:data_num] + + prompt = Prompt( + template="\nAnswer:", + components={ + "INST": "{INST}\n\n", + "shots": "{shots}\n", + "question": "Question:{question}\n\n", + "ans": "Prefix:{ans}\n\n", + "docs": "{docs}\n", + "span": "The highlighted spans are: \n{span}\n\n", + }, + ) + queryprompt = Prompt( + template="Please generate one query to help find relevent documents. If previous queries are provided, you shoud focus on an alternative perspective or subtopic different from the provided ones, enhancing diversity in retrieved documents. your query is:\n", + components={ + "question": "Given the original question: {question}\n", + "ans": "The context is: {ans}\n", + "prev": "\nPrevious queries:\n{prev}\n\n", + "INST": "{INST}\n\n", + }, + ) + + retriever_prompt = Prompt(template="", components={"query": "{query}"}) + + query_generator = LLM( + model=args.model, prompt_maker=queryprompt, self_prompt={"INST": query_inst} + ) + retriever_prompt = Prompt(template="", components={"query": "{query}"}) + eval = DefaultEvaluator(args) + ranker = Ranker(max_turn=3, iterative=True) + # ranker.set_eval('length', output = 'answer') + # ranker.new_eval('score', score , output = 'answer', docs = 'doc_cache', qa_pairs = 'qa_pairs') + ranker.new_eval("score", score, output="answer", docs="doc_cache") + # PIPELINE CONSTRUCTING + llm = LLM( + model=args.model, + prompt_maker=prompt, + self_prompt={"INST": llm_instruction, "shots": shots}, + max_turn=30, + auto_cite=True, + auto_cite_from = 'span', + share_model_with=query_generator, + parallel=True, + ) + pipeline = Pipeline( + save_path=args.save_path, + llm=llm, + module=[ranker, query_generator], + head_prompt_maker=prompt, + evaluator=eval, + dataset=dataset, + ) + + retriever = Retriever( + prompt_maker=retriever_prompt, + pipeline=pipeline, + retrieve_by="bm25", + topk=args.topk, + documents=documents, + ) + query_generator.set_target(retriever, post_processing=make_as("query")) + query_generator.add_to_head("prev", sub=False) + retriever.set_target(llm, post_processing=segment) + llm.set_target(ranker, post_processing=make_as("answer")) + ranker.set_output(post_processing=lambda x: x["answer"], end=False) + + ranker.add_to_head( + "ans", sub=True, process=lambda text: one_paragraph(text["answer"]) + ) + ranker.set_target(query_generator, post_processing=lambda x: {"ans": x["answer"]}) + pipeline.set_initial_module(query_generator) + pipeline.set_data_keys(["question"]) + + attributer = AttributingModule(model=args.model) + attributer.connect_to(pipeline) + retriever.set_target(attributer, post_processing=make_as("docs")) + retriever.add_to_head("docs", sub=True) + attributer.set_target(llm) + # graph = PipelineGraph(pipeline=pipeline) + # html = graph.generate_html(results='results.json') + # graph.visualize() + # print(html) + # with open('pipeline_.html','w') as file: + # file.write(html) + # RUN PIPELINE + pipeline.run_on_dataset(datakeys=["question"], initial_module=query_generator) diff --git a/methods/self_RAG_demo.py b/methods/self_RAG_demo.py new file mode 100644 index 0000000000000000000000000000000000000000..310c688c139c4438fff3fc71f6325d862f9668e3 --- /dev/null +++ b/methods/self_RAG_demo.py @@ -0,0 +1,121 @@ +from citekit.cite_modules.LLM import LLM +from citekit.cite_modules.augment_model import Retriever,CitationSimplyfier,Verifier,Ranker, AttributingModule +from citekit.pipeline.pipeline import Pipeline, PIPELINE_OUTPUT, PIPELINE_DOC_CACHE +from citekit.prompt.prompt import Prompt, ALCEDocPrompt,DocPrompt,NewALCEVanillaPrompt +from citekit.Dataset.Dataset import PromptDataset +from citekit.evaluator.evaluator import DefaultEvaluator, compute_autoais, test_compute_autoais +from citekit.attribute.attribute import compute_contributions, InterpretableAttributer +from citekit.utils.utils import sentence, one_paragraph, each_make_as, each_make_as, make_as,remove_citations, compute_str_em + +import json +from parser import * +import argparse + +def segment(i,text): + return [make_as('docs')(doc) for doc in text.split('\n') if doc] + + + + +# SETTING ARGS +parser = argparse.ArgumentParser() +parser.add_argument("--save_path", type=str, default='res.json', help="Path to the config file") +parser.add_argument("--model", type=str, default='gpt-3.5-turbo', help="model name or path") +parser.add_argument("--shots", type=int, default=2, help="number of shots") +parser.add_argument("--ndoc", type=int, default=5, help="number of docs") +parser.add_argument("--pr", action='store_true', help="use cite PR") +parser.add_argument("--rouge", action='store_true', help="use rouge") +parser.add_argument("--temp", type=float, default=0.5, help="temperature") +parser.add_argument("--qa", action='store_true', help="eval qa") +parser.add_argument("--mauve", action='store_true', help="eval mauve") +parser.add_argument("--length", type=bool, default=True, help="eval length") +parser.add_argument("--claims", action='store_true', help="eval claims") +parser.add_argument("--qampari", type=str, default=False, help="eval qampari") +parser.add_argument("--dataset", type=str, default='data/asqa_eval_gtr_top100.json', help="dataset") +parser.add_argument("--demo", type=str, default='prompts/asqa_default.json', help="demo") +parser.add_argument("--doctype", type=str, default='text', help="demo") +parser.add_argument("--data_num", type=int, default=1000, help="num of data") +parser.add_argument("--mode", type=str, default='text', help="mode-granularity: text, extraction or summary") +parser.add_argument("--k", type=float, default=1.5, help="coefficient of em") +parser.add_argument("--topk", type=int, default=1, help="topk") +args = parser.parse_args() + +def score(data): + #pr = compute_autoais(data) + #p = pr["citation_prec"] + #r = pr["citation_rec"] + #em = compute_str_em(data) + #return p + r + args.k * em + return 1 + +# DATA LOADING +file_path = args.dataset +demo_path = args.demo + + +dataset = [] +with open(demo_path,'r',encoding='utf-8') as file: + demo = json.load(file) +data_num = min(args.data_num,len(dataset)) + +llm_instruction = demo['one_sentence_instruction'] +query_inst = demo["query_instruction"] +shots = '\n\n'.join(NewALCEVanillaPrompt().load_data([demo['demos'][1],demo['demos'][3]],'question', answer = lambda data: remove_citations(sentence('first')(data['answer'])['first']), INST = lambda _: llm_instruction, docs = lambda data: ''.join(ALCEDocPrompt().default_load_data(data['docs'][1:2])))) + + +documents = [DocPrompt().load_data(list(enumerate(data['docs'])),Title = lambda data: data[1]['title'], Passage = lambda data: data[1][args.mode]) for data in dataset] + +dataset = PromptDataset(dataset,'question','answer','answers','qa_pairs','claims', docs = lambda data: ALCEDocPrompt().default_load_data(data['docs'][:args.ndoc]))[:data_num] + +prompt = Prompt(template='\nAnswer:', components= {'INST':'{INST}\n\n','shots':'{shots}\n','question':'Question:{question}\n\n', 'ans':'Prefix:{ans}\n\n','docs':'{docs}\n', 'span':'The highlighted spans are: \n{span}\n\n'}) +queryprompt = Prompt(template='Please generate one query to help find relevent documents, making sure it is different from previous queries(if provided). your query is:\n',components={'question':'Given the original question: {question}\n','ans':'The context is: {ans}\n','prev':'\nPrevious queries:\n{prev}\n\n','INST':'{INST}\n\n'}) + +retriever_prompt = Prompt(template='',components={'query':'{query}'}) + +query_generator = LLM(model=args.model, prompt_maker=queryprompt, self_prompt={'INST':query_inst}) +retriever_prompt = Prompt(template='',components={'query':'{query}'}) +eval = DefaultEvaluator(args) +ranker = Ranker(max_turn=5, iterative= True) +#ranker.set_eval('length', output = 'answer') +#ranker.new_eval('score', score , output = 'answer', docs = 'doc_cache', qa_pairs = 'qa_pairs') +ranker.new_eval('score', score , output = 'answer', docs = 'doc_cache') +# PIPELINE CONSTRUCTING +llm = LLM(model=args.model,prompt_maker=prompt, self_prompt={'INST':llm_instruction, 'shots':shots}, max_turn= 2, auto_cite=True,share_model_with= query_generator, parallel= True) + + +pipeline = Pipeline(save_path=args.save_path, llm = llm ,module=[ranker,query_generator],head_prompt_maker=prompt, evaluator=eval,dataset = dataset) + +retriever = Retriever(prompt_maker=retriever_prompt, pipeline=pipeline, retrieve_by='bm25', topk=args.topk) +query_generator.set_target(retriever, post_processing=make_as('query')) +query_generator.add_to_head('prev', sub = False) +retriever.set_target(llm,post_processing=segment) +llm.set_target(ranker,post_processing=make_as('answer')) +ranker.set_output(post_processing=lambda x: x['answer'], end = False) + +ranker.add_to_head('ans',sub = True, process = lambda text: one_paragraph(text['answer']) ) +ranker.set_target(query_generator,post_processing=lambda x:{'ans': x['answer']}) +pipeline.set_initial_module(query_generator) +pipeline.set_data_keys(['question']) + +attributer = AttributingModule(model = 'gpt-4o-mini') +attributer.connect_to(pipeline) + +# 交互界面加入：make_as之类预定义的输出格式， add to head的设置，prompt输入时的特殊格式。 +#retriever.set_target(attributer, post_processing=make_as('docs')) +#retriever.add_to_head('docs', sub=True) +#attributer.set_target(llm) + +#pipeline.run(datakeys=['question'],initial_module=query_generator) + + + + +graph = PipelineGraph(pipeline=pipeline) + +#html = graph.generate_html(results='results.json') +#graph.visualize() +#print(html) +#with open('pipeline_.html','w') as file: +# file.write(html) +# RUN PIPELINE +#pipeline.run_on_dataset(datakeys=['question'],initial_module=query_generator) diff --git a/methods/self_RAG_para.py b/methods/self_RAG_para.py new file mode 100644 index 0000000000000000000000000000000000000000..c406ee81f496dd3b33c4ddfd0808b2a81ed947ce --- /dev/null +++ b/methods/self_RAG_para.py @@ -0,0 +1,206 @@ +from citekit.cite_modules.LLM import LLM +from citekit.cite_modules.augment_model import ( + Retriever, + CitationSimplyfier, + Verifier, + Ranker, +) +from citekit.pipeline.pipeline import Pipeline, PIPELINE_OUTPUT, PIPELINE_DOC_CACHE +from citekit.prompt.prompt import Prompt, ALCEDocPrompt, DocPrompt, NewALCEVanillaPrompt +from citekit.Dataset.Dataset import PromptDataset +from citekit.evaluator.evaluator import ( + DefaultEvaluator, + compute_autoais, + test_compute_autoais, +) +from citekit.utils.utils import ( + sentence, + one_paragraph, + each_make_as, + each_make_as, + make_as, + remove_citations, + compute_str_em, +) +import json +from parser import * +import argparse + + +def segment(i, text): + return [make_as("docs")(doc) for doc in text.split("\n") if doc] + +def segment_query(text): + return [make_as("query")(doc) for doc in text.split("\n") if doc] + +if True: + + # SETTING ARGS + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", type=str, default="resu.json", help="Path to the config file" + ) + parser.add_argument( + "--model", type=str, default="gpt-3.5-turbo", help="model name or path" + ) + parser.add_argument("--shots", type=int, default=2, help="number of shots") + parser.add_argument("--ndoc", type=int, default=5, help="number of docs") + parser.add_argument("--pr", action="store_true", help="use cite PR") + parser.add_argument("--rouge", action="store_true", help="use rouge") + parser.add_argument("--temp", type=float, default=0.5, help="temperature") + parser.add_argument("--qa", action="store_true", help="eval qa") + parser.add_argument("--mauve", action="store_true", help="eval mauve") + parser.add_argument("--length", type=bool, default=True, help="eval length") + parser.add_argument("--claims", action="store_true", help="eval claims") + parser.add_argument("--qampari", type=str, default=False, help="eval qampari") + parser.add_argument( + "--dataset", type=str, default="data/asqa_eval_gtr_top100.json", help="dataset" + ) + parser.add_argument( + "--demo", type=str, default="prompts/asqa_default.json", help="demo" + ) + parser.add_argument("--doctype", type=str, default="text", help="demo") + parser.add_argument("--data_num", type=int, default=1000, help="num of data") + parser.add_argument( + "--mode", + type=str, + default="text", + help="mode-granularity: text, extraction or summary", + ) + parser.add_argument("--k", type=float, default=1.5, help="coefficient of em") + parser.add_argument("--topk", type=int, default=2, help="topk") + args = parser.parse_args() + + def score(data): + pr = compute_autoais(data) + p = pr["citation_prec"] + r = pr["citation_rec"] + em = compute_str_em(data) + return p + r + args.k * em + return 1 + + # DATA LOADING + file_path = args.dataset + demo_path = args.demo + + with open(file_path, "r", encoding="utf-8") as file: + dataset = json.load(file) + with open(demo_path, "r", encoding="utf-8") as file: + demo = json.load(file) + data_num = min(args.data_num, len(dataset)) + + llm_instruction = demo["one_sentence_instruction"] + query_inst = demo["query_instruction"] + shots = "\n\n".join( + NewALCEVanillaPrompt().load_data( + [demo["demos"][1], demo["demos"][3]], + "question", + answer=lambda data: remove_citations( + sentence("first")(data["answer"])["first"] + ), + INST=lambda _: llm_instruction, + docs=lambda data: "".join( + ALCEDocPrompt().default_load_data(data["docs"][1:2]) + ), + ) + ) + + documents = [ + DocPrompt().load_data( + list(enumerate(data["docs"])), + Title=lambda data: data[1]["title"], + Passage=lambda data: data[1][args.mode], + ) + for data in dataset + ] + + dataset = PromptDataset( + dataset, + "question", + "answer", + "answers", + "qa_pairs", + "claims", + docs=lambda data: ALCEDocPrompt().default_load_data(data["docs"][: args.ndoc]), + )[:data_num] + + prompt = Prompt( + template="\nAnswer:", + components={ + "INST": "{INST}\n\n", + "shots": "{shots}\n", + "question": "Question:{question}\n\n", + "ans": "Prefix:{ans}\n\n", + "docs": "{docs}\n", + }, + ) + queryprompt = Prompt( + template="Please generate at most three queries regarding possible subquestions of the given question. Your queris should be diverse and informative in natual language, splited by a new line.\n", + components={ + "question": "Given the original question: {question}\n", + "ans": "The context is: {ans}\n", + "prev": "\nPrevious queries:\n{prev}\n\n", + "INST": "{INST}\n\n", + }, + ) + + retriever_prompt = Prompt(template="", components={"query": "{query}"}) + + query_generator = LLM( + model=args.model, prompt_maker=queryprompt, self_prompt={"INST": query_inst} + ) + retriever_prompt = Prompt(template="", components={"query": "{query}"}) + eval = DefaultEvaluator(args) + ranker = Ranker(max_turn=10, iterative=True, fixed_turn=2) + # ranker.set_eval('length', output = 'answer') + # ranker.new_eval('score', score , output = 'answer', docs = 'doc_cache', qa_pairs = 'qa_pairs') + ranker.new_eval("score", score, output="answer", docs="doc_cache") + # PIPELINE CONSTRUCTING + llm = LLM( + model=args.model, + prompt_maker=prompt, + self_prompt={"INST": llm_instruction, "shots": shots}, + max_turn=30, + auto_cite=True, + share_model_with=query_generator, + parallel=True, + ) + pipeline = Pipeline( + save_path=args.save_path, + llm=llm, + module=[ranker, query_generator], + head_prompt_maker=prompt, + evaluator=eval, + dataset=dataset, + ) + + retriever = Retriever( + prompt_maker=retriever_prompt, + pipeline=pipeline, + retrieve_by="bm25", + documents=documents, + topk=args.topk, + parallel = True + ) + query_generator.set_target(retriever, post_processing=segment_query) + query_generator.add_to_head("prev", sub=False) + retriever.set_target(llm, post_processing=segment) + llm.set_target(ranker, post_processing=make_as("answer")) + ranker.set_output(post_processing=lambda x: x["answer"], end=False) + + ranker.add_to_head( + "ans", sub=True, process=lambda text: one_paragraph(text["answer"]) + ) + #ranker.set_target(query_generator, post_processing=lambda x: {"ans": x["answer"]}) + pipeline.set_initial_module(query_generator) + pipeline.set_data_keys(["question"]) + + + graph = PipelineGraph(pipeline=pipeline) + # html = graph.generate_html(results='results.json') + # graph.visualize() + # print(html) + # with open('pipeline_.html','w') as file: + # file.write(html) + # RUN PIPELINE + # pipeline.run_on_dataset(datakeys=['question'], initial_module=query_generator) diff --git a/methods/self_RAG_prompt.py b/methods/self_RAG_prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..715e15cbad6a579f82554087370cedbec424361d --- /dev/null +++ b/methods/self_RAG_prompt.py @@ -0,0 +1,202 @@ +from citekit.cite_modules.LLM import LLM +from citekit.cite_modules.augment_model import ( + Retriever, + CitationSimplyfier, + Verifier, + Ranker, +) +from citekit.pipeline.pipeline import Pipeline, PIPELINE_OUTPUT, PIPELINE_DOC_CACHE +from citekit.prompt.prompt import Prompt, ALCEDocPrompt, DocPrompt, NewALCEVanillaPrompt +from citekit.Dataset.Dataset import PromptDataset +from citekit.evaluator.evaluator import ( + DefaultEvaluator, + compute_autoais, + test_compute_autoais, +) + +from citekit.utils.utils import ( + sentence, + one_paragraph, + each_make_as, + each_make_as, + make_as, + remove_citations, + compute_str_em, +) +import json +import argparse + + +def segment(i, text): + return [make_as("docs")(doc) for doc in text.split("\n") if doc] + + +if __name__ == "__main__": + + # SETTING ARGS + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", type=str, default="res.json", help="Path to the config file" + ) + parser.add_argument( + "--model", type=str, default="gpt-3.5-turbo", help="model name or path" + ) + parser.add_argument("--shots", type=int, default=2, help="number of shots") + parser.add_argument("--ndoc", type=int, default=5, help="number of docs") + parser.add_argument("--pr", action="store_true", help="use cite PR") + parser.add_argument("--rouge", action="store_true", help="use rouge") + parser.add_argument("--temp", type=float, default=0.5, help="temperature") + parser.add_argument("--qa", action="store_true", help="eval qa") + parser.add_argument("--mauve", action="store_true", help="eval mauve") + parser.add_argument("--length", type=bool, default=True, help="eval length") + parser.add_argument("--claims", action="store_true", help="eval claims") + parser.add_argument("--qampari", type=str, default=False, help="eval qampari") + parser.add_argument( + "--dataset", type=str, default="data/asqa_eval_gtr_top100.json", help="dataset" + ) + parser.add_argument( + "--demo", type=str, default="prompts/asqa_default.json", help="demo" + ) + parser.add_argument("--doctype", type=str, default="text", help="demo") + parser.add_argument("--data_num", type=int, default=1000, help="num of data") + parser.add_argument( + "--mode", + type=str, + default="text", + help="mode-granularity: text, extraction or summary", + ) + parser.add_argument("--k", type=float, default=1.5, help="coefficient of em") + parser.add_argument("--topk", type=int, default=2, help="topk") + args = parser.parse_args() + + def score(data): + pr = compute_autoais(data) + p = pr["citation_prec"] + r = pr["citation_rec"] + em = compute_str_em(data) + return p + r + args.k * em + return 1 + + # DATA LOADING + file_path = args.dataset + demo_path = args.demo + + with open(file_path, "r", encoding="utf-8") as file: + dataset = json.load(file) + with open(demo_path, "r", encoding="utf-8") as file: + demo = json.load(file) + data_num = min(args.data_num, len(dataset)) + + llm_instruction = demo["one_sentence_instruction"] + query_inst = demo["query_instruction"] + shots = "\n\n".join( + NewALCEVanillaPrompt().load_data( + [demo["demos"][1], demo["demos"][3]], + "question", + answer=lambda data: remove_citations( + sentence("first")(data["answer"])["first"] + ), + INST=lambda _: llm_instruction, + docs=lambda data: "".join( + ALCEDocPrompt().default_load_data(data["docs"][1:2]) + ), + ) + ) + + documents = [ + DocPrompt().load_data( + list(enumerate(data["docs"])), + Title=lambda data: data[1]["title"], + Passage=lambda data: data[1][args.mode], + ) + for data in dataset + ] + + dataset = PromptDataset( + dataset, + "question", + "answer", + "answers", + "qa_pairs", + "claims", + docs=lambda data: ALCEDocPrompt().default_load_data(data["docs"][: args.ndoc]), + )[:data_num] + + prompt = Prompt( + template="\nAnswer:", + components={ + "INST": "{INST}\n\n", + "shots": "{shots}\n", + "question": "Question:{question}\n\n", + "ans": "Prefix:{ans}\n\n", + "docs": "{docs}\n", + }, + ) + queryprompt = Prompt( + template="Please generate one query to help find relevent documents. If previous queries are provided, you shoud focus on an alternative perspective or subtopic different from the provided ones, enhancing diversity in retrieved documents. your query is:\n", + components={ + "question": "Given the original question: {question}\n", + "ans": "The context is: {ans}\n", + "prev": "\nPrevious queries:\n{prev}\n\n", + "INST": "{INST}\n\n", + }, + ) + + retriever_prompt = Prompt(template="", components={"query": "{query}"}) + + query_generator = LLM( + model=args.model, prompt_maker=queryprompt, self_prompt={"INST": query_inst} + ) + retriever_prompt = Prompt(template="", components={"query": "{query}"}) + eval = DefaultEvaluator(args) + ranker = Ranker(max_turn=2, iterative=True) + # ranker.set_eval('length', output = 'answer') + # ranker.new_eval('score', score , output = 'answer', docs = 'doc_cache', qa_pairs = 'qa_pairs') + ranker.new_eval("score", score, output="answer", docs="doc_cache") + # PIPELINE CONSTRUCTING + llm = LLM( + model=args.model, + prompt_maker=prompt, + self_prompt={"INST": llm_instruction, "shots": shots}, + max_turn=10, + auto_cite=True, + share_model_with=query_generator, + parallel=True, + ) + pipeline = Pipeline( + save_path=args.save_path, + llm=llm, + module=[ranker, query_generator], + head_prompt_maker=prompt, + evaluator=eval, + dataset=dataset, + ) + + retriever = Retriever( + prompt_maker=retriever_prompt, + pipeline=pipeline, + retrieve_by="bm25", + topk=args.topk, + documents=documents, + ) + query_generator.set_target(retriever, post_processing=make_as("query")) + query_generator.add_to_head("prev", sub=False) + retriever.set_target(llm, post_processing=segment) + llm.set_target(ranker, post_processing=make_as("answer")) + ranker.set_output(post_processing=lambda x: x["answer"], end=False) + + ranker.add_to_head( + "ans", sub=True, process=lambda text: one_paragraph(text["answer"]) + ) + ranker.set_target(query_generator, post_processing=lambda x: {"ans": x["answer"]}) + pipeline.set_initial_module(query_generator) + pipeline.set_data_keys(["question"]) + + # graph = PipelineGraph(pipeline=pipeline) + # html = graph.generate_html(results='results.json') + # graph.visualize() + # print(html) + # with open('pipeline_.html','w') as file: + # file.write(html) + # RUN PIPELINE + pipeline.run_on_dataset(datakeys=['question'],initial_module=query_generator) diff --git a/methods/self_RAG_simplify.py b/methods/self_RAG_simplify.py new file mode 100644 index 0000000000000000000000000000000000000000..ea4de88ca93bc638b6c9b0b33f80f95e49741053 --- /dev/null +++ b/methods/self_RAG_simplify.py @@ -0,0 +1,208 @@ +from citekit.cite_modules.LLM import LLM +from citekit.cite_modules.augment_model import ( + Retriever, + CitationSimplyfier, + Verifier, + Ranker, +) +from citekit.pipeline.pipeline import Pipeline, PIPELINE_OUTPUT, PIPELINE_DOC_CACHE +from citekit.prompt.prompt import Prompt, ALCEDocPrompt, DocPrompt, NewALCEVanillaPrompt +from citekit.Dataset.Dataset import PromptDataset +from citekit.evaluator.evaluator import ( + DefaultEvaluator, + compute_autoais, + test_compute_autoais, +) +from citekit.utils.utils import ( + sentence, + one_paragraph, + each_make_as, + each_make_as, + make_as, + remove_citations, + compute_str_em, +) +import json +import argparse +from parser import * + +def segment(i, text): + return [make_as("docs")(doc) for doc in text.split("\n") if doc] + +def segment_query(text): + return [make_as("query")(doc) for doc in text.split("\n") if doc] + +if __name__ == "__main__": + + # SETTING ARGS + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", type=str, default="resu.json", help="Path to the config file" + ) + parser.add_argument( + "--model", type=str, default="gpt-3.5-turbo", help="model name or path" + ) + parser.add_argument("--shots", type=int, default=2, help="number of shots") + parser.add_argument("--ndoc", type=int, default=5, help="number of docs") + parser.add_argument("--pr", action="store_true", help="use cite PR") + parser.add_argument("--rouge", action="store_true", help="use rouge") + parser.add_argument("--temp", type=float, default=0.5, help="temperature") + parser.add_argument("--qa", action="store_true", help="eval qa") + parser.add_argument("--mauve", action="store_true", help="eval mauve") + parser.add_argument("--length", type=bool, default=True, help="eval length") + parser.add_argument("--claims", action="store_true", help="eval claims") + parser.add_argument("--qampari", type=str, default=False, help="eval qampari") + parser.add_argument( + "--dataset", type=str, default="data/asqa_eval_gtr_top100.json", help="dataset" + ) + parser.add_argument( + "--demo", type=str, default="prompts/asqa_default.json", help="demo" + ) + parser.add_argument("--doctype", type=str, default="text", help="demo") + parser.add_argument("--data_num", type=int, default=1000, help="num of data") + parser.add_argument( + "--mode", + type=str, + default="text", + help="mode-granularity: text, extraction or summary", + ) + parser.add_argument("--k", type=float, default=1.5, help="coefficient of em") + parser.add_argument("--topk", type=int, default=2, help="topk") + args = parser.parse_args() + + def score(data): + pr = compute_autoais(data) + p = pr["citation_prec"] + r = pr["citation_rec"] + em = compute_str_em(data) + return p + r + args.k * em + return 1 + + # DATA LOADING + file_path = args.dataset + demo_path = args.demo + + with open(file_path, "r", encoding="utf-8") as file: + dataset = json.load(file) + with open(demo_path, "r", encoding="utf-8") as file: + demo = json.load(file) + data_num = min(args.data_num, len(dataset)) + + llm_instruction = demo["one_sentence_instruction"] + query_inst = demo["query_instruction"] + shots = "\n\n".join( + NewALCEVanillaPrompt().load_data( + [demo["demos"][1], demo["demos"][3]], + "question", + answer=lambda data: remove_citations( + sentence("first")(data["answer"])["first"] + ), + INST=lambda _: llm_instruction, + docs=lambda data: "".join( + ALCEDocPrompt().default_load_data(data["docs"][1:2]) + ), + ) + ) + + documents = [ + DocPrompt().load_data( + list(enumerate(data["docs"])), + Title=lambda data: data[1]["title"], + Passage=lambda data: data[1][args.mode], + ) + for data in dataset + ] + + dataset = PromptDataset( + dataset, + "question", + "answer", + "answers", + "qa_pairs", + "claims", + docs=lambda data: ALCEDocPrompt().default_load_data(data["docs"][: args.ndoc]), + )[:data_num] + + prompt = Prompt( + template="\nAnswer:", + components={ + "INST": "{INST}\n\n", + "shots": "{shots}\n", + "question": "Question:{question}\n\n", + "ans": "Prefix:{ans}\n\n", + "docs": "{docs}\n", + }, + ) + queryprompt = Prompt( + template="Please generate at most three queries regarding possible subquestions of the given question. Your queris should be diverse and informative in natual language, splited by a new line.\n", + components={ + "question": "Given the original question: {question}\n", + "ans": "The context is: {ans}\n", + "prev": "\nPrevious queries:\n{prev}\n\n", + "INST": "{INST}\n\n", + }, + ) + + retriever_prompt = Prompt(template="", components={"query": "{query}"}) + + query_generator = LLM( + model=args.model, prompt_maker=queryprompt, self_prompt={"INST": query_inst} + ) + retriever_prompt = Prompt(template="", components={"query": "{query}"}) + eval = DefaultEvaluator(args) + ranker = Ranker(max_turn=6, iterative=True, fixed_turn=2) + # ranker.set_eval('length', output = 'answer') + # ranker.new_eval('score', score , output = 'answer', docs = 'doc_cache', qa_pairs = 'qa_pairs') + ranker.new_eval("score", score, output="answer", docs="doc_cache") + # PIPELINE CONSTRUCTING + llm = LLM( + model=args.model, + prompt_maker=prompt, + self_prompt={"INST": llm_instruction, "shots": shots}, + max_turn=30, + auto_cite=True, + share_model_with=query_generator, + parallel=True, + ) + pipeline = Pipeline( + save_path=args.save_path, + llm=llm, + module=[ranker, query_generator], + head_prompt_maker=prompt, + evaluator=eval, + dataset=dataset, + ) + + retriever = Retriever( + prompt_maker=retriever_prompt, + pipeline=pipeline, + retrieve_by="bm25", + documents=documents, + topk=args.topk, + ) + query_generator.set_target(retriever, post_processing=segment_query) + query_generator.add_to_head("prev", sub=False) + retriever.set_target(llm, post_processing=segment) + llm.set_target(ranker, post_processing=make_as("answer")) + #ranker.set_output(post_processing=lambda x: x["answer"], end=False) + + ranker.add_to_head( + "ans", sub=True, process=lambda text: one_paragraph(text["answer"]) + ) + #ranker.set_target(query_generator, post_processing=lambda x: {"ans": x["answer"]}) + pipeline.set_initial_module(query_generator) + pipeline.set_data_keys(["question"]) + + simplifier = CitationSimplyfier() + ranker.set_target(simplifier) + simplifier.set_output(end=False) + + # graph = PipelineGraph(pipeline=pipeline) + # html = graph.generate_html_embed(results='old/res_attr.json') + # graph.visualize() + # print(html) + # with open('pipeline_.html','w') as file: + # file.write(html) + + raise KeyError + pipeline.run_on_dataset(datakeys=['question'], initial_module=query_generator) diff --git a/parser.py b/parser.py new file mode 100644 index 0000000000000000000000000000000000000000..3291b205575981a949c2db7ac30fa0648d45f66e --- /dev/null +++ b/parser.py @@ -0,0 +1,422 @@ +import hashlib +import copy +import json +from citekit.prompt.prompt import Prompt, ALCEDocPrompt,DocPrompt,NewALCEVanillaPrompt + +try: + example_results = json.load(open('res.json')) +except: + example_results = [] + + +def weight_check(module, target): + if module.model_type == 'verifier': + test_module = copy.deepcopy(module) + test_module.last_message = True + true_target = test_module.send() + if str(target) == str(true_target) or ('output' in str(target).lower() and 'output' in str(true_target).lower()): + return 1 + test_module.last_message = False + true_target = test_module.send() + if str(target) == str(true_target) or ('output' in str(target).lower() and 'output' in str(true_target).lower()): + return -1 + + return 0 + +def get_params(module): + import copy + componets = copy.deepcopy(module.prompt_maker.components if hasattr(module, 'prompt_maker') and hasattr(module.prompt_maker, 'components') else None) + if componets: + if isinstance(componets, tuple): + componets = componets[0] + componets.update(module.self_prompt) + destination = module.get_destinations() + if len(destination) == 1: + destination = str(destination[0]) + elif not destination: + destination = 'N/A' + else: + destination = 'Not Available for multiple destinations' + + if hasattr(module, 'iterative'): + mode = 'iterative' if module.iterative else ('parallel' if module.parallel else None) + else: + mode = None + + if hasattr(module, 'if_add_output_to_head') and hasattr(module, 'head_key'): + global_prompt = 'N/A' if module.if_add_output_to_head == False else module.head_key + else: + global_prompt = 'N/A' + + + params = { + 'type': module.model_type, + 'Model': getattr(module, 'model_name', None), + 'Mode': mode, + 'Max Turn': getattr(module, 'max_turn', None), + 'Destination': destination, + 'Prompt': module.prompt_maker.make_prompt(componets) if hasattr(module,'prompt_maker') and module.prompt_maker else None, + 'Global Prompt': global_prompt, + } + print(params) + non_empty_params = {} + for k, v in params.items(): + if v: + non_empty_params[k] = v + return non_empty_params + +class PipelineGraph: + + def __init__(self, pipeline = None): + self.pipeline = pipeline + self.nodes = {'input': {}, 'output': {}} + self.edges = {} # 邻接表：{from_node: {to_node: weight}} + self.node_count = 0 + if pipeline: + self.load_pipeline(pipeline) + print('FINDING ATTR:', [str(module) for module in pipeline.module]) + + def get_auto_node_name(self): + """生成自动节点名。""" + self.node_count += 1 + return f"Node{self.node_count}" + + def update(self): + self.__init__(pipeline=self.pipeline) + + def load_pipeline(self, pipeline): + """从现有Pipeline对象加载图。""" + initial_module = pipeline.get_initial_module() + processed_nodes = [] + if not initial_module: + initial_module = pipeline.llm + self.add_node(name = str(initial_module), + params = get_params(module=initial_module)) + self.add_edge(from_node='input', to_node=str(initial_module), weight=0) + processed_nodes.append(initial_module) + subnodes = initial_module.get_destinations() + if not subnodes: + self.add_edge(from_node=str(initial_module), to_node='output', weight=0) + return + # print([str(subnode) for subnode in subnodes]) + for subnode in subnodes: + weight = weight_check(initial_module, subnode) + self.load_node(subnode, processed_nodes=processed_nodes) + self.add_edge(from_node=str(initial_module), to_node=str(subnode), weight=0) + if hasattr(initial_module,'output_cond') and initial_module.output_cond: + weight = weight_check(initial_module, 'output') + self.add_edge(from_node=str(initial_module), to_node='output', weight=weight) + + + def load_node(self, module, processed_nodes = None): + if module in processed_nodes: + return + self.add_node(name = str(module), + params = get_params(module=module)) + processed_nodes.append(module) + subnodes = module.get_destinations() + if not subnodes: + self.add_edge(from_node=str(module), to_node='output', weight=0) + return + for subnode in subnodes: + self.load_node(subnode, processed_nodes) + weight = weight_check(module, subnode) + self.add_edge(from_node=str(module), to_node=str(subnode), weight=weight) + if hasattr(module,'output_cond') and module.output_cond: + weight = weight_check(module, 'output') + self.add_edge(from_node=str(module), to_node='output', weight=0) + + + def export(self): + """将图结构导出为字典格式，包含节点和边的信息。""" + nodes_export = {name: params.copy() for name, params in self.nodes.items()} + edges_export = [] + for from_node, connections in self.edges.items(): + for to_node, weight in connections.items(): + edges_export.append({ + 'from': from_node, + 'to': to_node, + 'weight': weight + }) + return { + 'nodes': nodes_export, + 'edges': edges_export + } + + @classmethod + def import_from_dict(cls, data): + """从字典数据创建新的PipelineGraph实例。""" + graph = cls() # 创建新实例，此时包含默认的input/output节点 + # 清空默认节点（假设导入数据包含完整节点信息） + graph.nodes = {} + # 导入节点 + for name, params in data['nodes'].items(): + graph.add_node(name, params) + # 导入边 + for edge in data['edges']: + graph.add_edge(edge['from'], edge['to'], edge['weight']) + return graph + + def add_node(self, name, params=None): + """添加或更新节点，参数可选。""" + if params is None: + params = {} + self.nodes[name] = params.copy() + + def remove_node(self, name): + """删除节点，自动移除相关边。不能删除input/output节点。""" + if name in ['input', 'output']: + raise ValueError("Cannot remove 'input' or 'output' nodes.") + if name not in self.nodes: + return + del self.nodes[name] + # 移除该节点作为起点的边 + if name in self.edges: + del self.edges[name] + # 移除其他节点到该节点的边 + for from_node in list(self.edges.keys()): + if name in self.edges[from_node]: + del self.edges[from_node][name] + # 若该节点无其他出边，删除空字典以保持整洁 + if not self.edges[from_node]: + del self.edges[from_node] + + def add_edge(self, from_node, to_node, weight): + """添加或更新边，权重必须是0、-1或+1。""" + if weight not in {0, -1, 1}: + raise ValueError("Weight must be 0, -1, or 1.") + if from_node not in self.nodes: + raise KeyError(f"Node '{from_node}' does not exist.") + if to_node not in self.nodes: + raise KeyError(f"Node '{to_node}' does not exist.") + if from_node not in self.edges: + self.edges[from_node] = {} + self.edges[from_node][to_node] = weight + + def remove_edge(self, from_node, to_node): + """删除指定边。""" + if from_node in self.edges and to_node in self.edges[from_node]: + del self.edges[from_node][to_node] + # 清理空字典 + if not self.edges[from_node]: + del self.edges[from_node] + + def get_nodes(self): + """返回所有节点名。""" + return list(self.nodes.keys()) + + def get_edges(self): + """返回所有边的列表，形式为(from, to, weight)。""" + edges = [] + for from_node in self.edges: + for to_node, weight in self.edges[from_node].items(): + edges.append((from_node, to_node, weight)) + return edges + + def update_node_params(self, name, params): + """更新节点的参数表。""" + if name not in self.nodes: + raise KeyError(f"Node '{name}' does not exist.") + self.nodes[name].update(params.copy()) + + def visualize(self, use_graphviz=True): + """ + 可视化图结构： + - 默认尝试用graphviz生成矢量图（需安装graphviz） + - 若未安装graphviz，则用文本模式输出 + """ + # 尝试用graphviz生成专业图表 + if use_graphviz: + try: + import graphviz + except ImportError: + print("Graphviz未安装，切换到文本模式。安装命令：pip install graphviz") + return self._visualize_text() + + # 创建有向图，设置排版方向为从左到右 + dot = graphviz.Digraph(comment='Pipeline Graph', graph_attr={'rankdir': 'LR'}) + + # 添加所有节点（包含参数信息） + for node_name in self.nodes: + params = self.nodes[node_name] + # 将参数字典转换为易读字符串（例如 "size=2, color=red"） + params_str = ", ".join([f"{k}={v}" for k, v in params.items()]) + + # 特殊样式标记input/output节点 + if node_name in ('input', 'output'): + dot.node( + node_name, + label=f"{node_name}\n({params_str})" if params_str else node_name, + shape='doublecircle', # 双圆表示端点 + style='filled', # 填充颜色 + fillcolor='#e6f3ff' # 浅蓝色背景 + ) + else: + dot.node( + node_name, + label=f"{node_name}\n({params_str})" if params_str else node_name, + shape='box', # 方框表示普通节点 + style='rounded,filled',# 圆角填充 + fillcolor='#f0f0f0' # 浅灰色背景 + ) + + # 添加所有带权重的边 + for from_node, connections in self.edges.items(): + for to_node, weight in connections.items(): + dot.edge( + from_node, + to_node, + label=str(weight), # 显示权重 + color='#666666', # 边颜色 + fontcolor='#ff5555' # 权重文字颜色 + ) + + # 生成并自动打开可视化文件（Mac默认用Preview打开PDF） + dot.render('pipeline_graph', view=True, cleanup=True) + return + + # 文本模式回退方案 + self._visualize_text() + + def _visualize_text(self): + """纯文本模式的可视化（备用方案）""" + print("="*40 + "\nPipeline Graph 文本视图\n" + "="*40) + + # 列出所有节点及其参数 + print("\n[节点列表]") + for node, params in self.nodes.items(): + param_desc = " ".join([f"{k}={v}" for k, v in params.items()]) + print(f"· {node:10} {param_desc}") + + # 列出所有边及其权重 + print("\n[边列表]") + if not self.edges: + print("（暂无边连接）") + else: + for from_node, connections in self.edges.items(): + for to_node, weight in connections.items(): + arrow = { + 1: "──(+)──>", # +1权重用(+)表示 + -1: "──(-)──>", # -1权重用(-)表示 + 0: "───────>" # 0权重无特殊标记 + }[weight] + print(f"{from_node:10} {arrow} {to_node}") + print("="*40 + "\n") + + + + def generate_html(self, results = example_results): + assert isinstance(results, str) + + import json + result_path = results + results = json.load(open(results)) + """生成交互式可视化HTML，使用D3.js力导向图布局""" + nodes_data = [] + for name in self.nodes: + params = self.nodes[name] + nodes_data.append({ + "id": name, + "type": params.get("type", name), + "params": params + }) + + edges_data = [] + for from_node, to_nodes in self.edges.items(): + for to_node, weight in to_nodes.items(): + edges_data.append({ + "source": from_node, + "target": to_node, + "weight": weight + }) + + nodes_js = json.dumps(nodes_data) + edges_js = json.dumps(edges_data) + + with open('htmls/html_templates/pipeline_template.txt', 'r', encoding='utf-8') as f: + template = f.read() + + template = template.replace('', nodes_js).replace('', edges_js).replace('', result_path).replace('', "".join(f'' for i in range(len(results)))) + return template + + + def get_nodes(self): + nodes_data = [] + for name in self.nodes: + params = self.nodes[name] + nodes_data.append({ + "id": name, + "type": params.get("type", name), + "params": params + }) + return json.dumps(nodes_data) + + def get_edges(self): + edges_data = [] + for from_node, to_nodes in self.edges.items(): + for to_node, weight in to_nodes.items(): + edges_data.append({ + "source": from_node, + "target": to_node, + "weight": weight + }) + return json.dumps(edges_data) + + def get_json(self): + return { + 'nodes': self.get_nodes(), + 'edges': self.get_edges() + } + + def generate_html_embed(self, results = example_results): + assert isinstance(results, str) or isinstance(results, list) + + import json + result_path = results + if isinstance(results, str): + results = json.load(open(results)) + """生成交互式可视化HTML，使用D3.js力导向图布局""" + nodes_data = [] + for name in self.nodes: + params = self.nodes[name] + nodes_data.append({ + "id": name, + "type": params.get("type", name), + "params": params + }) + + edges_data = [] + for from_node, to_nodes in self.edges.items(): + for to_node, weight in to_nodes.items(): + edges_data.append({ + "source": from_node, + "target": to_node, + "weight": weight + }) + + nodes_js = json.dumps(nodes_data) + edges_js = json.dumps(edges_data) + + with open('htmls/html_templates/pipeline_template_emb.txt', 'r', encoding='utf-8') as f: + template = f.read() + + template = template.replace('', nodes_js).replace('', edges_js).replace('', json.dumps(results)).replace('', "".join(f'' for i in range(len(results)))) + return template + +if __name__ == '__main__': + # 初始化图 + graph = PipelineGraph() + + # 添加节点 + graph.add_node('processor', {'type': 'transform', 'rate': 0.8}) + graph.add_node('validator', {'threshold': 0.5}) + + # 添加边 + graph.add_edge('input', 'processor', 1) + graph.add_edge('processor', 'validator', -1) + graph.add_edge('validator', 'output', 0) + + # 更新参数 + graph.update_node_params('processor', {'rate': 1.0}) + + graph.visualize() \ No newline at end of file diff --git a/prompts/AnG.json b/prompts/AnG.json new file mode 100644 index 0000000000000000000000000000000000000000..51482f65c549b262600fe284ec124a5d679aa2fb --- /dev/null +++ b/prompts/AnG.json @@ -0,0 +1,28 @@ +{ + "ANG":{ + "selection_instruction":"In this task, you are presented with several documents, which need to be summarized. As an intermediate step, you need to identify salient content within the documents. For each document, copy verbatim the salient spans, and use as a delimiter between each consecutive span. IMPORTANT: The output must be of the format Document []: -delimited consecutive salient spans. IMPORTANT: Each salient content must be a single consecutive verbatim span from the corresponding passages. IMPORTANT: make sure the total number of copied words (from all documents) is around 200 words, and not more than 900.", + "selection_shot":"Question: Which is the most rainy place on earth?\n\nDocument [1]: The United States on Wednesday warned US citizens of a possible spike in \"terrorist activity\" in Colombia in the coming week as the left-wing Revolutionary Armed Forces of Colombia (FARC) mark the one-year anniversary of the death of one of their commanders ... \nDocument [2]: Colombia's hardline government said Friday it is willing to meet with members of the country's main leftist rebel group in an unprecedented offer aimed at freeing dozens of rebel hostages, including three Americans. Before any face-to-face meeting can take place, however, the rebels must agree to swap 15 hostages for 15 rebels jailed on minor charges, said Peace Commissioner Luis Carlos Restrepo ...\n... \n\nAnswer: \nDocument [1]: The United States on Wednesday warned US citizens mark the one-year anniversary of the death of one of their commanders ... \nDocument [2]: Colombia's hardline government said it is willing to meet with members of the country's main leftist rebel group ... \n...", + "clustering_instruction":"In this task, you are presented with several passages, where some parts are \"highlighted\" (namely, there are and tokens before and after each such span). The goal is to fuse all those highlights into a single summary. As an intermediate step, you need to cluster highlights that can be merged into a sentence (namely, each cluster will be later merged into one sentence). Make sure the clusters are in the same order you would then write the corresponding summary sentences. IMORTANT: make sure there are at least 7 clusters, and no more than 5-6 highlights per cluster. IMPORTANT: The output must be of the format [\"cluster\":[comma-delimited highlights indices]]", + "clustering_shot":"Question: Which is the most rainy place on earth?\n\nDocument [1]: The United States on Wednesday warned US citizens of a possible spike in \"terrorist activity\" in Colombia in the coming week as the left-wing Revolutionary Armed Forces of Colombia (FARC) mark the one-year anniversary of the death of one of their commanders ... \nDocument [2]: Colombia's hardline government said Friday it is willing to meet with members of the country's main leftist rebel group in an unprecedented offer aimed at freeing dozens of rebel hostages, including three Americans. Before any face-to-face meeting can take place, however, the rebels must agree to swap 15 hostages for 15 rebels jailed on minor charges, said Peace Commissioner Luis Carlos Restrepo ... \n... \n\nThe highlighted spans are: \nDocument [1]: \n1. The United States on Wednesday warned US citizens \n2. mark the one-year anniversary of the death of one of their commanders \n... \nDocument [2]: \n7. Colombia's hardline government said 14 \n8. it is willing to meet with members of the country's main leftist rebel group ... \n\nAnswer: \nThe highlighted spans are clustered as follows: \n[{\"cluster\":[1,2,5]}, {\"cluster\":[7,8]} ...]", + "gen_instruction":"In this task, you are presented with several passages, where some parts are \"highlighted\" (namely, there are and tokens before and after each such span). You are also presented with a prefix of a paragraph. You job is to generate the next sentence in the paragraph, that covers all and only the \"highlighted\" spans. Make sure it connects well with the prefix, and that it covers all and only the \"highlighted\" spans.", + "gen_shot":"Question: Which is the most rainy place on earth?\n\nDocument [1]: The United States on Wednesday warned US citizens of a possible spike in \"terrorist activity\" in Colombia in the coming week as the left-wing Revolutionary Armed Forces of Colombia (FARC) mark the one-year anniversary of the death of one of their commanders ... \nDocument [2]: Colombia's hardline government said Friday it is willing to meet with members of the country's main leftist rebel group in an unprecedented offer aimed at freeing dozens of rebel hostages, including three Americans ... \n... \n\nPrefix: Colombia's strict government agreed to meet with the Country's main leftist rebel group. \n\nThe highlighted spans are: \nDocument [1]: \n1. The United States on Wednesday warned US citizens \n2. mark the one-year anniversary of the death of a terrorist commanders, American citizens recieved a warning from their government \n... \n\nAnswer: \nThe next sentence is: Following the one-year anniversary of the death of one of the group's commanders, American citizens received a warning from their government on Wednesday." + +}, +"AnG":{ + "selection_instruction":"In this task, you are presented with several documents, which need to be summarized. As an intermediate step, you need to identify salient content within the documents. For each document, copy verbatim the salient spans, and use as a delimiter between each consecutive span. IMPORTANT: The output must be of the format Document []: -delimited consecutive salient spans. IMPORTANT: Each salient content must be a single consecutive verbatim span from the corresponding passages. IMPORTANT: make sure the total number of copied words (from all documents) is around 200 words, and not more than 900.", + "selection_shot":"Document [1]: Cherrapunji Cherrapunji ( with the native name Sohra being more commonly used, and can also be spelled Cherrapunjee or Cherrapunji) is a subdivisional town in the East Khasi Hills district in the Indian state of Meghalaya. It is the traditional capital of aNongkhlaw \"hima\" (Khasi tribal chieftainship constituting a petty state), both known as Sohra or Churra. Cherrapunji has often been credited as being the wettest place on Earth, but for now nearby Mawsynram currently holds that distinction. Cherrapunji still holds the all-time record for the most rainfall in a calendar month for July 1861 and most rain in a year from August 1860 to July 1861, however: it received in\" \nDocument [2]: Radio relay station known as Akashvani Cherrapunji. It broadcasts on FM frequencies. Cherrapunji Cherrapunji (; with the native name Sohra being more commonly used, and can also be spelled Cherrapunjee or Cherrapunji) is a subdivisional town in the East Khasi Hills district in the Indian state of Meghalaya. It is the traditional capital of aNongkhlaw \"hima\" (Khasi tribal chieftainship constituting a petty state), both known as Sohra or Churra. Cherrapunji has often been credited as being the wettest place on Earth, but for now nearby Mawsynram currently holds that distinction. Cherrapunji still holds the all-time record for the most rainfall\" \nDocument [3]: \"Mawsynram Mawsynram () is a village in the East Khasi Hills district of Meghalaya state in north-eastern India, 65 kilometres from Shillong. Mawsynram receives one of the highest rainfalls in India. It is reportedly the wettest place on Earth, with an average annual rainfall of 11,872 mm, but that claim is disputed by Lloró, Colombia, which reported an average yearly rainfall of 12,717 mm between 1952 and 1989 and López de Micay, also in Colombia, which reported an annual 12,892 mm per year between 1960 and 2012. According to the \"Guinness Book of World Records\", Mawsynram received of rainfall in 1985. Mawsynram is located at 25° 18′\" \n\nAnswer: \nDocument [1]: Cherrapunji has often been credited as being the wettest place on Earth still holds the all-time record for the most rainfall in a calendar month for July 1861 and most rain in a year from August 1860 to July 1861 \nDocument [2]: Cherrapunji has often been credited as being the wettest place on Earthstill holds the all-time record for the most rainfall \nDocument [3]: Mawsynram receives one of the highest rainfalls in India but that claim is disputed by Lloró, Colombia, which reported an average yearly rainfall of 12,717 mm between 1952 and 1989 López de Micay, also in Colombia, which reported an annual 12,892 mm per year between 1960 and 2012. ", + "clustering_instruction":"In this task, you are presented with several passages, where some parts are \"highlighted\" (namely, there are and tokens before and after each such span). The goal is to fuse all those highlights into a single summary. As an intermediate step, you need to cluster highlights that can be merged into a sentence (namely, each cluster will be later merged into one sentence). Make sure the clusters are in the same order you would then write the corresponding summary sentences. IMORTANT: make sure there are at least 7 clusters, and no more than 5-6 highlights per cluster. IMPORTANT: The output must be of the format [\"cluster\":[comma-delimited highlights indices]]", + "clustering_shot":"Document [1]: Cherrapunji Cherrapunji ( with the native name Sohra being more commonly used, and can also be spelled Cherrapunjee or Cherrapunji) is a subdivisional town in the East Khasi Hills district in the Indian state of Meghalaya. It is the traditional capital of aNongkhlaw \"hima\" (Khasi tribal chieftainship constituting a petty state), both known as Sohra or Churra. Cherrapunji has often been credited as being the wettest place on Earth, but for now nearby Mawsynram currently holds that distinction. Cherrapunji still holds the all-time record for the most rainfall in a calendar month for July 1861 and most rain in a year from August 1860 to July 1861, however: it received in\" \nDocument [2]: Radio relay station known as Akashvani Cherrapunji. It broadcasts on FM frequencies. Cherrapunji Cherrapunji (; with the native name Sohra being more commonly used, and can also be spelled Cherrapunjee or Cherrapunji) is a subdivisional town in the East Khasi Hills district in the Indian state of Meghalaya. It is the traditional capital of aNongkhlaw \"hima\" (Khasi tribal chieftainship constituting a petty state), both known as Sohra or Churra. Cherrapunji has often been credited as being the wettest place on Earth, but for now nearby Mawsynram currently holds that distinction. Cherrapunji still holds the all-time record for the most rainfall\" \nDocument [3]: \"Mawsynram Mawsynram () is a village in the East Khasi Hills district of Meghalaya state in north-eastern India, 65 kilometres from Shillong. Mawsynram receives one of the highest rainfalls in India. It is reportedly the wettest place on Earth, with an average annual rainfall of 11,872 mm, but that claim is disputed by Lloró, Colombia, which reported an average yearly rainfall of 12,717 mm between 1952 and 1989 and López de Micay, also in Colombia, which reported an annual 12,892 mm per year between 1960 and 2012. According to the \"Guinness Book of World Records\", Mawsynram received of rainfall in 1985. Mawsynram is located at 25° 18′\" \n\nThe highlighted spans are: \nDocument [1]: 1. Cherrapunji has often been credited as being the wettest place on Earth \n2. still holds the all-time record for the most rainfall in a calendar month for July 1861 and most rain in a year from August 1860 to July 1861 \nDocument [2]: \n3. Cherrapunji has often been credited as being the wettest place on Earth \n4. still holds the all-time record for the most rainfall \nDocument [3]: \n5. Mawsynram receives one of the highest rainfalls in India \n6. but that claim is disputed by Lloró, Colombia, which reported an average yearly rainfall of 12,717 mm between 1952 and 1989 \n7. López de Micay, also in Colombia, which reported an annual 12,892 mm per year between 1960 and 2012. \n\nAnswer: \nThe highlighted spans are clustered as follows: \n[{\"cluster\":[6,7]}, {\"cluster\":[5]},{\"cluster\":[1,2]}]", + "gen_instruction":"In this task, you are presented with several passages, where some parts are \"highlighted\" (namely, there are and tokens before and after each such span). You may also be presented with a prefix of the answer. You job is to generate the next sentence of the answer, that covers all and only the \"highlighted\" spans. Make sure it connects well with the prefix(if eixists), and that it covers all and only the \"highlighted\" spans. Always cite for any factual claim. When citing several search results, use [1][2][3]. Cite at least one document and at most three documents in each sentence. If multiple documents support the sentence, only cite a minimum sufficient subset of the documents.", + "gen_shot":"Document [1]: Cherrapunji Cherrapunji ( with the native name Sohra being more commonly used, and can also be spelled Cherrapunjee or Cherrapunji) is a subdivisional town in the East Khasi Hills district in the Indian state of Meghalaya. It is the traditional capital of aNongkhlaw \"hima\" (Khasi tribal chieftainship constituting a petty state), both known as Sohra or Churra. Cherrapunji has often been credited as being the wettest place on Earth, but for now nearby Mawsynram currently holds that distinction. Cherrapunji still holds the all-time record for the most rainfall in a calendar month for July 1861 and most rain in a year from August 1860 to July 1861, however: it received in\" \nDocument [2]: Radio relay station known as Akashvani Cherrapunji. It broadcasts on FM frequencies. Cherrapunji Cherrapunji (; with the native name Sohra being more commonly used, and can also be spelled Cherrapunjee or Cherrapunji) is a subdivisional town in the East Khasi Hills district in the Indian state of Meghalaya. It is the traditional capital of aNongkhlaw \"hima\" (Khasi tribal chieftainship constituting a petty state), both known as Sohra or Churra. Cherrapunji has often been credited as being the wettest place on Earth, but for now nearby Mawsynram currently holds that distinction. Cherrapunji still holds the all-time record for the most rainfall\" \nDocument [3]: \"Mawsynram Mawsynram () is a village in the East Khasi Hills district of Meghalaya state in north-eastern India, 65 kilometres from Shillong. Mawsynram receives one of the highest rainfalls in India. It is reportedly the wettest place on Earth, with an average annual rainfall of 11,872 mm, but that claim is disputed by Lloró, Colombia, which reported an average yearly rainfall of 12,717 mm between 1952 and 1989 and López de Micay, also in Colombia, which reported an annual 12,892 mm per year between 1960 and 2012. According to the \"Guinness Book of World Records\", Mawsynram received of rainfall in 1985. Mawsynram is located at 25° 18′\" \n\nPrefix: Several places on Earth claim to be the most rainy, such as Lloró, Colombia, which reported an average annual rainfall of 12,717 mm between 1952 and 1989, and López de Micay, Colombia, which reported an annual 12,892 mm between 1960 and 2012 [3]. \n\nThe highlighted spans are: \nDocument [3]: \n5. Mawsynram receives one of the highest rainfalls in India \n\nAnswer: \nThe next sentence is: \nHowever, the official record is held by Mawsynram, India with an average annual rainfall of 11,872 mm [3]." + +}, +"plan":{ + "shot1":"In this task, you should write no more than four subquestions according to the given documents and a question. Ensure that each subquestion can be respond by reading the documents, and is related to the question. Seperate all the questions with only space, to write then in only one paragraph.\n\nQuestion: Who is the original artist of sound of silence?\n\nDocument [1]: Sounds of Silence is the second studio album by Simon & Garfunkel, released on January 17, 1966. The album's title is a slight modification of the title of the duo's first major hit, \"The Sound of Silence\", which originally was released as \"The Sounds of Silence\". The song had earlier been released in an acoustic version on the album \"Wednesday Morning, 3 A.M.\", and later on the soundtrack to the movie \"The Graduate\". \nDocument [2]: Sound of Silence\" is a song performed by Australian recording artist Dami Im. Written by Anthony Egizii and David Musumeci of DNA Songs, it is best known as Australia's entry at the Eurovision Song Contest 2016 which was held in Stockholm, Sweden, where it finished 2nd, receiving a total of 511 points. \nDocument [3]: Simon & Garfunkel Simon & Garfunkel were an American folk rock duo consisting of singer-songwriter Paul Simon and singer Art Garfunkel. They were one of the bestselling music groups of the 1960s and became counterculture icons of the decade's social revolution, alongside artists such as the Beatles, the Beach Boys, and Bob Dylan. Their biggest hits\u2014including \"The Sound of Silence\" (1964), \"Mrs. Robinson\" (1968)\n\nsubquestions:\nWho is the original artist of sound of silence, the album? Who is the original artist of sound of silence, the song, released in 2016?Who is the original artist of sound of silence, the song, released in 1964?\n\n", + "shot2":"In this task, you should write no more than four subquestions according to the given documents and a question. Ensure that each subquestion can be respond by reading the documents, and is related to the question. Seperate all the questions with only space, to write then in only one paragraph.\n\nQuestion: When was the first apple i phone made?\n\nDocument [1]: On January 9, 2007, Steve Jobs announced the first iPhone at the Macworld convention, receiving substantial media attention. Jobs announced that the first iPhone would be released later that year. On June 29, 2007, the first iPhone was released.\nDocument [2]: The history of the iPhone began with a request from Apple Inc. CEO Steve Jobs to the company's engineers, asking them to investigate the use of touchscreen devices and tablet computers (which later came to fruition with the iPad). Many have noted the device's similarities to Apple's previous touch-screen portable device, the Newton MessagePad. Like the MessagePad, the iPhone is nearly all screen. Its form factor is credited to Apple's Chief Design Officer, Jonathan Ive. The iPhone beta was created in 2004 to test the commands of the Apple team, and while technically may have first iPhone ever created, it was never released to the public, so this beta was not considered the true first iPhone.\nDocument [3]: just a computer company anymore. At the same address, Jobs revealed a product that would revolutionize an industry in which Apple had never previously competed: the Apple iPhone. The iPhone combined Apple's first widescreen iPod with the world's first mobile device boasting visual voicemail, and an internet communicator able to run a fully functional version of Apple's web browser, Safari, on the then-named (later renamed iOS). The first version of the iPhone became publicly available on June 29, 2007 in selected countries/markets. It was another 12 months before the iPhone 3G became available on July 11, 2008.\n\nsubquestions:\nWhen was the first apple i phone released? When was the first apple i phone beta made? When was the first Apple iPhone publicly available?\n\n", + "INST":"In this task, you should write no more than four subquestions according to the given documents and a question. Ensure that each subquestion can be respond by reading the documents, and is related to the question. Seperate all the questions with only space, to write then in only one paragraph.", + "answer_shot_1":"Instruction: Write an accurate, engaging, and concise answer for the given question using only the provided search results (some of which might be irrelevant) and cite them properly, by answering all the subquestions. Each subquestion should be answered. Use an unbiased and journalistic tone. Always cite for any factual claim. When citing several search results, use [1][2][3]. Cite at least one document and at most three documents in each sentence. If multiple documents support the sentence, only cite a minimum sufficient subset of the documents.\n\nQuestion: Who is the original artist of sound of silence?\n\nDocument [1]: Sounds of Silence is the second studio album by Simon & Garfunkel, released on January 17, 1966. The album's title is a slight modification of the title of the duo's first major hit, \"The Sound of Silence\", which originally was released as \"The Sounds of Silence\". The song had earlier been released in an acoustic version on the album \"Wednesday Morning, 3 A.M.\", and later on the soundtrack to the movie \"The Graduate\". \nDocument [2]: Sound of Silence\" is a song performed by Australian recording artist Dami Im. Written by Anthony Egizii and David Musumeci of DNA Songs, it is best known as Australia's entry at the Eurovision Song Contest 2016 which was held in Stockholm, Sweden, where it finished 2nd, receiving a total of 511 points. \nDocument [3]: Simon & Garfunkel Simon & Garfunkel were an American folk rock duo consisting of singer-songwriter Paul Simon and singer Art Garfunkel. They were one of the bestselling music groups of the 1960s and became counterculture icons of the decade's social revolution, alongside artists such as the Beatles, the Beach Boys, and Bob Dylan. Their biggest hits\u2014including \"The Sound of Silence\" (1964), \"Mrs. Robinson\" (1968)\n\nsubquestions:\nWho is the original artist of sound of silence, the album? Who is the original artist of sound of silence, the song, released in 2016?Who is the original artist of sound of silence, the song, released in 1964?\n\nAnswer: \nThe original artist of the song sound of silence released in 1966 is Paul Simon and Art Garfunkel. The song had earlier been released in an acoustic version on the album \"Wednesday Morning, 3 A.M ,\" which had been produced in 1964 [1]. In 2016, Australian recording artist Dami Im recorded a different song by the same name [2].\n\n", + "answer_shot_2":"Instruction: Write an accurate, engaging, and concise answer for the given question using only the provided search results (some of which might be irrelevant) and cite them properly, by answering all the subquestions. Each subquestion should be answered. Use an unbiased and journalistic tone. Always cite for any factual claim. When citing several search results, use [1][2][3]. Cite at least one document and at most three documents in each sentence. If multiple documents support the sentence, only cite a minimum sufficient subset of the documents.\n\nQuestion: When was the first apple i phone made?\n\nDocument [1]: On January 9, 2007, Steve Jobs announced the first iPhone at the Macworld convention, receiving substantial media attention. Jobs announced that the first iPhone would be released later that year. On June 29, 2007, the first iPhone was released.\nDocument [2]: The history of the iPhone began with a request from Apple Inc. CEO Steve Jobs to the company's engineers, asking them to investigate the use of touchscreen devices and tablet computers (which later came to fruition with the iPad). Many have noted the device's similarities to Apple's previous touch-screen portable device, the Newton MessagePad. Like the MessagePad, the iPhone is nearly all screen. Its form factor is credited to Apple's Chief Design Officer, Jonathan Ive. The iPhone beta was created in 2004 to test the commands of the Apple team, and while technically may have first iPhone ever created, it was never released to the public, so this beta was not considered the true first iPhone.\nDocument [3]: just a computer company anymore. At the same address, Jobs revealed a product that would revolutionize an industry in which Apple had never previously competed: the Apple iPhone. The iPhone combined Apple's first widescreen iPod with the world's first mobile device boasting visual voicemail, and an internet communicator able to run a fully functional version of Apple's web browser, Safari, on the then-named (later renamed iOS). The first version of the iPhone became publicly available on June 29, 2007 in selected countries/markets. It was another 12 months before the iPhone 3G became available on July 11, 2008.\n\nsubquestions:\nWhen was the first apple i phone released? When was the first apple i phone beta made? When was the first Apple iPhone publicly available?\n\nAnswer: \nThe iPhone beta was created in 2004 to test the commands of the Apple team, and while technically may have been the first iPhone ever created, it was never released to the public, so this beta was not considered the true first iPhone [2]. After that, the first iPhone was officially launched and made accessible to the public on June 29, 2007 [1].\n\n", + "answer_inst":"Instruction: Write an accurate, engaging, and concise answer for the given question using only the provided search results (some of which might be irrelevant) and cite them properly, by answering all the subquestions. Each subquestion should be answered. Use an unbiased and journalistic tone. Always cite for any factual claim. When citing several search results, use [1][2][3]. Cite at least one document and at most three documents in each sentence. If multiple documents support the sentence, only cite a minimum sufficient subset of the documents." +} +} \ No newline at end of file diff --git a/prompts/asqa.json b/prompts/asqa.json new file mode 100644 index 0000000000000000000000000000000000000000..a4ccd0706ec15f89f3c94e7c1476e5243ced1a5d --- /dev/null +++ b/prompts/asqa.json @@ -0,0 +1,9 @@ +{ + "demo_1":"Instruction: Write an accurate, engaging, and concise answer for the given question using only the provided search results (some of which might be irrelevant) and cite them properly. Use an unbiased and journalistic tone. Always cite for any factual claim. When citing several search results, use [1][2][3]. Cite at least one document and at most three documents in each sentence. If multiple documents support the sentence, only cite a minimum sufficient subset of the documents.\n\nQuestion:When did the us break away from england?\n\nDocument [1](Title:United States withdrawal from Saudi Arabia): United States withdrawal from Saudi Arabia Beginning during Operation Desert Shield in August 1990, while preparing for the Gulf War, the United States sent a large troop contingent to Saudi Arabia. After the war, remnant troops, primarily U.S. Air Force personnel, augmented by a smaller number of coordinating and training personnel from the U.S. Navy, U.S. Army and U.S. Marine Corps remained in Saudi Arabia under the aegis of Joint Task Force Southwest Asia (JTF-SWA), as part of Operation Southern Watch (OSW). The United Kingdom and France also maintained a small contingent of Royal Air Force and French Air Force\nDocument [2](Title:Decolonization of the Americas): and France has fully \"integrated\" most of its former colonies as fully constituent \"departments\" of France. The United States of America declared independence from Great Britain on July 2, 1776 (although the event is now commemorated on July 4, the date when the Declaration of Independence was officially adopted by Congress), in so doing becoming the first independent, foreign-recognized nation in the Americas and the first European colonial entity to break from its mother country. Britain formally acknowledged American independence in 1783 after its defeat in the American Revolutionary War. Although initially occupying only the land east of the Mississippi\nDocument [3](Title:American Revolution): second British army at Yorktown in the fall of 1781, effectively ending the war. The Treaty of Paris was signed September 3, 1783, formally ending the conflict and confirming the new nation's complete separation from the British Empire. The United States took possession of nearly all the territory east of the Mississippi River and south of the Great Lakes, with the British retaining control of Canada and Spain taking Florida. Among the significant results of the revolution was the creation of the United States Constitution, establishing a relatively strong federal national government that included an executive, a national judiciary, and\n\nAnswer:The United States took the first step towards gaining independence from Great Britain when it declared independence from Great Britain on July 2, 1776 (although the event is now commemorated on July 4, 1776, the date when the Declaration of Independence was officially adopted by Congress) [2]. The Treaty of Paris was later signed on September 3, 1783, formally separating the United States from the British Empire [3].\n\n\nInstruction: Write an accurate, engaging, and concise answer for the given question using only the provided search results (some of which might be irrelevant) and cite them properly. Use an unbiased and journalistic tone. Always cite for any factual claim. When citing several search results, use [1][2][3]. Cite at least one document and at most three documents in each sentence. If multiple documents support the sentence, only cite a minimum sufficient subset of the documents.\n\nQuestion:Who played galen in planet of the apes?\n\nDocument [1](Title:Planet of the Apes): installment. Jacobs died on June 27, 1973, bringing an end to the APJAC Productions era of the \"Planet of the Apes\" franchise. Former Fox executive Stan Hough took over as producer for the television project, titled \"Planet of the Apes\". CBS picked up the series for its 1974 autumn lineup. Ron Harper and James Naughton played Alan Virdon and Peter Burke, two 20th-century American astronauts who pass through a time warp to a future where apes subjugate humans (unlike the original film, the humans can speak). Roddy McDowall returned to the franchise as Galen, a chimpanzee who joins the astronauts.\nDocument [2](Title:Planet of the Apes (1968 film)): chimpanzees: animal psychologist Zira (Kim Hunter) and surgeon Galen (Wright King). While unable to speak as his throat wound is healing, called \"Bright Eyes\" by Zira and placed with one of the captive primitive humans he later names \"Nova\", Taylor observes the enhanced society of talking apes and in a strict caste system: the gorillas being the military police, hunters and workers; the orangutans overseeing the affairs of government, science, and religion; and intellectual chimpanzees being mostly scientists. While their society is a theocracy similar to the beginnings of the human Industrial Era, the apes consider the primitive humans as\nDocument [3](Title:Planet of the Apes (1968 film)): Planet of the Apes (1968 film) Planet of the Apes is a 1968 American science fiction film directed by Franklin J. Schaffner. It stars Charlton Heston, Roddy McDowall, Kim Hunter, Maurice Evans, James Whitmore, James Daly and Linda Harrison. The screenplay by Michael Wilson and Rod Serling was loosely based on the 1963 French novel \"La Planète des Singes\" by Pierre Boulle. Jerry Goldsmith composed the groundbreaking avant-garde score. It was the first in a series of five films made between 1968 and 1973, all produced by Arthur P. Jacobs and released by 20th Century Fox. The film tells the\n\nAnswer:In the 1968 film Planet of the Apes, Galen was played by Wright King [2]. And in the tv series Planet of the Apes, Galen was played by Roddy McDowall [1].", + "instruction_1" : "Instruction: Write an accurate, engaging, and concise answer for the given question using only the provided search results (some of which might be irrelevant) and cite them properly. Use an unbiased and journalistic tone. Always cite for any factual claim. When citing several search results, use [1][2][3]. Cite at least one document and at most three documents in each sentence. If multiple documents support the sentence, only cite a minimum sufficient subset of the documents.\n\nQuestion:When did the us break away from england?\n\nDocument [1](Title:United States withdrawal from Saudi Arabia): United States withdrawal from Saudi Arabia Beginning during Operation Desert Shield in August 1990, while preparing for the Gulf War, the United States sent a large troop contingent to Saudi Arabia. After the war, remnant troops, primarily U.S. Air Force personnel, augmented by a smaller number of coordinating and training personnel from the U.S. Navy, U.S. Army and U.S. Marine Corps remained in Saudi Arabia under the aegis of Joint Task Force Southwest Asia (JTF-SWA), as part of Operation Southern Watch (OSW). The United Kingdom and France also maintained a small contingent of Royal Air Force and French Air Force\nDocument [2](Title:Decolonization of the Americas): and France has fully \"integrated\" most of its former colonies as fully constituent \"departments\" of France. The United States of America declared independence from Great Britain on July 2, 1776 (although the event is now commemorated on July 4, the date when the Declaration of Independence was officially adopted by Congress), in so doing becoming the first independent, foreign-recognized nation in the Americas and the first European colonial entity to break from its mother country. Britain formally acknowledged American independence in 1783 after its defeat in the American Revolutionary War. Although initially occupying only the land east of the Mississippi\nDocument [3](Title:American Revolution): second British army at Yorktown in the fall of 1781, effectively ending the war. The Treaty of Paris was signed September 3, 1783, formally ending the conflict and confirming the new nation's complete separation from the British Empire. The United States took possession of nearly all the territory east of the Mississippi River and south of the Great Lakes, with the British retaining control of Canada and Spain taking Florida. Among the significant results of the revolution was the creation of the United States Constitution, establishing a relatively strong federal national government that included an executive, a national judiciary, and\n\nAnswer:The United States took the first step towards gaining independence from Great Britain when it declared independence from Great Britain on July 2, 1776 (although the event is now commemorated on July 4, 1776, the date when the Declaration of Independence was officially adopted by Congress) [2]. The Treaty of Paris was later signed on September 3, 1783, formally separating the United States from the British Empire [3].\n\n\nInstruction: Write an accurate, engaging, and concise answer for the given question using only the provided search results (some of which might be irrelevant) and cite them properly. Use an unbiased and journalistic tone. Always cite for any factual claim. When citing several search results, use [1][2][3]. Cite at least one document and at most three documents in each sentence. If multiple documents support the sentence, only cite a minimum sufficient subset of the documents.\n\nQuestion:Who played galen in planet of the apes?\n\nDocument [1](Title:Planet of the Apes): installment. Jacobs died on June 27, 1973, bringing an end to the APJAC Productions era of the \"Planet of the Apes\" franchise. Former Fox executive Stan Hough took over as producer for the television project, titled \"Planet of the Apes\". CBS picked up the series for its 1974 autumn lineup. Ron Harper and James Naughton played Alan Virdon and Peter Burke, two 20th-century American astronauts who pass through a time warp to a future where apes subjugate humans (unlike the original film, the humans can speak). Roddy McDowall returned to the franchise as Galen, a chimpanzee who joins the astronauts.\nDocument [2](Title:Planet of the Apes (1968 film)): chimpanzees: animal psychologist Zira (Kim Hunter) and surgeon Galen (Wright King). While unable to speak as his throat wound is healing, called \"Bright Eyes\" by Zira and placed with one of the captive primitive humans he later names \"Nova\", Taylor observes the enhanced society of talking apes and in a strict caste system: the gorillas being the military police, hunters and workers; the orangutans overseeing the affairs of government, science, and religion; and intellectual chimpanzees being mostly scientists. While their society is a theocracy similar to the beginnings of the human Industrial Era, the apes consider the primitive humans as\nDocument [3](Title:Planet of the Apes (1968 film)): Planet of the Apes (1968 film) Planet of the Apes is a 1968 American science fiction film directed by Franklin J. Schaffner. It stars Charlton Heston, Roddy McDowall, Kim Hunter, Maurice Evans, James Whitmore, James Daly and Linda Harrison. The screenplay by Michael Wilson and Rod Serling was loosely based on the 1963 French novel \"La Planète des Singes\" by Pierre Boulle. Jerry Goldsmith composed the groundbreaking avant-garde score. It was the first in a series of five films made between 1968 and 1973, all produced by Arthur P. Jacobs and released by 20th Century Fox. The film tells the\n\nAnswer:In the 1968 film Planet of the Apes, Galen was played by Wright King [2]. And in the tv series Planet of the Apes, Galen was played by Roddy McDowall [1].\n\nInstruction: Write an accurate, engaging, and concise answer for the given question using only the provided search results (some of which might be irrelevant) and cite them properly. Use an unbiased and journalistic tone. Always cite for any factual claim. When citing several search results, use [1][2][3]. Cite at least one document and at most three documents in each sentence. If multiple documents support the sentence, only cite a minimum sufficient subset of the documents.", + "INST" : "Instruction: Write an accurate, engaging, and concise answer for the given question using only the provided search results (some of which might be irrelevant) and cite them properly. Use an unbiased and journalistic tone. Always cite for any factual claim. When citing several search results, use [1][2][3]. Cite at least one document and at most three documents in each sentence. If multiple documents support the sentence, only cite a minimum sufficient subset of the documents.", + "attr_INST":"In this task, you are presented with spans from documents that can be used to generare an answer sentence. You may also be presented with a prefix of the answer. You job is to generate the next sentence of the answer, that covers all and only the given spans. Make sure it connects well with the prefix(if eixists). Always cite for any factual claim. When citing several search results, use [1][2][3].", + + "instruction_2":"In this task, you are presented with spans that can be used to generare an answer sentence. You may also be presented with a prefix of the answer. You job is to generate the next sentence of the answer, that covers all and only the given spans. Make sure it connects well with the prefix(if eixists). Always cite for any factual claim. When citing several search results, use [1][2][3]. Cite at least one document and at most three documents in each sentence. If multiple documents support the sentence, only cite a minimum sufficient subset of the documents.\n\nPrefix: Several places on Earth claim to be the most rainy, such as Lloró, Colombia, which reported an average annual rainfall of 12,717 mm between 1952 and 1989, and López de Micay, Colombia, which reported an annual 12,892 mm between 1960 and 2012 [3]. \n\nThe highlighted spans are: \nDocument [3]: \n5. Mawsynram receives one of the highest rainfalls in India \n\nAnswer: \nAnswer: \nHowever, the official record is held by Mawsynram, India with an average annual rainfall of 11,872 mm [3].\n\nIn this task, you are presented with several passages, where some parts are \"highlighted\" (namely, there are and tokens before and after each such span). You may also be presented with a prefix of the answer. You job is to generate the next short sentence of the answer, that covers all and only the \"highlighted\" spans. Make sure it connects well with the prefix(if eixists), and that it covers all and only the \"highlighted\" spans. Always cite for any factual claim. When citing several search results, use [1][2][3]. Cite at least one document and at most three documents in each sentence. If multiple documents support the sentence, only cite a minimum sufficient subset of the documents.", + "demo_2":"In this task, you are presented with several passages, where some parts are \"highlighted\" (namely, there are and tokens before and after each such span). You may also be presented with a prefix of the answer. You job is to generate the next sentence of the answer, that covers all and only the \"highlighted\" spans. Make sure it connects well with the prefix(if eixists), and that it covers all and only the \"highlighted\" spans. Always cite for any factual claim. When citing several search results, use [1][2][3]. Cite at least one document and at most three documents in each sentence. If multiple documents support the sentence, only cite a minimum sufficient subset of the documents.\n\nPrefix: Several places on Earth claim to be the most rainy, such as Lloró, Colombia, which reported an average annual rainfall of 12,717 mm between 1952 and 1989, and López de Micay, Colombia, which reported an annual 12,892 mm between 1960 and 2012 [3]. \n\nThe highlighted spans are: \nDocument [3]: \n5. Mawsynram receives one of the highest rainfalls in India \n\nAnswer: \nHowever, the official record is held by Mawsynram, India with an average annual rainfall of 11,872 mm [3]." +} \ No newline at end of file diff --git a/prompts/asqa_default.json b/prompts/asqa_default.json new file mode 100644 index 0000000000000000000000000000000000000000..a58790f789bfa49e32d43aa3fd6febd9bd104eff --- /dev/null +++ b/prompts/asqa_default.json @@ -0,0 +1,114 @@ +{ + "instruction": "Instruction: Write an accurate, engaging, and concise answer for the given question using only the provided search results (some of which might be irrelevant) and cite them properly. Use an unbiased and journalistic tone. Always cite for any factual claim. When citing several search results, use [1][2][3]. Cite at least one document and at most three documents in each sentence. If multiple documents support the sentence, only cite a minimum sufficient subset of the documents.", + "query_instruction" : "In this task, you will be given a question, and you should generate a query to find relevent documents to help generating the answer. You may be given some sentences that have been generated as context, you should try to find documents that could support another claim other than sentences generated but still relevent to the question. You should only output a natural language sentence as query.", + "one_sentence_instruction": "Instruction: Write only a sentence as an accurate, engaging, and concise answer for the given question using only the provided search result. Use an unbiased and journalistic tone. If you are presented with a prefix of the paragraph, please write the next sentence, making sure it connect well with the preifx.", + "demo_sep": "\n\n\n", + "demo_prompt": "{INST}\n\nQuestion: {Q}\n\n{D}\nAnswer: {A}", + "doc_prompt": "Document [{ID}](Title: {T}): {P}\n", + "demos": [ + { + "question": "Which is the most rainy place on earth?", + "answer": "Several places on Earth claim to be the most rainy, such as Lloró, Colombia, which reported an average annual rainfall of 12,717 mm between 1952 and 1989, and López de Micay, Colombia, which reported an annual 12,892 mm between 1960 and 2012 [3]. However, the official record is held by Mawsynram, India with an average annual rainfall of 11,872 mm [3], although nearby town Sohra, India, also known as Cherrapunji, holds the record for most rain in a calendar month for July 1861 and most rain in a year from August 1860 to July 1861 [1].", + "docs": [ + { + "title": "Cherrapunji", + "text": "Cherrapunji Cherrapunji (; with the native name Sohra being more commonly used, and can also be spelled Cherrapunjee or Cherrapunji) is a subdivisional town in the East Khasi Hills district in the Indian state of Meghalaya. It is the traditional capital of aNongkhlaw \"hima\" (Khasi tribal chieftainship constituting a petty state), both known as Sohra or Churra. Cherrapunji has often been credited as being the wettest place on Earth, but for now nearby Mawsynram currently holds that distinction. Cherrapunji still holds the all-time record for the most rainfall in a calendar month for July 1861 and most rain in a year from August 1860 to July 1861, however: it received in" + }, + { + "title": "Cherrapunji", + "text": "Radio relay station known as Akashvani Cherrapunji. It broadcasts on FM frequencies. Cherrapunji Cherrapunji (; with the native name Sohra being more commonly used, and can also be spelled Cherrapunjee or Cherrapunji) is a subdivisional town in the East Khasi Hills district in the Indian state of Meghalaya. It is the traditional capital of aNongkhlaw \"hima\" (Khasi tribal chieftainship constituting a petty state), both known as Sohra or Churra. Cherrapunji has often been credited as being the wettest place on Earth, but for now nearby Mawsynram currently holds that distinction. Cherrapunji still holds the all-time record for the most rainfall" + }, + { + "title": "Mawsynram", + "text": "Mawsynram Mawsynram () is a village in the East Khasi Hills district of Meghalaya state in north-eastern India, 65 kilometres from Shillong. Mawsynram receives one of the highest rainfalls in India. It is reportedly the wettest place on Earth, with an average annual rainfall of 11,872 mm, but that claim is disputed by Lloró, Colombia, which reported an average yearly rainfall of 12,717 mm between 1952 and 1989 and López de Micay, also in Colombia, which reported an annual 12,892 mm per year between 1960 and 2012. According to the \"Guinness Book of World Records\", Mawsynram received of rainfall in 1985. Mawsynram is located at 25° 18′" + }, + { + "title": "Earth rainfall climatology", + "text": "Pacific Northwest, and the Sierra Nevada range are the wetter portions of the nation, with average rainfall exceeding per year. The drier areas are the Desert Southwest, Great Basin, valleys of northeast Arizona, eastern Utah, central Wyoming, eastern Oregon and Washington and the northeast of the Olympic Peninsula. The Big Bog on the island of Maui receives, on average, every year, making it the wettest location in the US, and all of Oceania. The annual average rainfall maxima across the continent lie across the northwest from northwest Brazil into northern Peru, Colombia, and Ecuador, then along the Atlantic coast of" + }, + { + "title": "Going to Extremes", + "text": "in the world. Oymyakon in Siberia, where the average winter temperature is −47 °F (− 44 °C). Arica in Chile, where there had been fourteen consecutive years without rain. Fog is the only local source of water. Mawsynram in India, where average annual rainfall is 14 meters, falling within a four-month period in the monsoon season. The rainfall is approximately equal to that of its neighbor Cherrapunji. Dallol in Ethiopia, known as the 'Hell-hole of creation' where the temperature averages 94 °F (34 °C) over the year. In his second series, Middleton visited places without permanent towns, locations where \"survival\"" + } + ] + }, + { + "question": "When did the us break away from england?", + "answer": "The United States took the first step towards gaining independence from Great Britain when it declared independence from Great Britain on July 2, 1776 (although the event is now commemorated on July 4, 1776, the date when the Declaration of Independence was officially adopted by Congress) [2]. The Treaty of Paris was later signed on September 3, 1783, formally separating the United States from the British Empire [3].", + "docs": [ + { + "title": "United States withdrawal from Saudi Arabia", + "text": "United States withdrawal from Saudi Arabia Beginning during Operation Desert Shield in August 1990, while preparing for the Gulf War, the United States sent a large troop contingent to Saudi Arabia. After the war, remnant troops, primarily U.S. Air Force personnel, augmented by a smaller number of coordinating and training personnel from the U.S. Navy, U.S. Army and U.S. Marine Corps remained in Saudi Arabia under the aegis of Joint Task Force Southwest Asia (JTF-SWA), as part of Operation Southern Watch (OSW). The United Kingdom and France also maintained a small contingent of Royal Air Force and French Air Force" + }, + { + "title": "Decolonization of the Americas", + "text": "and France has fully \"integrated\" most of its former colonies as fully constituent \"departments\" of France. The United States of America declared independence from Great Britain on July 2, 1776 (although the event is now commemorated on July 4, the date when the Declaration of Independence was officially adopted by Congress), in so doing becoming the first independent, foreign-recognized nation in the Americas and the first European colonial entity to break from its mother country. Britain formally acknowledged American independence in 1783 after its defeat in the American Revolutionary War. Although initially occupying only the land east of the Mississippi" + }, + { + "title": "American Revolution", + "text": "second British army at Yorktown in the fall of 1781, effectively ending the war. The Treaty of Paris was signed September 3, 1783, formally ending the conflict and confirming the new nation's complete separation from the British Empire. The United States took possession of nearly all the territory east of the Mississippi River and south of the Great Lakes, with the British retaining control of Canada and Spain taking Florida. Among the significant results of the revolution was the creation of the United States Constitution, establishing a relatively strong federal national government that included an executive, a national judiciary, and" + }, + { + "title": "Decolonization", + "text": "accelerate decolonialization and bring an end to the colonial empires of its Western allies, most importantly during the 1956 Suez Crisis, but American military bases were established around the world and direct and indirect interventions continued in Korea, Indochina, Latin America (\"inter alia\", the 1965 occupation of the Dominican Republic), Africa, and the Middle East to oppose Communist invasions and insurgencies. Since the dissolution of the Soviet Union, the United States has been far less active in the Americas, but invaded Afghanistan and Iraq following the September 11 attacks in 2001, establishing army and air bases in Central Asia. Before" + }, + { + "title": "Decolonization", + "text": "the responsibility of the United Kingdom (with a copy of the new constitution annexed), and finally, if approved, issuance of an Order of Council fixing the exact date of independence. After World War I, several former German and Ottoman territories in the Middle East, Africa, and the Pacific were governed by the UK as League of Nations mandates. Some were administered directly by the UK, and others by British dominions – Nauru and the Territory of New Guinea by Australia, South West Africa by the Union of South Africa, and Western Samoa by New Zealand. Egypt became independent in 1922," + } + ] + }, + { + "question": "Who set the record for longest field goal?", + "answer": "The record for the longest field goal in an NFL game was set by Matt Prater at 64 yards [1], but the record for the longest field goal at any level was 69 yards, kicked by collegiate kicker Ove Johansson in a 1976 Abilene Christian University football game against East Texas State University [2].", + "docs": [ + { + "title": "Field goal", + "text": "toward its own end. The longest field goal kick in NFL history is 64 yards, a record set by Matt Prater on December 8, 2013. The previous record was 63, originally set by Tom Dempsey (1970) and then matched by Jason Elam (1998), Sebastian Janikowski (2011), David Akers (2012), and Graham Gano (2018). High school, college and most professional football leagues offer only a three-point field goal; however, some professional leagues have encouraged more rare kicks through \"four-point field goals\". NFL Europe encouraged long field goals of 50 yards or more by making those worth four points instead of three" + }, + { + "title": "Field goal range", + "text": "35 and 40 yard lines (closer in a crosswind) often will go for the more risky fourth down conversion rather than risk either the touchback or the missed field goal. The longest field goal in recorded football history was 69 yards, set by collegiate kicker Ove Johansson, who was born in Sweden, in a 1976 Abilene Christian University football game against East Texas State University (now Texas A&M Commerce) at Shotwell Stadium in Abilene. The longest successful field goal in the NFL was 64 yards and was completed by Matt Prater in 2013. The NCAA record is 67 yards held" + }, + { + "title": "Field goal", + "text": "both end zones) is only 66 yards. Scaccia, while playing indoor football, attempted a 64-yard kick that was inches short of success, hitting the crossbar. Longer field goals have been attempted at times; the longest attempt in the NFL, which was well short and was kicked into the wind, was 76 yards, attempted by Sebastian Janikowski of the Oakland Raiders, in a September 28, 2008 game against the San Diego Chargers. NFL Europe rewarded kickers that successfully kicked a field goal of longer than 50 yards with a bonus point, making such field goals worth 4 points instead of 3;" + }, + { + "title": "Field goal", + "text": "this accomplishment is not the official record. All of the above kicks were successful with the use of a kicking tee, which was banned by the NCAA after the 1988 season. The longest known drop-kicked field goal in college football was a 62-yard kick from Pat O'Dea, an Australian kicker who played on the Wisconsin Badgers football team. O'Dea's kick took place in a blizzard against Northwestern on November 15, 1898. The longest field goal in U Sports football history is 59 yards, by Niko Difonte of Calgary Dinos, playing against the UBC Thunderbirds on November 11, 2017. The field" + }, + { + "title": "Field goal range", + "text": "NFL and have been banned from NCAA since 1989) is 68 yards held by Fabrizio Scaccia, and the high school record 68 yards held by Dirk Borgognone; high school has wider goal posts and treats a field goal attempt that lands short in the field of play the same as a punt, making longer attempts much less risky. The indoor football record, with narrower and higher goal posts, is 63 yards (set by Aaron Mills), which is practically as long of a field goal as is possible in that variant of the sport, since the field in indoor football (including" + } + ] + }, + { + "question": "Who played galen in planet of the apes?", + "answer": "In the 1968 film Planet of the Apes, Galen was played by Wright King [2]. And in the tv series Planet of the Apes, Galen was played by Roddy McDowall [1].", + "docs": [ + { + "title": "Planet of the Apes", + "text": "installment. Jacobs died on June 27, 1973, bringing an end to the APJAC Productions era of the \"Planet of the Apes\" franchise. Former Fox executive Stan Hough took over as producer for the television project, titled \"Planet of the Apes\". CBS picked up the series for its 1974 autumn lineup. Ron Harper and James Naughton played Alan Virdon and Peter Burke, two 20th-century American astronauts who pass through a time warp to a future where apes subjugate humans (unlike the original film, the humans can speak). Roddy McDowall returned to the franchise as Galen, a chimpanzee who joins the astronauts." + }, + { + "title": "Planet of the Apes (1968 film)", + "text": "chimpanzees: animal psychologist Zira (Kim Hunter) and surgeon Galen (Wright King). While unable to speak as his throat wound is healing, called \"Bright Eyes\" by Zira and placed with one of the captive primitive humans he later names \"Nova\", Taylor observes the enhanced society of talking apes and in a strict caste system: the gorillas being the military police, hunters and workers; the orangutans overseeing the affairs of government, science, and religion; and intellectual chimpanzees being mostly scientists. While their society is a theocracy similar to the beginnings of the human Industrial Era, the apes consider the primitive humans as" + }, + { + "title": "Planet of the Apes (1968 film)", + "text": "Planet of the Apes (1968 film) Planet of the Apes is a 1968 American science fiction film directed by Franklin J. Schaffner. It stars Charlton Heston, Roddy McDowall, Kim Hunter, Maurice Evans, James Whitmore, James Daly and Linda Harrison. The screenplay by Michael Wilson and Rod Serling was loosely based on the 1963 French novel \"La Plan\u00e8te des Singes\" by Pierre Boulle. Jerry Goldsmith composed the groundbreaking avant-garde score. It was the first in a series of five films made between 1968 and 1973, all produced by Arthur P. Jacobs and released by 20th Century Fox. The film tells the" + }, + { + "title": "Planet of the Apes", + "text": "Rupert Wyatt. To portray ape characters realistically, the production avoided practical effects in favor of performance capture acting, partnering with New Zealand visual effects company Weta Digital. Wyatt cast James Franco as Will Rodman, while veteran performance capture actor Andy Serkis signed on to star as Caesar. \"Rise\" debuted on August 5, 2011. Critics reviewed it positively, especially praising the visual effects and Serkis's performance. It was a major box office hit, taking in $482 million globally, more than five times its $93 million budget. Weta's special effects earned the film two Visual Effects Society Awards and an Oscar nomination" + }, + { + "title": "Planet of the Apes", + "text": "film stars Mark Wahlberg as astronaut Leo Davidson, who accidentally travels through a wormhole to a distant planet where talking apes enslave humans. He leads a human revolt and upends ape civilization by discovering that the apes evolved from the normal earth primates who had accompanied his mission, and arrived years before. Helena Bonham Carter played chimpanzee Ari, while Tim Roth played the human-hating chimpanzee General Thade. The film received mixed reviews; most critics believed it failed to compare to the original. Much of the negative commentary focused on the confusing plot and twist ending, though many reviewers praised the" + } + ] + } + ] +} \ No newline at end of file diff --git a/prompts/asqa_icl.v3_3.json b/prompts/asqa_icl.v3_3.json new file mode 100644 index 0000000000000000000000000000000000000000..2ddf12e290827b436e1df746030b14acaaf83759 --- /dev/null +++ b/prompts/asqa_icl.v3_3.json @@ -0,0 +1,139 @@ +{ + "instruction": "Generate an informative answer with reference spans for the given question.\nWrite an accurate, engaging, and informative answer for the given question using the provided Documents as a reference (some of which might be irrelevant), and cite them properly with markdown footnote format.\nOnly cite for the factual claims that the Documents support. \nIn the answer, if you need to cite, mark sentences that cite relevant documents with a reference mark ^[From document [document id]: continuous text fragment in this document]. \nOptionally cite relevant documents and cite at most two documents in each sentence.\nDo not overcite, if multiple documents support the sentence, only cite a minimum sufficient subset of the documents and cite at most two.\nFor sentences marked with reference numbers in the answer, list the corresponding document source, along with the exact citation span from it.", + "template_prompt": "Instruction:\n{INST}\n\nThere are some examples. Complete the last Answer according to example markdown format.\n\n{DEMOS}{QUESTION}", + "template_question": "## Question:\n{Q}\n### Documents:\n{D}## Response:\n", + "template_demo": "## Question:\n{Q}\n### Documents:\n{D}## Response:\n{A}\n\n\n", + "template_doc": "[{ID}] ({T}): {P}\n", + "demos": [ + { + "question": "Which is the most rainy place on earth?", + "answer": "Several places on Earth claim to be the most rainy, such as Lloró, Colombia, which reported an average annual rainfall of 12,717 mm between 1952 and 1989, and López de Micay, Colombia, which reported an annual 12,892 mm between 1960 and 2012^[From document [3]: \"but that claim is disputed by Lloró, Colombia, which reported an average yearly rainfall of 12,717 mm between 1952 and 1989 and López de Micay, also in Colombia, which reported an annual 12,892 mm per year between 1960 and 2012.\"]. However, the official record is held by Mawsynram, India with an average annual rainfall of 11,872 mm^[From document [3]: \"Mawsynram receives one of the highest rainfalls in India. It is reportedly the wettest place on Earth, with an average annual rainfall of 11,872 mm\"], although nearby town Sohra, India, also known as Cherrapunji, holds the record for most rain in a calendar month for July 1861 and most rain in a year from August 1860 to July 1861^[From document [1]: \"Cherrapunji still holds the all-time record for the most rainfall in a calendar month for July 1861 and most rain in a year from August 1860 to July 1861\"].", + "docs": [ + { + "title": "Cherrapunji", + "text": "Cherrapunji Cherrapunji (; with the native name Sohra being more commonly used, and can also be spelled Cherrapunjee or Cherrapunji) is a subdivisional town in the East Khasi Hills district in the Indian state of Meghalaya. It is the traditional capital of aNongkhlaw \"hima\" (Khasi tribal chieftainship constituting a petty state), both known as Sohra or Churra. Cherrapunji has often been credited as being the wettest place on Earth, but for now nearby Mawsynram currently holds that distinction. Cherrapunji still holds the all-time record for the most rainfall in a calendar month for July 1861 and most rain in a year from August 1860 to July 1861, however: it received in" + }, + { + "title": "Cherrapunji", + "text": "Radio relay station known as Akashvani Cherrapunji. It broadcasts on FM frequencies. Cherrapunji Cherrapunji (; with the native name Sohra being more commonly used, and can also be spelled Cherrapunjee or Cherrapunji) is a subdivisional town in the East Khasi Hills district in the Indian state of Meghalaya. It is the traditional capital of aNongkhlaw \"hima\" (Khasi tribal chieftainship constituting a petty state), both known as Sohra or Churra. Cherrapunji has often been credited as being the wettest place on Earth, but for now nearby Mawsynram currently holds that distinction. Cherrapunji still holds the all-time record for the most rainfall" + }, + { + "title": "Mawsynram", + "text": "Mawsynram Mawsynram () is a village in the East Khasi Hills district of Meghalaya state in north-eastern India, 65 kilometres from Shillong. Mawsynram receives one of the highest rainfalls in India. It is reportedly the wettest place on Earth, with an average annual rainfall of 11,872 mm, but that claim is disputed by Lloró, Colombia, which reported an average yearly rainfall of 12,717 mm between 1952 and 1989 and López de Micay, also in Colombia, which reported an annual 12,892 mm per year between 1960 and 2012. According to the \"Guinness Book of World Records\", Mawsynram received of rainfall in 1985. Mawsynram is located at 25° 18′" + }, + { + "title": "Earth rainfall climatology", + "text": "Pacific Northwest, and the Sierra Nevada range are the wetter portions of the nation, with average rainfall exceeding per year. The drier areas are the Desert Southwest, Great Basin, valleys of northeast Arizona, eastern Utah, central Wyoming, eastern Oregon and Washington and the northeast of the Olympic Peninsula. The Big Bog on the island of Maui receives, on average, every year, making it the wettest location in the US, and all of Oceania. The annual average rainfall maxima across the continent lie across the northwest from northwest Brazil into northern Peru, Colombia, and Ecuador, then along the Atlantic coast of" + }, + { + "title": "Going to Extremes", + "text": "in the world. Oymyakon in Siberia, where the average winter temperature is −47 °F (− 44 °C). Arica in Chile, where there had been fourteen consecutive years without rain. Fog is the only local source of water. Mawsynram in India, where average annual rainfall is 14 meters, falling within a four-month period in the monsoon season. The rainfall is approximately equal to that of its neighbor Cherrapunji. Dallol in Ethiopia, known as the 'Hell-hole of creation' where the temperature averages 94 °F (34 °C) over the year. In his second series, Middleton visited places without permanent towns, locations where \"survival\"" + } + ] + }, + { + "question": "When did the us break away from england?", + "answer": "The United States took the first step towards gaining independence from Great Britain when it declared independence from Great Britain on July 2, 1776 (although the event is now commemorated on July 4, 1776, the date when the Declaration of Independence was officially adopted by Congress)^[From document [2]: \"The United States of America declared independence from Great Britain on July 2, 1776\"]^[From document [1]: \"On July 4th, the Continental Congress formally adopted the Declaration of Independence\"]. The Treaty of Paris was later signed on September 3, 1783, formally separating the United States from the British Empire^[From document [3]: \"The Treaty of Paris was signed September 3, 1783, formally ending the conflict and confirming the new nation's complete separation from the British Empire.\"].", + "docs": [ + { + "title": "Fourth of July – Independence Day", + "text": "The Fourth of July—also known as Independence Day or July 4th—has been a federal holiday in the United States since 1941. On July 4th, the Continental Congress formally adopted the Declaration of Independence, which had been written largely by Jefferson. From then on the 4th became the day that was celebrated as the birth of American independence.The Fourth of July 2023 is on Tuesday, July 4." + }, + { + "title": "Decolonization of the Americas", + "text": "and France has fully \"integrated\" most of its former colonies as fully constituent \"departments\" of France. The United States of America declared independence from Great Britain on July 2, 1776, in so doing becoming the first independent, foreign-recognized nation in the Americas and the first European colonial entity to break from its mother country. Britain formally acknowledged American independence in 1783 after its defeat in the American Revolutionary War. Although initially occupying only the land east of the Mississippi" + }, + { + "title": "American Revolution", + "text": "second British army at Yorktown in the fall of 1781, effectively ending the war. The Treaty of Paris was signed September 3, 1783, formally ending the conflict and confirming the new nation's complete separation from the British Empire. The United States took possession of nearly all the territory east of the Mississippi River and south of the Great Lakes, with the British retaining control of Canada and Spain taking Florida. Among the significant results of the revolution was the creation of the United States Constitution, establishing a relatively strong federal national government that included an executive, a national judiciary, and" + }, + { + "title": "Decolonization", + "text": "accelerate decolonialization and bring an end to the colonial empires of its Western allies, most importantly during the 1956 Suez Crisis, but American military bases were established around the world and direct and indirect interventions continued in Korea, Indochina, Latin America (\"inter alia\", the 1965 occupation of the Dominican Republic), Africa, and the Middle East to oppose Communist invasions and insurgencies. Since the dissolution of the Soviet Union, the United States has been far less active in the Americas, but invaded Afghanistan and Iraq following the September 11 attacks in 2001, establishing army and air bases in Central Asia. Before" + }, + { + "title": "Decolonization", + "text": "the responsibility of the United Kingdom (with a copy of the new constitution annexed), and finally, if approved, issuance of an Order of Council fixing the exact date of independence. After World War I, several former German and Ottoman territories in the Middle East, Africa, and the Pacific were governed by the UK as League of Nations mandates. Some were administered directly by the UK, and others by British dominions – Nauru and the Territory of New Guinea by Australia, South West Africa by the Union of South Africa, and Western Samoa by New Zealand. Egypt became independent in 1922," + } + ] + }, + { + "question": "Why did New York City try to ban food donations to the poor?", + "answer": "New York City, under Mayor Michael Bloomberg's administration, banned citizens from donating food directly to homeless shelters because the city could not assess the salt, fat, and fiber content of donated items. This initiative was part of a broader public health campaign aimed at improving nutrition and preventing health issues among the city's most vulnerable populations ^[From document [1]: \"Outlawed are food donations to homeless shelters because the city can\\u2019t assess their salt, fat and fiber content\"]^[From document [2]: \"\\u2013 New York Mayor Michael Bloomberg\\u2019s administration is now banning all food being offered to the city\\u2019s homeless shelters.\"]. However, Bloomberg's administration was heavily criticized for losing their common sense by becoming too focused on what people eat, with many arguing that the policy overlooked the immediate needs of the homeless for the sake of nutritional guidelines ^[From document [1]: \"Outlawed are food donations to homeless shelters because the city can\\u2019t assess their salt, fat and fiber content, reports CBS 2\\u2019s Marcia Kramer. Glenn Richter arrived at a West Side synagogue on Monday to collect surplus bagels \\u2014 fresh nutritious bagels \\u2014 to donate to the poor.\"]^[From document [2]: \"Amuck: Bloomberg Bans Food Donations in New York City Food Might Be Salty or Too High in Calories, City Explains Washington, D.C. \\u2013 New York Mayor Michael Bloomberg\\u2019s administration is now banning all food being offered to the city\\u2019s homeless shelters. New York City\\u2019s bureaucrats have become so singularly focused on what people eat, says the National Center for Public Policy Research, that they\\u2019ve lost their common sense. \\u201cSo much for serving the homeless: The Bloomberg administration is now taking the term \\u2018food police\\u2019 to new depths\"]. Critics contended that the decision to ban food donations failed to strike a balance between ensuring food safety and addressing the urgent hunger faced by those living in shelters.", + "docs": [ + { + "title": "The Future Of America", + "text": "believe that they are \u201chelping\u201d the homeless by passing such laws. In New York City, Mayor Bloomberg has banned citizens from donating food directly to homeless shelters and he is actually convinced that it was the right thing to do for the homeless\u2026 Mayor Michael Bloomberg\u2019s food police have struck again! Outlawed are food donations to homeless shelters because the city can\u2019t assess their salt, fat and fiber content, reports CBS 2\u2019s Marcia Kramer. Glenn Richter arrived at a West Side synagogue on Monday to collect surplus bagels \u2014 fresh nutritious bagels \u2014 to donate to the poor." + }, + { + "title": "mayor bloomberg", + "text": "Amuck: Bloomberg Bans Food Donations in New York City Food Might Be Salty or Too High in Calories, City Explains Washington, D.C. \u2013 New York Mayor Michael Bloomberg\u2019s administration is now banning all food being offered to the city\u2019s homeless shelters. New York City\u2019s bureaucrats have become so singularly focused on what people eat, says the National Center for Public Policy Research, that they\u2019ve lost their common sense. \u201cSo much for serving the homeless: The Bloomberg administration is now taking the term \u2018food police\u2019 to new depths, blocking food donations to all government-run facilities that serve the" + }, + { + "title": "New York City bans food donations - WND", + "text": "New York City bans food donations - WND Front Page Health U.S. New York City bans food donations Inability to control 'nutritional content' cited as reason New York City homeless shelters have Mayor Michael Bloomberg to thank for a halt in food donations, for which hungry families are waiting, according to one public policy advocate. \"The Bloomberg administration is now taking the term 'food police' to new depths, blocking food donations to all government-run facilities that serve the city's homeless,\" says Jeff Stier, a National Center for Public Policy Research senior fellow. Currently, no food can be given to government-run, New York City facilities, despite hungry crowds perfectly" + }, + { + "title": "New York City bans food donations - WND", + "text": "New York City bans food donations - WND Services didn't return WND calls. Stier told WND that he specifically was told by Diamond that the policy was tied to the nutritional guidelines set by the mayor. \"They can say that this ban on donations is a long-standing policy, but they can\u2019t document it,\" Stier told WND. \"I've also been told that there are numerous food shelves that have been accepting food donations, not just one.\" Stier is a member of a New York Synagogue that has donated food for over a decade. He is outraged that the DHS' response to his demand to know why the practice can" + }, + { + "title": "New York City bans food donations - WND", + "text": "New York City bans food donations - WND ban on donated food. In fact, it thrives because of food donations. New York City Rescue Mission has been providing food, clothing, shelter and spiritual hope for needy New Yorkers since 1872. \"We feed over 500 people a day, all through donations,\" said James Varnhagen, NYCRM director. \"Boxed food, canned food, prepared food, we take any food,\" he told WND. \"We couldn't survive without donations,\" he said." + } + ] + }, + { + "question": "Who set the record for longest field goal?", + "answer": "The answer for this question varies. The record for the longest field goal in an NFL game was set by Matt Prater at 64 yards^[From document [3]: \"The longest field goal kick in NFL history is 64 yards, a record set by Matt Prater on December 8, 2013.\"], but the record for the longest field goal at any level was 69 yards, kicked by collegiate kicker Ove Johansson in a 1976 Abilene Christian University football game against East Texas State University^[From document [2]: \"The longest field goal in recorded football history was 69 yards, set by collegiate kicker Ove Johansson, who was born in Sweden, in a 1976 Abilene Christian University football game against East Texas State University (now Texas A&M Commerce) at Shotwell Stadium in Abilene.\"].", + "docs": [ + { + "title": "Field goal", + "text": "both end zones) is only 66 yards. Scaccia, while playing indoor football, attempted a 64-yard kick that was inches short of success, hitting the crossbar. Longer field goals have been attempted at times; the longest attempt in the NFL, which was well short and was kicked into the wind, was 76 yards, attempted by Sebastian Janikowski of the Oakland Raiders, in a September 28, 2008 game against the San Diego Chargers. NFL Europe rewarded kickers that successfully kicked a field goal of longer than 50 yards with a bonus point, making such field goals worth 4 points instead of 3;" + }, + { + "title": "Field goal range", + "text": "35 and 40 yard lines (closer in a crosswind) often will go for the more risky fourth down conversion rather than risk either the touchback or the missed field goal. The longest field goal in recorded football history was 69 yards, set by collegiate kicker Ove Johansson, who was born in Sweden, in a 1976 Abilene Christian University football game against East Texas State University (now Texas A&M Commerce) at Shotwell Stadium in Abilene. The longest successful field goal in the NFL was 64 yards and was completed by Matt Prater in 2013. The NCAA record is 67 yards held" + }, + { + "title": "Field goal", + "text": "toward its own end. The longest field goal kick in NFL history is 64 yards, a record set by Matt Prater on December 8, 2013. The previous record was 63, originally set by Tom Dempsey (1970) and then matched by Jason Elam (1998), Sebastian Janikowski (2011), David Akers (2012), and Graham Gano (2018). High school, college and most professional football leagues offer only a three-point field goal; however, some professional leagues have encouraged more rare kicks through \"four-point field goals\". NFL Europe encouraged long field goals of 50 yards or more by making those worth four points instead of three" + }, + { + "title": "Field goal", + "text": "this accomplishment is not the official record. All of the above kicks were successful with the use of a kicking tee, which was banned by the NCAA after the 1988 season. The longest known drop-kicked field goal in college football was a 62-yard kick from Pat O'Dea, an Australian kicker who played on the Wisconsin Badgers football team. O'Dea's kick took place in a blizzard against Northwestern on November 15, 1898. The longest field goal in U Sports football history is 59 yards, by Niko Difonte of Calgary Dinos, playing against the UBC Thunderbirds on November 11, 2017. The field" + }, + { + "title": "Field goal range", + "text": "NFL and have been banned from NCAA since 1989) is 68 yards held by Fabrizio Scaccia, and the high school record 68 yards held by Dirk Borgognone; high school has wider goal posts and treats a field goal attempt that lands short in the field of play the same as a punt, making longer attempts much less risky. The indoor football record, with narrower and higher goal posts, is 63 yards (set by Aaron Mills), which is practically as long of a field goal as is possible in that variant of the sport, since the field in indoor football (including" + } + ] + }, + { + "question": "Who played galen in planet of the apes?", + "answer": "In the 1968 film Planet of the Apes, Galen was played by Wright King. And in the tv series Planet of the Apes, Galen was played by Roddy McDowall^[From document [4]: \"Roddy McDowall returned to the franchise as Galen\"].", + "docs": [ + { + "title": "Planet of the Apes", + "text": "Rupert Wyatt. To portray ape characters realistically, the production avoided practical effects in favor of performance capture acting, partnering with New Zealand visual effects company Weta Digital. Wyatt cast James Franco as Will Rodman, while veteran performance capture actor Andy Serkis signed on to star as Caesar. \"Rise\" debuted on August 5, 2011. Critics reviewed it positively, especially praising the visual effects and Serkis's performance. It was a major box office hit, taking in $482 million globally, more than five times its $93 million budget. Weta's special effects earned the film two Visual Effects Society Awards and an Oscar nomination" + }, + { + "title": "Planet of the Apes (TV series)", + "text": "Planet of the Apes is a 1974 American science fiction television series that was broadcast on CBS. The series features Roddy McDowall, Ron Harper, James Naughton, and Mark Lenard. It is based on the 1968 film of the same name and its sequels, which were, in turn, based on the 1963 novel Planet of the Apes by Pierre Boulle." + }, + { + "title": "Planet of the Apes (1968 film)", + "text": "Planet of the Apes (1968 film) Planet of the Apes is a 1968 American science fiction film directed by Franklin J. Schaffner. It stars Charlton Heston, Roddy McDowall, Kim Hunter, Maurice Evans, James Whitmore, James Daly and Linda Harrison. The screenplay by Michael Wilson and Rod Serling was loosely based on the 1963 French novel \"La Plan\u00e8te des Singes\" by Pierre Boulle. Jerry Goldsmith composed the groundbreaking avant-garde score. It was the first in a series of five films made between 1968 and 1973, all produced by Arthur P. Jacobs and released by 20th Century Fox. The film tells the" + }, + { + "title": "Planet of the Apes", + "text": "installment. Jacobs died on June 27, 1973, bringing an end to the APJAC Productions era of the \"Planet of the Apes\" franchise. Former Fox executive Stan Hough took over as producer for the television project, titled \"Planet of the Apes\". CBS picked up the series for its 1974 autumn lineup. Ron Harper and James Naughton played Alan Virdon and Peter Burke, two 20th-century American astronauts who pass through a time warp to a future where apes subjugate humans (unlike the original film, the humans can speak). Roddy McDowall returned to the franchise as Galen, a chimpanzee who joins the astronauts." + }, + { + "title": "Planet of the Apes", + "text": "film stars Mark Wahlberg as astronaut Leo Davidson, who accidentally travels through a wormhole to a distant planet where talking apes enslave humans. He leads a human revolt and upends ape civilization by discovering that the apes evolved from the normal earth primates who had accompanied his mission, and arrived years before. Helena Bonham Carter played chimpanzee Ari, while Tim Roth played the human-hating chimpanzee General Thade. The film received mixed reviews; most critics believed it failed to compare to the original. Much of the negative commentary focused on the confusing plot and twist ending, though many reviewers praised the" + } + ] + } + ] +} \ No newline at end of file diff --git a/prompts/asqa_interact_doc_id.json b/prompts/asqa_interact_doc_id.json new file mode 100644 index 0000000000000000000000000000000000000000..9a072c7636c8070f13357b8734848a7026aababc --- /dev/null +++ b/prompts/asqa_interact_doc_id.json @@ -0,0 +1,176 @@ +{ + "instruction": "Instruction: Write an accurate, engaging, and concise answer for the given question using only the provided search results and cite them properly. Use an unbiased and journalistic tone. Always cite for any factual claim.\nYou are provided summaries/snippets of the search results. You can use \"Check: Document [1][2]\" to check the corresponding full documents (you should only check relevant documents and you can at most check 3 documents at a time) and use \"Output:\" to output a sentence in the answer. In the answer, cite properly by using [1][2][3]. Cite at least one document and at most three documents in each sentence. If multiple documents support the sentence, only cite a minimum sufficient subset of the documents. Use \"End\" to end the generation.", + "demo_sep": "\n\n\n", + "demo_prompt": "{INST}\n\nQuestion: {Q}\n\n{D}\nAnswer: {A}", + "doc_prompt": "Document [{ID}](Title: {T}): {P}\n", + "demos": [ + { + "question": "Which is the most rainy place on earth?", + "answer": [ + "Check: Document [3]", + "Output: Several places on Earth claim to be the most rainy, such as Lloró, Colombia, which reported an average annual rainfall of 12,717 mm between 1952 and 1989, and López de Micay, Colombia, which reported an annual 12,892 mm between 1960 and 2012 [3].", + "Check: Document [1][3]", + "Output: However, the official record is held by Mawsynram, India with an average annual rainfall of 11,872 mm [3], although nearby town Sohra, India, also known as Cherrapunji, holds the record for most rain in a calendar month for July 1861 and most rain in a year from August 1860 to July 1861 [1].", + "End" + ], + "docs": [ + { + "title": "Cherrapunji", + "text": "Cherrapunji Cherrapunji (; with the native name Sohra being more commonly used, and can also be spelled Cherrapunjee or Cherrapunji) is a subdivisional town in the East Khasi Hills district in the Indian state of Meghalaya. It is the traditional capital of aNongkhlaw \"hima\" (Khasi tribal chieftainship constituting a petty state), both known as Sohra or Churra. Cherrapunji has often been credited as being the wettest place on Earth, but for now nearby Mawsynram currently holds that distinction. Cherrapunji still holds the all-time record for the most rainfall in a calendar month for July 1861 and most rain in a year from August 1860 to July 1861, however: it received in", + "summary": "Cherrapunji, also known as Sohra or Churra, is a town in Meghalaya, India. It was once believed to be the wettest place on Earth, but nearby Mawsynram currently holds that title. However, Cherrapunji still holds the records for most rainfall in a calendar month (July 1861) and most rain in a year (August 1860 to July 1861).", + "extraction": "The nearby place Mawsynram currently holds the distinction of being the wettest place on Earth." + }, + { + "title": "Cherrapunji", + "text": "Radio relay station known as Akashvani Cherrapunji. It broadcasts on FM frequencies. Cherrapunji Cherrapunji (; with the native name Sohra being more commonly used, and can also be spelled Cherrapunjee or Cherrapunji) is a subdivisional town in the East Khasi Hills district in the Indian state of Meghalaya. It is the traditional capital of aNongkhlaw \"hima\" (Khasi tribal chieftainship constituting a petty state), both known as Sohra or Churra. Cherrapunji has often been credited as being the wettest place on Earth, but for now nearby Mawsynram currently holds that distinction. Cherrapunji still holds the all-time record for the most rainfall", + "summary": "Cherrapunji is a town in Meghalaya, India and is known for being one of the rainiest places on earth, although nearby Mawsynram currently holds the record. Cherrapunji still holds the all-time record for the most rainfall.", + "extraction": "The nearby Mawsynram currently holds the distinction of being the wettest place on Earth." + }, + { + "title": "Mawsynram", + "text": "Mawsynram Mawsynram () is a village in the East Khasi Hills district of Meghalaya state in north-eastern India, 65 kilometres from Shillong. Mawsynram receives one of the highest rainfalls in India. It is reportedly the wettest place on Earth, with an average annual rainfall of 11,872 mm, but that claim is disputed by Lloró, Colombia, which reported an average yearly rainfall of 12,717 mm between 1952 and 1989 and López de Micay, also in Colombia, which reported an annual 12,892 mm per year between 1960 and 2012. According to the \"Guinness Book of World Records\", Mawsynram received of rainfall in 1985. Mawsynram is located at 25° 18′", + "summary": "Mawsynram, India receives one of the highest rainfalls in India and is reportedly the wettest place on Earth with an average annual rainfall of 11,872 mm. However, this claim is disputed by two places in Colombia that reported higher annual rainfall.", + "extraction": "Mawsynram is reportedly the wettest place on Earth, with an average annual rainfall of 11,872 mm." + }, + { + "title": "Earth rainfall climatology", + "text": "Pacific Northwest, and the Sierra Nevada range are the wetter portions of the nation, with average rainfall exceeding per year. The drier areas are the Desert Southwest, Great Basin, valleys of northeast Arizona, eastern Utah, central Wyoming, eastern Oregon and Washington and the northeast of the Olympic Peninsula. The Big Bog on the island of Maui receives, on average, every year, making it the wettest location in the US, and all of Oceania. The annual average rainfall maxima across the continent lie across the northwest from northwest Brazil into northern Peru, Colombia, and Ecuador, then along the Atlantic coast of", + "summary": "The Big Bog on the island of Maui receives the most rainfall on earth, with an average of per year.", + "extraction": "The Big Bog on the island of Maui receives, on average, every year, making it the wettest location in the US, and all of Oceania." + }, + { + "title": "Going to Extremes", + "text": "in the world. Oymyakon in Siberia, where the average winter temperature is −47 °F (− 44 °C). Arica in Chile, where there had been fourteen consecutive years without rain. Fog is the only local source of water. Mawsynram in India, where average annual rainfall is 14 meters, falling within a four-month period in the monsoon season. The rainfall is approximately equal to that of its neighbor Cherrapunji. Dallol in Ethiopia, known as the 'Hell-hole of creation' where the temperature averages 94 °F (34 °C) over the year. In his second series, Middleton visited places without permanent towns, locations where \"survival\"", + "summary": "The document mentions Mawsynram in India as the most rainy place on earth with an average annual rainfall of 14 meters falling within a four-month period in the monsoon season.", + "extraction": "Mawsynram in India, where average annual rainfall is 14 meters, falling within a four-month period in the monsoon season." + } + ] + }, + { + "question": "When did the us break away from england?", + "answer": [ + "Check: Document [2]", + "Output: The United States took the first step towards gaining independence from Great Britain when it declared independence from Great Britain on July 2, 1776 (although the event is now commemorated on July 4, 1776, the date when the Declaration of Independence was officially adopted by Congress) [2].", + "Check: Document [1][3]", + "Output: The Treaty of Paris was later signed on September 3, 1783, formally separating the United States from the British Empire [3].", + "End" + ], + "docs": [ + { + "title": "Decolonization", + "text": "accelerate decolonialization and bring an end to the colonial empires of its Western allies, most importantly during the 1956 Suez Crisis, but American military bases were established around the world and direct and indirect interventions continued in Korea, Indochina, Latin America (\"inter alia\", the 1965 occupation of the Dominican Republic), Africa, and the Middle East to oppose Communist invasions and insurgencies. Since the dissolution of the Soviet Union, the United States has been far less active in the Americas, but invaded Afghanistan and Iraq following the September 11 attacks in 2001, establishing army and air bases in Central Asia. Before", + "summary": "The US fought against Mexico over border lands and gained California and Texas, but the document does not give a specific date for when the US broke away from England.", + "extraction": "A related process took place in what is now Mexico, Central America, and parts of North America between 1810 and 1821 with the Mexican War of Independence." + }, + { + "title": "Decolonization of the Americas", + "text": "and France has fully \"integrated\" most of its former colonies as fully constituent \"departments\" of France. The United States of America declared independence from Great Britain on July 2, 1776 (although the event is now commemorated on July 4, the date when the Declaration of Independence was officially adopted by Congress), in so doing becoming the first independent, foreign-recognized nation in the Americas and the first European colonial entity to break from its mother country. Britain formally acknowledged American independence in 1783 after its defeat in the American Revolutionary War. Although initially occupying only the land east of the Mississippi", + "summary": "The United States of America declared independence from Great Britain on July 2, 1776, becoming the first European colonial entity to break from its mother country.", + "extraction": "The United States of America declared independence from Great Britain on July 2, 1776." + }, + { + "title": "American Revolution", + "text": "second British army at Yorktown in the fall of 1781, effectively ending the war. The Treaty of Paris was signed September 3, 1783, formally ending the conflict and confirming the new nation's complete separation from the British Empire. The United States took possession of nearly all the territory east of the Mississippi River and south of the Great Lakes, with the British retaining control of Canada and Spain taking Florida. Among the significant results of the revolution was the creation of the United States Constitution, establishing a relatively strong federal national government that included an executive, a national judiciary, and", + "summary": "The Treaty of Paris was signed on September 3, 1783, which confirmed the complete separation of the United States from the British Empire.", + "extraction": "The United States broke away from England formally on September 3, 1783, when the Treaty of Paris was signed, confirming their complete separation from the British Empire." + }, + { + "title": "Decolonization", + "text": "accelerate decolonialization and bring an end to the colonial empires of its Western allies, most importantly during the 1956 Suez Crisis, but American military bases were established around the world and direct and indirect interventions continued in Korea, Indochina, Latin America (\"inter alia\", the 1965 occupation of the Dominican Republic), Africa, and the Middle East to oppose Communist invasions and insurgencies. Since the dissolution of the Soviet Union, the United States has been far less active in the Americas, but invaded Afghanistan and Iraq following the September 11 attacks in 2001, establishing army and air bases in Central Asia. Before", + "summary": "Since the dissolution of the Soviet Union, the United States has been far less active in the Americas", + "extraction": "Since the dissolution of the Soviet Union, the United States has been far less active in the Americas" + }, + { + "title": "Decolonization", + "text": "the responsibility of the United Kingdom (with a copy of the new constitution annexed), and finally, if approved, issuance of an Order of Council fixing the exact date of independence. After World War I, several former German and Ottoman territories in the Middle East, Africa, and the Pacific were governed by the UK as League of Nations mandates. Some were administered directly by the UK, and others by British dominions – Nauru and the Territory of New Guinea by Australia, South West Africa by the Union of South Africa, and Western Samoa by New Zealand. Egypt became independent in 1922,", + "summary": "After World War I, several former German and Ottoman territories in the Middle East, Africa, and the Pacific were governed by the UK as League of Nations mandates.", + "extraction": "After World War I, several former German and Ottoman territories in the Middle East, Africa, and the Pacific were governed by the UK as League of Nations mandates." + } + ] + }, + { + "question": "Who set the record for longest field goal?", + "answer": [ + "Check: Document [1]", + "Output: The record for the longest field goal in an NFL game was set by Matt Prater at 64 yards [1].", + "Check: Document [2]", + "Output: but the record for the longest field goal at any level was 69 yards, kicked by collegiate kicker Ove Johansson in a 1976 Abilene Christian University football game against East Texas State University [2].", + "End" + ], + "docs": [ + { + "title": "Field goal", + "text": "toward its own end. The longest field goal kick in NFL history is 64 yards, a record set by Matt Prater on December 8, 2013. The previous record was 63, originally set by Tom Dempsey (1970) and then matched by Jason Elam (1998), Sebastian Janikowski (2011), David Akers (2012), and Graham Gano (2018). High school, college and most professional football leagues offer only a three-point field goal; however, some professional leagues have encouraged more rare kicks through \"four-point field goals\". NFL Europe encouraged long field goals of 50 yards or more by making those worth four points instead of three", + "summary": "The record for longest field goal in NFL history is 64 yards, set by Matt Prater on December 8, 2013.", + "extraction": "Matt Prater set the record for longest field goal with a 64-yard kick on December 8, 2013." + }, + { + "title": "Field goal range", + "text": "35 and 40 yard lines (closer in a crosswind) often will go for the more risky fourth down conversion rather than risk either the touchback or the missed field goal. The longest field goal in recorded football history was 69 yards, set by collegiate kicker Ove Johansson, who was born in Sweden, in a 1976 Abilene Christian University football game against East Texas State University (now Texas A&M Commerce) at Shotwell Stadium in Abilene. The longest successful field goal in the NFL was 64 yards and was completed by Matt Prater in 2013. The NCAA record is 67 yards held", + "summary": "Ove Johansson set the record for longest field goal in a 1976 college football game at 69 yards.", + "extraction": "The longest field goal in recorded football history was 69 yards, set by collegiate kicker Ove Johansson, who was born in Sweden, in a 1976 Abilene Christian University football game against East Texas State University (now Texas A&M Commerce) at Shotwell Stadium in Abilene." + }, + { + "title": "Field goal", + "text": "both end zones) is only 66 yards. Scaccia, while playing indoor football, attempted a 64-yard kick that was inches short of success, hitting the crossbar. Longer field goals have been attempted at times; the longest attempt in the NFL, which was well short and was kicked into the wind, was 76 yards, attempted by Sebastian Janikowski of the Oakland Raiders, in a September 28, 2008 game against the San Diego Chargers. NFL Europe rewarded kickers that successfully kicked a field goal of longer than 50 yards with a bonus point, making such field goals worth 4 points instead of 3;", + "summary": "Sebastian Janikowski set the record for the longest attempted field goal in the NFL at 76 yards in a game against the San Diego Chargers on September 28, 2008.", + "extraction": "Sebastian Janikowski of the Oakland Raiders set the record for longest field goal attempted in the NFL at 76 yards in a September 28, 2008 game against the San Diego Chargers." + }, + { + "title": "Field goal", + "text": "this accomplishment is not the official record. All of the above kicks were successful with the use of a kicking tee, which was banned by the NCAA after the 1988 season. The longest known drop-kicked field goal in college football was a 62-yard kick from Pat O'Dea, an Australian kicker who played on the Wisconsin Badgers football team. O'Dea's kick took place in a blizzard against Northwestern on November 15, 1898. The longest field goal in U Sports football history is 59 yards, by Niko Difonte of Calgary Dinos, playing against the UBC Thunderbirds on November 11, 2017. The field", + "summary": "The document mentions several long field goals in football history, including a 62-yard drop-kicked field goal by Pat O'Dea in 1898 and a 59-yard field goal by Niko Difonte in 2017. However, it does not provide information on who currently holds the record for the longest field goal.", + "extraction": "All of the above kicks were successful with the use of a kicking tee, which was banned by the NCAA after the 1988 season." + }, + { + "title": "Field goal range", + "text": "NFL and have been banned from NCAA since 1989) is 68 yards held by Fabrizio Scaccia, and the high school record 68 yards held by Dirk Borgognone; high school has wider goal posts and treats a field goal attempt that lands short in the field of play the same as a punt, making longer attempts much less risky. The indoor football record, with narrower and higher goal posts, is 63 yards (set by Aaron Mills), which is practically as long of a field goal as is possible in that variant of the sport, since the field in indoor football (including", + "summary": "The document mentions that the record for the longest field goal is held by Fabrizio Scaccia at 68 yards, but it does not mention who set the record.", + "extraction": "Fabrizio Scaccia set the record for longest field goal at 68 yards." + } + ] + }, + { + "question": "Who played galen in planet of the apes?", + "answer": [ + "Check: Document [2]", + "Output: In the 1968 film Planet of the Apes, Galen was played by Wright King [2].", + "Check: Document [1]", + "Output: And in the tv series Planet of the Apes, Galen was played by Roddy McDowall [1].", + "End" + ], + "docs": [ + { + "title": "Planet of the Apes", + "text": "installment. Jacobs died on June 27, 1973, bringing an end to the APJAC Productions era of the \"Planet of the Apes\" franchise. Former Fox executive Stan Hough took over as producer for the television project, titled \"Planet of the Apes\". CBS picked up the series for its 1974 autumn lineup. Ron Harper and James Naughton played Alan Virdon and Peter Burke, two 20th-century American astronauts who pass through a time warp to a future where apes subjugate humans (unlike the original film, the humans can speak). Roddy McDowall returned to the franchise as Galen, a chimpanzee who joins the astronauts.", + "summary": "Roddy McDowall played Galen in Planet of the Apes.", + "extraction": "Roddy McDowall played Galen in Planet of the Apes." + }, + { + "title": "Planet of the Apes (1968 film)", + "text": "chimpanzees: animal psychologist Zira (Kim Hunter) and surgeon Galen (Wright King). While unable to speak as his throat wound is healing, called \"Bright Eyes\" by Zira and placed with one of the captive primitive humans he later names \"Nova\", Taylor observes the enhanced society of talking apes and in a strict caste system: the gorillas being the military police, hunters and workers; the orangutans overseeing the affairs of government, science, and religion; and intellectual chimpanzees being mostly scientists. While their society is a theocracy similar to the beginnings of the human Industrial Era, the apes consider the primitive humans as", + "summary": "Wright King played Galen in the 1968 film \"Planet of the Apes\".", + "extraction": "Wright King played Galen in Planet of the Apes." + }, + { + "title": "Planet of the Apes (1968 film)", + "text": "Planet of the Apes (1968 film) Planet of the Apes is a 1968 American science fiction film directed by Franklin J. Schaffner. It stars Charlton Heston, Roddy McDowall, Kim Hunter, Maurice Evans, James Whitmore, James Daly and Linda Harrison. The screenplay by Michael Wilson and Rod Serling was loosely based on the 1963 French novel \"La Plan\u00e8te des Singes\" by Pierre Boulle. Jerry Goldsmith composed the groundbreaking avant-garde score. It was the first in a series of five films made between 1968 and 1973, all produced by Arthur P. Jacobs and released by 20th Century Fox. The film tells the", + "summary": "Charlton Heston played the lead role in Planet of the Apes (1968 film).", + "extraction": "Roddy McDowall played Galen in Planet of the Apes." + }, + { + "title": "Planet of the Apes", + "text": "Rupert Wyatt. To portray ape characters realistically, the production avoided practical effects in favor of performance capture acting, partnering with New Zealand visual effects company Weta Digital. Wyatt cast James Franco as Will Rodman, while veteran performance capture actor Andy Serkis signed on to star as Caesar. \"Rise\" debuted on August 5, 2011. Critics reviewed it positively, especially praising the visual effects and Serkis's performance. It was a major box office hit, taking in $482 million globally, more than five times its $93 million budget. Weta's special effects earned the film two Visual Effects Society Awards and an Oscar nomination", + "summary": "The document provides information about the production and cast of Planet of the Apes, but it does not mention who played Galen.", + "extraction": "Critics reviewed it positively, especially praising the visual effects and Serkis's performance." + }, + { + "title": "Planet of the Apes", + "text": "film stars Mark Wahlberg as astronaut Leo Davidson, who accidentally travels through a wormhole to a distant planet where talking apes enslave humans. He leads a human revolt and upends ape civilization by discovering that the apes evolved from the normal earth primates who had accompanied his mission, and arrived years before. Helena Bonham Carter played chimpanzee Ari, while Tim Roth played the human-hating chimpanzee General Thade. The film received mixed reviews; most critics believed it failed to compare to the original. Much of the negative commentary focused on the confusing plot and twist ending, though many reviewers praised the", + "summary": "Mark Wahlberg played Leo Davidson in Planet of the Apes. Irrelevant to the question of who played Galen.", + "extraction": "most critics believed it failed to compare to the original." + } + ] + } + ] +} \ No newline at end of file diff --git a/prompts/eli5_default.json b/prompts/eli5_default.json new file mode 100644 index 0000000000000000000000000000000000000000..53c352124fe5073b311a73f724696dd982128596 --- /dev/null +++ b/prompts/eli5_default.json @@ -0,0 +1,112 @@ +{ + "instruction": "Instruction: Write an accurate, engaging, and concise answer for the given question using only the provided search results (some of which might be irrelevant) and cite them properly. Use an unbiased and journalistic tone. Always cite for any factual claim. When citing several search results, use [1][2][3]. Cite at least one document and at most three documents in each sentence. If multiple documents support the sentence, only cite a minimum sufficient subset of the documents.", + "demo_sep": "\n\n\n", + "demo_prompt": "{INST}\n\nQuestion: {Q}\n\n{D}\nAnswer: {A}", + "doc_prompt": "Document [{ID}](Title: {T}): {P}\n", + "demos": [ + { + "question": "Why did New York City try to ban food donations to the poor?", + "answer": "New York City, under Mayor Michael Bloomberg's administration, banned citizens from donating food directly to homeless shelters because the city could not assess the salt, fat, and fiber content [1][2][3]. Bloomberg's administration was heavily criticized for losing their common sense by becoming too focused on what people eat [2].", + "docs": [ + { + "title": "The Future Of America", + "text": "believe that they are \u201chelping\u201d the homeless by passing such laws. In New York City, Mayor Bloomberg has banned citizens from donating food directly to homeless shelters and he is actually convinced that it was the right thing to do for the homeless\u2026 Mayor Michael Bloomberg\u2019s food police have struck again! Outlawed are food donations to homeless shelters because the city can\u2019t assess their salt, fat and fiber content, reports CBS 2\u2019s Marcia Kramer. Glenn Richter arrived at a West Side synagogue on Monday to collect surplus bagels \u2014 fresh nutritious bagels \u2014 to donate to the poor." + }, + { + "title": "mayor bloomberg", + "text": "Amuck: Bloomberg Bans Food Donations in New York City Food Might Be Salty or Too High in Calories, City Explains Washington, D.C. \u2013 New York Mayor Michael Bloomberg\u2019s administration is now banning all food being offered to the city\u2019s homeless shelters. New York City\u2019s bureaucrats have become so singularly focused on what people eat, says the National Center for Public Policy Research, that they\u2019ve lost their common sense. \u201cSo much for serving the homeless: The Bloomberg administration is now taking the term \u2018food police\u2019 to new depths, blocking food donations to all government-run facilities that serve the" + }, + { + "title": "New York City bans food donations - WND", + "text": "New York City bans food donations - WND Front Page Health U.S. New York City bans food donations Inability to control 'nutritional content' cited as reason New York City homeless shelters have Mayor Michael Bloomberg to thank for a halt in food donations, for which hungry families are waiting, according to one public policy advocate. \"The Bloomberg administration is now taking the term 'food police' to new depths, blocking food donations to all government-run facilities that serve the city's homeless,\" says Jeff Stier, a National Center for Public Policy Research senior fellow. Currently, no food can be given to government-run, New York City facilities, despite hungry crowds perfectly" + }, + { + "title": "New York City bans food donations - WND", + "text": "New York City bans food donations - WND Services didn't return WND calls. Stier told WND that he specifically was told by Diamond that the policy was tied to the nutritional guidelines set by the mayor. \"They can say that this ban on donations is a long-standing policy, but they can\u2019t document it,\" Stier told WND. \"I've also been told that there are numerous food shelves that have been accepting food donations, not just one.\" Stier is a member of a New York Synagogue that has donated food for over a decade. He is outraged that the DHS' response to his demand to know why the practice can" + }, + { + "title": "New York City bans food donations - WND", + "text": "New York City bans food donations - WND ban on donated food. In fact, it thrives because of food donations. New York City Rescue Mission has been providing food, clothing, shelter and spiritual hope for needy New Yorkers since 1872. \"We feed over 500 people a day, all through donations,\" said James Varnhagen, NYCRM director. \"Boxed food, canned food, prepared food, we take any food,\" he told WND. \"We couldn't survive without donations,\" he said." + } + ] + }, + { + "question": "What's the difference between Shia vs. Sunni Islam?", + "answer": "The main difference between Shia and Sunni Muslim is related to ideological heritage and issues of leadership [1]. This difference is first formed after the death of the Prophet Muhammad in 632 A.D. [1][2]. The ideological practice of the Sunni branch strictly follows Prophet Muhammad and his teachings, while the Shia branch follows Prophet Muhammad's son-in-law Ali [2]. Nowadays, Sunni and Shia are the major branches of Islam [3].", + "docs": [ + { + "title": "The Sunni vs Shia Divide - Explained - Globaloi", + "text": "centuries-long strained relationship between Sunnis and Shias. As a scholar of Islam and a public educator, I often field questions about Sunnis, Shias and the sects of Islam. What exactly is the Shia-Sunni divide? And what is its history? History of divide Both Sunnis and Shias \u2013 drawing their faith and practice from the Qur\u2019an and the life of the Prophet Muhammad \u2013 agree on most of the fundamentals of Islam. The differences are related more to historical events, ideological heritage and issues of leadership. The first and central difference emerged after the death of Prophet Muhammad in A.D. 632." + }, + { + "title": "What\u2019s the difference between Sunni and Shia Islam? \u2013 Macrosnaps", + "text": "What\u2019s the difference between Sunni and Shia Islam? Sunni and Shia identities (the 2 main branches of Islam) first formed around a dispute over leadership succession after the death of the Prophet Muhammad in 632 A.D. Sunni is the larger branch (estimated 85-90% of total world Muslim population) and it's adherents are referred to as \"people of the tradition of Muhammad\", while Shia are \"followers\" of Muhammad's son-in-law and cousin Ali. Sunnis rely heavily on the practice of the Prophet Muhammad and his teachings, the Shia view their ayatollahs as reflections of God on earth. What challenges does the anti-IS" + }, + { + "title": "Difference between Sunni and Shia Muslims | Sunni vs Shia Muslims", + "text": "of Muhammad, the last prophet of God. A follower of Islam is known as a Muslim. Many Muslims believe that their sole purpose is to worship and serve God, for which they have established five pillars of Islam that guides a Muslim on almost every aspect of life and society. Due to differences, Muslims have been divided into two primary sects: The Sunnis and the Shias. These two sects have many similarities and both consider themselves are Muslims, following the will of God. However, they are also different from each other in certain aspects. Both the Sunnis and the Shias," + }, + { + "title": "What is the difference between Shia and Sunni Islam? - Islam Stack Exchange", + "text": "What is the difference between Shia and Sunni Islam? - Islam Stack Exchange between Mutah marriage and Misyar marriage? What theological and historical factors distinguish Ibadi Islam from either Shia or Sunni schools? What are the principle/fundamental differences between Sunni and Shia? Nikah between a Sunni girl and Shia boy What is the difference between \u201cMubtalat-of-Wudu\u201d of Shia and Sunni? How can the Hadith be reliable when Sunnis and Shia follow different points of reference? Rejection of Mutawatir Hadith in Sunni Islam and Shia Islam" + }, + { + "title": "What is the difference between Sunni and Shia Islam? | Patrick Syder Travel", + "text": "What is the difference between Sunni and Shia Islam? | Patrick Syder Travel What is the difference between Sunni and Shia Islam? This Channel 4 link answers some of the key questions about the difference between Sunni and Shia Islam and alarmingly, the politics on what is happening and why, in Syria\u2026\u2026. http://www.channel4.com/news/sunni-shia-islam-muslim-syria-middle-east-key-questions \u2190 Ethiopia Appeal \u2013 Help sponsor a nurse to train and to help others G\u00f6bekli Tepe, Turkey: a new wonder of the ancient world by Jeremy Seal (Telegraph Travel Section 23/04/2013) \u2192" + } + ] + }, + { + "question": "What causes Bi-polar disorder?", + "answer": "Bipolar disorder is an emotional disorder that causes extreme mood swings between excitement and depression [1][3]. The spectrum of mood swing may span from days to months [1][2]. We are still not certain of the exact factors that cause such disorder, but genetics is considered a major factor [2][3].", + "docs": [ + { + "title": "Bi-polar disorder | definition of Bi-polar disorder by Medical dictionary", + "text": "bi-polar disorder | definition of bi-polar disorder by medical dictionary https://medical-dictionary.thefreedictionary.com/bi-polar+disorder (redirected from bi-polar disorder) related to bi-polar disorder: depression bipolar disorder, formerly known as manic depression, is a mood disorder that causes radical emotional changes and mood swings, from manic, restless highs to depressive, listless lows. most bipolar individuals experience alternating episodes of mania and depression. bipolar disorder is characterized by alternating manic episodes in which the individual feels abnormally euphoric, optimistic, and energetic and depressive periods in which the individual feels sad, hopeless, guilty, and sometimes suicidal. manic or depressive periods may last for days, weeks, or months" + }, + { + "title": "Mania and Bi-Polar", + "text": "can go from depressed to \u201csuper happy\u201d all in one day, or even in a few days, does not have a bi-polar disorder Bi-polar looks different depending on the severity of the symptoms. Most bi-polar diagnoses that are made are for bi-polar 2, with bi-polar 1 being much more rare. Bi-polar 1 is so severe that the individual will have periods of such agitation, or such reckless and seemingly foolish behavior that they put themselves or those around them in danger. It is not completely clear what causes bi-polar, but genetics seem to have a large role. The biggest factor" + }, + { + "title": "Bi-Polar disorder", + "text": "Bi-Polar disorder Bi-polar is generally a cyclic disease where individuals display depressive and elevated episodes at regular intervals. It is a disorder resulting from the imbalance of the chemicals in the brain that causes a lot of fluctuations of mood. It is a fact that we all experience happy and sad moods, but people with bi-polar disorder experience the changes in mood at an increased level. The cause of this disorder is not known completely. However, it is estimated that there are different factors responsible for it. It is often connected to a genetic component. People suffering from the Bi-polar disorder are" + }, + { + "title": "For Individuals \u2014 Adam Schwartz", + "text": "For Individuals \u2014 Adam Schwartz The information is extensive and covers a huge range of topics. Some of the topics include the different types of bi-polar, what it feels like, signs and symptoms, treatments and more. Black Dog Institute bi-polar causes resource specifically covers the variety of areas that could potentially be a cause of bi-polar disorder. Including genetics, environmental factors, pregnancy, and more. Black Dog Institute bi-polar treatments resource specifically covers multiple potential treatments options for bi-polar. Including management, types of psychological treatment, lifestyle changes, and more. Black Dog Institute bi-polar self-test resource is a short self-test for people who may be concerned if" + }, + { + "title": "Depression Bi-polar Disorder Symptoms 2019 | Win Over Depression", + "text": "Depression Bi-polar Disorder Symptoms 2019 | Win Over Depression signs and symptoms of bipolar disorder. Learn more about the common symptoms of bipolar depression that some patients may experience. Home \u00bb Trending Health News \u00bb 10 Warning Signs of Bipolar Disorder: Depression. One of the most serious symptoms of bipolar disorder is. Bi Polar Depression. SEVERE SWINGS What is bipolar disorder, is it the same as manic depression, what are the symptoms and is there a cure? Bipolar disorder, or manic depression, causes symptoms of mania and depression. Read about bipolar disorder treatment, medications, and causes of this. Learn more about the different types of bipolar disorder. Find out" + } + ] + }, + { + "question": "How do student loans affect getting a mortgage?", + "answer": "When applying for a mortgage, student loans can affect the debt to income ratio, which is a key factor in determining the amount that an individual can afford to pay for the mortgage [1]. While student loan repayments do not appear in an individual's credit history and do not affect credit scores, lenders do consider the amount of an individual's student loan repayments when assessing their mortgage application [1][2][3]. Some 83% of non-homeowners say student loan debt is preventing them from buying a home, according to the National Association of Realtors [2]. It is important to note that student loans do not prevent an individual from getting a mortgage [1].", + "docs": [ + { + "title": "Student Loans \u2013 How do they work? | The Financial Review", + "text": "typical debt. Student loan repayments do not appear in an individual\u2019s credit history, therefore there are no implications whatsoever. This also extends to applications for credit cards \u2013 student \u2018loans\u2019 are not acknowledged. One noteworthy aspect that is affected by student loans however, is mortgage applications. Nevertheless, it does not prevent an individual from getting a mortgage. For example, lenders will consider the amount of an individual\u2019s student loan repayments in order to assess the debt to income ratio and therefore establish the amount that the individual can afford to pay for the mortgage. Just as they do with other" + }, + { + "title": "How Does Student Loan Debt Affect Buying a Home? | Experian", + "text": "Rates & Affordability How Student Loans Affect Getting a Mortgage Student Loan Impact on Credit Scores Other Factors for Getting Approved for a Mortgage If you're a recent college grad and hope to become a homeowner in the near future, you should know that student loan debt could affect buying a home by making it more difficult to get a mortgage. Some 83% of non-homeowners say student loan debt is preventing them from buying a home, according to the National Association of Realtors (NAR). But while student loan payments can make it harder to save for a down payment on" + }, + { + "title": "Studentloanify - How your student loans affect your home mortgage prospects", + "text": "Though it may not seem fair, your student loan situation impacts your home mortgage outlook. Many people carry student loan debt, but it\u2019s the amount of the loan and how you handle your student loan repayment plan that will influence your ability to get a home mortgage as well as what your interest rate will be. Here are some specific factors about your student loan that will affect your home mortgage prospects. On your mortgage loan application, you will have to report how much your monthly student loan payment is. This amount will be deducted from your monthly gross income" + }, + { + "title": "How do student loans affect your credit score? | Student Loan Planner", + "text": "How do student loans affect your credit score? | Student Loan Planner Your credit score is the three-digit number that dictates a lot in your adult life. Whether you\u2019re applying for a mortgage or looking to get an auto loan, this seemingly arbitrary number determines whether you get approved for a loan and also affects your interest rate. If you\u2019re a student loan borrower you may wonder, \u201cDo student loans affect credit score?\u201d You might be especially curious if you\u2019re in the process of applying for a mortgage. Here\u2019s how student loans affect your credit score and what to know for big life events, like getting a mortgage. Do student loans affect" + }, + { + "title": "Does Student Loan Debt Affect Getting A Mortgage?", + "text": "Does Student Loan Debt Affect Getting A Mortgage? Home \u00bb Does Student Loan Debt Affect Getting A Mortgage? Last year, I helped answer a reader\u2019s question about applying for a mortgage while on Income Based Repayment. However, over the last several months, I\u2019ve been getting bombarded with questions about how student loan debt impacts your ability to get a mortgage. Maybe it\u2019s because the housing market is improving, or maybe it\u2019s because people are finally taking their student loan debt seriously. Anyway, I wanted to share a few reader questions and then look at whether student loan debt affects getting a mortgage. Here are the reader questions I\u2019ve" + } + ] + } + ] +} \ No newline at end of file diff --git a/prompts/qampari_default.json b/prompts/qampari_default.json new file mode 100644 index 0000000000000000000000000000000000000000..779bbbbd7313e7a5e658a73ec5f4b58eef6ebc11 --- /dev/null +++ b/prompts/qampari_default.json @@ -0,0 +1,112 @@ +{ + "instruction": "Instruction: Provide a list of accurate answers for the given question using only the provided search results (some of which might be irrelevant) and cite them properly. Always cite one and only one document for each answer. Separate answers by commas. For questions that have more than 5 answers, write at least 5 answers.", + "demo_sep": "\n\n\n", + "demo_prompt": "{INST}\n\nQuestion: {Q}\n\n{D}\nAnswer: {A}", + "doc_prompt": "Document [{ID}](Title: {T}): {P}\n", + "demos": [ + { + "question": "Which books were written by Nevil Shute?", + "answer": "Marazan [1], Stephen Morris [1], Beyond the Black Stump [2], Lonely Road [2], The Chequer Board [2], In the Wet [2], Trustee from the Toolroom [2], Round the Bend [2], No Highway [3], Ruined City [3], On the Beach [3].", + "docs": [ + { + "title": "Nevil Shute", + "text": "early stages. My congratulations.\" His celebrity as a writer caused the Ministry of Information to send him to the Normandy Landings on 6 June 1944 and later to Burma as a correspondent. He finished the war with the rank of lieutenant commander in the Royal Navy Volunteer Reserves (RNVR). Shute's first novel, \"Stephen Morris\", was written in 1923, but not published until 1961. His first published novel was \"Marazan\", which came out in 1926. After that he averaged one novel every two years through the 1950s, with the exception of a six-year hiatus while he was establishing his own aircraft" + }, + { + "title": "Nevil Shute", + "text": "theme is the bridging of social barriers such as class (\"Lonely Road\" and \"Landfall\"), race (\"The Chequer Board\"), or religion (\"Round the Bend\"). The Australian novels are individual hymns to that country, with subtle disparagement of the mores of the United States (\"Beyond the Black Stump\") and overt antipathy towards the post-World War II socialist government of Shute's native Britain (\"The Far Country\" and \"In the Wet\"). Shute's heroes tended to be like himself: middle class solicitors, doctors, accountants, bank managers, engineers, generally university graduates. However (as in \"Trustee from the Toolroom\"), Shute valued the honest artisans and their social" + }, + { + "title": "Nevil Shute", + "text": "construction company, Airspeed Ltd. His popularity grew slowly with each novel, but he became much more famous after the publication of \"On the Beach\" in 1957. Shute's novels are written in a simple, highly readable style, with clearly delineated plot lines. Where there is a romantic element, sex is referred to only obliquely. Many of the stories are introduced by a narrator who is not a character in the story. The most common theme in Shute's novels is the dignity of work, spanning all classes, whether an Eastern European bar \"hostess\" (\"Ruined City\") or brilliant boffin (\"No Highway\"). Another recurrent" + }, + { + "title": "The Chequer Board", + "text": "the Burmese people\", both of which are central to the book's story. Shute was concerned that sales of the book in the United States would be negatively impacted by the book's open-minded handling of racial issues; as it turned out, sales soared. Shute and his wife traveled the U.S. on Greyhound buses to \"\"get in touch with the man on the street,\"\" finding the experience refreshing. Afterwards he wrote \"\"Sincerity is the first attribute for making money in the business of writing novels.\"\" The Chequer Board The Chequer Board is a novel by Nevil Shute, first published in the United" + }, + { + "title": "In the Wet", + "text": "had used the idea of multiple votes for merit in his short story \"The Curious Republic of Gondour\". In the Wet In The Wet is a novel by Nevil Shute that was first published in the United Kingdom in 1953. It contains many of the typical elements of a hearty and adventurous Shute yarn such as flying, the future, mystic states, and ordinary people doing extraordinary things. The story is opened by its initial narrator \u2013 an Anglican priest in the Bush Brotherhood named Roger Hargreaves \u2013 who describes his ordinary circumstances in a large parish of the Australian outback" + } + ] + }, + { + "question": "Which film has Gong Li as a member of its cast?", + "answer": "The Story of Qiu Ju [1], Farewell My Concubine [2], Flirting Scholar [2], The Monkey King 2 [3], Mulan [3], Saturday Fiction [3], Coming Home [3].", + "docs": [ + { + "title": "Gong Li", + "text": "Gong Li Gong Li (born 31 December 1965) is a Chinese-born Singaporean film actress. She achieved international prominence through her close collaborations with Chinese director Zhang Yimou and won the Volpi Cup for Best Actress at Venice for her performance in his 1992 film \"The Story of Qiu Ju\". She has been credited with helping to bring Chinese cinema to prominence in Europe and the United States. In 2006, she was voted the most beautiful woman in China. Gong has won numerous accolades for her work as an actress; she won the New York Film Critics Circle Award for Best" + }, + { + "title": "Gong Li", + "text": "making her realize that she has assisted the dark cynical system. In 1993, she received a New York Film Critics Circle award for her role in \"Farewell My Concubine\" (1993). Directed by Chen Kaige, the film was her first major role with a director other than Zhang Yimou. In the same year, she was awarded with the Berlinale Camera at the 43rd Berlin International Film Festival. \"Premiere\" magazine ranked her performance in \"Farewell My Concubine\" as the 89th greatest performance of all time. She also worked with renowned director Stephen Chow in comedy films \"\" (1991) and \"Flirting Scholar\" (1993)." + }, + { + "title": "Gong Li", + "text": "International Film Festival. Later that same year, she reunited with Zhang Yimou for the film \"Coming Home\", which is set during the throes of the Cultural Revolution; this film was their first collaboration since 2006. In 2016, Gong took on her first action role in \"The Monkey King 2\", playing the White Bone Demon. In 2018, Gong was cast in Lou Ye's period drama \"Saturday Fiction\", where she plays an actress who is working undercover gathering intelligence for the Allies. That year, she was also cast in the live-action adaptation of the 1998 Disney animated film \"Mulan\", as an unspecified" + }, + { + "title": "Zhang Yimou", + "text": "in Zhang's earlier films. \"Raise the Red Lantern\" was nominated in the Best Foreign Language Film category at the 1992 Academy Awards, becoming the second Chinese film to earn this distinction (after Zhang's \"Ju Dou\"). It eventually lost out to Gabriele Salvatores's \"Mediterraneo\". Zhang's next directorial work, \"The Story of Qiu Ju\", in 1992, once again starring Gong Li in the lead role. The film, which tells the tale of a peasant woman seeking justice for her husband after he was beaten by a village official, was a hit at film festivals and won the Golden Lion award at the" + }, + { + "title": "Gong Li", + "text": "Gong Li Gong Li (born 31 December 1965) is a Chinese-born Singaporean film actress. She achieved international prominence through her close collaborations with Chinese director Zhang Yimou and won the Volpi Cup for Best Actress at Venice for her performance in his 1992 film \"The Story of Qiu Ju\". She has been credited with helping to bring Chinese cinema to prominence in Europe and the United States. In 2006, she was voted the most beautiful woman in China. Gong has won numerous accolades for her work as an actress; she won the New York Film Critics Circle Award for Best" + } + ] + }, + { + "question": "In which years did Patti LaBelle publish music?", + "answer": "2006 [1], 1977 [2], 2004 [3], 2005 [3], 2000 [3], 2006 [3].", + "docs": [ + { + "title": "The Gospel According to Patti LaBelle", + "text": "The Gospel According to Patti LaBelle The Gospel According to Patti LaBelle is the first gospel album released by singer Patti LaBelle, released in November 2006. This project began three years ago when Patti's late musical director and close friend Budd Ellison told a skeptical LaBelle that \"it's now or never, Patti.\" The album is dedicated to his memory as he succumbed to prostate cancer before the album saw a release. The album was released on November 21, 2006 through indie label Umbrella/Bungalow Records, also home to Carl Thomas, Rodney Jerkins, Dean \"DC\" Charles, and other artists. \"The Gospel According" + }, + { + "title": "Patti LaBelle (album)", + "text": "scaled the high sixties on the \"Billboard\" R&B chart, it soon became one of her famous show-stoppers while performing the song. LaBelle performed the song at her first solo concert in London, getting a standing ovation, which helped to give LaBelle motivation to continue her career. The album, when released, performed successfully, reaching number 62 on the \"Billboard\" 200 and number 31 on the R&B albums chart, while critics hailed the album. Patti LaBelle (album) Patti LaBelle is the debut solo album by singer Patti LaBelle, released in 1977. The first album LaBelle recorded after sixteen years fronting the band" + }, + { + "title": "Patti LaBelle", + "text": "win. In 2000, LaBelle released her final MCA album, \"When a Woman Loves\", before signing with Def Soul Classics to release the 2004 album, \"Timeless Journey\". Following the release of her 2005 covers album, \"Classic Moments\", LaBelle engaged in a rivalry with Antonio \"L.A.\" Reid over the direction of her career, leading to her leaving the label.In the same year, the World Music Awards recognized her years in the music business by awarding her the Legend Award. In 2006, she released her first gospel album, \"The Gospel According to Patti LaBelle\" on the Bungalo label, the album later peaking at" + }, + { + "title": "Patti LaBelle", + "text": "Patti LaBelle Patti LaBelle (born Patricia Louise Holt; May 24, 1944) is an American singer, actress, and entrepreneur. LaBelle began her career in the early 1960s as lead singer and front woman of the vocal group, Patti LaBelle and the Bluebelles. Following the group's name change to Labelle in the early 1970s, they released the iconic disco song \"Lady Marmalade\" and the group later became the first African-American vocal group to land the cover of \"Rolling Stone\" magazine. After the group split in 1976, LaBelle began a successful solo career, starting with her critically acclaimed debut album, which included the" + }, + { + "title": "The Gospel According to Patti LaBelle", + "text": "Billboard's Top Gospel Albums chart for 17 weeks. \"Where Love Begins,\" a duet with Yolanda Adams was played frequently on R&B and gospel radio stations and debuted at #68 on Billboard's Hot R&B/Hip-Hop tracks. The second single \"Anything\" featuring Kanye West, Mary Mary and Consequence hit #64 on Billboards Hot R&B/Hip-Hop tracks. In 2008, the album was nominated for a Dove Award for Contemporary Gospel Album of the Year at the 39th GMA Dove Awards. The Gospel According to Patti LaBelle The Gospel According to Patti LaBelle is the first gospel album released by singer Patti LaBelle, released in November" + } + ] + }, + { + "question": "Glenn Ford was a member of cast in which film?", + "answer": "So Ends Our Night [1], Heaven with a Barbed Wire Fence [1], Happy Birthday to Me [2], The Greatest Gift [2], The Gift [2], The Brotherhood of the Bell [3].", + "docs": [ + { + "title": "Glenn Ford", + "text": "name came from his father's hometown of Glenford, Alberta. His first major movie part was in the 1939 film, \"Heaven with a Barbed Wire Fence\". Top Hollywood director John Cromwell was impressed enough with his work to borrow him from Columbia for the independently produced drama, \"So Ends Our Night\" (1941), where Ford delivered a poignant portrayal of a 19-year-old German exile on the run in Nazi-occupied Europe. Working with Academy Award-winning Fredric March and wooing (onscreen) 30-year-old Margaret Sullavan, recently nominated for an Oscar, Ford's shy, ardent young refugee riveted attention even in such stellar company. \"Glenn Ford, a" + }, + { + "title": "Glenn Ford", + "text": "were Westerns. He suggested doing a Western series, instead, which resulted in the \"modern-day Western\" series, \"Cade's County\". Ford played southwestern Sheriff Cade for one season (1971\u20131972) in a mix of police mystery and western drama. In \"The Family Holvak\" (1975\u20131976), Ford portrayed a Depression-era preacher in a family drama, reprising the same character he had played in the TV film, \"The Greatest Gift\". In 1978 Ford was host, presenter and narrator of the disaster documentary series 'When Havoc Struck'. In 1981, Ford co-starred with Melissa Sue Anderson in the slasher film \"Happy Birthday to Me\". In 1991, Ford agreed" + }, + { + "title": "CBS Thursday Night Movie", + "text": "Night Movie\" opened its fall schedule with the premiere of a low-budget, made-for-TV movie, rather than a proven Hollywood blockbuster guaranteed to lure mass viewership, it became CBS's way of declaring its commitment to product that, although cheaply manufactured, was nevertheless new and topical. In this case, the movie was \"The Brotherhood of the Bell\", and the film's star was Glenn Ford, a movie actor who had never appeared in a television-film. In fact, before shooting on the project even began, Ford had been warned by friends in the industry that he would hate the experience. Instead, the actor reported" + }, + { + "title": "The Trouble with Girls (film) ", + "text": "with Charlene, but when she refuses to give in, he deceives her and uses the local police force to be sure that she must leave on the train with the rest of the troupe. Cast notes In June 1959 it was announced that Don Mankiewicz would write a screenplay of an unpublished story by Mauri Grashin, Day Keene, and Dwight Babcock. By December 1960, with the project titled \"Chautauqua\", MGM was ready to make the film with Glenn Ford. Rumours circulating in Hollywood at the time stated that Presley would co-star with Ford, Hope Lange, and Arthur O'Connell, but nothing" + }, + { + "title": "Trouble in the Glen", + "text": "Mel Ferrer. It was Orson Welles' fifth British movie in six months. Filming started 15 December 1953. The film received very poor reviews. Trouble in the Glen Trouble in the Glen is a 1954 British comedy film directed by Herbert Wilcox and starring Margaret Lockwood, Orson Welles, Forrest Tucker and Victor McLaglen. It is loosely based on Maurice Walsh's 1950 novel of the same name. It was filmed in Trucolor for Republic Pictures. After moving from South America to the Scottish Highlands, millionaire Sanin Cejador y Mengues (Welles) reassumes the title of laird of Glen Easan, which he inherited from" + } + ] + } + ] +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..98b93f15ec69ccf00115da2dc4f0a6e2b45d4b1f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,152 @@ +absl-py==2.0.0 +anyio==3.6.2 +appnope==0.1.3 +argon2-cffi==21.3.0 +argon2-cffi-bindings==21.2.0 +arrow==1.2.3 +asttokens==2.2.1 +astunparse==1.6.3 +attrs==22.2.0 +backcall==0.2.0 +beautifulsoup4==4.11.2 +bleach==6.0.0 +cachetools==5.3.1 +certifi==2023.7.22 +cffi==1.15.1 +charset-normalizer==3.3.0 +comm==0.1.2 +cycler==0.11.0 +debugpy==1.6.6 +decorator==5.1.1 +defusedxml==0.7.1 +distlib==0.3.6 +docopt==0.6.2 +executing==1.2.0 +fastjsonschema==2.16.3 +filelock==3.9.0 +flatbuffers==23.5.26 +fqdn==1.5.1 +fsspec==2024.3.1 +gast==0.5.4 +google-auth==2.23.3 +google-auth-oauthlib==1.0.0 +google-pasta==0.2.0 +grpcio==1.59.0 +h5py==3.10.0 +huggingface-hub==0.22.2 +idna==3.4 +ipykernel==6.21.2 +ipython==8.12.3 +ipython-genutils==0.2.0 +ipywidgets==8.0.4 +isoduration==20.11.0 +jedi==0.18.2 +Jinja2==3.1.2 +joblib==1.2.0 +jsonpointer==2.3 +jsonschema==4.17.3 +jupyter==1.0.0 +jupyter-console==6.6.1 +jupyter-events==0.6.3 +jupyter_client==8.0.3 +jupyter_core==5.2.0 +jupyter_server==2.3.0 +jupyter_server_terminals==0.4.4 +jupyterlab-pygments==0.2.2 +jupyterlab-widgets==3.0.5 +keras==2.14.0 +kiwisolver==1.3.2 +libclang==16.0.6 +Markdown==3.5 +MarkupSafe==2.1.2 +matplotlib==3.4.3 +matplotlib-inline==0.1.6 +mistune==2.0.5 +ml-dtypes==0.2.0 +mpmath==1.3.0 +nbclassic==0.5.2 +nbclient==0.7.2 +nbconvert==7.16.4 +nbformat==5.7.3 +nest-asyncio==1.5.6 +networkx==3.3 +notebook==6.5.2 +notebook_shim==0.2.2 +numpy==1.26.1 +oauthlib==3.2.2 +opt-einsum==3.3.0 +packaging==23.0 +pandas==1.5.3 +pandocfilters==1.5.0 +parso==0.8.3 +pexpect==4.8.0 +pickleshare==0.7.5 +Pillow==8.4.0 +pipreqs==0.5.0 +platformdirs==3.0.0 +prometheus-client==0.16.0 +prompt_toolkit==3.0.47 +protobuf==4.24.4 +psutil==5.9.4 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyasn1==0.5.0 +pyasn1-modules==0.3.0 +pycparser==2.21 +Pygments==2.14.0 +pyparsing==3.0.4 +pyrsistent==0.19.3 +python-dateutil==2.8.2 +python-json-logger==2.0.7 +pytz==2022.7.1 +PyYAML==6.0 +pyzmq==25.0.0 +qtconsole==5.4.0 +QtPy==2.3.0 +regex==2023.12.25 +requests==2.31.0 +requests-oauthlib==1.3.1 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rsa==4.9 +safetensors==0.4.2 +scikit-learn==1.2.1 +scipy==1.10.1 +Send2Trash==1.8.0 +six==1.16.0 +sniffio==1.3.0 +soupsieve==2.4 +stack-data==0.6.2 +sympy==1.12.1 +tensorboard==2.14.1 +tensorboard-data-server==0.7.1 +tensorflow==2.14.0 +tensorflow-estimator==2.14.0 +tensorflow-io-gcs-filesystem==0.34.0 +tensorflow-macos==2.14.0 +termcolor==2.3.0 +terminado==0.17.1 +threadpoolctl==3.1.0 +tinycss2==1.2.1 +tokenizers==0.15.2 +torch==2.5.0.dev20240702 +torchaudio==2.4.0.dev20240702 +torchvision==0.20.0.dev20240702 +tornado==6.2 +tqdm==4.66.2 +traitlets==5.9.0 +transformers==4.39.3 +typing_extensions==4.8.0 +uri-template==1.2.0 +urllib3==2.0.7 +virtualenv==20.19.0 +wcwidth==0.2.6 +webcolors==1.12 +webencodings==0.5.1 +websocket-client==1.5.1 +Werkzeug==3.0.0 +widgetsnbextension==4.0.5 +wikipedia==1.4.0 +Wikipedia-API==0.6.0 +wrapt==1.14.1 +yarg==0.1.9 diff --git a/test.py b/test.py new file mode 100644 index 0000000000000000000000000000000000000000..b15add35275aef114e80c0f0db7a7d388b65906b --- /dev/null +++ b/test.py @@ -0,0 +1,37 @@ +import re +from bs4 import BeautifulSoup + +def parse_html_prompt(input_str): + soup = BeautifulSoup(input_str, "html.parser") + + # 处理

内的内容 + p_content = soup.find("p").decode_contents().replace("
", "\n") + p_content = re.sub(r']*>(.*?)', r'<\1>', p_content) + template = p_content.strip().replace('
', '').replace(' ', '').replace('
', '') + + # 解析 component-item + components = {} + for item in soup.find_all("div", class_="component-item"): + key_span = item.find("div", class_="component-key").find("span") + key = key_span.get_text(strip=True) if key_span else "" + value_div = item.find("div", class_="component-value") + value_content = value_div.decode_contents() + value_content = re.sub(r']*>(.*?)', r'{\1}', value_content) + components[key] = value_content.strip().replace('
', '').replace('
', '') + + # 解析 self-info-item + self_prompt = {} + for item in soup.find_all("div", class_="self-info-item"): + key_span = item.find("div", class_="component-key").find("span") + key = key_span.get_text(strip=True) if key_span else "" + value_div = item.find("div", class_="component-value") + value = value_div.get_text(strip=True) if value_div else "" + self_prompt[key] = value.replace('
', '').replace('
', '') + + return { + 'template': template, + 'components': components, + 'self_prompt': self_prompt + } + +#print(parse_html(info)) \ No newline at end of file diff --git a/word_level.py b/word_level.py new file mode 100644 index 0000000000000000000000000000000000000000..a0a6df24684f99b723105dafc25fd03ae2d53181 --- /dev/null +++ b/word_level.py @@ -0,0 +1,168 @@ +import json +import re +import numpy as np + +def all_normalize(obj): + all_values = [] + for output_sent_result in obj: + for each_doc in output_sent_result: + for each_span in each_doc: + all_values.append(each_span[1]) + max_val = max(all_values) + min_val = min(all_values) + for output_sent_result in obj: + for i, each_doc in enumerate(output_sent_result): + for j, each_span in enumerate(each_doc): + each_span = (each_span[0], (each_span[1] - min_val) / (max_val - min_val)) + output_sent_result[i][j] = each_span + return obj + + +def load_json(file_path): + + with open(file_path, 'r') as file: + data = file.read() + if file_path.endswith('.jsonl'): + data = f'[{'},{'.join(data.split("}\n{"))}]' + objects = json.loads(data) + return objects + +def ma(text): + pattern = r"Document \[\d+\]\(Title:[^)]+\):" + + match = re.search(pattern, text) + + if match: + index = match.end() + return index + else: + return 0 + +def write_json(file_path, data): + with open(file_path, 'w') as json_file: + json.dump(data, json_file, indent=4) + +def split_by_docs(scores, docs_text, doc_tokens): + assert len(scores) == len(doc_tokens) + sep = '\n\n' + docs = docs_text.strip().split(sep) + doc_lens = [len(doc) for doc in docs] + doc_end_idx = [sum(doc_lens[:i+1]) for i in range(len(doc_lens))] + print(doc_end_idx) + + last_tokens = [0] + for i, token in enumerate(doc_tokens): + next_token = doc_tokens[i+1] if i+1 < len(doc_tokens) else None + if token == "<0x0A>" and next_token == "<0x0A>": # FOR LLAMA2 ONLY + last_tokens.append(i + 1) + for i, idx in enumerate(last_tokens[1:]): + pre_idx = last_tokens[i] + curr_tokens = doc_tokens[pre_idx:idx + 1] + curr_tokens = [token for token in curr_tokens if token != "<0x0A>"] + curr_doc = ''.join(curr_tokens) + while curr_doc.startswith('\u2581'): + curr_doc = curr_doc[1:] + #print(curr_doc) + #print(docs[i]) + #assert len(curr_doc) == len(docs[i]), f"{len(curr_doc)} != {len(docs[i])}" + doc_num = len(last_tokens) - 1 + scores_per_doc = [[] for _ in range(doc_num)] + curr_doc_idx = 0 + skip = False + curr_char_idx = -2 # magic number + for i, (score, token) in enumerate(zip(scores, doc_tokens)): + if skip: + skip = False + continue + if i == 0: + token = token[1:] # remove the first space + if token == "<0x0A>": + curr_doc_idx += 1 + curr_char_idx = -2 + skip = True # skip the next token + continue + scores_per_doc[curr_doc_idx].append((curr_char_idx, score)) + curr_char_idx += len(token) + #print(scores_per_doc[0]) + for i, doc in enumerate(docs): + start = ma(doc) - 2 + #print(start) + scores_per_doc[i] = list(filter(lambda x: x[0] >= start, scores_per_doc[i])) + all_values = [] + for scores in scores_per_doc: + # normalize + all_values.extend([score[1] for score in scores]) + max_val = max(all_values) + min_val = min(all_values) + for scores in scores_per_doc: + for i, score in enumerate(scores): + scores[i] = (score[0], (score[1] - min_val) / (max_val - min_val)) + + return scores_per_doc + +def span_to_doc(results): + for res in results: + span_level = res['span_level'] + doc_level = [] + for output_sent_result in span_level: + doc_level.append([np.mean([span[1] for span in doc]) for doc in output_sent_result]) + res['doc_level'] = doc_level + return results + + + + +def word_level_attribute(raw, _i): + res = load_json(f'MIRAGE/internal_res/res_attr_dict-{_i}.json') + + input_text = res["input_context"] + input = res["input_context_tokens"] + output = res["output_current"] + output_tokens =res["output_current_tokens"] + token_lens = [len(x) for x in output_tokens] + cci_scores = res["cci_scores"] + splited_output = raw[_i]["output"] + all_lens = [len(x) for x in splited_output] + end_token_idx = [sum(token_lens[:i+1]) for i in range(len(token_lens))] + end_idx = [sum(all_lens[:i+1]) for i in range(len(all_lens))] + end_idx = [len(list(filter(lambda x: x < idx, end_token_idx))) for idx in end_idx] + belong_sents = [[] for _ in range(len(splited_output))] + for token_cci in cci_scores: + token_idx = token_cci['cti_idx'] + for i, idx in enumerate(end_idx): + if token_idx < idx: + belong_sents[i].append(token_cci) + break + scores = [] + for i, sent in enumerate(belong_sents): + weighted_scores = [token_cci["cti_score"]*np.array(token_cci["input_context_scores"]) for token_cci in sent] + #weighted_scores = [np.array(token_cci["input_context_scores"]) for token_cci in sent] + sum_scores = np.sum(weighted_scores, axis=0) + #max_scores = np.max(weighted_scores, axis=0) + scores.append(sum_scores) + #scores.append(max_scores) + finals = [] + + for score in scores: + doc_scores = split_by_docs(score, input_text, input) + finals.append(doc_scores) + + + doc_finals = [[] for _ in range(len(finals))] + for i, output_sent_result in enumerate(finals): + docs = [] + for doc in output_sent_result: + doc_score = sum([score[1] for score in doc]) + docs.append(doc_score) + doc_finals[i] = docs + print(doc_finals) + + + + raw[_i]["word_level"] = finals + raw[_i]["doc_level"] = doc_finals + +raw = load_json('results.json') +for i in range(len(raw)): + word_level_attribute(raw, i) +write_json('result_.json', raw) \ No newline at end of file