Upload folder using huggingface_hub

.gitignore

@@ -0,0 +1,2 @@
+saved_models
+multilingual_word_embeddings

LICENSE.md

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2021 Jivnesh Sandhan
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,3 +1,52 @@
----
-license: apache-2.0
----
+Official code for the paper ["A Little Pretraining Goes a Long Way: A Case Study on Dependency Parsing Task for Low-resource Morphologically Rich Languages"](https://arxiv.org/abs/2102.06551).
+If you use this code please cite our paper.
+
+## Requirements
+
+* Python 3.7 
+* Pytorch 1.1.0 
+* Cuda 9.0 
+* Gensim 3.8.1
+
+We assume that you have installed conda beforehand. 
+
+```
+conda install pytorch==1.1.0 torchvision==0.3.0 cudatoolkit=9.0 -c pytorch
+pip install gensim==3.8.1
+```
+## Data
+* Pretrained FastText embeddings for Sanskrit can be obtained from [here](https://drive.google.com/drive/folders/1JJMBjUZdqUY7WLYefBbA2zKaMHH3Mm18?usp=sharing). Make sure that `.vec` file is placed at approprite position.
+* For Multilingual experiments, we use [UD treebanks](https://universaldependencies.org/) and [Pretrained FastText embeddings](https://fasttext.cc/docs/en/crawl-vectors.html)
+
+
+## How to train model
+If you want to run complete model pipeline: (1) Pretraining (2) Integration, then simply run bash script `run_san_LCM.sh`.
+
+```bash
+bash run_san_LCM.sh
+
+```
+
+
+## Citation
+
+If you use our tool, we'd appreciate if you cite the following paper:
+
+```
+@inproceedings{sandhan-etal-2021-little,
+    title = "A Little Pretraining Goes a Long Way: A Case Study on Dependency Parsing Task for Low-resource Morphologically Rich Languages",
+    author = "Sandhan, Jivnesh  and Krishna, Amrith  and Gupta, Ashim  and Behera, Laxmidhar  and Goyal, Pawan",
+    booktitle = "Proceedings of the 16th Conference of the European Chapter of the Association for Computational Linguistics: Student Research Workshop",
+    month = apr,
+    year = "2021",
+    address = "Online",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/2021.eacl-srw.16",
+    doi = "10.18653/v1/2021.eacl-srw.16",
+    pages = "111--120",
+    abstract = "Neural dependency parsing has achieved remarkable performance for many domains and languages. The bottleneck of massive labelled data limits the effectiveness of these approaches for low resource languages. In this work, we focus on dependency parsing for morphological rich languages (MRLs) in a low-resource setting. Although morphological information is essential for the dependency parsing task, the morphological disambiguation and lack of powerful analyzers pose challenges to get this information for MRLs. To address these challenges, we propose simple auxiliary tasks for pretraining. We perform experiments on 10 MRLs in low-resource settings to measure the efficacy of our proposed pretraining method and observe an average absolute gain of 2 points (UAS) and 3.6 points (LAS).",
+}
+```
+
+## Acknowledgements
+Much of the base code is from ["DCST Implementation"](https://github.com/rotmanguy/DCST)

examples/GraphParser.py

@@ -0,0 +1,703 @@
+from __future__ import print_function
+import sys
+from os import path, makedirs
+
+sys.path.append(".")
+sys.path.append("..")
+
+import argparse
+from copy import deepcopy
+import json
+import numpy as np
+import torch
+import torch.nn as nn
+from collections import namedtuple
+from utils.io_ import seeds, Writer, get_logger, prepare_data, rearrange_splits
+from utils.models.parsing_gating import BiAffine_Parser_Gated
+from utils import load_word_embeddings
+from utils.tasks import parse
+import time
+from torch.nn.utils import clip_grad_norm_
+from torch.optim import Adam, SGD
+import uuid
+import pdb
+uid = uuid.uuid4().hex[:6]
+
+logger = get_logger('GraphParser')
+
+def read_arguments():
+    args_ = argparse.ArgumentParser(description='Sovling GraphParser')
+    args_.add_argument('--dataset', choices=['ontonotes', 'ud'], help='Dataset', required=True)
+    args_.add_argument('--domain', help='domain/language', required=True)
+    args_.add_argument('--rnn_mode', choices=['RNN', 'LSTM', 'GRU'], help='architecture of rnn',
+                       required=True)
+    args_.add_argument('--gating',action='store_true', help='use gated mechanism')
+    args_.add_argument('--num_gates', type=int, default=0, help='number of gates for gating mechanism')
+    args_.add_argument('--num_epochs', type=int, default=200, help='Number of training epochs')
+    args_.add_argument('--batch_size', type=int, default=64, help='Number of sentences in each batch')
+    args_.add_argument('--hidden_size', type=int, default=256, help='Number of hidden units in RNN')
+    args_.add_argument('--arc_space', type=int, default=128, help='Dimension of tag space')
+    args_.add_argument('--arc_tag_space', type=int, default=128, help='Dimension of tag space')
+    args_.add_argument('--num_layers', type=int, default=1, help='Number of layers of RNN')
+    args_.add_argument('--num_filters', type=int, default=50, help='Number of filters in CNN')
+    args_.add_argument('--kernel_size', type=int, default=3, help='Size of Kernel for CNN')
+    args_.add_argument('--use_pos', action='store_true', help='use part-of-speech embedding.')
+    args_.add_argument('--use_char', action='store_true', help='use character embedding and CNN.')
+    args_.add_argument('--word_dim', type=int, default=300, help='Dimension of word embeddings')
+    args_.add_argument('--pos_dim', type=int, default=50, help='Dimension of POS embeddings')
+    args_.add_argument('--char_dim', type=int, default=50, help='Dimension of Character embeddings')
+    args_.add_argument('--initializer', choices=['xavier'], help='initialize model parameters')
+    args_.add_argument('--opt', choices=['adam', 'sgd'], help='optimization algorithm')
+    args_.add_argument('--momentum', type=float, default=0.9, help='momentum of optimizer')
+    args_.add_argument('--betas', nargs=2, type=float, default=[0.9, 0.9], help='betas of optimizer')
+    args_.add_argument('--learning_rate', type=float, default=0.01, help='Learning rate')
+    args_.add_argument('--decay_rate', type=float, default=0.05, help='Decay rate of learning rate')
+    args_.add_argument('--schedule', type=int, help='schedule for learning rate decay')
+    args_.add_argument('--clip', type=float, default=5.0, help='gradient clipping')
+    args_.add_argument('--gamma', type=float, default=0.0, help='weight for regularization')
+    args_.add_argument('--epsilon', type=float, default=1e-8, help='epsilon for adam')
+    args_.add_argument('--p_rnn', nargs=2, type=float, required=True, help='dropout rate for RNN')
+    args_.add_argument('--p_in', type=float, default=0.33, help='dropout rate for input embeddings')
+    args_.add_argument('--p_out', type=float, default=0.33, help='dropout rate for output layer')
+    args_.add_argument('--arc_decode', choices=['mst', 'greedy'], help='arc decoding algorithm', required=True)
+    args_.add_argument('--unk_replace', type=float, default=0.,
+                       help='The rate to replace a singleton word with UNK')
+    args_.add_argument('--punct_set', nargs='+', type=str, help='List of punctuations')
+    args_.add_argument('--word_embedding', choices=['random', 'glove', 'fasttext', 'word2vec'],
+                       help='Embedding for words')
+    args_.add_argument('--word_path', help='path for word embedding dict - in case word_embedding is not random')
+    args_.add_argument('--freeze_word_embeddings', action='store_true', help='frozen the word embedding (disable fine-tuning).')
+    args_.add_argument('--freeze_sequence_taggers', action='store_true', help='frozen the BiLSTMs of the pre-trained taggers.')
+    args_.add_argument('--char_embedding', choices=['random','hellwig'], help='Embedding for characters',
+                       required=True)
+    args_.add_argument('--pos_embedding', choices=['random','one_hot'], help='Embedding for pos',
+                       required=True)
+    args_.add_argument('--char_path', help='path for character embedding dict')
+    args_.add_argument('--pos_path', help='path for pos embedding dict')
+    args_.add_argument('--set_num_training_samples', type=int, help='downsampling training set to a fixed number of samples')
+    args_.add_argument('--model_path', help='path for saving model file.', required=True)
+    args_.add_argument('--load_path', help='path for loading saved source model file.', default=None)
+    args_.add_argument('--load_sequence_taggers_paths', nargs='+', help='path for loading saved sequence_tagger saved_models files.', default=None)
+    args_.add_argument('--strict',action='store_true', help='if True loaded model state should contin '
+                                                            'exactly the same keys as current model')
+    args_.add_argument('--eval_mode', action='store_true', help='evaluating model without training it')
+    args_.add_argument('--eval_with_CI', action='store_true', help='evaluating model in constrained inference mode')
+    args_.add_argument('--LCM_Path_flag', action='store_true', help='for constrained inference with LCM, flag is used to change path')
+    args = args_.parse_args()
+    args_dict = {}
+    args_dict['dataset'] = args.dataset
+    args_dict['domain'] = args.domain
+    args_dict['rnn_mode'] = args.rnn_mode
+    args_dict['gating'] = args.gating
+    args_dict['num_gates'] = args.num_gates
+    args_dict['arc_decode'] = args.arc_decode
+    # args_dict['splits'] = ['train', 'dev', 'test']
+    args_dict['splits'] = ['train', 'dev', 'test']
+    args_dict['model_path'] = args.model_path
+    if not path.exists(args_dict['model_path']):
+        makedirs(args_dict['model_path'])
+    args_dict['data_paths'] = {}
+    if args_dict['dataset'] == 'ontonotes':
+        data_path = 'data/onto_pos_ner_dp'
+    else:
+        data_path = 'data/ud_pos_ner_dp'
+    for split in args_dict['splits']:
+        args_dict['data_paths'][split] = data_path + '_' + split + '_' + args_dict['domain']
+    ###################################    
+ 
+    ###################################
+    args_dict['alphabet_data_paths'] = {}
+    for split in args_dict['splits']:
+        if args_dict['dataset'] == 'ontonotes':
+            args_dict['alphabet_data_paths'][split] = data_path + '_' + split + '_' + 'all'
+        else:
+            if '_' in args_dict['domain']:
+                args_dict['alphabet_data_paths'][split] = data_path + '_' + split + '_' + args_dict['domain'].split('_')[0]
+            else:
+                args_dict['alphabet_data_paths'][split] = args_dict['data_paths'][split]
+    args_dict['model_name'] = 'domain_' + args_dict['domain']
+    args_dict['full_model_name'] = path.join(args_dict['model_path'],args_dict['model_name'])
+    args_dict['load_path'] = args.load_path
+    args_dict['load_sequence_taggers_paths'] = args.load_sequence_taggers_paths
+    if args_dict['load_sequence_taggers_paths'] is not None:
+        args_dict['gating'] = True
+        args_dict['num_gates'] = len(args_dict['load_sequence_taggers_paths']) + 1
+    else:
+        if not args_dict['gating']:
+            args_dict['num_gates'] = 0
+    args_dict['strict'] = args.strict
+    args_dict['num_epochs'] = args.num_epochs
+    args_dict['batch_size'] = args.batch_size
+    args_dict['hidden_size'] = args.hidden_size
+    args_dict['arc_space'] = args.arc_space
+    args_dict['arc_tag_space'] = args.arc_tag_space
+    args_dict['num_layers'] = args.num_layers
+    args_dict['num_filters'] = args.num_filters
+    args_dict['kernel_size'] = args.kernel_size
+    args_dict['learning_rate'] = args.learning_rate
+    args_dict['initializer'] = nn.init.xavier_uniform_ if args.initializer == 'xavier' else None
+    args_dict['opt'] = args.opt
+    args_dict['momentum'] = args.momentum
+    args_dict['betas'] = tuple(args.betas)
+    args_dict['epsilon'] = args.epsilon
+    args_dict['decay_rate'] = args.decay_rate
+    args_dict['clip'] = args.clip
+    args_dict['gamma'] = args.gamma
+    args_dict['schedule'] = args.schedule
+    args_dict['p_rnn'] = tuple(args.p_rnn)
+    args_dict['p_in'] = args.p_in
+    args_dict['p_out'] = args.p_out
+    args_dict['unk_replace'] = args.unk_replace
+    args_dict['set_num_training_samples'] = args.set_num_training_samples
+    args_dict['punct_set'] = None
+    if args.punct_set is not None:
+        args_dict['punct_set'] = set(args.punct_set)
+        logger.info("punctuations(%d): %s" % (len(args_dict['punct_set']), ' '.join(args_dict['punct_set'])))
+    args_dict['freeze_word_embeddings'] = args.freeze_word_embeddings
+    args_dict['freeze_sequence_taggers'] = args.freeze_sequence_taggers
+    args_dict['word_embedding'] = args.word_embedding
+    args_dict['word_path'] = args.word_path
+    args_dict['use_char'] = args.use_char
+    args_dict['char_embedding'] = args.char_embedding
+    args_dict['char_path'] = args.char_path
+    args_dict['pos_embedding'] = args.pos_embedding
+    args_dict['pos_path'] = args.pos_path
+    args_dict['use_pos'] = args.use_pos
+    args_dict['pos_dim'] = args.pos_dim
+    args_dict['word_dict'] = None
+    args_dict['word_dim'] = args.word_dim
+    if args_dict['word_embedding'] != 'random' and args_dict['word_path']:
+        args_dict['word_dict'], args_dict['word_dim'] = load_word_embeddings.load_embedding_dict(args_dict['word_embedding'],
+                                                                                                 args_dict['word_path'])
+    args_dict['char_dict'] = None
+    args_dict['char_dim'] = args.char_dim
+    if args_dict['char_embedding'] != 'random':
+        args_dict['char_dict'], args_dict['char_dim'] = load_word_embeddings.load_embedding_dict(args_dict['char_embedding'],
+                                                                                                 args_dict['char_path'])
+    args_dict['pos_dict'] = None
+    if args_dict['pos_embedding'] != 'random':
+        args_dict['pos_dict'], args_dict['pos_dim'] = load_word_embeddings.load_embedding_dict(args_dict['pos_embedding'],
+                                                                                                 args_dict['pos_path'])
+    args_dict['alphabet_path'] = path.join(args_dict['model_path'], 'alphabets' + '_src_domain_' + args_dict['domain'] + '/')
+    args_dict['model_name'] = path.join(args_dict['model_path'], args_dict['model_name'])
+    args_dict['eval_mode'] = args.eval_mode
+    args_dict['eval_with_CI'] = args.eval_with_CI
+    args_dict['LCM_Path_flag'] = args.LCM_Path_flag 
+    args_dict['device'] = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+    args_dict['word_status'] = 'frozen' if args.freeze_word_embeddings else 'fine tune'
+    args_dict['char_status'] = 'enabled' if args.use_char else 'disabled'
+    args_dict['pos_status'] = 'enabled' if args.use_pos else 'disabled'
+    logger.info("Saving arguments to file")
+    save_args(args, args_dict['full_model_name'])
+    logger.info("Creating Alphabets")
+    alphabet_dict = creating_alphabets(args_dict['alphabet_path'], args_dict['alphabet_data_paths'], args_dict['word_dict'])
+    args_dict = {**args_dict, **alphabet_dict}
+    ARGS = namedtuple('ARGS', args_dict.keys())
+    my_args = ARGS(**args_dict)
+    return my_args
+
+
+def creating_alphabets(alphabet_path, alphabet_data_paths, word_dict):
+    train_paths = alphabet_data_paths['train']
+    extra_paths = [v for k,v in alphabet_data_paths.items() if k != 'train']
+    alphabet_dict = {}
+    alphabet_dict['alphabets'] = prepare_data.create_alphabets(alphabet_path,
+                                                               train_paths,
+                                                               extra_paths=extra_paths,
+                                                               max_vocabulary_size=100000,
+                                                               embedd_dict=word_dict)
+    for k, v in alphabet_dict['alphabets'].items():
+        num_key = 'num_' + k.split('_')[0]
+        alphabet_dict[num_key] = v.size()
+        logger.info("%s : %d" % (num_key, alphabet_dict[num_key]))
+    return alphabet_dict
+
+def construct_embedding_table(alphabet, tokens_dict, dim, token_type='word'):
+    if tokens_dict is None:
+        return None
+    scale = np.sqrt(3.0 / dim)
+    table = np.empty([alphabet.size(), dim], dtype=np.float32)
+    table[prepare_data.UNK_ID, :] = np.random.uniform(-scale, scale, [1, dim]).astype(np.float32)
+    oov_tokens = 0
+    for token, index in alphabet.items():
+        if token in tokens_dict:
+            embedding = tokens_dict[token]
+        elif token.lower() in tokens_dict:
+            embedding = tokens_dict[token.lower()]
+        else:
+            embedding = np.random.uniform(-scale, scale, [1, dim]).astype(np.float32)
+            oov_tokens += 1
+        table[index, :] = embedding
+    print('token type : %s, number of oov: %d' % (token_type, oov_tokens))
+    table = torch.from_numpy(table)
+    return table
+
+def save_args(args, full_model_name):
+    arg_path = full_model_name + '.arg.json'
+    argparse_dict = vars(args)
+    with open(arg_path, 'w') as f:
+        json.dump(argparse_dict, f)
+
+def generate_optimizer(args, lr, params):
+    params = filter(lambda param: param.requires_grad, params)
+    if args.opt == 'adam':
+        return Adam(params, lr=lr, betas=args.betas, weight_decay=args.gamma, eps=args.epsilon)
+    elif args.opt == 'sgd':
+        return SGD(params, lr=lr, momentum=args.momentum, weight_decay=args.gamma, nesterov=True)
+    else:
+        raise ValueError('Unknown optimization algorithm: %s' % args.opt)
+
+
+def save_checkpoint(args, model, optimizer, opt, dev_eval_dict, test_eval_dict, full_model_name):
+    path_name = full_model_name + '.pt'
+    print('Saving model to: %s' % path_name)
+    state = {'model_state_dict': model.state_dict(),
+             'optimizer_state_dict': optimizer.state_dict(),
+             'opt': opt,
+             'dev_eval_dict': dev_eval_dict,
+             'test_eval_dict': test_eval_dict}
+    torch.save(state, path_name)
+
+def load_checkpoint(args, model, optimizer, dev_eval_dict, test_eval_dict, start_epoch, load_path, strict=True):
+    print('Loading saved model from: %s' % load_path)
+    checkpoint = torch.load(load_path, map_location=args.device)
+    if checkpoint['opt'] != args.opt:
+        raise ValueError('loaded optimizer type is: %s instead of: %s' % (checkpoint['opt'], args.opt))
+    model.load_state_dict(checkpoint['model_state_dict'], strict=strict)
+
+
+    if strict:
+        generate_optimizer(args, args.learning_rate, model.parameters())
+        optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
+        for state in optimizer.state.values():
+            for k, v in state.items():
+                if isinstance(v, torch.Tensor):
+                    state[k] = v.to(args.device)
+        dev_eval_dict = checkpoint['dev_eval_dict']
+        test_eval_dict = checkpoint['test_eval_dict']
+        start_epoch = dev_eval_dict['in_domain']['epoch']
+    return model, optimizer, dev_eval_dict, test_eval_dict, start_epoch
+
+
+def build_model_and_optimizer(args):
+    word_table = construct_embedding_table(args.alphabets['word_alphabet'], args.word_dict, args.word_dim, token_type='word')
+    char_table = construct_embedding_table(args.alphabets['char_alphabet'], args.char_dict, args.char_dim, token_type='char')
+    pos_table = construct_embedding_table(args.alphabets['pos_alphabet'], args.pos_dict, args.pos_dim, token_type='pos')
+    model = BiAffine_Parser_Gated(args.word_dim, args.num_word, args.char_dim, args.num_char,
+                            args.use_pos, args.use_char, args.pos_dim, args.num_pos,
+                            args.num_filters, args.kernel_size, args.rnn_mode,
+                            args.hidden_size, args.num_layers, args.num_arc,
+                            args.arc_space, args.arc_tag_space, args.num_gates,
+                            embedd_word=word_table, embedd_char=char_table, embedd_pos=pos_table,
+                            p_in=args.p_in, p_out=args.p_out,  p_rnn=args.p_rnn,
+                            biaffine=True, arc_decode=args.arc_decode, initializer=args.initializer)
+    print(model)
+    optimizer = generate_optimizer(args, args.learning_rate, model.parameters())
+    start_epoch = 0
+    dev_eval_dict = {'in_domain': initialize_eval_dict()}
+    test_eval_dict = {'in_domain': initialize_eval_dict()}
+    if args.load_path:
+        model, optimizer, dev_eval_dict, test_eval_dict, start_epoch = \
+            load_checkpoint(args, model, optimizer,
+                            dev_eval_dict, test_eval_dict,
+                            start_epoch, args.load_path, strict=args.strict)
+    if args.load_sequence_taggers_paths:
+        pretrained_dict = {}
+        model_dict = model.state_dict()
+        for idx, path in enumerate(args.load_sequence_taggers_paths):
+            print('Loading saved sequence_tagger from: %s' % path)
+            checkpoint = torch.load(path, map_location=args.device)
+            for k, v in checkpoint['model_state_dict'].items():
+                if 'rnn_encoder.' in k:
+                    pretrained_dict['extra_rnn_encoders.' + str(idx) + '.' + k.replace('rnn_encoder.', '')] = v
+        model_dict.update(pretrained_dict)
+        model.load_state_dict(model_dict)
+    if args.freeze_sequence_taggers:
+        print('Freezing Classifiers')
+        for name, parameter in model.named_parameters():
+            if 'extra_rnn_encoders' in name:
+                parameter.requires_grad = False
+    if args.freeze_word_embeddings:
+        model.rnn_encoder.word_embedd.weight.requires_grad = False
+        # model.rnn_encoder.char_embedd.weight.requires_grad = False
+        # model.rnn_encoder.pos_embedd.weight.requires_grad = False
+    device = args.device
+    model.to(device)
+    return model, optimizer, dev_eval_dict, test_eval_dict, start_epoch
+
+
+def initialize_eval_dict():
+    eval_dict = {}
+    eval_dict['dp_uas'] = 0.0
+    eval_dict['dp_las'] = 0.0
+    eval_dict['epoch'] = 0
+    eval_dict['dp_ucorrect'] = 0.0
+    eval_dict['dp_lcorrect'] = 0.0
+    eval_dict['dp_total'] = 0.0
+    eval_dict['dp_ucomplete_match'] = 0.0
+    eval_dict['dp_lcomplete_match'] = 0.0
+    eval_dict['dp_ucorrect_nopunc'] = 0.0
+    eval_dict['dp_lcorrect_nopunc'] = 0.0
+    eval_dict['dp_total_nopunc'] = 0.0
+    eval_dict['dp_ucomplete_match_nopunc'] = 0.0
+    eval_dict['dp_lcomplete_match_nopunc'] = 0.0
+    eval_dict['dp_root_correct'] = 0.0
+    eval_dict['dp_total_root'] = 0.0
+    eval_dict['dp_total_inst'] = 0.0
+    eval_dict['dp_total'] = 0.0
+    eval_dict['dp_total_inst'] = 0.0
+    eval_dict['dp_total_nopunc'] = 0.0
+    eval_dict['dp_total_root'] = 0.0
+    return eval_dict
+
+def in_domain_evaluation(args, datasets, model, optimizer, dev_eval_dict, test_eval_dict, epoch,
+                         best_model, best_optimizer, patient):
+    # In-domain evaluation
+    curr_dev_eval_dict = evaluation(args, datasets['dev'], 'dev', model, args.domain, epoch, 'current_results')
+    is_best_in_domain = dev_eval_dict['in_domain']['dp_lcorrect_nopunc'] <= curr_dev_eval_dict['dp_lcorrect_nopunc'] or \
+              (dev_eval_dict['in_domain']['dp_lcorrect_nopunc'] == curr_dev_eval_dict['dp_lcorrect_nopunc'] and
+               dev_eval_dict['in_domain']['dp_ucorrect_nopunc'] <= curr_dev_eval_dict['dp_ucorrect_nopunc'])
+
+    if is_best_in_domain:
+        for key, value in curr_dev_eval_dict.items():
+            dev_eval_dict['in_domain'][key] = value
+        curr_test_eval_dict = evaluation(args, datasets['test'], 'test', model, args.domain, epoch, 'current_results')
+        for key, value in curr_test_eval_dict.items():
+            test_eval_dict['in_domain'][key] = value
+        best_model = deepcopy(model)
+        best_optimizer = deepcopy(optimizer)
+        patient = 0
+    else:
+        patient += 1
+    if epoch == args.num_epochs:
+        # save in-domain checkpoint
+        if args.set_num_training_samples is not None:
+            splits_to_write = datasets.keys()
+        else:
+            splits_to_write = ['dev', 'test']
+        for split in splits_to_write:
+            if split == 'dev':
+                eval_dict = dev_eval_dict['in_domain']
+            elif split == 'test':
+                eval_dict = test_eval_dict['in_domain']
+            else:
+                eval_dict = None
+            write_results(args, datasets[split], args.domain, split, best_model, args.domain, eval_dict)
+        print("Saving best model")
+        save_checkpoint(args, best_model, best_optimizer, args.opt, dev_eval_dict, test_eval_dict, args.full_model_name)
+
+    print('\n')
+    return dev_eval_dict, test_eval_dict, best_model, best_optimizer, patient
+
+
+def evaluation(args, data, split, model, domain, epoch, str_res='results'):
+    # evaluate performance on data
+    model.eval()
+
+    eval_dict = initialize_eval_dict()
+    eval_dict['epoch'] = epoch
+    for batch in prepare_data.iterate_batch(data, args.batch_size, args.device):
+        word, char, pos, ner, heads, arc_tags, auto_label, masks, lengths = batch
+        out_arc, out_arc_tag, masks, lengths = model.forward(word, char, pos, mask=masks, length=lengths)
+        heads_pred, arc_tags_pred, _ = model.decode(args.model_path, word, pos,ner, out_arc, out_arc_tag, mask=masks, length=lengths,
+                                                    leading_symbolic=prepare_data.NUM_SYMBOLIC_TAGS)
+        lengths = lengths.cpu().numpy()
+        word = word.data.cpu().numpy()
+        pos = pos.data.cpu().numpy()
+        ner = ner.data.cpu().numpy()
+        heads = heads.data.cpu().numpy()
+        arc_tags = arc_tags.data.cpu().numpy()
+        heads_pred = heads_pred.data.cpu().numpy()
+        arc_tags_pred = arc_tags_pred.data.cpu().numpy()
+        stats, stats_nopunc, stats_root, num_inst = parse.eval_(word, pos, heads_pred, arc_tags_pred, heads,
+                                                                arc_tags, args.alphabets['word_alphabet'], args.alphabets['pos_alphabet'],
+                                                                lengths, punct_set=args.punct_set, symbolic_root=True)
+        ucorr, lcorr, total, ucm, lcm = stats
+        ucorr_nopunc, lcorr_nopunc, total_nopunc, ucm_nopunc, lcm_nopunc = stats_nopunc
+        corr_root, total_root = stats_root
+        eval_dict['dp_ucorrect'] += ucorr
+        eval_dict['dp_lcorrect'] += lcorr
+        eval_dict['dp_total'] += total
+        eval_dict['dp_ucomplete_match'] += ucm
+        eval_dict['dp_lcomplete_match'] += lcm
+        eval_dict['dp_ucorrect_nopunc'] += ucorr_nopunc
+        eval_dict['dp_lcorrect_nopunc'] += lcorr_nopunc
+        eval_dict['dp_total_nopunc'] += total_nopunc
+        eval_dict['dp_ucomplete_match_nopunc'] += ucm_nopunc
+        eval_dict['dp_lcomplete_match_nopunc'] += lcm_nopunc
+        eval_dict['dp_root_correct'] += corr_root
+        eval_dict['dp_total_root'] += total_root
+        eval_dict['dp_total_inst'] += num_inst
+
+    eval_dict['dp_uas'] = eval_dict['dp_ucorrect'] * 100 / eval_dict['dp_total']  # considering w. punctuation
+    eval_dict['dp_las'] = eval_dict['dp_lcorrect'] * 100 / eval_dict['dp_total']  # considering w. punctuation
+    print_results(eval_dict, split, domain, str_res)
+    return eval_dict
+
+def constrained_evaluation(args, data, split, model, domain, epoch, str_res='results'):
+    # evaluate performance on data
+    model.eval()
+
+    eval_dict = initialize_eval_dict()
+    eval_dict['epoch'] = epoch
+    for batch in prepare_data.iterate_batch(data, args.batch_size, args.device):
+        word, char, pos, ner, heads, arc_tags, auto_label, masks, lengths = batch
+        out_arc, out_arc_tag, masks, lengths = model.forward(word, char, pos, mask=masks, length=lengths)
+        heads_pred, arc_tags_pred, _ = model.constrained_decode(args, word, pos,ner, out_arc, out_arc_tag, mask=masks, length=lengths,
+                                                    leading_symbolic=prepare_data.NUM_SYMBOLIC_TAGS)
+        lengths = lengths.cpu().numpy()
+        word = word.data.cpu().numpy()
+        pos = pos.data.cpu().numpy()
+        ner = ner.data.cpu().numpy()
+        heads = heads.data.cpu().numpy()
+        arc_tags = arc_tags.data.cpu().numpy()
+        heads_pred = heads_pred.data.cpu().numpy()
+        arc_tags_pred = arc_tags_pred.data.cpu().numpy()
+        stats, stats_nopunc, stats_root, num_inst = parse.eval_(word, pos, heads_pred, arc_tags_pred, heads,
+                                                                arc_tags, args.alphabets['word_alphabet'], args.alphabets['pos_alphabet'],
+                                                                lengths, punct_set=args.punct_set, symbolic_root=True)
+        ucorr, lcorr, total, ucm, lcm = stats
+        ucorr_nopunc, lcorr_nopunc, total_nopunc, ucm_nopunc, lcm_nopunc = stats_nopunc
+        corr_root, total_root = stats_root
+        eval_dict['dp_ucorrect'] += ucorr
+        eval_dict['dp_lcorrect'] += lcorr
+        eval_dict['dp_total'] += total
+        eval_dict['dp_ucomplete_match'] += ucm
+        eval_dict['dp_lcomplete_match'] += lcm
+        eval_dict['dp_ucorrect_nopunc'] += ucorr_nopunc
+        eval_dict['dp_lcorrect_nopunc'] += lcorr_nopunc
+        eval_dict['dp_total_nopunc'] += total_nopunc
+        eval_dict['dp_ucomplete_match_nopunc'] += ucm_nopunc
+        eval_dict['dp_lcomplete_match_nopunc'] += lcm_nopunc
+        eval_dict['dp_root_correct'] += corr_root
+        eval_dict['dp_total_root'] += total_root
+        eval_dict['dp_total_inst'] += num_inst
+
+    eval_dict['dp_uas'] = eval_dict['dp_ucorrect'] * 100 / eval_dict['dp_total']  # considering w. punctuation
+    eval_dict['dp_las'] = eval_dict['dp_lcorrect'] * 100 / eval_dict['dp_total']  # considering w. punctuation
+    print_results(eval_dict, split, domain, str_res)
+    return eval_dict
+
+def print_results(eval_dict, split, domain, str_res='results'):
+    print('----------------------------------------------------------------------------------------------------------------------------')
+    print('Testing model on domain %s' % domain)
+    print('--------------- Dependency Parsing - %s ---------------' % split)
+    print(
+        str_res + ' on ' + split + '  W. Punct: ucorr: %d, lcorr: %d, total: %d, uas: %.2f%%, las: %.2f%%, ucm: %.2f%%, lcm: %.2f%% (epoch: %d)' % (
+            eval_dict['dp_ucorrect'], eval_dict['dp_lcorrect'], eval_dict['dp_total'],
+            eval_dict['dp_ucorrect'] * 100 / eval_dict['dp_total'],
+            eval_dict['dp_lcorrect'] * 100 / eval_dict['dp_total'],
+            eval_dict['dp_ucomplete_match'] * 100 / eval_dict['dp_total_inst'],
+            eval_dict['dp_lcomplete_match'] * 100 / eval_dict['dp_total_inst'],
+            eval_dict['epoch']))
+    print(
+        str_res + ' on ' + split + '  Wo Punct: ucorr: %d, lcorr: %d, total: %d, uas: %.2f%%, las: %.2f%%, ucm: %.2f%%, lcm: %.2f%% (epoch: %d)' % (
+            eval_dict['dp_ucorrect_nopunc'], eval_dict['dp_lcorrect_nopunc'], eval_dict['dp_total_nopunc'],
+            eval_dict['dp_ucorrect_nopunc'] * 100 / eval_dict['dp_total_nopunc'],
+            eval_dict['dp_lcorrect_nopunc'] * 100 / eval_dict['dp_total_nopunc'],
+            eval_dict['dp_ucomplete_match_nopunc'] * 100 / eval_dict['dp_total_inst'],
+            eval_dict['dp_lcomplete_match_nopunc'] * 100 / eval_dict['dp_total_inst'],
+            eval_dict['epoch']))
+    print(str_res + ' on ' + split + '  Root: corr: %d, total: %d, acc: %.2f%% (epoch: %d)' % (
+        eval_dict['dp_root_correct'], eval_dict['dp_total_root'],
+        eval_dict['dp_root_correct'] * 100 / eval_dict['dp_total_root'], eval_dict['epoch']))
+    print('\n')
+def constrained_write_results(args, data, data_domain, split, model, model_domain, eval_dict):
+    str_file = args.full_model_name + '_' + split + '_model_domain_' + model_domain + '_data_domain_' + data_domain
+    res_filename = str_file + '_res.txt'
+    pred_filename = str_file + '_pred.txt'
+    gold_filename = str_file + '_gold.txt'
+    if eval_dict is not None:
+        # save results dictionary into a file
+        with open(res_filename, 'w') as f:
+            json.dump(eval_dict, f)
+
+    # save predictions and gold labels into files
+    pred_writer = Writer(args.alphabets)
+    gold_writer = Writer(args.alphabets)
+    pred_writer.start(pred_filename)
+    gold_writer.start(gold_filename)
+    for batch in prepare_data.iterate_batch(data, args.batch_size, args.device):
+        word, char, pos, ner, heads, arc_tags, auto_label, masks, lengths = batch
+        # pdb.set_trace()
+        out_arc, out_arc_tag, masks, lengths = model.forward(word, char, pos, mask=masks, length=lengths)
+        heads_pred, arc_tags_pred, _ = model.constrained_decode(args,word, pos, ner, out_arc, out_arc_tag, mask=masks, length=lengths,
+                                                    leading_symbolic=prepare_data.NUM_SYMBOLIC_TAGS)
+        lengths = lengths.cpu().numpy()
+        word = word.data.cpu().numpy()
+        pos = pos.data.cpu().numpy()
+        ner = ner.data.cpu().numpy()
+        heads = heads.data.cpu().numpy()
+        arc_tags = arc_tags.data.cpu().numpy()
+        heads_pred = heads_pred.data.cpu().numpy()
+        arc_tags_pred = arc_tags_pred.data.cpu().numpy()
+        # print('words',word)
+        # print('Pos',pos)
+        
+        # print('heads_pred',heads_pred)
+        # print('arc_tags_pred',arc_tags_pred)
+        # pdb.set_trace()
+        # writing predictions
+        pred_writer.write(word, pos, ner, heads_pred, arc_tags_pred, lengths, symbolic_root=True)
+        # writing gold labels
+        gold_writer.write(word, pos, ner, heads, arc_tags, lengths, symbolic_root=True)
+
+    pred_writer.close()
+    gold_writer.close()
+def write_results(args, data, data_domain, split, model, model_domain, eval_dict):
+    str_file = args.full_model_name + '_' + split + '_model_domain_' + model_domain + '_data_domain_' + data_domain
+    res_filename = str_file + '_res.txt'
+    pred_filename = str_file + '_pred.txt'
+    gold_filename = str_file + '_gold.txt'
+    if eval_dict is not None:
+        # save results dictionary into a file
+        with open(res_filename, 'w') as f:
+            json.dump(eval_dict, f)
+
+    # save predictions and gold labels into files
+    pred_writer = Writer(args.alphabets)
+    gold_writer = Writer(args.alphabets)
+    pred_writer.start(pred_filename)
+    gold_writer.start(gold_filename)
+    for batch in prepare_data.iterate_batch(data, args.batch_size, args.device):
+        word, char, pos, ner, heads, arc_tags, auto_label, masks, lengths = batch
+        # pdb.set_trace()
+        out_arc, out_arc_tag, masks, lengths = model.forward(word, char, pos, mask=masks, length=lengths)
+        heads_pred, arc_tags_pred, _ = model.decode(args.model_path,word, pos, ner, out_arc, out_arc_tag, mask=masks, length=lengths,
+                                                    leading_symbolic=prepare_data.NUM_SYMBOLIC_TAGS)
+        lengths = lengths.cpu().numpy()
+        word = word.data.cpu().numpy()
+        pos = pos.data.cpu().numpy()
+        ner = ner.data.cpu().numpy()
+        heads = heads.data.cpu().numpy()
+        arc_tags = arc_tags.data.cpu().numpy()
+        heads_pred = heads_pred.data.cpu().numpy()
+        arc_tags_pred = arc_tags_pred.data.cpu().numpy()
+        # print('words',word)
+        # print('Pos',pos)
+        
+        # print('heads_pred',heads_pred)
+        # print('arc_tags_pred',arc_tags_pred)
+        # pdb.set_trace()
+        # writing predictions
+        pred_writer.write(word, pos, ner, heads_pred, arc_tags_pred, lengths, symbolic_root=True)
+        # writing gold labels
+        gold_writer.write(word, pos, ner, heads, arc_tags, lengths, symbolic_root=True)
+
+    pred_writer.close()
+    gold_writer.close()
+
+def main():
+    logger.info("Reading and creating arguments")
+    args = read_arguments()
+    logger.info("Reading Data")
+    datasets = {}
+    for split in args.splits:
+        print("Splits are:",split)
+        dataset = prepare_data.read_data_to_variable(args.data_paths[split], args.alphabets, args.device,
+                                                     symbolic_root=True)
+        datasets[split] = dataset
+    if args.set_num_training_samples is not None:
+        print('Setting train and dev to %d samples' % args.set_num_training_samples)
+        datasets = rearrange_splits.rearranging_splits(datasets, args.set_num_training_samples)
+    logger.info("Creating Networks")
+    num_data = sum(datasets['train'][1])
+    model, optimizer, dev_eval_dict, test_eval_dict, start_epoch = build_model_and_optimizer(args)
+    best_model = deepcopy(model)
+    best_optimizer = deepcopy(optimizer)
+
+    logger.info('Training INFO of in domain %s' % args.domain)
+    logger.info('Training on Dependecy Parsing')
+    logger.info("train: gamma: %f, batch: %d, clip: %.2f, unk replace: %.2f" % (args.gamma, args.batch_size, args.clip, args.unk_replace))
+    logger.info('number of training samples for %s is: %d' % (args.domain, num_data))
+    logger.info("dropout(in, out, rnn): (%.2f, %.2f, %s)" % (args.p_in, args.p_out, args.p_rnn))
+    logger.info("num_epochs: %d" % (args.num_epochs))
+    print('\n')
+
+    if not args.eval_mode:
+        logger.info("Training")
+        num_batches = prepare_data.calc_num_batches(datasets['train'], args.batch_size)
+        lr = args.learning_rate
+        patient = 0
+        decay = 0
+        for epoch in range(start_epoch + 1, args.num_epochs + 1):
+            print('Epoch %d (Training: rnn mode: %s, optimizer: %s, learning rate=%.6f, eps=%.1e, decay rate=%.2f (schedule=%d, decay=%d)): ' % (
+                epoch, args.rnn_mode, args.opt, lr, args.epsilon, args.decay_rate, args.schedule, decay))
+            model.train()
+            total_loss = 0.0
+            total_arc_loss = 0.0
+            total_arc_tag_loss = 0.0
+            total_train_inst = 0.0
+
+            train_iter = prepare_data.iterate_batch_rand_bucket_choosing(
+                    datasets['train'], args.batch_size, args.device, unk_replace=args.unk_replace)
+            start_time = time.time()
+            batch_num = 0
+            for batch_num, batch in enumerate(train_iter):
+                batch_num = batch_num + 1
+                optimizer.zero_grad()
+                # compute loss of main task
+                word, char, pos, ner_tags, heads, arc_tags, auto_label, masks, lengths = batch
+                out_arc, out_arc_tag, masks, lengths = model.forward(word, char, pos, mask=masks, length=lengths)
+                loss_arc, loss_arc_tag = model.loss(out_arc, out_arc_tag, heads, arc_tags, mask=masks, length=lengths)
+                loss = loss_arc + loss_arc_tag
+                # pdb.set_trace()
+
+                # update losses
+                num_insts = masks.data.sum() - word.size(0)
+                total_arc_loss += loss_arc.item() * num_insts
+                total_arc_tag_loss += loss_arc_tag.item() * num_insts
+                total_loss += loss.item() * num_insts
+                total_train_inst += num_insts
+                # optimize parameters
+                loss.backward()
+                clip_grad_norm_(model.parameters(), args.clip)
+                optimizer.step()
+
+                time_ave = (time.time() - start_time) / batch_num
+                time_left = (num_batches - batch_num) * time_ave
+
+                # update log
+                if batch_num % 50 == 0:
+                    log_info = 'train: %d/%d, domain: %s, total loss: %.2f, arc_loss: %.2f, arc_tag_loss: %.2f, time left: %.2fs' % \
+                               (batch_num, num_batches, args.domain, total_loss / total_train_inst, total_arc_loss / total_train_inst,
+                                total_arc_tag_loss / total_train_inst, time_left)
+                    sys.stdout.write(log_info)
+                    sys.stdout.write('\n')
+                    sys.stdout.flush()
+            print('\n')
+            print('train: %d/%d, domain: %s, total_loss: %.2f, arc_loss: %.2f, arc_tag_loss: %.2f, time: %.2fs' %
+                  (batch_num, num_batches, args.domain, total_loss / total_train_inst, total_arc_loss / total_train_inst,
+                   total_arc_tag_loss / total_train_inst, time.time() - start_time))
+
+            dev_eval_dict, test_eval_dict, best_model, best_optimizer, patient = in_domain_evaluation(args, datasets, model, optimizer, dev_eval_dict, test_eval_dict, epoch, best_model, best_optimizer, patient)
+            if patient >= args.schedule:
+                lr = args.learning_rate / (1.0 + epoch * args.decay_rate)
+                optimizer = generate_optimizer(args, lr, model.parameters())
+                print('updated learning rate to %.6f' % lr)
+                patient = 0
+            print_results(test_eval_dict['in_domain'], 'test', args.domain, 'best_results')
+            print('\n')
+        for split in datasets.keys():
+            if args.eval_with_CI and split not in ['train', 'extra_train', 'extra_dev']:
+                print('Currently going on ... ',split)
+                eval_dict = constrained_evaluation(args, datasets[split], split, best_model, args.domain, epoch, 'best_results')
+                constrained_write_results(args, datasets[split], args.domain, split, model, args.domain, eval_dict)
+            else:
+                eval_dict = evaluation(args, datasets[split], split, best_model, args.domain, epoch, 'best_results')
+                write_results(args, datasets[split], args.domain, split, model, args.domain, eval_dict)
+
+    else:
+        logger.info("Evaluating")
+        epoch = start_epoch
+        for split in ['train','dev','test']:
+            if args.eval_with_CI and split != 'train':
+                eval_dict = constrained_evaluation(args, datasets[split], split, best_model, args.domain, epoch, 'best_results')
+                constrained_write_results(args, datasets[split], args.domain, split, model, args.domain, eval_dict)
+            else:
+                eval_dict = evaluation(args, datasets[split], split, best_model, args.domain, epoch, 'best_results')
+                write_results(args, datasets[split], args.domain, split, model, args.domain, eval_dict)
+
+
+if __name__ == '__main__':
+    main()

examples/GraphParser_MRL.py

@@ -0,0 +1,603 @@
+from __future__ import print_function
+import sys
+from os import path, makedirs
+
+sys.path.append(".")
+sys.path.append("..")
+
+import argparse
+from copy import deepcopy
+import json
+import numpy as np
+import torch
+import torch.nn as nn
+from collections import namedtuple
+from utils.io_ import seeds, Writer, get_logger, prepare_data, rearrange_splits
+from utils.models.parsing_gating import BiAffine_Parser_Gated
+from utils import load_word_embeddings
+from utils.tasks import parse
+import time
+from torch.nn.utils import clip_grad_norm_
+from torch.optim import Adam, SGD
+import uuid
+
+uid = uuid.uuid4().hex[:6]
+
+logger = get_logger('GraphParser')
+
+def read_arguments():
+    args_ = argparse.ArgumentParser(description='Sovling GraphParser')
+    args_.add_argument('--dataset', choices=['ontonotes', 'ud'], help='Dataset', required=True)
+    args_.add_argument('--domain', help='domain/language', required=True)
+    args_.add_argument('--rnn_mode', choices=['RNN', 'LSTM', 'GRU'], help='architecture of rnn',
+                       required=True)
+    args_.add_argument('--gating',action='store_true', help='use gated mechanism')
+    args_.add_argument('--num_gates', type=int, default=0, help='number of gates for gating mechanism')
+    args_.add_argument('--num_epochs', type=int, default=200, help='Number of training epochs')
+    args_.add_argument('--batch_size', type=int, default=64, help='Number of sentences in each batch')
+    args_.add_argument('--hidden_size', type=int, default=256, help='Number of hidden units in RNN')
+    args_.add_argument('--arc_space', type=int, default=128, help='Dimension of tag space')
+    args_.add_argument('--arc_tag_space', type=int, default=128, help='Dimension of tag space')
+    args_.add_argument('--num_layers', type=int, default=1, help='Number of layers of RNN')
+    args_.add_argument('--num_filters', type=int, default=50, help='Number of filters in CNN')
+    args_.add_argument('--kernel_size', type=int, default=3, help='Size of Kernel for CNN')
+    args_.add_argument('--use_pos', action='store_true', help='use part-of-speech embedding.')
+    args_.add_argument('--use_char', action='store_true', help='use character embedding and CNN.')
+    args_.add_argument('--word_dim', type=int, default=300, help='Dimension of word embeddings')
+    args_.add_argument('--pos_dim', type=int, default=50, help='Dimension of POS embeddings')
+    args_.add_argument('--char_dim', type=int, default=50, help='Dimension of Character embeddings')
+    args_.add_argument('--initializer', choices=['xavier'], help='initialize model parameters')
+    args_.add_argument('--opt', choices=['adam', 'sgd'], help='optimization algorithm')
+    args_.add_argument('--momentum', type=float, default=0.9, help='momentum of optimizer')
+    args_.add_argument('--betas', nargs=2, type=float, default=[0.9, 0.9], help='betas of optimizer')
+    args_.add_argument('--learning_rate', type=float, default=0.01, help='Learning rate')
+    args_.add_argument('--decay_rate', type=float, default=0.05, help='Decay rate of learning rate')
+    args_.add_argument('--schedule', type=int, help='schedule for learning rate decay')
+    args_.add_argument('--clip', type=float, default=5.0, help='gradient clipping')
+    args_.add_argument('--gamma', type=float, default=0.0, help='weight for regularization')
+    args_.add_argument('--epsilon', type=float, default=1e-8, help='epsilon for adam')
+    args_.add_argument('--p_rnn', nargs=2, type=float, required=True, help='dropout rate for RNN')
+    args_.add_argument('--p_in', type=float, default=0.33, help='dropout rate for input embeddings')
+    args_.add_argument('--p_out', type=float, default=0.33, help='dropout rate for output layer')
+    args_.add_argument('--arc_decode', choices=['mst', 'greedy'], help='arc decoding algorithm', required=True)
+    args_.add_argument('--unk_replace', type=float, default=0.,
+                       help='The rate to replace a singleton word with UNK')
+    args_.add_argument('--punct_set', nargs='+', type=str, help='List of punctuations')
+    args_.add_argument('--word_embedding', choices=['random', 'glove', 'fasttext', 'word2vec'],
+                       help='Embedding for words')
+    args_.add_argument('--word_path', help='path for word embedding dict - in case word_embedding is not random')
+    args_.add_argument('--freeze_word_embeddings', action='store_true', help='frozen the word embedding (disable fine-tuning).')
+    args_.add_argument('--freeze_sequence_taggers', action='store_true', help='frozen the BiLSTMs of the pre-trained taggers.')
+    args_.add_argument('--char_embedding', choices=['random','hellwig'], help='Embedding for characters',
+                       required=True)
+    args_.add_argument('--pos_embedding', choices=['random','one_hot'], help='Embedding for pos',
+                       required=True)
+    args_.add_argument('--char_path', help='path for character embedding dict')
+    args_.add_argument('--pos_path', help='path for pos embedding dict')
+    args_.add_argument('--set_num_training_samples', type=int, help='downsampling training set to a fixed number of samples')
+    args_.add_argument('--model_path', help='path for saving model file.', required=True)
+    args_.add_argument('--load_path', help='path for loading saved source model file.', default=None)
+    args_.add_argument('--load_sequence_taggers_paths', nargs='+', help='path for loading saved sequence_tagger saved_models files.', default=None)
+    args_.add_argument('--strict',action='store_true', help='if True loaded model state should contin '
+                                                            'exactly the same keys as current model')
+    args_.add_argument('--eval_mode', action='store_true', help='evaluating model without training it')
+    args = args_.parse_args()
+    args_dict = {}
+    args_dict['dataset'] = args.dataset
+    args_dict['domain'] = args.domain
+    args_dict['rnn_mode'] = args.rnn_mode
+    args_dict['gating'] = args.gating
+    args_dict['num_gates'] = args.num_gates
+    args_dict['arc_decode'] = args.arc_decode
+    # args_dict['splits'] = ['train', 'dev', 'test']
+    args_dict['splits'] = ['train', 'dev', 'test','poetry','prose']
+    args_dict['model_path'] = args.model_path
+    if not path.exists(args_dict['model_path']):
+        makedirs(args_dict['model_path'])
+    args_dict['data_paths'] = {}
+    if args_dict['dataset'] == 'ontonotes':
+        data_path = 'data/Pre_MRL/onto_pos_ner_dp'
+    else:
+        data_path = 'data/Prep_MRL/ud_pos_ner_dp'
+    for split in args_dict['splits']:
+        args_dict['data_paths'][split] = data_path + '_' + split + '_' + args_dict['domain']
+    ###################################    
+    args_dict['data_paths']['poetry'] = data_path + '_' + 'test' + '_' + args_dict['domain']
+    args_dict['data_paths']['prose'] = data_path + '_' + 'test' + '_' + args_dict['domain']
+    ###################################
+    args_dict['alphabet_data_paths'] = {}
+    for split in args_dict['splits']:
+        if args_dict['dataset'] == 'ontonotes':
+            args_dict['alphabet_data_paths'][split] = data_path + '_' + split + '_' + 'all'
+        else:
+            if '_' in args_dict['domain']:
+                args_dict['alphabet_data_paths'][split] = data_path + '_' + split + '_' + args_dict['domain'].split('_')[0]
+            else:
+                args_dict['alphabet_data_paths'][split] = args_dict['data_paths'][split]
+    args_dict['model_name'] = 'domain_' + args_dict['domain']
+    args_dict['full_model_name'] = path.join(args_dict['model_path'],args_dict['model_name'])
+    args_dict['load_path'] = args.load_path
+    args_dict['load_sequence_taggers_paths'] = args.load_sequence_taggers_paths
+    if args_dict['load_sequence_taggers_paths'] is not None:
+        args_dict['gating'] = True
+        args_dict['num_gates'] = len(args_dict['load_sequence_taggers_paths']) + 1
+    else:
+        if not args_dict['gating']:
+            args_dict['num_gates'] = 0
+    args_dict['strict'] = args.strict
+    args_dict['num_epochs'] = args.num_epochs
+    args_dict['batch_size'] = args.batch_size
+    args_dict['hidden_size'] = args.hidden_size
+    args_dict['arc_space'] = args.arc_space
+    args_dict['arc_tag_space'] = args.arc_tag_space
+    args_dict['num_layers'] = args.num_layers
+    args_dict['num_filters'] = args.num_filters
+    args_dict['kernel_size'] = args.kernel_size
+    args_dict['learning_rate'] = args.learning_rate
+    args_dict['initializer'] = nn.init.xavier_uniform_ if args.initializer == 'xavier' else None
+    args_dict['opt'] = args.opt
+    args_dict['momentum'] = args.momentum
+    args_dict['betas'] = tuple(args.betas)
+    args_dict['epsilon'] = args.epsilon
+    args_dict['decay_rate'] = args.decay_rate
+    args_dict['clip'] = args.clip
+    args_dict['gamma'] = args.gamma
+    args_dict['schedule'] = args.schedule
+    args_dict['p_rnn'] = tuple(args.p_rnn)
+    args_dict['p_in'] = args.p_in
+    args_dict['p_out'] = args.p_out
+    args_dict['unk_replace'] = args.unk_replace
+    args_dict['set_num_training_samples'] = args.set_num_training_samples
+    args_dict['punct_set'] = None
+    if args.punct_set is not None:
+        args_dict['punct_set'] = set(args.punct_set)
+        logger.info("punctuations(%d): %s" % (len(args_dict['punct_set']), ' '.join(args_dict['punct_set'])))
+    args_dict['freeze_word_embeddings'] = args.freeze_word_embeddings
+    args_dict['freeze_sequence_taggers'] = args.freeze_sequence_taggers
+    args_dict['word_embedding'] = args.word_embedding
+    args_dict['word_path'] = args.word_path
+    args_dict['use_char'] = args.use_char
+    args_dict['char_embedding'] = args.char_embedding
+    args_dict['char_path'] = args.char_path
+    args_dict['pos_embedding'] = args.pos_embedding
+    args_dict['pos_path'] = args.pos_path
+    args_dict['use_pos'] = args.use_pos
+    args_dict['pos_dim'] = args.pos_dim
+    args_dict['word_dict'] = None
+    args_dict['word_dim'] = args.word_dim
+    if args_dict['word_embedding'] != 'random' and args_dict['word_path']:
+        args_dict['word_dict'], args_dict['word_dim'] = load_word_embeddings.load_embedding_dict(args_dict['word_embedding'],
+                                                                                                 args_dict['word_path'])
+    args_dict['char_dict'] = None
+    args_dict['char_dim'] = args.char_dim
+    if args_dict['char_embedding'] != 'random':
+        args_dict['char_dict'], args_dict['char_dim'] = load_word_embeddings.load_embedding_dict(args_dict['char_embedding'],
+                                                                                                 args_dict['char_path'])
+    args_dict['pos_dict'] = None
+    if args_dict['pos_embedding'] != 'random':
+        args_dict['pos_dict'], args_dict['pos_dim'] = load_word_embeddings.load_embedding_dict(args_dict['pos_embedding'],
+                                                                                                 args_dict['pos_path'])
+    args_dict['alphabet_path'] = path.join(args_dict['model_path'], 'alphabets' + '_src_domain_' + args_dict['domain'] + '/')
+    args_dict['model_name'] = path.join(args_dict['model_path'], args_dict['model_name'])
+    args_dict['eval_mode'] = args.eval_mode
+    args_dict['device'] = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+    args_dict['word_status'] = 'frozen' if args.freeze_word_embeddings else 'fine tune'
+    args_dict['char_status'] = 'enabled' if args.use_char else 'disabled'
+    args_dict['pos_status'] = 'enabled' if args.use_pos else 'disabled'
+    logger.info("Saving arguments to file")
+    save_args(args, args_dict['full_model_name'])
+    logger.info("Creating Alphabets")
+    alphabet_dict = creating_alphabets(args_dict['alphabet_path'], args_dict['alphabet_data_paths'], args_dict['word_dict'])
+    args_dict = {**args_dict, **alphabet_dict}
+    ARGS = namedtuple('ARGS', args_dict.keys())
+    my_args = ARGS(**args_dict)
+    return my_args
+
+
+def creating_alphabets(alphabet_path, alphabet_data_paths, word_dict):
+    train_paths = alphabet_data_paths['train']
+    extra_paths = [v for k,v in alphabet_data_paths.items() if k != 'train']
+    alphabet_dict = {}
+    alphabet_dict['alphabets'] = prepare_data.create_alphabets(alphabet_path,
+                                                               train_paths,
+                                                               extra_paths=extra_paths,
+                                                               max_vocabulary_size=100000,
+                                                               embedd_dict=word_dict)
+    for k, v in alphabet_dict['alphabets'].items():
+        num_key = 'num_' + k.split('_')[0]
+        alphabet_dict[num_key] = v.size()
+        logger.info("%s : %d" % (num_key, alphabet_dict[num_key]))
+    return alphabet_dict
+
+def construct_embedding_table(alphabet, tokens_dict, dim, token_type='word'):
+    if tokens_dict is None:
+        return None
+    scale = np.sqrt(3.0 / dim)
+    table = np.empty([alphabet.size(), dim], dtype=np.float32)
+    table[prepare_data.UNK_ID, :] = np.random.uniform(-scale, scale, [1, dim]).astype(np.float32)
+    oov_tokens = 0
+    for token, index in alphabet.items():
+        if token in ['aTA']:
+            embedding = np.random.uniform(-scale, scale, [1, dim]).astype(np.float32)
+            oov_tokens += 1
+        elif token in tokens_dict:
+            embedding = tokens_dict[token]
+        elif token.lower() in tokens_dict:
+            embedding = tokens_dict[token.lower()]
+        else:
+            embedding = np.random.uniform(-scale, scale, [1, dim]).astype(np.float32)
+            oov_tokens += 1
+        # print(token)
+        table[index, :] = embedding
+    print('token type : %s, number of oov: %d' % (token_type, oov_tokens))
+    table = torch.from_numpy(table)
+    return table
+
+def save_args(args, full_model_name):
+    arg_path = full_model_name + '.arg.json'
+    argparse_dict = vars(args)
+    with open(arg_path, 'w') as f:
+        json.dump(argparse_dict, f)
+
+def generate_optimizer(args, lr, params):
+    params = filter(lambda param: param.requires_grad, params)
+    if args.opt == 'adam':
+        return Adam(params, lr=lr, betas=args.betas, weight_decay=args.gamma, eps=args.epsilon)
+    elif args.opt == 'sgd':
+        return SGD(params, lr=lr, momentum=args.momentum, weight_decay=args.gamma, nesterov=True)
+    else:
+        raise ValueError('Unknown optimization algorithm: %s' % args.opt)
+
+
+def save_checkpoint(model, optimizer, opt, dev_eval_dict, test_eval_dict, full_model_name):
+    path_name = full_model_name + '.pt'
+    print('Saving model to: %s' % path_name)
+    state = {'model_state_dict': model.state_dict(),
+             'optimizer_state_dict': optimizer.state_dict(),
+             'opt': opt,
+             'dev_eval_dict': dev_eval_dict,
+             'test_eval_dict': test_eval_dict}
+    torch.save(state, path_name)
+
+
+def load_checkpoint(args, model, optimizer, dev_eval_dict, test_eval_dict, start_epoch, load_path, strict=True):
+    print('Loading saved model from: %s' % load_path)
+    checkpoint = torch.load(load_path, map_location=args.device)
+    if checkpoint['opt'] != args.opt:
+        raise ValueError('loaded optimizer type is: %s instead of: %s' % (checkpoint['opt'], args.opt))
+    model.load_state_dict(checkpoint['model_state_dict'], strict=strict)
+
+    if strict:
+        generate_optimizer(args, args.learning_rate, model.parameters())
+        optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
+        for state in optimizer.state.values():
+            for k, v in state.items():
+                if isinstance(v, torch.Tensor):
+                    state[k] = v.to(args.device)
+        dev_eval_dict = checkpoint['dev_eval_dict']
+        test_eval_dict = checkpoint['test_eval_dict']
+        start_epoch = dev_eval_dict['in_domain']['epoch']
+    return model, optimizer, dev_eval_dict, test_eval_dict, start_epoch
+
+
+def build_model_and_optimizer(args):
+    word_table = construct_embedding_table(args.alphabets['word_alphabet'], args.word_dict, args.word_dim, token_type='word')
+    char_table = construct_embedding_table(args.alphabets['char_alphabet'], args.char_dict, args.char_dim, token_type='char')
+    pos_table = construct_embedding_table(args.alphabets['pos_alphabet'], args.pos_dict, args.pos_dim, token_type='pos')
+    model = BiAffine_Parser_Gated(args.word_dim, args.num_word, args.char_dim, args.num_char,
+                            args.use_pos, args.use_char, args.pos_dim, args.num_pos,
+                            args.num_filters, args.kernel_size, args.rnn_mode,
+                            args.hidden_size, args.num_layers, args.num_arc,
+                            args.arc_space, args.arc_tag_space, args.num_gates,
+                            embedd_word=word_table, embedd_char=char_table, embedd_pos=pos_table,
+                            p_in=args.p_in, p_out=args.p_out,  p_rnn=args.p_rnn,
+                            biaffine=True, arc_decode=args.arc_decode, initializer=args.initializer)
+    print(model)
+    optimizer = generate_optimizer(args, args.learning_rate, model.parameters())
+    start_epoch = 0
+    dev_eval_dict = {'in_domain': initialize_eval_dict()}
+    test_eval_dict = {'in_domain': initialize_eval_dict()}
+    if args.load_path:
+        model, optimizer, dev_eval_dict, test_eval_dict, start_epoch = \
+            load_checkpoint(args, model, optimizer,
+                            dev_eval_dict, test_eval_dict,
+                            start_epoch, args.load_path, strict=args.strict)
+    if args.load_sequence_taggers_paths:
+        pretrained_dict = {}
+        model_dict = model.state_dict()
+        for idx, path in enumerate(args.load_sequence_taggers_paths):
+            print('Loading saved sequence_tagger from: %s' % path)
+            checkpoint = torch.load(path, map_location=args.device)
+            for k, v in checkpoint['model_state_dict'].items():
+                if 'rnn_encoder.' in k:
+                    pretrained_dict['extra_rnn_encoders.' + str(idx) + '.' + k.replace('rnn_encoder.', '')] = v
+        model_dict.update(pretrained_dict)
+        model.load_state_dict(model_dict)
+    if args.freeze_sequence_taggers:
+        print('Freezing Classifiers')
+        for name, parameter in model.named_parameters():
+            if 'extra_rnn_encoders' in name:
+                parameter.requires_grad = False
+    if args.freeze_word_embeddings:
+        model.rnn_encoder.word_embedd.weight.requires_grad = False
+        # model.rnn_encoder.char_embedd.weight.requires_grad = False
+        # model.rnn_encoder.pos_embedd.weight.requires_grad = False
+    device = args.device
+    model.to(device)
+    return model, optimizer, dev_eval_dict, test_eval_dict, start_epoch
+
+
+def initialize_eval_dict():
+    eval_dict = {}
+    eval_dict['dp_uas'] = 0.0
+    eval_dict['dp_las'] = 0.0
+    eval_dict['epoch'] = 0
+    eval_dict['dp_ucorrect'] = 0.0
+    eval_dict['dp_lcorrect'] = 0.0
+    eval_dict['dp_total'] = 0.0
+    eval_dict['dp_ucomplete_match'] = 0.0
+    eval_dict['dp_lcomplete_match'] = 0.0
+    eval_dict['dp_ucorrect_nopunc'] = 0.0
+    eval_dict['dp_lcorrect_nopunc'] = 0.0
+    eval_dict['dp_total_nopunc'] = 0.0
+    eval_dict['dp_ucomplete_match_nopunc'] = 0.0
+    eval_dict['dp_lcomplete_match_nopunc'] = 0.0
+    eval_dict['dp_root_correct'] = 0.0
+    eval_dict['dp_total_root'] = 0.0
+    eval_dict['dp_total_inst'] = 0.0
+    eval_dict['dp_total'] = 0.0
+    eval_dict['dp_total_inst'] = 0.0
+    eval_dict['dp_total_nopunc'] = 0.0
+    eval_dict['dp_total_root'] = 0.0
+    return eval_dict
+
+def in_domain_evaluation(args, datasets, model, optimizer, dev_eval_dict, test_eval_dict, epoch,
+                         best_model, best_optimizer, patient):
+    # In-domain evaluation
+    curr_dev_eval_dict = evaluation(args, datasets['dev'], 'dev', model, args.domain, epoch, 'current_results')
+    is_best_in_domain = dev_eval_dict['in_domain']['dp_lcorrect_nopunc'] <= curr_dev_eval_dict['dp_lcorrect_nopunc'] or \
+              (dev_eval_dict['in_domain']['dp_lcorrect_nopunc'] == curr_dev_eval_dict['dp_lcorrect_nopunc'] and
+               dev_eval_dict['in_domain']['dp_ucorrect_nopunc'] <= curr_dev_eval_dict['dp_ucorrect_nopunc'])
+
+    if is_best_in_domain:
+        for key, value in curr_dev_eval_dict.items():
+            dev_eval_dict['in_domain'][key] = value
+        curr_test_eval_dict = evaluation(args, datasets['test'], 'test', model, args.domain, epoch, 'current_results')
+        for key, value in curr_test_eval_dict.items():
+            test_eval_dict['in_domain'][key] = value
+        best_model = deepcopy(model)
+        best_optimizer = deepcopy(optimizer)
+        patient = 0
+    else:
+        patient += 1
+    if epoch == args.num_epochs:
+        # save in-domain checkpoint
+        if args.set_num_training_samples is not None:
+            splits_to_write = datasets.keys()
+        else:
+            splits_to_write = ['dev', 'test']
+        for split in splits_to_write:
+            if split == 'dev':
+                eval_dict = dev_eval_dict['in_domain']
+            elif split == 'test':
+                eval_dict = test_eval_dict['in_domain']
+            else:
+                eval_dict = None
+            write_results(args, datasets[split], args.domain, split, best_model, args.domain, eval_dict)
+        print("Saving best model")
+        save_checkpoint(best_model, best_optimizer, args.opt, dev_eval_dict, test_eval_dict, args.full_model_name)
+
+    print('\n')
+    return dev_eval_dict, test_eval_dict, best_model, best_optimizer, patient
+
+
+def evaluation(args, data, split, model, domain, epoch, str_res='results'):
+    # evaluate performance on data
+    model.eval()
+
+    eval_dict = initialize_eval_dict()
+    eval_dict['epoch'] = epoch
+    for batch in prepare_data.iterate_batch(data, args.batch_size, args.device):
+        word, char, pos, ner, heads, arc_tags, auto_label, masks, lengths = batch
+        out_arc, out_arc_tag, masks, lengths = model.forward(word, char, pos, mask=masks, length=lengths)
+        heads_pred, arc_tags_pred, _ = model.decode(args.model_path,word, pos, ner,out_arc, out_arc_tag, mask=masks, length=lengths,
+                                                    leading_symbolic=prepare_data.NUM_SYMBOLIC_TAGS)
+        lengths = lengths.cpu().numpy()
+        word = word.data.cpu().numpy()
+        pos = pos.data.cpu().numpy()
+        ner = ner.data.cpu().numpy()
+        heads = heads.data.cpu().numpy()
+        arc_tags = arc_tags.data.cpu().numpy()
+        heads_pred = heads_pred.data.cpu().numpy()
+        arc_tags_pred = arc_tags_pred.data.cpu().numpy()
+        stats, stats_nopunc, stats_root, num_inst = parse.eval_(word, pos, heads_pred, arc_tags_pred, heads,
+                                                                arc_tags, args.alphabets['word_alphabet'], args.alphabets['pos_alphabet'],
+                                                                lengths, punct_set=args.punct_set, symbolic_root=True)
+        ucorr, lcorr, total, ucm, lcm = stats
+        ucorr_nopunc, lcorr_nopunc, total_nopunc, ucm_nopunc, lcm_nopunc = stats_nopunc
+        corr_root, total_root = stats_root
+        eval_dict['dp_ucorrect'] += ucorr
+        eval_dict['dp_lcorrect'] += lcorr
+        eval_dict['dp_total'] += total
+        eval_dict['dp_ucomplete_match'] += ucm
+        eval_dict['dp_lcomplete_match'] += lcm
+        eval_dict['dp_ucorrect_nopunc'] += ucorr_nopunc
+        eval_dict['dp_lcorrect_nopunc'] += lcorr_nopunc
+        eval_dict['dp_total_nopunc'] += total_nopunc
+        eval_dict['dp_ucomplete_match_nopunc'] += ucm_nopunc
+        eval_dict['dp_lcomplete_match_nopunc'] += lcm_nopunc
+        eval_dict['dp_root_correct'] += corr_root
+        eval_dict['dp_total_root'] += total_root
+        eval_dict['dp_total_inst'] += num_inst
+
+    eval_dict['dp_uas'] = eval_dict['dp_ucorrect'] * 100 / eval_dict['dp_total']  # considering w. punctuation
+    eval_dict['dp_las'] = eval_dict['dp_lcorrect'] * 100 / eval_dict['dp_total']  # considering w. punctuation
+    print_results(eval_dict, split, domain, str_res)
+    return eval_dict
+
+
+def print_results(eval_dict, split, domain, str_res='results'):
+    print('----------------------------------------------------------------------------------------------------------------------------')
+    print('Testing model on domain %s' % domain)
+    print('--------------- Dependency Parsing - %s ---------------' % split)
+    print(
+        str_res + ' on ' + split + '  W. Punct: ucorr: %d, lcorr: %d, total: %d, uas: %.2f%%, las: %.2f%%, ucm: %.2f%%, lcm: %.2f%% (epoch: %d)' % (
+            eval_dict['dp_ucorrect'], eval_dict['dp_lcorrect'], eval_dict['dp_total'],
+            eval_dict['dp_ucorrect'] * 100 / eval_dict['dp_total'],
+            eval_dict['dp_lcorrect'] * 100 / eval_dict['dp_total'],
+            eval_dict['dp_ucomplete_match'] * 100 / eval_dict['dp_total_inst'],
+            eval_dict['dp_lcomplete_match'] * 100 / eval_dict['dp_total_inst'],
+            eval_dict['epoch']))
+    print(
+        str_res + ' on ' + split + '  Wo Punct: ucorr: %d, lcorr: %d, total: %d, uas: %.2f%%, las: %.2f%%, ucm: %.2f%%, lcm: %.2f%% (epoch: %d)' % (
+            eval_dict['dp_ucorrect_nopunc'], eval_dict['dp_lcorrect_nopunc'], eval_dict['dp_total_nopunc'],
+            eval_dict['dp_ucorrect_nopunc'] * 100 / eval_dict['dp_total_nopunc'],
+            eval_dict['dp_lcorrect_nopunc'] * 100 / eval_dict['dp_total_nopunc'],
+            eval_dict['dp_ucomplete_match_nopunc'] * 100 / eval_dict['dp_total_inst'],
+            eval_dict['dp_lcomplete_match_nopunc'] * 100 / eval_dict['dp_total_inst'],
+            eval_dict['epoch']))
+    print(str_res + ' on ' + split + '  Root: corr: %d, total: %d, acc: %.2f%% (epoch: %d)' % (
+        eval_dict['dp_root_correct'], eval_dict['dp_total_root'],
+        eval_dict['dp_root_correct'] * 100 / eval_dict['dp_total_root'], eval_dict['epoch']))
+    print('\n')
+
+def write_results(args, data, data_domain, split, model, model_domain, eval_dict):
+    str_file = args.full_model_name + '_' + split + '_model_domain_' + model_domain + '_data_domain_' + data_domain
+    res_filename = str_file + '_res.txt'
+    pred_filename = str_file + '_pred.txt'
+    gold_filename = str_file + '_gold.txt'
+    if eval_dict is not None:
+        # save results dictionary into a file
+        with open(res_filename, 'w') as f:
+            json.dump(eval_dict, f)
+
+    # save predictions and gold labels into files
+    pred_writer = Writer(args.alphabets)
+    gold_writer = Writer(args.alphabets)
+    pred_writer.start(pred_filename)
+    gold_writer.start(gold_filename)
+    for batch in prepare_data.iterate_batch(data, args.batch_size, args.device):
+        word, char, pos, ner, heads, arc_tags, auto_label, masks, lengths = batch
+        out_arc, out_arc_tag, masks, lengths = model.forward(word, char, pos, mask=masks, length=lengths)
+        heads_pred, arc_tags_pred, _ = model.decode(args.model_path,word, pos,ner,out_arc, out_arc_tag, mask=masks, length=lengths,
+                                                    leading_symbolic=prepare_data.NUM_SYMBOLIC_TAGS)
+        lengths = lengths.cpu().numpy()
+        word = word.data.cpu().numpy()
+        pos = pos.data.cpu().numpy()
+        ner = ner.data.cpu().numpy()
+        heads = heads.data.cpu().numpy()
+        arc_tags = arc_tags.data.cpu().numpy()
+        heads_pred = heads_pred.data.cpu().numpy()
+        arc_tags_pred = arc_tags_pred.data.cpu().numpy()
+        # writing predictions
+        pred_writer.write(word, pos, ner, heads_pred, arc_tags_pred, lengths, symbolic_root=True)
+        # writing gold labels
+        gold_writer.write(word, pos, ner, heads, arc_tags, lengths, symbolic_root=True)
+
+    pred_writer.close()
+    gold_writer.close()
+
+def main():
+    logger.info("Reading and creating arguments")
+    args = read_arguments()
+    logger.info("Reading Data")
+    datasets = {}
+    for split in args.splits:
+        print("Splits are:",split)
+        dataset = prepare_data.read_data_to_variable(args.data_paths[split], args.alphabets, args.device,
+                                                     symbolic_root=True)
+        datasets[split] = dataset
+    if args.set_num_training_samples is not None:
+        print('Setting train and dev to %d samples' % args.set_num_training_samples)
+        datasets = rearrange_splits.rearranging_splits(datasets, args.set_num_training_samples)
+    logger.info("Creating Networks")
+    num_data = sum(datasets['train'][1])
+    model, optimizer, dev_eval_dict, test_eval_dict, start_epoch = build_model_and_optimizer(args)
+    best_model = deepcopy(model)
+    best_optimizer = deepcopy(optimizer)
+
+    logger.info('Training INFO of in domain %s' % args.domain)
+    logger.info('Training on Dependecy Parsing')
+    logger.info("train: gamma: %f, batch: %d, clip: %.2f, unk replace: %.2f" % (args.gamma, args.batch_size, args.clip, args.unk_replace))
+    logger.info('number of training samples for %s is: %d' % (args.domain, num_data))
+    logger.info("dropout(in, out, rnn): (%.2f, %.2f, %s)" % (args.p_in, args.p_out, args.p_rnn))
+    logger.info("num_epochs: %d" % (args.num_epochs))
+    print('\n')
+
+    if not args.eval_mode:
+        logger.info("Training")
+        num_batches = prepare_data.calc_num_batches(datasets['train'], args.batch_size)
+        lr = args.learning_rate
+        patient = 0
+        decay = 0
+        for epoch in range(start_epoch + 1, args.num_epochs + 1):
+            print('Epoch %d (Training: rnn mode: %s, optimizer: %s, learning rate=%.6f, eps=%.1e, decay rate=%.2f (schedule=%d, decay=%d)): ' % (
+                epoch, args.rnn_mode, args.opt, lr, args.epsilon, args.decay_rate, args.schedule, decay))
+            model.train()
+            total_loss = 0.0
+            total_arc_loss = 0.0
+            total_arc_tag_loss = 0.0
+            total_train_inst = 0.0
+
+            train_iter = prepare_data.iterate_batch_rand_bucket_choosing(
+                    datasets['train'], args.batch_size, args.device, unk_replace=args.unk_replace)
+            start_time = time.time()
+            batch_num = 0
+            for batch_num, batch in enumerate(train_iter):
+                batch_num = batch_num + 1
+                optimizer.zero_grad()
+                # compute loss of main task
+                word, char, pos, ner_tags, heads, arc_tags, auto_label, masks, lengths = batch
+                out_arc, out_arc_tag, masks, lengths = model.forward(word, char, pos, mask=masks, length=lengths)
+                loss_arc, loss_arc_tag = model.loss(out_arc, out_arc_tag, heads, arc_tags, mask=masks, length=lengths)
+                loss = loss_arc + loss_arc_tag
+
+                # update losses
+                num_insts = masks.data.sum() - word.size(0)
+                total_arc_loss += loss_arc.item() * num_insts
+                total_arc_tag_loss += loss_arc_tag.item() * num_insts
+                total_loss += loss.item() * num_insts
+                total_train_inst += num_insts
+                # optimize parameters
+                loss.backward()
+                clip_grad_norm_(model.parameters(), args.clip)
+                optimizer.step()
+
+                time_ave = (time.time() - start_time) / batch_num
+                time_left = (num_batches - batch_num) * time_ave
+
+                # update log
+                if batch_num % 50 == 0:
+                    log_info = 'train: %d/%d, domain: %s, total loss: %.2f, arc_loss: %.2f, arc_tag_loss: %.2f, time left: %.2fs' % \
+                               (batch_num, num_batches, args.domain, total_loss / total_train_inst, total_arc_loss / total_train_inst,
+                                total_arc_tag_loss / total_train_inst, time_left)
+                    sys.stdout.write(log_info)
+                    sys.stdout.write('\n')
+                    sys.stdout.flush()
+            print('\n')
+            print('train: %d/%d, domain: %s, total_loss: %.2f, arc_loss: %.2f, arc_tag_loss: %.2f, time: %.2fs' %
+                  (batch_num, num_batches, args.domain, total_loss / total_train_inst, total_arc_loss / total_train_inst,
+                   total_arc_tag_loss / total_train_inst, time.time() - start_time))
+
+            dev_eval_dict, test_eval_dict, best_model, best_optimizer, patient = in_domain_evaluation(args, datasets, model, optimizer, dev_eval_dict, test_eval_dict, epoch, best_model, best_optimizer, patient)
+            if patient >= args.schedule:
+                lr = args.learning_rate / (1.0 + epoch * args.decay_rate)
+                optimizer = generate_optimizer(args, lr, model.parameters())
+                print('updated learning rate to %.6f' % lr)
+                patient = 0
+            print_results(test_eval_dict['in_domain'], 'test', args.domain, 'best_results')
+            print('\n')
+        for split in datasets.keys():
+            eval_dict = evaluation(args, datasets[split], split, best_model, args.domain, epoch, 'best_results')
+            write_results(args, datasets[split], args.domain, split, model, args.domain, eval_dict)
+
+    else:
+        logger.info("Evaluating")
+        epoch = start_epoch
+        for split in ['train', 'dev', 'test','poetry','prose']:
+            eval_dict = evaluation(args, datasets[split], split, model, args.domain, epoch, 'best_results')
+            write_results(args, datasets[split], args.domain, split, model, args.domain, eval_dict)
+
+
+if __name__ == '__main__':
+    main()

examples/SequenceTagger.py

@@ -0,0 +1,597 @@
+from __future__ import print_function
+import sys
+from os import path, makedirs, system, remove
+
+sys.path.append(".")
+sys.path.append("..")
+
+import time
+import argparse
+import uuid
+import json
+import numpy as np
+import torch
+import torch.nn as nn
+from collections import namedtuple
+from copy import deepcopy
+from torch.nn.utils import clip_grad_norm_
+from torch.optim import Adam, SGD
+from utils.io_ import seeds, Writer, get_logger, Index2Instance, prepare_data, write_extra_labels
+from utils.models.sequence_tagger import Sequence_Tagger
+from utils import load_word_embeddings
+from utils.tasks.seqeval import accuracy_score, f1_score, precision_score, recall_score,classification_report
+
+uid = uuid.uuid4().hex[:6]
+
+logger = get_logger('SequenceTagger')
+
+def read_arguments():
+    args_ = argparse.ArgumentParser(description='Sovling SequenceTagger')
+    args_.add_argument('--dataset', choices=['ontonotes', 'ud'], help='Dataset', required=True)
+    args_.add_argument('--domain', help='domain', required=True)
+    args_.add_argument('--rnn_mode', choices=['RNN', 'LSTM', 'GRU'], help='architecture of rnn',
+                       required=True)
+    args_.add_argument('--task', default='distance_from_the_root', choices=['distance_from_the_root', 'number_of_children',\
+     'relative_pos_based', 'language_model','add_label','add_head_coarse_pos','Multitask_POS_predict','Multitask_case_predict',\
+     'Multitask_label_predict','Multitask_coarse_predict','MRL_Person','MRL_Gender','MRL_case','MRL_POS','MRL_no','MRL_label',\
+     'predict_coarse_of_modifier','predict_ma_tag_of_modifier','add_head_ma','predict_case_of_modifier'], help='sequence_tagger task')
+    args_.add_argument('--num_epochs', type=int, default=200, help='Number of training epochs')
+    args_.add_argument('--batch_size', type=int, default=64, help='Number of sentences in each batch')
+    args_.add_argument('--hidden_size', type=int, default=256, help='Number of hidden units in RNN')
+    args_.add_argument('--tag_space', type=int, default=128, help='Dimension of tag space')
+    args_.add_argument('--num_layers', type=int, default=1, help='Number of layers of RNN')
+    args_.add_argument('--num_filters', type=int, default=50, help='Number of filters in CNN')
+    args_.add_argument('--kernel_size', type=int, default=3, help='Size of Kernel for CNN')
+    args_.add_argument('--use_pos', action='store_true', help='use part-of-speech embedding.')
+    args_.add_argument('--use_char', action='store_true', help='use character embedding and CNN.')
+    args_.add_argument('--word_dim', type=int, default=300, help='Dimension of word embeddings')
+    args_.add_argument('--pos_dim', type=int, default=50, help='Dimension of POS embeddings')
+    args_.add_argument('--char_dim', type=int, default=50, help='Dimension of Character embeddings')
+    args_.add_argument('--initializer', choices=['xavier'], help='initialize model parameters')
+    args_.add_argument('--opt', choices=['adam', 'sgd'], help='optimization algorithm')
+    args_.add_argument('--momentum', type=float, default=0.9, help='momentum of optimizer')
+    args_.add_argument('--betas', nargs=2, type=float, default=[0.9, 0.9], help='betas of optimizer')
+    args_.add_argument('--learning_rate', type=float, default=0.01, help='Learning rate')
+    args_.add_argument('--decay_rate', type=float, default=0.05, help='Decay rate of learning rate')
+    args_.add_argument('--schedule', type=int, help='schedule for learning rate decay')
+    args_.add_argument('--clip', type=float, default=5.0, help='gradient clipping')
+    args_.add_argument('--gamma', type=float, default=0.0, help='weight for regularization')
+    args_.add_argument('--epsilon', type=float, default=1e-8, help='epsilon for adam')
+    args_.add_argument('--p_rnn', nargs=2, type=float, required=True, help='dropout rate for RNN')
+    args_.add_argument('--p_in', type=float, default=0.33, help='dropout rate for input embeddings')
+    args_.add_argument('--p_out', type=float, default=0.33, help='dropout rate for output layer')
+    args_.add_argument('--unk_replace', type=float, default=0.,
+                       help='The rate to replace a singleton word with UNK')
+    args_.add_argument('--punct_set', nargs='+', type=str, help='List of punctuations')
+    args_.add_argument('--word_embedding', choices=['random', 'glove', 'fasttext', 'word2vec'],
+                       help='Embedding for words')
+    args_.add_argument('--word_path', help='path for word embedding dict - in case word_embedding is not random')
+    args_.add_argument('--freeze_word_embeddings', action='store_true', help='frozen the word embedding (disable fine-tuning).')
+    args_.add_argument('--char_embedding', choices=['random','hellwig'], help='Embedding for characters',
+                       required=True)
+    args_.add_argument('--pos_embedding', choices=['random','one_hot'], help='Embedding for pos',
+                       required=True)
+    args_.add_argument('--char_path', help='path for character embedding dict')
+    args_.add_argument('--pos_path', help='path for pos embedding dict')
+    args_.add_argument('--use_unlabeled_data', action='store_true', help='flag to use unlabeled data.')
+    args_.add_argument('--use_labeled_data', action='store_true', help='flag to use labeled data.')
+    args_.add_argument('--model_path', help='path for saving model file.', required=True)
+    args_.add_argument('--parser_path', help='path for loading parser files.', default=None)
+    args_.add_argument('--load_path', help='path for loading saved source model file.', default=None)
+    args_.add_argument('--strict',action='store_true', help='if True loaded model state should contain '
+                                                            'exactly the same keys as current model')
+    args_.add_argument('--eval_mode', action='store_true', help='evaluating model without training it')
+    args = args_.parse_args()
+    args_dict = {}
+    args_dict['dataset'] = args.dataset
+    args_dict['domain'] = args.domain
+    args_dict['task'] = args.task
+    args_dict['rnn_mode'] = args.rnn_mode
+    args_dict['load_path'] = args.load_path
+    args_dict['strict'] = args.strict
+    args_dict['model_path'] = args.model_path
+    if not path.exists(args_dict['model_path']):
+        makedirs(args_dict['model_path'])
+    args_dict['parser_path'] = args.parser_path
+    args_dict['model_name'] = 'domain_' + args_dict['domain']
+    args_dict['full_model_name'] = path.join(args_dict['model_path'],args_dict['model_name'])
+    args_dict['use_unlabeled_data'] = args.use_unlabeled_data
+    args_dict['use_labeled_data'] = args.use_labeled_data
+    print(args_dict['parser_path'])
+    if args_dict['task'] == 'number_of_children':
+        args_dict['data_paths'] = write_extra_labels.add_number_of_children(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                            use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                            use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'distance_from_the_root':
+        args_dict['data_paths'] = write_extra_labels.add_distance_from_the_root(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'Multitask_label_predict':
+        args_dict['data_paths'] = write_extra_labels.Multitask_label_predict(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'Multitask_coarse_predict':
+        args_dict['data_paths'] = write_extra_labels.Multitask_coarse_predict(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'Multitask_POS_predict':
+        args_dict['data_paths'] = write_extra_labels.Multitask_POS_predict(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'relative_pos_based':
+        args_dict['data_paths'] = write_extra_labels.add_relative_pos_based(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'add_label':
+        args_dict['data_paths'] = write_extra_labels.add_label(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'add_relative_TAG':
+        args_dict['data_paths'] = write_extra_labels.add_relative_TAG(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'add_head_coarse_pos':
+        args_dict['data_paths'] = write_extra_labels.add_head_coarse_pos(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'predict_ma_tag_of_modifier':
+        args_dict['data_paths'] = write_extra_labels.predict_ma_tag_of_modifier(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'Multitask_case_predict':
+        args_dict['data_paths'] = write_extra_labels.Multitask_case_predict(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'predict_coarse_of_modifier':
+        args_dict['data_paths'] = write_extra_labels.predict_coarse_of_modifier(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'predict_case_of_modifier':
+        args_dict['data_paths'] = write_extra_labels.predict_case_of_modifier(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'add_head_ma':
+        args_dict['data_paths'] = write_extra_labels.add_head_ma(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'MRL_case':
+        args_dict['data_paths'] = write_extra_labels.MRL_case(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'MRL_POS':
+        args_dict['data_paths'] = write_extra_labels.MRL_POS(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'MRL_no':
+        args_dict['data_paths'] = write_extra_labels.MRL_no(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'MRL_label':
+        args_dict['data_paths'] = write_extra_labels.MRL_label(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'MRL_Person':
+        args_dict['data_paths'] = write_extra_labels.MRL_Person(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                             use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                             use_labeled_data=args_dict['use_labeled_data'])
+    elif args_dict['task'] == 'MRL_Gender':
+        args_dict['data_paths'] = write_extra_labels.MRL_Gender(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                                                 use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                                                 use_labeled_data=args_dict['use_labeled_data'])
+    else: #args_dict['task'] == 'language_model':
+        args_dict['data_paths'] = write_extra_labels.add_language_model(args_dict['model_path'], args_dict['parser_path'], args_dict['domain'], args_dict['domain'],
+                                                            use_unlabeled_data=args_dict['use_unlabeled_data'],
+                                                            use_labeled_data=args_dict['use_labeled_data'])
+    args_dict['splits'] = args_dict['data_paths'].keys()
+    alphabet_data_paths = deepcopy(args_dict['data_paths'])
+    if args_dict['dataset'] == 'ontonotes':
+        data_path = 'data/onto_pos_ner_dp'
+    else:
+        data_path = 'data/ud_pos_ner_dp'
+    # Adding more resources to make sure equal alphabet size for all domains
+    for split in args_dict['splits']:
+        if args_dict['dataset'] == 'ontonotes':
+            alphabet_data_paths['additional_' + split] = data_path + '_' + split + '_' + 'all'
+        else:
+            if '_' in args_dict['domain']:
+                alphabet_data_paths[split] = data_path + '_' + split + '_' + args_dict['domain'].split('_')[0]
+            else:
+                alphabet_data_paths[split] = args_dict['data_paths'][split]
+    args_dict['alphabet_data_paths'] = alphabet_data_paths
+    args_dict['num_epochs'] = args.num_epochs
+    args_dict['batch_size'] = args.batch_size
+    args_dict['hidden_size'] = args.hidden_size
+    args_dict['tag_space'] = args.tag_space
+    args_dict['num_layers'] = args.num_layers
+    args_dict['num_filters'] = args.num_filters
+    args_dict['kernel_size'] = args.kernel_size
+    args_dict['learning_rate'] = args.learning_rate
+    args_dict['initializer'] = nn.init.xavier_uniform_ if args.initializer == 'xavier' else None
+    args_dict['opt'] = args.opt
+    args_dict['momentum'] = args.momentum
+    args_dict['betas'] = tuple(args.betas)
+    args_dict['epsilon'] = args.epsilon
+    args_dict['decay_rate'] = args.decay_rate
+    args_dict['clip'] = args.clip
+    args_dict['gamma'] = args.gamma
+    args_dict['schedule'] = args.schedule
+    args_dict['p_rnn'] = tuple(args.p_rnn)
+    args_dict['p_in'] = args.p_in
+    args_dict['p_out'] = args.p_out
+    args_dict['unk_replace'] = args.unk_replace
+    args_dict['punct_set'] = None
+    if args.punct_set is not None:
+        args_dict['punct_set'] = set(args.punct_set)
+        logger.info("punctuations(%d): %s" % (len(args_dict['punct_set']), ' '.join(args_dict['punct_set'])))
+    args_dict['freeze_word_embeddings'] = args.freeze_word_embeddings
+    args_dict['word_embedding'] = args.word_embedding
+    args_dict['word_path'] = args.word_path
+    args_dict['use_char'] = args.use_char
+    args_dict['char_embedding'] = args.char_embedding
+    args_dict['pos_embedding'] = args.pos_embedding
+    args_dict['char_path'] = args.char_path
+    args_dict['pos_path'] = args.pos_path
+    args_dict['use_pos'] = args.use_pos
+    args_dict['pos_dim'] = args.pos_dim
+    args_dict['word_dict'] = None
+    args_dict['word_dim'] = args.word_dim
+    if args_dict['word_embedding'] != 'random' and args_dict['word_path']:
+        args_dict['word_dict'], args_dict['word_dim'] = load_word_embeddings.load_embedding_dict(args_dict['word_embedding'],
+                                                                                                 args_dict['word_path'])
+    args_dict['char_dict'] = None
+    args_dict['char_dim'] = args.char_dim
+    if args_dict['char_embedding'] != 'random':
+        args_dict['char_dict'], args_dict['char_dim'] = load_word_embeddings.load_embedding_dict(args_dict['char_embedding'],
+                                                                                                 args_dict['char_path'])
+    args_dict['pos_dict'] = None
+    if args_dict['pos_embedding'] != 'random':
+        args_dict['pos_dict'], args_dict['pos_dim'] = load_word_embeddings.load_embedding_dict(args_dict['pos_embedding'],
+                                                                                                 args_dict['pos_path'])
+    args_dict['alphabet_path'] = path.join(args_dict['model_path'], 'alphabets' + '_src_domain_' + args_dict['domain'] + '/')
+    args_dict['alphabet_parser_path'] = path.join(args_dict['parser_path'], 'alphabets' + '_src_domain_' + args_dict['domain'] + '/')
+    args_dict['model_name'] = path.join(args_dict['model_path'], args_dict['model_name'])
+    args_dict['eval_mode'] = args.eval_mode
+    args_dict['device'] = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+    args_dict['word_status'] = 'frozen' if args.freeze_word_embeddings else 'fine tune'
+    args_dict['char_status'] = 'enabled' if args.use_char else 'disabled'
+    args_dict['pos_status'] = 'enabled' if args.use_pos else 'disabled'
+    logger.info("Saving arguments to file")
+    save_args(args, args_dict['full_model_name'])
+    logger.info("Creating Alphabets")
+    alphabet_dict = creating_alphabets(args_dict['alphabet_path'], args_dict['alphabet_parser_path'], args_dict['alphabet_data_paths'])
+    args_dict = {**args_dict, **alphabet_dict}
+    ARGS = namedtuple('ARGS', args_dict.keys())
+    my_args = ARGS(**args_dict)
+    return my_args
+
+
+def creating_alphabets(alphabet_path, alphabet_parser_path, alphabet_data_paths):
+    data_paths_list = alphabet_data_paths.values()
+    alphabet_dict = {}
+    alphabet_dict['alphabets'] = prepare_data.create_alphabets_for_sequence_tagger(alphabet_path, alphabet_parser_path, data_paths_list)
+    for k, v in alphabet_dict['alphabets'].items():
+        num_key = 'num_' + k.split('_alphabet')[0]
+        alphabet_dict[num_key] = v.size()
+        logger.info("%s : %d" % (num_key, alphabet_dict[num_key]))
+    return alphabet_dict
+
+def construct_embedding_table(alphabet, tokens_dict, dim, token_type='word'):
+    if tokens_dict is None:
+        return None
+    scale = np.sqrt(3.0 / dim)
+    table = np.empty([alphabet.size(), dim], dtype=np.float32)
+    table[prepare_data.UNK_ID, :] = np.random.uniform(-scale, scale, [1, dim]).astype(np.float32)
+    oov_tokens = 0
+    for token, index in alphabet.items():
+        if token in tokens_dict:
+            embedding = tokens_dict[token]
+            if token =='ata':
+                embedding = np.random.uniform(-scale, scale, [1, dim]).astype(np.float32)     
+        else:
+            embedding = np.random.uniform(-scale, scale, [1, dim]).astype(np.float32)
+            oov_tokens += 1
+        table[index, :] = embedding
+    print('token type : %s, number of oov: %d' % (token_type, oov_tokens))
+    table = torch.from_numpy(table)
+    return table
+
+def get_free_gpu():
+    system('nvidia-smi -q -d Memory |grep -A4 GPU|grep Free > tmp.txt')
+    memory_available = [int(x.split()[2]) for x in open('tmp.txt', 'r').readlines()]
+    remove("tmp.txt")
+    free_device = 'cuda:' + str(np.argmax(memory_available))
+    return free_device
+
+def save_args(args, full_model_name):
+    arg_path = full_model_name + '.arg.json'
+    argparse_dict = vars(args)
+    with open(arg_path, 'w') as f:
+        json.dump(argparse_dict, f)
+
+def generate_optimizer(args, lr, params):
+    params = filter(lambda param: param.requires_grad, params)
+    if args.opt == 'adam':
+        return Adam(params, lr=lr, betas=args.betas, weight_decay=args.gamma, eps=args.epsilon)
+    elif args.opt == 'sgd':
+        return SGD(params, lr=lr, momentum=args.momentum, weight_decay=args.gamma, nesterov=True)
+    else:
+        raise ValueError('Unknown optimization algorithm: %s' % args.opt)
+
+
+def save_checkpoint(model, optimizer, opt, dev_eval_dict, test_eval_dict, full_model_name):
+    path_name = full_model_name + '.pt'
+    print('Saving model to: %s' % path_name)
+    state = {'model_state_dict': model.state_dict(),
+             'optimizer_state_dict': optimizer.state_dict(),
+             'opt': opt, 'dev_eval_dict': dev_eval_dict, 'test_eval_dict': test_eval_dict}
+    torch.save(state, path_name)
+
+
+def load_checkpoint(args, model, optimizer, dev_eval_dict, test_eval_dict, start_epoch, load_path, strict=True):
+    print('Loading saved model from: %s' % load_path)
+    checkpoint = torch.load(load_path, map_location=args.device)
+    if checkpoint['opt'] != args.opt:
+        raise ValueError('loaded optimizer type is: %s instead of: %s' % (checkpoint['opt'], args.opt))
+    model.load_state_dict(checkpoint['model_state_dict'], strict=strict)
+    if strict:
+        optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
+        dev_eval_dict = checkpoint['dev_eval_dict']
+        test_eval_dict = checkpoint['test_eval_dict']
+        start_epoch = dev_eval_dict['in_domain']['epoch']
+    return model, optimizer, dev_eval_dict, test_eval_dict, start_epoch
+
+
+def build_model_and_optimizer(args):
+    word_table = construct_embedding_table(args.alphabets['word_alphabet'], args.word_dict, args.word_dim, token_type='word')
+    char_table = construct_embedding_table(args.alphabets['char_alphabet'], args.char_dict, args.char_dim, token_type='char')
+    pos_table = construct_embedding_table(args.alphabets['pos_alphabet'], args.pos_dict, args.pos_dim, token_type='pos')
+    model = Sequence_Tagger(args.word_dim, args.num_word, args.char_dim, args.num_char,
+                            args.use_pos, args.use_char, args.pos_dim, args.num_pos,
+                            args.num_filters, args.kernel_size, args.rnn_mode,
+                            args.hidden_size, args.num_layers, args.tag_space, args.num_auto_label,
+                            embedd_word=word_table, embedd_char=char_table, embedd_pos=pos_table,
+                            p_in=args.p_in, p_out=args.p_out, p_rnn=args.p_rnn,
+                            initializer=args.initializer)
+    optimizer = generate_optimizer(args, args.learning_rate, model.parameters())
+    start_epoch = 0
+    dev_eval_dict = {'in_domain': initialize_eval_dict()}
+    test_eval_dict = {'in_domain': initialize_eval_dict()}
+    if args.load_path:
+        model, optimizer, dev_eval_dict, test_eval_dict, start_epoch = \
+            load_checkpoint(args, model, optimizer,
+                            dev_eval_dict, test_eval_dict,
+                            start_epoch, args.load_path, strict=args.strict)
+    if args.freeze_word_embeddings:
+        model.rnn_encoder.word_embedd.weight.requires_grad = False
+        # model.rnn_encoder.char_embedd.weight.requires_grad = False
+        # model.rnn_encoder.pos_embedd.weight.requires_grad = False
+    device = args.device
+    model.to(device)
+    return model, optimizer, dev_eval_dict, test_eval_dict, start_epoch
+
+
+def initialize_eval_dict():
+    eval_dict = {}
+    eval_dict['auto_label_accuracy'] = 0.0
+    eval_dict['auto_label_precision'] = 0.0
+    eval_dict['auto_label_recall'] = 0.0
+    eval_dict['auto_label_f1'] = 0.0
+    return eval_dict
+
+def in_domain_evaluation(args, datasets, model, optimizer, dev_eval_dict, test_eval_dict, epoch,
+                         best_model, best_optimizer, patient):
+    # In-domain evaluation
+    curr_dev_eval_dict = evaluation(args, datasets['dev'], 'dev', model, args.domain, epoch, 'current_results')
+    is_best_in_domain = dev_eval_dict['in_domain']['auto_label_f1'] <= curr_dev_eval_dict['auto_label_f1']
+
+    if is_best_in_domain:
+        for key, value in curr_dev_eval_dict.items():
+            dev_eval_dict['in_domain'][key] = value
+        curr_test_eval_dict = evaluation(args, datasets['test'], 'test', model, args.domain, epoch, 'current_results')
+        for key, value in curr_test_eval_dict.items():
+            test_eval_dict['in_domain'][key] = value
+        best_model = deepcopy(model)
+        best_optimizer = deepcopy(optimizer)
+        patient = 0
+    else:
+        patient += 1
+    if epoch == args.num_epochs:
+        # save in-domain checkpoint
+        for split in ['dev', 'test']:
+            eval_dict = dev_eval_dict['in_domain'] if split == 'dev' else test_eval_dict['in_domain']
+            write_results(args, datasets[split], args.domain, split, best_model, args.domain, eval_dict)
+        save_checkpoint(best_model, best_optimizer, args.opt, dev_eval_dict, test_eval_dict, args.full_model_name)
+
+    print('\n')
+    return dev_eval_dict, test_eval_dict, best_model, best_optimizer, patient, curr_dev_eval_dict
+
+
+def evaluation(args, data, split, model, domain, epoch, str_res='results'):
+    # evaluate performance on data
+    model.eval()
+    auto_label_idx2inst = Index2Instance(args.alphabets['auto_label_alphabet'])
+    eval_dict = initialize_eval_dict()
+    eval_dict['epoch'] = epoch
+    pred_labels = []
+    gold_labels = []
+    for batch in prepare_data.iterate_batch(data, args.batch_size, args.device):
+        word, char, pos, ner, heads, arc_tags, auto_label, masks, lengths = batch
+        output, masks, lengths = model.forward(word, char, pos, mask=masks, length=lengths)
+        auto_label_preds = model.decode(output, mask=masks, length=lengths, leading_symbolic=prepare_data.NUM_SYMBOLIC_TAGS)
+        lengths = lengths.cpu().numpy()
+        word = word.data.cpu().numpy()
+        pos = pos.data.cpu().numpy()
+        ner = ner.data.cpu().numpy()
+        heads = heads.data.cpu().numpy()
+        arc_tags = arc_tags.data.cpu().numpy()
+        auto_label = auto_label.data.cpu().numpy()
+        auto_label_preds = auto_label_preds.data.cpu().numpy()
+        gold_labels += auto_label_idx2inst.index2instance(auto_label, lengths, symbolic_root=True)
+        pred_labels += auto_label_idx2inst.index2instance(auto_label_preds, lengths, symbolic_root=True)
+
+    eval_dict['auto_label_accuracy'] = accuracy_score(gold_labels, pred_labels) * 100
+    eval_dict['auto_label_precision'] = precision_score(gold_labels, pred_labels) * 100
+    eval_dict['auto_label_recall'] = recall_score(gold_labels, pred_labels) * 100
+    eval_dict['auto_label_f1'] = f1_score(gold_labels, pred_labels) * 100
+    eval_dict['classification_report'] = classification_report(gold_labels, pred_labels)
+    print_results(eval_dict, split, domain, str_res)
+    return eval_dict
+
+
+def print_results(eval_dict, split, domain, str_res='results'):
+    print('----------------------------------------------------------------------------------------------------------------------------')
+    print('Testing model on domain %s' % domain)
+    print('--------------- sequence_tagger - %s ---------------' % split)
+    print(
+        str_res + ' on ' + split + ' accuracy: %.2f%%, precision: %.2f%%, recall: %.2f%%, F1: %.2f%% (epoch: %d)'
+        % (eval_dict['auto_label_accuracy'], eval_dict['auto_label_precision'], eval_dict['auto_label_recall'], eval_dict['auto_label_f1'],
+           eval_dict['epoch']))
+    print(eval_dict['classification_report'])
+
+
+def write_results(args, data, data_domain, split, model, model_domain, eval_dict):
+    str_file = args.full_model_name + '_' + split + '_model_domain_' + model_domain + '_data_domain_' + data_domain
+    res_filename = str_file + '_res.txt'
+    pred_filename = str_file + '_pred.txt'
+    gold_filename = str_file + '_gold.txt'
+
+    # save results dictionary into a file
+    with open(res_filename, 'w') as f:
+        json.dump(eval_dict, f)
+
+    # save predictions and gold labels into files
+    pred_writer = Writer(args.alphabets)
+    gold_writer = Writer(args.alphabets)
+    pred_writer.start(pred_filename)
+    gold_writer.start(gold_filename)
+    for batch in prepare_data.iterate_batch(data, args.batch_size, args.device):
+        word, char, pos, ner, heads, arc_tags, auto_label, masks, lengths = batch
+        output, masks, lengths = model.forward(word, char, pos, mask=masks, length=lengths)
+        auto_label_preds = model.decode(output, mask=masks, length=lengths,
+                                        leading_symbolic=prepare_data.NUM_SYMBOLIC_TAGS)
+        lengths = lengths.cpu().numpy()
+        word = word.data.cpu().numpy()
+        pos = pos.data.cpu().numpy()
+        ner = ner.data.cpu().numpy()
+        heads = heads.data.cpu().numpy()
+        arc_tags = arc_tags.data.cpu().numpy()
+        auto_label_preds = auto_label_preds.data.cpu().numpy()
+        # writing predictions
+        pred_writer.write(word, pos, ner, heads, arc_tags, lengths, auto_label=auto_label_preds, symbolic_root=True)
+        # writing gold labels
+        gold_writer.write(word, pos, ner, heads, arc_tags, lengths, auto_label=auto_label, symbolic_root=True)
+
+    pred_writer.close()
+    gold_writer.close()
+
+def main():
+    logger.info("Reading and creating arguments")
+    args = read_arguments()
+    logger.info("Reading Data")
+    datasets = {}
+    for split in args.splits:
+        dataset = prepare_data.read_data_to_variable(args.data_paths[split], args.alphabets, args.device,
+                                                     symbolic_root=True)
+        datasets[split] = dataset
+
+    logger.info("Creating Networks")
+    num_data = sum(datasets['train'][1])
+    model, optimizer, dev_eval_dict, test_eval_dict, start_epoch = build_model_and_optimizer(args)
+    best_model = deepcopy(model)
+    best_optimizer = deepcopy(optimizer)
+    logger.info('Training INFO of in domain %s' % args.domain)
+    logger.info('Training on Dependecy Parsing')
+    print(model)
+    logger.info("train: gamma: %f, batch: %d, clip: %.2f, unk replace: %.2f" % (args.gamma, args.batch_size, args.clip, args.unk_replace))
+    logger.info('number of training samples for %s is: %d' % (args.domain, num_data))
+    logger.info("dropout(in, out, rnn): (%.2f, %.2f, %s)" % (args.p_in, args.p_out, args.p_rnn))
+    logger.info("num_epochs: %d" % (args.num_epochs))
+    print('\n')
+
+    if not args.eval_mode:
+        logger.info("Training")
+        num_batches = prepare_data.calc_num_batches(datasets['train'], args.batch_size)
+        lr = args.learning_rate
+        patient = 0
+        terminal_patient = 0
+        decay = 0
+        for epoch in range(start_epoch + 1, args.num_epochs + 1):
+            print('Epoch %d (Training: rnn mode: %s, optimizer: %s, learning rate=%.6f, eps=%.1e, decay rate=%.2f (schedule=%d, decay=%d)): ' % (
+                epoch, args.rnn_mode, args.opt, lr, args.epsilon, args.decay_rate, args.schedule, decay))
+            model.train()
+            total_loss = 0.0
+            total_train_inst = 0.0
+
+            iter = prepare_data.iterate_batch_rand_bucket_choosing(
+                    datasets['train'], args.batch_size, args.device, unk_replace=args.unk_replace)
+            start_time = time.time()
+            batch_num = 0
+            for batch_num, batch in enumerate(iter):
+                batch_num = batch_num + 1
+                optimizer.zero_grad()
+                # compute loss of main task
+                word, char, pos, ner_tags, heads, arc_tags, auto_label, masks, lengths = batch
+                output, masks, lengths = model.forward(word, char, pos, mask=masks, length=lengths)
+                loss = model.loss(output, auto_label, mask=masks, length=lengths)
+
+                # update losses
+                num_insts = masks.data.sum() - word.size(0)
+                total_loss += loss.item() * num_insts
+                total_train_inst += num_insts
+                # optimize parameters
+                loss.backward()
+                clip_grad_norm_(model.parameters(), args.clip)
+                optimizer.step()
+
+                time_ave = (time.time() - start_time) / batch_num
+                time_left = (num_batches - batch_num) * time_ave
+
+                # update log
+                if batch_num % 50 == 0:
+                    log_info = 'train: %d/%d, domain: %s, total loss: %.2f, time left: %.2fs' % \
+                               (batch_num, num_batches, args.domain, total_loss / total_train_inst, time_left)
+                    sys.stdout.write(log_info)
+                    sys.stdout.write('\n')
+                    sys.stdout.flush()
+            print('\n')
+            print('train: %d/%d, domain: %s, total_loss: %.2f, time: %.2fs' %
+                  (batch_num, num_batches, args.domain, total_loss / total_train_inst, time.time() - start_time))
+
+            dev_eval_dict, test_eval_dict, best_model, best_optimizer, patient,curr_dev_eval_dict = in_domain_evaluation(args, datasets, model, optimizer, dev_eval_dict, test_eval_dict, epoch, best_model, best_optimizer, patient)
+            store ={'dev_eval_dict':curr_dev_eval_dict }
+            ############################################# 
+            str_file = args.full_model_name + '_' +'all_epochs'
+            with open(str_file,'a') as f:
+                f.write(str(store)+'\n')
+            if patient == 0:
+                terminal_patient = 0
+            else:
+                terminal_patient += 1
+            if terminal_patient >= 4 * args.schedule:
+                # Save best model and terminate learning
+                cur_epoch = epoch
+                epoch = args.num_epochs
+                in_domain_evaluation(args, datasets, model, optimizer, dev_eval_dict, test_eval_dict, epoch, best_model,
+                                     best_optimizer, patient)
+                log_info = 'Terminating training in epoch %d' % (cur_epoch)
+                sys.stdout.write(log_info)
+                sys.stdout.write('\n')
+                sys.stdout.flush()
+                return
+            if patient >= args.schedule:
+                lr = args.learning_rate / (1.0 + epoch * args.decay_rate)
+                optimizer = generate_optimizer(args, lr, model.parameters())
+                print('updated learning rate to %.6f' % lr)
+                patient = 0
+            print_results(test_eval_dict['in_domain'], 'test', args.domain, 'best_results')
+            print('\n')
+
+    else:
+        logger.info("Evaluating")
+        epoch = start_epoch
+        for split in ['train', 'dev', 'test']:
+            evaluation(args, datasets[split], split, model, args.domain, epoch, 'best_results')
+
+
+if __name__ == '__main__':
+    main()

examples/eval/conll03eval.v2

@@ -0,0 +1,336 @@
+#!/usr/bin/perl -w
+# conlleval: evaluate result of processing CoNLL-2000 shared task
+# usage:     conlleval [-l] [-r] [-d delimiterTag] [-o oTag] < file
+#            README: http://cnts.uia.ac.be/conll2000/chunking/output.html
+# options:   l: generate LaTeX output for tables like in
+#               http://cnts.uia.ac.be/conll2003/ner/example.tex
+#            r: accept raw result tags (without B- and I- prefix;
+#                                       assumes one word per chunk)
+#            d: alternative delimiter tag (default is single space)
+#            o: alternative outside tag (default is O)
+# note:      the file should contain lines with items separated
+#            by $delimiter characters (default space). The final
+#            two items should contain the correct tag and the 
+#            guessed tag in that order. Sentences should be
+#            separated from each other by empty lines or lines
+#            with $boundary fields (default -X-).
+# url:       http://lcg-www.uia.ac.be/conll2000/chunking/
+# started:   1998-09-25
+# version:   2004-01-26
+# author:    Erik Tjong Kim Sang <erikt@uia.ua.ac.be>
+
+use strict;
+
+my $false = 0;
+my $true = 42;
+
+my $boundary = "-X-";     # sentence boundary
+my $correct;              # current corpus chunk tag (I,O,B)
+my $correctChunk = 0;     # number of correctly identified chunks
+my $correctTags = 0;      # number of correct chunk tags
+my $correctType;          # type of current corpus chunk tag (NP,VP,etc.)
+my $delimiter = " ";      # field delimiter
+my $FB1 = 0.0;            # FB1 score (Van Rijsbergen 1979)
+my $firstItem;            # first feature (for sentence boundary checks)
+my $foundCorrect = 0;     # number of chunks in corpus
+my $foundGuessed = 0;     # number of identified chunks
+my $guessed;              # current guessed chunk tag
+my $guessedType;          # type of current guessed chunk tag
+my $i;                    # miscellaneous counter
+my $inCorrect = $false;   # currently processed chunk is correct until now
+my $lastCorrect = "O";    # previous chunk tag in corpus
+my $latex = 0;            # generate LaTeX formatted output
+my $lastCorrectType = ""; # type of previously identified chunk tag
+my $lastGuessed = "O";    # previously identified chunk tag
+my $lastGuessedType = ""; # type of previous chunk tag in corpus
+my $lastType;             # temporary storage for detecting duplicates
+my $line;                 # line
+my $nbrOfFeatures = -1;   # number of features per line
+my $precision = 0.0;      # precision score
+my $oTag = "O";           # outside tag, default O
+my $raw = 0;              # raw input: add B to every token
+my $recall = 0.0;         # recall score
+my $tokenCounter = 0;     # token counter (ignores sentence breaks)
+
+my %correctChunk = ();    # number of correctly identified chunks per type
+my %foundCorrect = ();    # number of chunks in corpus per type
+my %foundGuessed = ();    # number of identified chunks per type
+
+my @features;             # features on line
+my @sortedTypes;          # sorted list of chunk type names
+
+# sanity check
+while (@ARGV and $ARGV[0] =~ /^-/) {
+   if ($ARGV[0] eq "-l") { $latex = 1; shift(@ARGV); }
+   elsif ($ARGV[0] eq "-r") { $raw = 1; shift(@ARGV); }
+   elsif ($ARGV[0] eq "-d") { 
+      shift(@ARGV); 
+      if (not defined $ARGV[0]) { 
+         die "conlleval: -d requires delimiter character"; 
+      }
+      $delimiter = shift(@ARGV);
+   } elsif ($ARGV[0] eq "-o") {
+      shift(@ARGV);
+      if (not defined $ARGV[0]) {
+         die "conlleval: -o requires delimiter character";
+      }
+      $oTag = shift(@ARGV);
+   } else { die "conlleval: unknown argument $ARGV[0]\n"; }
+}
+if (@ARGV) { die "conlleval: unexpected command line argument\n"; }
+# process input
+while (<STDIN>) {
+   chomp($line = $_);
+   @features = split(/$delimiter/,$line);
+   if ($nbrOfFeatures < 0) { $nbrOfFeatures = $#features; }
+   elsif ($nbrOfFeatures != $#features and @features != 0) {
+      printf STDERR "unexpected number of features: %d (%d)\n",
+         $#features+1,$nbrOfFeatures+1;
+      exit(1);
+   }
+   if (@features == 0 or 
+       $features[0] eq $boundary) { @features = ($boundary,"O","O"); }
+   if (@features < 2) { 
+      die "conlleval: unexpected number of features in line $line\n"; 
+   }
+   if ($raw) {
+      if ($features[$#features] eq $oTag) { $features[$#features] = "O"; } 
+      if ($features[$#features-1] eq $oTag) { $features[$#features-1] = "O"; } 
+      if ($features[$#features] ne "O") { 
+         $features[$#features] = "B-$features[$#features]";
+      }
+      if ($features[$#features-1] ne "O") { 
+         $features[$#features-1] = "B-$features[$#features-1]";
+      }
+   }
+   # 20040126 ET code which allows hyphens in the types
+   if ($features[$#features] =~ /^([^-]*)-(.*)$/) {
+      $guessed = $1;
+      $guessedType = $2;
+   } else { 
+      $guessed = $features[$#features]; 
+      $guessedType = ""; 
+   }
+   pop(@features);
+   if ($features[$#features] =~ /^([^-]*)-(.*)$/) {
+      $correct = $1;
+      $correctType = $2;
+   } else { 
+      $correct = $features[$#features]; 
+      $correctType = ""; 
+   }
+   pop(@features);
+#  ($guessed,$guessedType) = split(/-/,pop(@features));
+#  ($correct,$correctType) = split(/-/,pop(@features));
+   $guessedType = $guessedType ? $guessedType : "";
+   $correctType = $correctType ? $correctType : "";
+   $firstItem = shift(@features);
+
+   # 1999-06-26 sentence breaks should always be counted as out of chunk
+   if ( $firstItem eq $boundary ) { $guessed = "O"; }
+
+   if ($inCorrect) {
+      if ( &endOfChunk($lastCorrect,$correct,$lastCorrectType,$correctType) and
+           &endOfChunk($lastGuessed,$guessed,$lastGuessedType,$guessedType) and
+           $lastGuessedType eq $lastCorrectType) {
+         $inCorrect=$false;
+         $correctChunk++;
+         $correctChunk{$lastCorrectType} = $correctChunk{$lastCorrectType} ?
+             $correctChunk{$lastCorrectType}+1 : 1;
+      } elsif ( 
+           &endOfChunk($lastCorrect,$correct,$lastCorrectType,$correctType) != 
+           &endOfChunk($lastGuessed,$guessed,$lastGuessedType,$guessedType) or
+           $guessedType ne $correctType ) {
+         $inCorrect=$false; 
+      }
+   }
+
+   if ( &startOfChunk($lastCorrect,$correct,$lastCorrectType,$correctType) and 
+        &startOfChunk($lastGuessed,$guessed,$lastGuessedType,$guessedType) and
+        $guessedType eq $correctType) { $inCorrect = $true; }
+
+   if ( &startOfChunk($lastCorrect,$correct,$lastCorrectType,$correctType) ) {
+      $foundCorrect++; 
+      $foundCorrect{$correctType} = $foundCorrect{$correctType} ?
+          $foundCorrect{$correctType}+1 : 1;
+   }
+   if ( &startOfChunk($lastGuessed,$guessed,$lastGuessedType,$guessedType) ) {
+      $foundGuessed++; 
+      $foundGuessed{$guessedType} = $foundGuessed{$guessedType} ?
+          $foundGuessed{$guessedType}+1 : 1;
+   }
+   if ( $firstItem ne $boundary ) { 
+      if ( $correct eq $guessed and $guessedType eq $correctType ) { 
+         $correctTags++; 
+      }
+      $tokenCounter++; 
+   }
+
+   $lastGuessed = $guessed;
+   $lastCorrect = $correct;
+   $lastGuessedType = $guessedType;
+   $lastCorrectType = $correctType;
+}
+if ($inCorrect) { 
+   $correctChunk++;
+   $correctChunk{$lastCorrectType} = $correctChunk{$lastCorrectType} ?
+       $correctChunk{$lastCorrectType}+1 : 1;
+}
+
+if (not $latex) {
+   # compute overall precision, recall and FB1 (default values are 0.0)
+   $precision = 100*$correctChunk/$foundGuessed if ($foundGuessed > 0);
+   $recall = 100*$correctChunk/$foundCorrect if ($foundCorrect > 0);
+   $FB1 = 2*$precision*$recall/($precision+$recall)
+      if ($precision+$recall > 0);
+   
+   # print overall performance
+   printf "processed $tokenCounter tokens with $foundCorrect phrases; ";
+   printf "found: $foundGuessed phrases; correct: $correctChunk.\n";
+   if ($tokenCounter>0) {
+      printf "accuracy: %6.2f%%; ",100*$correctTags/$tokenCounter;
+      printf "precision: %6.2f%%; ",$precision;
+      printf "recall: %6.2f%%; ",$recall;
+      printf "FB1: %6.2f\n",$FB1;
+   }
+}
+
+# sort chunk type names
+undef($lastType);
+@sortedTypes = ();
+foreach $i (sort (keys %foundCorrect,keys %foundGuessed)) {
+   if (not($lastType) or $lastType ne $i) { 
+      push(@sortedTypes,($i));
+   }
+   $lastType = $i;
+}
+# print performance per chunk type
+if (not $latex) {
+   for $i (@sortedTypes) {
+      $correctChunk{$i} = $correctChunk{$i} ? $correctChunk{$i} : 0;
+      if (not($foundGuessed{$i})) { $foundGuessed{$i} = 0; $precision = 0.0; }
+      else { $precision = 100*$correctChunk{$i}/$foundGuessed{$i}; }
+      if (not($foundCorrect{$i})) { $recall = 0.0; }
+      else { $recall = 100*$correctChunk{$i}/$foundCorrect{$i}; }
+      if ($precision+$recall == 0.0) { $FB1 = 0.0; }
+      else { $FB1 = 2*$precision*$recall/($precision+$recall); }
+      printf "%17s: ",$i;
+      printf "precision: %6.2f%%; ",$precision;
+      printf "recall: %6.2f%%; ",$recall;
+      printf "FB1: %6.2f  %d\n",$FB1,$foundGuessed{$i};
+   }
+} else {
+   print "        & Precision &  Recall  & F\$_{\\beta=1} \\\\\\hline";
+   for $i (@sortedTypes) {
+      $correctChunk{$i} = $correctChunk{$i} ? $correctChunk{$i} : 0;
+      if (not($foundGuessed{$i})) { $precision = 0.0; }
+      else { $precision = 100*$correctChunk{$i}/$foundGuessed{$i}; }
+      if (not($foundCorrect{$i})) { $recall = 0.0; }
+      else { $recall = 100*$correctChunk{$i}/$foundCorrect{$i}; }
+      if ($precision+$recall == 0.0) { $FB1 = 0.0; }
+      else { $FB1 = 2*$precision*$recall/($precision+$recall); }
+      printf "\n%-7s &  %6.2f\\%% & %6.2f\\%% & %6.2f \\\\",
+             $i,$precision,$recall,$FB1;
+   }
+   print "\\hline\n";
+   $precision = 0.0;
+   $recall = 0;
+   $FB1 = 0.0;
+   $precision = 100*$correctChunk/$foundGuessed if ($foundGuessed > 0);
+   $recall = 100*$correctChunk/$foundCorrect if ($foundCorrect > 0);
+   $FB1 = 2*$precision*$recall/($precision+$recall)
+      if ($precision+$recall > 0);
+   printf "Overall &  %6.2f\\%% & %6.2f\\%% & %6.2f \\\\\\hline\n",
+          $precision,$recall,$FB1;
+}
+
+exit 0;
+
+# endOfChunk: checks if a chunk ended between the previous and current word
+# arguments:  previous and current chunk tags, previous and current types
+# note:       this code is capable of handling other chunk representations
+#             than the default CoNLL-2000 ones, see EACL'99 paper of Tjong
+#             Kim Sang and Veenstra http://xxx.lanl.gov/abs/cs.CL/9907006
+
+sub endOfChunk {
+   my $prevTag = shift(@_);
+   my $tag = shift(@_);
+   my $prevType = shift(@_);
+   my $type = shift(@_);
+   my $chunkEnd = $false;
+
+   if ( $prevTag eq "B" and $tag eq "B" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "B" and $tag eq "O" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "B" and $tag eq "S" ) { $chunkEnd = $true; }
+
+   if ( $prevTag eq "I" and $tag eq "B" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "I" and $tag eq "S" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "I" and $tag eq "O" ) { $chunkEnd = $true; }
+
+   if ( $prevTag eq "E" and $tag eq "E" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "E" and $tag eq "I" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "E" and $tag eq "O" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "E" and $tag eq "S" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "E" and $tag eq "B" ) { $chunkEnd = $true; }
+
+   if ( $prevTag eq "S" and $tag eq "E" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "S" and $tag eq "I" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "S" and $tag eq "O" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "S" and $tag eq "S" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "S" and $tag eq "B" ) { $chunkEnd = $true; }
+   
+
+   if ($prevTag ne "O" and $prevTag ne "." and $prevType ne $type) { 
+      $chunkEnd = $true; 
+   }
+
+   # corrected 1998-12-22: these chunks are assumed to have length 1
+   if ( $prevTag eq "]" ) { $chunkEnd = $true; }
+   if ( $prevTag eq "[" ) { $chunkEnd = $true; }
+
+   return($chunkEnd);   
+}
+
+# startOfChunk: checks if a chunk started between the previous and current word
+# arguments:    previous and current chunk tags, previous and current types
+# note:         this code is capable of handling other chunk representations
+#               than the default CoNLL-2000 ones, see EACL'99 paper of Tjong
+#               Kim Sang and Veenstra http://xxx.lanl.gov/abs/cs.CL/9907006
+
+sub startOfChunk {
+   my $prevTag = shift(@_);
+   my $tag = shift(@_);
+   my $prevType = shift(@_);
+   my $type = shift(@_);
+   my $chunkStart = $false;
+
+   if ( $prevTag eq "B" and $tag eq "B" ) { $chunkStart = $true; }
+   if ( $prevTag eq "I" and $tag eq "B" ) { $chunkStart = $true; }
+   if ( $prevTag eq "O" and $tag eq "B" ) { $chunkStart = $true; }
+   if ( $prevTag eq "S" and $tag eq "B" ) { $chunkStart = $true; }
+   if ( $prevTag eq "E" and $tag eq "B" ) { $chunkStart = $true; }
+   
+   if ( $prevTag eq "B" and $tag eq "S" ) { $chunkStart = $true; }
+   if ( $prevTag eq "I" and $tag eq "S" ) { $chunkStart = $true; }
+   if ( $prevTag eq "O" and $tag eq "S" ) { $chunkStart = $true; }
+   if ( $prevTag eq "S" and $tag eq "S" ) { $chunkStart = $true; }
+   if ( $prevTag eq "E" and $tag eq "S" ) { $chunkStart = $true; }
+
+   if ( $prevTag eq "O" and $tag eq "I" ) { $chunkStart = $true; }
+   if ( $prevTag eq "S" and $tag eq "I" ) { $chunkStart = $true; }
+   if ( $prevTag eq "E" and $tag eq "I" ) { $chunkStart = $true; }
+
+   if ( $prevTag eq "S" and $tag eq "E" ) { $chunkStart = $true; }
+   if ( $prevTag eq "E" and $tag eq "E" ) { $chunkStart = $true; }
+   if ( $prevTag eq "O" and $tag eq "E" ) { $chunkStart = $true; }
+
+   if ($tag ne "O" and $tag ne "." and $prevType ne $type) { 
+      $chunkStart = $true; 
+   }
+
+   # corrected 1998-12-22: these chunks are assumed to have length 1
+   if ( $tag eq "[" ) { $chunkStart = $true; }
+   if ( $tag eq "]" ) { $chunkStart = $true; }
+
+   return($chunkStart);   
+}

examples/eval/conll06eval.pl

@@ -0,0 +1,1826 @@
+#!/usr/bin/env perl
+
+# Author: Yuval Krymolowski
+# Addition of precision and recall 
+#   and of frame confusion list: Sabine Buchholz
+# Addition of DEPREL + ATTACHMENT:
+#   Prokopis Prokopidis (prokopis at ilsp dot gr)
+# Acknowledgements: 
+#   to Markus Kuhn for suggesting the use of 
+#   the Unicode category property
+
+if ($] < 5.008001)
+{
+  printf STDERR <<EOM
+
+ This script requires PERL 5.8.1 for running.
+ The new version is needed for proper handling
+ of Unicode characters.
+
+ Please obtain a new version or contact the shared task team
+ if you are unable to upgrade PERL.
+
+EOM
+;
+  exit(1) ;
+}
+
+require Encode;
+
+use strict ;
+use warnings;
+use Getopt::Std ;
+
+my ($usage) = <<EOT
+
+  CoNLL-X evaluation script:
+
+   [perl] eval.pl [OPTIONS] -g <gold standard> -s <system output>
+
+  This script evaluates a system output with respect to a gold standard.
+  Both files should be in UTF-8 encoded CoNLL-X tabular format.
+
+  Punctuation tokens (those where all characters have the Unicode
+  category property "Punctuation") are ignored for scoring (unless the
+  -p flag is used).
+
+  The output breaks down the errors according to their type and context.
+
+  Optional parameters:
+     -o FILE : output: print output to FILE (default is standard output)
+     -q : quiet:       only print overall performance, without the details
+     -b : evalb:       produce output in a format similar to evalb 
+                       (http://nlp.cs.nyu.edu/evalb/); use together with -q
+     -p : punctuation: also score on punctuation (default is not to score on it)
+     -v : version:     show the version number
+     -h : help:        print this help text and exit
+
+EOT
+;
+
+my ($line_num) ;
+my ($sep) = '0x01' ;
+
+my ($START) = '.S' ;
+my ($END) = '.E' ;
+
+my ($con_err_num) = 3 ;
+my ($freq_err_num) = 10 ;
+my ($spec_err_loc_con) = 8 ;
+
+################################################################################
+###                              subfunctions                                ###
+################################################################################
+
+# Whether a string consists entirely of characters with the Unicode
+# category property "Punctuation" (see "man perlunicode")
+sub is_uni_punct
+{
+  my ($word) = @_ ;
+
+  return scalar(Encode::decode_utf8($word)=~ /^\p{Punctuation}+$/) ;
+}
+
+# The length of a unicode string, excluding non-spacing marks
+# (for example vowel marks in Arabic)
+
+sub uni_len
+{
+  my ($word) = @_ ;
+  my ($ch, $l) ;
+
+  $l = 0 ;
+  foreach $ch (split(//,  Encode::decode_utf8($word)))
+  {
+    if ($ch !~ /^\p{NonspacingMark}/)
+    {
+      $l++ ;
+    }
+  }
+
+  return $l ;
+}
+
+sub filter_context_counts
+{ # filter_context_counts
+
+  my ($vec, $num, $max_len) = @_ ;
+  my ($con, $l, $thresh) ;
+
+  $thresh = (sort {$b <=> $a} values %{$vec})[$num-1] ;
+
+  foreach $con (keys %{$vec})
+  {
+    if (${$vec}{$con} < $thresh)
+    {
+      delete ${$vec}{$con} ;
+      next ;
+    }
+
+    $l = uni_len($con) ;
+
+    if ($l > ${$max_len})
+    {
+      ${$max_len} = $l ;
+    }
+  }
+
+} # filter_context_counts
+
+sub print_context
+{ # print_context
+
+  my ($counts, $counts_pos, $max_con_len, $max_con_pos_len) = @_ ;
+  my (@v_con, @v_con_pos, $con, $con_pos, $i, $n) ;
+
+  printf OUT "  %-*s | %-4s | %-4s | %-4s | %-4s", $max_con_pos_len, 'CPOS', 'any', 'head', 'dep', 'both' ;
+  printf OUT "  ||" ;
+  printf OUT "  %-*s | %-4s | %-4s | %-4s | %-4s", $max_con_len, 'word', 'any', 'head', 'dep', 'both' ;
+  printf OUT "\n" ;
+  printf OUT "  %s-+------+------+------+-----", '-' x $max_con_pos_len;
+  printf OUT "--++" ;
+  printf OUT "--%s-+------+------+------+-----", '-' x $max_con_len;
+  printf OUT "\n" ;
+
+  @v_con = sort {${$counts}{tot}{$b} <=> ${$counts}{tot}{$a}} keys %{${$counts}{tot}} ;
+  @v_con_pos = sort {${$counts_pos}{tot}{$b} <=> ${$counts_pos}{tot}{$a}} keys %{${$counts_pos}{tot}} ;
+
+  $n = scalar @v_con ;
+  if (scalar @v_con_pos > $n)
+  {
+    $n = scalar @v_con_pos ;
+  }
+
+  foreach $i (0 .. $n-1)
+  {
+    if (defined $v_con_pos[$i])
+    {
+      $con_pos = $v_con_pos[$i] ;
+      printf OUT "  %-*s | %4d | %4d | %4d | %4d",
+	$max_con_pos_len, $con_pos, ${$counts_pos}{tot}{$con_pos},
+	  ${$counts_pos}{err_head}{$con_pos}, ${$counts_pos}{err_dep}{$con_pos},
+	    ${$counts_pos}{err_dep}{$con_pos}+${$counts_pos}{err_head}{$con_pos}-${$counts_pos}{tot}{$con_pos} ;
+    }
+    else
+    {
+      printf OUT "  %-*s | %4s | %4s | %4s | %4s",
+	$max_con_pos_len, ' ', ' ', ' ', ' ', ' ' ;
+    }
+
+    printf OUT "  ||" ;
+
+    if (defined $v_con[$i])
+    {
+      $con = $v_con[$i] ;
+      printf OUT "  %-*s | %4d | %4d | %4d | %4d",
+	$max_con_len+length($con)-uni_len($con), $con, ${$counts}{tot}{$con},
+	  ${$counts}{err_head}{$con}, ${$counts}{err_dep}{$con},
+	    ${$counts}{err_dep}{$con}+${$counts}{err_head}{$con}-${$counts}{tot}{$con} ;
+    }
+    else
+    {
+      printf OUT "  %-*s | %4s | %4s | %4s | %4s",
+	$max_con_len, ' ', ' ', ' ', ' ', ' ' ;
+    }
+
+    printf OUT "\n" ;
+  }
+
+  printf OUT "  %s-+------+------+------+-----", '-' x $max_con_pos_len;
+  printf OUT "--++" ;
+  printf OUT "--%s-+------+------+------+-----", '-' x $max_con_len;
+  printf OUT "\n" ;
+
+  printf OUT "\n\n" ;
+
+} # print_context
+
+sub num_as_word
+{
+  my ($num) = @_ ;
+
+  $num = abs($num) ;
+
+  if ($num == 1)
+  {
+    return ('one word') ;
+  }
+  elsif ($num == 2)
+  {
+    return ('two words') ;
+  }
+  elsif ($num == 3)
+  {
+    return ('three words') ;
+  }
+  elsif ($num == 4)
+  {
+    return ('four words') ;
+  }
+  else
+  {
+    return ($num.' words') ;
+  }
+}
+
+sub describe_err
+{ # describe_err
+
+  my ($head_err, $head_aft_bef, $dep_err) = @_ ;
+  my ($dep_g, $dep_s, $desc) ;
+  my ($head_aft_bef_g, $head_aft_bef_s) = split(//, $head_aft_bef) ;
+
+  if ($head_err eq '-')
+  {
+    $desc = 'correct head' ;
+
+    if ($head_aft_bef_s eq '0')
+    {
+      $desc .= ' (0)' ;
+    }
+    elsif ($head_aft_bef_s eq 'e')
+    {
+      $desc .= ' (the focus word)' ;
+    }
+    elsif ($head_aft_bef_s eq 'a')
+    {
+      $desc .= ' (after the focus word)' ;
+    }
+    elsif ($head_aft_bef_s eq 'b')
+    {
+      $desc .= ' (before the focus word)' ;
+    }
+  }
+  elsif ($head_aft_bef_s eq '0')
+  {
+    $desc = 'head = 0 instead of ' ;
+    if ($head_aft_bef_g eq 'a')
+    {
+      $desc.= 'after ' ;
+    }
+    if ($head_aft_bef_g eq 'b')
+    {
+      $desc.= 'before ' ;
+    }
+    $desc .= 'the focus word' ;
+  }
+  elsif ($head_aft_bef_g eq '0')
+  {
+    $desc = 'head is ' ;
+    if ($head_aft_bef_g eq 'a')
+    {
+      $desc.= 'after ' ;
+    }
+    if ($head_aft_bef_g eq 'b')
+    {
+      $desc.= 'before ' ;
+    }
+    $desc .= 'the focus word instead of 0' ;
+  }
+  else
+  {
+    $desc = num_as_word($head_err) ;
+    if ($head_err < 0)
+    {
+      $desc .= ' before' ;
+    }
+    else
+    {
+      $desc .= ' after' ;
+    }
+
+    $desc = 'head '.$desc.' the correct head ' ;
+
+    if ($head_aft_bef_s eq '0')
+    {
+      $desc .= '(0' ;
+    }
+    elsif ($head_aft_bef_s eq 'e')
+    {
+      $desc .= '(the focus word' ;
+    }
+    elsif ($head_aft_bef_s eq 'a')
+    {
+      $desc .= '(after the focus word' ;
+    }
+    elsif ($head_aft_bef_s eq 'b')
+    {
+      $desc .= '(before the focus word' ;
+    }
+
+    if ($head_aft_bef_g ne $head_aft_bef_s)
+    {
+      $desc .= ' instead of' ;
+      if ($head_aft_bef_s eq '0')
+      {
+	$desc .= '0' ;
+      }
+      elsif ($head_aft_bef_s eq 'e')
+      {
+	$desc .= 'the focus word' ;
+      }
+      elsif ($head_aft_bef_s eq 'a')
+      {
+	$desc .= 'after the focus word' ;
+      }
+      elsif ($head_aft_bef_s eq 'b')
+      {
+	$desc .= 'before the focus word' ;
+      }
+    }
+
+    $desc .= ')' ;
+  }
+
+  $desc .= ', ' ;
+
+  if ($dep_err eq '-')
+  {
+    $desc .= 'correct dependency' ;
+  }
+  else
+  {
+    ($dep_g, $dep_s) = ($dep_err =~ /^(.*)->(.*)$/) ;
+    $desc .= sprintf('dependency "%s" instead of "%s"', $dep_s, $dep_g) ;
+  }
+
+  return($desc) ;
+
+} # describe_err
+
+sub get_context
+{ # get_context
+
+  my ($sent, $i_w) = @_ ;
+  my ($w_2, $w_1, $w1, $w2) ;
+  my ($p_2, $p_1, $p1, $p2) ;
+
+  if ($i_w >= 2)
+  {
+    $w_2 = ${${$sent}[$i_w-2]}{word} ;
+    $p_2 = ${${$sent}[$i_w-2]}{pos} ;
+  }
+  else
+  {
+    $w_2 = $START ;
+    $p_2 = $START ;
+  }
+
+  if ($i_w >= 1)
+  {
+    $w_1 = ${${$sent}[$i_w-1]}{word} ;
+    $p_1 = ${${$sent}[$i_w-1]}{pos} ;
+  }
+  else
+  {
+    $w_1 = $START ;
+    $p_1 = $START ;
+  }
+
+  if ($i_w <= scalar @{$sent}-2)
+  {
+    $w1 = ${${$sent}[$i_w+1]}{word} ;
+    $p1 = ${${$sent}[$i_w+1]}{pos} ;
+  }
+  else
+  {
+    $w1 = $END ;
+    $p1 = $END ;
+  }
+
+  if ($i_w <= scalar @{$sent}-3)
+  {
+    $w2 = ${${$sent}[$i_w+2]}{word} ;
+    $p2 = ${${$sent}[$i_w+2]}{pos} ;
+  }
+  else
+  {
+    $w2 = $END ;
+    $p2 = $END ;
+  }
+
+  return ($w_2, $w_1, $w1, $w2, $p_2, $p_1, $p1, $p2) ;
+
+} # get_context
+
+sub read_sent
+{ # read_sent
+
+  my ($sent_gold, $sent_sys) = @_ ;
+  my ($line_g, $line_s, $new_sent) ;
+  my (%fields_g, %fields_s) ;
+
+  $new_sent = 1 ;
+
+  @{$sent_gold} = () ;
+  @{$sent_sys} = () ;
+
+  while (1)
+  { # main reading loop
+
+    $line_g = <GOLD> ;
+    $line_s = <SYS> ;
+
+    $line_num++ ;
+
+    # system output has fewer lines than gold standard
+    if ((defined $line_g) && (! defined $line_s))
+    {
+	printf STDERR "line mismatch, line %d:\n", $line_num ;
+	printf STDERR " gold: %s", $line_g ;
+	printf STDERR " sys : past end of file\n" ;
+	exit(1) ;
+    }
+
+    # system output has more lines than gold standard
+    if ((! defined $line_g) && (defined $line_s))
+    {
+	printf STDERR "line mismatch, line %d:\n", $line_num ;
+	printf STDERR " gold: past end of file\n" ;
+	printf STDERR " sys : %s", $line_s ;
+	exit(1) ;
+    }
+    
+    # end of file reached for both
+    if ((! defined $line_g) && (! defined $line_s))
+    {
+	return (1) ;
+    }
+
+    # one contains end of sentence but other one does not
+    if (($line_g =~ /^\s+$/) != ($line_s =~ /^\s+$/))
+    {
+      printf STDERR "line mismatch, line %d:\n", $line_num ;
+      printf STDERR " gold: %s", $line_g ;
+      printf STDERR " sys : %s", $line_s ;
+      exit(1) ;
+    }
+
+    # end of sentence reached
+    if ($line_g =~ /^\s+$/)
+    {
+	return(0) ;
+    }
+
+    # now both lines contain information
+
+    if ($new_sent)
+    {
+      $new_sent = 0 ;
+    }
+
+    # 'official' column names
+    # options.output = ['id','form','lemma','cpostag','postag',
+    #                   'feats','head','deprel','phead','pdeprel']
+
+    @fields_g{'word', 'pos', 'head', 'dep'} = (split (/\s+/, $line_g))[1, 3, 6, 7] ;
+
+    push @{$sent_gold}, { %fields_g } ;
+
+    @fields_s{'word', 'pos', 'head', 'dep'} = (split (/\s+/, $line_s))[1, 3, 6, 7] ;
+
+    if (($fields_g{word} ne $fields_s{word})
+	||
+	($fields_g{pos} ne $fields_s{pos}))
+    {
+      printf STDERR "Word/pos mismatch, line %d:\n", $line_num ;
+      printf STDERR " gold: %s", $line_g ;
+      printf STDERR " sys : %s", $line_s ;
+      exit(1) ;
+    }
+
+    push @{$sent_sys}, { %fields_s } ;
+
+  } # main reading loop
+  
+} # read_sent
+
+################################################################################
+###                                  main                                    ###
+################################################################################
+
+our ($opt_g, $opt_s, $opt_o, $opt_h, $opt_v, $opt_q, $opt_p, $opt_b) ;
+
+my ($sent_num, $eof, $word_num, @err_sent) ;
+my (@sent_gold, @sent_sys, @starts) ;
+my ($word, $pos, $wp, $head_g, $dep_g, $head_s, $dep_s) ;
+my (%counts, $err_head, $err_dep, $con, $con1, $con_pos, $con_pos1, $thresh) ;
+my ($head_err, $dep_err, @cur_err, %err_counts, $err_counter, $err_desc) ;
+my ($loc_con, %loc_con_err_counts, %err_desc) ;
+my ($head_aft_bef_g, $head_aft_bef_s, $head_aft_bef) ;
+my ($con_bef, $con_aft, $con_bef_2, $con_aft_2, @bits, @e_bits, @v_con, @v_con_pos) ;
+my ($con_pos_bef, $con_pos_aft, $con_pos_bef_2, $con_pos_aft_2) ;
+my ($max_word_len, $max_pos_len, $max_con_len, $max_con_pos_len) ;
+my ($max_word_spec_len, $max_con_bef_len, $max_con_aft_len) ;
+my (%freq_err, $err) ;
+
+my ($i, $j, $i_w, $l, $n_args) ;
+my ($w_2, $w_1, $w1, $w2) ;
+my ($wp_2, $wp_1, $wp1, $wp2) ;
+my ($p_2, $p_1, $p1, $p2) ;
+
+my ($short_output) ;
+my ($score_on_punct) ;
+$counts{punct} = 0; # initialize
+
+getopts("g:o:s:qvhpb") ;
+
+if (defined $opt_v)
+{
+    my $id = '$Id: eval.pl,v 1.9 2006/05/09 20:30:01 yuval Exp $';
+    my @parts = split ' ',$id;
+    print "Version $parts[2]\n";
+    exit(0);
+}
+
+if ((defined $opt_h) || ((! defined $opt_g) && (! defined $opt_s)))
+{
+  die $usage ;
+}
+
+if (! defined $opt_g)
+{
+  die "Gold standard file (-g) missing\n" ;
+}
+
+if (! defined $opt_s)
+{
+  die "System output file (-s) missing\n" ;
+}
+
+if (! defined $opt_o)
+{
+  $opt_o = '-' ;
+}
+
+if (defined $opt_q)
+{
+    $short_output = 1 ;
+} else {
+    $short_output = 0 ;
+}
+
+if (defined $opt_p)
+{
+    $score_on_punct = 1 ;
+} else {
+    $score_on_punct = 0 ;
+}
+
+$line_num = 0 ;
+$sent_num = 0 ;
+$eof = 0 ;
+
+@err_sent = () ;
+@starts = () ;
+
+%{$err_sent[0]} = () ;
+
+$max_pos_len = length('CPOS') ;
+
+################################################################################
+###                              reading input                               ###
+################################################################################
+
+open (GOLD, "<$opt_g") || die "Could not open gold standard file $opt_g\n" ;
+open (SYS,  "<$opt_s") || die "Could not open system output file $opt_s\n" ;
+open (OUT,  ">$opt_o") || die "Could not open output file $opt_o\n" ;
+
+
+if (defined $opt_b) {  # produce output similar to evalb
+    print OUT "     Sent.          Attachment      Correct        Scoring          \n";
+    print OUT "    ID Tokens  -   Unlab. Lab.   HEAD HEAD+DEPREL   tokens   - - - -\n";
+    print OUT "  ============================================================================\n";
+}
+
+
+while (! $eof)
+{ # main reading loop
+
+  $starts[$sent_num] = $line_num+1 ;
+  $eof = read_sent(\@sent_gold, \@sent_sys) ;
+
+  $sent_num++ ;
+
+  %{$err_sent[$sent_num]} = () ;
+  $word_num = scalar @sent_gold ;
+
+  # for accuracy per sentence
+  my %sent_counts = ( tot      => 0,
+		      err_any  => 0,
+		      err_head => 0
+		      ); 
+
+  # printf "$sent_num $word_num\n" ;
+
+  my @frames_g = ('** '); # the initial frame for the virtual root
+  my @frames_s = ('** '); # the initial frame for the virtual root
+  foreach $i_w (0 .. $word_num-1)
+  { # loop on words
+      push @frames_g, ''; # initialize
+      push @frames_s, ''; # initialize
+  }
+
+  foreach $i_w (0 .. $word_num-1)
+  { # loop on words
+
+    ($word, $pos, $head_g, $dep_g)
+      = @{$sent_gold[$i_w]}{'word', 'pos', 'head', 'dep'} ;
+    $wp = $word.' / '.$pos ;
+
+    # printf "%d: %s %s %s %s\n", $i_w,  $word, $pos, $head_g, $dep_g ;
+
+    if ((! $score_on_punct) && is_uni_punct($word))
+    {
+      $counts{punct}++ ;
+      # ignore punctuations
+      next ;
+    }
+
+    if (length($pos) > $max_pos_len)
+    {
+      $max_pos_len = length($pos) ;
+    }
+
+    ($head_s, $dep_s) = @{$sent_sys[$i_w]}{'head', 'dep'} ;
+
+    $counts{tot}++ ;
+    $counts{word}{$wp}{tot}++ ;
+    $counts{pos}{$pos}{tot}++ ;
+    $counts{head}{$head_g-$i_w-1}{tot}++ ;
+
+    # for frame confusions
+    # add child to frame of parent
+    $frames_g[$head_g] .= "$dep_g ";
+    $frames_s[$head_s] .= "$dep_s ";
+    # add to frame of token itself
+    $frames_g[$i_w+1] .= "*$dep_g* "; # $i_w+1 because $i_w starts counting at zero
+    $frames_s[$i_w+1] .= "*$dep_g* ";
+
+    # for precision and recall of DEPREL
+    $counts{dep}{$dep_g}{tot}++ ;     # counts for gold standard deprels
+    $counts{dep2}{$dep_g}{$dep_s}++ ; # counts for confusions
+    $counts{dep_s}{$dep_s}{tot}++ ;   # counts for system deprels
+    $counts{all_dep}{$dep_g} = 1 ;    # list of all deprels that occur ...
+    $counts{all_dep}{$dep_s} = 1 ;    # ... in either gold or system output
+
+    # for precision and recall of HEAD direction
+    my $dir_g;
+    if ($head_g == 0) {
+	$dir_g = 'to_root';
+    } elsif ($head_g < $i_w+1) { # $i_w+1 because $i_w starts counting at zero
+                                 # also below
+	$dir_g = 'left';
+    } elsif ($head_g > $i_w+1) {
+	$dir_g = 'right';
+    } else {
+        # token links to itself; should never happen in correct gold standard
+	$dir_g = 'self'; 
+    }
+    my $dir_s;
+    if ($head_s == 0) {
+	$dir_s = 'to_root';
+    } elsif ($head_s < $i_w+1) {
+	$dir_s = 'left';
+    } elsif ($head_s > $i_w+1) {
+	$dir_s = 'right';
+    } else {
+        # token links to itself; should not happen in good system 
+        # (but not forbidden in shared task)
+	$dir_s = 'self'; 
+    }
+    $counts{dir_g}{$dir_g}{tot}++ ;   # counts for gold standard head direction
+    $counts{dir2}{$dir_g}{$dir_s}++ ; # counts for confusions
+    $counts{dir_s}{$dir_s}{tot}++ ;   # counts for system head direction
+
+    # for precision and recall of HEAD distance
+    my $dist_g;
+    if ($head_g == 0) {
+	$dist_g = 'to_root';
+    } elsif ( abs($head_g - ($i_w+1)) <= 1 ) {
+	$dist_g = '1'; # includes the 'self' cases
+    } elsif ( abs($head_g - ($i_w+1)) <= 2 ) {
+	$dist_g = '2';
+    } elsif ( abs($head_g - ($i_w+1)) <= 6 ) {
+	$dist_g = '3-6';
+    } else {
+	$dist_g = '7-...';
+    }
+    my $dist_s;
+    if ($head_s == 0) {
+	$dist_s = 'to_root';
+    } elsif ( abs($head_s - ($i_w+1)) <= 1 ) {
+	$dist_s = '1'; # includes the 'self' cases
+    } elsif ( abs($head_s - ($i_w+1)) <= 2 ) {
+	$dist_s = '2';
+    } elsif ( abs($head_s - ($i_w+1)) <= 6 ) {
+	$dist_s = '3-6';
+    } else {
+	$dist_s = '7-...';
+    }
+    $counts{dist_g}{$dist_g}{tot}++ ;    # counts for gold standard head distance
+    $counts{dist2}{$dist_g}{$dist_s}++ ; # counts for confusions
+    $counts{dist_s}{$dist_s}{tot}++ ;    # counts for system head distance
+
+
+    $err_head = ($head_g ne $head_s) ; # error in head
+    $err_dep = ($dep_g ne $dep_s) ;    # error in deprel
+
+    $head_err = '-' ;
+    $dep_err = '-' ;
+
+    # for accuracy per sentence
+    $sent_counts{tot}++ ;
+    if ($err_dep || $err_head) {
+	$sent_counts{err_any}++ ;
+    }
+    if ($err_head) {
+	$sent_counts{err_head}++ ;
+    }
+
+    # total counts and counts for CPOS involved in errors
+
+    if ($head_g eq '0')
+    {
+      $head_aft_bef_g = '0' ;
+    }
+    elsif ($head_g eq $i_w+1)
+    {
+      $head_aft_bef_g = 'e' ;
+    }
+    else
+    {
+      $head_aft_bef_g = ($head_g <= $i_w+1 ? 'b' : 'a') ;
+    }
+
+    if ($head_s eq '0')
+    {
+      $head_aft_bef_s = '0' ;
+    }
+    elsif ($head_s eq $i_w+1)
+    {
+      $head_aft_bef_s = 'e' ;
+    }
+    else
+    {
+      $head_aft_bef_s = ($head_s <= $i_w+1 ? 'b' : 'a') ;
+    }
+
+    $head_aft_bef = $head_aft_bef_g.$head_aft_bef_s ;
+
+    if ($err_head)
+    {
+      if ($head_aft_bef_s eq '0')
+      {
+	$head_err = 0 ;
+      }
+      else
+      {
+	$head_err = $head_s-$head_g ;
+      }
+
+      $err_sent[$sent_num]{head}++ ;
+      $counts{err_head}{tot}++ ;
+      $counts{err_head}{$head_err}++ ;
+
+      $counts{word}{err_head}{$wp}++ ;
+      $counts{pos}{$pos}{err_head}{tot}++ ;
+      $counts{pos}{$pos}{err_head}{$head_err}++ ;
+    }
+
+    if ($err_dep)
+    {
+      $dep_err = $dep_g.'->'.$dep_s ;
+      $err_sent[$sent_num]{dep}++ ;
+      $counts{err_dep}{tot}++ ;
+      $counts{err_dep}{$dep_err}++ ;
+
+      $counts{word}{err_dep}{$wp}++ ;
+      $counts{pos}{$pos}{err_dep}{tot}++ ;
+      $counts{pos}{$pos}{err_dep}{$dep_err}++ ;
+
+      if ($err_head)
+      {
+	$counts{err_both}++ ;
+	$counts{pos}{$pos}{err_both}++ ;
+      }
+    }
+
+    ### DEPREL + ATTACHMENT
+    if ((!$err_dep) && ($err_head)) {
+	$counts{err_head_corr_dep}{tot}++ ;
+	$counts{err_head_corr_dep}{$dep_s}++ ;
+    }
+    ### DEPREL + ATTACHMENT
+
+    # counts for words involved in errors
+
+    if (! ($err_head || $err_dep))
+    {
+      next ;
+    }
+
+    $err_sent[$sent_num]{word}++ ;
+    $counts{err_any}++ ;
+    $counts{word}{err_any}{$wp}++ ;
+    $counts{pos}{$pos}{err_any}++ ;
+
+    ($w_2, $w_1, $w1, $w2, $p_2, $p_1, $p1, $p2) = get_context(\@sent_gold, $i_w) ;
+
+    if ($w_2 ne $START)
+    {
+      $wp_2 = $w_2.' / '.$p_2 ;
+    }
+    else
+    {
+      $wp_2 = $w_2 ;
+    }
+
+    if ($w_1 ne $START)
+    {
+      $wp_1 = $w_1.' / '.$p_1 ;
+    }
+    else
+    {
+      $wp_1 = $w_1 ;
+    }
+
+    if ($w1 ne $END)
+    {
+      $wp1 = $w1.' / '.$p1 ;
+    }
+    else
+    {
+      $wp1 = $w1 ;
+    }
+
+    if ($w2 ne $END)
+    {
+      $wp2 = $w2.' / '.$p2 ;
+    }
+    else
+    {
+      $wp2 = $w2 ;
+    }
+
+    $con_bef = $wp_1 ;
+    $con_bef_2 = $wp_2.' + '.$wp_1 ;
+    $con_aft = $wp1 ;
+    $con_aft_2 = $wp1.' + '.$wp2 ;
+
+    $con_pos_bef = $p_1 ;
+    $con_pos_bef_2 = $p_2.'+'.$p_1 ;
+    $con_pos_aft = $p1 ;
+    $con_pos_aft_2 = $p1.'+'.$p2 ;
+
+    if ($w_1 ne $START)
+    {
+      # do not count '.S' as a word context
+      $counts{con_bef_2}{tot}{$con_bef_2}++ ;
+      $counts{con_bef_2}{err_head}{$con_bef_2} += $err_head ;
+      $counts{con_bef_2}{err_dep}{$con_bef_2} += $err_dep ;
+      $counts{con_bef}{tot}{$con_bef}++ ;
+      $counts{con_bef}{err_head}{$con_bef} += $err_head ;
+      $counts{con_bef}{err_dep}{$con_bef} += $err_dep ;
+    }
+
+    if ($w1 ne $END)
+    {
+      # do not count '.E' as a word context
+      $counts{con_aft_2}{tot}{$con_aft_2}++ ;
+      $counts{con_aft_2}{err_head}{$con_aft_2} += $err_head ;
+      $counts{con_aft_2}{err_dep}{$con_aft_2} += $err_dep ;
+      $counts{con_aft}{tot}{$con_aft}++ ;
+      $counts{con_aft}{err_head}{$con_aft} += $err_head ;
+      $counts{con_aft}{err_dep}{$con_aft} += $err_dep ;
+    }
+
+    $counts{con_pos_bef_2}{tot}{$con_pos_bef_2}++ ;
+    $counts{con_pos_bef_2}{err_head}{$con_pos_bef_2} += $err_head ;
+    $counts{con_pos_bef_2}{err_dep}{$con_pos_bef_2} += $err_dep ;
+    $counts{con_pos_bef}{tot}{$con_pos_bef}++ ;
+    $counts{con_pos_bef}{err_head}{$con_pos_bef} += $err_head ;
+    $counts{con_pos_bef}{err_dep}{$con_pos_bef} += $err_dep ;
+
+    $counts{con_pos_aft_2}{tot}{$con_pos_aft_2}++ ;
+    $counts{con_pos_aft_2}{err_head}{$con_pos_aft_2} += $err_head ;
+    $counts{con_pos_aft_2}{err_dep}{$con_pos_aft_2} += $err_dep ;
+    $counts{con_pos_aft}{tot}{$con_pos_aft}++ ;
+    $counts{con_pos_aft}{err_head}{$con_pos_aft} += $err_head ;
+    $counts{con_pos_aft}{err_dep}{$con_pos_aft} += $err_dep ;
+
+    $err = $head_err.$sep.$head_aft_bef.$sep.$dep_err ;
+    $freq_err{$err}++ ;
+
+  } # loop on words
+
+  foreach $i_w (0 .. $word_num) # including one for the virtual root
+  { # loop on words
+      if ($frames_g[$i_w] ne $frames_s[$i_w]) {
+	  $counts{frame2}{"$frames_g[$i_w]/ $frames_s[$i_w]"}++ ;
+      }
+  }
+
+  if (defined $opt_b) { # produce output similar to evalb
+      if ($word_num > 0) {
+	  my ($unlabeled,$labeled) = ('NaN', 'NaN');
+	  if ($sent_counts{tot} > 0) { # there are scoring tokens
+	      $unlabeled = 100-$sent_counts{err_head}*100.0/$sent_counts{tot};
+	      $labeled   = 100-$sent_counts{err_any} *100.0/$sent_counts{tot};
+	  }
+	  printf OUT "  %4d %4d    0  %6.2f %6.2f  %4d    %4d        %4d    0 0 0 0\n", 
+	  $sent_num, $word_num, 
+	  $unlabeled, $labeled, 
+	  $sent_counts{tot}-$sent_counts{err_head}, 
+	  $sent_counts{tot}-$sent_counts{err_any}, 
+	  $sent_counts{tot},;
+      }
+  }
+
+} # main reading loop
+
+################################################################################
+###                             printing output                              ###
+################################################################################
+
+if (defined $opt_b) {  # produce output similar to evalb
+    print OUT "\n\n";
+}
+printf OUT "  Labeled   attachment score: %d / %d * 100 = %.2f %%\n", 
+    $counts{tot}-$counts{err_any},      $counts{tot}, 100-$counts{err_any}*100.0/$counts{tot} ;
+printf OUT "  Unlabeled attachment score: %d / %d * 100 = %.2f %%\n", 
+    $counts{tot}-$counts{err_head}{tot}, $counts{tot}, 100-$counts{err_head}{tot}*100.0/$counts{tot} ;
+printf OUT "  Label accuracy score:       %d / %d * 100 = %.2f %%\n", 
+    $counts{tot}-$counts{err_dep}{tot}, $counts{tot}, 100-$counts{err_dep}{tot}*100.0/$counts{tot} ;
+
+if ($short_output)
+{
+    exit(0) ;
+}
+printf OUT "\n  %s\n\n", '=' x 80 ;
+printf OUT "  Evaluation of the results in %s\n  vs. gold standard %s:\n\n", $opt_s, $opt_g ;
+
+printf OUT "  Legend: '%s' - the beginning of a sentence, '%s' - the end of a sentence\n\n", $START, $END ;
+
+printf OUT "  Number of non-scoring tokens: $counts{punct}\n\n";
+
+printf OUT "  The overall accuracy and its distribution over CPOSTAGs\n\n" ;
+printf OUT "%s\n", "  -----------+-------+-------+------+-------+------+-------+-------" ;
+
+printf OUT "  %-10s | %-5s | %-5s |   %%  | %-5s |   %%  | %-5s |   %%\n",
+  'Accuracy', 'words', 'right', 'right', 'both' ;
+printf OUT "  %-10s | %-5s | %-5s |      | %-5s |      | %-5s |\n",
+  ' ', ' ', 'head', ' dep', 'right' ;
+
+printf OUT "%s\n", "  -----------+-------+-------+------+-------+------+-------+-------" ;
+
+printf OUT "  %-10s | %5d | %5d | %3.0f%% | %5d | %3.0f%% | %5d | %3.0f%%\n",
+  'total', $counts{tot},
+  $counts{tot}-$counts{err_head}{tot}, 100-$counts{err_head}{tot}*100.0/$counts{tot},
+  $counts{tot}-$counts{err_dep}{tot}, 100-$counts{err_dep}{tot}*100.0/$counts{tot},
+  $counts{tot}-$counts{err_any}, 100-$counts{err_any}*100.0/$counts{tot} ;
+
+printf OUT "%s\n", "  -----------+-------+-------+------+-------+------+-------+-------" ;
+
+foreach $pos (sort {$counts{pos}{$b}{tot} <=> $counts{pos}{$a}{tot}} keys %{$counts{pos}})
+{
+    if (! defined($counts{pos}{$pos}{err_head}{tot}))
+    {
+	$counts{pos}{$pos}{err_head}{tot} = 0 ;
+    }
+    if (! defined($counts{pos}{$pos}{err_dep}{tot}))
+    {
+	$counts{pos}{$pos}{err_dep}{tot} = 0 ;
+    }
+    if (! defined($counts{pos}{$pos}{err_any}))
+    {
+	$counts{pos}{$pos}{err_any} = 0 ;
+    }
+
+    printf OUT "  %-10s | %5d | %5d | %3.0f%% | %5d | %3.0f%% | %5d | %3.0f%%\n",
+    $pos, $counts{pos}{$pos}{tot},
+    $counts{pos}{$pos}{tot}-$counts{pos}{$pos}{err_head}{tot}, 100-$counts{pos}{$pos}{err_head}{tot}*100.0/$counts{pos}{$pos}{tot},
+    $counts{pos}{$pos}{tot}-$counts{pos}{$pos}{err_dep}{tot}, 100-$counts{pos}{$pos}{err_dep}{tot}*100.0/$counts{pos}{$pos}{tot},
+    $counts{pos}{$pos}{tot}-$counts{pos}{$pos}{err_any}, 100-$counts{pos}{$pos}{err_any}*100.0/$counts{pos}{$pos}{tot} ;
+}
+
+printf OUT "%s\n", "  -----------+-------+-------+------+-------+------+-------+-------" ;
+
+printf OUT "\n\n" ;
+
+printf OUT "  The overall error rate and its distribution over CPOSTAGs\n\n" ;
+printf OUT "%s\n", "  -----------+-------+-------+------+-------+------+-------+-------" ;
+
+printf OUT "  %-10s | %-5s | %-5s |   %%  | %-5s |   %%  | %-5s |   %%\n",
+  'Error', 'words', 'head', ' dep', 'both' ;
+printf OUT "  %-10s | %-5s | %-5s |      | %-5s |      | %-5s |\n",
+
+  'Rate', ' ', 'err', ' err', 'wrong' ;
+
+printf OUT "%s\n", "  -----------+-------+-------+------+-------+------+-------+-------" ;
+
+printf OUT "  %-10s | %5d | %5d | %3.0f%% | %5d | %3.0f%% | %5d | %3.0f%%\n",
+  'total', $counts{tot},
+  $counts{err_head}{tot}, $counts{err_head}{tot}*100.0/$counts{tot},
+  $counts{err_dep}{tot}, $counts{err_dep}{tot}*100.0/$counts{tot},
+  $counts{err_both}, $counts{err_both}*100.0/$counts{tot} ;
+
+printf OUT "%s\n", "  -----------+-------+-------+------+-------+------+-------+-------" ;
+
+foreach $pos (sort {$counts{pos}{$b}{tot} <=> $counts{pos}{$a}{tot}} keys %{$counts{pos}})
+{
+    if (! defined($counts{pos}{$pos}{err_both}))
+    {
+	$counts{pos}{$pos}{err_both} = 0 ;
+    }
+
+    printf OUT "  %-10s | %5d | %5d | %3.0f%% | %5d | %3.0f%% | %5d | %3.0f%%\n",
+    $pos, $counts{pos}{$pos}{tot},
+    $counts{pos}{$pos}{err_head}{tot}, $counts{pos}{$pos}{err_head}{tot}*100.0/$counts{pos}{$pos}{tot},
+    $counts{pos}{$pos}{err_dep}{tot}, $counts{pos}{$pos}{err_dep}{tot}*100.0/$counts{pos}{$pos}{tot},
+    $counts{pos}{$pos}{err_both}, $counts{pos}{$pos}{err_both}*100.0/$counts{pos}{$pos}{tot} ;
+    
+}
+
+printf OUT "%s\n", "  -----------+-------+-------+------+-------+------+-------+-------" ;
+
+### added by Sabine Buchholz
+printf OUT "\n\n";
+printf OUT "  Precision and recall of DEPREL\n\n";
+printf OUT "  ----------------+------+---------+--------+------------+---------------\n";
+printf OUT "  deprel          | gold | correct | system | recall (%%) | precision (%%) \n";
+printf OUT "  ----------------+------+---------+--------+------------+---------------\n";
+foreach my $dep (sort keys %{$counts{all_dep}}) {
+    # initialize
+    my ($tot_corr, $tot_g, $tot_s, $prec, $rec) = (0, 0, 0, 'NaN', 'NaN');
+
+    if (defined($counts{dep2}{$dep}{$dep})) {
+	$tot_corr = $counts{dep2}{$dep}{$dep};
+    } 
+    if (defined($counts{dep}{$dep}{tot})) {
+    	$tot_g = $counts{dep}{$dep}{tot};
+	$rec = sprintf("%.2f",$tot_corr / $tot_g * 100);
+    }
+    if (defined($counts{dep_s}{$dep}{tot})) {
+	$tot_s = $counts{dep_s}{$dep}{tot};
+	$prec = sprintf("%.2f",$tot_corr / $tot_s * 100);
+    }
+    printf OUT "  %-15s | %4d | %7d | %6d | %10s | %13s\n",
+    $dep, $tot_g, $tot_corr, $tot_s, $rec, $prec;
+}
+
+### DEPREL + ATTACHMENT:
+### Same as Sabine's DEPREL apart from $tot_corr calculation
+printf OUT "\n\n";
+printf OUT "  Precision and recall of DEPREL + ATTACHMENT\n\n";
+printf OUT "  ----------------+------+---------+--------+------------+---------------\n";
+printf OUT "  deprel          | gold | correct | system | recall (%%) | precision (%%) \n";
+printf OUT "  ----------------+------+---------+--------+------------+---------------\n";
+foreach my $dep (sort keys %{$counts{all_dep}}) {
+    # initialize
+    my ($tot_corr, $tot_g, $tot_s, $prec, $rec) = (0, 0, 0, 'NaN', 'NaN');
+
+    if (defined($counts{dep2}{$dep}{$dep})) {
+	if (defined($counts{err_head_corr_dep}{$dep})) {
+	    $tot_corr = $counts{dep2}{$dep}{$dep} - $counts{err_head_corr_dep}{$dep};
+	} else {
+	    $tot_corr = $counts{dep2}{$dep}{$dep};
+	}
+    } 
+    if (defined($counts{dep}{$dep}{tot})) {
+    	$tot_g = $counts{dep}{$dep}{tot};
+	$rec = sprintf("%.2f",$tot_corr / $tot_g * 100);
+    }
+    if (defined($counts{dep_s}{$dep}{tot})) {
+	$tot_s = $counts{dep_s}{$dep}{tot};
+	$prec = sprintf("%.2f",$tot_corr / $tot_s * 100);
+    }
+    printf OUT "  %-15s | %4d | %7d | %6d | %10s | %13s\n",
+    $dep, $tot_g, $tot_corr, $tot_s, $rec, $prec;
+}
+### DEPREL + ATTACHMENT
+
+printf OUT "\n\n";
+printf OUT "  Precision and recall of binned HEAD direction\n\n";
+printf OUT "  ----------------+------+---------+--------+------------+---------------\n";
+printf OUT "  direction       | gold | correct | system | recall (%%) | precision (%%) \n";
+printf OUT "  ----------------+------+---------+--------+------------+---------------\n";
+foreach my $dir ('to_root', 'left', 'right', 'self') {
+    # initialize
+    my ($tot_corr, $tot_g, $tot_s, $prec, $rec) = (0, 0, 0, 'NaN', 'NaN');
+
+    if (defined($counts{dir2}{$dir}{$dir})) {
+	$tot_corr = $counts{dir2}{$dir}{$dir};
+    } 
+    if (defined($counts{dir_g}{$dir}{tot})) {
+    	$tot_g = $counts{dir_g}{$dir}{tot};
+	$rec = sprintf("%.2f",$tot_corr / $tot_g * 100);
+    }
+    if (defined($counts{dir_s}{$dir}{tot})) {
+	$tot_s = $counts{dir_s}{$dir}{tot};
+	$prec = sprintf("%.2f",$tot_corr / $tot_s * 100);
+    }
+    printf OUT "  %-15s | %4d | %7d | %6d | %10s | %13s\n",
+    $dir, $tot_g, $tot_corr, $tot_s, $rec, $prec;
+}
+
+printf OUT "\n\n";
+printf OUT "  Precision and recall of binned HEAD distance\n\n";
+printf OUT "  ----------------+------+---------+--------+------------+---------------\n";
+printf OUT "  distance        | gold | correct | system | recall (%%) | precision (%%) \n";
+printf OUT "  ----------------+------+---------+--------+------------+---------------\n";
+foreach my $dist ('to_root', '1', '2', '3-6', '7-...') {
+    # initialize
+    my ($tot_corr, $tot_g, $tot_s, $prec, $rec) = (0, 0, 0, 'NaN', 'NaN');
+
+    if (defined($counts{dist2}{$dist}{$dist})) {
+	$tot_corr = $counts{dist2}{$dist}{$dist};
+    } 
+    if (defined($counts{dist_g}{$dist}{tot})) {
+    	$tot_g = $counts{dist_g}{$dist}{tot};
+	$rec = sprintf("%.2f",$tot_corr / $tot_g * 100);
+    }
+    if (defined($counts{dist_s}{$dist}{tot})) {
+	$tot_s = $counts{dist_s}{$dist}{tot};
+	$prec = sprintf("%.2f",$tot_corr / $tot_s * 100);
+    }
+    printf OUT "  %-15s | %4d | %7d | %6d | %10s | %13s\n",
+    $dist, $tot_g, $tot_corr, $tot_s, $rec, $prec;
+}
+
+printf OUT "\n\n";
+printf OUT "  Frame confusions (gold versus system; *...* marks the head token)\n\n";
+foreach my $frame (sort {$counts{frame2}{$b} <=> $counts{frame2}{$a}} keys %{$counts{frame2}})
+{
+    if ($counts{frame2}{$frame} >= 5) # (make 5 a changeable threshold later)
+    {
+	printf OUT "  %3d  %s\n", $counts{frame2}{$frame}, $frame;
+    }
+}
+### end of: added by Sabine Buchholz
+
+
+#
+# Leave only the 5 words mostly involved in errors
+#
+
+
+$thresh = (sort {$b <=> $a} values %{$counts{word}{err_any}})[4] ;
+
+# ensure enough space for title
+$max_word_len = length('word') ;
+
+foreach $word (keys %{$counts{word}{err_any}})
+{
+  if ($counts{word}{err_any}{$word} < $thresh)
+  {
+    delete $counts{word}{err_any}{$word} ;
+    next ;
+  }
+
+  $l = uni_len($word) ;
+  if ($l > $max_word_len)
+  {
+    $max_word_len = $l ;
+  }
+}
+
+# filter a case when the difference between the error counts
+# for 2-word and 1-word contexts is small
+# (leave the 2-word context)
+
+foreach $con (keys %{$counts{con_aft_2}{tot}})
+{
+  ($w1) = split(/\+/, $con) ;
+  
+  if (defined $counts{con_aft}{tot}{$w1} &&
+      $counts{con_aft}{tot}{$w1}-$counts{con_aft_2}{tot}{$con} <= 1)
+  {
+    delete $counts{con_aft}{tot}{$w1} ;
+  }
+}
+
+foreach $con (keys %{$counts{con_bef_2}{tot}})
+{
+  ($w_2, $w_1) = split(/\+/, $con) ;
+
+  if (defined $counts{con_bef}{tot}{$w_1} &&
+      $counts{con_bef}{tot}{$w_1}-$counts{con_bef_2}{tot}{$con} <= 1)
+  {
+    delete $counts{con_bef}{tot}{$w_1} ;
+  }
+}
+
+foreach $con_pos (keys %{$counts{con_pos_aft_2}{tot}})
+{
+  ($p1) = split(/\+/, $con_pos) ;
+
+  if (defined($counts{con_pos_aft}{tot}{$p1}) &&
+      $counts{con_pos_aft}{tot}{$p1}-$counts{con_pos_aft_2}{tot}{$con_pos} <= 1)
+  {
+    delete $counts{con_pos_aft}{tot}{$p1} ;
+  }
+}
+
+foreach $con_pos (keys %{$counts{con_pos_bef_2}{tot}})
+{
+  ($p_2, $p_1) = split(/\+/, $con_pos) ;
+
+  if (defined($counts{con_pos_bef}{tot}{$p_1}) &&
+      $counts{con_pos_bef}{tot}{$p_1}-$counts{con_pos_bef_2}{tot}{$con_pos} <= 1)
+  {
+    delete $counts{con_pos_bef}{tot}{$p_1} ;
+  }
+}
+
+# for each context type, take the three contexts most involved in errors
+
+$max_con_len = 0 ;
+
+filter_context_counts($counts{con_bef_2}{tot}, $con_err_num, \$max_con_len) ;
+
+filter_context_counts($counts{con_bef}{tot}, $con_err_num, \$max_con_len) ;
+
+filter_context_counts($counts{con_aft}{tot}, $con_err_num, \$max_con_len) ;
+
+filter_context_counts($counts{con_aft_2}{tot}, $con_err_num, \$max_con_len) ;
+
+# for each CPOS context type, take the three CPOS contexts most involved in errors
+
+$max_con_pos_len = 0 ;
+
+$thresh = (sort {$b <=> $a} values %{$counts{con_pos_bef_2}{tot}})[$con_err_num-1] ;
+
+foreach $con_pos (keys %{$counts{con_pos_bef_2}{tot}})
+{
+  if ($counts{con_pos_bef_2}{tot}{$con_pos} < $thresh)
+  {
+    delete $counts{con_pos_bef_2}{tot}{$con_pos} ;
+    next ;
+  }
+  if (length($con_pos) > $max_con_pos_len)
+  {
+    $max_con_pos_len = length($con_pos) ;
+  }
+}
+
+$thresh = (sort {$b <=> $a} values %{$counts{con_pos_bef}{tot}})[$con_err_num-1] ;
+
+foreach $con_pos (keys %{$counts{con_pos_bef}{tot}})
+{
+  if ($counts{con_pos_bef}{tot}{$con_pos} < $thresh)
+  {
+    delete $counts{con_pos_bef}{tot}{$con_pos} ;
+    next ;
+  }
+  if (length($con_pos) > $max_con_pos_len)
+  {
+    $max_con_pos_len = length($con_pos) ;
+  }
+}
+
+$thresh = (sort {$b <=> $a} values %{$counts{con_pos_aft}{tot}})[$con_err_num-1] ;
+
+foreach $con_pos (keys %{$counts{con_pos_aft}{tot}})
+{
+  if ($counts{con_pos_aft}{tot}{$con_pos} < $thresh)
+  {
+    delete $counts{con_pos_aft}{tot}{$con_pos} ;
+    next ;
+  }
+  if (length($con_pos) > $max_con_pos_len)
+  {
+    $max_con_pos_len = length($con_pos) ;
+  }
+}
+
+$thresh = (sort {$b <=> $a} values %{$counts{con_pos_aft_2}{tot}})[$con_err_num-1] ;
+
+foreach $con_pos (keys %{$counts{con_pos_aft_2}{tot}})
+{
+  if ($counts{con_pos_aft_2}{tot}{$con_pos} < $thresh)
+  {
+    delete $counts{con_pos_aft_2}{tot}{$con_pos} ;
+    next ;
+  }
+  if (length($con_pos) > $max_con_pos_len)
+  {
+    $max_con_pos_len = length($con_pos) ;
+  }
+}
+
+# printing
+
+# ------------- focus words
+
+printf OUT "\n\n" ;
+printf OUT "  %d focus words where most of the errors occur:\n\n", scalar keys %{$counts{word}{err_any}} ;
+
+printf OUT "  %-*s | %-4s | %-4s | %-4s | %-4s\n", $max_word_len, ' ', 'any', 'head', 'dep', 'both' ;
+printf OUT "  %s-+------+------+------+------\n", '-' x $max_word_len;
+
+foreach $word (sort {$counts{word}{err_any}{$b} <=> $counts{word}{err_any}{$a}} keys %{$counts{word}{err_any}})
+{
+    if (!defined($counts{word}{err_head}{$word}))
+    {
+	$counts{word}{err_head}{$word} = 0 ;
+    }
+    if (! defined($counts{word}{err_dep}{$word}))
+    {
+	$counts{word}{err_dep}{$word} = 0 ;
+    }
+    if (! defined($counts{word}{err_any}{$word}))
+    {
+	$counts{word}{err_any}{$word} = 0;
+    }
+    printf OUT "  %-*s | %4d | %4d | %4d | %4d\n",
+    $max_word_len+length($word)-uni_len($word), $word, $counts{word}{err_any}{$word},
+    $counts{word}{err_head}{$word},
+    $counts{word}{err_dep}{$word},
+    $counts{word}{err_dep}{$word}+$counts{word}{err_head}{$word}-$counts{word}{err_any}{$word} ;
+}
+
+printf OUT "  %s-+------+------+------+------\n", '-' x $max_word_len;
+
+# ------------- contexts
+
+printf OUT "\n\n" ;
+
+printf OUT "  one-token preceeding contexts where most of the errors occur:\n\n" ;
+
+print_context($counts{con_bef}, $counts{con_pos_bef}, $max_con_len, $max_con_pos_len) ;
+
+printf OUT "  two-token preceeding contexts where most of the errors occur:\n\n" ;
+
+print_context($counts{con_bef_2}, $counts{con_pos_bef_2}, $max_con_len, $max_con_pos_len) ;
+
+printf OUT "  one-token following contexts where most of the errors occur:\n\n" ;
+
+print_context($counts{con_aft}, $counts{con_pos_aft}, $max_con_len, $max_con_pos_len) ;
+
+printf OUT "  two-token following contexts where most of the errors occur:\n\n" ;
+
+print_context($counts{con_aft_2}, $counts{con_pos_aft_2}, $max_con_len, $max_con_pos_len) ;
+
+# ------------- Sentences
+
+printf OUT "  Sentence with the highest number of word errors:\n" ;
+$i = (sort { (defined($err_sent[$b]{word}) && $err_sent[$b]{word})
+		 <=> (defined($err_sent[$a]{word}) && $err_sent[$a]{word}) } 1 .. $sent_num)[0] ;
+printf OUT "   Sentence %d line %d, ", $i, $starts[$i-1] ;
+printf OUT "%d head errors, %d dependency errors, %d word errors\n",
+  $err_sent[$i]{head}, $err_sent[$i]{dep}, $err_sent[$i]{word} ;
+
+printf OUT "\n\n" ;
+
+printf OUT "  Sentence with the highest number of head errors:\n" ;
+$i = (sort { (defined($err_sent[$b]{head}) && $err_sent[$b]{head}) 
+		 <=> (defined($err_sent[$a]{head}) && $err_sent[$a]{head}) } 1 .. $sent_num)[0] ;
+printf OUT "   Sentence %d line %d, ", $i, $starts[$i-1] ;
+printf OUT "%d head errors, %d dependency errors, %d word errors\n",
+  $err_sent[$i]{head}, $err_sent[$i]{dep}, $err_sent[$i]{word} ;
+
+printf OUT "\n\n" ;
+
+printf OUT "  Sentence with the highest number of dependency errors:\n" ;
+$i = (sort { (defined($err_sent[$b]{dep}) && $err_sent[$b]{dep}) 
+		 <=> (defined($err_sent[$a]{dep}) && $err_sent[$a]{dep}) } 1 .. $sent_num)[0] ;
+printf OUT "   Sentence %d line %d, ", $i, $starts[$i-1] ;
+printf OUT "%d head errors, %d dependency errors, %d word errors\n",
+  $err_sent[$i]{head}, $err_sent[$i]{dep}, $err_sent[$i]{word} ;
+
+#
+# Second pass, collect statistics of the frequent errors
+#
+
+# filter the errors, leave the most frequent $freq_err_num errors
+
+$i = 0 ;
+
+$thresh = (sort {$b <=> $a} values %freq_err)[$freq_err_num-1] ;
+
+foreach $err (keys %freq_err)
+{
+  if ($freq_err{$err} < $thresh)
+  {
+    delete $freq_err{$err} ;
+  }
+}
+
+# in case there are several errors with the threshold count
+
+$freq_err_num = scalar keys %freq_err ;
+
+%err_counts = () ;
+
+$eof = 0 ;
+
+seek (GOLD, 0, 0) ;
+seek (SYS, 0, 0) ;
+
+while (! $eof)
+{ # second reading loop
+
+  $eof = read_sent(\@sent_gold, \@sent_sys) ;
+  $sent_num++ ;
+
+  $word_num = scalar @sent_gold ;
+
+  # printf "$sent_num $word_num\n" ;
+  
+  foreach $i_w (0 .. $word_num-1)
+  { # loop on words
+    ($word, $pos, $head_g, $dep_g)
+      = @{$sent_gold[$i_w]}{'word', 'pos', 'head', 'dep'} ;
+
+    # printf "%d: %s %s %s %s\n", $i_w,  $word, $pos, $head_g, $dep_g ;
+
+    if ((! $score_on_punct) && is_uni_punct($word))
+    {
+      # ignore punctuations
+      next ;
+    }
+
+    ($head_s, $dep_s) = @{$sent_sys[$i_w]}{'head', 'dep'} ;
+
+    $err_head = ($head_g ne $head_s) ;
+    $err_dep = ($dep_g ne $dep_s) ;
+
+    $head_err = '-' ;
+    $dep_err = '-' ;
+
+    if ($head_g eq '0')
+    {
+      $head_aft_bef_g = '0' ;
+    }
+    elsif ($head_g eq $i_w+1)
+    {
+      $head_aft_bef_g = 'e' ;
+    }
+    else
+    {
+      $head_aft_bef_g = ($head_g <= $i_w+1 ? 'b' : 'a') ;
+    }
+
+    if ($head_s eq '0')
+    {
+      $head_aft_bef_s = '0' ;
+    }
+    elsif ($head_s eq $i_w+1)
+    {
+      $head_aft_bef_s = 'e' ;
+    }
+    else
+    {
+      $head_aft_bef_s = ($head_s <= $i_w+1 ? 'b' : 'a') ;
+    }
+
+    $head_aft_bef = $head_aft_bef_g.$head_aft_bef_s ;
+
+    if ($err_head)
+    {
+      if ($head_aft_bef_s eq '0')
+      {
+	$head_err = 0 ;
+      }
+      else
+      {
+	$head_err = $head_s-$head_g ;
+      }
+    }
+
+    if ($err_dep)
+    {
+      $dep_err = $dep_g.'->'.$dep_s ;
+    }
+
+    if (! ($err_head || $err_dep))
+    {
+      next ;
+    }
+
+    # handle only the most frequent errors
+
+    $err = $head_err.$sep.$head_aft_bef.$sep.$dep_err ;
+
+    if (! exists $freq_err{$err})
+    {
+      next ;
+    }
+
+    ($w_2, $w_1, $w1, $w2, $p_2, $p_1, $p1, $p2) = get_context(\@sent_gold, $i_w) ;
+
+    $con_bef = $w_1 ;
+    $con_bef_2 = $w_2.' + '.$w_1 ;
+    $con_aft = $w1 ;
+    $con_aft_2 = $w1.' + '.$w2 ;
+
+    $con_pos_bef = $p_1 ;
+    $con_pos_bef_2 = $p_2.'+'.$p_1 ;
+    $con_pos_aft = $p1 ;
+    $con_pos_aft_2 = $p1.'+'.$p2 ;
+
+    @cur_err = ($con_pos_bef, $con_bef, $word, $pos, $con_pos_aft, $con_aft) ;
+
+    # printf "# %-25s %-15s %-10s %-25s %-3s %-30s\n",
+    #  $con_bef, $word, $pos, $con_aft, $head_err, $dep_err ;
+    
+    @bits = (0, 0, 0, 0, 0, 0) ;
+    $j = 0 ;
+
+    while ($j == 0)
+    {
+      for ($i = 0; $i <= $#bits; $i++)
+      {
+	if ($bits[$i] == 0)
+	{
+	  $bits[$i] = 1 ;
+	  $j = 0 ;
+	  last ;
+	}
+	else
+	{
+	  $bits[$i] = 0 ;
+	  $j = 1 ;
+	}
+      }
+
+      @e_bits = @cur_err ;
+
+      for ($i = 0; $i <= $#bits; $i++)
+      {
+	if (! $bits[$i])
+	{
+	  $e_bits[$i] = '*' ;
+	}
+      }
+
+      # include also the last case which is the most general
+      # (wildcards for everything)
+      $err_counts{$err}{join($sep, @e_bits)}++ ;
+
+    }
+
+  } # loop on words
+} # second reading loop
+
+printf OUT "\n\n" ;
+printf OUT "  Specific errors, %d most frequent errors:", $freq_err_num ;
+printf OUT "\n  %s\n", '=' x 41 ;
+
+
+# deleting local contexts which are too general
+
+foreach $err (keys %err_counts)
+{
+  foreach $loc_con (sort {$err_counts{$err}{$b} <=> $err_counts{$err}{$a}}
+		    keys %{$err_counts{$err}})
+  {
+    @cur_err = split(/\Q$sep\E/, $loc_con) ;
+
+    # In this loop, one or two elements of the local context are
+    # replaced with '*' to make it more general. If the entry for
+    # the general context has the same count it is removed.
+
+    foreach $i (0 .. $#cur_err)
+    {
+      $w1 = $cur_err[$i] ;
+      if ($cur_err[$i] eq '*')
+      {
+	next ;
+      }
+      $cur_err[$i] = '*' ;
+      $con1 = join($sep, @cur_err) ;
+      if ( defined($err_counts{$err}{$con1}) && defined($err_counts{$err}{$loc_con})
+	   && ($err_counts{$err}{$con1} == $err_counts{$err}{$loc_con}))
+      {
+	delete $err_counts{$err}{$con1} ;
+      }
+      for ($j = $i+1; $j <=$#cur_err; $j++)
+      {
+	if ($cur_err[$j] eq '*')
+	{
+	  next ;
+	}
+	$w2 = $cur_err[$j] ;
+	$cur_err[$j] = '*' ;
+	$con1 = join($sep, @cur_err) ;
+	if ( defined($err_counts{$err}{$con1}) && defined($err_counts{$err}{$loc_con})
+	     && ($err_counts{$err}{$con1} == $err_counts{$err}{$loc_con}))
+	{
+	  delete $err_counts{$err}{$con1} ;
+	}
+	$cur_err[$j] = $w2 ;
+      }
+      $cur_err[$i] = $w1 ;
+    }
+  }
+}
+
+# Leaving only the topmost local contexts for each error
+
+foreach $err (keys %err_counts)
+{
+  $thresh = (sort {$b <=> $a} values %{$err_counts{$err}})[$spec_err_loc_con-1] || 0 ;
+
+  # of the threshold is too low, take the 2nd highest count
+  # (the highest may be the total which is the generic case
+  #   and not relevant for printing)
+
+  if ($thresh < 5)
+  {
+    $thresh = (sort {$b <=> $a} values %{$err_counts{$err}})[1] ;
+  }
+
+  foreach $loc_con (keys %{$err_counts{$err}})
+  {
+    if ($err_counts{$err}{$loc_con} < $thresh)
+    {
+      delete $err_counts{$err}{$loc_con} ;
+    }
+    else
+    {
+      if ($loc_con ne join($sep, ('*', '*', '*', '*', '*', '*')))
+      {
+	$loc_con_err_counts{$loc_con}{$err} = $err_counts{$err}{$loc_con} ;
+      }
+    }
+  }
+}
+
+# printing an error summary
+
+# calculating the context field length
+
+$max_word_spec_len= length('word') ;
+$max_con_aft_len = length('word') ;
+$max_con_bef_len = length('word') ;
+$max_con_pos_len = length('CPOS') ;
+
+foreach $err (keys %err_counts)
+{
+  foreach $loc_con (sort keys %{$err_counts{$err}})
+  {
+    ($con_pos_bef, $con_bef, $word, $pos, $con_pos_aft, $con_aft) =
+      split(/\Q$sep\E/, $loc_con) ;
+
+    $l = uni_len($word) ;
+    if ($l > $max_word_spec_len)
+    {
+      $max_word_spec_len = $l ;
+    }
+
+    $l = uni_len($con_bef) ;
+    if ($l > $max_con_bef_len)
+    {
+      $max_con_bef_len = $l ;
+    }
+
+    $l = uni_len($con_aft) ;
+    if ($l > $max_con_aft_len)
+    {
+      $max_con_aft_len = $l ;
+    }
+
+    if (length($con_pos_aft) > $max_con_pos_len)
+    {
+      $max_con_pos_len = length($con_pos_aft) ;
+    }
+
+    if (length($con_pos_bef) > $max_con_pos_len)
+    {
+      $max_con_pos_len = length($con_pos_bef) ;
+    }
+  }
+}
+
+$err_counter = 0 ;
+
+foreach $err (sort {$freq_err{$b} <=> $freq_err{$a}} keys %freq_err)
+{
+
+  ($head_err, $head_aft_bef, $dep_err) = split(/\Q$sep\E/, $err) ;
+
+  $err_counter++ ;
+  $err_desc{$err} = sprintf("%2d. ", $err_counter).
+    describe_err($head_err, $head_aft_bef, $dep_err) ;
+  
+  # printf OUT "  %-3s %-30s %d\n", $head_err, $dep_err, $freq_err{$err} ;
+  printf OUT "\n" ;
+  printf OUT "  %s : %d times\n", $err_desc{$err}, $freq_err{$err} ;
+
+  printf OUT "  %s-+-%s-+-%s-+-%s-+-%s-+-%s-+------\n",
+    '-' x $max_con_pos_len, '-' x $max_con_bef_len,
+       '-' x $max_pos_len, '-' x $max_word_spec_len,
+	'-' x $max_con_pos_len, '-' x $max_con_aft_len ;
+
+  printf OUT "  %-*s | %-*s | %-*s | %s\n",
+      $max_con_pos_len+$max_con_bef_len+3, '  Before',
+	$max_word_spec_len+$max_pos_len+3, '   Focus',
+	  $max_con_pos_len+$max_con_aft_len+3, '  After',
+	    'Count' ;
+
+  printf OUT "  %-*s   %-*s | %-*s   %-*s | %-*s   %-*s |\n",
+    $max_con_pos_len, 'CPOS', $max_con_bef_len, 'word',
+       $max_pos_len, 'CPOS', $max_word_spec_len, 'word',
+	$max_con_pos_len, 'CPOS', $max_con_aft_len, 'word' ;
+  
+  printf OUT "  %s-+-%s-+-%s-+-%s-+-%s-+-%s-+------\n",
+    '-' x $max_con_pos_len, '-' x $max_con_bef_len,
+       '-' x $max_pos_len, '-' x $max_word_spec_len,
+	'-' x $max_con_pos_len, '-' x $max_con_aft_len ;
+
+  foreach $loc_con (sort {$err_counts{$err}{$b} <=> $err_counts{$err}{$a}}
+		    keys %{$err_counts{$err}})
+  {
+    if ($loc_con eq join($sep, ('*', '*', '*', '*', '*', '*')))
+    {
+      next ;
+    }
+
+    $con1 = $loc_con ;
+    $con1 =~ s/\*/ /g ;
+
+    ($con_pos_bef, $con_bef, $word, $pos, $con_pos_aft, $con_aft) =
+      split(/\Q$sep\E/, $con1) ;
+
+    printf OUT "  %-*s | %-*s | %-*s | %-*s | %-*s | %-*s | %3d\n",
+      $max_con_pos_len, $con_pos_bef, $max_con_bef_len+length($con_bef)-uni_len($con_bef), $con_bef,
+	  $max_pos_len, $pos, $max_word_spec_len+length($word)-uni_len($word), $word,
+	    $max_con_pos_len, $con_pos_aft, $max_con_aft_len+length($con_aft)-uni_len($con_aft), $con_aft,
+	      $err_counts{$err}{$loc_con} ;
+  }
+  
+  printf OUT "  %s-+-%s-+-%s-+-%s-+-%s-+-%s-+------\n",
+    '-' x $max_con_pos_len, '-' x $max_con_bef_len,
+       '-' x $max_pos_len, '-' x $max_word_spec_len,
+	'-' x $max_con_pos_len, '-' x $max_con_aft_len ;
+
+}
+
+printf OUT "\n\n" ;
+printf OUT "  Local contexts involved in several frequent errors:" ;
+printf OUT "\n  %s\n", '=' x 51 ;
+printf OUT "\n\n" ;
+
+foreach $loc_con (sort {scalar keys %{$loc_con_err_counts{$b}} <=>
+			  scalar keys %{$loc_con_err_counts{$a}}}
+		  keys %loc_con_err_counts)
+{
+
+  if (scalar keys %{$loc_con_err_counts{$loc_con}} == 1)
+  {
+    next ;
+  }
+  
+  printf OUT "  %s-+-%s-+-%s-+-%s-+-%s-+-%s-\n",
+    '-' x $max_con_pos_len, '-' x $max_con_bef_len,
+       '-' x $max_pos_len, '-' x $max_word_spec_len,
+	'-' x $max_con_pos_len, '-' x $max_con_aft_len ;
+
+  printf OUT "  %-*s | %-*s | %-*s \n",
+      $max_con_pos_len+$max_con_bef_len+3, '  Before',
+	$max_word_spec_len+$max_pos_len+3, '   Focus',
+	  $max_con_pos_len+$max_con_aft_len+3, '  After' ;
+
+  printf OUT "  %-*s   %-*s | %-*s   %-*s | %-*s   %-*s \n",
+    $max_con_pos_len, 'CPOS', $max_con_bef_len, 'word',
+       $max_pos_len, 'CPOS', $max_word_spec_len, 'word',
+	$max_con_pos_len, 'CPOS', $max_con_aft_len, 'word' ;
+  
+  printf OUT "  %s-+-%s-+-%s-+-%s-+-%s-+-%s-\n",
+    '-' x $max_con_pos_len, '-' x $max_con_bef_len,
+       '-' x $max_pos_len, '-' x $max_word_spec_len,
+	'-' x $max_con_pos_len, '-' x $max_con_aft_len ;
+
+  $con1 = $loc_con ;
+  $con1 =~ s/\*/ /g ;
+
+  ($con_pos_bef, $con_bef, $word, $pos, $con_pos_aft, $con_aft) =
+      split(/\Q$sep\E/, $con1) ;
+
+  printf OUT "  %-*s | %-*s | %-*s | %-*s | %-*s | %-*s \n",
+    $max_con_pos_len, $con_pos_bef, $max_con_bef_len+length($con_bef)-uni_len($con_bef), $con_bef,
+      $max_pos_len, $pos, $max_word_spec_len+length($word)-uni_len($word), $word,
+	$max_con_pos_len, $con_pos_aft, $max_con_aft_len+length($con_aft)-uni_len($con_aft), $con_aft ;
+	  
+  printf OUT "  %s-+-%s-+-%s-+-%s-+-%s-+-%s-\n",
+    '-' x $max_con_pos_len, '-' x $max_con_bef_len,
+       '-' x $max_pos_len, '-' x $max_word_spec_len,
+	'-' x $max_con_pos_len, '-' x $max_con_aft_len ;
+
+  foreach $err (sort {$loc_con_err_counts{$loc_con}{$b} <=>
+			$loc_con_err_counts{$loc_con}{$a}}
+		keys %{$loc_con_err_counts{$loc_con}})
+  {
+    printf OUT "  %s : %d times\n", $err_desc{$err},
+      $loc_con_err_counts{$loc_con}{$err} ;
+  }
+
+  printf OUT "\n" ;
+}
+
+close GOLD ;
+close SYS ;
+
+close OUT ;

examples/test_original_dcst.sh

@@ -0,0 +1,110 @@
+
+domain="san"
+word_path="../Documents/DCST/data/multilingual_word_embeddings/cc.sanskrit.300.new.vec"
+char_path="../Documents/DCST/data/multilingual_word_embeddings/hellwig_char_embedding.128"
+pos_path='../Documents/DCST/data/multilingual_word_embeddings/pos_embedding_FT.100'
+declare -i num_epochs=1
+declare -i word_dim=300
+declare -i set_num_training_samples=500
+start_time=`date +%s`
+current_time=$(date "+%Y.%m.%d-%H.%M.%S")
+# model_path="ud_parser_san_"$current_time 
+
+# ######## Changes ###########
+# ## --use_pos is removed from all the models
+# ## singletions are excluded.
+# ##  auxiliary task epoch change to 20
+
+touch saved_models/log.txt
+################################################################ 
+echo "#################################################################"
+echo "Currently base model in progress..."
+echo "#################################################################"
+python examples/GraphParser.py --dataset ud --domain $domain --rnn_mode LSTM \
+--num_epochs $num_epochs --batch_size 16 --hidden_size 512 --arc_space 512 \
+--arc_tag_space 128 --num_layers 3 --num_filters 100 --use_char  \
+--word_dim $word_dim --pos_dim 100 --initializer xavier --opt adam \
+--learning_rate 0.002 --decay_rate 0.5 --schedule 6 --clip 5.0 --gamma 0.0 \
+--epsilon 1e-6 --p_rnn 0.33 0.33 --p_in 0.33 --p_out 0.33 --arc_decode mst \
+--punct_set '.' '``'  ':' ','  --word_embedding fasttext  \
+--set_num_training_samples $set_num_training_samples \
+--word_path $word_path --char_path $char_path --pos_path $pos_path  --pos_embedding random --char_embedding random --char_dim 100 \
+--model_path saved_models/$model_path 2>&1 | tee saved_models/log.txt
+mv saved_models/log.txt saved_models/$model_path/log.txt
+
+
+## model_path="ud_parser_san_2020.03.03-12.24.41"
+
+####### self-training ###########
+## 'number_of_children' 'relative_pos_based' 'distance_from_the_root'
+
+####### Multitask setup #########
+## 'Multitask_label_predict' 'Multitask_case_predict' 'Multitask_POS_predict' 'Multitask_coarse_predict' 
+## 'predict_ma_tag_of_modifier' 'add_label' 'predict_case_of_modifier' 
+
+####### Other Tasks #############
+## #   'add_head_ma' 'add_head_coarse_pos' 'predict_coarse_of_modifier' 
+
+for task in 'Multitask_label_predict' 'Multitask_case_predict' 'Multitask_POS_predict' 'number_of_children' 'relative_pos_based' 'distance_from_the_root' ; do
+	echo "#################################################################"
+	echo "Currently $task in progress..."
+	echo "#################################################################"  
+	touch saved_models/log.txt
+    python examples/SequenceTagger.py --dataset ud --domain $domain --task $task \
+    --rnn_mode LSTM --num_epochs $num_epochs --batch_size 16 --hidden_size 512 \
+	--tag_space 128 --num_layers 3 --num_filters 100 --use_char  \
+	--pos_dim 100 --initializer xavier --opt adam --learning_rate 0.002 --decay_rate 0.5 \
+	--schedule 6 --clip 5.0 --gamma 0.0 --epsilon 1e-6 --p_rnn 0.33 0.33 \
+	--p_in 0.33 --p_out 0.33 --punct_set '.' '``'  ':' ','  \
+	--word_dim $word_dim --word_embedding fasttext --word_path $word_path \
+	--parser_path saved_models/$model_path/ \
+	--use_unlabeled_data --char_path $char_path --pos_path $pos_path --pos_embedding random --char_embedding random --char_dim 100  \
+	--model_path saved_models/$model_path/$task/ 2>&1 | tee saved_models/log.txt
+	mv saved_models/log.txt saved_models/$model_path/$task/log.txt
+done
+
+echo "#################################################################"
+echo "Currently final model in progress..."
+echo "#################################################################"
+touch saved_models/log.txt
+python examples/GraphParser.py --dataset ud --domain $domain --rnn_mode LSTM \
+--num_epochs $num_epochs --batch_size 16 --hidden_size 512 \
+--arc_space 512 --arc_tag_space 128 --num_layers 3 --num_filters 100 --use_char  \
+--word_dim $word_dim --pos_dim 100 --initializer xavier --opt adam \
+--learning_rate 0.002 --decay_rate 0.5 --schedule 6 --clip 5.0 --gamma 0.0 --epsilon 1e-6 \
+--p_rnn 0.33 0.33 --p_in 0.33 --p_out 0.33 --arc_decode mst \
+--punct_set '.' '``'  ':' ','  --word_embedding fasttext \
+--word_path $word_path --gating --num_gates 4 \
+--char_path $char_path --pos_path $pos_path  --pos_embedding random --char_embedding random --char_dim 100 \
+--load_sequence_taggers_paths saved_models/$model_path/Multitask_case_predict/domain_$domain.pt \
+saved_models/$model_path/Multitask_POS_predict/domain_$domain.pt \
+saved_models/$model_path/Multitask_label_predict/domain_$domain.pt \
+--model_path saved_models/$model_path/final_ensembled_multi/ 2>&1 | tee saved_models/log.txt
+mv saved_models/log.txt saved_models/$model_path/final_ensembled_multi/log.txt
+end_time=`date +%s`
+echo execution time was `expr $end_time - $start_time` s.
+
+
+echo "#################################################################"
+echo "Currently final model in progress..."
+echo "#################################################################"
+touch saved_models/log.txt
+python examples/GraphParser.py --dataset ud --domain $domain --rnn_mode LSTM \
+--num_epochs $num_epochs --batch_size 16 --hidden_size 512 \
+--arc_space 512 --arc_tag_space 128 --num_layers 3 --num_filters 100 --use_char  \
+--word_dim $word_dim --pos_dim 100 --initializer xavier --opt adam \
+--learning_rate 0.002 --decay_rate 0.5 --schedule 6 --clip 5.0 --gamma 0.0 --epsilon 1e-6 \
+--p_rnn 0.33 0.33 --p_in 0.33 --p_out 0.33 --arc_decode mst \
+--punct_set '.' '``'  ':' ','  --word_embedding fasttext \
+--word_path $word_path --gating --num_gates 7 \
+--char_path $char_path --pos_path $pos_path  --pos_embedding random --char_embedding random --char_dim 100 \
+--load_sequence_taggers_paths saved_models/$model_path/Multitask_case_predict/domain_$domain.pt \
+saved_models/$model_path/Multitask_POS_predict/domain_$domain.pt \
+saved_models/$model_path/Multitask_label_predict/domain_$domain.pt \
+saved_models/$model_path/number_of_children/domain_$domain.pt \
+saved_models/$model_path/relative_pos_based/domain_$domain.pt \
+saved_models/$model_path/distance_from_the_root/domain_$domain.pt \
+--model_path saved_models/$model_path/final_ensembled_multi_self/ 2>&1 | tee saved_models/log.txt
+mv saved_models/log.txt saved_models/$model_path/final_ensembled_multi_self/log.txt
+end_time=`date +%s`
+echo execution time was `expr $end_time - $start_time` s.

run_san_LCM.sh

@@ -0,0 +1,73 @@
+
+domain="san"
+
+word_path="./data/multilingual_word_embeddings/cc.sanskrit.300.new.vec" 
+declare -i num_epochs=100
+declare -i word_dim=300
+declare -i set_num_training_samples=500
+start_time=`date +%s`
+current_time=$(date "+%Y.%m.%d-%H.%M.%S")
+model_path="ud_parser_san_"$current_time 
+
+
+echo "#################################################################"
+echo "Currently BiAFFINE model training in progress..."
+echo "#################################################################"
+python examples/GraphParser.py --dataset ud --domain $domain --rnn_mode LSTM \
+--num_epochs $num_epochs --batch_size 16 --hidden_size 512 --arc_space 512 \
+--arc_tag_space 128 --num_layers 2 --num_filters 100 --use_char \
+--set_num_training_samples $set_num_training_samples \
+--word_dim $word_dim --char_dim 100 --pos_dim 100 --initializer xavier --opt adam \
+--learning_rate 0.002 --decay_rate 0.5 --schedule 6 --clip 5.0 --gamma 0.0 \
+--epsilon 1e-6 --p_rnn 0.33 0.33 --p_in 0.33 --p_out 0.33 --arc_decode mst \
+--punct_set '.' '``'  ':' ','  --word_embedding fasttext --char_embedding random  --pos_embedding random --word_path $word_path \
+--model_path saved_models/$model_path 2>&1 | tee saved_models/base_log.txt
+
+mv saved_models/base_log.txt saved_models/$model_path/base_log.txt
+
+##################################################################
+## Pretrainig Step
+## For DCST setting set tasks as : 'number_of_children' 'relative_pos_based' 'distance_from_the_root'
+## For LCM set tasks as: 'Multitask_case_predict' 'Multitask_POS_predict' 'add_label'
+for task in 'Multitask_case_predict' 'Multitask_POS_predict' 'add_label'; do
+	touch saved_models/$model_path/log.txt
+	echo "#################################################################"
+	echo "Currently $task in progress..."
+	echo "#################################################################"  
+    python examples/SequenceTagger.py --dataset ud --domain $domain --task $task \
+    --rnn_mode LSTM --num_epochs $num_epochs --batch_size 16 --hidden_size 512 \
+	--tag_space 128 --num_layers 2 --num_filters 100 --use_char --char_dim 100 \
+	--pos_dim 100 --initializer xavier --opt adam --learning_rate 0.002 --decay_rate 0.5 \
+	--schedule 6 --clip 5.0 --gamma 0.0 --epsilon 1e-6 --p_rnn 0.33 0.33 \
+	--p_in 0.33 --p_out 0.33 --punct_set '.' '``'  ':' ','  \
+	--word_dim $word_dim --word_embedding fasttext --word_path $word_path --pos_embedding random \
+	--parser_path saved_models/$model_path/ \
+	--use_unlabeled_data --use_labeled_data --char_embedding random \
+	--model_path saved_models/$model_path/$task/ 2>&1 | tee saved_models/$model_path/log.txt
+	mv saved_models/$model_path/log.txt saved_models/$model_path/$task/log.txt
+done
+
+# ########################################################################
+
+echo "#################################################################"
+echo "Final Parsing Model Integrated with Pretrained Encoders of Auxiliary tasks..."
+echo "#################################################################"
+touch saved_models/$model_path/log.txt
+python examples/GraphParser.py --dataset ud --domain $domain --rnn_mode LSTM \
+--num_epochs $num_epochs --batch_size 16 --hidden_size 512 \
+--arc_space 512 --arc_tag_space 128 --num_layers 2 --num_filters 100 --use_char  \
+--word_dim $word_dim --char_dim 100 --pos_dim 100 --initializer xavier --opt adam \
+--learning_rate 0.002 --decay_rate 0.5 --schedule 6 --clip 5.0 --gamma 0.0 --epsilon 1e-6 \
+--p_rnn 0.33 0.33 --p_in 0.33 --p_out 0.33 --arc_decode mst --pos_embedding random \
+--punct_set '.' '``'  ':' ','  --word_embedding fasttext --char_embedding random --word_path $word_path  \
+--gating --num_gates 4 \
+--set_num_training_samples $set_num_training_samples \
+--load_sequence_taggers_paths saved_models/$model_path/add_label/domain_$domain.pt \
+saved_models/$model_path/Multitask_case_predict/domain_$domain.pt \
+saved_models/$model_path/Multitask_POS_predict/domain_$domain.pt \
+--model_path saved_models/$model_path/final_ensembled_BiAFF_LCM 2>&1 | tee saved_models/$model_path/log.txt
+mv saved_models/$model_path/log.txt saved_models/$model_path/final_ensembled_BiAFF_LCM/log.txt
+
+end_time=`date +%s`
+echo execution time was `expr $end_time - $start_time` s.
+

utils/__init__.py

@@ -0,0 +1,7 @@
+from . import io_
+from . import nn
+from . import load_word_embeddings
+from . import nlinalg
+from . import models
+
+__version__ = "0.1.dev1"

utils/io_/__init__.py

@@ -0,0 +1,5 @@
+from .alphabet import *
+from .instance import *
+from .logger import *
+from .writer import *
+from . import prepare_data

utils/io_/alphabet.py

@@ -0,0 +1,147 @@
+"""
+Alphabet maps objects to integer ids. It provides two way mapping from the index to the objects.
+"""
+import json
+import os
+from .logger import get_logger
+
+class Alphabet(object):
+    def __init__(self, name, defualt_value=False, keep_growing=True, singleton=False):
+        self.__name = name
+
+        self.instance2index = {}
+        self.instances = []
+        self.default_value = defualt_value
+        self.offset = 1 if self.default_value else 0
+        self.keep_growing = keep_growing
+        self.singletons = set() if singleton else None
+
+        # Index 0 is occupied by default, all else following.
+        self.default_index = 0 if self.default_value else None
+
+        self.next_index = self.offset
+
+        self.logger = get_logger("Alphabet")
+
+    def add(self, instance):
+        if instance not in self.instance2index and instance != '<_UNK>':
+            self.instances.append(instance)
+            self.instance2index[instance] = self.next_index
+            self.next_index += 1
+
+    def add_singleton(self, id):
+        if self.singletons is None:
+            raise RuntimeError("Alphabet %s does not have singleton." % self.__name)
+        else:
+            self.singletons.add(id)
+
+    def add_singletons(self, ids):
+        if self.singletons is None:
+            raise RuntimeError("Alphabet %s does not have singleton." % self.__name)
+        else:
+            self.singletons.update(ids)
+
+    def is_singleton(self, id):
+        if self.singletons is None:
+            raise RuntimeError("Alphabet %s does not have singleton." % self.__name)
+        else:
+            return id in self.singletons
+
+    def get_index(self, instance):
+        try:
+            return self.instance2index[instance]
+        except KeyError:
+            if self.keep_growing:
+                index = self.next_index
+                self.add(instance)
+                return index
+            else:
+                if self.default_value:
+                    return self.default_index
+                else:
+                    raise KeyError("instance not found: %s" % instance)
+
+    def get_instance(self, index):
+        if self.default_value and index == self.default_index:
+            # First index is occupied by the wildcard element.
+            return "<_UNK>"
+        else:
+            try:
+                return self.instances[index - self.offset]
+            except IndexError:
+                raise IndexError("unknown index: %d" % index)
+
+    def size(self):
+        return len(self.instances) + self.offset
+
+    def singleton_size(self):
+        return len(self.singletons)
+
+    def items(self):
+        return self.instance2index.items()
+
+    def keys(self):
+        return self.instance2index.keys()
+
+    def values(self):
+        return self.instance2index.values()
+
+    def token_in_alphabet(self, token):
+        return token in set(self.instance2index.keys())
+
+    def enumerate_items(self, start):
+        if start < self.offset or start >= self.size():
+            raise IndexError("Enumerate is allowed between [%d : size of the alphabet)" % self.offset)
+        return zip(range(start, len(self.instances) + self.offset), self.instances[start - self.offset:])
+
+    def close(self):
+        self.keep_growing = False
+
+    def open(self):
+        self.keep_growing = True
+
+    def get_content(self):
+        if self.singletons is None:
+            return {"instance2index": self.instance2index, "instances": self.instances}
+        else:
+            return {"instance2index": self.instance2index, "instances": self.instances,
+                    "singletions": list(self.singletons)}
+
+    def __from_json(self, data):
+        self.instances = data["instances"]
+        self.instance2index = data["instance2index"]
+        if "singletions" in data:
+            self.singletons = set(data["singletions"])
+        else:
+            self.singletons = None
+
+    def save(self, output_directory, name=None):
+        """
+        Save both alhpabet records to the given directory.
+        :param output_directory: Directory to save model and weights.
+        :param name: The alphabet saving name, optional.
+        :return:
+        """
+        saving_name = name if name else self.__name
+        try:
+            if not os.path.exists(output_directory):
+                os.makedirs(output_directory)
+
+            json.dump(self.get_content(),
+                      open(os.path.join(output_directory, saving_name + ".json"), 'w'), indent=4)
+        except Exception as e:
+            self.logger.warn("Alphabet is not saved: %s" % repr(e))
+
+    def load(self, input_directory, name=None):
+        """
+        Load model architecture and weights from the give directory. This allow we use old saved_models even the structure
+        changes.
+        :param input_directory: Directory to save model and weights
+        :return:
+        """
+        loading_name = name if name else self.__name
+        filename = os.path.join(input_directory, loading_name + ".json")
+        f = json.load(open(filename))
+        self.__from_json(f)
+        self.next_index = len(self.instances) + self.offset
+        self.keep_growing = False

utils/io_/coarse_to_ma_dict.json

@@ -0,0 +1 @@
+{"Noun": ["acc. pl. m.", "acc. pl. *", "inst. sg. m.", "abl. du. f.", "nom. sg. *.", "loc. pl. n.", "loc. du. f.", "nom. sg. m", "g. sg. *.", "i. sg. m", "g. pl. *", "nom. pl. n.", "g. du. f.", "acc. du. f.", "loc. du. m.", "nom. sg. *", "nom. sg. m.", "loc. pl. m.", "inst. pl. n.", "g. du. n.", "nom. du. f.", "inst. pl. f.", "acc. sg. n", "abl.sg.m", "loc. pl. *", "abl. pl. n.", "i. pl. n.", "dat. du. m.", "abl. pl. m.", "inst. sg. n.", "acc. pl. n.", "nom. du. *.", "inst. du. n.", "nom. pl. *", "loc. du. *.", "abl. du. n.", "nom. pl. *.", "i. sg. m.", "dat. sg. f.", "inst. sg. *.", "abl. sg. m.", "acc. sg. *", "dat. sg. *", "dat. pl. *", "dat. du. n.", "i. pl. m.", "inst. du. m.", "nom. sg. f.", "g. sg. *", "acc. sg. *.", "i. sg. n.", "i. pl. *", "loc. du. n.", "abl. sg. *", "loc. sg. m.", "nom.sg.m", "i. sg. n..", "abl. sg. f.", "nom. sg. sg.", "dat. du. f.", "nom. sg. n.", "loc. sg. f.", "acc. du. *", "acc. pl. *.", "dat. sg. m.", "nom. sg. n", "nom. pl. f.", "i. pl. f.", "acc. du. m.", "inst. sg. f.", "g. sg. n.", "abl. sg. n", "inst. pl. m.", "loc. pl. f.", "acc. sg. f.", "g. sg. f.", "voc. du. m.", "abl. sg. n.", "voc. sg. *", "dat. sg. n.", "i. sg. *", "acc. du. n.", "g. sg. m.", "loc. sg. n", "dat. pl. m.", "nom. du. *", "nom. du. n.", "acc. sg. m.", "nom. pl. m.", "acc. pl. f.", "acc. sg. f", "g. pl. m.", "voc. du. f.", "g. pl. *.", "voc. du. n.", "dat. pl. f.", "nom. du. m.", "abl. pl. f.", "dat. pl. n.", "dat. sg. m", "loc. sg. *.", "g. pl. n.", "loc. sg. n.", "abl. sg. m", "g. du. m.", "loc. sg. *", "voc. sg. m.", "voc. pl. m.", "voc. sg. n.", "acc. sg. n.", "g. pl. f.", "i. sg. f.", "abl. du. m.", "voc. sg. f.", "voc. pl. f."], "FV": ["impft. [2] ac. du. 3", "imp. [1] ac. du. 1", "imp. [3] ac. pl. 2", "ca. pr. md. sg. 1", "ca. impft. ac. sg. 1", "pp. . pl. ", "imp. [4] ac. sg. 2", "impft . ps. pl. 3", "pr. ac. sg. 1", "pp. . sg. ", "pr. [6] ac. pl. 3", "imp. [1] ac. pl. 2", "cond. ac. sg. 3", "pft. ac. sg. 2", "impft. [6] md. du. 2", "pr. [7] md. sg. 3", "pr. ps. sg. 2", "impft. [1] md. pl. 3", "ca. impft. ac. pl. 1", "pr. [10] ac. sg. 3", "pr. [5] ac. pl. 3", "imp. md. sg. 3", "fut. ac. du. 1", "imp. [10] ac. pl. 2", "impft. [1] md. sg. 3", "impft. [6] ac. sg. 2", "pr. [6] ac. sg. 1", "ca. impft. ac. pl. 3", "opt. [8] ac. sg. 3", "des. pr. md. du. 1", "ca. imp. ac. pl. 1", "pr. [1] md. pl. 3", "aor. [5] ac. sg. 2", "pfp. md. sg. 3", "imp. [2] md. sg. 1", "impft. [3] ac. sg. 3", "fut. ac. pl. 1", "pr. [vn.] ac. pl. 3", "ca. imp. ps. sg. 3", "impft. [10] ac. sg. 3", "imp. ac. pl. 3", "impft. [8] ac. pl. 3", "impft. [10] ac. sg. 2", "impft. [1] ac. pl. 3", "pr. [10] ac. du. 3", "impft. [10] ac. du. 3", "pp. . sg. 1", "pr. [4] md. sg. 3", "opt. [1] ac. pl. 2", "pfp. md. sg. 1", "pr. [2] ac. sg. 3", "opt. ac. pl. 1", "pp. ac. sg. 2", "pr. [5] ac. du. 3", "pr. md. sg. ", "pr. [8] md. sg. 3", "ca. imp. ac. sg. 2", "imp. ac. pl. 1", "imp. ps. sg. 2", "pft. md. pl. ", "ca. impft. md. sg. 3", "pr. [6] md. sg. 3", "opt. [1] ac. sg. 3", "pft. ac. sg. 1", "pft. ac. du. 3", "pr. [vn.] ac. sg. 3", "imp. ps. pl. 3", "opt. ps. sg. 3", "pr. ac. pl. 1", "pft. md. du. 3", "pr. [1] md. sg. 1", "imp. [1] ac. sg. 3", "pr. [3] ac. sg. 1", "pft. ps. sg. 3", "ca. pr. ac. du. 3", "impft. [9] ac. sg. 3", "impft . ac. sg. 1", "inj. [2] ac. sg. 3", "imp. [1] ac. sg. 2", "pr. [6] ac. du. 3", "fut. ac. du. 3", "imp. [vn.] ac. sg. 2", "imp. ac. sg. 2", "pr. ps. pl. 3", "pr. [6] ac. pl. 2", "ppr. . pl. ", "ppr. . sg. ", "pr. md. sg. 1", "imp. [9] ac. sg. 3", "impft. md. sg. ", "aor. [2] ac. sg. 3", "impft. md. sg. 2", "pr. md. sg. 3", "impft. [2] ac. sg. 1", "fut. ac. sg. 2", "opt. [2] ac. sg. 3", "imp. [8] ac. sg. 3", "imp. [10] ac. sg. 2", "pr. [4] ac. sg. 3", "pr. [8] ac. sg. 3", "fut. md. sg. 1", "opt. [8] md. sg. 3", "ca. pr. md. sg. 3", "aor. [5] ac. sg. 3", "impft. [1] md. sg. 2", "pr. [1] md. du. 3", "cond. ac. pl. 3", "imp. [4] ac. pl. 2", "impft. [1] ac. du. 1", "impft. [4] ac. pl. 3", "pr. [3] ac. pl. 3", "impft. [1] md. sg. 1", "pr. md. . 3", "imp. [1] ac. du. 3", "pr. ac. du. 3", "pr. [9] ac. sg. 2", "pr. [2] ac. du. 1", "aor. [4] ac. du. 2", "pr. [1] ac. sg. 1", "opt. [5] ac. sg. 3", "opt. [10] ac. pl. 2", "imp. [10] ac. du. 3", "aor. [1] ac. sg. 1", "pr. [10] ac. pl. 3", "opt. [2] ac. pl. 3", "opt. [1] ac. du. 3", "pr. [4] md. pl. 3", "impft. ac. sg. 2", "impft . ac. du. 3", "cond. ac. sg. 2", "pr. . sg. 3", "impft. [2] ac. sg. 3", "opt. [6] ac. sg. 3", "pr. [5] ac. sg. 3", "pft. ps. du. 3", "des. imp. md. du. 3", "impft . ac. sg. 3", "impft . ac. sg. ", "pr. md. sg. 2", "impft. [vn.] ac. sg. 3", "pr. [2] ac. sg. 2", "pr. [6] ac. sg. 3", "pr. ac. sg. 2", "pr. [7] ac. pl. 3", "imp. [3] ac. sg. 2", "opt. [2] ac. sg. 1", "pr. [1] md. pl. 2", "pft. md. pl. 3", "pr. [1] ac. sg. 3", "pr. [1] ac. pl. 1", "pr. [3] ac. du. 3", "impft. md. du. 3", "opt. [10] ac. du. 2", "imp. ac. du. 2", "opt. [1] ac. pl. 3", "impft. [vn.] md. sg. 3", "impft . ac. sg. 2", "aor. [4] ac. sg. 3", "ca. fut. ac. sg. 1", "imp. ac. sg. 3", "fut. ac. pl. 2", "impft. md. pl. 3", "ca. fut. ac. sg. 3", "opt. [2] ac. sg. 2", "impft. ac. pl. 3", "pr. [10] ac. sg. 1", "impft. md. sg. 3", "pr. [6] ac. sg. 2", "ca. pr. ps. sg. 3", "impft. ac. sg. 3", "imp. [4] ac. sg. 3", "pr. ps. sg. 1", "pr. [2] md. sg. 3", "imp. [1] ac. du. 2", "impft. [4] ac. sg. 3", "ca. opt. ac. sg. 3", "ca. pr. ac. sg. 3", "opt. ac. pl. 3", "imp. [1] md. sg. 3", "pr. [10] ac. sg. 2", "ca. pr. ps. du. 3", "opt. [5] ac. sg. 1", "impft. ps. pl. 3", "pr. [5] md. sg. 3", "pft. ps. pl. 3", "pft. ac. pl. 2", "ca. opt. ps. sg. 3", "impft. [6] ac. pl. 1", "imp. md. pl. 3", "pr. ps. du. 3", "ca. pr. ac. sg. 1", "opt. md. sg. 3", "fut. md. sg. 3", "impft . ac. pl. 3", "pr. [1] md. sg. 2", "pr. md. sg. 3", "opt. [8] ac. pl. 3", "opt. ac. du. 3", "impft. [2] ac. pl. 3", "imp. [2] ac. sg. 2", "imp. ac. pl. 2", "opt. [10] ac. sg. 3", "pft. md. sg. 1", "pr. [8] ac. sg. 1", "imp. [1] md. sg. 2", "pr. [vn.] md. sg. 3", "ca. pr. ps. sg. 1", "ca. imp. ac. sg. 3", "imp. [2] ac. du. 2", "imp. [6] ac. sg. 3", "opt. [1] ac. sg. 1", "ca. opt. ac. sg. 1", "imp. md. sg. 2", "pft. md. sg. 3", "imp. [2] ac. sg. 3", "impft. [1] ac. sg. 3", "ca. pr. ac. sg. 2", "pr. [1] md. pl. 1", "per.  fut. md. sg. 3", "impft. [2] ac. sg. 2", "opt. ac. sg. 2", "imp. [3] ac. sg. 3", "imp. [3] ac. du. 2", "imp. [vn.] ac. pl. 3", "ca. pr. ac. pl. 3", "pr. [1] ac. sg. 2", "imp. [1] ac. pl. 3", "impft. [6] md. pl. 3", "per. fut. ac. sg. 3", "pr. md. pl. 3", "impft. [4] md. pl. 3", "opt. [vn.] ac. sg. 3", "aor. [1] ac. sg. 3", "impft. [4] md. sg. 3", "impft . md. sg. 3", "pr. [1] ac. pl. 3", "pft. ac. pl. 3", "opt. ac. sg. 3", "impft. ps. sg. 3", "impft. [6] ac. sg. 3", "pr. [1] ac. du. 3", "imp. [4] md. sg. 3", "per.  fut. ac. sg. 3", "impft. [1] ac. sg. 1", "pr. [2] ac. du. 3", "pr. [2] ac. pl. 1", "impft. [1] ac. du. 3", "fut. ac. du. 2", "imp. [9] ac. sg. 2", "opt. [8] ac. sg. 2", "pr. [9] ac. sg. 3", "des. pr. md. pl. 3", "imp. [8] ac. sg. 2", "inj. [4] md. sg. 3", "fut. ac. pl. 3", "pr. [4] ac. pl. 3", "imp. md. sg. 1", "imp. [6] ac. sg. 2", "imp. ps. sg. 3", "imp. [vn.] ac. sg. 3", "pfp. . pl. ", "impft. [3] ac. pl. 3", "impft. [4] ac. sg. 1", "fut. ac. sg. 1", "per. fut. ac. sg. 2", "pr. [2] ac. pl. 3", "pr. [2] ac. sg. 1", "opt. md. . 3", "des. impft. ac. pl. 2", "opt. [3] ac. pl. 2", "pr. [5] ac. sg. 1", "pr. md. du. 3", "imp. [10] ac. sg. 3", "imp. [2] md. sg. 3", "pft. ac. sg. 3", "impft. [8] ac. sg. 3", "imp. [5] ac. sg. 2", "impft. [8] ac. du. 3", "impft . ps. sg. 3", "ca. opt. ac. sg. 2", "per.  fut. ac. pl. 3", "pr. [9] ac. sg. 1", "impft . ps. du. 3", "aor. [1] ac. pl. 3", "pr. [3] ac. sg. 3", "opt. [3] ac. sg. 1", "impft. ac. sg. ", "aor. [1] ac. du. 3", "pr. [1] md. sg. 3", "impft. [1] md. pl. 1", "des. pr. md. sg. 3", "opt. [8] ac. sg. 1", "imp. [4] ac. pl. 3", "imp. [4] ac. du. 3", "pr. ac. sg. 3", "opt. ac. du. 1", "impft. [10] ac. pl. 3", "pr. ac. pl. 3", "pfp. . sg. ", "impft . ac. pl. 2", "fut. ac. sg. 3", "fut. md. pl. 3", "impft. [5] ac. sg. 3", "impft. [1] ac. sg. 2", "pr. [8] ac. pl. 3", "pr. [8] ac. du. 3", "opt. [8] ac. pl. 1", "ca. impft. ac. sg. 3", "opt. ac. sg. 1", "impft. [5] ac. sg. 1", "pp. . du. ", "pr. md. pl. 1", "impft . md. pl. 3", "pr. ps. sg. 3", "des. impft. md. pl. 1", "opt. [10] ac. sg. 2", "impft. [8] md. sg. 3", "pr. [6] md. pl. 3", "imp. [1] md. pl. 2", "opt. [1] md. sg. 3", "opt. [3] md. sg. 3", "ca. fut. md. sg. 3", "pr. [9] ac. pl. 3"], "IV": ["pp.", "inf.", "ca. ppr. ps.", "pp. . . ", "des. ppr. ac.", "ca. pfp. [1]", "ppr. ps.", "ppr.", "abs.", "ppr. [6] ac.", "ppa.", "pfp. [3]", "pfp. [1]", "ppr. [1] ac.", "impft . ac. . ", "ppr. [1] md.", "ca. inf.", "part.", "ca. pfp. [3]", "pfp.", "ppr. [8] ac.", "ca. abs.", "ca. pp.", "ca. ppa.", "ca. ppr. ac.", "ppr. [10] ac.", "ppr. [2] ac.", "pfu. ac.", "tasil", "pfp. [2]", "ca. pfp. [2]", "ppr. [6] ac. ", "pp. [2]"], "Ind": ["prep.", "ind.", "conj."], "adv": ["adv."]}

utils/io_/convert_ud_to_onto_format.py

@@ -0,0 +1,74 @@
+import argparse
+import os
+# /home/jivnesh/Documents/DCST/d
+# 'ar','eu','cs','de','hu','lv','pl','sv','fi','ru'
+# python utils/io_/convert_ud_to_onto_format.py --ud_data_path d
+def write_ud_files(args):
+    languages_for_low_resource = ['el']
+
+    languages = sorted(list(set(languages_for_low_resource)))
+    splits = ['train', 'dev', 'test']
+    lng_to_files = dict((language, {}) for language in languages)
+    for language, d in lng_to_files.items():
+        for split in splits:
+            d[split] = []
+        lng_to_files[language] = d
+    sub_folders = os.listdir(args.ud_data_path)
+    for sub_folder in sub_folders:
+        folder = os.path.join(args.ud_data_path, sub_folder)
+        files = os.listdir(folder)
+        for file in files:
+            for language in languages:
+                if file.startswith(language) and file.endswith('conllu'):
+                    for split in splits:
+                        if split in file:
+                            full_path = os.path.join(folder, file)
+                            lng_to_files[language][split].append(full_path)
+                            break
+
+    for language, split_dict in lng_to_files.items():
+        for split, files in split_dict.items():
+            if split == 'dev' and len(files) == 0:
+                files = split_dict['train']
+                print('No dev files were found, copying train files instead')
+            sentences = []
+            num_sentences = 0
+            for file in files:
+                with open(file, 'r') as file:
+                    for line in file:
+                        new_line = []
+                        line = line.strip()
+                        if len(line) == 0:
+                            sentences.append(new_line)
+                            num_sentences += 1
+                            continue
+                        tokens = line.split('\t')
+                        if not tokens[0].isdigit():
+                            continue
+                        id = tokens[0]
+                        word = tokens[1]
+                        pos = tokens[3]
+                        ner = tokens[5]
+                        head = tokens[6]
+                        arc_tag = tokens[7]
+                        new_line = [id, word, pos, ner, head, arc_tag]
+                        sentences.append(new_line)
+            print('Language: %s Split: %s Num. Sentences: %s ' % (language, split, num_sentences))
+            if not os.path.exists('data'):
+                os.makedirs('data')
+            write_data_path = 'data/MRL/ud_pos_ner_dp_' + split + '_' + language
+            print('creating %s' % write_data_path)
+            with open(write_data_path, 'w') as f:
+                for line in sentences:
+                    f.write('\t'.join(line) + '\n')
+
+def main():
+    # Parse arguments
+    args_ = argparse.ArgumentParser()
+    args_.add_argument('--ud_data_path', help='Directory path of the UD treebanks.', required=True)
+
+    args = args_.parse_args()
+    write_ud_files(args)
+
+if __name__ == "__main__":
+    main()

utils/io_/instance.py

@@ -0,0 +1,19 @@
+class Sentence(object):
+    def __init__(self, words, word_ids, char_seqs, char_id_seqs):
+        self.words = words
+        self.word_ids = word_ids
+        self.char_seqs = char_seqs
+        self.char_id_seqs = char_id_seqs
+
+    def length(self):
+        return len(self.words)
+
+class NER_DependencyInstance(object):
+    def __init__(self, sentence, tokens_dict, ids_dict, heads):
+        self.sentence = sentence
+        self.tokens = tokens_dict
+        self.ids = ids_dict
+        self.heads = heads
+
+    def length(self):
+        return self.sentence.length()

utils/io_/logger.py

@@ -0,0 +1,15 @@
+import logging
+import sys
+
+
+def get_logger(name, level=logging.INFO, handler=sys.stdout,
+               formatter='%(asctime)s - %(name)s - %(levelname)s - %(message)s'):
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.INFO)
+    formatter = logging.Formatter(formatter)
+    stream_handler = logging.StreamHandler(handler)
+    stream_handler.setLevel(level)
+    stream_handler.setFormatter(formatter)
+    logger.addHandler(stream_handler)
+
+    return logger

utils/io_/prepare_data.py

@@ -0,0 +1,397 @@
+import os.path
+import numpy as np
+from .alphabet import Alphabet
+from .logger import get_logger
+import torch
+
+# Special vocabulary symbols - we always put them at the end.
+PAD = "_<PAD>_"
+ROOT = "_<ROOT>_"
+END = "_<END>_"
+_START_VOCAB = [PAD, ROOT, END]
+
+MAX_CHAR_LENGTH = 45
+NUM_CHAR_PAD = 2
+
+UNK_ID = 0
+PAD_ID_WORD = 1
+PAD_ID_CHAR = 1
+PAD_ID_TAG = 0
+
+NUM_SYMBOLIC_TAGS = 3
+
+_buckets = [10, 15, 20, 25, 30, 35, 40, 50, 60, 70, 80, 90, 100, 140]
+
+from .reader import Reader
+
+def create_alphabets(alphabet_directory, train_paths, extra_paths=None, max_vocabulary_size=100000, embedd_dict=None,
+                     min_occurence=1, lower_case=False):
+    def expand_vocab(vocab_list, char_alphabet, pos_alphabet, ner_alphabet, arc_alphabet):
+        vocab_set = set(vocab_list)
+        for data_path in extra_paths:
+            with open(data_path, 'r') as file:
+                for line in file:
+                    line = line.strip()
+                    if len(line) == 0:
+                        continue
+
+                    tokens = line.split('\t')
+                    if lower_case:
+                        tokens[1] = tokens[1].lower()
+                    for char in tokens[1]:
+                        char_alphabet.add(char)
+
+                    word = tokens[1]
+                    pos = tokens[2]
+                    ner = tokens[3]
+                    arc_tag = tokens[5]
+
+                    pos_alphabet.add(pos)
+                    ner_alphabet.add(ner)
+                    arc_alphabet.add(arc_tag)
+                    if embedd_dict is not None:
+                        if word not in vocab_set and (word in embedd_dict or word.lower() in embedd_dict):
+                            vocab_set.add(word)
+                            vocab_list.append(word)
+                    else:
+                        if word not in vocab_set:
+                            vocab_set.add(word)
+                            vocab_list.append(word)
+        return vocab_list, char_alphabet, pos_alphabet, ner_alphabet, arc_alphabet
+
+    logger = get_logger("Create Alphabets")
+    word_alphabet = Alphabet('word', defualt_value=True, singleton=True)
+    char_alphabet = Alphabet('character', defualt_value=True)
+    pos_alphabet = Alphabet('pos', defualt_value=True)
+    ner_alphabet = Alphabet('ner', defualt_value=True)
+    arc_alphabet = Alphabet('arc', defualt_value=True)
+    auto_label_alphabet = Alphabet('auto_labeler', defualt_value=True)
+    if not os.path.isdir(alphabet_directory):
+        logger.info("Creating Alphabets: %s" % alphabet_directory)
+
+        char_alphabet.add(PAD)
+        pos_alphabet.add(PAD)
+        ner_alphabet.add(PAD)
+        arc_alphabet.add(PAD)
+        auto_label_alphabet.add(PAD)
+
+        char_alphabet.add(ROOT)
+        pos_alphabet.add(ROOT)
+        ner_alphabet.add(ROOT)
+        arc_alphabet.add(ROOT)
+        auto_label_alphabet.add(ROOT)
+
+        char_alphabet.add(END)
+        pos_alphabet.add(END)
+        ner_alphabet.add(END)
+        arc_alphabet.add(END)
+        auto_label_alphabet.add(END)
+
+        vocab = dict()
+        if isinstance(train_paths, str):
+            train_paths = [train_paths]
+        for train_path in train_paths:
+            with open(train_path, 'r') as file:
+                for line in file:
+                    line = line.strip()
+                    if len(line) == 0:
+                        continue
+
+                    tokens = line.split('\t')
+                    if lower_case:
+                        tokens[1] = tokens[1].lower()
+                    for char in tokens[1]:
+                        char_alphabet.add(char)
+
+                    word = tokens[1]
+                    # print(word)
+                    pos = tokens[2]
+                    ner = tokens[3]
+                    arc_tag = tokens[5]
+
+                    pos_alphabet.add(pos)
+                    ner_alphabet.add(ner)
+                    arc_alphabet.add(arc_tag)
+
+                    if word in vocab:
+                        vocab[word] += 1
+                    else:
+                        vocab[word] = 1
+
+        # collect singletons
+        singletons = set([word for word, count in vocab.items() if count <= min_occurence])
+
+        # if a singleton is in pretrained embedding dict, set the count to min_occur + c
+        if embedd_dict is not None:
+            for word in vocab.keys():
+                if word in embedd_dict or word.lower() in embedd_dict:
+                    vocab[word] += min_occurence
+
+        vocab_list = sorted(vocab, key=vocab.get, reverse=True)
+        vocab_list = [word for word in vocab_list if vocab[word] > min_occurence]
+        vocab_list = _START_VOCAB + vocab_list
+
+        if extra_paths is not None:
+            vocab_list, char_alphabet, pos_alphabet, ner_alphabet, arc_alphabet = \
+                expand_vocab(vocab_list, char_alphabet, pos_alphabet, ner_alphabet, arc_alphabet)
+
+        if len(vocab_list) > max_vocabulary_size:
+            vocab_list = vocab_list[:max_vocabulary_size]
+
+        for word in vocab_list:
+            word_alphabet.add(word)
+            if word in singletons:
+                word_alphabet.add_singleton(word_alphabet.get_index(word))
+
+        word_alphabet.save(alphabet_directory)
+        char_alphabet.save(alphabet_directory)
+        pos_alphabet.save(alphabet_directory)
+        ner_alphabet.save(alphabet_directory)
+        arc_alphabet.save(alphabet_directory)
+        auto_label_alphabet.save(alphabet_directory)
+
+    else:
+        print('loading saved alphabet from %s' % alphabet_directory)
+        word_alphabet.load(alphabet_directory)
+        char_alphabet.load(alphabet_directory)
+        pos_alphabet.load(alphabet_directory)
+        ner_alphabet.load(alphabet_directory)
+        arc_alphabet.load(alphabet_directory)
+        auto_label_alphabet.load(alphabet_directory)
+
+    word_alphabet.close()
+    char_alphabet.close()
+    pos_alphabet.close()
+    ner_alphabet.close()
+    arc_alphabet.close()
+    auto_label_alphabet.close()
+
+    alphabet_dict = {'word_alphabet': word_alphabet, 'char_alphabet': char_alphabet, 'pos_alphabet': pos_alphabet,
+                     'ner_alphabet': ner_alphabet, 'arc_alphabet': arc_alphabet, 'auto_label_alphabet': auto_label_alphabet}
+    return alphabet_dict
+
+def create_alphabets_for_sequence_tagger(alphabet_directory, parser_alphabet_directory, paths):
+    logger = get_logger("Create Alphabets")
+    print('loading saved alphabet from %s' % parser_alphabet_directory)
+    word_alphabet = Alphabet('word', defualt_value=True, singleton=True)
+    char_alphabet = Alphabet('character', defualt_value=True)
+    pos_alphabet = Alphabet('pos', defualt_value=True)
+    ner_alphabet = Alphabet('ner', defualt_value=True)
+    arc_alphabet = Alphabet('arc', defualt_value=True)
+    auto_label_alphabet = Alphabet('auto_labeler', defualt_value=True)
+
+    word_alphabet.load(parser_alphabet_directory)
+    char_alphabet.load(parser_alphabet_directory)
+    pos_alphabet.load(parser_alphabet_directory)
+    ner_alphabet.load(parser_alphabet_directory)
+    arc_alphabet.load(parser_alphabet_directory)
+    try:
+        auto_label_alphabet.load(alphabet_directory)
+    except:
+        print('Creating auto labeler alphabet')
+        auto_label_alphabet.add(PAD)
+        auto_label_alphabet.add(ROOT)
+        auto_label_alphabet.add(END)
+        for path in paths:
+            with open(path, 'r') as file:
+                for line in file:
+                    line = line.strip()
+                    if len(line) == 0:
+                        continue
+                    tokens = line.split('\t')
+                    if len(tokens) > 6:
+                        auto_label = tokens[6]
+                        auto_label_alphabet.add(auto_label)
+
+    word_alphabet.save(alphabet_directory)
+    char_alphabet.save(alphabet_directory)
+    pos_alphabet.save(alphabet_directory)
+    ner_alphabet.save(alphabet_directory)
+    arc_alphabet.save(alphabet_directory)
+    auto_label_alphabet.save(alphabet_directory)
+    word_alphabet.close()
+    char_alphabet.close()
+    pos_alphabet.close()
+    ner_alphabet.close()
+    arc_alphabet.close()
+    auto_label_alphabet.close()
+    alphabet_dict = {'word_alphabet': word_alphabet, 'char_alphabet': char_alphabet, 'pos_alphabet': pos_alphabet,
+                     'ner_alphabet': ner_alphabet, 'arc_alphabet': arc_alphabet, 'auto_label_alphabet': auto_label_alphabet}
+    return alphabet_dict
+
+def read_data(source_path, alphabets, max_size=None,
+              lower_case=False, symbolic_root=False, symbolic_end=False):
+    data = [[] for _ in _buckets]
+    max_char_length = [0 for _ in _buckets]
+    print('Reading data from %s' % ', '.join(source_path) if type(source_path) is list else source_path)
+    counter = 0
+    if type(source_path) is not list:
+        source_path = [source_path]
+    for path in source_path:
+        reader = Reader(path, alphabets)
+        inst = reader.getNext(lower_case=lower_case, symbolic_root=symbolic_root, symbolic_end=symbolic_end)
+        while inst is not None and (not max_size or counter < max_size):
+            counter += 1
+            inst_size = inst.length()
+            sent = inst.sentence
+            for bucket_id, bucket_size in enumerate(_buckets):
+                if inst_size < bucket_size:
+                    data[bucket_id].append([sent.word_ids, sent.char_id_seqs, inst.ids['pos_alphabet'], inst.ids['ner_alphabet'],
+                                            inst.heads, inst.ids['arc_alphabet'], inst.ids['auto_label_alphabet']])
+                    max_len = max([len(char_seq) for char_seq in sent.char_seqs])
+                    if max_char_length[bucket_id] < max_len:
+                        max_char_length[bucket_id] = max_len
+                    break
+
+            inst = reader.getNext(lower_case=lower_case, symbolic_root=symbolic_root, symbolic_end=symbolic_end)
+        reader.close()
+    print("Total number of data: %d" % counter)
+    return data, max_char_length
+
+def read_data_to_variable(source_path, alphabets, device, max_size=None,
+                          lower_case=False, symbolic_root=False, symbolic_end=False):
+    data, max_char_length = read_data(source_path, alphabets,
+                                      max_size=max_size, lower_case=lower_case,
+                                      symbolic_root=symbolic_root, symbolic_end=symbolic_end)
+    bucket_sizes = [len(data[b]) for b in range(len(_buckets))]
+
+    data_variable = []
+
+    for bucket_id in range(len(_buckets)):
+        bucket_size = bucket_sizes[bucket_id]
+        if bucket_size <= 0:
+            data_variable.append((1, 1))
+            continue
+
+        bucket_length = _buckets[bucket_id]
+        char_length = min(MAX_CHAR_LENGTH, max_char_length[bucket_id] + NUM_CHAR_PAD)
+        wid_inputs = np.empty([bucket_size, bucket_length], dtype=np.int64)
+        cid_inputs = np.empty([bucket_size, bucket_length, char_length], dtype=np.int64)
+        pid_inputs = np.empty([bucket_size, bucket_length], dtype=np.int64)
+        nid_inputs = np.empty([bucket_size, bucket_length], dtype=np.int64)
+        hid_inputs = np.empty([bucket_size, bucket_length], dtype=np.int64)
+        aid_inputs = np.empty([bucket_size, bucket_length], dtype=np.int64)
+        mid_inputs = np.empty([bucket_size, bucket_length], dtype=np.int64)
+
+        masks = np.zeros([bucket_size, bucket_length], dtype=np.float32)
+        single = np.zeros([bucket_size, bucket_length], dtype=np.int64)
+
+        lengths = np.empty(bucket_size, dtype=np.int64)
+
+        for i, inst in enumerate(data[bucket_id]):
+            wids, cid_seqs, pids, nids, hids, aids, mids = inst
+            inst_size = len(wids)
+            lengths[i] = inst_size
+            # word ids
+            wid_inputs[i, :inst_size] = wids
+            wid_inputs[i, inst_size:] = PAD_ID_WORD
+            for c, cids in enumerate(cid_seqs):
+                cid_inputs[i, c, :len(cids)] = cids
+                cid_inputs[i, c, len(cids):] = PAD_ID_CHAR
+            cid_inputs[i, inst_size:, :] = PAD_ID_CHAR
+            # pos ids
+            pid_inputs[i, :inst_size] = pids
+            pid_inputs[i, inst_size:] = PAD_ID_TAG
+            # ner ids
+            nid_inputs[i, :inst_size] = nids
+            nid_inputs[i, inst_size:] = PAD_ID_TAG
+            # arc ids
+            aid_inputs[i, :inst_size] = aids
+            aid_inputs[i, inst_size:] = PAD_ID_TAG
+            # auto_label ids
+            mid_inputs[i, :inst_size] = mids
+            mid_inputs[i, inst_size:] = PAD_ID_TAG
+            # heads
+            hid_inputs[i, :inst_size] = hids
+            hid_inputs[i, inst_size:] = PAD_ID_TAG
+            # masks
+            masks[i, :inst_size] = 1.0
+            for j, wid in enumerate(wids):
+                if alphabets['word_alphabet'].is_singleton(wid):
+                    single[i, j] = 1
+
+        words = torch.LongTensor(wid_inputs)
+        chars = torch.LongTensor(cid_inputs)
+        pos = torch.LongTensor(pid_inputs)
+        ner = torch.LongTensor(nid_inputs)
+        heads = torch.LongTensor(hid_inputs)
+        arc = torch.LongTensor(aid_inputs)
+        auto_label = torch.LongTensor(mid_inputs)
+        masks = torch.FloatTensor(masks)
+        single = torch.LongTensor(single)
+        lengths = torch.LongTensor(lengths)
+        words = words.to(device)
+        chars = chars.to(device)
+        pos = pos.to(device)
+        ner = ner.to(device)
+        heads = heads.to(device)
+        arc = arc.to(device)
+        auto_label = auto_label.to(device)
+        masks = masks.to(device)
+        single = single.to(device)
+        lengths = lengths.to(device)
+
+        data_variable.append((words, chars, pos, ner, heads, arc, auto_label, masks, single, lengths))
+
+    return data_variable, bucket_sizes
+
+def iterate_batch(data, batch_size, device, unk_replace=0.0, shuffle=False):
+    data_variable, bucket_sizes = data
+
+    bucket_indices = np.arange(len(_buckets))
+    if shuffle:
+        np.random.shuffle((bucket_indices))
+
+    for bucket_id in bucket_indices:
+        bucket_size = bucket_sizes[bucket_id]
+        bucket_length = _buckets[bucket_id]
+        if bucket_size <= 0:
+            continue
+
+        words, chars, pos, ner, heads, arc, auto_label, masks, single, lengths = data_variable[bucket_id]
+        if unk_replace:
+            ones = single.data.new(bucket_size, bucket_length).fill_(1)
+            noise = masks.data.new(bucket_size, bucket_length).bernoulli_(unk_replace).long()
+            words = words * (ones - single * noise)
+
+        indices = None
+        if shuffle:
+            indices = torch.randperm(bucket_size).long()
+            indices = indices.to(device)
+        for start_idx in range(0, bucket_size, batch_size):
+            if shuffle:
+                excerpt = indices[start_idx:start_idx + batch_size]
+            else:
+                excerpt = slice(start_idx, start_idx + batch_size)
+            yield words[excerpt], chars[excerpt], pos[excerpt], ner[excerpt], heads[excerpt], arc[excerpt], auto_label[excerpt], \
+                  masks[excerpt], lengths[excerpt]
+
+def iterate_batch_rand_bucket_choosing(data, batch_size, device, unk_replace=0.0):
+    data_variable, bucket_sizes = data
+    indices_left = [set(np.arange(bucket_size)) for bucket_size in bucket_sizes]
+    while sum(bucket_sizes) > 0:
+        non_empty_buckets = [i for i, bucket_size in enumerate(bucket_sizes) if bucket_size > 0]
+        bucket_id = np.random.choice(non_empty_buckets)
+        bucket_size = bucket_sizes[bucket_id]
+        bucket_length = _buckets[bucket_id]
+
+        words, chars, pos, ner, heads, arc, auto_label, masks, single, lengths = data_variable[bucket_id]
+        min_batch_size = min(bucket_size, batch_size)
+        indices = torch.LongTensor(np.random.choice(list(indices_left[bucket_id]), min_batch_size, replace=False))
+        set_indices = set(indices.numpy())
+        indices_left[bucket_id] = indices_left[bucket_id].difference(set_indices)
+        indices = indices.to(device)
+        words = words[indices]
+        if unk_replace:
+            ones = single.data.new(min_batch_size, bucket_length).fill_(1)
+            noise = masks.data.new(min_batch_size, bucket_length).bernoulli_(unk_replace).long()
+            words = words * (ones - single[indices] * noise)
+        bucket_sizes = [len(s) for s in indices_left]
+        yield words, chars[indices], pos[indices], ner[indices], heads[indices], arc[indices], auto_label[indices], masks[indices], lengths[indices]
+
+
+def calc_num_batches(data, batch_size):
+    _, bucket_sizes = data
+    bucket_sizes_mod_batch_size = [int(bucket_size / batch_size) + 1 if bucket_size > 0 else 0 for bucket_size in bucket_sizes]
+    num_batches = sum(bucket_sizes_mod_batch_size)
+    return num_batches

utils/io_/reader.py

@@ -0,0 +1,93 @@
+from .instance import NER_DependencyInstance
+from .instance import Sentence
+from .prepare_data import ROOT, END, MAX_CHAR_LENGTH
+
+class Reader(object):
+    def __init__(self, file_path, alphabets):
+        self.__source_file = open(file_path, 'r')
+        self.alphabets = alphabets
+
+    def close(self):
+        self.__source_file.close()
+
+    def getNext(self, lower_case=False, symbolic_root=False, symbolic_end=False):
+        line = self.__source_file.readline()
+        # skip multiple blank lines.
+        while len(line) > 0 and len(line.strip()) == 0:
+            line = self.__source_file.readline()
+        if len(line) == 0:
+            return None
+
+        lines = []
+        while len(line.strip()) > 0:
+            line = line.strip()
+            lines.append(line.split('\t'))
+            line = self.__source_file.readline()
+
+        length = len(lines)
+        if length == 0:
+            return None
+
+        heads = []
+        tokens_dict = {}
+        ids_dict = {}
+        for alphabet_name in self.alphabets.keys():
+            tokens_dict[alphabet_name] = []
+            ids_dict[alphabet_name] = []
+        if symbolic_root:
+            for alphabet_name, alphabet in self.alphabets.items():
+                if alphabet_name.startswith('char'):
+                    tokens_dict[alphabet_name].append([ROOT, ])
+                    ids_dict[alphabet_name].append([alphabet.get_index(ROOT), ])
+                else:
+                    tokens_dict[alphabet_name].append(ROOT)
+                    ids_dict[alphabet_name].append(alphabet.get_index(ROOT))
+            heads.append(0)
+
+        for tokens in lines:
+            chars = []
+            char_ids = []
+            if lower_case:
+                tokens[1] = tokens[1].lower()
+            for char in tokens[1]:
+                chars.append(char)
+                char_ids.append(self.alphabets['char_alphabet'].get_index(char))
+            if len(chars) > MAX_CHAR_LENGTH:
+                chars = chars[:MAX_CHAR_LENGTH]
+                char_ids = char_ids[:MAX_CHAR_LENGTH]
+            tokens_dict['char_alphabet'].append(chars)
+            ids_dict['char_alphabet'].append(char_ids)
+
+            word = tokens[1]
+            # print(word+ ' ')
+            pos = tokens[2]
+            ner = tokens[3]
+            head = int(tokens[4])
+            arc_tag = tokens[5]
+            if len(tokens) > 6:
+                auto_label = tokens[6]
+                tokens_dict['auto_label_alphabet'].append(auto_label)
+                ids_dict['auto_label_alphabet'].append(self.alphabets['auto_label_alphabet'].get_index(auto_label))
+            tokens_dict['word_alphabet'].append(word)
+            ids_dict['word_alphabet'].append(self.alphabets['word_alphabet'].get_index(word))
+            tokens_dict['pos_alphabet'].append(pos)
+            ids_dict['pos_alphabet'].append(self.alphabets['pos_alphabet'].get_index(pos))
+            tokens_dict['ner_alphabet'].append(ner)
+            ids_dict['ner_alphabet'].append(self.alphabets['ner_alphabet'].get_index(ner))
+            tokens_dict['arc_alphabet'].append(arc_tag)
+            ids_dict['arc_alphabet'].append(self.alphabets['arc_alphabet'].get_index(arc_tag))
+            heads.append(head)
+
+        if symbolic_end:
+            for alphabet_name, alphabet in self.alphabets.items():
+                if alphabet_name.startswith('char'):
+                    tokens_dict[alphabet_name].append([END, ])
+                    ids_dict[alphabet_name].append([alphabet.get_index(END), ])
+                else:
+                    tokens_dict[alphabet_name] = [END]
+                    ids_dict[alphabet_name] = [alphabet.get_index(END)]
+            heads.append(0)
+
+        return NER_DependencyInstance(Sentence(tokens_dict['word_alphabet'], ids_dict['word_alphabet'],
+                                               tokens_dict['char_alphabet'], ids_dict['char_alphabet']),
+                                      tokens_dict, ids_dict, heads)

utils/io_/rearrange_splits.py

@@ -0,0 +1,68 @@
+import numpy as np
+from torch import stack
+
+def rearranging_splits(datasets, num_training_samples):
+    new_datasets = {}
+    data_splits = datasets.keys()
+    for split in data_splits:
+        if split == 'test':
+            new_datasets['test'] = datasets['test']
+    
+        else:
+            num_buckets = len(datasets[split][1])
+            num_tensors = len(datasets[split][0][0])
+            num_samples = sum(datasets[split][1])
+            if num_samples < num_training_samples:
+                print("set_num_training_samples (%d) should be smaller than the actual %s size (%d)"
+                                 % (num_training_samples, split, num_samples))
+            new_datasets[split] = [[[[] for _ in range(num_tensors)] for _ in range(num_buckets)], []]
+            new_datasets['extra_' + split] = [[[[] for _ in range(num_tensors)] for _ in range(num_buckets)], []]
+    for split in data_splits:
+        if split == 'test':
+            continue
+        else:
+            curr_bucket_sizes = datasets[split][1]
+            curr_samples = datasets[split][0]
+            num_tensors = len(datasets[split][0][0])
+            curr_num_samples = sum(curr_bucket_sizes)
+            sample_indices_in_buckets = {}
+            i = 0
+            for bucket_idx, bucket_size in enumerate(curr_bucket_sizes):
+                for sample_idx in range(bucket_size):
+                    sample_indices_in_buckets[i] = (bucket_idx, sample_idx)
+                    i += 1
+            rng = np.arange(curr_num_samples)
+            rng = np.random.permutation(rng)
+            sample_indices = {}
+            sample_indices[split] = [sample_indices_in_buckets[key] for key in rng[:num_training_samples]]
+            sample_indices['extra_' + split] = [sample_indices_in_buckets[key] for key in rng[num_training_samples:]]
+            if len(sample_indices['extra_' + split]) == 0:
+                if len(sample_indices[split]) > 1:
+                    sample_indices['extra_' + split].append(sample_indices[split].pop(-1))
+                else:
+                    sample_indices['extra_' + split].append(sample_indices[split][0])
+
+            for key, indices in sample_indices.items():
+                for bucket_idx, sample_idx in indices:
+                    curr_bucket = curr_samples[bucket_idx]
+                    for tensor_idx, tensor in enumerate(curr_bucket):
+                        new_datasets[key][0][bucket_idx][tensor_idx].append(tensor[sample_idx])
+    del datasets
+    new_splits = []
+    new_splits += [split for split in data_splits if split != 'test']
+    new_splits += ['extra_' + split for split in data_splits if split != 'test']
+
+    for split in new_splits:
+        for bucket_idx in range(num_buckets):
+            for tensor_idx in range(num_tensors):
+                if len(new_datasets[split][0][bucket_idx][tensor_idx]) > 0:
+                    new_datasets[split][0][bucket_idx][tensor_idx] = stack(new_datasets[split][0][bucket_idx][tensor_idx])
+                else:
+                    new_datasets[split][0][bucket_idx] = (1,1)
+                    break
+            # set lengths of buckets
+            if new_datasets[split][0][bucket_idx] == (1,1):
+                new_datasets[split][1].append(0)
+            else:
+                new_datasets[split][1].append(len(new_datasets[split][0][bucket_idx][tensor_idx]))
+    return new_datasets

utils/io_/remove_xx.py

@@ -0,0 +1,60 @@
+import numpy as np
+
+def read_file(filename):
+    sentences = []
+    sentence = []
+    lengths = []
+    num_sentneces_to_remove = 0
+    num_sentences = 0
+    num_tokens_to_remove = 0
+    num_tokens = 0
+    with open(filename, 'r') as file:
+        for line in file:
+            line = line.strip()
+            if len(line) == 0:
+                xx_count = 0
+                for row in sentence:
+                    if row[2] == 'XX':
+                        xx_count += 1
+                if xx_count / len(sentence) >= 0.5:
+                    num_sentneces_to_remove += 1
+                    num_tokens_to_remove += len(sentence)
+                else:
+                    sentences.append(sentence)
+                    lengths.append(len(sentence))
+                num_sentences += 1
+                num_tokens += len(sentence)
+                sentence = []
+                continue
+            tokens = line.split('\t')
+            idx = tokens[0]
+            word = tokens[1]
+            pos = tokens[2]
+            ner = tokens[3]
+            arc = tokens[4]
+            arc_tag = tokens[5]
+            sentence.append((idx, word, pos, ner, arc, arc_tag))
+    print("removed %d sentences out of %d sentences" % (num_sentneces_to_remove, num_sentences))
+    print("removed %d tokens out of %d tokens" % (num_tokens_to_remove, num_tokens))
+    return sentences
+
+def write_file(filename, sentences):
+    with open(filename, 'w') as file:
+        for sentence in sentences:
+            for row in sentence:
+                file.write('\t'.join([token for token in row]) + '\n')
+            file.write('\n')
+
+dataset_dict = {'ontonotes': 'onto'}
+datasets = ['ontonotes']
+splits = ['test']
+domains = ['all', 'wb']
+
+for dataset in datasets:
+    for domain in domains:
+        for split in splits:
+            print('dataset: %s, domain: %s, split: %s' % (dataset, domain, split))
+            filemame = 'data/'+ dataset_dict[dataset] + '_pos_ner_dp_' + split + '_' + domain
+            sentences = read_file(filemame)
+            write_filename = filemame + '_without_xx'
+            write_file(write_filename, sentences)

utils/io_/seeds.py

@@ -0,0 +1,12 @@
+import torch
+import numpy as np
+import random
+seed = 0
+torch.manual_seed(seed)
+torch.cuda.manual_seed(seed)
+torch.cuda.manual_seed_all(seed)  # if you are using multi-GPU.
+np.random.seed(seed)  # Numpy module.
+random.seed(seed)  # Python random module.
+torch.manual_seed(seed)
+torch.backends.cudnn.benchmark = False
+torch.backends.cudnn.deterministic = True

utils/io_/write_extra_labels.py

@@ -0,0 +1,1592 @@
+import os
+from .prepare_data import ROOT, END
+import pdb
+def get_split(path):
+    if 'train' in path:
+        if 'extra_train' in path:
+            split = 'extra_train'
+        else:
+            split = 'train'
+    elif 'dev' in path:
+        if 'extra_dev' in path:
+            split = 'extra_dev'
+        else:
+            split = 'dev'
+    else:
+        split = 'test'
+    return split
+
+def add_number_of_children(model_path, parser_path, src_domain, tgt_domain, use_unlabeled_data=True, use_labeled_data=True):
+    if src_domain == tgt_domain:
+        pred_paths = []
+        if use_unlabeled_data:
+            pred_paths = [file for file in os.listdir(parser_path) if file.endswith("pred.txt") and 'poetry' not in file and 'prose' not in file and 'extra' in file and tgt_domain in file]
+
+        gold_paths = [file for file in os.listdir(parser_path) if file.endswith("gold.txt") and 'poetry' not in file and 'prose' not in file  and 'extra' not in file and tgt_domain in file and 'train' not in file]
+        if use_labeled_data:
+            gold_paths += [file for file in os.listdir(parser_path) if file.endswith("gold.txt") and 'poetry' not in file and 'prose' not in file and 'extra' not in file and tgt_domain in file and 'train' in file]
+
+        if not use_unlabeled_data and not use_labeled_data:
+            raise ValueError
+    else:
+        pred_paths = [file for file in os.listdir(parser_path) if file.endswith("pred.txt") and tgt_domain in file]
+
+        gold_paths = []
+        if use_labeled_data:
+            gold_paths = ['data/onto_pos_ner_dp_train_' + src_domain]
+
+        if not use_unlabeled_data and not use_labeled_data:
+            raise ValueError
+
+    paths = pred_paths + gold_paths
+    print("Adding labels to paths: %s" % ', '.join(paths))
+    root_line = ['0', ROOT, 'XX', 'O', '0', 'root']
+    writing_paths = {}
+    sentences = {}
+    for path in paths:
+        if tgt_domain in path:
+            reading_path = parser_path + path
+            writing_path = model_path + 'parser_' + path
+            split = get_split(writing_path)
+        else:
+            reading_path = path
+            writing_path = model_path + 'parser_' + 'domain_' + src_domain + '_train_model_domain_' + src_domain + '_data_domain_' + src_domain + '_gold.txt'
+            split = 'extra_train'
+        writing_paths[split] = writing_path
+        len_sent = 0
+        number_of_children = {}
+        lines = []
+        sentences_list = []
+        with open(reading_path, 'r') as file:
+            for line in file:
+                # line = line.decode('utf-8')
+                line = line.strip()
+                # print(line)
+                if len(line) == 0:
+                    for idx in range(len_sent):
+                        node = str(idx + 1)
+                        if node not in number_of_children:
+                            lines[idx].append('0')
+                        else:
+                            lines[idx].append(str(number_of_children[node]))
+                    if len(lines) > 0:
+                        tmp_root_line = root_line + [str(number_of_children['0'])]
+                        sentences_list.append(tmp_root_line)
+                    for line_ in lines:
+                        sentences_list.append(line_)
+                    sentences_list.append([])
+                    lines = []
+                    number_of_children = {}
+                    len_sent = 0
+                    continue
+                tokens = line.split('\t')
+                idx = tokens[0]
+                word = tokens[1]
+                pos = tokens[2]
+                ner = tokens[3]
+                head = tokens[4]
+                arc_tag = tokens[5]
+                if head not in number_of_children:
+                    number_of_children[head] = 1
+                else:
+                    number_of_children[head] += 1
+                lines.append([idx, word, pos, ner, head, arc_tag])
+                len_sent += 1
+        sentences[split] = sentences_list
+
+    train_sentences = []
+    if 'train' in sentences:
+        train_sentences = sentences['train']
+    else:
+        writing_paths['train'] = writing_paths['extra_train'].replace('extra_train', 'train')
+    if 'extra_train' in sentences:
+        train_sentences += sentences['extra_train']
+        del writing_paths['extra_train']
+    if 'extra_dev' in sentences:
+        train_sentences += sentences['extra_dev']
+        del writing_paths['extra_dev']
+    with open(writing_paths['train'], 'w') as f:
+        for sent in train_sentences:
+            f.write('\t'.join(sent) + '\n')
+    for split in ['dev', 'test']:
+        if split in sentences:
+            split_sentences = sentences[split]
+            with open(writing_paths[split], 'w') as f:
+                for sent in split_sentences:
+                    f.write('\t'.join(sent) + '\n')
+    return writing_paths
+
+
+def add_distance_from_the_root(model_path, parser_path, src_domain, tgt_domain, use_unlabeled_data=True, use_labeled_data=True):
+    if src_domain == tgt_domain:
+        pred_paths = []
+        if use_unlabeled_data:
+            pred_paths = [file for file in os.listdir(parser_path) if
+                          file.endswith("pred.txt") and 'poetry' not in file and 'prose' not in file  and 'extra' in file and tgt_domain in file]
+
+        gold_paths = [file for file in os.listdir(parser_path) if
+                      file.endswith("gold.txt") and 'poetry' not in file and 'prose' not in file  and 'extra' not in file and tgt_domain in file and 'train' not in file]
+        if use_labeled_data:
+            gold_paths += [file for file in os.listdir(parser_path) if
+                           file.endswith("gold.txt") and 'extra' not in file and 'poetry' not in file and 'prose' not in file and tgt_domain in file and 'train' in file]
+
+        if not use_unlabeled_data and not use_labeled_data:
+            raise ValueError
+    else:
+        pred_paths = [file for file in os.listdir(parser_path) if file.endswith("pred.txt") and tgt_domain in file]
+
+        gold_paths = []
+        if use_labeled_data:
+            gold_paths = ['data/onto_pos_ner_dp_train_' + src_domain]
+
+        if not use_unlabeled_data and not use_labeled_data:
+            raise ValueError
+
+    paths = pred_paths + gold_paths
+    print("Adding labels to paths: %s" % ', '.join(paths))
+    root_line = ['0', ROOT, 'XX', 'O', '0', 'root', '0']
+    writing_paths = {}
+    sentences = {}
+    for path in paths:
+        if tgt_domain in path:
+            reading_path = parser_path + path
+            writing_path = model_path + 'parser_' + path
+            split = get_split(writing_path)
+        else:
+            reading_path = path
+            writing_path = model_path + 'parser_' + 'domain_' + src_domain + '_train_model_domain_' + src_domain + '_data_domain_' + src_domain + '_gold.txt'
+            split = 'extra_train'
+        writing_paths[split] = writing_path
+        len_sent = 0
+        tree_dict = {'0': '0'}
+        lines = []
+        sentences_list = []
+        with open(reading_path, 'r') as file:
+            for line in file:
+                # line = line.decode('utf-8')
+                line = line.strip()
+                if len(line) == 0:
+                    for idx in range(len_sent):
+                        depth = 1
+                        node = str(idx + 1)
+                        while tree_dict[node] != '0':
+                            node = tree_dict[node]
+                            depth += 1
+                        lines[idx].append(str(depth))
+                    if len(lines) > 0:
+                        sentences_list.append(root_line)
+                    for line_ in lines:
+                        sentences_list.append(line_)
+                    sentences_list.append([])
+                    lines = []
+                    tree_dict = {'0': '0'}
+                    len_sent = 0
+                    continue
+                tokens = line.split('\t')
+                idx = tokens[0]
+                word = tokens[1]
+                pos = tokens[2]
+                ner = tokens[3]
+                head = tokens[4]
+                arc_tag = tokens[5]
+                tree_dict[idx] = head
+                lines.append([idx, word, pos, ner, head, arc_tag])
+                len_sent += 1
+        sentences[split] = sentences_list
+
+    train_sentences = []
+    if 'train' in sentences:
+        train_sentences = sentences['train']
+    else:
+        writing_paths['train'] = writing_paths['extra_train'].replace('extra_train', 'train')
+    if 'extra_train' in sentences:
+        train_sentences += sentences['extra_train']
+        del writing_paths['extra_train']
+    if 'extra_dev' in sentences:
+        train_sentences += sentences['extra_dev']
+        del writing_paths['extra_dev']
+    with open(writing_paths['train'], 'w') as f:
+        for sent in train_sentences:
+            f.write('\t'.join(sent) + '\n')
+    for split in ['dev', 'test']:
+        if split in sentences:
+            split_sentences = sentences[split]
+            with open(writing_paths[split], 'w') as f:
+                for sent in split_sentences:
+                    f.write('\t'.join(sent) + '\n')
+    return writing_paths
+
+def add_relative_pos_based(model_path, parser_path, src_domain, tgt_domain, use_unlabeled_data=True, use_labeled_data=True):
+    # most of the code for this function is taken from:
+    # https://github.com/mstrise/dep2label/blob/master/encoding.py
+    def pos_cluster(pos):
+        # clustering the parts of speech
+        if pos[0] == 'V':
+            pos = 'VB'
+        elif pos == 'NNS':
+            pos = 'NN'
+        elif pos == 'NNPS':
+            pos = 'NNP'
+        elif 'JJ' in pos:
+            pos = 'JJ'
+        elif pos[:2] == 'RB' or pos == 'WRB' or pos == 'RP':
+            pos = 'RB'
+        elif pos[:3] == 'PRP':
+            pos = 'PRP'
+        elif pos in ['.', ':', ',', "''", '``']:
+            pos = '.'
+        elif pos[0] == '-':
+            pos = '-RB-'
+        elif pos[:2] == 'WP':
+            pos = 'WP'
+        return pos
+
+    if src_domain == tgt_domain:
+        pred_paths = []
+        if use_unlabeled_data:
+            pred_paths = [file for file in os.listdir(parser_path) if
+                          file.endswith("pred.txt") and 'poetry' not in file and 'prose' not in file  and 'extra' in file and tgt_domain in file]
+
+        gold_paths = [file for file in os.listdir(parser_path) if
+                      file.endswith("gold.txt") and 'poetry' not in file and 'prose' not in file  and 'extra' not in file and tgt_domain in file and 'train' not in file]
+        if use_labeled_data:
+            gold_paths += [file for file in os.listdir(parser_path) if
+                           file.endswith("gold.txt") and 'extra' not in file and 'poetry' not in file and 'prose' not in file and tgt_domain in file and 'train' in file]
+
+        if not use_unlabeled_data and not use_labeled_data:
+            raise ValueError
+    else:
+        pred_paths = [file for file in os.listdir(parser_path) if file.endswith("pred.txt") and tgt_domain in file]
+        gold_paths = []
+        if use_labeled_data:
+            gold_paths = ['data/onto_pos_ner_dp_train_' + src_domain]
+
+        if not use_unlabeled_data:
+            raise ValueError
+
+    paths = pred_paths + gold_paths
+    print("Adding labels to paths: %s" % ', '.join(paths))
+    root_line = ['0', ROOT, 'XX', 'O', '0', 'root', '+0_XX']
+    writing_paths = {}
+    sentences = {}
+    for path in paths:
+        if tgt_domain in path:
+            reading_path = parser_path + path
+            writing_path = model_path + 'parser_' + path
+            split = get_split(writing_path)
+        else:
+            reading_path = path
+            writing_path = model_path + 'parser_' + 'domain_' + src_domain + '_train_model_domain_' + src_domain + '_data_domain_' + src_domain + '_gold.txt'
+            split = 'extra_train'
+        writing_paths[split] = writing_path
+        len_sent = 0
+        tree_dict = {'0': '0'}
+        lines = []
+        sentences_list = []
+        with open(reading_path, 'r') as file:
+            for line in file:
+                # line = line.decode('utf-8')
+                line = line.strip()
+                if len(line) == 0:
+                    for idx in range(len_sent):
+                        info_of_a_word = lines[idx]
+                        # head is on the right side from the word
+                        head = int(info_of_a_word[4]) - 1
+                        if head == -1:
+                            info_about_head = root_line
+                        else:
+                            info_about_head = lines[head]
+                        if idx < head:
+                            relative_position_head = 1
+                            postag_head = pos_cluster(info_about_head[2])
+
+                            for x in range(idx + 1, head):
+                                another_word = lines[x]
+                                postag_word_before_head = pos_cluster(another_word[2])
+                                if postag_word_before_head == postag_head:
+                                    relative_position_head += 1
+                            label = str(
+                                "+" +
+                                repr(relative_position_head) +
+                                "_" +
+                                postag_head)
+                            lines[idx].append(label)
+
+                        # head is on the left side from the word
+                        elif idx > head:
+                            relative_position_head = 1
+                            postag_head = pos_cluster(info_about_head[2])
+                            for x in range(head + 1, idx):
+                                another_word = lines[x]
+                                postag_word_before_head = pos_cluster(another_word[2])
+                                if postag_word_before_head == postag_head:
+                                    relative_position_head += 1
+                            label = str(
+                                "-" +
+                                repr(relative_position_head) +
+                                "_" +
+                                postag_head)
+                            lines[idx].append(label)
+                    if len(lines) > 0:
+                        sentences_list.append(root_line)
+                    for line_ in lines:
+                        sentences_list.append(line_)
+                    sentences_list.append([])
+                    lines = []
+                    tree_dict = {'0': '0'}
+                    len_sent = 0
+                    continue
+                tokens = line.split('\t')
+                idx = tokens[0]
+                word = tokens[1]
+                pos = tokens[2]
+                ner = tokens[3]
+                head = tokens[4]
+                arc_tag = tokens[5]
+                tree_dict[idx] = head
+                lines.append([idx, word, pos, ner, head, arc_tag])
+                len_sent += 1
+        sentences[split] = sentences_list
+
+    train_sentences = []
+    if 'train' in sentences:
+        train_sentences = sentences['train']
+    else:
+        writing_paths['train'] = writing_paths['extra_train'].replace('extra_train', 'train')
+    if 'extra_train' in sentences:
+        train_sentences += sentences['extra_train']
+        del writing_paths['extra_train']
+    if 'extra_dev' in sentences:
+        train_sentences += sentences['extra_dev']
+        del writing_paths['extra_dev']
+    with open(writing_paths['train'], 'w') as f:
+        for sent in train_sentences:
+            f.write('\t'.join(sent) + '\n')
+    for split in ['dev', 'test']:
+        if split in sentences:
+            split_sentences = sentences[split]
+            with open(writing_paths[split], 'w') as f:
+                for sent in split_sentences:
+                    f.write('\t'.join(sent) + '\n')
+    return writing_paths
+
+def add_language_model(model_path, parser_path, src_domain, tgt_domain, use_unlabeled_data=True, use_labeled_data=True):
+    if src_domain == tgt_domain:
+        pred_paths = []
+        if use_unlabeled_data:
+            pred_paths = [file for file in os.listdir(parser_path) if
+                          file.endswith("pred.txt") and 'extra' in file and tgt_domain in file]
+
+        gold_paths = [file for file in os.listdir(parser_path) if
+                      file.endswith("gold.txt") and 'extra' not in file and tgt_domain in file and 'train' not in file]
+        if use_labeled_data:
+            gold_paths += [file for file in os.listdir(parser_path) if
+                           file.endswith("gold.txt") and 'extra' not in file and tgt_domain in file and 'train' in file]
+
+        if not use_unlabeled_data and not use_labeled_data:
+            raise ValueError
+    else:
+        pred_paths = [file for file in os.listdir(parser_path) if file.endswith("pred.txt") and tgt_domain in file]
+
+        gold_paths = []
+        if use_labeled_data:
+            gold_paths = ['data/onto_pos_ner_dp_train_' + src_domain]
+
+        if not use_unlabeled_data and not use_labeled_data:
+            raise ValueError
+
+    paths = pred_paths + gold_paths
+    print("Adding labels to paths: %s" % ', '.join(paths))
+    root_line = ['0', ROOT, 'XX', 'O', '0', 'root']
+    writing_paths = {}
+    sentences = {}
+    for path in paths:
+        if tgt_domain in path:
+            reading_path = parser_path + path
+            writing_path = model_path + 'parser_' + path
+            split = get_split(writing_path)
+        else:
+            reading_path = path
+            writing_path = model_path + 'parser_' + 'domain_' + src_domain + '_train_model_domain_' + src_domain + '_data_domain_' + src_domain + '_gold.txt'
+            split = 'extra_train'
+        writing_paths[split] = writing_path
+        len_sent = 0
+        lines = []
+        sentences_list = []
+        with open(reading_path, 'r') as file:
+            for line in file:
+                # line = line.decode('utf-8')
+                line = line.strip()
+                if len(line) == 0:
+                    for idx in range(len_sent):
+                        if idx < len_sent - 1:
+                            lines[idx].append(lines[idx+1][1])
+                        else:
+                            lines[idx].append(END)
+                    if len(lines) > 0:
+                        tmp_root_line = root_line + [lines[0][1]]
+                        sentences_list.append(tmp_root_line)
+                    for line_ in lines:
+                        sentences_list.append(line_)
+                    sentences_list.append([])
+                    lines = []
+                    len_sent = 0
+                    continue
+                tokens = line.split('\t')
+                idx = tokens[0]
+                word = tokens[1]
+                pos = tokens[2]
+                ner = tokens[3]
+                head = tokens[4]
+                arc_tag = tokens[5]
+                lines.append([idx, word, pos, ner, head, arc_tag])
+                len_sent += 1
+        sentences[split] = sentences_list
+
+    train_sentences = []
+    if 'train' in sentences:
+        train_sentences = sentences['train']
+    else:
+        writing_paths['train'] = writing_paths['extra_train'].replace('extra_train', 'train')
+    if 'extra_train' in sentences:
+        train_sentences += sentences['extra_train']
+        del writing_paths['extra_train']
+    if 'extra_dev' in sentences:
+        train_sentences += sentences['extra_dev']
+        del writing_paths['extra_dev']
+    with open(writing_paths['train'], 'w') as f:
+        for sent in train_sentences:
+            f.write('\t'.join(sent) + '\n')
+    for split in ['dev', 'test']:
+        if split in sentences:
+            split_sentences = sentences[split]
+            with open(writing_paths[split], 'w') as f:
+                for sent in split_sentences:
+                    f.write('\t'.join(sent) + '\n')
+    return writing_paths
+
+def add_relative_TAG(model_path, parser_path, src_domain, tgt_domain, use_unlabeled_data=True, use_labeled_data=True):
+    # most of the code for this function is taken from:
+    # https://github.com/mstrise/dep2label/blob/master/encoding.py
+    def pos_cluster(pos):
+        # clustering the parts of speech
+        if pos[0] == 'V':
+            pos = 'VB'
+        elif pos == 'NNS':
+            pos = 'NN'
+        elif pos == 'NNPS':
+            pos = 'NNP'
+        elif 'JJ' in pos:
+            pos = 'JJ'
+        elif pos[:2] == 'RB' or pos == 'WRB' or pos == 'RP':
+            pos = 'RB'
+        elif pos[:3] == 'PRP':
+            pos = 'PRP'
+        elif pos in ['.', ':', ',', "''", '``']:
+            pos = '.'
+        elif pos[0] == '-':
+            pos = '-RB-'
+        elif pos[:2] == 'WP':
+            pos = 'WP'
+        return pos
+
+    if src_domain == tgt_domain:
+        pred_paths = []
+        if use_unlabeled_data:
+            pred_paths = [file for file in os.listdir(parser_path) if
+                          file.endswith("pred.txt") and 'extra' in file and tgt_domain in file]
+
+        gold_paths = [file for file in os.listdir(parser_path) if
+                      file.endswith("gold.txt") and 'extra' not in file and tgt_domain in file and 'train' not in file]
+        if use_labeled_data:
+            gold_paths += [file for file in os.listdir(parser_path) if
+                           file.endswith("gold.txt") and 'extra' not in file and tgt_domain in file and 'train' in file]
+
+        if not use_unlabeled_data and not use_labeled_data:
+            raise ValueError
+    else:
+        pred_paths = [file for file in os.listdir(parser_path) if file.endswith("pred.txt") and tgt_domain in file]
+        gold_paths = []
+        if use_labeled_data:
+            gold_paths = ['data/onto_pos_ner_dp_train_' + src_domain]
+
+        if not use_unlabeled_data:
+            raise ValueError
+
+    paths = pred_paths + gold_paths
+    print("Adding labels to paths: %s" % ', '.join(paths))
+    root_line = ['0', ROOT, 'XX', 'O', '0', 'root', '+0_XX']
+    writing_paths = {}
+    sentences = {}
+    for path in paths:
+        if tgt_domain in path:
+            reading_path = parser_path + path
+            writing_path = model_path + 'parser_' + path
+            split = get_split(writing_path)
+        else:
+            reading_path = path
+            writing_path = model_path + 'parser_' + 'domain_' + src_domain + '_train_model_domain_' + src_domain + '_data_domain_' + src_domain + '_gold.txt'
+            split = 'extra_train'
+        writing_paths[split] = writing_path
+        len_sent = 0
+        tree_dict = {'0': '0'}
+        lines = []
+        sentences_list = []
+        with open(reading_path, 'r') as file:
+
+            for line in file:
+                # print(line)
+                # print(reading_path)
+                # line = line.decode('utf-8')
+                line = line.strip()
+                if len(line) == 0:
+                    for idx in range(len_sent):
+                        info_of_a_word = lines[idx]
+                        # head is on the right side from the word
+                        head = int(info_of_a_word[4]) - 1
+                        if head == -1:
+                            info_about_head = root_line
+                        else:
+                            # print(len(lines), head)
+                            info_about_head = lines[head]
+                                
+                        if idx < head:
+                            relative_position_head = 1
+                            tag_head = info_about_head[5]
+
+                            for x in range(idx + 1, head):
+                                another_word = lines[x]
+                                tag_word_before_head = another_word[5]
+                                if tag_word_before_head == tag_head:
+                                    relative_position_head += 1
+                            label = str(
+                                "+" +
+                                repr(relative_position_head) +
+                                "_" +
+                                tag_head)
+                            lines[idx].append(label)
+
+                        # head is on the left side from the word
+                        elif idx > head:
+                            relative_position_head = 1
+                            tag_head = info_about_head[5]
+                            for x in range(head + 1, idx):
+                                another_word = lines[x]
+                                tag_word_before_head = another_word[5]
+                                if tag_word_before_head == tag_head:
+                                    relative_position_head += 1
+                            label = str(
+                                "-" +
+                                repr(relative_position_head) +
+                                "_" +
+                                tag_head)
+                            lines[idx].append(label)
+                    if len(lines) > 0:
+                        sentences_list.append(root_line)
+                    for line_ in lines:
+                        sentences_list.append(line_)
+                    sentences_list.append([])
+                    lines = []
+                    tree_dict = {'0': '0'}
+                    len_sent = 0
+                    continue
+                tokens = line.split('\t')
+                idx = tokens[0]
+                word = tokens[1]
+                pos = tokens[2]
+                ner = tokens[3]
+                head = tokens[4]
+                arc_tag = tokens[5]
+                tree_dict[idx] = head
+                lines.append([idx, word, pos, ner, head, arc_tag])
+                len_sent += 1
+        sentences[split] = sentences_list
+
+    train_sentences = []
+    if 'train' in sentences:
+        train_sentences = sentences['train']
+    else:
+        writing_paths['train'] = writing_paths['extra_train'].replace('extra_train', 'train')
+    if 'extra_train' in sentences:
+        train_sentences += sentences['extra_train']
+        del writing_paths['extra_train']
+    if 'extra_dev' in sentences:
+        train_sentences += sentences['extra_dev']
+        del writing_paths['extra_dev']
+    with open(writing_paths['train'], 'w') as f:
+        for sent in train_sentences:
+            f.write('\t'.join(sent) + '\n')
+    for split in ['dev', 'test']:
+        if split in sentences:
+            split_sentences = sentences[split]
+            with open(writing_paths[split], 'w') as f:
+                for sent in split_sentences:
+                    f.write('\t'.join(sent) + '\n')
+    return writing_paths
+
+
+def add_head(model_path, parser_path, src_domain, tgt_domain, use_unlabeled_data=True, use_labeled_data=True):
+    # most of the code for this function is taken from:
+    # https://github.com/mstrise/dep2label/blob/master/encoding.py
+    def pos_cluster(pos):
+        # clustering the parts of speech
+        if pos[0] == 'V':
+            pos = 'VB'
+        elif pos == 'NNS':
+            pos = 'NN'
+        elif pos == 'NNPS':
+            pos = 'NNP'
+        elif 'JJ' in pos:
+            pos = 'JJ'
+        elif pos[:2] == 'RB' or pos == 'WRB' or pos == 'RP':
+            pos = 'RB'
+        elif pos[:3] == 'PRP':
+            pos = 'PRP'
+        elif pos in ['.', ':', ',', "''", '``']:
+            pos = '.'
+        elif pos[0] == '-':
+            pos = '-RB-'
+        elif pos[:2] == 'WP':
+            pos = 'WP'
+        return pos
+
+    if src_domain == tgt_domain:
+        pred_paths = []
+        if use_unlabeled_data:
+            pred_paths = [file for file in os.listdir(parser_path) if
+                          file.endswith("pred.txt") and 'extra' in file and tgt_domain in file]
+
+        gold_paths = [file for file in os.listdir(parser_path) if
+                      file.endswith("gold.txt") and 'extra' not in file and tgt_domain in file and 'train' not in file]
+        if use_labeled_data:
+            gold_paths += [file for file in os.listdir(parser_path) if
+                           file.endswith("gold.txt") and 'extra' not in file and tgt_domain in file and 'train' in file]
+
+        if not use_unlabeled_data and not use_labeled_data:
+            raise ValueError
+    else:
+        pred_paths = [file for file in os.listdir(parser_path) if file.endswith("pred.txt") and tgt_domain in file]
+        gold_paths = []
+        if use_labeled_data:
+            gold_paths = ['data/onto_pos_ner_dp_train_' + src_domain]
+
+        if not use_unlabeled_data:
+            raise ValueError
+
+    paths = pred_paths + gold_paths
+    print("Adding labels to paths: %s" % ', '.join(paths))
+    root_line = ['0', ROOT, 'XX', 'O', '0', 'root', '+0_XX']
+    writing_paths = {}
+    sentences = {}
+    for path in paths:
+        if tgt_domain in path:
+            reading_path = parser_path + path
+            writing_path = model_path + 'parser_' + path
+            split = get_split(writing_path)
+        else:
+            reading_path = path
+            writing_path = model_path + 'parser_' + 'domain_' + src_domain + '_train_model_domain_' + src_domain + '_data_domain_' + src_domain + '_gold.txt'
+            split = 'extra_train'
+        writing_paths[split] = writing_path
+        len_sent = 0
+        tree_dict = {'0': '0'}
+        lines = []
+        sentences_list = []
+        with open(reading_path, 'r') as file:
+
+            for line in file:
+                # print(line)
+                # print(reading_path)
+                # line = line.decode('utf-8')
+                line = line.strip()
+                if len(line) == 0:
+                    for idx in range(len_sent):
+                        info_of_a_word = lines[idx]
+                        # head is on the right side from the word
+                        head = int(info_of_a_word[4]) - 1
+                        if head == -1:
+                            info_about_head = root_line
+                        else:
+                            # print(len(lines), head)
+                            info_about_head = lines[head]
+                        head_word = info_about_head[1]     
+                        lines[idx].append(head_word)  
+                        # if idx < head:
+                        #     relative_position_head = 1
+                            
+
+                        #     for x in range(idx + 1, head):
+                        #         another_word = lines[x]
+                        #         postag_word_before_head = pos_cluster(another_word[2])
+                        #         if postag_word_before_head == postag_head:
+                        #             relative_position_head += 1
+                        #     label = str(
+                        #         "+" +
+                        #         repr(relative_position_head) +
+                        #         "_" +
+                        #         postag_head)
+                            
+
+                        # # head is on the left side from the word
+                        # elif idx > head:
+                        #     relative_position_head = 1
+                        #     postag_head = pos_cluster(info_about_head[2])
+                        #     for x in range(head + 1, idx):
+                        #         another_word = lines[x]
+                        #         postag_word_before_head = pos_cluster(another_word[2])
+                        #         if postag_word_before_head == postag_head:
+                        #             relative_position_head += 1
+                        #     label = str(
+                        #         "-" +
+                        #         repr(relative_position_head) +
+                        #         "_" +
+                        #         postag_head)
+                        #     lines[idx].append(label)
+                    if len(lines) > 0:
+                        sentences_list.append(root_line)
+                    for line_ in lines:
+                        sentences_list.append(line_)
+                    sentences_list.append([])
+                    lines = []
+                    tree_dict = {'0': '0'}
+                    len_sent = 0
+                    continue
+                tokens = line.split('\t')
+                idx = tokens[0]
+                word = tokens[1]
+                pos = tokens[2]
+                ner = tokens[3]
+                head = tokens[4]
+                arc_tag = tokens[5]
+                tree_dict[idx] = head
+                lines.append([idx, word, pos, ner, head, arc_tag])
+                len_sent += 1
+        sentences[split] = sentences_list
+
+    train_sentences = []
+    if 'train' in sentences:
+        train_sentences = sentences['train']
+    else:
+        writing_paths['train'] = writing_paths['extra_train'].replace('extra_train', 'train')
+    if 'extra_train' in sentences:
+        train_sentences += sentences['extra_train']
+        del writing_paths['extra_train']
+    if 'extra_dev' in sentences:
+        train_sentences += sentences['extra_dev']
+        del writing_paths['extra_dev']
+    with open(writing_paths['train'], 'w') as f:
+        for sent in train_sentences:
+            f.write('\t'.join(sent) + '\n')
+    for split in ['dev', 'test']:
+        if split in sentences:
+            split_sentences = sentences[split]
+            with open(writing_paths[split], 'w') as f:
+                for sent in split_sentences:
+                    f.write('\t'.join(sent) + '\n')
+    return writing_paths
+import json
+def get_modified_coarse(ma):
+    ma = ma.replace('sgpl','sg').replace('sgdu','sg')
+    with open('/home/jivnesh/DCST_scratch/utils/io_/coarse_to_ma_dict.json', 'r') as fh:
+        coarse_dict = json.load(fh)
+    for key in coarse_dict.keys():
+        if ma in coarse_dict[key]:
+            return key
+def add_head_coarse_pos(model_path, parser_path, src_domain, tgt_domain, use_unlabeled_data=True, use_labeled_data=True):
+    # most of the code for this function is taken from:
+    # https://github.com/mstrise/dep2label/blob/master/encoding.py
+    def pos_cluster(pos):
+        # clustering the parts of speech
+        if pos[0] == 'V':
+            pos = 'VB'
+        elif pos == 'NNS':
+            pos = 'NN'
+        elif pos == 'NNPS':
+            pos = 'NNP'
+        elif 'JJ' in pos:
+            pos = 'JJ'
+        elif pos[:2] == 'RB' or pos == 'WRB' or pos == 'RP':
+            pos = 'RB'
+        elif pos[:3] == 'PRP':
+            pos = 'PRP'
+        elif pos in ['.', ':', ',', "''", '``']:
+            pos = '.'
+        elif pos[0] == '-':
+            pos = '-RB-'
+        elif pos[:2] == 'WP':
+            pos = 'WP'
+        return pos
+
+    if src_domain == tgt_domain:
+        pred_paths = []
+        if use_unlabeled_data:
+            pred_paths = [file for file in os.listdir(parser_path) if
+                          file.endswith("pred.txt") and 'extra' in file and tgt_domain in file]
+
+        gold_paths = [file for file in os.listdir(parser_path) if
+                      file.endswith("gold.txt") and 'extra' not in file and tgt_domain in file and 'train' not in file]
+        if use_labeled_data:
+            gold_paths += [file for file in os.listdir(parser_path) if
+                           file.endswith("gold.txt") and 'extra' not in file and tgt_domain in file and 'train' in file]
+
+        if not use_unlabeled_data and not use_labeled_data:
+            raise ValueError
+    else:
+        pred_paths = [file for file in os.listdir(parser_path) if file.endswith("pred.txt") and tgt_domain in file]
+        gold_paths = []
+        if use_labeled_data:
+            gold_paths = ['data/onto_pos_ner_dp_train_' + src_domain]
+
+        if not use_unlabeled_data:
+            raise ValueError
+
+    paths = pred_paths + gold_paths
+    print("Adding labels to paths: %s" % ', '.join(paths))
+    root_line = ['0', ROOT, 'XX', 'O', '0', 'root', 'O']
+    writing_paths = {}
+    sentences = {}
+    for path in paths:
+        if tgt_domain in path:
+            reading_path = parser_path + path
+            writing_path = model_path + 'parser_' + path
+            split = get_split(writing_path)
+        else:
+            reading_path = path
+            writing_path = model_path + 'parser_' + 'domain_' + src_domain + '_train_model_domain_' + src_domain + '_data_domain_' + src_domain + '_gold.txt'
+            split = 'extra_train'
+        writing_paths[split] = writing_path
+        len_sent = 0
+        tree_dict = {'0': '0'}
+        lines = []
+        sentences_list = []
+        with open(reading_path, 'r') as file:
+
+            for line in file:
+                # print(line)
+                # print(reading_path)
+                # line = line.decode('utf-8')
+                line = line.strip()
+                if len(line) == 0:
+                    for idx in range(len_sent):
+                        info_of_a_word = lines[idx]
+                        # head is on the right side from the word
+                        head = int(info_of_a_word[4]) - 1
+                        if head == -1:
+                            info_about_head = root_line
+                        else:
+                            # print(len(lines), head)
+                            info_about_head = lines[head]
+                        postag_head = info_about_head[2]
+                        lines[idx].append(postag_head)    
+                    if len(lines) > 0:
+                        sentences_list.append(root_line)
+                    for line_ in lines:
+                        sentences_list.append(line_)
+                    sentences_list.append([])
+                    lines = []
+                    tree_dict = {'0': '0'}
+                    len_sent = 0
+                    continue
+                tokens = line.split('\t')
+                idx = tokens[0]
+                word = tokens[1]
+                pos = tokens[2]
+                ner = tokens[3]
+                head = tokens[4]
+                arc_tag = tokens[5]
+                tree_dict[idx] = head
+                lines.append([idx, word, pos, ner, head, arc_tag])
+                len_sent += 1
+        sentences[split] = sentences_list
+
+    train_sentences = []
+    if 'train' in sentences:
+        train_sentences = sentences['train']
+    else:
+        writing_paths['train'] = writing_paths['extra_train'].replace('extra_train', 'train')
+    if 'extra_train' in sentences:
+        train_sentences += sentences['extra_train']
+        del writing_paths['extra_train']
+    if 'extra_dev' in sentences:
+        train_sentences += sentences['extra_dev']
+        del writing_paths['extra_dev']
+    with open(writing_paths['train'], 'w') as f:
+        for sent in train_sentences:
+            f.write('\t'.join(sent) + '\n')
+    for split in ['dev', 'test']:
+        if split in sentences:
+            split_sentences = sentences[split]
+            with open(writing_paths[split], 'w') as f:
+                for sent in split_sentences:
+                    f.write('\t'.join(sent) + '\n')
+    return writing_paths
+
+def add_head_ma(model_path, parser_path, src_domain, tgt_domain, use_unlabeled_data=True, use_labeled_data=True):
+    # most of the code for this function is taken from:
+    # https://github.com/mstrise/dep2label/blob/master/encoding.py
+    def pos_cluster(pos):
+        # clustering the parts of speech
+        if pos[0] == 'V':
+            pos = 'VB'
+        elif pos == 'NNS':
+            pos = 'NN'
+        elif pos == 'NNPS':
+            pos = 'NNP'
+        elif 'JJ' in pos:
+            pos = 'JJ'
+        elif pos[:2] == 'RB' or pos == 'WRB' or pos == 'RP':
+            pos = 'RB'
+        elif pos[:3] == 'PRP':
+            pos = 'PRP'
+        elif pos in ['.', ':', ',', "''", '``']:
+            pos = '.'
+        elif pos[0] == '-':
+            pos = '-RB-'
+        elif pos[:2] == 'WP':
+            pos = 'WP'
+        return pos
+
+    if src_domain == tgt_domain:
+        pred_paths = []
+        if use_unlabeled_data:
+            pred_paths = [file for file in os.listdir(parser_path) if
+                          file.endswith("pred.txt") and 'extra' in file and tgt_domain in file]
+
+        gold_paths = [file for file in os.listdir(parser_path) if
+                      file.endswith("gold.txt") and 'extra' not in file and tgt_domain in file and 'train' not in file]
+        if use_labeled_data:
+            gold_paths += [file for file in os.listdir(parser_path) if
+                           file.endswith("gold.txt") and 'extra' not in file and tgt_domain in file and 'train' in file]
+
+        if not use_unlabeled_data and not use_labeled_data:
+            raise ValueError
+    else:
+        pred_paths = [file for file in os.listdir(parser_path) if file.endswith("pred.txt") and tgt_domain in file]
+        gold_paths = []
+        if use_labeled_data:
+            gold_paths = ['data/onto_pos_ner_dp_train_' + src_domain]
+
+        if not use_unlabeled_data:
+            raise ValueError
+
+    paths = pred_paths + gold_paths
+    print("Adding labels to paths: %s" % ', '.join(paths))
+    root_line = ['0', ROOT, 'XX', 'O', '0', 'root', 'XX']
+    writing_paths = {}
+    sentences = {}
+    for path in paths:
+        if tgt_domain in path:
+            reading_path = parser_path + path
+            writing_path = model_path + 'parser_' + path
+            split = get_split(writing_path)
+        else:
+            reading_path = path
+            writing_path = model_path + 'parser_' + 'domain_' + src_domain + '_train_model_domain_' + src_domain + '_data_domain_' + src_domain + '_gold.txt'
+            split = 'extra_train'
+        writing_paths[split] = writing_path
+        len_sent = 0
+        tree_dict = {'0': '0'}
+        lines = []
+        sentences_list = []
+        with open(reading_path, 'r') as file:
+
+            for line in file:
+                # print(line)
+                # print(reading_path)
+                # line = line.decode('utf-8')
+                line = line.strip()
+                if len(line) == 0:
+                    for idx in range(len_sent):
+                        info_of_a_word = lines[idx]
+                        # head is on the right side from the word
+                        head = int(info_of_a_word[4]) - 1
+                        if head == -1:
+                            info_about_head = root_line
+                        else:
+                            # print(len(lines), head)
+                            info_about_head = lines[head]
+                        postag_head = pos_cluster(info_about_head[2])
+                        lines[idx].append(postag_head)    
+                    if len(lines) > 0:
+                        sentences_list.append(root_line)
+                    for line_ in lines:
+                        sentences_list.append(line_)
+                    sentences_list.append([])
+                    lines = []
+                    tree_dict = {'0': '0'}
+                    len_sent = 0
+                    continue
+                tokens = line.split('\t')
+                idx = tokens[0]
+                word = tokens[1]
+                pos = tokens[2]
+                ner = tokens[3]
+                head = tokens[4]
+                arc_tag = tokens[5]
+                tree_dict[idx] = head
+                lines.append([idx, word, pos, ner, head, arc_tag])
+                len_sent += 1
+        sentences[split] = sentences_list
+
+    train_sentences = []
+    if 'train' in sentences:
+        train_sentences = sentences['train']
+    else:
+        writing_paths['train'] = writing_paths['extra_train'].replace('extra_train', 'train')
+    if 'extra_train' in sentences:
+        train_sentences += sentences['extra_train']
+        del writing_paths['extra_train']
+    if 'extra_dev' in sentences:
+        train_sentences += sentences['extra_dev']
+        del writing_paths['extra_dev']
+    with open(writing_paths['train'], 'w') as f:
+        for sent in train_sentences:
+            f.write('\t'.join(sent) + '\n')
+    for split in ['dev', 'test']:
+        if split in sentences:
+            split_sentences = sentences[split]
+            with open(writing_paths[split], 'w') as f:
+                for sent in split_sentences:
+                    f.write('\t'.join(sent) + '\n')
+    return writing_paths
+
+def add_label(model_path, parser_path, src_domain, tgt_domain, use_unlabeled_data=True, use_labeled_data=True):
+    if src_domain == tgt_domain:
+        pred_paths = []
+        if use_unlabeled_data:
+            pred_paths = [file for file in os.listdir(parser_path) if
+                          file.endswith("pred.txt") and 'extra' in file and tgt_domain in file]
+
+        gold_paths = [file for file in os.listdir(parser_path) if
+                      file.endswith("gold.txt") and 'extra' not in file and tgt_domain in file and 'train' not in file]
+        if use_labeled_data:
+            gold_paths += [file for file in os.listdir(parser_path) if
+                           file.endswith("gold.txt") and 'extra' not in file and tgt_domain in file and 'train' in file]
+
+        if not use_unlabeled_data and not use_labeled_data:
+            raise ValueError
+    else:
+        pred_paths = [file for file in os.listdir(parser_path) if file.endswith("pred.txt") and tgt_domain in file]
+
+        gold_paths = []
+        if use_labeled_data:
+            gold_paths = ['data/onto_pos_ner_dp_train_' + src_domain]
+
+        if not use_unlabeled_data and not use_labeled_data:
+            raise ValueError
+
+    paths = pred_paths + gold_paths
+    print('############ Add Label Task #################')
+    print("Adding labels to paths: %s" % ', '.join(paths))
+    root_line = ['0', ROOT, 'XX', 'O', '0', 'root']
+    writing_paths = {}
+    sentences = {}
+    for path in paths:
+        if tgt_domain in path:
+            reading_path = parser_path + path
+            writing_path = model_path + 'parser_' + path
+            split = get_split(writing_path)
+        else:
+            reading_path = path
+            writing_path = model_path + 'parser_' + 'domain_' + src_domain + '_train_model_domain_' + src_domain + '_data_domain_' + src_domain + '_gold.txt'
+            split = 'extra_train'
+        writing_paths[split] = writing_path
+        len_sent = 0
+        lines = []
+        sentences_list = []
+        with open(reading_path, 'r') as file:
+            for line in file:
+                # line = line.decode('utf-8')
+                line = line.strip()
+                # Now blank space got detected
+                if len(line) == 0:
+                    # Append next word to last column
+                    for idx in range(len_sent):
+                            lines[idx].append(lines[idx][5])
+                    # Add root line first
+                    if len(lines) > 0:
+                        tmp_root_line = root_line + [root_line[5]]
+                        sentences_list.append(tmp_root_line)
+                    for line_ in lines:
+                        sentences_list.append(line_)
+                    sentences_list.append([])
+                    lines = []
+                    len_sent = 0
+                    continue
+                tokens = line.split('\t')
+                idx = tokens[0]
+                word = tokens[1]
+                pos = tokens[2]
+                ner = tokens[3]
+                head = tokens[4]
+                arc_tag = tokens[5]
+                lines.append([idx, word, pos, ner, head, arc_tag])
+                len_sent += 1
+        sentences[split] = sentences_list
+
+    train_sentences = []
+    if 'train' in sentences:
+        train_sentences = sentences['train']
+    else:
+        # pdb.set_trace()
+        writing_paths['train'] = writing_paths['extra_train'].replace('extra_train', 'train')
+    if 'extra_train' in sentences:
+        train_sentences += sentences['extra_train']
+        del writing_paths['extra_train']
+    if 'extra_dev' in sentences:
+        train_sentences += sentences['extra_dev']
+        del writing_paths['extra_dev']
+    with open(writing_paths['train'], 'w') as f:
+        for sent in train_sentences:
+            f.write('\t'.join(sent) + '\n')
+    for split in ['dev', 'test']:
+        if split in sentences:
+            split_sentences = sentences[split]
+            with open(writing_paths[split], 'w') as f:
+                for sent in split_sentences:
+                    f.write('\t'.join(sent) + '\n')
+    return writing_paths
+
+def predict_ma_tag_of_modifier(model_path, parser_path, src_domain, tgt_domain, use_unlabeled_data=True, use_labeled_data=True):
+    if src_domain == tgt_domain:
+        pred_paths = []
+        if use_unlabeled_data:
+            pred_paths = [file for file in os.listdir(parser_path) if
+                          file.endswith("pred.txt") and 'extra' in file and tgt_domain in file]
+
+        gold_paths = [file for file in os.listdir(parser_path) if
+                      file.endswith("gold.txt") and 'extra' not in file and tgt_domain in file and 'train' not in file]
+        if use_labeled_data:
+            gold_paths += [file for file in os.listdir(parser_path) if
+                           file.endswith("gold.txt") and 'extra' not in file and tgt_domain in file and 'train' in file]
+
+        if not use_unlabeled_data and not use_labeled_data:
+            raise ValueError
+    else:
+        pred_paths = [file for file in os.listdir(parser_path) if file.endswith("pred.txt") and tgt_domain in file]
+
+        gold_paths = []
+        if use_labeled_data:
+            gold_paths = ['data/onto_pos_ner_dp_train_' + src_domain]
+
+        if not use_unlabeled_data and not use_labeled_data:
+            raise ValueError
+
+    paths = pred_paths + gold_paths
+    print('############ Add Label Task #################')
+    print("Adding labels to paths: %s" % ', '.join(paths))
+    root_line = ['0', ROOT, 'XX', 'O', '0', 'root']
+    writing_paths = {}
+    sentences = {}
+    for path in paths:
+        if tgt_domain in path:
+            reading_path = parser_path + path
+            writing_path = model_path + 'parser_' + path
+            split = get_split(writing_path)
+        else:
+            reading_path = path
+            writing_path = model_path + 'parser_' + 'domain_' + src_domain + '_train_model_domain_' + src_domain + '_data_domain_' + src_domain + '_gold.txt'
+            split = 'extra_train'
+        writing_paths[split] = writing_path
+        len_sent = 0
+        lines = []
+        sentences_list = []
+        with open(reading_path, 'r') as file:
+            for line in file:
+                # line = line.decode('utf-8')
+                line = line.strip()
+                # Now blank space got detected
+                if len(line) == 0:
+                    # Append next word to last column
+                    for idx in range(len_sent):
+                            lines[idx].append(clean_ma(lines[idx][3]))
+                    # Add root line first
+                    if len(lines) > 0:
+                        tmp_root_line = root_line + [root_line[3]]
+                        sentences_list.append(tmp_root_line)
+                    for line_ in lines:
+                        sentences_list.append(line_)
+                    sentences_list.append([])
+                    lines = []
+                    len_sent = 0
+                    continue
+                tokens = line.split('\t')
+                idx = tokens[0]
+                word = tokens[1]
+                pos = tokens[2]
+                ner = tokens[3]
+                head = tokens[4]
+                arc_tag = tokens[5]
+                lines.append([idx, word, pos, ner, head, arc_tag])
+                len_sent += 1
+        sentences[split] = sentences_list
+
+    train_sentences = []
+    if 'train' in sentences:
+        train_sentences = sentences['train']
+    else:
+        writing_paths['train'] = writing_paths['extra_train'].replace('extra_train', 'train')
+    if 'extra_train' in sentences:
+        train_sentences += sentences['extra_train']
+        del writing_paths['extra_train']
+    if 'extra_dev' in sentences:
+        train_sentences += sentences['extra_dev']
+        del writing_paths['extra_dev']
+    with open(writing_paths['train'], 'w') as f:
+        for sent in train_sentences:
+            f.write('\t'.join(sent) + '\n')
+    for split in ['dev', 'test']:
+        if split in sentences:
+            split_sentences = sentences[split]
+            with open(writing_paths[split], 'w') as f:
+                for sent in split_sentences:
+                    f.write('\t'.join(sent) + '\n')
+    return writing_paths
+
+def predict_coarse_of_modifier(model_path, parser_path, src_domain, tgt_domain, use_unlabeled_data=True, use_labeled_data=True):
+    if src_domain == tgt_domain:
+        pred_paths = []
+        if use_unlabeled_data:
+            pred_paths = [file for file in os.listdir(parser_path) if
+                          file.endswith("pred.txt") and 'extra' in file and tgt_domain in file]
+
+        gold_paths = [file for file in os.listdir(parser_path) if
+                      file.endswith("gold.txt") and 'extra' not in file and tgt_domain in file and 'train' not in file]
+        if use_labeled_data:
+            gold_paths += [file for file in os.listdir(parser_path) if
+                           file.endswith("gold.txt") and 'extra' not in file and tgt_domain in file and 'train' in file]
+
+        if not use_unlabeled_data and not use_labeled_data:
+            raise ValueError
+    else:
+        pred_paths = [file for file in os.listdir(parser_path) if file.endswith("pred.txt") and tgt_domain in file]
+
+        gold_paths = []
+        if use_labeled_data:
+            gold_paths = ['data/onto_pos_ner_dp_train_' + src_domain]
+
+        if not use_unlabeled_data and not use_labeled_data:
+            raise ValueError
+
+    paths = pred_paths + gold_paths
+    print('############ Add Label Task #################')
+    print("Adding labels to paths: %s" % ', '.join(paths))
+    root_line = ['0', ROOT, 'XX', 'O', '0', 'root']
+    writing_paths = {}
+    sentences = {}
+    for path in paths:
+        if tgt_domain in path:
+            reading_path = parser_path + path
+            writing_path = model_path + 'parser_' + path
+            split = get_split(writing_path)
+        else:
+            reading_path = path
+            writing_path = model_path + 'parser_' + 'domain_' + src_domain + '_train_model_domain_' + src_domain + '_data_domain_' + src_domain + '_gold.txt'
+            split = 'extra_train'
+        writing_paths[split] = writing_path
+        len_sent = 0
+        lines = []
+        sentences_list = []
+        with open(reading_path, 'r') as file:
+            for line in file:
+                # line = line.decode('utf-8')
+                line = line.strip()
+                # Now blank space got detected
+                if len(line) == 0:
+                    # Append next word to last column
+                    for idx in range(len_sent):
+                            lines[idx].append(lines[idx][3])
+                    # Add root line first
+                    if len(lines) > 0:
+                        tmp_root_line = root_line + [root_line[3]]
+                        sentences_list.append(tmp_root_line)
+                    for line_ in lines:
+                        sentences_list.append(line_)
+                    sentences_list.append([])
+                    lines = []
+                    len_sent = 0
+                    continue
+                tokens = line.split('\t')
+                idx = tokens[0]
+                word = tokens[1]
+                pos = tokens[2]
+                ner = tokens[3]
+                head = tokens[4]
+                arc_tag = tokens[5]
+                lines.append([idx, word, pos, ner, head, arc_tag])
+                len_sent += 1
+        sentences[split] = sentences_list
+
+    train_sentences = []
+    if 'train' in sentences:
+        train_sentences = sentences['train']
+    else:
+        writing_paths['train'] = writing_paths['extra_train'].replace('extra_train', 'train')
+    if 'extra_train' in sentences:
+        train_sentences += sentences['extra_train']
+        del writing_paths['extra_train']
+    if 'extra_dev' in sentences:
+        train_sentences += sentences['extra_dev']
+        del writing_paths['extra_dev']
+    with open(writing_paths['train'], 'w') as f:
+        for sent in train_sentences:
+            f.write('\t'.join(sent) + '\n')
+    for split in ['dev', 'test']:
+        if split in sentences:
+            split_sentences = sentences[split]
+            with open(writing_paths[split], 'w') as f:
+                for sent in split_sentences:
+                    f.write('\t'.join(sent) + '\n')
+    return writing_paths
+import re
+def get_case(ma):
+        indeclinable = ['ind','prep','interj','prep','conj','part']
+        case_list = ['nom','voc','acc','i','inst','dat','abl','g','loc']
+        gender_list = ['n','f','m','*']
+        person_list = ['1','2','3']
+        no_list = ['du','sg','pl']
+        pops = [' ac',' ps']
+        ma=ma.replace('sgpl','sg').replace('sgdu','sg')
+        temp = re.sub("([\(\[]).*?([\)\]])", "\g<1>\g<2>", ma).replace('[] ','').strip(' []')
+        temp = temp.split('.')
+        if temp[-1] == '':
+            temp.pop(-1)
+        # Remove active passive
+        case=''
+        no=''
+        person=''
+        gender=''
+        tense=''
+        coarse=''
+        for a,b in enumerate(temp):
+            if b in pops:
+                temp.pop(a)
+        # Get gender
+        for a,b in enumerate(temp):
+            if b.strip() in gender_list:
+                gender = b.strip()
+                temp.pop(a)
+        # Get case
+        for a,b in enumerate(temp):
+            if b.strip() in case_list:
+                case = b.strip()
+                temp.pop(a)
+        if case!= '':
+            coarse ='Noun'
+        # Get person
+        for a,b in enumerate(temp):
+            if b.strip() in person_list:
+                person = b.strip()
+                temp.pop(a)
+        # Get no
+        for a,b in enumerate(temp):
+            if b.strip() in no_list:
+                no = b.strip()
+                temp.pop(a)
+        # Get Tense
+        for b in temp:
+            tense=tense+ ' '+b.strip()
+        tense=tense.strip()
+        
+#         print(tense)
+        if tense == 'adv':
+            coarse = 'adv'
+        for ind in indeclinable:
+            if tense == ind:
+                coarse = 'Ind'
+        if tense == 'abs' or tense == 'ca abs':
+            coarse = 'IV'
+        if tense!='' and coarse=='':
+            if person !='' or no!='':
+                coarse= 'FV'
+            else:
+                coarse = 'IV'
+        if case == 'i':
+            return 'inst'
+        
+        if case !='':
+            return case
+        else:
+            return coarse
+def clean_ma(ma):
+    ma = re.sub("([\(\[]).*?([\)\]])", "\g<1>\g<2>", ma).replace('[] ','').strip(' []').replace(' ac','').replace(' ps','').replace('sgpl','sg').replace('sgdu','sg')
+    ma = ma.replace('i.','inst.').replace('.','').replace(' ','')
+    return ma
+def predict_case_of_modifier(model_path, parser_path, src_domain, tgt_domain, use_unlabeled_data=True, use_labeled_data=True):
+    if src_domain == tgt_domain:
+        pred_paths = []
+        if use_unlabeled_data:
+            pred_paths = [file for file in os.listdir(parser_path) if
+                          file.endswith("pred.txt") and 'extra' in file and tgt_domain in file]
+
+        gold_paths = [file for file in os.listdir(parser_path) if
+                      file.endswith("gold.txt") and 'extra' not in file and tgt_domain in file and 'train' not in file]
+        if use_labeled_data:
+            gold_paths += [file for file in os.listdir(parser_path) if
+                           file.endswith("gold.txt") and 'extra' not in file and tgt_domain in file and 'train' in file]
+
+        if not use_unlabeled_data and not use_labeled_data:
+            raise ValueError
+    else:
+        pred_paths = [file for file in os.listdir(parser_path) if file.endswith("pred.txt") and tgt_domain in file]
+
+        gold_paths = []
+        if use_labeled_data:
+            gold_paths = ['data/onto_pos_ner_dp_train_' + src_domain]
+
+        if not use_unlabeled_data and not use_labeled_data:
+            raise ValueError
+
+    paths = pred_paths + gold_paths
+    print('############ Add Label Task #################')
+    print("Adding labels to paths: %s" % ', '.join(paths))
+    root_line = ['0', ROOT, 'XX', 'O', '0', 'root']
+    writing_paths = {}
+    sentences = {}
+    for path in paths:
+        if tgt_domain in path:
+            reading_path = parser_path + path
+            writing_path = model_path + 'parser_' + path
+            split = get_split(writing_path)
+        else:
+            reading_path = path
+            writing_path = model_path + 'parser_' + 'domain_' + src_domain + '_train_model_domain_' + src_domain + '_data_domain_' + src_domain + '_gold.txt'
+            split = 'extra_train'
+        writing_paths[split] = writing_path
+        len_sent = 0
+        lines = []
+        sentences_list = []
+        with open(reading_path, 'r') as file:
+            for line in file:
+                # line = line.decode('utf-8')
+                line = line.strip()
+                # Now blank space got detected
+                if len(line) == 0:
+                    # Append next word to last column
+                    for idx in range(len_sent):
+                            lines[idx].append(get_case(lines[idx][3]))
+                    # Add root line first
+                    if len(lines) > 0:
+                        tmp_root_line = root_line + [root_line[3]]
+                        sentences_list.append(tmp_root_line)
+                    for line_ in lines:
+                        sentences_list.append(line_)
+                    sentences_list.append([])
+                    lines = []
+                    len_sent = 0
+                    continue
+                tokens = line.split('\t')
+                idx = tokens[0]
+                word = tokens[1]
+                pos = tokens[2]
+                ner = tokens[3]
+                head = tokens[4]
+                arc_tag = tokens[5]
+                lines.append([idx, word, pos, ner, head, arc_tag])
+                len_sent += 1
+        sentences[split] = sentences_list
+
+    train_sentences = []
+    if 'train' in sentences:
+        train_sentences = sentences['train']
+    else:
+        writing_paths['train'] = writing_paths['extra_train'].replace('extra_train', 'train')
+    if 'extra_train' in sentences:
+        train_sentences += sentences['extra_train']
+        del writing_paths['extra_train']
+    if 'extra_dev' in sentences:
+        train_sentences += sentences['extra_dev']
+        del writing_paths['extra_dev']
+    with open(writing_paths['train'], 'w') as f:
+        for sent in train_sentences:
+            f.write('\t'.join(sent) + '\n')
+    for split in ['dev', 'test']:
+        if split in sentences:
+            split_sentences = sentences[split]
+            with open(writing_paths[split], 'w') as f:
+                for sent in split_sentences:
+                    f.write('\t'.join(sent) + '\n')
+    return writing_paths
+
+def Multitask_case_predict(model_path, parser_path, src_domain, tgt_domain, use_unlabeled_data=True, use_labeled_data=True):
+    writing_paths = {}
+    # multitask_silver_20ktrain_san 
+    writing_paths['train'] = 'data/ud_pos_ner_dp_train_san_case'
+    writing_paths['dev'] = 'data/ud_pos_ner_dp_dev_san_case'
+    writing_paths['test'] = 'data/ud_pos_ner_dp_test_san_case'
+    return writing_paths
+
+def Multitask_POS_predict(model_path, parser_path, src_domain, tgt_domain, use_unlabeled_data=True, use_labeled_data=True):
+    writing_paths = {}
+    # multitask_silver_20ktrain_san
+    # writing_paths['train'] = 'data/Multitask_POS_predict_train_san'
+    # writing_paths['dev'] = 'data/Multitask_POS_predict_dev_san'
+    # writing_paths['test'] = 'data/Multitask_POS_predict_test_san'
+    writing_paths['train'] = 'data/ud_pos_ner_dp_train_san_POS'
+    writing_paths['dev'] = 'data/ud_pos_ner_dp_dev_san_POS'
+    writing_paths['test'] = 'data/ud_pos_ner_dp_test_san_POS'
+    return writing_paths
+
+def Multitask_coarse_predict(model_path, parser_path, src_domain, tgt_domain, use_unlabeled_data=True, use_labeled_data=True):
+    writing_paths = {}
+    # multitask_silver_20ktrain_san
+    writing_paths['train'] = 'data/Multitask_coarse_predict_train_san'
+    writing_paths['dev'] = 'data/Multitask_coarse_predict_dev_san'
+    writing_paths['test'] = 'data/Multitask_coarse_predict_test_san'
+    return writing_paths
+
+def Multitask_label_predict(model_path, parser_path, src_domain, tgt_domain, use_unlabeled_data=True, use_labeled_data=True):
+    writing_paths = {}
+    # multitask_silver_20ktrain_san
+    writing_paths['train'] = 'data/Multitask_label_predict_train_san'
+    writing_paths['dev'] = 'data/Multitask_label_predict_dev_san'
+    writing_paths['test'] = 'data/Multitask_label_predict_test_san'
+    return writing_paths
+
+
+def MRL_case(model_path, parser_path, src_domain, tgt_domain, use_unlabeled_data=True, use_labeled_data=True):
+    writing_paths = {}
+    # multitask_silver_20ktrain_san
+    writing_paths['train'] = 'data/Prep_MRL/ud_pos_ner_dp_train_'+src_domain+'_case'
+    writing_paths['dev'] = 'data/Prep_MRL/ud_pos_ner_dp_dev_'+src_domain+'_case'
+    writing_paths['test'] = 'data/Prep_MRL/ud_pos_ner_dp_test_'+src_domain+'_case'
+    return writing_paths
+
+def MRL_POS(model_path, parser_path, src_domain, tgt_domain, use_unlabeled_data=True, use_labeled_data=True):
+    writing_paths = {}
+    # multitask_silver_20ktrain_san
+    writing_paths['train'] = 'data/Prep_MRL/ud_pos_ner_dp_train_'+src_domain+'_POS'
+    writing_paths['dev'] = 'data/Prep_MRL/ud_pos_ner_dp_dev_'+src_domain+'_POS'
+    writing_paths['test'] = 'data/Prep_MRL/ud_pos_ner_dp_test_'+src_domain+'_POS'
+    return writing_paths
+
+def MRL_label(model_path, parser_path, src_domain, tgt_domain, use_unlabeled_data=True, use_labeled_data=True):
+    writing_paths = {}
+    # multitask_silver_20ktrain_san
+    writing_paths['train'] = 'data/Prep_MRL/ud_pos_ner_dp_train_'+src_domain+'_dep'
+    writing_paths['dev'] = 'data/Prep_MRL/ud_pos_ner_dp_dev_'+src_domain+'_dep'
+    writing_paths['test'] = 'data/Prep_MRL/ud_pos_ner_dp_test_'+src_domain+'_dep'
+    return writing_paths
+
+def MRL_no(model_path, parser_path, src_domain, tgt_domain, use_unlabeled_data=True, use_labeled_data=True):
+    writing_paths = {}
+    # multitask_silver_20ktrain_san
+    writing_paths['train'] = 'data/Prep_MRL/ud_pos_ner_dp_train_'+src_domain+'_no'
+    writing_paths['dev'] = 'data/Prep_MRL/ud_pos_ner_dp_dev_'+src_domain+'_no'
+    writing_paths['test'] = 'data/Prep_MRL/ud_pos_ner_dp_test_'+src_domain+'_no'
+    return writing_paths
+
+def MRL_Person(model_path, parser_path, src_domain, tgt_domain, use_unlabeled_data=True, use_labeled_data=True):
+    writing_paths = {}
+    # multitask_silver_20ktrain_san
+    writing_paths['train'] = 'data/Prep_MRL/ud_pos_ner_dp_train_'+src_domain+'_per'
+    writing_paths['dev'] = 'data/Prep_MRL/ud_pos_ner_dp_dev_'+src_domain+'_per'
+    writing_paths['test'] = 'data/Prep_MRL/ud_pos_ner_dp_test_'+src_domain+'_per'
+    return writing_paths
+def MRL_Gender(model_path, parser_path, src_domain, tgt_domain, use_unlabeled_data=True, use_labeled_data=True):
+    writing_paths = {}
+    # multitask_silver_20ktrain_san
+    writing_paths['train'] = 'data/Prep_MRL/ud_pos_ner_dp_train_'+src_domain+'_gen'
+    writing_paths['dev'] = 'data/Prep_MRL/ud_pos_ner_dp_dev_'+src_domain+'_gen'
+    writing_paths['test'] = 'data/Prep_MRL/ud_pos_ner_dp_test_'+src_domain+'_gen'
+    return writing_paths

utils/io_/writer.py

@@ -0,0 +1,46 @@
+
+class Writer(object):
+    def __init__(self, alphabets):
+        self.__source_file = None
+        self.alphabets = alphabets
+
+    def start(self, file_path):
+        self.__source_file = open(file_path, 'w')
+
+    def close(self):
+        self.__source_file.close()
+
+    def write(self, word, pos, ner, head, arc, lengths, auto_label=None, symbolic_root=False, symbolic_end=False):
+        batch_size, _ = word.shape
+        start = 1 if symbolic_root else 0
+        end = 1 if symbolic_end else 0
+        for i in range(batch_size):
+            for j in range(start, lengths[i] - end):
+                w = self.alphabets['word_alphabet'].get_instance(word[i, j])
+                p = self.alphabets['pos_alphabet'].get_instance(pos[i, j])
+                n = self.alphabets['ner_alphabet'].get_instance(ner[i, j])
+                t = self.alphabets['arc_alphabet'].get_instance(arc[i, j])
+                h = head[i, j]
+                if auto_label is not None:
+                    m = self.alphabets['auto_label_alphabet'].get_instance(auto_label[i, j])
+                    self.__source_file.write('%d\t%s\t%s\t%s\t%d\t%s\t%s\n' % (j, w, p, n, h, t, m))
+                else:
+                    self.__source_file.write('%d\t%s\t%s\t%s\t%d\t%s\n' % (j, w, p, n, h, t))
+            self.__source_file.write('\n')
+
+class Index2Instance(object):
+    def __init__(self, alphabet):
+        self.__alphabet = alphabet
+
+    def index2instance(self, indices, lengths, symbolic_root=False, symbolic_end=False):
+        batch_size, _ = indices.shape
+        start = 1 if symbolic_root else 0
+        end = 1 if symbolic_end else 0
+        instnaces = []
+        for i in range(batch_size):
+            tmp_instances = []
+            for j in range(start, lengths[i] - end):
+                instamce = self.__alphabet.get_instance(indices[i, j])
+                tmp_instances.append(instamce)
+            instnaces.append(tmp_instances)
+        return instnaces

utils/load_word_embeddings.py

@@ -0,0 +1,123 @@
+import pickle
+import numpy as np
+from gensim.models import KeyedVectors
+import gzip
+import io
+import os
+
+def calc_mean_vec_for_lower_mapping(embedd_dict):
+    lower_counts = {}
+    for word in embedd_dict:
+        word_lower = word.lower()
+        if word_lower not in lower_counts:
+            lower_counts[word_lower] = [word]
+        else:
+            lower_counts[word_lower] = lower_counts[word_lower] + [word]
+    # calculating mean vector for all words that have the same mapping after performing lower()
+    for word in lower_counts:
+        embedd_dict[word] = np.mean([embedd_dict[word_] for word_ in lower_counts[word]])
+    return embedd_dict
+
+def load_embedding_dict(embedding, embedding_path, lower_case=False):
+    """
+    load word embeddings from file
+    :param embedding:
+    :param embedding_path:
+    :return: embedding dict, embedding dimention, caseless
+    """
+    print("loading embedding: %s from %s" % (embedding, embedding_path))
+    if lower_case:
+        pkl_path = embedding_path + '_lower' + '.pkl'
+    else:
+        pkl_path = embedding_path + '.pkl'
+    if os.path.isfile(pkl_path):
+        # load dict and dim from a pickle file
+        with open(pkl_path, 'rb') as f:
+            embedd_dict, embedd_dim = pickle.load(f)
+        print("num dimensions of word embeddings:", embedd_dim)
+        return embedd_dict, embedd_dim
+
+    if embedding == 'glove':
+        # loading GloVe
+        embedd_dict = {}
+        word = None
+        with io.open(embedding_path, 'r', encoding='utf-8') as f:
+            for line in f:
+                word, vec = line.split(' ', 1)
+                embedd_dict[word] = np.fromstring(vec, sep=' ')
+        embedd_dim = len(embedd_dict[word])
+        if lower_case:
+            embedd_dict = calc_mean_vec_for_lower_mapping(embedd_dict)
+        for k, v in embedd_dict.items():
+            if len(v) != embedd_dim:
+                print(len(v),embedd_dim)
+
+    elif embedding == 'fasttext':
+        # loading GloVe
+        embedd_dict = {}
+        word = None
+        with io.open(embedding_path, 'r', encoding='utf-8') as f:
+            # skip first line
+            for i, line in enumerate(f):
+                if i == 0:
+                    continue
+                word, vec = line.split(' ', 1)
+                embedd_dict[word] = np.fromstring(vec, sep=' ')
+        embedd_dim = len(embedd_dict[word])
+        if lower_case:
+            embedd_dict = calc_mean_vec_for_lower_mapping(embedd_dict)
+        for k, v in embedd_dict.items():
+            if len(v) != embedd_dim:
+                print(len(v),embedd_dim)
+
+    elif embedding == 'hellwig':
+        # loading hellwig
+        embedd_dict = {}
+        word = None
+        with io.open(embedding_path, 'r', encoding='utf-8') as f:
+            # skip first line
+            for i, line in enumerate(f):
+                if i == 0:
+                    continue
+                word, vec = line.split(' ', 1)
+                embedd_dict[word] = np.fromstring(vec, sep=' ')
+        embedd_dim = len(embedd_dict[word])
+        if lower_case:
+            embedd_dict = calc_mean_vec_for_lower_mapping(embedd_dict)
+        for k, v in embedd_dict.items():
+            if len(v) != embedd_dim:
+                print(len(v),embedd_dim)
+
+    elif embedding == 'one_hot':
+        # loading hellwig
+        embedd_dict = {}
+        word = None
+        with io.open(embedding_path, 'r', encoding='utf-8') as f:
+            # skip first line
+            for i, line in enumerate(f):
+                if i == 0:
+                    continue
+                word, vec = line.split('@', 1)
+                embedd_dict[word] = np.fromstring(vec, sep=' ')
+        embedd_dim = len(embedd_dict[word])
+        if lower_case:
+            embedd_dict = calc_mean_vec_for_lower_mapping(embedd_dict)
+        for k, v in embedd_dict.items():
+            if len(v) != embedd_dim:
+                print(len(v),embedd_dim)
+
+    elif embedding == 'word2vec':
+        # loading word2vec
+        embedd_dict = KeyedVectors.load_word2vec_format(embedding_path, binary=True)
+        if lower_case:
+            embedd_dict = calc_mean_vec_for_lower_mapping(embedd_dict)
+        embedd_dim = embedd_dict.vector_size
+
+    else:
+        raise ValueError("embedding should choose from [fasttext, glove, word2vec]")
+
+    print("num dimensions of word embeddings:", embedd_dim)
+    # save dict and dim to a pickle file
+    with open(pkl_path, 'wb') as f:
+        pickle.dump([embedd_dict, embedd_dim], f, pickle.HIGHEST_PROTOCOL)
+    return embedd_dict, embedd_dim

utils/models/__init__.py

@@ -0,0 +1,3 @@
+from .parsing import *
+from .parsing_gating import *
+from .sequence_tagger import *