fixing stronger

# Conflicts:
#	pytorch_transformers/__init__.py
#	transformers/tests/modeling_bert_test.py
#	transformers/tests/modeling_roberta_test.py

examples/README.md

@@ -427,6 +427,76 @@ exact_match = 86.91
 This fine-tuned model is available as a checkpoint under the reference
 `bert-large-uncased-whole-word-masking-finetuned-squad`.
 
+### `run_semeval.py`: relationship classification using R-Bert
+
+R-BERT is a relationship classification head for BERT and RoBERTa, described [here](https://arxiv.org/pdf/1905.08284.pdf").
+
+This example code fine-tunes R-BERT on the semeval 2010 Task 8 dataset:
+ 
+```bash
+python ./examples/run_semeval.py \
+--data_dir $SEMEVAL_DIR \
+--output_dir $RESULTS_DIR \
+--model_name_or_path bert-base-uncased \
+--do_train \
+--do_eval \
+--overwrite_output_dir \
+--num_train_epochs 8.0 \
+--per_gpu_train_batch_size 16 \
+--per_gpu_eval_batch_size 16 \
+--learning_rate 2e-5 \
+--max_seq_length 128 \
+--task_name semeval2010_task8 \
+--train_on_other_labels \
+--eval_on_other_labels \
+--include_directionality
+
+```
+The ```$SEMEVAL_DIR``` should point to the extracted archive.
+
+The ```--include_directionality``` flag trains a classifier using all 18 semeval classes. The 
+```--train_on_other_labels``` and ```--eval_on_other_labels``` flags also include instances labeled as 'Other' in the 
+training and evaluation respectively. Include all of these to be able to use the official evaluation script. 
+
+Note, although an F1 score is calculated in the python code, two additional files are also written out at the checkpoint 
+intervals ```{global_step}_semeval_results.tsv``` that may be used with the official Semeval evaluation script 
+(supplied in the semeval data archive). The dataset is available under Creative Commons Atrribution 3.0 
+Unported Licence (http://creativecommons.org/licenses/by/3.0/) and is available 
+[here](http://docs.google.com/leaf?id=0B_jQiLugGTAkMDQ5ZjZiMTUtMzQ1Yy00YWNmLWJlZDYtOWY1ZDMwY2U4YjFk&sort=name&layout=list&num=50). 
+
+for example:
+
+```bash
+./semeval2010_task8_scorer-v1.2.pl $SEMEVAL_DIR/{global_step}_semeval_results.tsv $SEMEVAL_DIR/TEST_FILE_SEMEVAL_SCRIPT_FORMAT.tsv 
+```
+
+
+Using the ```bert-base-uncased``` should give around 87.6 undirected F1 (excluding other), with the official script. 
+However, the RoBERTa model can also be used with this head:
+
+```bash
+python ./examples/run_semeval.py \
+--data_dir $SEMEVAL_DIR \
+--output_dir $RESULTS_DIR \
+--model_name_or_path roberta-large \
+--model_type roberta \
+--do_train \
+--do_eval \
+--overwrite_output_dir \
+--num_train_epochs 8.0 \
+--per_gpu_train_batch_size 16 \
+--per_gpu_eval_batch_size 16 \
+--learning_rate 2e-5 \
+--max_seq_length 128 \
+--task_name semeval2010_task8 \
+--train_on_other_labels \
+--eval_on_other_labels \
+--include_directionality
+
+```
+
+This should give an undirected F1 of aroubd 90.8 (excluding other) with the official script.
+
 #### Fine-tuning XLNet on SQuAD
 
 This example code fine-tunes XLNet on the SQuAD dataset. See above to download the data for SQuAD .

examples/run_semeval.py

@@ -0,0 +1,483 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Finetuning the library models for sequence classification on Semeval (Bert)."""
+
+from __future__ import absolute_import, division, print_function
+
+import argparse
+import glob
+import logging
+import os
+import random
+
+import numpy as np
+import torch
+from tensorboardX import SummaryWriter
+from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler)
+from torch.utils.data.distributed import DistributedSampler
+from tqdm import tqdm, trange
+
+from transformers import AdamW, WarmupLinearSchedule
+from transformers import (WEIGHTS_NAME, BertConfig, RobertaConfig,
+                          BertForRelationshipClassification, RobertaForRelationshipClassification,
+                          BertTokenizer, RobertaTokenizer)
+from examples.utils_semeval import (processors, output_modes, convert_examples_to_features,
+                                    convert_features_to_dataset, compute_metrics, RBertUtils,
+                                    get_entity_seperator_token_ids)
+
+logger = logging.getLogger(__name__)
+
+MODEL_CLASSES = {
+    'bert': (BertConfig, BertForRelationshipClassification, BertTokenizer),
+    'roberta': (RobertaConfig, RobertaForRelationshipClassification, RobertaTokenizer)
+}
+
+RBERT_UTILS = RBertUtils()
+
+def set_seed(args):
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    torch.manual_seed(args.seed)
+    if args.n_gpu > 0:
+        torch.cuda.manual_seed_all(args.seed)
+
+
+def train(args, train_dataset, model, tokenizer):
+    """ Train the model """
+    if args.local_rank in [-1, 0]:
+        tb_writer = SummaryWriter()
+
+    args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu)
+    train_sampler = RandomSampler(train_dataset) if args.local_rank == -1 else DistributedSampler(train_dataset)
+    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size)
+
+    if args.max_steps > 0:
+        t_total = args.max_steps
+        args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1
+    else:
+        t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs
+
+    # Prepare optimizer and schedule (linear warmup and decay)
+    no_decay = ['bias', 'LayerNorm.weight']
+    optimizer_grouped_parameters = [
+        {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
+         'weight_decay': args.weight_decay},
+        {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
+    ]
+    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
+    scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total)
+    if args.fp16:
+        try:
+            from apex import amp
+        except ImportError:
+            raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
+        model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level)
+
+    # multi-gpu training (should be after apex fp16 initialization)
+    if args.n_gpu > 1:
+        model = torch.nn.DataParallel(model)
+
+    # Distributed training (should be after apex fp16 initialization)
+    if args.local_rank != -1:
+        model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank],
+                                                          output_device=args.local_rank,
+                                                          find_unused_parameters=True)
+
+    # Train!
+    logger.info("***** Running training *****")
+    logger.info("  Num examples = %d", len(train_dataset))
+    logger.info("  Num Epochs = %d", args.num_train_epochs)
+    logger.info("  Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size)
+    logger.info("  Total train batch size (w. parallel, distributed & accumulation) = %d",
+                args.train_batch_size * args.gradient_accumulation_steps * (
+                    torch.distributed.get_world_size() if args.local_rank != -1 else 1))
+    logger.info("  Gradient Accumulation steps = %d", args.gradient_accumulation_steps)
+    logger.info("  Total optimization steps = %d", t_total)
+
+    global_step = 0
+    tr_loss, logging_loss = 0.0, 0.0
+    model.zero_grad()
+    train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0])
+    set_seed(args)  # Added here for reproductibility (even between python 2 and 3)
+    for _ in train_iterator:
+        epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0])
+        for step, batch in enumerate(epoch_iterator):
+            model.train()
+            batch = tuple(t.to(args.device) for t in batch)
+            inputs = {'input_ids': batch[0],
+                      'attention_mask': batch[1],
+                      'token_type_ids': batch[2],
+                      'labels': batch[3]}
+            outputs = model(**inputs)
+            loss = outputs[0]  # model outputs are always tuple in pytorch-transformers (see doc)
+
+            if args.n_gpu > 1:
+                loss = loss.mean()  # mean() to average on multi-gpu parallel training
+            if args.gradient_accumulation_steps > 1:
+                loss = loss / args.gradient_accumulation_steps
+
+            if args.fp16:
+                with amp.scale_loss(loss, optimizer) as scaled_loss:
+                    scaled_loss.backward()
+                torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)
+            else:
+                loss.backward()
+                torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
+
+            tr_loss += loss.item()
+            if (step + 1) % args.gradient_accumulation_steps == 0:
+                scheduler.step()  # Update learning rate schedule
+                optimizer.step()
+                model.zero_grad()
+                global_step += 1
+
+                if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0:
+                    # Log metrics
+                    if args.local_rank == -1 and args.evaluate_during_training:  # Only evaluate when single GPU otherwise metrics may not average well
+                        results = evaluate(args, model, tokenizer)
+                        for key, value in results.items():
+                            tb_writer.add_scalar('eval_{}'.format(key), value, global_step)
+                    tb_writer.add_scalar('lr', scheduler.get_lr()[0], global_step)
+                    tb_writer.add_scalar('loss', (tr_loss - logging_loss) / args.logging_steps, global_step)
+                    logging_loss = tr_loss
+
+                if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0:
+                    # Save model checkpoint
+                    output_dir = os.path.join(args.output_dir, 'checkpoint-{}'.format(global_step))
+                    if not os.path.exists(output_dir):
+                        os.makedirs(output_dir)
+                    model_to_save = model.module if hasattr(model,
+                                                            'module') else model  # Take care of distributed/parallel training
+                    model_to_save.save_pretrained(output_dir)
+                    torch.save(args, os.path.join(output_dir, 'training_args.bin'))
+                    logger.info("Saving model checkpoint to %s", output_dir)
+
+            if args.max_steps > 0 and global_step > args.max_steps:
+                epoch_iterator.close()
+                break
+        if args.max_steps > 0 and global_step > args.max_steps:
+            train_iterator.close()
+            break
+
+    if args.local_rank in [-1, 0]:
+        tb_writer.close()
+
+    return global_step, tr_loss / global_step
+
+
+def evaluate(args, model, eval_dataset, prefix=""):
+    eval_task = args.task_name
+    eval_output_dir = args.output_dir
+
+    results = {}
+
+    if not os.path.exists(eval_output_dir) and args.local_rank in [-1, 0]:
+        os.makedirs(eval_output_dir)
+
+    args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)
+    # Note that DistributedSampler samples randomly
+    eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset)
+    eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)
+
+    # Eval!
+    logger.info("***** Running evaluation {} *****".format(prefix))
+    logger.info("  Num examples = %d", len(eval_dataset))
+    logger.info("  Batch size = %d", args.eval_batch_size)
+    eval_loss = 0.0
+    nb_eval_steps = 0
+    preds = None
+    out_label_ids = None
+    out_instance_ids = None
+    for batch in tqdm(eval_dataloader, desc="Evaluating"):
+        model.eval()
+        batch = tuple(t.to(args.device) for t in batch)
+
+        with torch.no_grad():
+            inputs = {'input_ids': batch[0],
+                      'attention_mask': batch[1],
+                      'token_type_ids': batch[2],
+                      'labels': batch[3]}
+            outputs = model(**inputs)
+            tmp_eval_loss, logits = outputs[:2]
+
+            eval_loss += tmp_eval_loss.mean().item()
+        nb_eval_steps += 1
+
+        if preds is None:
+            preds = logits.detach().cpu().numpy()
+            out_label_ids = inputs['labels'].detach().cpu().numpy()
+            out_instance_ids = batch[4].detach().cpu().numpy()
+        else:
+            preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)
+            out_label_ids = np.append(out_label_ids, inputs['labels'].detach().cpu().numpy(), axis=0)
+            out_instance_ids = np.append(out_instance_ids, batch[4].detach().cpu().numpy(), axis=0)
+
+    eval_loss = eval_loss / nb_eval_steps
+    preds = np.argmax(preds, axis=1)
+    result = compute_metrics(eval_task, preds, out_label_ids)
+    results.update(result)
+
+    output_eval_file = os.path.join(eval_output_dir, "eval_results.txt")
+    with open(output_eval_file, "w") as writer:
+        logger.info("***** Eval results {} *****".format(prefix))
+        for key in sorted(result.keys()):
+            logger.info("  %s = %s", key, str(result[key]))
+            writer.write("%s = %s\n" % (key, str(result[key])))
+    return results, np.stack([out_instance_ids,preds],axis=1)
+
+
+def load_and_cache_examples(args, task, tokenizer, evaluate=False):
+    if args.local_rank not in [-1, 0] and not evaluate:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache
+
+    processor = processors[task](include_directionality=args.include_directionality)
+    output_mode = output_modes[task]
+    # Load data features from cache or dataset file
+    cached_features_file = os.path.join(args.data_dir, 'cached_{}_{}_{}_{}'.format(
+        'dev' if evaluate else 'train',
+        list(filter(None, args.model_name_or_path.split('/'))).pop(),
+        str(args.max_seq_length),
+        str(task)))
+    if os.path.exists(cached_features_file):
+        logger.info("Loading features from cached file %s", cached_features_file)
+        features = torch.load(cached_features_file)
+    else:
+        logger.info("Creating features from dataset file at %s", args.data_dir)
+        label_list = processor.get_labels()
+        if evaluate:
+            examples = list(processor.get_dev_examples(args.data_dir,include_other=args.eval_on_other_labels))
+            # so that the official scoring script can be used
+            processor.write_dev_examples_to_official_format(args.output_dir,examples=examples)
+        else:
+            examples = list(processor.get_train_examples(args.data_dir,include_other=args.train_on_other_labels))
+        features = convert_examples_to_features(examples, label_list,args.max_seq_length, tokenizer,RBERT_UTILS)
+        if args.local_rank in [-1, 0]:
+            logger.info("Saving features into cached file %s", cached_features_file)
+            torch.save(features, cached_features_file)
+
+    if args.local_rank == 0 and not evaluate:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache
+
+    # Convert to Tensors and build dataset
+    dataset = convert_features_to_dataset(features, output_mode=output_mode)
+    return dataset
+
+
+def main():
+    parser = argparse.ArgumentParser()
+
+    ## Required parameters
+    parser.add_argument("--data_dir", default=None, type=str, required=True,
+                        help="The input data dir. Should contain the .tsv files (or other data files) for the task.")
+    parser.add_argument("--model_name_or_path", default='bert-base-uncased', type=str)
+    parser.add_argument("--model_type", default='bert', type=str)
+    parser.add_argument("--output_dir", default=None, type=str, required=True,
+                        help="The output directory where the model predictions and checkpoints will be written.")
+    parser.add_argument("--task_name", default='semeval2010_task8', type=str)
+    ## Other parameters
+    parser.add_argument("--config_name", default="", type=str,
+                        help="Pretrained config name or path if not the same as model_name")
+    parser.add_argument("--tokenizer_name", default="", type=str,
+                        help="Pretrained tokenizer name or path if not the same as model_name")
+    parser.add_argument("--cache_dir", default="", type=str,
+                        help="Where do you want to store the pre-trained models downloaded from s3")
+    parser.add_argument("--max_seq_length", default=128, type=int,
+                        help="The maximum total input sequence length after tokenization. Sequences longer "
+                             "than this will be truncated, sequences shorter will be padded.")
+    parser.add_argument("--do_train", action='store_true',
+                        help="Whether to run training.")
+    parser.add_argument("--do_eval", action='store_true',
+                        help="Whether to run eval on the dev set.")
+    parser.add_argument("--evaluate_during_training", action='store_true',
+                        help="Rul evaluation during training at each logging step.")
+    parser.add_argument("--do_lower_case", action='store_true',
+                        help="Set this flag if you are using an uncased model.")
+
+    parser.add_argument("--per_gpu_train_batch_size", default=8, type=int,
+                        help="Batch size per GPU/CPU for training.")
+    parser.add_argument("--per_gpu_eval_batch_size", default=8, type=int,
+                        help="Batch size per GPU/CPU for evaluation.")
+    parser.add_argument('--gradient_accumulation_steps', type=int, default=1,
+                        help="Number of updates steps to accumulate before performing a backward/update pass.")
+    parser.add_argument("--learning_rate", default=5e-5, type=float,
+                        help="The initial learning rate for Adam.")
+    parser.add_argument("--weight_decay", default=0.0, type=float,
+                        help="Weight deay if we apply some.")
+    parser.add_argument("--adam_epsilon", default=1e-8, type=float,
+                        help="Epsilon for Adam optimizer.")
+    parser.add_argument("--max_grad_norm", default=1.0, type=float,
+                        help="Max gradient norm.")
+    parser.add_argument("--num_train_epochs", default=3.0, type=float,
+                        help="Total number of training epochs to perform.")
+    parser.add_argument("--max_steps", default=-1, type=int,
+                        help="If > 0: set total number of training steps to perform. Override num_train_epochs.")
+    parser.add_argument("--warmup_steps", default=0, type=int,
+                        help="Linear warmup over warmup_steps.")
+
+    parser.add_argument('--logging_steps', type=int, default=50,
+                        help="Log every X updates steps.")
+    parser.add_argument('--save_steps', type=int, default=50,
+                        help="Save checkpoint every X updates steps.")
+    parser.add_argument("--eval_all_checkpoints", action='store_true',
+                        help="Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number")
+    parser.add_argument("--no_cuda", action='store_true',
+                        help="Avoid using CUDA when available")
+    parser.add_argument('--overwrite_output_dir', action='store_true',
+                        help="Overwrite the content of the output directory")
+    parser.add_argument('--overwrite_cache', action='store_true',
+                        help="Overwrite the cached training and evaluation sets")
+    parser.add_argument('--seed', type=int, default=42,
+                        help="random seed for initialization")
+
+    parser.add_argument('--fp16', action='store_true',
+                        help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit")
+    parser.add_argument('--fp16_opt_level', type=str, default='O1',
+                        help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
+                             "See details at https://nvidia.github.io/apex/amp.html")
+    parser.add_argument("--local_rank", type=int, default=-1,
+                        help="For distributed training: local_rank")
+    parser.add_argument('--server_ip', type=str, default='', help="For distant debugging.")
+    parser.add_argument('--server_port', type=str, default='', help="For distant debugging.")
+    parser.add_argument("--include_directionality", action='store_true',help='train on all 19 classes for semeval 2010 task 8. Exclude this flag to train on the 9 relation labels only')
+    parser.add_argument("--train_on_other_labels", action='store_true',help='For semeval 2010 task 8, do not include instances with the label "Other" in the training data')
+    parser.add_argument("--eval_on_other_labels", action='store_true',help='For semeval 2010 task 8, Do not include instances with the label "Other" in the evaluation data')
+
+    args = parser.parse_args()
+
+    if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train and not args.overwrite_output_dir:
+        raise ValueError("Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(args.output_dir))
+
+    # Setup distant debugging if needed
+    if args.server_ip and args.server_port:
+        # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
+        import ptvsd
+        print("Waiting for debugger attach")
+        ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True)
+        ptvsd.wait_for_attach()
+
+    # Setup CUDA, GPU & distributed training
+    if args.local_rank == -1 or args.no_cuda:
+        device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
+        args.n_gpu = torch.cuda.device_count()
+    else:  # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
+        torch.cuda.set_device(args.local_rank)
+        device = torch.device("cuda", args.local_rank)
+        torch.distributed.init_process_group(backend='nccl')
+        args.n_gpu = 1
+    args.device = device
+
+    # Setup logging
+    logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
+                        datefmt = '%m/%d/%Y %H:%M:%S',
+                        level = logging.INFO if args.local_rank in [-1, 0] else logging.WARN)
+    logger.warning("Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
+                    args.local_rank, device, args.n_gpu, bool(args.local_rank != -1), args.fp16)
+
+    # Set seed
+    set_seed(args)
+
+    # Prepare semeval task
+    args.task_name = args.task_name.lower()
+    if args.task_name not in processors:
+        raise ValueError("Task not found: %s" % (args.task_name))
+    processor = processors[args.task_name](include_directionality=args.include_directionality)
+    args.output_mode = output_modes[args.task_name]
+    label_list = processor.get_labels()
+    logger.info(f"labels are:  {','.join(label_list)}")
+    num_labels = len(label_list)
+
+    # Load pretrained model and tokenizer
+    if args.local_rank not in [-1, 0]:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
+
+    args.model_type = args.model_type.lower()
+    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
+    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
+                                                do_lower_case=args.do_lower_case)
+
+    entity_1_token_id,entity_2_token_id = get_entity_seperator_token_ids(RBERT_UTILS, tokenizer)
+
+    config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path,
+                                          num_labels=num_labels, finetuning_task=args.task_name,
+                                          entity_1_token_id=entity_1_token_id,
+                                          entity_2_token_id=entity_2_token_id,
+                                          )
+
+    model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path), config=config)
+
+    if args.local_rank == 0:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
+
+    model.to(args.device)
+
+    logger.info("Training/evaluation parameters %s", args)
+
+
+    # Training
+    if args.do_train:
+        train_dataset = load_and_cache_examples(args, args.task_name, tokenizer, evaluate=False)
+        global_step, tr_loss = train(args, train_dataset, model, tokenizer)
+        logger.info(" global_step = %s, average loss = %s", global_step, tr_loss)
+
+
+    # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()
+    if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
+        # Create output directory if needed
+        if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
+            os.makedirs(args.output_dir)
+
+        logger.info("Saving model checkpoint to %s", args.output_dir)
+        # Save a trained model, configuration and tokenizer using `save_pretrained()`.
+        # They can then be reloaded using `from_pretrained()`
+        model_to_save = model.module if hasattr(model, 'module') else model  # Take care of distributed/parallel training
+        model_to_save.save_pretrained(args.output_dir)
+        tokenizer.save_pretrained(args.output_dir)
+
+        # Good practice: save your training arguments together with the trained model
+        torch.save(args, os.path.join(args.output_dir, 'training_args.bin'))
+
+        # Load a trained model and vocabulary that you have fine-tuned
+        model = model_class.from_pretrained(args.output_dir)
+        tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path, do_lower_case=args.do_lower_case)
+        model.to(args.device)
+
+
+    # Evaluation
+    results = {}
+    if args.do_eval and args.local_rank in [-1, 0]:
+        checkpoints = [args.output_dir]
+        if args.eval_all_checkpoints:
+            checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True)))
+            logging.getLogger("pytorch_transformers.modeling_utils").setLevel(logging.WARN)  # Reduce logging
+        logger.info("Evaluate the following checkpoints: %s", checkpoints)
+        for checkpoint in checkpoints:
+            global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else "best"
+            model = model_class.from_pretrained(checkpoint)
+            model.to(args.device)
+            eval_dataset = load_and_cache_examples(args, args.task_name, tokenizer, evaluate=True)
+            result , instance_result_array = evaluate(args, model, eval_dataset, prefix=global_step)
+            result = dict((k + '_{}'.format(global_step), v) for k, v in result.items())
+            results.update(result)
+            # output file in the format for the official semeval script
+            output_semeval_file = os.path.join(args.output_dir , f"{global_step}_semeval_results.tsv")
+            with open(output_semeval_file, "w") as writer:
+                label_map = {i: label for i, label in enumerate(label_list)}
+                for x in instance_result_array:
+                    writer.write(str(x[0])+'\t'+label_map[x[1]]+'\n')
+    return results
+
+if __name__ == "__main__":
+    main()

examples/utils_semeval.py

@@ -0,0 +1,309 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Finetuning the library models for sequence classification on GLUE (Bert, XLM, XLNet, RoBERTa)."""
+
+from __future__ import absolute_import, division, print_function
+
+import logging
+import os
+import re
+import torch
+from torch.utils.data import (TensorDataset)
+from sklearn.metrics import  f1_score
+
+from transformers import RobertaTokenizer
+
+logger = logging.getLogger(__name__)
+
+
+class RBertUtils:
+    """
+    Utility class to prepare text for RBERT
+
+
+    Args:
+    same as BertTokenizer, apart from
+    replace_conflicting_entity_offset_char: since the forward method of RBert classification models detects
+                                            the special characters that bound each entity, any pre-existing characters
+                                            of this type will cause a conflict. To prevent this, we replace any affected
+                                            characters with this, prior to encoding. Defaults to '|'
+    ent1_sep_token: the char to delimit entity 1. Default is '$'
+    ent2_sep_token: the char to delimit entity 2. Default is '#'
+
+
+    """
+
+    def __init__(self, replace_conflicting_entity_offset_char='|',
+                 ent1_sep_token="$", ent2_sep_token="#", ):
+        self.ent1_sep_token = ent1_sep_token
+        self.ent2_sep_token = ent2_sep_token
+        self.replace_conflicting_entity_offset_char = replace_conflicting_entity_offset_char
+
+    def _clean_special_chars(self, text: str):
+        return text.replace(self.ent1_sep_token,
+                            self.replace_conflicting_entity_offset_char).replace(self.ent2_sep_token,
+                                                                                 self.replace_conflicting_entity_offset_char)
+
+    def _insert_relationship_special_chars(self, text, e1_offset_tup, e2_offset_tup):
+        reverse_ordered_tups = sorted([e1_offset_tup, e2_offset_tup], reverse=True, key=lambda x: x[1])
+        last_token_to_process = True
+        for start, end in reverse_ordered_tups:
+            before = text[:start]
+            during = text[start:end]
+            after = text[end:]
+            if last_token_to_process:
+                text = before + " " + self.ent2_sep_token + " " + during + " " + self.ent2_sep_token + " " + after
+            else:
+                text = before + " " + self.ent1_sep_token + " " + during + " " + self.ent1_sep_token + " " + after
+            last_token_to_process = False
+        return text
+
+    def insert_special_chars_into_text(self, text, e1_offset_tup, e2_offset_tup):
+        text = self._clean_special_chars(text)
+        text = self._insert_relationship_special_chars(text, e1_offset_tup, e2_offset_tup)
+        return text
+
+class InputExample(object):
+    """A single SemEval 2010 Task 8 example"""
+
+    def __init__(self, id: int, text: str, label: str, comment: str):
+        self.comment = comment
+        self.label = label
+        self.text = text
+        self.id = id
+
+
+class InputFeatures(object):
+    """A single set of features of data."""
+
+    def __init__(self, input_ids, input_mask, segment_ids, label_id, instance_id):
+        self.instance_id = instance_id
+        self.input_ids = input_ids
+        self.input_mask = input_mask
+        self.segment_ids = segment_ids
+        self.label_id = label_id
+
+
+
+class SemEval2010Task8DataProcessor():
+    """Processor for the SemEval 2010 Task 8 Dataset. Note, it's not clear from the RBert paper how many classes the
+    model is trained on. Clearly this has a big impact on the result.
+
+
+    """
+    def __init__(self,include_directionality=True):
+        self.include_directionality = include_directionality
+        self.e1_e2_labels = [
+                         'Product-Producer(e1,e2)',
+                         'Entity-Origin(e1,e2)',
+                         'Entity-Destination(e1,e2)',
+                         'Message-Topic(e1,e2)',
+                         'Component-Whole(e1,e2)',
+                         'Content-Container(e1,e2)',
+                         'Instrument-Agency(e1,e2)',
+                         'Cause-Effect(e1,e2)',
+                         'Member-Collection(e1,e2)'
+                       ]
+        self.e2_e1_labels = [
+                         'Product-Producer(e2,e1)',
+                         'Entity-Origin(e2,e1)',
+                         'Entity-Destination(e2,e1)',
+                         'Message-Topic(e2,e1)',
+                         'Component-Whole(e2,e1)',
+                         'Content-Container(e2,e1)',
+                         'Instrument-Agency(e2,e1)',
+                         'Cause-Effect(e2,e1)',
+                         'Member-Collection(e2,e1)'
+                       ]
+        self.undirected_labels_mapping = dict(zip(self.e2_e1_labels,self.e1_e2_labels))
+        self.other_label = ['Other']
+        self.undirected_labels_mapping[self.other_label[0]] = self.other_label[0]
+
+        if self.include_directionality:
+            self.all_labels = self.e1_e2_labels + self.e2_e1_labels + self.other_label
+        else:
+            self.all_labels = self.e1_e2_labels  + self.other_label
+
+    def _instance_generator(self, data,include_other):
+        for i in range(0, len(data), 4):
+            id, text = data[i].split('\t')
+            text = text.strip()
+            label = str(data[i + 1]).strip()
+            # map all labels to single direction if not using directionality
+            if not self.include_directionality and label in self.undirected_labels_mapping:
+                label = self.undirected_labels_mapping[label]
+            # Don't include Other label in training set
+            if label == 'Other' and not include_other:
+                logger.info(f'Skipping Other labeled instance at {id}')
+            else:
+                comment = data[i + 2].strip()
+                yield InputExample(id=id, text=text, label=label, comment=comment)
+
+
+    def _strip_direction(self,label):
+        return re.sub(r'\(.*', "", label)
+
+    def _create_examples(self, path,include_other):
+        with open(path, 'r') as f:
+            data = f.readlines()
+        return self._instance_generator(data,include_other)
+
+    def get_train_examples(self, data_dir,include_other):
+        return self._create_examples(os.path.join(data_dir, "SemEval2010_task8_training","TRAIN_FILE.TXT"),
+                                     include_other)
+
+    def get_dev_examples(self, data_dir,include_other):
+        return self._create_examples(os.path.join(data_dir, "SemEval2010_task8_testing_keys","TEST_FILE_FULL.TXT"),
+                                     include_other)
+
+    def write_dev_examples_to_official_format(self,data_dir,examples):
+        with open(os.path.join(data_dir,"TEST_FILE_SEMEVAL_SCRIPT_FORMAT.tsv"),'w') as f:
+            for example in examples:
+                f.write(str(example.id)+'\t'+example.label+'\n')
+
+
+    def get_labels(self):
+        return self.all_labels
+
+def mod_text(text, hit):
+    before = text[:hit.start()]
+    during = hit.group(2)
+    after = text[hit.end():]
+
+    start = len(before)
+    end = len(before) + len(during)
+    return (before + during + after, [start, end],)
+
+
+def find_ents_and_modify_string(text):
+    e1_hits = re.search("(<e1>)(.*)(</e1>)", text)
+    new_text, e1_offsets = mod_text(text, e1_hits)
+    e2_hits = re.search("(<e2>)(.*)(</e2>)", new_text)
+    new_text, e2_offsets = mod_text(new_text, e2_hits)
+
+    return new_text, e1_offsets, e2_offsets
+
+def get_entity_seperator_token_ids(rbert_utils,tokenizer):
+    if isinstance(tokenizer, RobertaTokenizer):
+        entity_1_token_id = \
+        tokenizer.encode(rbert_utils.ent1_sep_token, text_pair=None, add_special_tokens=True, add_prefix_space=True)[1]
+        entity_2_token_id = \
+        tokenizer.encode(rbert_utils.ent2_sep_token, text_pair=None, add_special_tokens=True, add_prefix_space=True)[1]
+    else:
+        entity_1_token_id = tokenizer.encode(rbert_utils.ent1_sep_token)[0]
+        entity_2_token_id = tokenizer.encode(rbert_utils.ent2_sep_token)[0]
+
+    return entity_1_token_id, entity_2_token_id
+
+def get_input_ids(text, tokenizer):
+    if isinstance(tokenizer, RobertaTokenizer):
+        input_ids = tokenizer.encode(text, text_pair=None, add_special_tokens=True, add_prefix_space=True)
+    else:
+        input_ids = tokenizer.encode(text, text_pair=None, add_special_tokens=True)
+    return input_ids
+
+def convert_examples_to_features(examples, label_list, max_seq_length,
+                                 tokenizer, rbert_utils,pad_token=0,
+                                 mask_padding_with_zero=True):
+    label_map = {label: i for i, label in enumerate(label_list)}
+    features = []
+    for (ex_index, example) in enumerate(examples):
+        if ex_index % 5000 == 0:
+            logger.info("Writing example %d" % (ex_index))
+
+
+        text_with_tags_removed, e1_offsets, e2_offsets = find_ents_and_modify_string(example.text)
+        new_text = rbert_utils.insert_special_chars_into_text(text_with_tags_removed,e1_offsets,e2_offsets)
+
+        entity_1_token_id, entity_2_token_id = get_entity_seperator_token_ids(rbert_utils,tokenizer)
+        input_ids = get_input_ids(new_text,tokenizer)
+
+
+
+
+        #check that the special tokens have been encoded in the right order
+        special_tokens_ordered = list(filter(lambda n: n in [entity_1_token_id, entity_2_token_id], input_ids))
+        assert special_tokens_ordered[0] == entity_1_token_id
+        assert special_tokens_ordered[1] == entity_1_token_id
+        assert special_tokens_ordered[2] == entity_2_token_id
+        assert special_tokens_ordered[3] == entity_2_token_id
+
+        # Account for [CLS] and [SEP] with "- 2"
+        if len(input_ids) > max_seq_length - 2:
+            input_ids = input_ids[:(max_seq_length - 2)]
+
+        # The mask has 1 for real tokens and 0 for padding tokens. Only real
+        # tokens are attended to.
+        input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)
+
+        # Zero-pad up to the sequence length.
+        padding_length = max_seq_length - len(input_ids)
+
+        input_ids = input_ids + ([pad_token] * padding_length)
+        input_mask = input_mask + ([0 if mask_padding_with_zero else 1] * padding_length)
+
+        assert len(input_ids) == max_seq_length
+        assert len(input_mask) == max_seq_length
+
+        label_id = label_map[example.label]
+        features.append(
+            InputFeatures(input_ids=input_ids,
+                          input_mask=input_mask,
+                          segment_ids=[0] * len(input_ids),
+                          label_id=label_id,
+                          instance_id=int(example.id)
+                          ))
+    return features
+
+
+def convert_features_to_dataset(features, output_mode='classification'):
+    all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
+    all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
+    all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)
+    if output_mode == "classification":
+        all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.long)
+    else:
+        all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.float)
+    instance_ids = torch.tensor([f.instance_id for f in features], dtype=torch.int)
+    dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids,instance_ids)
+    return dataset
+
+def simple_accuracy(preds, labels):
+    return (preds == labels).mean()
+
+def acc_and_f1(preds, labels):
+    acc = simple_accuracy(preds, labels)
+    f1 = f1_score(y_true=labels, y_pred=preds,average='macro')
+    return {
+        "acc": acc,
+        "f1": f1,
+        "acc_and_f1": (acc + f1) / 2,
+    }
+def compute_metrics(task_name, preds, labels):
+    assert len(preds) == len(labels)
+    if task_name == "semeval2010_task8":
+        return acc_and_f1(labels, preds)
+    else:
+        raise KeyError(task_name)
+
+
+processors = {
+    "semeval2010_task8": SemEval2010Task8DataProcessor
+}
+
+output_modes = {
+    "semeval2010_task8": "classification"
+}

transformers/__init__.py

@@ -71,7 +71,7 @@ if is_torch_available():
     from .modeling_bert import (BertPreTrainedModel, BertModel, BertForPreTraining,
                                 BertForMaskedLM, BertForNextSentencePrediction,
                                 BertForSequenceClassification, BertForMultipleChoice,
-                                BertForTokenClassification, BertForQuestionAnswering,
+                                BertForTokenClassification, BertForQuestionAnswering, BertForRelationshipClassification,
                                 load_tf_weights_in_bert, BERT_PRETRAINED_MODEL_ARCHIVE_MAP)
     from .modeling_openai import (OpenAIGPTPreTrainedModel, OpenAIGPTModel,
                                 OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel,
@@ -96,6 +96,7 @@ if is_torch_available():
                             XLM_PRETRAINED_MODEL_ARCHIVE_MAP)
     from .modeling_roberta import (RobertaForMaskedLM, RobertaModel,
                                 RobertaForSequenceClassification, RobertaForMultipleChoice,
+                                RobertaForRelationshipClassification,
                                 RobertaForTokenClassification,
                                 ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP)
     from .modeling_distilbert import (DistilBertPreTrainedModel, DistilBertForMaskedLM, DistilBertModel,

transformers/configuration_bert.py

@@ -89,27 +89,30 @@ class BertConfig(PretrainedConfig):
                  type_vocab_size=2,
                  initializer_range=0.02,
                  layer_norm_eps=1e-12,
+                 entity_1_token_id=1001,
+                 entity_2_token_id=1002,
                  **kwargs):
         super(BertConfig, self).__init__(**kwargs)
+        self.vocab_size = vocab_size_or_config_json_file if isinstance(vocab_size_or_config_json_file, int) else -1
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.hidden_act = hidden_act
+        self.intermediate_size = intermediate_size
+        self.hidden_dropout_prob = hidden_dropout_prob
+        self.attention_probs_dropout_prob = attention_probs_dropout_prob
+        self.max_position_embeddings = max_position_embeddings
+        self.type_vocab_size = type_vocab_size
+        self.initializer_range = initializer_range
+        self.layer_norm_eps = layer_norm_eps
+        self.entity_2_token_id = entity_2_token_id
+        self.entity_1_token_id = entity_1_token_id
         if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2
                         and isinstance(vocab_size_or_config_json_file, unicode)):
             with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader:
                 json_config = json.loads(reader.read())
             for key, value in json_config.items():
                 self.__dict__[key] = value
-        elif isinstance(vocab_size_or_config_json_file, int):
+        elif not isinstance(vocab_size_or_config_json_file, int):
-            self.vocab_size = vocab_size_or_config_json_file
-            self.hidden_size = hidden_size
-            self.num_hidden_layers = num_hidden_layers
-            self.num_attention_heads = num_attention_heads
-            self.hidden_act = hidden_act
-            self.intermediate_size = intermediate_size
-            self.hidden_dropout_prob = hidden_dropout_prob
-            self.attention_probs_dropout_prob = attention_probs_dropout_prob
-            self.max_position_embeddings = max_position_embeddings
-            self.type_vocab_size = type_vocab_size
-            self.initializer_range = initializer_range
-            self.layer_norm_eps = layer_norm_eps
-        else:
             raise ValueError("First argument must be either a vocabulary size (int)"
                              " or the path to a pretrained model config file (str)")

transformers/configuration_distilbert.py

@@ -53,29 +53,27 @@ class DistilBertConfig(PretrainedConfig):
                  seq_classif_dropout=0.2,
                  **kwargs):
         super(DistilBertConfig, self).__init__(**kwargs)
-
+        self.vocab_size = vocab_size_or_config_json_file if isinstance(vocab_size_or_config_json_file, int) else -1
+        self.max_position_embeddings = max_position_embeddings
+        self.sinusoidal_pos_embds = sinusoidal_pos_embds
+        self.n_layers = n_layers
+        self.n_heads = n_heads
+        self.dim = dim
+        self.hidden_dim = hidden_dim
+        self.dropout = dropout
+        self.attention_dropout = attention_dropout
+        self.activation = activation
+        self.initializer_range = initializer_range
+        self.tie_weights_ = tie_weights_
+        self.qa_dropout = qa_dropout
+        self.seq_classif_dropout = seq_classif_dropout
         if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2
                         and isinstance(vocab_size_or_config_json_file, unicode)):
             with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader:
                 json_config = json.loads(reader.read())
             for key, value in json_config.items():
                 self.__dict__[key] = value
-        elif isinstance(vocab_size_or_config_json_file, int):
+        elif not isinstance(vocab_size_or_config_json_file, int):
-            self.vocab_size = vocab_size_or_config_json_file
-            self.max_position_embeddings = max_position_embeddings
-            self.sinusoidal_pos_embds = sinusoidal_pos_embds
-            self.n_layers = n_layers
-            self.n_heads = n_heads
-            self.dim = dim
-            self.hidden_dim = hidden_dim
-            self.dropout = dropout
-            self.attention_dropout = attention_dropout
-            self.activation = activation
-            self.initializer_range = initializer_range
-            self.tie_weights_ = tie_weights_
-            self.qa_dropout = qa_dropout
-            self.seq_classif_dropout = seq_classif_dropout
-        else:
             raise ValueError("First argument must be either a vocabulary size (int)"
                              " or the path to a pretrained model config file (str)")
     @property

transformers/configuration_gpt2.py

@@ -96,33 +96,31 @@ class GPT2Config(PretrainedConfig):
                 initializing all weight matrices.
         """
         super(GPT2Config, self).__init__(**kwargs)
+        self.vocab_size = vocab_size_or_config_json_file if isinstance(vocab_size_or_config_json_file, int) else -1
+        self.n_ctx = n_ctx
+        self.n_positions = n_positions
+        self.n_embd = n_embd
+        self.n_layer = n_layer
+        self.n_head = n_head
+        self.resid_pdrop = resid_pdrop
+        self.embd_pdrop = embd_pdrop
+        self.attn_pdrop = attn_pdrop
+        self.layer_norm_epsilon = layer_norm_epsilon
+        self.initializer_range = initializer_range
 
+        self.num_labels = num_labels
+        self.summary_type = summary_type
+        self.summary_use_proj = summary_use_proj
+        self.summary_activation = summary_activation
+        self.summary_first_dropout = summary_first_dropout
+        self.summary_proj_to_labels = summary_proj_to_labels
         if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2
                         and isinstance(vocab_size_or_config_json_file, unicode)):
             with open(vocab_size_or_config_json_file, "r", encoding="utf-8") as reader:
                 json_config = json.loads(reader.read())
             for key, value in json_config.items():
                 self.__dict__[key] = value
-        elif isinstance(vocab_size_or_config_json_file, int):
+        elif not isinstance(vocab_size_or_config_json_file, int):
-            self.vocab_size = vocab_size_or_config_json_file
-            self.n_ctx = n_ctx
-            self.n_positions = n_positions
-            self.n_embd = n_embd
-            self.n_layer = n_layer
-            self.n_head = n_head
-            self.resid_pdrop = resid_pdrop
-            self.embd_pdrop = embd_pdrop
-            self.attn_pdrop = attn_pdrop
-            self.layer_norm_epsilon = layer_norm_epsilon
-            self.initializer_range = initializer_range
-
-            self.num_labels = num_labels
-            self.summary_type = summary_type
-            self.summary_use_proj = summary_use_proj
-            self.summary_activation = summary_activation
-            self.summary_first_dropout = summary_first_dropout
-            self.summary_proj_to_labels = summary_proj_to_labels
-        else:
             raise ValueError(
                 "First argument must be either a vocabulary size (int)"
                 "or the path to a pretrained model config file (str)"

transformers/configuration_openai.py

@@ -83,35 +83,33 @@ class OpenAIGPTConfig(PretrainedConfig):
         """Constructs OpenAIGPTConfig.
         """
         super(OpenAIGPTConfig, self).__init__(**kwargs)
+        self.vocab_size = vocab_size_or_config_json_file if isinstance(vocab_size_or_config_json_file, int) else -1
+        self.n_ctx = n_ctx
+        self.n_positions = n_positions
+        self.n_embd = n_embd
+        self.n_layer = n_layer
+        self.n_head = n_head
+        self.afn = afn
+        self.resid_pdrop = resid_pdrop
+        self.embd_pdrop = embd_pdrop
+        self.attn_pdrop = attn_pdrop
+        self.layer_norm_epsilon = layer_norm_epsilon
+        self.initializer_range = initializer_range
+        self.predict_special_tokens = predict_special_tokens
 
+        self.num_labels = num_labels
+        self.summary_type = summary_type
+        self.summary_use_proj = summary_use_proj
+        self.summary_activation = summary_activation
+        self.summary_first_dropout = summary_first_dropout
+        self.summary_proj_to_labels = summary_proj_to_labels
         if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2
                         and isinstance(vocab_size_or_config_json_file, unicode)):
             with open(vocab_size_or_config_json_file, "r", encoding="utf-8") as reader:
                 json_config = json.loads(reader.read())
             for key, value in json_config.items():
                 self.__dict__[key] = value
-        elif isinstance(vocab_size_or_config_json_file, int):
+        elif not isinstance(vocab_size_or_config_json_file, int):
-            self.vocab_size = vocab_size_or_config_json_file
-            self.n_ctx = n_ctx
-            self.n_positions = n_positions
-            self.n_embd = n_embd
-            self.n_layer = n_layer
-            self.n_head = n_head
-            self.afn = afn
-            self.resid_pdrop = resid_pdrop
-            self.embd_pdrop = embd_pdrop
-            self.attn_pdrop = attn_pdrop
-            self.layer_norm_epsilon = layer_norm_epsilon
-            self.initializer_range = initializer_range
-            self.predict_special_tokens = predict_special_tokens
-
-            self.num_labels = num_labels
-            self.summary_type = summary_type
-            self.summary_use_proj = summary_use_proj
-            self.summary_activation = summary_activation
-            self.summary_first_dropout = summary_first_dropout
-            self.summary_proj_to_labels = summary_proj_to_labels
-        else:
             raise ValueError(
                 "First argument must be either a vocabulary size (int)"
                 "or the path to a pretrained model config file (str)"

transformers/configuration_xlm.py

@@ -117,46 +117,44 @@ class XLMConfig(PretrainedConfig):
         """Constructs XLMConfig.
         """
         super(XLMConfig, self).__init__(**kwargs)
-
+        self.n_words = vocab_size_or_config_json_file if isinstance(vocab_size_or_config_json_file, int) else -1
+        self.emb_dim = emb_dim
+        self.n_layers = n_layers
+        self.n_heads = n_heads
+        self.dropout = dropout
+        self.attention_dropout = attention_dropout
+        self.gelu_activation = gelu_activation
+        self.sinusoidal_embeddings = sinusoidal_embeddings
+        self.causal = causal
+        self.asm = asm
+        self.n_langs = n_langs
+        self.use_lang_emb = use_lang_emb
+        self.layer_norm_eps = layer_norm_eps
+        self.bos_index = bos_index
+        self.eos_index = eos_index
+        self.pad_index = pad_index
+        self.unk_index = unk_index
+        self.mask_index = mask_index
+        self.is_encoder = is_encoder
+        self.max_position_embeddings = max_position_embeddings
+        self.embed_init_std = embed_init_std
+        self.init_std = init_std
+        self.finetuning_task = finetuning_task
+        self.num_labels = num_labels
+        self.summary_type = summary_type
+        self.summary_use_proj = summary_use_proj
+        self.summary_activation = summary_activation
+        self.summary_proj_to_labels = summary_proj_to_labels
+        self.summary_first_dropout = summary_first_dropout
+        self.start_n_top = start_n_top
+        self.end_n_top = end_n_top
         if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2
                         and isinstance(vocab_size_or_config_json_file, unicode)):
             with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader:
                 json_config = json.loads(reader.read())
             for key, value in json_config.items():
                 self.__dict__[key] = value
-        elif isinstance(vocab_size_or_config_json_file, int):
+        elif not isinstance(vocab_size_or_config_json_file, int):
-            self.n_words = vocab_size_or_config_json_file
-            self.emb_dim = emb_dim
-            self.n_layers = n_layers
-            self.n_heads = n_heads
-            self.dropout = dropout
-            self.attention_dropout = attention_dropout
-            self.gelu_activation = gelu_activation
-            self.sinusoidal_embeddings = sinusoidal_embeddings
-            self.causal = causal
-            self.asm = asm
-            self.n_langs = n_langs
-            self.use_lang_emb = use_lang_emb
-            self.layer_norm_eps = layer_norm_eps
-            self.bos_index = bos_index
-            self.eos_index = eos_index
-            self.pad_index = pad_index
-            self.unk_index = unk_index
-            self.mask_index = mask_index
-            self.is_encoder = is_encoder
-            self.max_position_embeddings = max_position_embeddings
-            self.embed_init_std = embed_init_std
-            self.init_std = init_std
-            self.finetuning_task = finetuning_task
-            self.num_labels = num_labels
-            self.summary_type = summary_type
-            self.summary_use_proj = summary_use_proj
-            self.summary_activation = summary_activation
-            self.summary_proj_to_labels = summary_proj_to_labels
-            self.summary_first_dropout = summary_first_dropout
-            self.start_n_top = start_n_top
-            self.end_n_top = end_n_top
-        else:
             raise ValueError("First argument must be either a vocabulary size (int)"
                              " or the path to a pretrained model config file (str)")
 

transformers/configuration_xlnet.py

@@ -77,7 +77,6 @@ class XLNetConfig(PretrainedConfig):
                  n_layer=24,
                  n_head=16,
                  d_inner=4096,
-                 max_position_embeddings=512,
                  ff_activation="gelu",
                  untie_r=True,
                  attn_type="bi",
@@ -104,44 +103,42 @@ class XLNetConfig(PretrainedConfig):
         """Constructs XLNetConfig.
         """
         super(XLNetConfig, self).__init__(**kwargs)
+        self.n_token = vocab_size_or_config_json_file if isinstance(vocab_size_or_config_json_file, int) else -1
+        self.d_model = d_model
+        self.n_layer = n_layer
+        self.n_head = n_head
+        assert d_model % n_head == 0
+        self.d_head = d_model // n_head
+        self.ff_activation = ff_activation
+        self.d_inner = d_inner
+        self.untie_r = untie_r
+        self.attn_type = attn_type
 
+        self.initializer_range = initializer_range
+        self.layer_norm_eps = layer_norm_eps
+
+        self.dropout = dropout
+        self.mem_len = mem_len
+        self.reuse_len = reuse_len
+        self.bi_data = bi_data
+        self.clamp_len = clamp_len
+        self.same_length = same_length
+
+        self.finetuning_task = finetuning_task
+        self.num_labels = num_labels
+        self.summary_type = summary_type
+        self.summary_use_proj = summary_use_proj
+        self.summary_activation = summary_activation
+        self.summary_last_dropout = summary_last_dropout
+        self.start_n_top = start_n_top
+        self.end_n_top = end_n_top
         if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2
                         and isinstance(vocab_size_or_config_json_file, unicode)):
             with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader:
                 json_config = json.loads(reader.read())
             for key, value in json_config.items():
-                setattr(config, key, value)
+                self.__dict__[key] = value
-        elif isinstance(vocab_size_or_config_json_file, int):
+        elif not isinstance(vocab_size_or_config_json_file, int):
-            self.n_token = vocab_size_or_config_json_file
-            self.d_model = d_model
-            self.n_layer = n_layer
-            self.n_head = n_head
-            assert d_model % n_head == 0
-            self.d_head = d_model // n_head
-            self.ff_activation = ff_activation
-            self.d_inner = d_inner
-            self.untie_r = untie_r
-            self.attn_type = attn_type
-
-            self.initializer_range = initializer_range
-            self.layer_norm_eps = layer_norm_eps
-
-            self.dropout = dropout
-            self.mem_len = mem_len
-            self.reuse_len = reuse_len
-            self.bi_data = bi_data
-            self.clamp_len = clamp_len
-            self.same_length = same_length
-
-            self.finetuning_task = finetuning_task
-            self.num_labels = num_labels
-            self.summary_type = summary_type
-            self.summary_use_proj = summary_use_proj
-            self.summary_activation = summary_activation
-            self.summary_last_dropout = summary_last_dropout
-            self.start_n_top = start_n_top
-            self.end_n_top = end_n_top
-        else:
             raise ValueError("First argument must be either a vocabulary size (int)"
                              " or the path to a pretrained model config file (str)")
 

transformers/modeling_bert.py

@@ -478,6 +478,58 @@ class BertPreTrainingHeads(nn.Module):
         return prediction_scores, seq_relationship_score
 
 
+class RBertClassificationHead(nn.Module):
+    def __init__(self, config):
+        """
+        :param config: a BertConfig or aRobertaConfig.
+        """
+        super(RBertClassificationHead, self).__init__()
+        self.ent_2_index_id = config.entity_2_token_id
+        self.ent_1_index_id = config.entity_1_token_id
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+        self.sentence_layer = torch.nn.Sequential(
+            self.dropout,
+            torch.nn.Linear(config.hidden_size, config.hidden_size),
+            torch.nn.modules.activation.Tanh(),
+        )
+        self.entity_layer = torch.nn.Sequential(
+            self.dropout,
+            torch.nn.Linear(config.hidden_size, config.hidden_size),
+            torch.nn.modules.activation.Tanh()
+        )
+
+        self.classifier = nn.Linear(config.hidden_size * 3, config.num_labels)
+
+    def get_indices_tensor(self, instance_input_ids, entity_char_id):
+        index_tensors = (instance_input_ids == entity_char_id).nonzero()
+        start_index = index_tensors[0].item() + 1  # do not include the symbol itself in the calculation
+        end_index = index_tensors[1].item()
+        return torch.arange(start_index, end_index, device=instance_input_ids.device.type)
+
+    def average_entity_vectors(self, last_hidden_states, input_ids, entity_char_id):
+        batch_return_list = []
+        for instance_hidden_state, instance_input_ids in zip(last_hidden_states, input_ids):
+            lookup_tensor = self.get_indices_tensor(instance_input_ids, entity_char_id)
+            average_of_entity_vectors = torch.index_select(instance_hidden_state, 0, lookup_tensor).unsqueeze(0).mean(1)
+            batch_return_list.append(average_of_entity_vectors)
+        return_tensor = torch.cat(batch_return_list)
+        return_tensor[return_tensor.ne(return_tensor)] = 0 #set any nan's created by mean operation to 0
+        return return_tensor
+
+    def forward(self, input_ids, last_hidden_states):
+        cls_tensor = self.sentence_layer(
+            last_hidden_states[:, 0])  # get the sentence embedding and pass through FC layer
+        ent1_tensor = self.entity_layer(self.average_entity_vectors(last_hidden_states, input_ids,
+                                                                    self.ent_1_index_id))  # average wordpieces of ent1 and pass through FC entity layer
+        ent2_tensor = self.entity_layer(self.average_entity_vectors(last_hidden_states, input_ids,
+                                                                    self.ent_2_index_id))  # ditto - ent2 shared parameters with ent1
+
+        entities_and_cls_tensor = torch.cat((cls_tensor, ent1_tensor, ent2_tensor),
+                                            dim=1)  # concat all and pass through classifier layer
+        logits = self.classifier(entities_and_cls_tensor)
+        return logits
+
+
 class BertPreTrainedModel(PreTrainedModel):
     """ An abstract class to handle weights initialization and
         a simple interface for dowloading and loading pretrained models.
@@ -963,6 +1015,80 @@ class BertForNextSentencePrediction(BertPreTrainedModel):
         return outputs  # (next_sentence_loss), seq_relationship_score, (hidden_states), (attentions)
 
 
+@add_start_docstrings(r"""Bert Model transformer with an R-BERT head on top, making use of
+        entity offsets for relationship classification. See the details in the R-Bert paper https://arxiv.org/pdf/1905.08284.pdf""",
+        BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING)
+class BertForRelationshipClassification(BertPreTrainedModel):
+    r"""
+        **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
+        Labels for computing the sequence classification/regression loss.
+        Indices should be in ``[0, ..., config.num_labels - 1]``.
+        If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss),
+        If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy).
+
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
+            Classification (or regression if config.num_labels==1) loss.
+        **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)``
+            Classification (or regression if config.num_labels==1) scores (before SoftMax).
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        test_string = "The $ cat $ sat on the # mat # ."
+        ent1_sep_token = '$'
+        ent2_sep_token = '#'
+
+        model_name = 'bert-base-uncased'
+
+        tokenizer = BertTokenizer.from_pretrained(model_name)
+        ent1_sep_token_id = tokenizer.encode(ent1_sep_token)[0]
+        ent2_sep_token_id = tokenizer.encode(ent2_sep_token)[0]
+        config = BertConfig.from_pretrained(model_name, entity_1_token_id=ent1_sep_token_id, entity_2_token_id=ent2_sep_token_id)
+        model = BertForRelationshipClassification.from_pretrained(model_name, config=config)
+        input_ids = torch.tensor(tokenizer.encode(test_string,add_special_tokens=True)).unsqueeze(0)  # Batch size 1
+        labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids, labels=labels)
+        loss, logits = outputs[:2]
+
+    """
+
+    def __init__(self, config):
+        super(BertForRelationshipClassification, self).__init__(config)
+        self.num_labels = config.num_labels
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+        self.bert = BertModel(config)
+        self.rbert = RBertClassificationHead(config)
+        self.init_weights()
+
+    def forward(self, input_ids, attention_mask=None, token_type_ids=None,
+                position_ids=None, head_mask=None, labels=None):
+
+        outputs = self.bert(input_ids, position_ids=position_ids, token_type_ids=token_type_ids,
+                            attention_mask=attention_mask, head_mask=head_mask)
+        last_hidden_states = outputs[0]
+        logits = self.rbert(input_ids, last_hidden_states)
+        outputs = (logits,) + outputs[2:]  # add hidden states and attention if they are here
+
+        if labels is not None:
+            if self.num_labels == 1:
+                #  We are doing regression
+                loss_fct = MSELoss()
+                loss = loss_fct(logits.view(-1), labels.view(-1))
+            else:
+                loss_fct = CrossEntropyLoss()
+                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+            outputs = (loss,) + outputs
+
+        return outputs  # (loss), logits, (hidden_states), (attentions)
+
+
 @add_start_docstrings("""Bert Model transformer with a sequence classification/regression head on top (a linear layer on top of
                       the pooled output) e.g. for GLUE tasks. """,
                       BERT_START_DOCSTRING,

transformers/modeling_roberta.py

@@ -24,7 +24,7 @@ import torch
 import torch.nn as nn
 from torch.nn import CrossEntropyLoss, MSELoss
 
-from .modeling_bert import BertEmbeddings, BertLayerNorm, BertModel, BertPreTrainedModel, gelu
+from .modeling_bert import BertEmbeddings, BertLayerNorm, BertModel, BertPreTrainedModel, gelu, RBertClassificationHead
 from .configuration_roberta import RobertaConfig
 from .file_utils import add_start_docstrings
 
@@ -435,8 +435,8 @@ class RobertaForMultipleChoice(BertPreTrainedModel):
 
         self.init_weights()
 
-    def forward(self, input_ids=None, token_type_ids=None, attention_mask=None, labels=None,
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None,
-                position_ids=None, head_mask=None, inputs_embeds=None):
+                labels=None):
         num_choices = input_ids.shape[1]
 
         flat_input_ids = input_ids.view(-1, input_ids.size(-1))
@@ -460,6 +460,86 @@ class RobertaForMultipleChoice(BertPreTrainedModel):
 
         return outputs  # (loss), reshaped_logits, (hidden_states), (attentions)
 
+@add_start_docstrings("""Roberta Model transformer with an R-BERT head on top, making use of
+        entity offsets for relationship classification. See the details in the R-Bert paper https://arxiv.org/pdf/1905.08284.pdf""",
+    ROBERTA_START_DOCSTRING, ROBERTA_INPUTS_DOCSTRING)
+class RobertaForRelationshipClassification(BertPreTrainedModel):
+    r"""
+        **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
+        Labels for computing the sequence classification/regression loss.
+        Indices should be in ``[0, ..., config.num_labels - 1]``.
+        If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss),
+        If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy).
+
+    Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
+        **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
+            Classification (or regression if config.num_labels==1) loss.
+        **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)``
+            Classification (or regression if config.num_labels==1) scores (before SoftMax).
+        **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
+            list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
+            of shape ``(batch_size, sequence_length, hidden_size)``:
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        **attentions**: (`optional`, returned when ``config.output_attentions=True``)
+            list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+
+    Examples::
+
+        test_string = "The $ cat $ sat on the # mat # ."
+        ent1_sep_token = '$'
+        ent2_sep_token = '#'
+
+        model_name = 'roberta-large'
+
+        tokenizer = RobertaTokenizer.from_pretrained(model_name)
+        ent1_sep_token_id = tokenizer.encode(ent1_sep_token, text_pair=None, add_special_tokens=True, add_prefix_space=True)[1]
+        ent2_sep_token_id = tokenizer.encode(ent2_sep_token, text_pair=None, add_special_tokens=True, add_prefix_space=True)[1]
+        config = RobertaConfig.from_pretrained(model_name,entity_1_token_id=ent1_sep_token_id, entity_2_token_id=ent2_sep_token_id)
+        model = RobertaForRelationshipClassification.from_pretrained(model_name, config=config)
+        input_ids = torch.tensor(tokenizer.encode(test_string,add_special_tokens=True)).unsqueeze(0)  # Batch size 1
+        labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids, labels=labels)
+        loss, logits = outputs[:2]
+    """
+    config_class = RobertaConfig
+    pretrained_model_archive_map = ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP
+    base_model_prefix = "roberta"
+
+    def __init__(self, config):
+        super(RobertaForRelationshipClassification, self).__init__(config)
+
+        self.num_labels = config.num_labels
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+        self.roberta = RobertaModel(config)
+        self.rbert = RBertClassificationHead(config)
+
+        self.init_weights()
+
+    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None,
+                labels=None):
+
+        outputs = self.roberta(input_ids,
+                               attention_mask=attention_mask,
+                               token_type_ids=token_type_ids,
+                               position_ids=position_ids,
+                               head_mask=head_mask,
+                               inputs_embeds=inputs_embeds)
+        last_hidden_states = outputs[0]
+        logits = self.rbert(input_ids, last_hidden_states)
+        outputs = (logits,) + outputs[2:]  # add hidden states and attention if they are here
+
+        if labels is not None:
+            if self.num_labels == 1:
+                #  We are doing regression
+                loss_fct = MSELoss()
+                loss = loss_fct(logits.view(-1), labels.view(-1))
+            else:
+                loss_fct = CrossEntropyLoss()
+                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+            outputs = (loss,) + outputs
+
+        return outputs  # (loss), logits, (hidden_states), (attentions)
 
 @add_start_docstrings("""Roberta Model with a token classification head on top (a linear layer on top of
     the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """,

transformers/tests/modeling_bert_test.py

@@ -29,7 +29,8 @@ if is_torch_available():
     from transformers import (BertConfig, BertModel, BertForMaskedLM,
                               BertForNextSentencePrediction, BertForPreTraining,
                               BertForQuestionAnswering, BertForSequenceClassification,
-                              BertForTokenClassification, BertForMultipleChoice)
+                              BertForTokenClassification, BertForMultipleChoice,
+                              BertForRelationshipClassification)
     from transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP
 else:
     pytestmark = pytest.mark.skip("Require Torch")
@@ -66,6 +67,8 @@ class BertModelTest(CommonTestCases.CommonModelTester):
                      initializer_range=0.02,
                      num_labels=3,
                      num_choices=4,
+                     entity_1_token_id=4,
+                     entity_2_token_id=8,
                      scope=None,
                      device='cpu',
                      ):
@@ -90,12 +93,21 @@ class BertModelTest(CommonTestCases.CommonModelTester):
             self.initializer_range = initializer_range
             self.num_labels = num_labels
             self.num_choices = num_choices
+            self.entity_1_token_id = entity_1_token_id
+            self.entity_2_token_id = entity_2_token_id
             self.scope = scope
             self.device = device
 
         def prepare_config_and_inputs(self):
             input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size).to(self.device)
 
+            assert self.seq_length >= 7
+            # we need to mock the behaviour of a string prepared for RBERT, by inserting entity bounding characters
+            input_ids[:, 0] = self.entity_1_token_id
+            input_ids[:, 2] = self.entity_1_token_id
+            input_ids[:, 4] = self.entity_2_token_id
+            input_ids[:, 6] = self.entity_2_token_id
+
             input_mask = None
             if self.use_input_mask:
                 input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2).to(self.device)
@@ -123,6 +135,8 @@ class BertModelTest(CommonTestCases.CommonModelTester):
                 attention_probs_dropout_prob=self.attention_probs_dropout_prob,
                 max_position_embeddings=self.max_position_embeddings,
                 type_vocab_size=self.type_vocab_size,
+                entity_1_token_id=self.entity_1_token_id,
+                entity_2_token_id=self.entity_2_token_id,
                 is_decoder=False,
                 initializer_range=self.initializer_range)
 
@@ -299,6 +313,22 @@ class BertModelTest(CommonTestCases.CommonModelTester):
                 [self.batch_size, self.num_choices])
             self.check_loss_output(result)
 
+        def create_and_check_bert_for_relationship_classification(self, config, input_ids, token_type_ids, input_mask,
+                                                                  sequence_labels, token_labels, choice_labels):
+            config.num_labels = self.num_labels
+            model = BertForRelationshipClassification(config=config)
+            model.eval()
+            loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids,
+                                 labels=sequence_labels)
+            result = {
+                "loss": loss,
+                "logits": logits,
+            }
+            self.parent.assertListEqual(
+                list(result["logits"].size()),
+                [self.batch_size, self.num_labels])
+            self.check_loss_output(result)
+
         def prepare_config_and_inputs_for_common(self):
             config_and_inputs = self.prepare_config_and_inputs()
             (config, input_ids, token_type_ids, input_mask,
@@ -356,6 +386,10 @@ class BertModelTest(CommonTestCases.CommonModelTester):
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_bert_for_token_classification(*config_and_inputs)
 
+    def test_for_relationship_classification(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_bert_for_relationship_classification(*config_and_inputs)
+
     @pytest.mark.slow
     def test_model_from_pretrained(self):
         cache_dir = "/tmp/transformers_test/"

transformers/tests/modeling_roberta_test.py

@@ -25,7 +25,7 @@ from transformers import is_torch_available
 if is_torch_available():
     import torch
     from transformers import (RobertaConfig, RobertaModel, RobertaForMaskedLM,
-                              RobertaForSequenceClassification, RobertaForTokenClassification)
+                              RobertaForSequenceClassification, RobertaForTokenClassification, RobertaForRelationshipClassification)
     from transformers.modeling_roberta import ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP
 else:
     pytestmark = pytest.mark.skip("Require Torch")
@@ -36,7 +36,8 @@ from .configuration_common_test import ConfigTester
 
 class RobertaModelTest(CommonTestCases.CommonModelTester):
 
-    all_model_classes = (RobertaForMaskedLM, RobertaModel) if is_torch_available() else ()
+    all_model_classes = (RobertaForMaskedLM, RobertaModel,
+                        RobertaForRelationshipClassification) if is_torch_available() else ()
 
     class RobertaModelTester(object):
 
@@ -62,6 +63,8 @@ class RobertaModelTest(CommonTestCases.CommonModelTester):
                      initializer_range=0.02,
                      num_labels=3,
                      num_choices=4,
+                     entity_1_token_id=4,
+                     entity_2_token_id=8,
                      scope=None,
                     ):
             self.parent = parent
@@ -85,11 +88,20 @@ class RobertaModelTest(CommonTestCases.CommonModelTester):
             self.initializer_range = initializer_range
             self.num_labels = num_labels
             self.num_choices = num_choices
+            self.entity_1_token_id = entity_1_token_id
+            self.entity_2_token_id = entity_2_token_id
             self.scope = scope
 
         def prepare_config_and_inputs(self):
             input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)
 
+            assert self.seq_length >= 7
+            # we need to mock the behaviour of a string prepared for RBERT, by inserting entity bounding characters
+            input_ids[:, 0] = self.entity_1_token_id
+            input_ids[:, 2] = self.entity_1_token_id
+            input_ids[:, 4] = self.entity_2_token_id
+            input_ids[:, 6] = self.entity_2_token_id
+
             input_mask = None
             if self.use_input_mask:
                 input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2)
@@ -117,7 +129,9 @@ class RobertaModelTest(CommonTestCases.CommonModelTester):
                 attention_probs_dropout_prob=self.attention_probs_dropout_prob,
                 max_position_embeddings=self.max_position_embeddings,
                 type_vocab_size=self.type_vocab_size,
-                initializer_range=self.initializer_range)
+                initializer_range=self.initializer_range,
+                entity_1_token_id=self.entity_1_token_id,
+                entity_2_token_id=self.entity_2_token_id)
 
             return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
 
@@ -157,6 +171,23 @@ class RobertaModelTest(CommonTestCases.CommonModelTester):
                 [self.batch_size, self.seq_length, self.vocab_size])
             self.check_loss_output(result)
 
+        def create_and_check_roberta_for_relationship_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels,
+                                                   token_labels, choice_labels):
+            config.num_labels = self.num_choices
+            model = RobertaForRelationshipClassification(config=config)
+            model.eval()
+            loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=choice_labels)
+
+            result = {
+                "loss": loss,
+                "logits": logits,
+            }
+
+            self.parent.assertListEqual(
+                list(result["logits"].size()),
+                [self.batch_size, self.num_choices])
+            self.check_loss_output(result)
+
         def create_and_check_roberta_for_token_classification(self, config, input_ids, token_type_ids, input_mask,
                                                               sequence_labels, token_labels, choice_labels):
             config.num_labels = self.num_labels
@@ -171,6 +202,7 @@ class RobertaModelTest(CommonTestCases.CommonModelTester):
             self.parent.assertListEqual(
                 list(result["logits"].size()),
                 [self.batch_size, self.seq_length, self.num_labels])
+
             self.check_loss_output(result)
 
         def prepare_config_and_inputs_for_common(self):
@@ -195,6 +227,10 @@ class RobertaModelTest(CommonTestCases.CommonModelTester):
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_roberta_for_masked_lm(*config_and_inputs)
 
+    def test_for_relationship_classification(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_roberta_for_relationship_classification(*config_and_inputs)
+
     @pytest.mark.slow
     def test_model_from_pretrained(self):
         cache_dir = "/tmp/transformers_test/"
@@ -204,7 +240,6 @@ class RobertaModelTest(CommonTestCases.CommonModelTester):
             self.assertIsNotNone(model)
 
 
-
 class RobertaModelIntegrationTest(unittest.TestCase):
 
     @pytest.mark.slow