mt-dnn-train.log

08/19/2019 08:19:56 0
08/19/2019 08:19:56 Launching the MT-DNN training
08/19/2019 08:19:56 Loading data/canonical_data/mt_dnn_uncased_lower\mnli_train.json as task 0
08/19/2019 08:34:38 0
08/19/2019 08:34:38 Launching the MT-DNN training
08/19/2019 08:34:38 Loading data/canonical_data/mt_dnn_uncased_lower\mnli_train.json as task 0
08/19/2019 08:36:15 0
08/19/2019 08:36:15 Launching the MT-DNN training
08/19/2019 08:36:15 Loading data/scitail_train.json as task 0
08/19/2019 08:37:52 0
08/19/2019 08:37:52 Launching the MT-DNN training
08/19/2019 08:37:52 Loading data\canonical_data\bert_uncased\mnli_train.json as task 0
08/19/2019 08:38:27 0
08/19/2019 08:38:27 Launching the MT-DNN training
08/19/2019 08:38:27 Loading data\canonical_data\bert_uncased\scitail_train.json as task 0
08/19/2019 08:38:28 2
08/19/2019 08:38:28 ####################
08/19/2019 08:38:28 {'log_file': 'mt-dnn-train.log', 'init_checkpoint': 'mt_dnn_models/bert_model_base.pt', 'data_dir': 'data\\canonical_data\\bert_uncased', 'data_sort_on': False, 'name': 'farmer', 'task_def': 'experiments/glue/glue_task_def.yml', 'train_datasets': ['scitail'], 'test_datasets': ['scitail_001'], 'update_bert_opt': 0, 'multi_gpu_on': False, 'mem_cum_type': 'simple', 'answer_num_turn': 5, 'answer_mem_drop_p': 0.1, 'answer_att_hidden_size': 128, 'answer_att_type': 'bilinear', 'answer_rnn_type': 'gru', 'answer_sum_att_type': 'bilinear', 'answer_merge_opt': 1, 'answer_mem_type': 1, 'answer_dropout_p': 0.1, 'answer_weight_norm_on': False, 'dump_state_on': False, 'answer_opt': [0], 'label_size': '2', 'mtl_opt': 0, 'ratio': 0, 'mix_opt': 0, 'max_seq_len': 512, 'init_ratio': 1, 'encoder_type': <EncoderModelType.BERT: 1>, 'cuda': True, 'log_per_updates': 500, 'save_per_updates': 10000, 'save_per_updates_on': False, 'epochs': 5, 'batch_size': 8, 'batch_size_eval': 8, 'optimizer': 'adamax', 'grad_clipping': 0, 'global_grad_clipping': 1.0, 'weight_decay': 0, 'learning_rate': 5e-05, 'momentum': 0, 'warmup': 0.1, 'warmup_schedule': 'warmup_linear', 'vb_dropout': True, 'dropout_p': 0.1, 'dropout_w': 0.0, 'bert_dropout_p': 0.1, 'model_ckpt': 'checkpoints/model_0.pt', 'resume': False, 'ema_opt': 0, 'ema_gamma': 0.995, 'have_lr_scheduler': True, 'multi_step_lr': '10,20,30', 'freeze_layers': -1, 'embedding_opt': 0, 'lr_gamma': 0.5, 'bert_l2norm': 0.0, 'scheduler_type': 'ms', 'output_dir': 'checkpoint', 'seed': 2018, 'grad_accumulation_step': 1, 'fp16': False, 'fp16_opt_level': 'O1', 'tasks_dropout_p': [0.1]}
08/19/2019 08:38:28 ####################
08/19/2019 08:38:28 ############# Gradient Accumulation Info #############
08/19/2019 08:38:28 number of step: 14750
08/19/2019 08:38:28 number of grad grad_accumulation step: 1
08/19/2019 08:38:28 adjusted number of step: 14750
08/19/2019 08:38:28 ############# Gradient Accumulation Info #############
08/19/2019 08:38:28 ####################
08/19/2019 08:38:28 Could not find the init model!
 The parameters will be initialized randomly!
08/19/2019 08:38:28 ####################
08/19/2019 08:40:39 0
08/19/2019 08:40:39 Launching the MT-DNN training
08/19/2019 08:40:39 Loading data\canonical_data\bert_uncased\scitail_train.json as task 0
08/19/2019 08:40:39 2
08/19/2019 08:40:39 ####################
08/19/2019 08:40:39 {'log_file': 'mt-dnn-train.log', 'init_checkpoint': 'mt_dnn_models/bert_model_base_uncased.pt', 'data_dir': 'data\\canonical_data\\bert_uncased', 'data_sort_on': False, 'name': 'farmer', 'task_def': 'experiments/glue/glue_task_def.yml', 'train_datasets': ['scitail'], 'test_datasets': ['scitail_001'], 'update_bert_opt': 0, 'multi_gpu_on': False, 'mem_cum_type': 'simple', 'answer_num_turn': 5, 'answer_mem_drop_p': 0.1, 'answer_att_hidden_size': 128, 'answer_att_type': 'bilinear', 'answer_rnn_type': 'gru', 'answer_sum_att_type': 'bilinear', 'answer_merge_opt': 1, 'answer_mem_type': 1, 'answer_dropout_p': 0.1, 'answer_weight_norm_on': False, 'dump_state_on': False, 'answer_opt': [0], 'label_size': '2', 'mtl_opt': 0, 'ratio': 0, 'mix_opt': 0, 'max_seq_len': 512, 'init_ratio': 1, 'encoder_type': <EncoderModelType.BERT: 1>, 'cuda': True, 'log_per_updates': 500, 'save_per_updates': 10000, 'save_per_updates_on': False, 'epochs': 5, 'batch_size': 8, 'batch_size_eval': 8, 'optimizer': 'adamax', 'grad_clipping': 0, 'global_grad_clipping': 1.0, 'weight_decay': 0, 'learning_rate': 5e-05, 'momentum': 0, 'warmup': 0.1, 'warmup_schedule': 'warmup_linear', 'vb_dropout': True, 'dropout_p': 0.1, 'dropout_w': 0.0, 'bert_dropout_p': 0.1, 'model_ckpt': 'checkpoints/model_0.pt', 'resume': False, 'ema_opt': 0, 'ema_gamma': 0.995, 'have_lr_scheduler': True, 'multi_step_lr': '10,20,30', 'freeze_layers': -1, 'embedding_opt': 0, 'lr_gamma': 0.5, 'bert_l2norm': 0.0, 'scheduler_type': 'ms', 'output_dir': 'checkpoint', 'seed': 2018, 'grad_accumulation_step': 1, 'fp16': False, 'fp16_opt_level': 'O1', 'tasks_dropout_p': [0.1]}
08/19/2019 08:40:39 ####################
08/19/2019 08:40:39 ############# Gradient Accumulation Info #############
08/19/2019 08:40:39 number of step: 14750
08/19/2019 08:40:39 number of grad grad_accumulation step: 1
08/19/2019 08:40:39 adjusted number of step: 14750
08/19/2019 08:40:39 ############# Gradient Accumulation Info #############
08/19/2019 08:40:47 
############# Model Arch of MT-DNN #############
SANBertNetwork(
  (dropout_list): ModuleList(
    (0): DropoutWrapper()
  )
  (bert): BertModel(
    (embeddings): BertEmbeddings(
      (word_embeddings): Embedding(30522, 768)
      (position_embeddings): Embedding(512, 768)
      (token_type_embeddings): Embedding(2, 768)
      (LayerNorm): BertLayerNorm()
      (dropout): Dropout(p=0.1)
    )
    (encoder): BertEncoder(
      (layer): ModuleList(
        (0): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (1): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (2): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (3): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (4): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (5): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (6): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (7): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (8): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (9): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (10): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (11): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
      )
    )
    (pooler): BertPooler(
      (dense): Linear(in_features=768, out_features=768, bias=True)
      (activation): Tanh()
    )
  )
  (scoring_list): ModuleList(
    (0): Linear(in_features=768, out_features=2, bias=True)
  )
)

08/19/2019 08:40:47 Total number of params: 109483778
08/19/2019 08:40:47 At epoch 0
08/19/2019 08:40:47 Task [ 0] updates[     1] train loss[1.49810] remaining[0:29:07]
08/19/2019 08:43:24 Task [ 0] updates[   500] train loss[0.65001] remaining[0:12:51]
08/19/2019 08:46:00 Task [ 0] updates[  1000] train loss[0.53163] remaining[0:10:11]
08/19/2019 08:48:36 Task [ 0] updates[  1500] train loss[0.47806] remaining[0:07:33]
08/19/2019 08:51:11 Task [ 0] updates[  2000] train loss[0.44618] remaining[0:04:56]
08/19/2019 08:53:47 Task [ 0] updates[  2500] train loss[0.42259] remaining[0:02:20]
08/19/2019 08:56:11 At epoch 1
08/19/2019 08:56:27 Task [ 0] updates[  3000] train loss[0.40276] remaining[0:15:12]
08/19/2019 08:59:01 Task [ 0] updates[  3500] train loss[0.38243] remaining[0:12:21]
08/19/2019 09:01:38 Task [ 0] updates[  4000] train loss[0.36635] remaining[0:09:51]
08/19/2019 10:02:28 0
08/19/2019 10:02:28 Launching the MT-DNN training
08/19/2019 10:02:28 Loading data/canonical_data/bert-uncased\sst_train.json as task 0
08/19/2019 10:03:18 0
08/19/2019 10:03:18 Launching the MT-DNN training
08/19/2019 10:03:18 Loading data\canonical_dataert_uncased\sst_train.json as task 0
08/19/2019 10:03:38 0
08/19/2019 10:03:38 Launching the MT-DNN training
08/19/2019 10:03:38 Loading /data/canonical_data/bert_uncased\sst_train.json as task 0
08/19/2019 10:05:31 0
08/19/2019 10:05:31 Launching the MT-DNN training
08/19/2019 10:05:37 0
08/19/2019 10:05:37 Launching the MT-DNN training
08/19/2019 10:05:37 Loading /data/canonical_data/bert_uncased\scitail_train.json as task 0
08/19/2019 10:06:03 0
08/19/2019 10:06:03 Launching the MT-DNN training
08/19/2019 10:06:03 Loading /data/canonical_data/bert_uncased/scitail_train.json as task 0
08/19/2019 10:06:26 0
08/19/2019 10:06:26 Launching the MT-DNN training
08/19/2019 10:06:26 Loading data\canonical_data\bert_uncased\sst_train.json as task 0
08/19/2019 10:06:27 2
08/19/2019 10:06:27 ####################
08/19/2019 10:06:27 {'log_file': 'mt-dnn-train.log', 'init_checkpoint': 'mt_dnn_models/bert_model_base_uncased.pt', 'data_dir': 'data\\canonical_data\\bert_uncased', 'data_sort_on': False, 'name': 'farmer', 'task_def': 'experiments/glue/glue_task_def.yml', 'train_datasets': ['sst'], 'test_datasets': ['sst'], 'update_bert_opt': 0, 'multi_gpu_on': False, 'mem_cum_type': 'simple', 'answer_num_turn': 5, 'answer_mem_drop_p': 0.1, 'answer_att_hidden_size': 128, 'answer_att_type': 'bilinear', 'answer_rnn_type': 'gru', 'answer_sum_att_type': 'bilinear', 'answer_merge_opt': 1, 'answer_mem_type': 1, 'answer_dropout_p': 0.1, 'answer_weight_norm_on': False, 'dump_state_on': False, 'answer_opt': [0], 'label_size': '2', 'mtl_opt': 0, 'ratio': 0, 'mix_opt': 0, 'max_seq_len': 512, 'init_ratio': 1, 'encoder_type': <EncoderModelType.BERT: 1>, 'cuda': True, 'log_per_updates': 500, 'save_per_updates': 10000, 'save_per_updates_on': False, 'epochs': 5, 'batch_size': 8, 'batch_size_eval': 8, 'optimizer': 'adamax', 'grad_clipping': 0, 'global_grad_clipping': 1.0, 'weight_decay': 0, 'learning_rate': 5e-05, 'momentum': 0, 'warmup': 0.1, 'warmup_schedule': 'warmup_linear', 'vb_dropout': True, 'dropout_p': 0.1, 'dropout_w': 0.0, 'bert_dropout_p': 0.1, 'model_ckpt': 'checkpoints/model_0.pt', 'resume': False, 'ema_opt': 0, 'ema_gamma': 0.995, 'have_lr_scheduler': True, 'multi_step_lr': '10,20,30', 'freeze_layers': -1, 'embedding_opt': 0, 'lr_gamma': 0.5, 'bert_l2norm': 0.0, 'scheduler_type': 'ms', 'output_dir': 'checkpoint', 'seed': 2018, 'grad_accumulation_step': 1, 'fp16': False, 'fp16_opt_level': 'O1', 'tasks_dropout_p': [0.1]}
08/19/2019 10:06:27 ####################
08/19/2019 10:06:27 ############# Gradient Accumulation Info #############
08/19/2019 10:06:27 number of step: 42095
08/19/2019 10:06:27 number of grad grad_accumulation step: 1
08/19/2019 10:06:27 adjusted number of step: 42095
08/19/2019 10:06:27 ############# Gradient Accumulation Info #############
08/19/2019 10:06:31 
############# Model Arch of MT-DNN #############
SANBertNetwork(
  (dropout_list): ModuleList(
    (0): DropoutWrapper()
  )
  (bert): BertModel(
    (embeddings): BertEmbeddings(
      (word_embeddings): Embedding(30522, 768)
      (position_embeddings): Embedding(512, 768)
      (token_type_embeddings): Embedding(2, 768)
      (LayerNorm): BertLayerNorm()
      (dropout): Dropout(p=0.1)
    )
    (encoder): BertEncoder(
      (layer): ModuleList(
        (0): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (1): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (2): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (3): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (4): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (5): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (6): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (7): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (8): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (9): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (10): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (11): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
      )
    )
    (pooler): BertPooler(
      (dense): Linear(in_features=768, out_features=768, bias=True)
      (activation): Tanh()
    )
  )
  (scoring_list): ModuleList(
    (0): Linear(in_features=768, out_features=2, bias=True)
  )
)

08/19/2019 10:06:31 Total number of params: 109483778
08/19/2019 10:06:31 At epoch 0
08/19/2019 10:06:31 Task [ 0] updates[     1] train loss[0.60717] remaining[1:22:38]
08/19/2019 10:07:42 0
08/19/2019 10:07:42 Launching the MT-DNN training
08/19/2019 10:07:42 Loading data\canonical_data\bert_uncased\scitail_001_train.json as task 0
08/19/2019 10:09:22 0
08/19/2019 10:09:22 Launching the MT-DNN training
08/19/2019 10:09:22 Loading data\canonical_data\bert_uncased\scitail_001_train.json as task 0
08/19/2019 10:09:31 0
08/19/2019 10:09:31 Launching the MT-DNN training
08/19/2019 10:09:31 Loading data\canonical_data\bert_uncased\sst_train.json as task 0
08/19/2019 10:09:32 2
08/19/2019 10:09:32 ####################
08/19/2019 10:09:32 {'log_file': 'mt-dnn-train.log', 'init_checkpoint': 'mt_dnn_models/bert_model_base_uncased.pt', 'data_dir': 'data\\canonical_data\\bert_uncased', 'data_sort_on': False, 'name': 'farmer', 'task_def': 'experiments/glue/glue_task_def.yml', 'train_datasets': ['sst'], 'test_datasets': ['sst'], 'update_bert_opt': 0, 'multi_gpu_on': False, 'mem_cum_type': 'simple', 'answer_num_turn': 5, 'answer_mem_drop_p': 0.1, 'answer_att_hidden_size': 128, 'answer_att_type': 'bilinear', 'answer_rnn_type': 'gru', 'answer_sum_att_type': 'bilinear', 'answer_merge_opt': 1, 'answer_mem_type': 1, 'answer_dropout_p': 0.1, 'answer_weight_norm_on': False, 'dump_state_on': False, 'answer_opt': [0], 'label_size': '2', 'mtl_opt': 0, 'ratio': 0, 'mix_opt': 0, 'max_seq_len': 512, 'init_ratio': 1, 'encoder_type': <EncoderModelType.BERT: 1>, 'cuda': True, 'log_per_updates': 500, 'save_per_updates': 10000, 'save_per_updates_on': False, 'epochs': 5, 'batch_size': 8, 'batch_size_eval': 8, 'optimizer': 'adamax', 'grad_clipping': 0, 'global_grad_clipping': 1.0, 'weight_decay': 0, 'learning_rate': 5e-05, 'momentum': 0, 'warmup': 0.1, 'warmup_schedule': 'warmup_linear', 'vb_dropout': True, 'dropout_p': 0.1, 'dropout_w': 0.0, 'bert_dropout_p': 0.1, 'model_ckpt': 'checkpoints/model_0.pt', 'resume': False, 'ema_opt': 0, 'ema_gamma': 0.995, 'have_lr_scheduler': True, 'multi_step_lr': '10,20,30', 'freeze_layers': -1, 'embedding_opt': 0, 'lr_gamma': 0.5, 'bert_l2norm': 0.0, 'scheduler_type': 'ms', 'output_dir': 'checkpoint', 'seed': 2018, 'grad_accumulation_step': 1, 'fp16': False, 'fp16_opt_level': 'O1', 'tasks_dropout_p': [0.1]}
08/19/2019 10:09:32 ####################
08/19/2019 10:09:32 ############# Gradient Accumulation Info #############
08/19/2019 10:09:32 number of step: 42095
08/19/2019 10:09:32 number of grad grad_accumulation step: 1
08/19/2019 10:09:32 adjusted number of step: 42095
08/19/2019 10:09:32 ############# Gradient Accumulation Info #############
08/19/2019 10:09:35 
############# Model Arch of MT-DNN #############
SANBertNetwork(
  (dropout_list): ModuleList(
    (0): DropoutWrapper()
  )
  (bert): BertModel(
    (embeddings): BertEmbeddings(
      (word_embeddings): Embedding(30522, 768)
      (position_embeddings): Embedding(512, 768)
      (token_type_embeddings): Embedding(2, 768)
      (LayerNorm): BertLayerNorm()
      (dropout): Dropout(p=0.1)
    )
    (encoder): BertEncoder(
      (layer): ModuleList(
        (0): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (1): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (2): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (3): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (4): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (5): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (6): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (7): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (8): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (9): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (10): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (11): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
      )
    )
    (pooler): BertPooler(
      (dense): Linear(in_features=768, out_features=768, bias=True)
      (activation): Tanh()
    )
  )
  (scoring_list): ModuleList(
    (0): Linear(in_features=768, out_features=2, bias=True)
  )
)

08/19/2019 10:09:35 Total number of params: 109483778
08/19/2019 10:09:35 At epoch 0
08/19/2019 10:09:36 Task [ 0] updates[     1] train loss[0.60717] remaining[1:26:22]
08/19/2019 10:12:11 Task [ 0] updates[   500] train loss[0.66817] remaining[0:40:58]
08/19/2019 10:15:15 Task [ 0] updates[  1000] train loss[0.54980] remaining[0:41:55]
08/19/2019 10:18:52 Task [ 0] updates[  1500] train loss[0.48298] remaining[0:42:48]
08/19/2019 10:22:16 Task [ 0] updates[  2000] train loss[0.44683] remaining[0:40:41]
08/19/2019 10:25:38 Task [ 0] updates[  2500] train loss[0.42590] remaining[0:37:58]
08/19/2019 10:29:04 Task [ 0] updates[  3000] train loss[0.40850] remaining[0:35:10]
08/19/2019 10:32:57 Task [ 0] updates[  3500] train loss[0.39546] remaining[0:32:49]
08/19/2019 10:36:19 Task [ 0] updates[  4000] train loss[0.38379] remaining[0:29:31]
08/19/2019 10:39:41 Task [ 0] updates[  4500] train loss[0.37479] remaining[0:26:12]
08/19/2019 10:44:02 Task [ 0] updates[  5000] train loss[0.36676] remaining[0:23:33]
08/19/2019 10:46:37 Task [ 0] updates[  5500] train loss[0.35867] remaining[0:19:38]
08/19/2019 10:49:12 Task [ 0] updates[  6000] train loss[0.35264] remaining[0:15:58]
08/19/2019 10:51:43 Task [ 0] updates[  6500] train loss[0.34506] remaining[0:12:26]
08/19/2019 10:54:13 Task [ 0] updates[  7000] train loss[0.33864] remaining[0:09:02]
08/19/2019 10:56:43 Task [ 0] updates[  7500] train loss[0.33283] remaining[0:05:46]
08/19/2019 10:59:13 Task [ 0] updates[  8000] train loss[0.32780] remaining[0:02:35]
08/19/2019 11:01:22 Task sst -- epoch 0 -- Dev ACC: 91.858
08/19/2019 11:01:32 [new test scores saved.]
08/19/2019 11:01:32 At epoch 1
08/19/2019 11:01:57 Task [ 0] updates[  8500] train loss[0.32275] remaining[0:41:56]
08/19/2019 11:04:27 Task [ 0] updates[  9000] train loss[0.31659] remaining[0:39:16]
08/19/2019 11:06:57 Task [ 0] updates[  9500] train loss[0.31136] remaining[0:36:41]
08/19/2019 11:09:25 Task [ 0] updates[ 10000] train loss[0.30516] remaining[0:34:06]
08/19/2019 11:11:55 Task [ 0] updates[ 10500] train loss[0.29946] remaining[0:31:35]
08/19/2019 11:14:24 Task [ 0] updates[ 11000] train loss[0.29480] remaining[0:29:06]
08/19/2019 11:16:53 Task [ 0] updates[ 11500] train loss[0.29088] remaining[0:26:35]
08/19/2019 11:19:22 Task [ 0] updates[ 12000] train loss[0.28619] remaining[0:24:05]
08/19/2019 11:21:51 Task [ 0] updates[ 12500] train loss[0.28202] remaining[0:21:35]
08/19/2019 11:24:20 Task [ 0] updates[ 13000] train loss[0.27850] remaining[0:19:06]
08/19/2019 11:26:49 Task [ 0] updates[ 13500] train loss[0.27509] remaining[0:16:36]
08/19/2019 11:29:19 Task [ 0] updates[ 14000] train loss[0.27180] remaining[0:14:07]
08/19/2019 11:31:49 Task [ 0] updates[ 14500] train loss[0.26913] remaining[0:11:38]
08/19/2019 11:34:18 Task [ 0] updates[ 15000] train loss[0.26634] remaining[0:09:08]
08/19/2019 11:36:47 Task [ 0] updates[ 15500] train loss[0.26341] remaining[0:06:39]
08/19/2019 11:39:16 Task [ 0] updates[ 16000] train loss[0.26118] remaining[0:04:10]
08/19/2019 11:41:45 Task [ 0] updates[ 16500] train loss[0.25849] remaining[0:01:40]
08/19/2019 11:43:30 Task sst -- epoch 1 -- Dev ACC: 91.858
08/19/2019 11:43:39 [new test scores saved.]
08/19/2019 11:43:40 At epoch 2
08/19/2019 11:44:28 Task [ 0] updates[ 17000] train loss[0.25545] remaining[0:40:57]
08/19/2019 11:46:56 Task [ 0] updates[ 17500] train loss[0.25142] remaining[0:38:25]
08/19/2019 11:49:25 Task [ 0] updates[ 18000] train loss[0.24841] remaining[0:35:57]
08/19/2019 11:51:53 Task [ 0] updates[ 18500] train loss[0.24501] remaining[0:33:26]
08/19/2019 11:54:22 Task [ 0] updates[ 19000] train loss[0.24176] remaining[0:31:00]
08/19/2019 11:56:51 Task [ 0] updates[ 19500] train loss[0.23862] remaining[0:28:30]
08/19/2019 11:59:20 Task [ 0] updates[ 20000] train loss[0.23602] remaining[0:26:03]
08/20/2019 12:01:48 Task [ 0] updates[ 20500] train loss[0.23333] remaining[0:23:34]
08/20/2019 12:04:17 Task [ 0] updates[ 21000] train loss[0.23095] remaining[0:21:05]
08/20/2019 12:06:45 Task [ 0] updates[ 21500] train loss[0.22856] remaining[0:18:36]
08/20/2019 12:09:13 Task [ 0] updates[ 22000] train loss[0.22653] remaining[0:16:07]
08/20/2019 12:11:42 Task [ 0] updates[ 22500] train loss[0.22472] remaining[0:13:39]
08/20/2019 12:14:11 Task [ 0] updates[ 23000] train loss[0.22308] remaining[0:11:10]
08/20/2019 12:16:40 Task [ 0] updates[ 23500] train loss[0.22132] remaining[0:08:42]
08/20/2019 12:19:08 Task [ 0] updates[ 24000] train loss[0.21936] remaining[0:06:13]
08/20/2019 12:21:37 Task [ 0] updates[ 24500] train loss[0.21768] remaining[0:03:44]
08/20/2019 12:24:05 Task [ 0] updates[ 25000] train loss[0.21586] remaining[0:01:16]
08/20/2019 12:25:26 Task sst -- epoch 2 -- Dev ACC: 92.890
08/20/2019 12:25:35 [new test scores saved.]
08/20/2019 12:25:35 At epoch 3
08/20/2019 12:26:47 Task [ 0] updates[ 25500] train loss[0.21381] remaining[0:40:23]
08/20/2019 12:29:16 Task [ 0] updates[ 26000] train loss[0.21135] remaining[0:37:58]
08/20/2019 12:31:44 Task [ 0] updates[ 26500] train loss[0.20893] remaining[0:35:28]
08/20/2019 12:34:12 Task [ 0] updates[ 27000] train loss[0.20671] remaining[0:33:00]
08/20/2019 12:36:41 Task [ 0] updates[ 27500] train loss[0.20465] remaining[0:30:32]
08/20/2019 12:39:09 Task [ 0] updates[ 28000] train loss[0.20260] remaining[0:28:03]
08/20/2019 12:41:37 Task [ 0] updates[ 28500] train loss[0.20075] remaining[0:25:35]
08/20/2019 12:44:06 Task [ 0] updates[ 29000] train loss[0.19919] remaining[0:23:07]
08/20/2019 12:46:34 Task [ 0] updates[ 29500] train loss[0.19771] remaining[0:20:38]
08/20/2019 12:49:02 Task [ 0] updates[ 30000] train loss[0.19610] remaining[0:18:10]
08/20/2019 12:51:32 Task [ 0] updates[ 30500] train loss[0.19471] remaining[0:15:42]
08/20/2019 12:54:00 Task [ 0] updates[ 31000] train loss[0.19319] remaining[0:13:14]
08/20/2019 12:56:29 Task [ 0] updates[ 31500] train loss[0.19193] remaining[0:10:46]
08/20/2019 12:58:57 Task [ 0] updates[ 32000] train loss[0.19082] remaining[0:08:17]
08/20/2019 01:01:26 Task [ 0] updates[ 32500] train loss[0.18936] remaining[0:05:49]
08/20/2019 01:03:54 Task [ 0] updates[ 33000] train loss[0.18786] remaining[0:03:20]
08/20/2019 01:06:23 Task [ 0] updates[ 33500] train loss[0.18678] remaining[0:00:52]
08/20/2019 01:07:20 Task sst -- epoch 3 -- Dev ACC: 92.317
08/20/2019 01:07:30 [new test scores saved.]
08/20/2019 01:07:30 At epoch 4
08/20/2019 01:09:06 Task [ 0] updates[ 34000] train loss[0.18528] remaining[0:40:08]
08/20/2019 01:11:34 Task [ 0] updates[ 34500] train loss[0.18379] remaining[0:37:36]
08/20/2019 01:14:03 Task [ 0] updates[ 35000] train loss[0.18204] remaining[0:35:07]
08/20/2019 01:16:31 Task [ 0] updates[ 35500] train loss[0.18050] remaining[0:32:37]
08/20/2019 01:18:59 Task [ 0] updates[ 36000] train loss[0.17920] remaining[0:30:09]
08/20/2019 01:21:28 Task [ 0] updates[ 36500] train loss[0.17781] remaining[0:27:40]
08/20/2019 01:23:56 Task [ 0] updates[ 37000] train loss[0.17640] remaining[0:25:12]
08/20/2019 01:26:25 Task [ 0] updates[ 37500] train loss[0.17531] remaining[0:22:43]
08/20/2019 01:28:52 Task [ 0] updates[ 38000] train loss[0.17428] remaining[0:20:14]
08/20/2019 01:31:21 Task [ 0] updates[ 38500] train loss[0.17308] remaining[0:17:46]
08/20/2019 01:33:50 Task [ 0] updates[ 39000] train loss[0.17174] remaining[0:15:18]
08/20/2019 01:36:18 Task [ 0] updates[ 39500] train loss[0.17064] remaining[0:12:50]
08/20/2019 01:38:46 Task [ 0] updates[ 40000] train loss[0.16940] remaining[0:10:21]
08/20/2019 01:41:15 Task [ 0] updates[ 40500] train loss[0.16828] remaining[0:07:53]
08/20/2019 01:43:43 Task [ 0] updates[ 41000] train loss[0.16713] remaining[0:05:24]
08/20/2019 01:46:11 Task [ 0] updates[ 41500] train loss[0.16611] remaining[0:02:56]
08/20/2019 01:48:40 Task [ 0] updates[ 42000] train loss[0.16493] remaining[0:00:28]
08/20/2019 01:49:12 Task sst -- epoch 4 -- Dev ACC: 92.546
08/20/2019 01:49:22 [new test scores saved.]
08/20/2019 08:56:39 0
08/20/2019 08:56:39 Launching the MT-DNN training
08/20/2019 08:56:39 Loading data\canonical_data\bert_uncased\sst_train.json as task 0
08/20/2019 08:56:39 2
08/20/2019 08:56:39 ####################
08/20/2019 08:56:39 {'log_file': 'mt-dnn-train.log', 'init_checkpoint': 'mt_dnn_models/bert_model_base_uncased.pt', 'data_dir': 'data\\canonical_data\\bert_uncased', 'data_sort_on': False, 'name': 'farmer', 'task_def': 'experiments/glue/glue_task_def.yml', 'train_datasets': ['sst'], 'test_datasets': ['sst'], 'update_bert_opt': 0, 'multi_gpu_on': False, 'mem_cum_type': 'simple', 'answer_num_turn': 5, 'answer_mem_drop_p': 0.1, 'answer_att_hidden_size': 128, 'answer_att_type': 'bilinear', 'answer_rnn_type': 'gru', 'answer_sum_att_type': 'bilinear', 'answer_merge_opt': 1, 'answer_mem_type': 1, 'answer_dropout_p': 0.1, 'answer_weight_norm_on': False, 'dump_state_on': False, 'answer_opt': [0], 'label_size': '2', 'mtl_opt': 0, 'ratio': 0, 'mix_opt': 0, 'max_seq_len': 512, 'init_ratio': 1, 'encoder_type': <EncoderModelType.BERT: 1>, 'cuda': True, 'log_per_updates': 500, 'save_per_updates': 10000, 'save_per_updates_on': False, 'epochs': 5, 'batch_size': 8, 'batch_size_eval': 8, 'optimizer': 'adamax', 'grad_clipping': 0, 'global_grad_clipping': 1.0, 'weight_decay': 0, 'learning_rate': 5e-05, 'momentum': 0, 'warmup': 0.1, 'warmup_schedule': 'warmup_linear', 'vb_dropout': True, 'dropout_p': 0.1, 'dropout_w': 0.0, 'bert_dropout_p': 0.1, 'model_ckpt': 'checkpoints/model_0.pt', 'resume': False, 'ema_opt': 0, 'ema_gamma': 0.995, 'have_lr_scheduler': True, 'multi_step_lr': '10,20,30', 'freeze_layers': -1, 'embedding_opt': 0, 'lr_gamma': 0.5, 'bert_l2norm': 0.0, 'scheduler_type': 'ms', 'output_dir': 'checkpoint', 'seed': 2018, 'grad_accumulation_step': 1, 'fp16': False, 'fp16_opt_level': 'O1', 'tasks_dropout_p': [0.1]}
08/20/2019 08:56:39 ####################
08/20/2019 08:56:39 ############# Gradient Accumulation Info #############
08/20/2019 08:56:39 number of step: 42095
08/20/2019 08:56:39 number of grad grad_accumulation step: 1
08/20/2019 08:56:39 adjusted number of step: 42095
08/20/2019 08:56:39 ############# Gradient Accumulation Info #############
08/20/2019 08:56:43 
############# Model Arch of MT-DNN #############
SANBertNetwork(
  (dropout_list): ModuleList(
    (0): DropoutWrapper()
  )
  (bert): BertModel(
    (embeddings): BertEmbeddings(
      (word_embeddings): Embedding(30522, 768)
      (position_embeddings): Embedding(512, 768)
      (token_type_embeddings): Embedding(2, 768)
      (LayerNorm): BertLayerNorm()
      (dropout): Dropout(p=0.1)
    )
    (encoder): BertEncoder(
      (layer): ModuleList(
        (0): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (1): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (2): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (3): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (4): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (5): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (6): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (7): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (8): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (9): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (10): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (11): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
      )
    )
    (pooler): BertPooler(
      (dense): Linear(in_features=768, out_features=768, bias=True)
      (activation): Tanh()
    )
  )
  (scoring_list): ModuleList(
    (0): Linear(in_features=768, out_features=2, bias=True)
  )
)

08/20/2019 08:56:43 Total number of params: 109483778
08/20/2019 08:56:43 At epoch 0
08/20/2019 08:56:44 Task [ 0] updates[     1] train loss[0.60717] remaining[1:23:21]
08/20/2019 08:59:24 Task [ 0] updates[   500] train loss[0.66817] remaining[0:42:26]
08/20/2019 09:02:04 Task [ 0] updates[  1000] train loss[0.54980] remaining[0:39:42]
08/20/2019 09:04:43 Task [ 0] updates[  1500] train loss[0.48298] remaining[0:36:53]
08/20/2019 09:07:21 Task [ 0] updates[  2000] train loss[0.44683] remaining[0:34:08]
08/20/2019 09:10:02 Task [ 0] updates[  2500] train loss[0.42590] remaining[0:31:31]
08/20/2019 09:13:15 Task [ 0] updates[  3000] train loss[0.40850] remaining[0:29:50]
08/20/2019 09:17:06 Task [ 0] updates[  3500] train loss[0.39546] remaining[0:28:39]
08/20/2019 09:21:01 Task [ 0] updates[  4000] train loss[0.38379] remaining[0:26:50]
08/20/2019 09:25:00 Task [ 0] updates[  4500] train loss[0.37479] remaining[0:24:37]
08/20/2019 09:29:12 Task [ 0] updates[  5000] train loss[0.36676] remaining[0:22:12]
08/20/2019 09:33:12 Task [ 0] updates[  5500] train loss[0.35867] remaining[0:19:21]
08/20/2019 09:36:57 Task [ 0] updates[  6000] train loss[0.35264] remaining[0:16:13]
08/20/2019 09:41:15 Task [ 0] updates[  6500] train loss[0.34506] remaining[0:13:08]
08/20/2019 09:44:55 Task [ 0] updates[  7000] train loss[0.33864] remaining[0:09:46]
08/20/2019 09:48:47 Task [ 0] updates[  7500] train loss[0.33283] remaining[0:06:22]
08/20/2019 09:52:22 Task [ 0] updates[  8000] train loss[0.32780] remaining[0:02:54]
08/20/2019 09:55:35 Task sst -- epoch 0 -- Dev ACC: 91.858
08/20/2019 09:55:49 [new test scores saved.]
08/20/2019 09:55:49 At epoch 1
08/20/2019 09:56:24 Task [ 0] updates[  8500] train loss[0.32275] remaining[1:01:12]
08/20/2019 09:59:52 Task [ 0] updates[  9000] train loss[0.31659] remaining[0:54:45]
08/20/2019 10:03:44 Task [ 0] updates[  9500] train loss[0.31136] remaining[0:53:49]
08/20/2019 10:07:09 Task [ 0] updates[ 10000] train loss[0.30516] remaining[0:49:03]
08/20/2019 10:10:20 Task [ 0] updates[ 10500] train loss[0.29946] remaining[0:44:15]
08/20/2019 10:13:31 Task [ 0] updates[ 11000] train loss[0.29480] remaining[0:40:03]
08/20/2019 10:16:41 Task [ 0] updates[ 11500] train loss[0.29088] remaining[0:36:10]
08/20/2019 10:19:51 Task [ 0] updates[ 12000] train loss[0.28619] remaining[0:32:29]
08/20/2019 10:23:02 Task [ 0] updates[ 12500] train loss[0.28202] remaining[0:28:56]
08/20/2019 10:26:24 Task [ 0] updates[ 13000] train loss[0.27850] remaining[0:25:37]
08/20/2019 10:29:46 Task [ 0] updates[ 13500] train loss[0.27509] remaining[0:22:18]
08/20/2019 10:32:50 Task [ 0] updates[ 14000] train loss[0.27180] remaining[0:18:49]
08/20/2019 10:35:52 Task [ 0] updates[ 14500] train loss[0.26913] remaining[0:15:24]
08/20/2019 10:38:56 Task [ 0] updates[ 15000] train loss[0.26634] remaining[0:12:02]
08/20/2019 10:42:05 Task [ 0] updates[ 15500] train loss[0.26341] remaining[0:08:44]
08/20/2019 10:45:14 Task [ 0] updates[ 16000] train loss[0.26118] remaining[0:05:27]
08/20/2019 10:48:18 Task [ 0] updates[ 16500] train loss[0.25849] remaining[0:02:11]
08/20/2019 10:50:28 Task sst -- epoch 1 -- Dev ACC: 91.858
08/20/2019 10:50:39 [new test scores saved.]
08/20/2019 10:50:39 At epoch 2
08/20/2019 10:51:38 Task [ 0] updates[ 17000] train loss[0.25545] remaining[0:50:13]
08/20/2019 10:54:43 Task [ 0] updates[ 17500] train loss[0.25142] remaining[0:47:35]
08/20/2019 10:57:25 Task [ 0] updates[ 18000] train loss[0.24841] remaining[0:42:12]
08/20/2019 11:00:03 Task [ 0] updates[ 18500] train loss[0.24501] remaining[0:38:12]
08/20/2019 11:02:42 Task [ 0] updates[ 19000] train loss[0.24176] remaining[0:34:50]
08/20/2019 11:05:20 Task [ 0] updates[ 19500] train loss[0.23862] remaining[0:31:44]
08/20/2019 11:08:01 Task [ 0] updates[ 20000] train loss[0.23602] remaining[0:28:51]
08/20/2019 11:10:39 Task [ 0] updates[ 20500] train loss[0.23333] remaining[0:25:58]
08/20/2019 11:13:20 Task [ 0] updates[ 21000] train loss[0.23095] remaining[0:23:12]
08/20/2019 11:16:01 Task [ 0] updates[ 21500] train loss[0.22856] remaining[0:20:26]
08/20/2019 11:18:44 Task [ 0] updates[ 22000] train loss[0.22653] remaining[0:17:43]
08/20/2019 11:21:24 Task [ 0] updates[ 22500] train loss[0.22472] remaining[0:14:58]
08/20/2019 11:24:05 Task [ 0] updates[ 23000] train loss[0.22308] remaining[0:12:14]
08/20/2019 11:26:43 Task [ 0] updates[ 23500] train loss[0.22132] remaining[0:09:30]
08/20/2019 11:29:23 Task [ 0] updates[ 24000] train loss[0.21936] remaining[0:06:47]
08/20/2019 11:32:05 Task [ 0] updates[ 24500] train loss[0.21768] remaining[0:04:05]
08/20/2019 11:34:45 Task [ 0] updates[ 25000] train loss[0.21586] remaining[0:01:23]
08/20/2019 11:36:13 Task sst -- epoch 2 -- Dev ACC: 92.890
08/20/2019 11:36:23 [new test scores saved.]
08/20/2019 11:36:23 At epoch 3
08/20/2019 11:37:41 Task [ 0] updates[ 25500] train loss[0.21381] remaining[0:43:40]
08/20/2019 11:40:21 Task [ 0] updates[ 26000] train loss[0.21135] remaining[0:40:56]
08/20/2019 11:43:02 Task [ 0] updates[ 26500] train loss[0.20893] remaining[0:38:21]
08/20/2019 11:45:42 Task [ 0] updates[ 27000] train loss[0.20671] remaining[0:35:40]
08/20/2019 11:48:21 Task [ 0] updates[ 27500] train loss[0.20465] remaining[0:32:54]
08/20/2019 11:51:05 Task [ 0] updates[ 28000] train loss[0.20260] remaining[0:30:24]
08/20/2019 11:53:52 Task [ 0] updates[ 28500] train loss[0.20075] remaining[0:27:53]
08/20/2019 11:54:56 0
08/20/2019 11:54:56 Launching the MT-DNN training
08/20/2019 11:54:56 Loading data\canonical_data\bert_uncased\sst_train.json as task 0
08/20/2019 11:54:57 2
08/20/2019 11:54:57 ####################
08/20/2019 11:54:57 {'log_file': 'mt-dnn-train.log', 'init_checkpoint': 'mt_dnn_models/bert_model_base_uncased.pt', 'data_dir': 'data\\canonical_data\\bert_uncased', 'data_sort_on': False, 'name': 'farmer', 'task_def': 'experiments/glue/glue_task_def.yml', 'train_datasets': ['sst'], 'test_datasets': ['sst'], 'update_bert_opt': 0, 'multi_gpu_on': False, 'mem_cum_type': 'simple', 'answer_num_turn': 5, 'answer_mem_drop_p': 0.1, 'answer_att_hidden_size': 128, 'answer_att_type': 'bilinear', 'answer_rnn_type': 'gru', 'answer_sum_att_type': 'bilinear', 'answer_merge_opt': 1, 'answer_mem_type': 1, 'answer_dropout_p': 0.1, 'answer_weight_norm_on': False, 'dump_state_on': False, 'answer_opt': [0], 'label_size': '2', 'mtl_opt': 0, 'ratio': 0, 'mix_opt': 0, 'max_seq_len': 512, 'init_ratio': 1, 'encoder_type': <EncoderModelType.BERT: 1>, 'cuda': True, 'log_per_updates': 500, 'save_per_updates': 10000, 'save_per_updates_on': False, 'epochs': 5, 'batch_size': 8, 'batch_size_eval': 8, 'optimizer': 'adamax', 'grad_clipping': 0, 'global_grad_clipping': 1.0, 'weight_decay': 0, 'learning_rate': 5e-05, 'momentum': 0, 'warmup': 0.1, 'warmup_schedule': 'warmup_linear', 'vb_dropout': True, 'dropout_p': 0.1, 'dropout_w': 0.0, 'bert_dropout_p': 0.1, 'model_ckpt': 'checkpoints/model_0.pt', 'resume': False, 'ema_opt': 0, 'ema_gamma': 0.995, 'have_lr_scheduler': True, 'multi_step_lr': '10,20,30', 'freeze_layers': -1, 'embedding_opt': 0, 'lr_gamma': 0.5, 'bert_l2norm': 0.0, 'scheduler_type': 'ms', 'output_dir': 'checkpoint', 'seed': 2018, 'grad_accumulation_step': 1, 'fp16': False, 'fp16_opt_level': 'O1', 'tasks_dropout_p': [0.1]}
08/20/2019 11:54:57 ####################
08/20/2019 11:54:57 ############# Gradient Accumulation Info #############
08/20/2019 11:54:57 number of step: 42095
08/20/2019 11:54:57 number of grad grad_accumulation step: 1
08/20/2019 11:54:57 adjusted number of step: 42095
08/20/2019 11:54:57 ############# Gradient Accumulation Info #############
08/20/2019 11:55:52 0
08/20/2019 11:55:52 Launching the MT-DNN training
08/20/2019 11:55:52 Loading data\canonical_data\bert_uncased\sst_train.json as task 0
08/20/2019 11:55:52 2
08/20/2019 11:55:52 ####################
08/20/2019 11:55:52 {'log_file': 'mt-dnn-train.log', 'init_checkpoint': 'mt_dnn_models/bert_model_base_uncased.pt', 'data_dir': 'data\\canonical_data\\bert_uncased', 'data_sort_on': False, 'name': 'farmer', 'task_def': 'experiments/glue/glue_task_def.yml', 'train_datasets': ['sst'], 'test_datasets': ['sst'], 'update_bert_opt': 0, 'multi_gpu_on': False, 'mem_cum_type': 'simple', 'answer_num_turn': 5, 'answer_mem_drop_p': 0.1, 'answer_att_hidden_size': 128, 'answer_att_type': 'bilinear', 'answer_rnn_type': 'gru', 'answer_sum_att_type': 'bilinear', 'answer_merge_opt': 1, 'answer_mem_type': 1, 'answer_dropout_p': 0.1, 'answer_weight_norm_on': False, 'dump_state_on': False, 'answer_opt': [0], 'label_size': '2', 'mtl_opt': 0, 'ratio': 0, 'mix_opt': 0, 'max_seq_len': 512, 'init_ratio': 1, 'encoder_type': <EncoderModelType.BERT: 1>, 'cuda': True, 'log_per_updates': 500, 'save_per_updates': 10000, 'save_per_updates_on': False, 'epochs': 5, 'batch_size': 8, 'batch_size_eval': 8, 'optimizer': 'adamax', 'grad_clipping': 0, 'global_grad_clipping': 1.0, 'weight_decay': 0, 'learning_rate': 5e-05, 'momentum': 0, 'warmup': 0.1, 'warmup_schedule': 'warmup_linear', 'vb_dropout': True, 'dropout_p': 0.1, 'dropout_w': 0.0, 'bert_dropout_p': 0.1, 'model_ckpt': 'checkpoints/model_0.pt', 'resume': False, 'ema_opt': 0, 'ema_gamma': 0.995, 'have_lr_scheduler': True, 'multi_step_lr': '10,20,30', 'freeze_layers': -1, 'embedding_opt': 0, 'lr_gamma': 0.5, 'bert_l2norm': 0.0, 'scheduler_type': 'ms', 'output_dir': 'checkpoint', 'seed': 2018, 'grad_accumulation_step': 1, 'fp16': False, 'fp16_opt_level': 'O1', 'tasks_dropout_p': [0.1]}
08/20/2019 11:55:52 ####################
08/20/2019 11:55:52 ############# Gradient Accumulation Info #############
08/20/2019 11:55:52 number of step: 42095
08/20/2019 11:55:52 number of grad grad_accumulation step: 1
08/20/2019 11:55:52 adjusted number of step: 42095
08/20/2019 11:55:52 ############# Gradient Accumulation Info #############
08/20/2019 11:55:56 
############# Model Arch of MT-DNN #############
SANBertNetwork(
  (dropout_list): ModuleList(
    (0): DropoutWrapper()
  )
  (bert): BertModel(
    (embeddings): BertEmbeddings(
      (word_embeddings): Embedding(30522, 768)
      (position_embeddings): Embedding(512, 768)
      (token_type_embeddings): Embedding(2, 768)
      (LayerNorm): BertLayerNorm()
      (dropout): Dropout(p=0.1)
    )
    (encoder): BertEncoder(
      (layer): ModuleList(
        (0): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (1): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (2): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (3): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (4): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (5): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (6): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (7): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (8): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (9): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (10): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (11): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
      )
    )
    (pooler): BertPooler(
      (dense): Linear(in_features=768, out_features=768, bias=True)
      (activation): Tanh()
    )
  )
  (scoring_list): ModuleList(
    (0): Linear(in_features=768, out_features=2, bias=True)
  )
)

08/20/2019 11:55:56 Total number of params: 109483778
08/20/2019 11:55:56 At epoch 0
08/20/2019 11:55:57 Task [ 0] updates[     1] train loss[0.60717] remaining[1:35:50]
08/20/2019 11:56:49 Task [ 0] updates[ 29000] train loss[0.19919] remaining[0:25:31]
08/20/2019 11:59:14 Task [ 0] updates[   500] train loss[0.66817] remaining[0:52:09]
08/20/2019 12:00:07 Task [ 0] updates[ 29500] train loss[0.19771] remaining[0:23:21]
08/20/2019 12:02:30 Task [ 0] updates[  1000] train loss[0.54980] remaining[0:48:43]
08/20/2019 12:03:25 Task [ 0] updates[ 30000] train loss[0.19610] remaining[0:20:56]
08/20/2019 12:05:46 Task [ 0] updates[  1500] train loss[0.48298] remaining[0:45:22]
08/20/2019 12:06:44 Task [ 0] updates[ 30500] train loss[0.19471] remaining[0:18:22]
08/20/2019 12:09:03 Task [ 0] updates[  2000] train loss[0.44683] remaining[0:42:04]
08/20/2019 12:10:00 Task [ 0] updates[ 31000] train loss[0.19319] remaining[0:15:39]
08/20/2019 12:12:21 Task [ 0] updates[  2500] train loss[0.42590] remaining[0:38:52]
08/20/2019 12:13:20 Task [ 0] updates[ 31500] train loss[0.19193] remaining[0:12:52]
08/20/2019 12:15:37 Task [ 0] updates[  3000] train loss[0.40850] remaining[0:35:33]
08/20/2019 12:16:37 Task [ 0] updates[ 32000] train loss[0.19082] remaining[0:09:59]
08/20/2019 12:18:54 Task [ 0] updates[  3500] train loss[0.39546] remaining[0:32:16]
08/20/2019 12:19:56 Task [ 0] updates[ 32500] train loss[0.18936] remaining[0:07:04]
08/20/2019 12:22:08 Task [ 0] updates[  4000] train loss[0.38379] remaining[0:28:56]
08/20/2019 12:23:14 Task [ 0] updates[ 33000] train loss[0.18786] remaining[0:04:05]
08/20/2019 01:47:21 Task [ 0] updates[  4500] train loss[0.37479] remaining[1:37:01]
08/20/2019 01:48:26 Task [ 0] updates[ 33500] train loss[0.18678] remaining[0:02:49]
08/20/2019 01:49:41 Task sst -- epoch 3 -- Dev ACC: 92.317
08/20/2019 01:49:53 [new test scores saved.]
08/20/2019 01:49:53 At epoch 4
08/20/2019 01:50:35 Task [ 0] updates[  5000] train loss[0.36676] remaining[1:18:24]
08/20/2019 01:51:58 Task [ 0] updates[ 34000] train loss[0.18528] remaining[0:52:02]
08/20/2019 01:53:49 Task [ 0] updates[  5500] train loss[0.35867] remaining[1:02:33]
08/20/2019 01:55:11 Task [ 0] updates[ 34500] train loss[0.18379] remaining[0:48:53]
08/20/2019 01:57:01 Task [ 0] updates[  6000] train loss[0.35264] remaining[0:48:48]
08/20/2019 08:04:13 0
08/20/2019 08:04:13 Launching the MT-DNN training
08/20/2019 08:04:13 Loading data\canonical_data\bert_uncased\sst_train.json as task 0
08/20/2019 08:04:14 2
08/20/2019 08:04:14 ####################
08/20/2019 08:04:14 {'log_file': 'mt-dnn-train.log', 'init_checkpoint': 'mt_dnn_models/bert_model_base_uncased.pt', 'data_dir': 'data\\canonical_data\\bert_uncased', 'data_sort_on': False, 'name': 'farmer', 'task_def': 'experiments/glue/glue_task_def.yml', 'train_datasets': ['sst'], 'test_datasets': ['sst'], 'update_bert_opt': 0, 'multi_gpu_on': False, 'mem_cum_type': 'simple', 'answer_num_turn': 5, 'answer_mem_drop_p': 0.1, 'answer_att_hidden_size': 128, 'answer_att_type': 'bilinear', 'answer_rnn_type': 'gru', 'answer_sum_att_type': 'bilinear', 'answer_merge_opt': 1, 'answer_mem_type': 1, 'answer_dropout_p': 0.1, 'answer_weight_norm_on': False, 'dump_state_on': False, 'answer_opt': [0], 'label_size': '2', 'mtl_opt': 0, 'ratio': 0, 'mix_opt': 0, 'max_seq_len': 512, 'init_ratio': 1, 'encoder_type': <EncoderModelType.BERT: 1>, 'cuda': True, 'log_per_updates': 500, 'save_per_updates': 10000, 'save_per_updates_on': False, 'epochs': 5, 'batch_size': 8, 'batch_size_eval': 8, 'optimizer': 'adamax', 'grad_clipping': 0, 'global_grad_clipping': 1.0, 'weight_decay': 0, 'learning_rate': 5e-05, 'momentum': 0, 'warmup': 0.1, 'warmup_schedule': 'warmup_linear', 'vb_dropout': True, 'dropout_p': 0.1, 'dropout_w': 0.0, 'bert_dropout_p': 0.1, 'model_ckpt': 'checkpoints/model_0.pt', 'resume': False, 'ema_opt': 0, 'ema_gamma': 0.995, 'have_lr_scheduler': True, 'multi_step_lr': '10,20,30', 'freeze_layers': -1, 'embedding_opt': 0, 'lr_gamma': 0.5, 'bert_l2norm': 0.0, 'scheduler_type': 'ms', 'output_dir': 'checkpoint', 'seed': 2018, 'grad_accumulation_step': 1, 'fp16': False, 'fp16_opt_level': 'O1', 'tasks_dropout_p': [0.1]}
08/20/2019 08:04:14 ####################
08/20/2019 08:04:14 ############# Gradient Accumulation Info #############
08/20/2019 08:04:14 number of step: 42095
08/20/2019 08:04:14 number of grad grad_accumulation step: 1
08/20/2019 08:04:14 adjusted number of step: 42095
08/20/2019 08:04:14 ############# Gradient Accumulation Info #############
08/20/2019 08:04:18 
############# Model Arch of MT-DNN #############
SANBertNetwork(
  (dropout_list): ModuleList(
    (0): DropoutWrapper()
  )
  (bert): BertModel(
    (embeddings): BertEmbeddings(
      (word_embeddings): Embedding(30522, 768)
      (position_embeddings): Embedding(512, 768)
      (token_type_embeddings): Embedding(2, 768)
      (LayerNorm): BertLayerNorm()
      (dropout): Dropout(p=0.1)
    )
    (encoder): BertEncoder(
      (layer): ModuleList(
        (0): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (1): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (2): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (3): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (4): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (5): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (6): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (7): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (8): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (9): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (10): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (11): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
      )
    )
    (pooler): BertPooler(
      (dense): Linear(in_features=768, out_features=768, bias=True)
      (activation): Tanh()
    )
  )
  (scoring_list): ModuleList(
    (0): Linear(in_features=768, out_features=2, bias=True)
  )
)

08/20/2019 08:04:18 Total number of params: 109483778
08/20/2019 08:04:18 At epoch 0
08/20/2019 08:04:19 Task [ 0] updates[     1] train loss[0.60717] remaining[1:32:05]
08/20/2019 08:07:05 Task [ 0] updates[   500] train loss[0.66817] remaining[0:44:09]
08/20/2019 08:10:01 Task [ 0] updates[  1000] train loss[0.54980] remaining[0:42:22]
08/20/2019 08:13:01 Task [ 0] updates[  1500] train loss[0.48298] remaining[0:40:12]
08/20/2019 08:16:06 Task [ 0] updates[  2000] train loss[0.44683] remaining[0:37:51]
08/20/2019 08:19:06 Task [ 0] updates[  2500] train loss[0.42590] remaining[0:35:02]
08/20/2019 08:22:19 Task [ 0] updates[  3000] train loss[0.40850] remaining[0:32:32]
08/20/2019 08:25:30 Task [ 0] updates[  3500] train loss[0.39546] remaining[0:29:47]
08/20/2019 08:28:33 Task [ 0] updates[  4000] train loss[0.38379] remaining[0:26:46]
08/20/2019 08:31:50 Task [ 0] updates[  4500] train loss[0.37479] remaining[0:23:58]
08/20/2019 08:35:18 Task [ 0] updates[  5000] train loss[0.36676] remaining[0:21:11]
08/20/2019 08:38:24 Task [ 0] updates[  5500] train loss[0.35867] remaining[0:18:05]
08/20/2019 08:41:25 Task [ 0] updates[  6000] train loss[0.35264] remaining[0:14:57]
08/20/2019 08:44:25 Task [ 0] updates[  6500] train loss[0.34506] remaining[0:11:50]
08/20/2019 08:47:23 Task [ 0] updates[  7000] train loss[0.33864] remaining[0:08:43]
08/20/2019 08:50:32 Task [ 0] updates[  7500] train loss[0.33283] remaining[0:05:39]
08/20/2019 08:53:42 Task [ 0] updates[  8000] train loss[0.32780] remaining[0:02:35]
08/20/2019 08:56:29 Task sst -- epoch 0 -- Dev ACC: 91.858
08/20/2019 08:56:42 [new test scores saved.]
08/20/2019 08:56:42 At epoch 1
08/20/2019 08:57:11 Task [ 0] updates[  8500] train loss[0.32275] remaining[0:50:42]
08/20/2019 09:00:05 Task [ 0] updates[  9000] train loss[0.31659] remaining[0:45:42]
08/20/2019 09:03:13 Task [ 0] updates[  9500] train loss[0.31136] remaining[0:44:13]
08/20/2019 09:06:13 Task [ 0] updates[ 10000] train loss[0.30516] remaining[0:41:11]
08/20/2019 09:09:10 Task [ 0] updates[ 10500] train loss[0.29946] remaining[0:37:58]
08/20/2019 09:12:20 Task [ 0] updates[ 11000] train loss[0.29480] remaining[0:35:22]
08/20/2019 09:15:23 Task [ 0] updates[ 11500] train loss[0.29088] remaining[0:32:22]
08/20/2019 09:18:22 Task [ 0] updates[ 12000] train loss[0.28619] remaining[0:29:17]
08/20/2019 09:21:32 Task [ 0] updates[ 12500] train loss[0.28202] remaining[0:26:24]
08/20/2019 09:24:42 Task [ 0] updates[ 13000] train loss[0.27850] remaining[0:23:27]
08/20/2019 09:27:43 Task [ 0] updates[ 13500] train loss[0.27509] remaining[0:20:22]
08/20/2019 09:30:36 Task [ 0] updates[ 14000] train loss[0.27180] remaining[0:17:14]
08/20/2019 09:33:23 Task [ 0] updates[ 14500] train loss[0.26913] remaining[0:14:06]
08/20/2019 09:34:42 0
08/20/2019 09:34:42 Launching the MT-DNN training
08/20/2019 09:34:42 Loading data\canonical_data\bert_uncased\sst_train.json as task 0
08/20/2019 09:34:42 2
08/20/2019 09:34:42 ####################
08/20/2019 09:34:42 {'log_file': 'mt-dnn-train.log', 'init_checkpoint': 'mt_dnn_models/bert_model_base_uncased.pt', 'data_dir': 'data\\canonical_data\\bert_uncased', 'data_sort_on': False, 'name': 'farmer', 'task_def': 'experiments/glue/glue_task_def.yml', 'train_datasets': ['sst'], 'test_datasets': ['sst'], 'update_bert_opt': 0, 'multi_gpu_on': False, 'mem_cum_type': 'simple', 'answer_num_turn': 5, 'answer_mem_drop_p': 0.1, 'answer_att_hidden_size': 128, 'answer_att_type': 'bilinear', 'answer_rnn_type': 'gru', 'answer_sum_att_type': 'bilinear', 'answer_merge_opt': 1, 'answer_mem_type': 1, 'answer_dropout_p': 0.1, 'answer_weight_norm_on': False, 'dump_state_on': False, 'answer_opt': [0], 'label_size': '2', 'mtl_opt': 0, 'ratio': 0, 'mix_opt': 0, 'max_seq_len': 512, 'init_ratio': 1, 'encoder_type': <EncoderModelType.BERT: 1>, 'cuda': True, 'log_per_updates': 500, 'save_per_updates': 10000, 'save_per_updates_on': False, 'epochs': 5, 'batch_size': 8, 'batch_size_eval': 8, 'optimizer': 'adamax', 'grad_clipping': 0, 'global_grad_clipping': 1.0, 'weight_decay': 0, 'learning_rate': 5e-05, 'momentum': 0, 'warmup': 0.1, 'warmup_schedule': 'warmup_linear', 'vb_dropout': True, 'dropout_p': 0.1, 'dropout_w': 0.0, 'bert_dropout_p': 0.1, 'model_ckpt': 'checkpoints/model_0.pt', 'resume': False, 'ema_opt': 0, 'ema_gamma': 0.995, 'have_lr_scheduler': True, 'multi_step_lr': '10,20,30', 'freeze_layers': -1, 'embedding_opt': 0, 'lr_gamma': 0.5, 'bert_l2norm': 0.0, 'scheduler_type': 'ms', 'output_dir': 'checkpoint', 'seed': 2018, 'grad_accumulation_step': 1, 'fp16': False, 'fp16_opt_level': 'O1', 'tasks_dropout_p': [0.1]}
08/20/2019 09:34:42 ####################
08/20/2019 09:34:42 ############# Gradient Accumulation Info #############
08/20/2019 09:34:42 number of step: 42095
08/20/2019 09:34:42 number of grad grad_accumulation step: 1
08/20/2019 09:34:42 adjusted number of step: 42095
08/20/2019 09:34:42 ############# Gradient Accumulation Info #############
08/20/2019 09:34:46 
############# Model Arch of MT-DNN #############
SANBertNetwork(
  (dropout_list): ModuleList(
    (0): DropoutWrapper()
  )
  (bert): BertModel(
    (embeddings): BertEmbeddings(
      (word_embeddings): Embedding(30522, 768)
      (position_embeddings): Embedding(512, 768)
      (token_type_embeddings): Embedding(2, 768)
      (LayerNorm): BertLayerNorm()
      (dropout): Dropout(p=0.1)
    )
    (encoder): BertEncoder(
      (layer): ModuleList(
        (0): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (1): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (2): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (3): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (4): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (5): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (6): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (7): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (8): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (9): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (10): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (11): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
      )
    )
    (pooler): BertPooler(
      (dense): Linear(in_features=768, out_features=768, bias=True)
      (activation): Tanh()
    )
  )
  (scoring_list): ModuleList(
    (0): Linear(in_features=768, out_features=2, bias=True)
  )
)

08/20/2019 09:34:46 Total number of params: 109483778
08/20/2019 09:34:46 At epoch 0
08/20/2019 09:34:46 Task [ 0] updates[     1] train loss[0.60717] remaining[1:21:27]
08/20/2019 09:37:22 Task [ 0] updates[   500] train loss[0.66817] remaining[0:41:09]
08/20/2019 09:40:04 Task [ 0] updates[  1000] train loss[0.54980] remaining[0:39:23]
08/20/2019 09:43:24 Task [ 0] updates[  1500] train loss[0.48298] remaining[0:39:48]
08/20/2019 09:46:54 Task [ 0] updates[  2000] train loss[0.44683] remaining[0:38:55]
08/20/2019 09:50:32 Task [ 0] updates[  2500] train loss[0.42590] remaining[0:37:20]
08/20/2019 09:55:00 Task [ 0] updates[  3000] train loss[0.40850] remaining[0:36:33]
08/20/2019 09:58:40 Task [ 0] updates[  3500] train loss[0.39546] remaining[0:33:36]
08/20/2019 10:01:20 Task [ 0] updates[  4000] train loss[0.38379] remaining[0:29:20]
08/20/2019 10:03:53 Task [ 0] updates[  4500] train loss[0.37479] remaining[0:25:21]
08/20/2019 10:06:27 Task [ 0] updates[  5000] train loss[0.36676] remaining[0:21:40]
08/20/2019 10:09:01 Task [ 0] updates[  5500] train loss[0.35867] remaining[0:18:10]
08/20/2019 10:11:34 Task [ 0] updates[  6000] train loss[0.35264] remaining[0:14:50]
08/20/2019 10:14:08 Task [ 0] updates[  6500] train loss[0.34506] remaining[0:11:37]
08/20/2019 10:16:42 Task [ 0] updates[  7000] train loss[0.33864] remaining[0:08:30]
08/20/2019 10:19:16 Task [ 0] updates[  7500] train loss[0.33283] remaining[0:05:27]
08/20/2019 10:21:50 Task [ 0] updates[  8000] train loss[0.32780] remaining[0:02:27]
08/20/2019 10:24:04 Task sst -- epoch 0 -- Dev ACC: 91.858
08/20/2019 10:24:13 [new test scores saved.]
08/20/2019 10:24:16 At epoch 1
08/20/2019 10:24:41 Task [ 0] updates[  8500] train loss[0.32275] remaining[0:43:07]
08/20/2019 10:27:14 Task [ 0] updates[  9000] train loss[0.31659] remaining[0:40:01]
08/20/2019 10:29:48 Task [ 0] updates[  9500] train loss[0.31136] remaining[0:37:34]
08/20/2019 10:32:22 Task [ 0] updates[ 10000] train loss[0.30516] remaining[0:35:05]
08/20/2019 10:34:57 Task [ 0] updates[ 10500] train loss[0.29946] remaining[0:32:33]
08/20/2019 10:37:32 Task [ 0] updates[ 11000] train loss[0.29480] remaining[0:30:01]
08/20/2019 10:40:05 Task [ 0] updates[ 11500] train loss[0.29088] remaining[0:27:25]
08/20/2019 10:42:40 Task [ 0] updates[ 12000] train loss[0.28619] remaining[0:24:52]
08/20/2019 10:45:14 Task [ 0] updates[ 12500] train loss[0.28202] remaining[0:22:18]
08/20/2019 10:47:49 Task [ 0] updates[ 13000] train loss[0.27850] remaining[0:19:44]
08/20/2019 10:50:24 Task [ 0] updates[ 13500] train loss[0.27509] remaining[0:17:10]
08/20/2019 10:53:01 Task [ 0] updates[ 14000] train loss[0.27180] remaining[0:14:37]
08/20/2019 10:55:51 Task [ 0] updates[ 14500] train loss[0.26913] remaining[0:12:08]
08/20/2019 10:58:48 Task [ 0] updates[ 15000] train loss[0.26634] remaining[0:09:38]
08/20/2019 11:01:42 Task [ 0] updates[ 15500] train loss[0.26341] remaining[0:07:04]
08/20/2019 11:04:37 Task [ 0] updates[ 16000] train loss[0.26118] remaining[0:04:27]
08/20/2019 11:07:31 Task [ 0] updates[ 16500] train loss[0.25849] remaining[0:01:48]
08/20/2019 11:09:34 Task sst -- epoch 1 -- Dev ACC: 91.858
08/20/2019 11:09:46 [new test scores saved.]
08/20/2019 11:09:48 At epoch 2
08/20/2019 11:10:44 Task [ 0] updates[ 17000] train loss[0.25545] remaining[0:47:54]
08/20/2019 11:13:37 Task [ 0] updates[ 17500] train loss[0.25142] remaining[0:44:45]
08/20/2019 11:16:33 Task [ 0] updates[ 18000] train loss[0.24841] remaining[0:42:10]
08/20/2019 11:19:27 Task [ 0] updates[ 18500] train loss[0.24501] remaining[0:39:13]
08/20/2019 11:22:19 Task [ 0] updates[ 19000] train loss[0.24176] remaining[0:36:14]
08/20/2019 11:25:13 Task [ 0] updates[ 19500] train loss[0.23862] remaining[0:33:20]
08/20/2019 11:28:08 Task [ 0] updates[ 20000] train loss[0.23602] remaining[0:30:28]
08/20/2019 11:31:02 Task [ 0] updates[ 20500] train loss[0.23333] remaining[0:27:35]
08/20/2019 11:33:55 Task [ 0] updates[ 21000] train loss[0.23095] remaining[0:24:40]
08/20/2019 11:36:53 Task [ 0] updates[ 21500] train loss[0.22856] remaining[0:21:49]
08/20/2019 11:39:48 Task [ 0] updates[ 22000] train loss[0.22653] remaining[0:18:55]
08/20/2019 11:42:41 Task [ 0] updates[ 22500] train loss[0.22472] remaining[0:16:00]
08/20/2019 11:45:34 Task [ 0] updates[ 23000] train loss[0.22308] remaining[0:13:06]
08/20/2019 11:48:28 Task [ 0] updates[ 23500] train loss[0.22132] remaining[0:10:11]
08/20/2019 11:51:21 Task [ 0] updates[ 24000] train loss[0.21936] remaining[0:07:17]
08/20/2019 11:54:16 Task [ 0] updates[ 24500] train loss[0.21768] remaining[0:04:23]
08/20/2019 11:57:09 Task [ 0] updates[ 25000] train loss[0.21586] remaining[0:01:29]
08/20/2019 11:58:45 Task sst -- epoch 2 -- Dev ACC: 92.890
08/20/2019 11:58:56 [new test scores saved.]
08/20/2019 11:58:59 At epoch 3
08/21/2019 12:00:24 Task [ 0] updates[ 25500] train loss[0.21381] remaining[0:47:50]
08/21/2019 12:03:18 Task [ 0] updates[ 26000] train loss[0.21135] remaining[0:44:41]
08/21/2019 12:06:13 Task [ 0] updates[ 26500] train loss[0.20893] remaining[0:41:46]
08/21/2019 12:09:06 Task [ 0] updates[ 27000] train loss[0.20671] remaining[0:38:45]
08/21/2019 12:12:00 Task [ 0] updates[ 27500] train loss[0.20465] remaining[0:35:52]
08/21/2019 12:14:54 Task [ 0] updates[ 28000] train loss[0.20260] remaining[0:32:57]
08/21/2019 12:17:49 Task [ 0] updates[ 28500] train loss[0.20075] remaining[0:30:04]
08/21/2019 12:20:43 Task [ 0] updates[ 29000] train loss[0.19919] remaining[0:27:10]
08/21/2019 12:23:39 Task [ 0] updates[ 29500] train loss[0.19771] remaining[0:24:17]
08/21/2019 12:26:34 Task [ 0] updates[ 30000] train loss[0.19610] remaining[0:21:23]
08/21/2019 12:29:27 Task [ 0] updates[ 30500] train loss[0.19471] remaining[0:18:27]
08/21/2019 12:32:22 Task [ 0] updates[ 31000] train loss[0.19319] remaining[0:15:33]
08/21/2019 12:35:16 Task [ 0] updates[ 31500] train loss[0.19193] remaining[0:12:38]
08/21/2019 12:38:08 Task [ 0] updates[ 32000] train loss[0.19082] remaining[0:09:43]
08/21/2019 12:41:01 Task [ 0] updates[ 32500] train loss[0.18936] remaining[0:06:49]
08/21/2019 12:43:55 Task [ 0] updates[ 33000] train loss[0.18786] remaining[0:03:55]
08/21/2019 12:46:50 Task [ 0] updates[ 33500] train loss[0.18678] remaining[0:01:01]
08/21/2019 12:47:57 Task sst -- epoch 3 -- Dev ACC: 92.317
08/21/2019 12:48:08 [new test scores saved.]
08/21/2019 12:48:10 At epoch 4
08/21/2019 12:50:04 Task [ 0] updates[ 34000] train loss[0.18528] remaining[0:47:14]
08/21/2019 12:52:58 Task [ 0] updates[ 34500] train loss[0.18379] remaining[0:44:12]
08/21/2019 12:55:51 Task [ 0] updates[ 35000] train loss[0.18204] remaining[0:41:06]
08/21/2019 12:58:44 Task [ 0] updates[ 35500] train loss[0.18050] remaining[0:38:10]
08/21/2019 01:01:38 Task [ 0] updates[ 36000] train loss[0.17920] remaining[0:35:18]
08/21/2019 01:04:32 Task [ 0] updates[ 36500] train loss[0.17781] remaining[0:32:24]
08/21/2019 01:07:26 Task [ 0] updates[ 37000] train loss[0.17640] remaining[0:29:30]
08/21/2019 01:10:20 Task [ 0] updates[ 37500] train loss[0.17531] remaining[0:26:37]
08/21/2019 01:12:58 Task [ 0] updates[ 38000] train loss[0.17428] remaining[0:23:29]
08/21/2019 01:15:34 Task [ 0] updates[ 38500] train loss[0.17308] remaining[0:20:25]
08/21/2019 01:18:04 Task [ 0] updates[ 39000] train loss[0.17174] remaining[0:17:22]
08/21/2019 01:20:36 Task [ 0] updates[ 39500] train loss[0.17064] remaining[0:14:26]
08/21/2019 01:23:05 Task [ 0] updates[ 40000] train loss[0.16940] remaining[0:11:33]
08/21/2019 01:25:35 Task [ 0] updates[ 40500] train loss[0.16828] remaining[0:08:44]
08/21/2019 01:28:07 Task [ 0] updates[ 41000] train loss[0.16713] remaining[0:05:58]
08/21/2019 01:30:42 Task [ 0] updates[ 41500] train loss[0.16611] remaining[0:03:14]
08/21/2019 01:33:11 Task [ 0] updates[ 42000] train loss[0.16493] remaining[0:00:30]
08/21/2019 01:33:44 Task sst -- epoch 4 -- Dev ACC: 92.546
08/21/2019 01:33:54 [new test scores saved.]
09/08/2019 10:19:22 0
09/08/2019 10:19:22 Launching the MT-DNN training
09/08/2019 10:19:22 Loading /data/canonical_data/bert_uncased\scitail_train.json as task 0
09/08/2019 10:21:35 0
09/08/2019 10:21:35 Launching the MT-DNN training
09/08/2019 10:21:35 Loading /scitail_train.json as task 0
09/08/2019 10:21:48 0
09/08/2019 10:21:48 Launching the MT-DNN training
09/08/2019 10:21:48 Loading /data/canonical_data/bert_uncased\scitail_train.json as task 0
09/08/2019 10:27:08 0
09/08/2019 10:27:08 Launching the MT-DNN training
09/08/2019 10:27:08 Loading /data/canonical_data/bert_uncased/scitail_train.json as task 0
09/08/2019 10:27:58 0
09/08/2019 10:27:58 Launching the MT-DNN training
09/08/2019 10:27:58 Loading data/canonical_data/bert_uncased/scitail_train.json as task 0
09/08/2019 10:27:58 2
09/08/2019 10:29:54 0
09/08/2019 10:29:54 Launching the MT-DNN training
09/08/2019 10:29:54 Loading data/canonical_data/bert_uncased/scitail_train.json as task 0
09/08/2019 10:29:54 2
09/08/2019 10:31:11 0
09/08/2019 10:31:11 Launching the MT-DNN training
09/08/2019 10:31:11 Loading data/canonical_data/bert_uncased/scitail_train.json as task 0
09/08/2019 10:31:11 2
09/08/2019 10:31:11 ####################
09/08/2019 10:31:11 {'log_file': 'mt-dnn-train.log', 'init_checkpoint': 'mt_dnn_models/bert_model_base_uncased.pt', 'data_dir': 'data/canonical_data/bert_uncased', 'data_sort_on': False, 'name': 'farmer', 'task_def': 'experiments/glue/glue_task_def.yml', 'train_datasets': ['scitail'], 'test_datasets': ['scitail'], 'update_bert_opt': 0, 'multi_gpu_on': False, 'mem_cum_type': 'simple', 'answer_num_turn': 5, 'answer_mem_drop_p': 0.1, 'answer_att_hidden_size': 128, 'answer_att_type': 'bilinear', 'answer_rnn_type': 'gru', 'answer_sum_att_type': 'bilinear', 'answer_merge_opt': 1, 'answer_mem_type': 1, 'answer_dropout_p': 0.1, 'answer_weight_norm_on': False, 'dump_state_on': False, 'answer_opt': [0], 'label_size': '2', 'mtl_opt': 0, 'ratio': 0, 'mix_opt': 0, 'max_seq_len': 512, 'init_ratio': 1, 'encoder_type': <EncoderModelType.BERT: 1>, 'cuda': True, 'log_per_updates': 500, 'save_per_updates': 10000, 'save_per_updates_on': False, 'epochs': 5, 'batch_size': 8, 'batch_size_eval': 8, 'optimizer': 'adamax', 'grad_clipping': 0, 'global_grad_clipping': 1.0, 'weight_decay': 0, 'learning_rate': 5e-05, 'momentum': 0, 'warmup': 0.1, 'warmup_schedule': 'warmup_linear', 'vb_dropout': True, 'dropout_p': 0.1, 'dropout_w': 0.0, 'bert_dropout_p': 0.1, 'model_ckpt': 'checkpoints/model_0.pt', 'resume': False, 'ema_opt': 0, 'ema_gamma': 0.995, 'have_lr_scheduler': True, 'multi_step_lr': '10,20,30', 'freeze_layers': -1, 'embedding_opt': 0, 'lr_gamma': 0.5, 'bert_l2norm': 0.0, 'scheduler_type': 'ms', 'output_dir': 'checkpoint', 'seed': 2018, 'grad_accumulation_step': 1, 'fp16': False, 'fp16_opt_level': 'O1', 'tasks_dropout_p': [0.1]}
09/08/2019 10:31:11 ####################
09/08/2019 10:31:11 ############# Gradient Accumulation Info #############
09/08/2019 10:31:11 number of step: 14750
09/08/2019 10:31:11 number of grad grad_accumulation step: 1
09/08/2019 10:31:11 adjusted number of step: 14750
09/08/2019 10:31:11 ############# Gradient Accumulation Info #############
09/08/2019 10:31:17 
############# Model Arch of MT-DNN #############
SANBertNetwork(
  (dropout_list): ModuleList(
    (0): DropoutWrapper()
  )
  (bert): BertModel(
    (embeddings): BertEmbeddings(
      (word_embeddings): Embedding(30522, 768)
      (position_embeddings): Embedding(512, 768)
      (token_type_embeddings): Embedding(2, 768)
      (LayerNorm): BertLayerNorm()
      (dropout): Dropout(p=0.1)
    )
    (encoder): BertEncoder(
      (layer): ModuleList(
        (0): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (1): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (2): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (3): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (4): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (5): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (6): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (7): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (8): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (9): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (10): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (11): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
      )
    )
    (pooler): BertPooler(
      (dense): Linear(in_features=768, out_features=768, bias=True)
      (activation): Tanh()
    )
  )
  (scoring_list): ModuleList(
    (0): Linear(in_features=768, out_features=2, bias=True)
  )
)

09/08/2019 10:31:17 Total number of params: 109483778
09/08/2019 10:31:17 At epoch 0
09/08/2019 10:31:18 Task [ 0] updates[     1] train loss[1.49810] remaining[0:28:36]
09/08/2019 10:33:50 Task [ 0] updates[   500] train loss[0.65001] remaining[0:12:30]
09/08/2019 10:36:23 Task [ 0] updates[  1000] train loss[0.53163] remaining[0:09:57]
09/08/2019 10:38:55 Task [ 0] updates[  1500] train loss[0.47806] remaining[0:07:22]
09/08/2019 10:41:27 Task [ 0] updates[  2000] train loss[0.44618] remaining[0:04:49]
09/08/2019 10:44:00 Task [ 0] updates[  2500] train loss[0.42259] remaining[0:02:17]
09/08/2019 10:46:25 Task scitail -- epoch 0 -- Dev ACC: 87.883
09/08/2019 10:46:36 [new test scores saved.]
09/08/2019 10:46:38 At epoch 1
09/08/2019 10:46:54 Task [ 0] updates[  3000] train loss[0.40276] remaining[0:14:43]
09/08/2019 10:49:25 Task [ 0] updates[  3500] train loss[0.38243] remaining[0:12:06]
09/08/2019 10:51:57 Task [ 0] updates[  4000] train loss[0.36635] remaining[0:09:36]
09/08/2019 10:54:29 Task [ 0] updates[  4500] train loss[0.35349] remaining[0:07:05]
09/08/2019 10:57:02 Task [ 0] updates[  5000] train loss[0.34322] remaining[0:04:33]
09/08/2019 10:59:33 Task [ 0] updates[  5500] train loss[0.33603] remaining[0:02:01]
09/08/2019 11:01:42 Task scitail -- epoch 1 -- Dev ACC: 86.580
09/08/2019 11:01:54 [new test scores saved.]
09/08/2019 11:01:56 At epoch 2
09/08/2019 11:02:26 Task [ 0] updates[  6000] train loss[0.32682] remaining[0:14:16]
09/08/2019 11:04:49 0
09/08/2019 11:04:49 Launching the MT-DNN training
09/08/2019 11:04:49 Loading data/canonical_data/bert_uncased/scitail_train.json as task 0
09/08/2019 11:04:49 2
09/08/2019 11:04:49 ####################
09/08/2019 11:04:49 {'log_file': 'mt-dnn-train.log', 'init_checkpoint': 'ooga_booga', 'data_dir': 'data/canonical_data/bert_uncased', 'data_sort_on': False, 'name': 'farmer', 'task_def': 'experiments/glue/glue_task_def.yml', 'train_datasets': ['scitail'], 'test_datasets': ['scitail'], 'update_bert_opt': 0, 'multi_gpu_on': False, 'mem_cum_type': 'simple', 'answer_num_turn': 5, 'answer_mem_drop_p': 0.1, 'answer_att_hidden_size': 128, 'answer_att_type': 'bilinear', 'answer_rnn_type': 'gru', 'answer_sum_att_type': 'bilinear', 'answer_merge_opt': 1, 'answer_mem_type': 1, 'answer_dropout_p': 0.1, 'answer_weight_norm_on': False, 'dump_state_on': False, 'answer_opt': [0], 'label_size': '2', 'mtl_opt': 0, 'ratio': 0, 'mix_opt': 0, 'max_seq_len': 512, 'init_ratio': 1, 'encoder_type': <EncoderModelType.BERT: 1>, 'cuda': True, 'log_per_updates': 500, 'save_per_updates': 10000, 'save_per_updates_on': False, 'epochs': 5, 'batch_size': 8, 'batch_size_eval': 8, 'optimizer': 'adamax', 'grad_clipping': 0, 'global_grad_clipping': 1.0, 'weight_decay': 0, 'learning_rate': 5e-05, 'momentum': 0, 'warmup': 0.1, 'warmup_schedule': 'warmup_linear', 'vb_dropout': True, 'dropout_p': 0.1, 'dropout_w': 0.0, 'bert_dropout_p': 0.1, 'model_ckpt': 'checkpoints/model_0.pt', 'resume': False, 'ema_opt': 0, 'ema_gamma': 0.995, 'have_lr_scheduler': True, 'multi_step_lr': '10,20,30', 'freeze_layers': -1, 'embedding_opt': 0, 'lr_gamma': 0.5, 'bert_l2norm': 0.0, 'scheduler_type': 'ms', 'output_dir': 'checkpoint', 'seed': 2018, 'grad_accumulation_step': 1, 'fp16': False, 'fp16_opt_level': 'O1', 'tasks_dropout_p': [0.1]}
09/08/2019 11:04:49 ####################
09/08/2019 11:04:49 ############# Gradient Accumulation Info #############
09/08/2019 11:04:49 number of step: 14750
09/08/2019 11:04:49 number of grad grad_accumulation step: 1
09/08/2019 11:04:49 adjusted number of step: 14750
09/08/2019 11:04:49 ############# Gradient Accumulation Info #############
09/08/2019 11:04:49 ####################
09/08/2019 11:04:49 Could not find the init model!
 The parameters will be initialized randomly!
09/08/2019 11:04:49 ####################
09/08/2019 11:04:53 
############# Model Arch of MT-DNN #############
SANBertNetwork(
  (dropout_list): ModuleList(
    (0): DropoutWrapper()
  )
  (bert): BertModel(
    (embeddings): BertEmbeddings(
      (word_embeddings): Embedding(30522, 768)
      (position_embeddings): Embedding(512, 768)
      (token_type_embeddings): Embedding(2, 768)
      (LayerNorm): BertLayerNorm()
      (dropout): Dropout(p=0.1)
    )
    (encoder): BertEncoder(
      (layer): ModuleList(
        (0): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (1): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (2): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (3): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (4): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (5): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (6): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (7): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (8): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (9): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (10): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (11): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
      )
    )
    (pooler): BertPooler(
      (dense): Linear(in_features=768, out_features=768, bias=True)
      (activation): Tanh()
    )
  )
  (scoring_list): ModuleList(
    (0): Linear(in_features=768, out_features=2, bias=True)
  )
)

09/08/2019 11:04:53 Total number of params: 109483778
09/08/2019 11:04:53 At epoch 0
09/08/2019 11:04:54 Task [ 0] updates[     1] train loss[0.65300] remaining[0:34:33]
09/08/2019 11:04:54 Task [ 0] updates[  6500] train loss[0.31250] remaining[0:11:37]
09/08/2019 11:07:21 Task [ 0] updates[  7000] train loss[0.30118] remaining[0:09:06]
09/08/2019 11:09:47 Task [ 0] updates[  7500] train loss[0.29068] remaining[0:06:37]
09/08/2019 11:12:13 Task [ 0] updates[  8000] train loss[0.28140] remaining[0:04:09]
09/08/2019 11:14:50 Task [ 0] updates[  8500] train loss[0.27354] remaining[0:01:44]
09/08/2019 11:16:48 Task scitail -- epoch 2 -- Dev ACC: 87.577
09/08/2019 11:17:00 [new test scores saved.]
09/08/2019 11:17:02 At epoch 3
09/08/2019 11:17:50 Task [ 0] updates[  9000] train loss[0.26636] remaining[0:14:51]
09/08/2019 11:20:32 Task [ 0] updates[  9500] train loss[0.25726] remaining[0:12:20]
09/08/2019 11:23:09 Task [ 0] updates[ 10000] train loss[0.24911] remaining[0:09:33]
09/08/2019 11:25:48 Task [ 0] updates[ 10500] train loss[0.24087] remaining[0:06:53]
09/08/2019 11:28:27 Task [ 0] updates[ 11000] train loss[0.23479] remaining[0:04:14]
09/08/2019 11:31:04 Task [ 0] updates[ 11500] train loss[0.22848] remaining[0:01:35]
09/08/2019 11:32:47 Task scitail -- epoch 3 -- Dev ACC: 87.960
09/08/2019 11:32:59 [new test scores saved.]
09/08/2019 11:33:02 At epoch 4
09/08/2019 11:34:04 Task [ 0] updates[ 12000] train loss[0.22185] remaining[0:14:21]
09/08/2019 11:42:33 Task [ 0] updates[ 12500] train loss[0.21508] remaining[0:30:37]
09/08/2019 11:45:07 Task [ 0] updates[ 13000] train loss[0.20866] remaining[0:17:37]
09/08/2019 11:47:40 Task [ 0] updates[ 13500] train loss[0.20328] remaining[0:10:45]
09/08/2019 11:50:12 Task [ 0] updates[ 14000] train loss[0.19808] remaining[0:05:51]
09/08/2019 11:52:44 Task [ 0] updates[ 14500] train loss[0.19316] remaining[0:01:49]
09/08/2019 11:54:08 Task scitail -- epoch 4 -- Dev ACC: 88.190
09/08/2019 11:54:20 [new test scores saved.]
09/09/2019 04:02:02 0
09/09/2019 04:02:02 Launching the MT-DNN training
09/09/2019 04:02:02 Loading data/canonical_data/bert_uncased/snli_train.json as task 0
09/09/2019 04:02:09 3
09/09/2019 04:02:09 ####################
09/09/2019 04:02:09 {'log_file': 'mt-dnn-train.log', 'init_checkpoint': 'mt_dnn_models/bert_model_base_uncased.pt', 'data_dir': 'data/canonical_data/bert_uncased', 'data_sort_on': False, 'name': 'farmer', 'task_def': 'experiments/glue/glue_task_def.yml', 'train_datasets': ['snli'], 'test_datasets': ['snli'], 'update_bert_opt': 0, 'multi_gpu_on': False, 'mem_cum_type': 'simple', 'answer_num_turn': 5, 'answer_mem_drop_p': 0.1, 'answer_att_hidden_size': 128, 'answer_att_type': 'bilinear', 'answer_rnn_type': 'gru', 'answer_sum_att_type': 'bilinear', 'answer_merge_opt': 1, 'answer_mem_type': 1, 'answer_dropout_p': 0.1, 'answer_weight_norm_on': False, 'dump_state_on': False, 'answer_opt': [0], 'label_size': '3', 'mtl_opt': 0, 'ratio': 0, 'mix_opt': 0, 'max_seq_len': 512, 'init_ratio': 1, 'encoder_type': <EncoderModelType.BERT: 1>, 'cuda': True, 'log_per_updates': 500, 'save_per_updates': 10000, 'save_per_updates_on': False, 'epochs': 5, 'batch_size': 8, 'batch_size_eval': 8, 'optimizer': 'adamax', 'grad_clipping': 0, 'global_grad_clipping': 1.0, 'weight_decay': 0, 'learning_rate': 5e-05, 'momentum': 0, 'warmup': 0.1, 'warmup_schedule': 'warmup_linear', 'vb_dropout': True, 'dropout_p': 0.1, 'dropout_w': 0.0, 'bert_dropout_p': 0.1, 'model_ckpt': 'checkpoints/model_0.pt', 'resume': False, 'ema_opt': 0, 'ema_gamma': 0.995, 'have_lr_scheduler': True, 'multi_step_lr': '10,20,30', 'freeze_layers': -1, 'embedding_opt': 0, 'lr_gamma': 0.5, 'bert_l2norm': 0.0, 'scheduler_type': 'ms', 'output_dir': 'checkpoint', 'seed': 2018, 'grad_accumulation_step': 1, 'fp16': False, 'fp16_opt_level': 'O1', 'tasks_dropout_p': [0.1]}
09/09/2019 04:02:09 ####################
09/09/2019 04:02:09 ############# Gradient Accumulation Info #############
09/09/2019 04:02:09 number of step: 343355
09/09/2019 04:02:09 number of grad grad_accumulation step: 1
09/09/2019 04:02:09 adjusted number of step: 343355
09/09/2019 04:02:09 ############# Gradient Accumulation Info #############
09/09/2019 04:02:13 
############# Model Arch of MT-DNN #############
SANBertNetwork(
  (dropout_list): ModuleList(
    (0): DropoutWrapper()
  )
  (bert): BertModel(
    (embeddings): BertEmbeddings(
      (word_embeddings): Embedding(30522, 768)
      (position_embeddings): Embedding(512, 768)
      (token_type_embeddings): Embedding(2, 768)
      (LayerNorm): BertLayerNorm()
      (dropout): Dropout(p=0.1)
    )
    (encoder): BertEncoder(
      (layer): ModuleList(
        (0): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (1): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (2): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (3): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (4): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (5): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (6): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (7): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (8): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (9): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (10): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (11): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
      )
    )
    (pooler): BertPooler(
      (dense): Linear(in_features=768, out_features=768, bias=True)
      (activation): Tanh()
    )
  )
  (scoring_list): ModuleList(
    (0): Linear(in_features=768, out_features=3, bias=True)
  )
)

09/09/2019 04:02:13 Total number of params: 109484547
09/09/2019 04:02:13 At epoch 0
09/09/2019 04:02:13 Task [ 0] updates[     1] train loss[1.08828] remaining[11:33:55]
09/09/2019 04:05:00 Task [ 0] updates[   500] train loss[1.17994] remaining[6:20:55]
09/09/2019 04:07:45 Task [ 0] updates[  1000] train loss[1.15661] remaining[6:14:33]
09/09/2019 04:10:18 Task [ 0] updates[  1500] train loss[1.13392] remaining[6:02:31]
09/09/2019 04:12:50 Task [ 0] updates[  2000] train loss[1.11102] remaining[5:54:03]
09/09/2019 04:15:21 Task [ 0] updates[  2500] train loss[1.08658] remaining[5:47:31]
09/09/2019 04:17:51 Task [ 0] updates[  3000] train loss[1.06568] remaining[5:42:21]
09/09/2019 04:20:24 Task [ 0] updates[  3500] train loss[1.04419] remaining[5:38:36]
09/09/2019 04:22:55 Task [ 0] updates[  4000] train loss[1.02299] remaining[5:34:36]
09/09/2019 04:25:24 Task [ 0] updates[  4500] train loss[1.00216] remaining[5:30:42]
09/09/2019 04:27:57 Task [ 0] updates[  5000] train loss[0.98285] remaining[5:27:41]
09/09/2019 04:30:31 Task [ 0] updates[  5500] train loss[0.96429] remaining[5:25:07]
09/09/2019 04:33:19 Task [ 0] updates[  6000] train loss[0.94520] remaining[5:24:50]
09/09/2019 04:35:53 Task [ 0] updates[  6500] train loss[0.92779] remaining[5:22:04]
09/09/2019 04:38:26 Task [ 0] updates[  7000] train loss[0.90986] remaining[5:19:05]
09/09/2019 04:40:56 Task [ 0] updates[  7500] train loss[0.89394] remaining[5:15:52]
09/09/2019 04:43:41 Task [ 0] updates[  8000] train loss[0.87916] remaining[5:14:29]
09/09/2019 04:46:40 Task [ 0] updates[  8500] train loss[0.86530] remaining[5:14:40]
09/09/2019 04:49:39 Task [ 0] updates[  9000] train loss[0.85113] remaining[5:14:29]
09/09/2019 04:52:38 Task [ 0] updates[  9500] train loss[0.83783] remaining[5:14:05]
09/09/2019 04:55:37 Task [ 0] updates[ 10000] train loss[0.82689] remaining[5:13:19]
09/09/2019 04:58:36 Task [ 0] updates[ 10500] train loss[0.81618] remaining[5:12:24]
09/09/2019 05:01:35 Task [ 0] updates[ 11000] train loss[0.80600] remaining[5:11:15]
09/09/2019 05:04:34 Task [ 0] updates[ 11500] train loss[0.79666] remaining[5:09:57]
09/09/2019 05:07:30 Task [ 0] updates[ 12000] train loss[0.78676] remaining[5:08:18]
09/09/2019 05:10:25 Task [ 0] updates[ 12500] train loss[0.77826] remaining[5:06:29]
09/09/2019 05:13:22 Task [ 0] updates[ 13000] train loss[0.76981] remaining[5:04:40]
09/09/2019 05:16:19 Task [ 0] updates[ 13500] train loss[0.76220] remaining[5:02:51]
09/09/2019 05:19:16 Task [ 0] updates[ 14000] train loss[0.75476] remaining[5:00:54]
09/09/2019 05:22:15 Task [ 0] updates[ 14500] train loss[0.74778] remaining[4:59:01]
09/09/2019 05:25:14 Task [ 0] updates[ 15000] train loss[0.74107] remaining[4:57:01]
09/09/2019 05:28:14 Task [ 0] updates[ 15500] train loss[0.73391] remaining[4:55:04]
09/09/2019 05:31:13 Task [ 0] updates[ 16000] train loss[0.72755] remaining[4:53:00]
09/09/2019 05:34:13 Task [ 0] updates[ 16500] train loss[0.72152] remaining[4:50:54]
09/09/2019 05:37:12 Task [ 0] updates[ 17000] train loss[0.71516] remaining[4:48:42]
09/09/2019 05:40:12 Task [ 0] updates[ 17500] train loss[0.70889] remaining[4:46:31]
09/09/2019 05:43:12 Task [ 0] updates[ 18000] train loss[0.70370] remaining[4:44:17]
09/09/2019 05:46:09 Task [ 0] updates[ 18500] train loss[0.69851] remaining[4:41:52]
09/09/2019 05:49:08 Task [ 0] updates[ 19000] train loss[0.69354] remaining[4:39:32]
09/09/2019 05:52:08 Task [ 0] updates[ 19500] train loss[0.68838] remaining[4:37:10]
09/09/2019 05:55:08 Task [ 0] updates[ 20000] train loss[0.68375] remaining[4:34:47]
09/09/2019 05:58:07 Task [ 0] updates[ 20500] train loss[0.67894] remaining[4:32:21]
09/09/2019 06:01:07 Task [ 0] updates[ 21000] train loss[0.67453] remaining[4:29:54]
09/09/2019 06:04:06 Task [ 0] updates[ 21500] train loss[0.67013] remaining[4:27:26]
09/09/2019 06:07:07 Task [ 0] updates[ 22000] train loss[0.66611] remaining[4:24:57]
09/09/2019 06:09:53 Task [ 0] updates[ 22500] train loss[0.66228] remaining[4:21:59]
09/09/2019 06:12:27 Task [ 0] updates[ 23000] train loss[0.65844] remaining[4:18:37]
09/09/2019 06:15:01 Task [ 0] updates[ 23500] train loss[0.65468] remaining[4:15:15]
09/09/2019 06:17:35 Task [ 0] updates[ 24000] train loss[0.65100] remaining[4:11:57]
09/09/2019 06:20:09 Task [ 0] updates[ 24500] train loss[0.64760] remaining[4:08:41]
09/09/2019 06:22:51 Task [ 0] updates[ 25000] train loss[0.64414] remaining[4:05:40]
09/09/2019 06:25:41 Task [ 0] updates[ 25500] train loss[0.64064] remaining[4:02:53]
09/09/2019 06:28:30 Task [ 0] updates[ 26000] train loss[0.63737] remaining[4:00:05]
09/09/2019 06:31:14 Task [ 0] updates[ 26500] train loss[0.63452] remaining[3:57:07]
09/09/2019 06:33:54 Task [ 0] updates[ 27000] train loss[0.63121] remaining[3:54:07]
09/09/2019 06:36:36 Task [ 0] updates[ 27500] train loss[0.62863] remaining[3:51:08]
09/09/2019 06:39:20 Task [ 0] updates[ 28000] train loss[0.62572] remaining[3:48:12]
09/09/2019 06:42:06 Task [ 0] updates[ 28500] train loss[0.62295] remaining[3:45:21]
09/09/2019 06:44:47 Task [ 0] updates[ 29000] train loss[0.62002] remaining[3:42:23]
09/09/2019 06:47:20 Task [ 0] updates[ 29500] train loss[0.61756] remaining[3:39:14]
09/09/2019 06:49:51 Task [ 0] updates[ 30000] train loss[0.61486] remaining[3:36:05]
09/09/2019 06:52:22 Task [ 0] updates[ 30500] train loss[0.61235] remaining[3:32:56]
09/09/2019 06:54:53 Task [ 0] updates[ 31000] train loss[0.61017] remaining[3:29:49]
09/09/2019 06:57:25 Task [ 0] updates[ 31500] train loss[0.60828] remaining[3:26:44]
09/09/2019 06:59:56 Task [ 0] updates[ 32000] train loss[0.60594] remaining[3:23:39]
09/09/2019 09:03:25 Task [ 0] updates[ 32500] train loss[0.60374] remaining[5:35:14]
09/09/2019 09:05:58 Task [ 0] updates[ 33000] train loss[0.60173] remaining[5:28:20]
09/09/2019 09:08:30 Task [ 0] updates[ 33500] train loss[0.59978] remaining[5:21:34]
09/09/2019 09:11:02 Task [ 0] updates[ 34000] train loss[0.59775] remaining[5:14:54]
09/09/2019 09:13:51 Task [ 0] updates[ 34500] train loss[0.59569] remaining[5:08:40]
09/09/2019 09:16:31 Task [ 0] updates[ 35000] train loss[0.59368] remaining[5:02:22]
09/09/2019 09:19:02 Task [ 0] updates[ 35500] train loss[0.59176] remaining[4:56:02]
09/09/2019 09:21:34 Task [ 0] updates[ 36000] train loss[0.58972] remaining[4:49:49]
09/09/2019 09:24:08 Task [ 0] updates[ 36500] train loss[0.58784] remaining[4:43:44]
09/09/2019 09:26:38 Task [ 0] updates[ 37000] train loss[0.58596] remaining[4:37:41]
09/09/2019 09:29:06 Task [ 0] updates[ 37500] train loss[0.58414] remaining[4:31:43]
09/09/2019 09:31:36 Task [ 0] updates[ 38000] train loss[0.58272] remaining[4:25:51]
09/09/2019 09:34:06 Task [ 0] updates[ 38500] train loss[0.58079] remaining[4:20:05]
09/09/2019 09:36:37 Task [ 0] updates[ 39000] train loss[0.57910] remaining[4:14:24]
09/09/2019 09:39:07 Task [ 0] updates[ 39500] train loss[0.57772] remaining[4:08:48]
09/09/2019 09:41:37 Task [ 0] updates[ 40000] train loss[0.57626] remaining[4:03:16]
09/09/2019 09:44:12 Task [ 0] updates[ 40500] train loss[0.57458] remaining[3:57:52]
09/09/2019 09:46:44 Task [ 0] updates[ 41000] train loss[0.57306] remaining[3:52:31]
09/09/2019 09:49:14 Task [ 0] updates[ 41500] train loss[0.57155] remaining[3:47:12]
09/09/2019 09:51:46 Task [ 0] updates[ 42000] train loss[0.56994] remaining[3:41:58]
09/09/2019 09:54:18 Task [ 0] updates[ 42500] train loss[0.56841] remaining[3:36:48]
09/09/2019 09:56:50 Task [ 0] updates[ 43000] train loss[0.56668] remaining[3:31:42]
09/09/2019 09:59:21 Task [ 0] updates[ 43500] train loss[0.56490] remaining[3:26:39]
09/09/2019 10:01:51 Task [ 0] updates[ 44000] train loss[0.56347] remaining[3:21:39]
09/09/2019 10:04:22 Task [ 0] updates[ 44500] train loss[0.56218] remaining[3:16:42]
09/09/2019 10:06:52 Task [ 0] updates[ 45000] train loss[0.56074] remaining[3:11:49]
09/09/2019 10:09:23 Task [ 0] updates[ 45500] train loss[0.55931] remaining[3:06:58]
09/09/2019 10:11:55 Task [ 0] updates[ 46000] train loss[0.55808] remaining[3:02:12]
09/09/2019 10:14:26 Task [ 0] updates[ 46500] train loss[0.55685] remaining[2:57:28]
09/09/2019 10:16:59 Task [ 0] updates[ 47000] train loss[0.55532] remaining[2:52:48]
09/09/2019 10:19:31 Task [ 0] updates[ 47500] train loss[0.55408] remaining[2:48:09]
09/09/2019 10:22:01 Task [ 0] updates[ 48000] train loss[0.55273] remaining[2:43:33]
09/09/2019 10:24:35 Task [ 0] updates[ 48500] train loss[0.55166] remaining[2:39:01]
09/09/2019 10:27:07 Task [ 0] updates[ 49000] train loss[0.55040] remaining[2:34:31]
09/09/2019 10:29:39 Task [ 0] updates[ 49500] train loss[0.54918] remaining[2:30:03]
09/09/2019 10:32:10 Task [ 0] updates[ 50000] train loss[0.54807] remaining[2:25:37]
09/09/2019 10:34:43 Task [ 0] updates[ 50500] train loss[0.54687] remaining[2:21:13]
09/09/2019 10:37:14 Task [ 0] updates[ 51000] train loss[0.54592] remaining[2:16:52]
09/09/2019 10:39:45 Task [ 0] updates[ 51500] train loss[0.54473] remaining[2:12:32]
09/09/2019 10:42:16 Task [ 0] updates[ 52000] train loss[0.54334] remaining[2:08:15]
09/09/2019 10:44:46 Task [ 0] updates[ 52500] train loss[0.54243] remaining[2:03:59]
09/09/2019 10:47:22 Task [ 0] updates[ 53000] train loss[0.54120] remaining[1:59:47]
09/09/2019 10:49:55 Task [ 0] updates[ 53500] train loss[0.54022] remaining[1:55:36]
09/09/2019 10:52:24 Task [ 0] updates[ 54000] train loss[0.53927] remaining[1:51:26]
09/09/2019 10:54:55 Task [ 0] updates[ 54500] train loss[0.53809] remaining[1:47:18]
09/09/2019 10:57:27 Task [ 0] updates[ 55000] train loss[0.53707] remaining[1:43:12]
09/10/2019 04:22:10 0
09/10/2019 04:22:10 Launching the MT-DNN training
09/10/2019 04:22:10 Loading data/canonical_data/bert_uncased/snli_train.json as task 0
09/10/2019 04:22:20 3
09/10/2019 04:22:20 ####################
09/10/2019 04:22:20 {'log_file': 'mt-dnn-train.log', 'init_checkpoint': 'mt_dnn_models/bert_model_base_uncased.pt', 'data_dir': 'data/canonical_data/bert_uncased', 'data_sort_on': False, 'name': 'farmer', 'task_def': 'experiments/glue/glue_task_def.yml', 'train_datasets': ['snli'], 'test_datasets': ['snli'], 'update_bert_opt': 0, 'multi_gpu_on': False, 'mem_cum_type': 'simple', 'answer_num_turn': 5, 'answer_mem_drop_p': 0.1, 'answer_att_hidden_size': 128, 'answer_att_type': 'bilinear', 'answer_rnn_type': 'gru', 'answer_sum_att_type': 'bilinear', 'answer_merge_opt': 1, 'answer_mem_type': 1, 'answer_dropout_p': 0.1, 'answer_weight_norm_on': False, 'dump_state_on': False, 'answer_opt': [0], 'label_size': '3', 'mtl_opt': 0, 'ratio': 0, 'mix_opt': 0, 'max_seq_len': 512, 'init_ratio': 1, 'encoder_type': <EncoderModelType.BERT: 1>, 'cuda': True, 'log_per_updates': 500, 'save_per_updates': 10000, 'save_per_updates_on': False, 'epochs': 5, 'batch_size': 8, 'batch_size_eval': 8, 'optimizer': 'adamax', 'grad_clipping': 0, 'global_grad_clipping': 1.0, 'weight_decay': 0, 'learning_rate': 5e-05, 'momentum': 0, 'warmup': 0.1, 'warmup_schedule': 'warmup_linear', 'vb_dropout': True, 'dropout_p': 0.1, 'dropout_w': 0.0, 'bert_dropout_p': 0.1, 'model_ckpt': 'checkpoints/model_0.pt', 'resume': False, 'ema_opt': 0, 'ema_gamma': 0.995, 'have_lr_scheduler': True, 'multi_step_lr': '10,20,30', 'freeze_layers': -1, 'embedding_opt': 0, 'lr_gamma': 0.5, 'bert_l2norm': 0.0, 'scheduler_type': 'ms', 'output_dir': 'checkpoint', 'seed': 2018, 'grad_accumulation_step': 1, 'fp16': False, 'fp16_opt_level': 'O1', 'tasks_dropout_p': [0.1]}
09/10/2019 04:22:20 ####################
09/10/2019 04:22:20 ############# Gradient Accumulation Info #############
09/10/2019 04:22:20 number of step: 343355
09/10/2019 04:22:20 number of grad grad_accumulation step: 1
09/10/2019 04:22:20 adjusted number of step: 343355
09/10/2019 04:22:20 ############# Gradient Accumulation Info #############
09/10/2019 04:22:25 
############# Model Arch of MT-DNN #############
SANBertNetwork(
  (dropout_list): ModuleList(
    (0): DropoutWrapper()
  )
  (bert): BertModel(
    (embeddings): BertEmbeddings(
      (word_embeddings): Embedding(30522, 768)
      (position_embeddings): Embedding(512, 768)
      (token_type_embeddings): Embedding(2, 768)
      (LayerNorm): BertLayerNorm()
      (dropout): Dropout(p=0.1)
    )
    (encoder): BertEncoder(
      (layer): ModuleList(
        (0): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (1): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (2): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (3): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (4): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (5): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (6): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (7): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (8): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (9): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (10): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
        (11): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): BertLayerNorm()
              (dropout): Dropout(p=0.1)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): BertLayerNorm()
            (dropout): Dropout(p=0.1)
          )
        )
      )
    )
    (pooler): BertPooler(
      (dense): Linear(in_features=768, out_features=768, bias=True)
      (activation): Tanh()
    )
  )
  (scoring_list): ModuleList(
    (0): Linear(in_features=768, out_features=3, bias=True)
  )
)

09/10/2019 04:22:25 Total number of params: 109484547
09/10/2019 04:22:25 At epoch 0
09/10/2019 04:22:26 Task [ 0] updates[     1] train loss[1.08828] remaining[16:52:29]
09/10/2019 04:23:40 Task [ 0] updates[ 55500] train loss[0.53594] remaining[5:46:49]
09/10/2019 04:25:46 Task [ 0] updates[   500] train loss[1.17994] remaining[7:37:19]
09/10/2019 04:26:58 Task [ 0] updates[ 56000] train loss[0.53497] remaining[5:31:25]
09/10/2019 04:29:04 Task [ 0] updates[  1000] train loss[1.15661] remaining[7:29:42]
09/10/2019 04:30:14 Task [ 0] updates[ 56500] train loss[0.53415] remaining[5:16:14]
09/10/2019 04:32:20 Task [ 0] updates[  1500] train loss[1.13392] remaining[7:24:20]
09/10/2019 04:33:32 Task [ 0] updates[ 57000] train loss[0.53331] remaining[5:01:15]
09/10/2019 04:35:36 Task [ 0] updates[  2000] train loss[1.11102] remaining[7:19:25]
09/10/2019 04:36:47 Task [ 0] updates[ 57500] train loss[0.53250] remaining[4:46:28]
09/10/2019 04:38:51 Task [ 0] updates[  2500] train loss[1.08658] remaining[7:15:05]
09/10/2019 04:40:01 Task [ 0] updates[ 58000] train loss[0.53156] remaining[4:31:53]
09/10/2019 04:42:07 Task [ 0] updates[  3000] train loss[1.06568] remaining[7:11:15]
09/10/2019 04:43:16 Task [ 0] updates[ 58500] train loss[0.53049] remaining[4:17:30]
09/10/2019 04:45:23 Task [ 0] updates[  3500] train loss[1.04419] remaining[7:07:46]
09/10/2019 04:46:32 Task [ 0] updates[ 59000] train loss[0.52964] remaining[4:03:18]
09/10/2019 04:48:40 Task [ 0] updates[  4000] train loss[1.02299] remaining[7:04:22]
09/10/2019 04:49:47 Task [ 0] updates[ 59500] train loss[0.52874] remaining[3:49:17]
09/10/2019 04:51:57 Task [ 0] updates[  4500] train loss[1.00216] remaining[7:01:02]
09/10/2019 04:53:03 Task [ 0] updates[ 60000] train loss[0.52784] remaining[3:35:27]
09/10/2019 04:55:12 Task [ 0] updates[  5000] train loss[0.98285] remaining[6:57:27]
09/10/2019 04:56:17 Task [ 0] updates[ 60500] train loss[0.52693] remaining[3:21:47]
09/10/2019 04:58:27 Task [ 0] updates[  5500] train loss[0.96429] remaining[6:53:50]
09/10/2019 04:59:32 Task [ 0] updates[ 61000] train loss[0.52620] remaining[3:08:17]
09/10/2019 05:01:42 Task [ 0] updates[  6000] train loss[0.94520] remaining[6:50:23]
09/10/2019 05:02:48 Task [ 0] updates[ 61500] train loss[0.52550] remaining[2:54:58]
09/10/2019 05:04:58 Task [ 0] updates[  6500] train loss[0.92779] remaining[6:46:57]
09/10/2019 05:06:03 Task [ 0] updates[ 62000] train loss[0.52457] remaining[2:41:48]
09/10/2019 05:08:14 Task [ 0] updates[  7000] train loss[0.90986] remaining[6:43:41]
09/10/2019 05:09:18 Task [ 0] updates[ 62500] train loss[0.52367] remaining[2:28:48]
09/10/2019 05:11:30 Task [ 0] updates[  7500] train loss[0.89394] remaining[6:40:16]
09/10/2019 05:12:33 Task [ 0] updates[ 63000] train loss[0.52303] remaining[2:15:57]
09/10/2019 05:14:47 Task [ 0] updates[  8000] train loss[0.87916] remaining[6:37:08]
09/10/2019 05:15:48 Task [ 0] updates[ 63500] train loss[0.52207] remaining[2:03:15]
09/10/2019 05:18:01 Task [ 0] updates[  8500] train loss[0.86530] remaining[6:33:31]
09/10/2019 05:19:02 Task [ 0] updates[ 64000] train loss[0.52114] remaining[1:50:42]
09/10/2019 05:21:14 Task [ 0] updates[  9000] train loss[0.85113] remaining[6:29:55]
09/10/2019 05:22:16 Task [ 0] updates[ 64500] train loss[0.52019] remaining[1:38:17]
09/10/2019 05:24:28 Task [ 0] updates[  9500] train loss[0.83783] remaining[6:26:27]
09/10/2019 05:25:29 Task [ 0] updates[ 65000] train loss[0.51942] remaining[1:26:01]
09/10/2019 05:27:41 Task [ 0] updates[ 10000] train loss[0.82689] remaining[6:22:53]
09/10/2019 05:28:42 Task [ 0] updates[ 65500] train loss[0.51880] remaining[1:13:54]
09/10/2019 05:30:56 Task [ 0] updates[ 10500] train loss[0.81618] remaining[6:19:32]
09/10/2019 05:31:56 Task [ 0] updates[ 66000] train loss[0.51806] remaining[1:01:54]
09/10/2019 05:34:09 Task [ 0] updates[ 11000] train loss[0.80600] remaining[6:16:05]
09/10/2019 05:35:09 Task [ 0] updates[ 66500] train loss[0.51745] remaining[0:50:02]
09/10/2019 05:37:23 Task [ 0] updates[ 11500] train loss[0.79666] remaining[6:12:43]
09/10/2019 05:38:22 Task [ 0] updates[ 67000] train loss[0.51655] remaining[0:38:18]
09/10/2019 05:40:37 Task [ 0] updates[ 12000] train loss[0.78676] remaining[6:09:16]
09/10/2019 05:41:35 Task [ 0] updates[ 67500] train loss[0.51577] remaining[0:26:42]
09/10/2019 05:43:50 Task [ 0] updates[ 12500] train loss[0.77826] remaining[6:05:52]
09/10/2019 05:44:49 Task [ 0] updates[ 68000] train loss[0.51497] remaining[0:15:13]
09/10/2019 05:47:07 Task [ 0] updates[ 13000] train loss[0.76981] remaining[6:02:42]
09/10/2019 05:48:00 Task [ 0] updates[ 68500] train loss[0.51415] remaining[0:03:51]
09/10/2019 05:50:08 Task snli -- epoch 0 -- Dev ACC: 88.458
09/10/2019 05:50:19 Task [ 0] updates[ 13500] train loss[0.76220] remaining[5:59:13]
09/10/2019 05:51:09 [new test scores saved.]
09/10/2019 05:51:12 At epoch 1
09/10/2019 05:53:19 Task [ 0] updates[ 69000] train loss[0.51325] remaining[7:19:02]
09/10/2019 05:53:32 Task [ 0] updates[ 14000] train loss[0.75476] remaining[5:55:49]
09/10/2019 05:56:34 Task [ 0] updates[ 69500] train loss[0.51243] remaining[7:18:50]
09/10/2019 05:56:47 Task [ 0] updates[ 14500] train loss[0.74778] remaining[5:52:32]
09/10/2019 05:59:12 Task [ 0] updates[ 70000] train loss[0.51156] remaining[6:45:11]
09/10/2019 06:01:40 Task [ 0] updates[ 70500] train loss[0.51058] remaining[6:22:40]
09/10/2019 06:04:08 Task [ 0] updates[ 71000] train loss[0.50970] remaining[6:08:38]
09/10/2019 06:06:36 Task [ 0] updates[ 71500] train loss[0.50862] remaining[5:58:34]
09/10/2019 06:09:05 Task [ 0] updates[ 72000] train loss[0.50778] remaining[5:50:56]
09/10/2019 06:11:35 Task [ 0] updates[ 72500] train loss[0.50705] remaining[5:45:05]
09/10/2019 06:14:04 Task [ 0] updates[ 73000] train loss[0.50617] remaining[5:39:49]
09/10/2019 06:16:31 Task [ 0] updates[ 73500] train loss[0.50525] remaining[5:34:41]
09/10/2019 06:18:59 Task [ 0] updates[ 74000] train loss[0.50435] remaining[5:30:15]
09/10/2019 06:21:25 Task [ 0] updates[ 74500] train loss[0.50366] remaining[5:25:51]
09/10/2019 08:08:49 Task [ 0] updates[ 75000] train loss[0.50280] remaining[22:35:39]
09/10/2019 08:11:18 Task [ 0] updates[ 75500] train loss[0.50195] remaining[21:08:46]
09/10/2019 08:13:45 Task [ 0] updates[ 76000] train loss[0.50121] remaining[19:53:11]
09/10/2019 08:16:13 Task [ 0] updates[ 76500] train loss[0.50034] remaining[18:46:56]
09/10/2019 08:18:39 Task [ 0] updates[ 77000] train loss[0.49960] remaining[17:48:17]
09/10/2019 08:21:06 Task [ 0] updates[ 77500] train loss[0.49868] remaining[16:56:03]
09/10/2019 08:23:36 Task [ 0] updates[ 78000] train loss[0.49815] remaining[16:09:29]
09/10/2019 08:26:03 Task [ 0] updates[ 78500] train loss[0.49735] remaining[15:27:04]
09/10/2019 08:28:30 Task [ 0] updates[ 79000] train loss[0.49660] remaining[14:48:30]
09/10/2019 08:30:58 Task [ 0] updates[ 79500] train loss[0.49591] remaining[14:13:21]
09/10/2019 08:33:24 Task [ 0] updates[ 80000] train loss[0.49507] remaining[13:41:00]
09/10/2019 08:35:51 Task [ 0] updates[ 80500] train loss[0.49449] remaining[13:11:13]
09/10/2019 08:38:20 Task [ 0] updates[ 81000] train loss[0.49363] remaining[12:43:46]
09/10/2019 08:40:51 Task [ 0] updates[ 81500] train loss[0.49290] remaining[12:18:28]
09/10/2019 08:43:21 Task [ 0] updates[ 82000] train loss[0.49229] remaining[11:54:48]
09/10/2019 08:45:51 Task [ 0] updates[ 82500] train loss[0.49152] remaining[11:32:36]
09/10/2019 08:48:21 Task [ 0] updates[ 83000] train loss[0.49080] remaining[11:11:50]
09/10/2019 08:50:53 Task [ 0] updates[ 83500] train loss[0.49020] remaining[10:52:24]
09/10/2019 08:53:23 Task [ 0] updates[ 84000] train loss[0.48955] remaining[10:33:57]
09/10/2019 08:55:52 Task [ 0] updates[ 84500] train loss[0.48888] remaining[10:16:29]
09/10/2019 08:58:24 Task [ 0] updates[ 85000] train loss[0.48824] remaining[10:00:04]
09/10/2019 09:00:53 Task [ 0] updates[ 85500] train loss[0.48751] remaining[9:44:20]
09/10/2019 09:03:24 Task [ 0] updates[ 86000] train loss[0.48688] remaining[9:29:27]
09/10/2019 09:05:58 Task [ 0] updates[ 86500] train loss[0.48626] remaining[9:15:24]
09/10/2019 09:08:28 Task [ 0] updates[ 87000] train loss[0.48583] remaining[9:01:48]
09/10/2019 09:10:59 Task [ 0] updates[ 87500] train loss[0.48521] remaining[8:48:50]
09/10/2019 09:13:28 Task [ 0] updates[ 88000] train loss[0.48463] remaining[8:36:20]
09/10/2019 09:15:57 Task [ 0] updates[ 88500] train loss[0.48403] remaining[8:24:19]
09/10/2019 09:18:26 Task [ 0] updates[ 89000] train loss[0.48341] remaining[8:12:48]
09/10/2019 09:20:57 Task [ 0] updates[ 89500] train loss[0.48294] remaining[8:01:45]
09/10/2019 09:23:28 Task [ 0] updates[ 90000] train loss[0.48238] remaining[7:51:09]
09/10/2019 09:25:57 Task [ 0] updates[ 90500] train loss[0.48177] remaining[7:40:49]
09/10/2019 09:28:27 Task [ 0] updates[ 91000] train loss[0.48124] remaining[7:30:53]
09/10/2019 09:30:57 Task [ 0] updates[ 91500] train loss[0.48061] remaining[7:21:15]
09/10/2019 09:33:27 Task [ 0] updates[ 92000] train loss[0.48000] remaining[7:11:57]
09/10/2019 09:35:58 Task [ 0] updates[ 92500] train loss[0.47953] remaining[7:02:59]
09/10/2019 09:38:29 Task [ 0] updates[ 93000] train loss[0.47889] remaining[6:54:15]
09/10/2019 09:41:01 Task [ 0] updates[ 93500] train loss[0.47830] remaining[6:45:48]
09/10/2019 09:43:32 Task [ 0] updates[ 94000] train loss[0.47775] remaining[6:37:34]
09/10/2019 09:46:05 Task [ 0] updates[ 94500] train loss[0.47715] remaining[6:29:35]
09/10/2019 09:48:34 Task [ 0] updates[ 95000] train loss[0.47667] remaining[6:21:44]
09/10/2019 09:51:04 Task [ 0] updates[ 95500] train loss[0.47629] remaining[6:14:05]
09/10/2019 09:53:36 Task [ 0] updates[ 96000] train loss[0.47573] remaining[6:06:40]
09/10/2019 09:56:04 Task [ 0] updates[ 96500] train loss[0.47512] remaining[5:59:22]
09/10/2019 09:58:33 Task [ 0] updates[ 97000] train loss[0.47461] remaining[5:52:14]
09/10/2019 10:01:04 Task [ 0] updates[ 97500] train loss[0.47414] remaining[5:45:19]
09/10/2019 10:03:35 Task [ 0] updates[ 98000] train loss[0.47377] remaining[5:38:33]
09/10/2019 10:06:05 Task [ 0] updates[ 98500] train loss[0.47319] remaining[5:31:54]
09/10/2019 10:08:34 Task [ 0] updates[ 99000] train loss[0.47272] remaining[5:25:21]
09/10/2019 10:11:03 Task [ 0] updates[ 99500] train loss[0.47215] remaining[5:18:57]
09/10/2019 10:13:31 Task [ 0] updates[100000] train loss[0.47174] remaining[5:12:39]
09/10/2019 10:15:58 Task [ 0] updates[100500] train loss[0.47127] remaining[5:06:28]
09/10/2019 10:18:25 Task [ 0] updates[101000] train loss[0.47079] remaining[5:00:23]
09/10/2019 10:20:52 Task [ 0] updates[101500] train loss[0.47046] remaining[4:54:25]
09/10/2019 10:23:23 Task [ 0] updates[102000] train loss[0.46997] remaining[4:48:37]
09/10/2019 10:25:55 Task [ 0] updates[102500] train loss[0.46936] remaining[4:42:56]
09/10/2019 10:28:26 Task [ 0] updates[103000] train loss[0.46889] remaining[4:37:20]
09/10/2019 10:30:59 Task [ 0] updates[103500] train loss[0.46847] remaining[4:31:51]
09/10/2019 10:33:32 Task [ 0] updates[104000] train loss[0.46808] remaining[4:26:26]
09/10/2019 10:36:03 Task [ 0] updates[104500] train loss[0.46750] remaining[4:21:06]
09/10/2019 10:38:36 Task [ 0] updates[105000] train loss[0.46716] remaining[4:15:51]
09/10/2019 10:41:08 Task [ 0] updates[105500] train loss[0.46677] remaining[4:10:40]
09/10/2019 10:43:36 Task [ 0] updates[106000] train loss[0.46636] remaining[4:05:30]
09/10/2019 10:46:05 Task [ 0] updates[106500] train loss[0.46588] remaining[4:00:25]
09/10/2019 10:48:34 Task [ 0] updates[107000] train loss[0.46546] remaining[3:55:23]
09/10/2019 10:51:02 Task [ 0] updates[107500] train loss[0.46505] remaining[3:50:26]
09/10/2019 10:53:31 Task [ 0] updates[108000] train loss[0.46467] remaining[3:45:33]
09/10/2019 10:55:59 Task [ 0] updates[108500] train loss[0.46423] remaining[3:40:42]
09/10/2019 10:58:28 Task [ 0] updates[109000] train loss[0.46381] remaining[3:35:56]
09/10/2019 11:00:58 Task [ 0] updates[109500] train loss[0.46338] remaining[3:31:14]
09/10/2019 11:36:57 Task [ 0] updates[110000] train loss[0.46295] remaining[3:48:44]
09/10/2019 11:39:24 Task [ 0] updates[110500] train loss[0.46258] remaining[3:43:26]
09/10/2019 11:41:50 Task [ 0] updates[111000] train loss[0.46224] remaining[3:38:12]
09/10/2019 11:44:16 Task [ 0] updates[111500] train loss[0.46192] remaining[3:33:02]
09/10/2019 11:46:43 Task [ 0] updates[112000] train loss[0.46156] remaining[3:27:56]
09/10/2019 11:49:09 Task [ 0] updates[112500] train loss[0.46125] remaining[3:22:53]
09/10/2019 11:51:36 Task [ 0] updates[113000] train loss[0.46088] remaining[3:17:54]
09/10/2019 11:54:01 Task [ 0] updates[113500] train loss[0.46056] remaining[3:12:58]
09/10/2019 11:56:28 Task [ 0] updates[114000] train loss[0.46014] remaining[3:08:05]
09/10/2019 11:58:54 Task [ 0] updates[114500] train loss[0.45989] remaining[3:03:16]
09/11/2019 12:01:21 Task [ 0] updates[115000] train loss[0.45945] remaining[2:58:30]
09/11/2019 12:03:47 Task [ 0] updates[115500] train loss[0.45908] remaining[2:53:46]
09/11/2019 12:06:15 Task [ 0] updates[116000] train loss[0.45868] remaining[2:49:07]
09/11/2019 12:08:45 Task [ 0] updates[116500] train loss[0.45836] remaining[2:44:31]
09/11/2019 12:11:11 Task [ 0] updates[117000] train loss[0.45800] remaining[2:39:56]
09/11/2019 12:13:38 Task [ 0] updates[117500] train loss[0.45768] remaining[2:35:24]
09/11/2019 12:16:05 Task [ 0] updates[118000] train loss[0.45730] remaining[2:30:54]
09/11/2019 12:18:32 Task [ 0] updates[118500] train loss[0.45695] remaining[2:26:28]
09/11/2019 12:21:00 Task [ 0] updates[119000] train loss[0.45667] remaining[2:22:03]
09/11/2019 12:23:27 Task [ 0] updates[119500] train loss[0.45626] remaining[2:17:41]
09/11/2019 12:25:53 Task [ 0] updates[120000] train loss[0.45596] remaining[2:13:21]
09/11/2019 12:28:19 Task [ 0] updates[120500] train loss[0.45555] remaining[2:09:02]
09/11/2019 12:30:46 Task [ 0] updates[121000] train loss[0.45524] remaining[2:04:47]
09/11/2019 12:33:15 Task [ 0] updates[121500] train loss[0.45486] remaining[2:00:33]
09/11/2019 12:35:41 Task [ 0] updates[122000] train loss[0.45454] remaining[1:56:21]
09/11/2019 12:38:08 Task [ 0] updates[122500] train loss[0.45421] remaining[1:52:12]
09/11/2019 12:40:35 Task [ 0] updates[123000] train loss[0.45393] remaining[1:48:04]
09/11/2019 12:43:01 Task [ 0] updates[123500] train loss[0.45366] remaining[1:43:58]
09/11/2019 12:48:20 Task [ 0] updates[124000] train loss[0.45332] remaining[1:40:35]
09/11/2019 12:50:54 Task [ 0] updates[124500] train loss[0.45302] remaining[1:36:32]
09/11/2019 12:53:28 Task [ 0] updates[125000] train loss[0.45273] remaining[1:32:31]
09/11/2019 12:55:59 Task [ 0] updates[125500] train loss[0.45243] remaining[1:28:31]
09/11/2019 12:58:26 Task [ 0] updates[126000] train loss[0.45217] remaining[1:24:31]
09/11/2019 01:00:52 Task [ 0] updates[126500] train loss[0.45185] remaining[1:20:33]
09/11/2019 01:03:19 Task [ 0] updates[127000] train loss[0.45159] remaining[1:16:36]
09/11/2019 01:05:45 Task [ 0] updates[127500] train loss[0.45128] remaining[1:12:41]
09/11/2019 01:08:11 Task [ 0] updates[128000] train loss[0.45093] remaining[1:08:48]
09/11/2019 01:10:37 Task [ 0] updates[128500] train loss[0.45059] remaining[1:04:56]
09/11/2019 01:13:04 Task [ 0] updates[129000] train loss[0.45035] remaining[1:01:05]
09/11/2019 01:15:31 Task [ 0] updates[129500] train loss[0.45012] remaining[0:57:16]
09/11/2019 01:17:57 Task [ 0] updates[130000] train loss[0.44985] remaining[0:53:28]