##主函数打上断点
if __name__ == "__main__":
    main()##主函数进入

if args.local_rank == -1 or args.no_cuda:
    device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
    args.n_gpu = torch.cuda.device_count()
else:  # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
    torch.cuda.set_device(args.local_rank)
    device = torch.device("cuda", args.local_rank)
    torch.distributed.init_process_group(backend='nccl')
    args.n_gpu = 1

processor = processors[args.task_name]()

TnewsProcessor(DataProcessor)

train_dataset = load_and_cache_examples(args, args.task_name, tokenizer, data_type='train')
global_step, tr_loss = train(args, train_dataset, model, tokenizer)

examples = processor.get_train_examples(args.data_dir)

features = convert_examples_to_features(examples,tokenizer,label_list=label_list,max_length=args.max_seq_length,output_mode=output_mode,pad_on_left=bool(args.model_type in ['xlnet']),                                                pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0],
pad_token_segment_id=4 if args.model_type in ['xlnet'] else 0,

processors.glue.glue_convert_examples_to_features

features.append(InputFeatures(input_ids=input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
label=label,
input_len=input_len))##长度为原始长度，这里应该是10，不是128

dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_lens,all_labels)

outputs = model(**inputs)

##reference：transformers.modeling_bert.BertEncoder
class BertEncoder(nn.Module):
    def __init__(self, config):
        super(BertEncoder, self).__init__()
        self.output_attentions = config.output_attentions
        self.output_hidden_states = config.output_hidden_states
        self.layer = nn.ModuleList([BertLayer(config) for _ in range(config.num_hidden_layers)])

    def forward(self, hidden_states, attention_mask=None, head_mask=None):
        all_hidden_states = ()
        all_attentions = ()
        for i, layer_module in enumerate(self.layer):
            if self.output_hidden_states:
                all_hidden_states = all_hidden_states + (hidden_states,)

            layer_outputs = layer_module(hidden_states, attention_mask, head_mask[i])
            hidden_states = layer_outputs[0]

            if self.output_attentions:
                all_attentions = all_attentions + (layer_outputs[1],)

        # Add last layer
        if self.output_hidden_states:
            all_hidden_states = all_hidden_states + (hidden_states,)

        outputs = (hidden_states,)
        if self.output_hidden_states:
            outputs = outputs + (all_hidden_states,)
        if self.output_attentions:
            outputs = outputs + (all_attentions,)
        return outputs  # last-layer hidden state, (all hidden states), (all attentions)

      

def forward(self, hidden_states, attention_mask=None, head_mask=None):
  ## 接受参数如上
  mixed_query_layer = self.query(hidden_states) ## 生成query [16,32,768],16是batch_size,32是这个batch中每个句子的长度，768是维度
  mixed_key_layer = self.key(hidden_states)
  mixed_value_layer = self.value(hidden_states)

  query_layer = self.transpose_for_scores(mixed_query_layer)## 将上面生成的query进行维度转化，现在维度: [16,12,32,64]:[Batch_size,Num_head,Seq_len,每个头维度]
  key_layer = self.transpose_for_scores(mixed_key_layer)
  value_layer = self.transpose_for_scores(mixed_value_layer)

  # Take the dot product between "query" and "key" to get the raw attention scores.
  attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
  ## 上面操作之后 attention_scores 维度为torch.Size([16, 12, 32, 32])
  attention_scores = attention_scores / math.sqrt(self.attention_head_size)
  if attention_mask is not None:
  # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
  attention_scores = attention_scores + attention_mask
  ## 这里直接就是相加了，pad的部分直接为非常大的负值，下面softmax的时候，直接就为接近0

  # Normalize the attention scores to probabilities.
  attention_probs = nn.Softmax(dim=-1)(attention_scores)

  # This is actually dropping out entire tokens to attend to, which might
  # seem a bit unusual, but is taken from the original Transformer paper.
  attention_probs = self.dropout(attention_probs)##维度torch.Size([16, 12, 32, 32])

  # Mask heads if we want to
  if head_mask is not None:
  	attention_probs = attention_probs * head_mask

  context_layer = torch.matmul(attention_probs, value_layer)##维度torch.Size([16, 12, 32, 64])

  context_layer = context_layer.permute(0, 2, 1, 3).contiguous()##维度torch.Size([16, 32, 12, 64])
  new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)## new_context_layer_shape：torch.Size([16, 32, 768])
  context_layer = context_layer.view(*new_context_layer_shape)
##维度变成torch.Size([16, 32, 768])
  outputs = (context_layer, attention_probs) if self.output_attentions else (context_layer,)
  return outputs

class BertSelfOutput(nn.Module):
    def __init__(self, config):
        super(BertSelfOutput, self).__init__()
        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
        ## 做了一个linear 维度没变
        self.LayerNorm = BertLayerNorm(config.hidden_size, eps=config.layer_norm_eps)
        self.dropout = nn.Dropout(config.hidden_dropout_prob)

    def forward(self, hidden_states, input_tensor):
        hidden_states = self.dense(hidden_states)
        hidden_states = self.dropout(hidden_states)
        hidden_states = self.LayerNorm(hidden_states + input_tensor)
        return hidden_states