v1.0

bert_layers.py

@@ -826,12 +826,13 @@ class BertForSequenceClassification(BertPreTrainedModel):
         self.num_labels = config.num_labels
         self.config = config
 
-        self.bert = BertModel(config)
+        self.bert = BertModel(config, add_pooling_layer=False)
+        self.pooler = nn.Linear(config.hidden_size, 1)
         classifier_dropout = (config.classifier_dropout
                               if config.classifier_dropout is not None else
                               config.hidden_dropout_prob)
         self.dropout = nn.Dropout(classifier_dropout)
-        self.classifier = nn.Linear(config.hidden_size, config.num_labels)
+        self.classifier = nn.Linear(10, config.num_labels)
 
         # Initialize weights and apply final processing
         self.post_init()
@@ -871,7 +872,8 @@ class BertForSequenceClassification(BertPreTrainedModel):
             return_dict=return_dict,
         )
 
-        pooled_output = outputs[1]
+        encoder_output = outputs[0]
+        pooled_output = self.pooler(encoder_output).squeeze(-1)[:, :10]
 
         pooled_output = self.dropout(pooled_output)
         logits = self.classifier(pooled_output)

config.json

@@ -1 +1 @@
-{"_name_or_path": "zhihan1996/DNABERT-2-117M", "alibi_starting_size": 512, "architectures": ["BertForMaskedLM"], "attention_probs_dropout_prob": 0.0, "auto_map": {"AutoConfig": "configuration_bert.BertConfig", "AutoModel": "bert_layers.BertModel", "AutoModelForMaskedLM": "bert_layers.BertForMaskedLM", "AutoModelForSequenceClassification": "bert_layers.BertForSequenceClassification"}, "classifier_dropout": null, "gradient_checkpointing": false, "hidden_act": "gelu", "hidden_dropout_prob": 0.1, "hidden_size": 768, "initializer_range": 0.02, "intermediate_size": 3072, "layer_norm_eps": 1e-12, "max_position_embeddings": 512, "model_type": "bert", "num_attention_heads": 12, "num_hidden_layers": 12, "position_embedding_type": "absolute", "torch_dtype": "float32", "transformers_version": "4.28.0", "type_vocab_size": 2, "use_cache": true, "vocab_size": 4096}
+{"_name_or_path": "zhihan1996/DNABERT-2-117M", "alibi_starting_size": 512, "architectures": ["BertForMaskedLM"], "attention_probs_dropout_prob": 0.0, "auto_map": {"AutoConfig": "configuration_bert.BertConfig", "AutoModel": "bert_layers.BertModel", "AutoModelForMaskedLM": "bert_layers.BertForMaskedLM", "AutoModelForSequenceClassification": "bert_layers.BertForSequenceClassification"}, "classifier_dropout": 0.01, "gradient_checkpointing": false, "hidden_act": "gelu", "hidden_dropout_prob": 0.1, "hidden_size": 768, "initializer_range": 0.02, "intermediate_size": 3072, "layer_norm_eps": 1e-12, "max_position_embeddings": 512, "model_type": "bert", "num_attention_heads": 12, "num_hidden_layers": 12, "position_embedding_type": "absolute", "torch_dtype": "float32", "transformers_version": "4.28.0", "type_vocab_size": 2, "use_cache": true, "vocab_size": 4096}