Advince/distilbert-base-uncased-lora-toxic-classification

README.md

@@ -0,0 +1,76 @@
+---
+base_model: distilbert-base-uncased
+library_name: peft
+license: apache-2.0
+metrics:
+- accuracy
+tags:
+- generated_from_trainer
+model-index:
+- name: distilbert-base-uncased-lora-text-classification
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# distilbert-base-uncased-lora-text-classification
+
+This model is a fine-tuned version of [distilbert-base-uncased](https://huggingface.co/distilbert-base-uncased) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.3115
+- Accuracy: {'accuracy': 0.8125}
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.001
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 15
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Accuracy                         |
+|:-------------:|:-----:|:----:|:---------------:|:--------------------------------:|
+| No log        | 1.0   | 431  | 0.6279          | {'accuracy': 0.7135416666666666} |
+| 0.7248        | 2.0   | 862  | 0.5949          | {'accuracy': 0.7604166666666666} |
+| 0.5433        | 3.0   | 1293 | 0.5986          | {'accuracy': 0.8072916666666666} |
+| 0.4762        | 4.0   | 1724 | 0.6967          | {'accuracy': 0.8125}             |
+| 0.3971        | 5.0   | 2155 | 0.7136          | {'accuracy': 0.8229166666666666} |
+| 0.3471        | 6.0   | 2586 | 0.8597          | {'accuracy': 0.8177083333333334} |
+| 0.2695        | 7.0   | 3017 | 1.0061          | {'accuracy': 0.8072916666666666} |
+| 0.2695        | 8.0   | 3448 | 0.7674          | {'accuracy': 0.8333333333333334} |
+| 0.2417        | 9.0   | 3879 | 1.2479          | {'accuracy': 0.828125}           |
+| 0.2079        | 10.0  | 4310 | 1.0548          | {'accuracy': 0.8177083333333334} |
+| 0.1941        | 11.0  | 4741 | 1.0516          | {'accuracy': 0.8229166666666666} |
+| 0.1711        | 12.0  | 5172 | 1.2246          | {'accuracy': 0.828125}           |
+| 0.1253        | 13.0  | 5603 | 1.2416          | {'accuracy': 0.8177083333333334} |
+| 0.0918        | 14.0  | 6034 | 1.3199          | {'accuracy': 0.8229166666666666} |
+| 0.0918        | 15.0  | 6465 | 1.3115          | {'accuracy': 0.8125}             |
+
+
+### Framework versions
+
+- PEFT 0.13.0
+- Transformers 4.44.2
+- Pytorch 2.4.1+cu121
+- Datasets 3.0.0
+- Tokenizers 0.19.1

adapter_config.json

@@ -0,0 +1,31 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "distilbert-base-uncased",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.01,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [
+    "classifier",
+    "score"
+  ],
+  "peft_type": "LORA",
+  "r": 4,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_lin"
+  ],
+  "task_type": "SEQ_CLS",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70c55ad8ee709d5ca29ddaaa939b5c82162f72af76aa36960605b9316e294432
+size 2521180

runs/Sep25_17-20-16_da58eb7a1d54/events.out.tfevents.1727284817.da58eb7a1d54.620.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0c7e1a4b25c92e35e8c1b82d3343b7d3f6a99bc81fb4082311998dc9932840a
+size 9795

runs/Sep25_17-47-24_da58eb7a1d54/events.out.tfevents.1727286445.da58eb7a1d54.620.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:842d4e18dfea33e591696c2322af150b3b3cba6b47b22397bf9f51c32da2aded
+size 6007

runs/Sep25_17-50-04_da58eb7a1d54/events.out.tfevents.1727286605.da58eb7a1d54.620.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41f1ef64935491aaf491a7288c67bba967df315b4e1feee890a567d5d64bb1a6
+size 22178

runs/Sep25_18-35-52_da58eb7a1d54/events.out.tfevents.1727289353.da58eb7a1d54.620.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5413d8757c24b1d962c373a9885b0c791c4374aed704f9452707a794d4ff11dc
+size 4999

runs/Sep25_18-36-44_da58eb7a1d54/events.out.tfevents.1727289405.da58eb7a1d54.620.4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ced1e95a96f297df4ed1029e8a7d3a9600fee5910503dc1bd06d5d9ef34f8dd
+size 24113

runs/Sep25_19-27-13_da58eb7a1d54/events.out.tfevents.1727292434.da58eb7a1d54.620.5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f10b9126faa0c990e1459820a289e4d269f2892c56ec26233f515f3ffd868daa
+size 9795

runs/Sep25_19-33-07_da58eb7a1d54/events.out.tfevents.1727292787.da58eb7a1d54.620.6

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c598ddfc8a9697cfe2c44b24025af884b69ebd558b1cfd5079d54436656f195
+size 11994

runs/Sep25_20-14-03_da58eb7a1d54/events.out.tfevents.1727295244.da58eb7a1d54.620.7

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1be67ed6b2cb1d000be57200799431d43ded28e58430af1cd2e64a69acaeab4e
+size 11994

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "DistilBertTokenizer",
+  "unk_token": "[UNK]"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66153193613f4463045ef8aba5885b66b724bc83111c9867a7656c6db070c4a2
+size 5240