Initial commit

Co-authored-by: tsbpp <tsbpp@users.noreply.huggingface.co>
Co-authored-by: meta-bot <meta-bot@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+webssl_teaser.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,51 @@
+---
+library_name: transformers
+license: cc-by-nc-4.0
+inference: false
+---
+# Web-SSL MAE ViT-3B: 2B MetaCLIP data, 224 Resolution
+A 3 billion parameter Vision Transformer (ViT) trained with Masked Autoencoder (MAE) self-supervised learning on web-scale image data without language supervision. Introduced in ["Scaling Language-Free Visual Representation Learning"](https://arxiv.org/abs/2504.01017) (Fan et al., 2025).
+
+## Model Details
+- **Architecture**: ViT (3072 width, 26 depth, 24 heads)
+- **Parameters**: 3B
+- **Resolution**: 224×224 pixels
+- **Training**: Self-supervised Web-MAE on 2B image samples from MetaCLIP web data
+
+## Model Descriptions
+Web-SSL MAE 3B is a 3 billion parameter Vision Transformer model trained using masked autoencoder self-supervised learning on 2 billion web images without language supervision. This model demonstrates that pure visual learning, when scaled appropriately, can match or exceed the performance of language-supervised models like CLIP across various vision tasks. Web-MAE exhibits particularly strong performance on OCR & Chart understanding tasks while maintaining competitive performance across traditional vision benchmarks and multimodal tasks.
+
+<img src="webssl_teaser.png" alt="WebSSL Model Overview" width="600">
+
+## Usage
+```python
+from transformers import AutoImageProcessor, ViTModel
+import torch
+from PIL import Image
+
+# Adjust the size, crop_size, etc. fields to your liking
+processor = AutoImageProcessor.from_pretrained('facebook/webssl-mae3b-full2b-224')
+model = ViTModel.from_pretrained('facebook/webssl-mae3b-full2b-224').cuda().eval()
+
+# Process an image
+image = Image.open('path/to/image.jpg')
+inputs = processor(images=image, return_tensors="pt").to('cuda')
+with torch.no_grad():
+    outputs = model(**inputs)
+
+# Extract features from the encoder
+encoder_hidden_states = outputs.last_hidden_state
+```
+
+## Citation
+
+```bibtex
+@article{fan2025scaling,
+  title={Scaling Language-Free Visual Representation Learning}, 
+  author={David Fan and Shengbang Tong and Jiachen Zhu and Koustuv Sinha and Zhuang Liu and Xinlei Chen and Michael Rabbat and Nicolas Ballas and Yann LeCun and Amir Bar and Saining Xie},
+  year={2025},
+  eprint={2504.01017},
+  archivePrefix={arXiv},
+  primaryClass={cs.CV}
+}
+```

config.json

@@ -0,0 +1,24 @@
+{
+  "_name_or_path": "/checkpoint/amaia/video/davidfan/shared_checkpoints/webssl/webssl_mae3b_full2b_224/hugging_face",
+  "architectures": [
+    "ViTModel"
+  ],
+  "attention_probs_dropout_prob": 0.0,
+  "encoder_stride": 16,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.0,
+  "hidden_size": 3072,
+  "image_size": 224,
+  "initializer_range": 0.02,
+  "intermediate_size": 12288,
+  "layer_norm_eps": 1e-06,
+  "mask_ratio": 0.0,
+  "model_type": "vit",
+  "num_attention_heads": 24,
+  "num_channels": 3,
+  "num_hidden_layers": 26,
+  "patch_size": 14,
+  "qkv_bias": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.46.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea6f49b43ed204d35df445c389325f64c4fd798c0ba8e53c921260a306e40d69
+size 4995019168

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d859ff5d667e65d80bfff6845ed9930cf43b7ee3aff9e2b3d3897f43ea00daa
+size 4984610968

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7de7e3d226fcc48ac9064f4a3d3bcf5e491353e543e2f1b800affa304d2a8ac4
+size 1850371736

model.safetensors.index.json

@@ -0,0 +1,431 @@
+{
+  "metadata": {
+    "total_size": 11829952512
+  },
+  "weight_map": {
+    "embeddings.cls_token": "model-00001-of-00003.safetensors",
+    "embeddings.patch_embeddings.projection.bias": "model-00001-of-00003.safetensors",
+    "embeddings.patch_embeddings.projection.weight": "model-00001-of-00003.safetensors",
+    "embeddings.position_embeddings": "model-00001-of-00003.safetensors",
+    "encoder.layer.0.attention.attention.key.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.0.attention.attention.key.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.0.attention.attention.query.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.0.attention.attention.query.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.0.attention.attention.value.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.0.attention.attention.value.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.0.attention.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.0.attention.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.0.intermediate.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.0.intermediate.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.0.layernorm_after.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.0.layernorm_after.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.0.layernorm_before.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.0.layernorm_before.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.0.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.0.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.1.attention.attention.key.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.1.attention.attention.key.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.1.attention.attention.query.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.1.attention.attention.query.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.1.attention.attention.value.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.1.attention.attention.value.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.1.attention.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.1.attention.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.1.intermediate.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.1.intermediate.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.1.layernorm_after.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.1.layernorm_after.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.1.layernorm_before.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.1.layernorm_before.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.1.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.1.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.10.attention.attention.key.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.10.attention.attention.key.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.10.attention.attention.query.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.10.attention.attention.query.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.10.attention.attention.value.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.10.attention.attention.value.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.10.attention.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.10.attention.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.10.intermediate.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.10.intermediate.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.10.layernorm_after.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.10.layernorm_after.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.10.layernorm_before.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.10.layernorm_before.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.10.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.10.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.11.attention.attention.key.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.11.attention.attention.key.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.11.attention.attention.query.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.11.attention.attention.query.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.11.attention.attention.value.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.11.attention.attention.value.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.11.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.11.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.11.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.11.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.11.layernorm_after.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.11.layernorm_after.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.11.layernorm_before.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.11.layernorm_before.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.11.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.11.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.12.attention.attention.key.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.12.attention.attention.key.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.12.attention.attention.query.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.12.attention.attention.query.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.12.attention.attention.value.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.12.attention.attention.value.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.12.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.12.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.12.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.12.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.12.layernorm_after.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.12.layernorm_after.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.12.layernorm_before.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.12.layernorm_before.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.12.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.12.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.13.attention.attention.key.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.13.attention.attention.key.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.13.attention.attention.query.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.13.attention.attention.query.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.13.attention.attention.value.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.13.attention.attention.value.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.13.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.13.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.13.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.13.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.13.layernorm_after.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.13.layernorm_after.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.13.layernorm_before.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.13.layernorm_before.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.13.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.13.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.14.attention.attention.key.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.14.attention.attention.key.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.14.attention.attention.query.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.14.attention.attention.query.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.14.attention.attention.value.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.14.attention.attention.value.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.14.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.14.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.14.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.14.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.14.layernorm_after.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.14.layernorm_after.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.14.layernorm_before.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.14.layernorm_before.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.14.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.14.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.15.attention.attention.key.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.15.attention.attention.key.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.15.attention.attention.query.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.15.attention.attention.query.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.15.attention.attention.value.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.15.attention.attention.value.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.15.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.15.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.15.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.15.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.15.layernorm_after.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.15.layernorm_after.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.15.layernorm_before.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.15.layernorm_before.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.15.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.15.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.16.attention.attention.key.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.16.attention.attention.key.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.16.attention.attention.query.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.16.attention.attention.query.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.16.attention.attention.value.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.16.attention.attention.value.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.16.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.16.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.16.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.16.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.16.layernorm_after.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.16.layernorm_after.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.16.layernorm_before.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.16.layernorm_before.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.16.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.16.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.17.attention.attention.key.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.17.attention.attention.key.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.17.attention.attention.query.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.17.attention.attention.query.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.17.attention.attention.value.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.17.attention.attention.value.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.17.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.17.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.17.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.17.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.17.layernorm_after.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.17.layernorm_after.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.17.layernorm_before.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.17.layernorm_before.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.17.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.17.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.18.attention.attention.key.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.18.attention.attention.key.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.18.attention.attention.query.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.18.attention.attention.query.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.18.attention.attention.value.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.18.attention.attention.value.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.18.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.18.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.18.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.18.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.18.layernorm_after.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.18.layernorm_after.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.18.layernorm_before.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.18.layernorm_before.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.18.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.18.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.19.attention.attention.key.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.19.attention.attention.key.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.19.attention.attention.query.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.19.attention.attention.query.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.19.attention.attention.value.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.19.attention.attention.value.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.19.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.19.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.19.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.19.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.19.layernorm_after.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.19.layernorm_after.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.19.layernorm_before.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.19.layernorm_before.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.19.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.19.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.2.attention.attention.key.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.2.attention.attention.key.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.2.attention.attention.query.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.2.attention.attention.query.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.2.attention.attention.value.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.2.attention.attention.value.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.2.attention.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.2.attention.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.2.intermediate.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.2.intermediate.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.2.layernorm_after.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.2.layernorm_after.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.2.layernorm_before.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.2.layernorm_before.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.2.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.2.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.20.attention.attention.key.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.20.attention.attention.key.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.20.attention.attention.query.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.20.attention.attention.query.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.20.attention.attention.value.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.20.attention.attention.value.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.20.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.20.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.20.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.20.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.20.layernorm_after.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.20.layernorm_after.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.20.layernorm_before.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.20.layernorm_before.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.20.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.20.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.21.attention.attention.key.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.21.attention.attention.key.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.21.attention.attention.query.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.21.attention.attention.query.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.21.attention.attention.value.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.21.attention.attention.value.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.21.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.21.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.21.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.21.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.21.layernorm_after.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.21.layernorm_after.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.21.layernorm_before.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.21.layernorm_before.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.21.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.layer.21.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.layer.22.attention.attention.key.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.22.attention.attention.key.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.22.attention.attention.query.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.22.attention.attention.query.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.22.attention.attention.value.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.22.attention.attention.value.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.22.attention.output.dense.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.22.attention.output.dense.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.22.intermediate.dense.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.22.intermediate.dense.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.22.layernorm_after.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.22.layernorm_after.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.22.layernorm_before.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.22.layernorm_before.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.22.output.dense.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.22.output.dense.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.23.attention.attention.key.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.23.attention.attention.key.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.23.attention.attention.query.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.23.attention.attention.query.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.23.attention.attention.value.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.23.attention.attention.value.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.23.attention.output.dense.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.23.attention.output.dense.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.23.intermediate.dense.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.23.intermediate.dense.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.23.layernorm_after.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.23.layernorm_after.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.23.layernorm_before.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.23.layernorm_before.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.23.output.dense.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.23.output.dense.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.24.attention.attention.key.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.24.attention.attention.key.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.24.attention.attention.query.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.24.attention.attention.query.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.24.attention.attention.value.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.24.attention.attention.value.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.24.attention.output.dense.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.24.attention.output.dense.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.24.intermediate.dense.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.24.intermediate.dense.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.24.layernorm_after.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.24.layernorm_after.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.24.layernorm_before.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.24.layernorm_before.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.24.output.dense.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.24.output.dense.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.25.attention.attention.key.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.25.attention.attention.key.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.25.attention.attention.query.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.25.attention.attention.query.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.25.attention.attention.value.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.25.attention.attention.value.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.25.attention.output.dense.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.25.attention.output.dense.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.25.intermediate.dense.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.25.intermediate.dense.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.25.layernorm_after.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.25.layernorm_after.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.25.layernorm_before.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.25.layernorm_before.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.25.output.dense.bias": "model-00003-of-00003.safetensors",
+    "encoder.layer.25.output.dense.weight": "model-00003-of-00003.safetensors",
+    "encoder.layer.3.attention.attention.key.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.3.attention.attention.key.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.3.attention.attention.query.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.3.attention.attention.query.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.3.attention.attention.value.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.3.attention.attention.value.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.3.attention.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.3.attention.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.3.intermediate.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.3.intermediate.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.3.layernorm_after.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.3.layernorm_after.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.3.layernorm_before.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.3.layernorm_before.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.3.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.3.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.4.attention.attention.key.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.4.attention.attention.key.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.4.attention.attention.query.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.4.attention.attention.query.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.4.attention.attention.value.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.4.attention.attention.value.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.4.attention.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.4.attention.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.4.intermediate.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.4.intermediate.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.4.layernorm_after.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.4.layernorm_after.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.4.layernorm_before.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.4.layernorm_before.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.4.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.4.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.5.attention.attention.key.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.5.attention.attention.key.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.5.attention.attention.query.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.5.attention.attention.query.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.5.attention.attention.value.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.5.attention.attention.value.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.5.attention.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.5.attention.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.5.intermediate.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.5.intermediate.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.5.layernorm_after.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.5.layernorm_after.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.5.layernorm_before.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.5.layernorm_before.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.5.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.5.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.6.attention.attention.key.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.6.attention.attention.key.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.6.attention.attention.query.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.6.attention.attention.query.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.6.attention.attention.value.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.6.attention.attention.value.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.6.attention.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.6.attention.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.6.intermediate.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.6.intermediate.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.6.layernorm_after.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.6.layernorm_after.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.6.layernorm_before.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.6.layernorm_before.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.6.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.6.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.7.attention.attention.key.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.7.attention.attention.key.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.7.attention.attention.query.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.7.attention.attention.query.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.7.attention.attention.value.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.7.attention.attention.value.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.7.attention.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.7.attention.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.7.intermediate.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.7.intermediate.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.7.layernorm_after.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.7.layernorm_after.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.7.layernorm_before.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.7.layernorm_before.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.7.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.7.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.8.attention.attention.key.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.8.attention.attention.key.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.8.attention.attention.query.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.8.attention.attention.query.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.8.attention.attention.value.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.8.attention.attention.value.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.8.attention.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.8.attention.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.8.intermediate.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.8.intermediate.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.8.layernorm_after.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.8.layernorm_after.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.8.layernorm_before.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.8.layernorm_before.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.8.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.8.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.9.attention.attention.key.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.9.attention.attention.key.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.9.attention.attention.query.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.9.attention.attention.query.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.9.attention.attention.value.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.9.attention.attention.value.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.9.attention.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.9.attention.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.9.intermediate.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.9.intermediate.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.9.layernorm_after.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.9.layernorm_after.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.9.layernorm_before.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.9.layernorm_before.weight": "model-00001-of-00003.safetensors",
+    "encoder.layer.9.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.layer.9.output.dense.weight": "model-00001-of-00003.safetensors",
+    "layernorm.bias": "model-00003-of-00003.safetensors",
+    "layernorm.weight": "model-00003-of-00003.safetensors",
+    "pooler.dense.bias": "model-00003-of-00003.safetensors",
+    "pooler.dense.weight": "model-00003-of-00003.safetensors"
+  }
+}

preprocessor_config.json

@@ -0,0 +1,27 @@
+{
+  "crop_size": {
+    "height": 224,
+    "width": 224
+  },
+  "do_center_crop": true,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.485,
+    0.456,
+    0.406
+  ],
+  "image_processor_type": "BitImageProcessor",
+  "image_std": [
+    0.229,
+    0.224,
+    0.225
+  ],
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "shortest_edge": 224
+  }
+}