Duplicate from pierreguillou/Inference-APP-Document-Understanding-at-linelevel-LiLT-base-LayoutXLM-base-v1

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,16 @@
+---
+title: >-
+  Document Understanding Inference APP (v1 - line level - LiLT base vs LayoutXLM
+  base)
+emoji: 🐢
+colorFrom: blue
+colorTo: yellow
+sdk: gradio
+sdk_version: 3.18.0
+app_file: app.py
+pinned: false
+duplicated_from: >-
+  pierreguillou/Inference-APP-Document-Understanding-at-linelevel-LiLT-base-LayoutXLM-base-v1
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,249 @@
+import os
+
+# workaround: install old version of pytorch since detectron2 hasn't released packages for pytorch 1.9 (issue: https://github.com/facebookresearch/detectron2/issues/3158)
+# os.system('pip install torch==1.8.0+cu101 torchvision==0.9.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html')
+os.system('pip install -q torch==1.10.0+cu111 torchvision==0.11+cu111 -f https://download.pytorch.org/whl/torch_stable.html')
+
+# install detectron2 that matches pytorch 1.8
+# See https://detectron2.readthedocs.io/tutorials/install.html for instructions
+#os.system('pip install -q detectron2 -f https://dl.fbaipublicfiles.com/detectron2/wheels/cu101/torch1.8/index.html')
+os.system('pip install git+https://github.com/facebookresearch/detectron2.git')
+
+import detectron2
+from detectron2.utils.logger import setup_logger
+setup_logger()
+
+import gradio as gr
+import re
+import string
+
+from operator import itemgetter
+import collections
+
+import pypdf
+from pypdf import PdfReader
+from pypdf.errors import PdfReadError
+
+import pdf2image
+from pdf2image import convert_from_path
+import langdetect
+from langdetect import detect_langs
+
+import pandas as pd
+import numpy as np
+import random
+import tempfile
+import itertools
+
+from matplotlib import font_manager
+from PIL import Image, ImageDraw, ImageFont
+import cv2
+
+## files
+
+import sys  
+sys.path.insert(0, 'files/')
+
+import functions
+from functions import *
+
+# update pip
+os.system('python -m pip install --upgrade pip')
+
+## model / feature extractor / tokenizer
+
+# models
+model_id_lilt = "pierreguillou/lilt-xlm-roberta-base-finetuned-with-DocLayNet-base-at-linelevel-ml384"
+model_id_layoutxlm = "pierreguillou/layout-xlm-base-finetuned-with-DocLayNet-base-at-linelevel-ml384"
+
+# get device
+import torch
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+## model LiLT
+import transformers
+from transformers import AutoTokenizer, AutoModelForTokenClassification
+tokenizer_lilt = AutoTokenizer.from_pretrained(model_id_lilt)
+model_lilt = AutoModelForTokenClassification.from_pretrained(model_id_lilt);
+model_lilt.to(device);
+
+## model LayoutXLM
+from transformers import LayoutLMv2ForTokenClassification # LayoutXLMTokenizerFast, 
+model_layoutxlm = LayoutLMv2ForTokenClassification.from_pretrained(model_id_layoutxlm);
+model_layoutxlm.to(device);
+
+# feature extractor
+from transformers import LayoutLMv2FeatureExtractor
+feature_extractor = LayoutLMv2FeatureExtractor(apply_ocr=False)
+
+# tokenizer
+from transformers import AutoTokenizer
+tokenizer_layoutxlm = AutoTokenizer.from_pretrained(tokenizer_id_layoutxlm)
+
+# get labels
+id2label_lilt = model_lilt.config.id2label
+label2id_lilt = model_lilt.config.label2id
+num_labels_lilt = len(id2label_lilt)
+
+id2label_layoutxlm = model_layoutxlm.config.id2label
+label2id_layoutxlm = model_layoutxlm.config.label2id
+num_labels_layoutxlm = len(id2label_layoutxlm)
+
+# APP outputs by model
+def app_outputs_by_model(uploaded_pdf, model_id, model, tokenizer, max_length, id2label, cls_box, sep_box):
+    filename, msg, images = pdf_to_images(uploaded_pdf)
+    num_images = len(images)
+
+    if not msg.startswith("Error with the PDF"):
+    
+        # Extraction of image data (text and bounding boxes)
+        dataset, lines, row_indexes, par_boxes, line_boxes = extraction_data_from_image(images)
+        # prepare our data in the format of the model
+        prepare_inference_features_partial = partial(prepare_inference_features, tokenizer=tokenizer, max_length=max_length, cls_box=cls_box, sep_box=sep_box)
+        encoded_dataset = dataset.map(prepare_inference_features_partial, batched=True, batch_size=64, remove_columns=dataset.column_names)
+        custom_encoded_dataset = CustomDataset(encoded_dataset, tokenizer)
+        # Get predictions (token level)
+        outputs, images_ids_list, chunk_ids, input_ids, bboxes = predictions_token_level(images, custom_encoded_dataset, model_id, model)
+        # Get predictions (line level)
+        probs_bbox, bboxes_list_dict, input_ids_dict_dict, probs_dict_dict, df = predictions_line_level(max_length, tokenizer, id2label, dataset, outputs, images_ids_list, chunk_ids, input_ids, bboxes, cls_box, sep_box)
+        # Get labeled images with lines bounding boxes
+        images = get_labeled_images(id2label, dataset, images_ids_list, bboxes_list_dict, probs_dict_dict)
+
+        img_files = list()
+        # get image of PDF without bounding boxes
+        for i in range(num_images):
+            if filename != "files/blank.png": img_file = f"img_{i}_" + filename.replace(".pdf", ".png")
+            else: img_file = filename.replace(".pdf", ".png")
+            img_file = img_file.replace("/", "_")
+            images[i].save(img_file)
+            img_files.append(img_file)
+
+        if num_images < max_imgboxes:
+            img_files += [image_blank]*(max_imgboxes - num_images)
+            images += [Image.open(image_blank)]*(max_imgboxes - num_images)
+            for count in range(max_imgboxes - num_images):
+                df[num_images + count] = pd.DataFrame()
+        else:
+            img_files = img_files[:max_imgboxes]
+            images = images[:max_imgboxes]
+            df = dict(itertools.islice(df.items(), max_imgboxes))
+
+        # save 
+        csv_files = list()
+        for i in range(max_imgboxes):
+            csv_file = f"csv_{i}_" + filename.replace(".pdf", ".csv")
+            csv_file = csv_file.replace("/", "_")
+            csv_files.append(gr.File.update(value=csv_file, visible=True))
+            df[i].to_csv(csv_file, encoding="utf-8", index=False)
+
+    else:  
+        img_files, images, csv_files = [""]*max_imgboxes, [""]*max_imgboxes, [""]*max_imgboxes
+        img_files[0], img_files[1] = image_blank, image_blank
+        images[0], images[1] = Image.open(image_blank), Image.open(image_blank)
+        csv_file = "csv_wo_content.csv"
+        csv_files[0], csv_files[1] = gr.File.update(value=csv_file, visible=True), gr.File.update(value=csv_file, visible=True)
+        df, df_empty = dict(), pd.DataFrame()
+        df[0], df[1] = df_empty.to_csv(csv_file, encoding="utf-8", index=False), df_empty.to_csv(csv_file, encoding="utf-8", index=False)
+    
+    return msg, img_files[0], images[0], csv_files[0], df[0]
+
+def app_outputs(uploaded_pdf):
+    msg_lilt, img_files_lilt, images_lilt, csv_files_lilt, df_lilt = app_outputs_by_model(uploaded_pdf, 
+                                                                                          model_id=model_id_lilt, model=model_lilt, tokenizer=tokenizer_lilt, 
+                                                                                          max_length=max_length_lilt, id2label=id2label_lilt, cls_box=cls_box, sep_box=sep_box_lilt)
+    
+    msg_layoutxlm, img_files_layoutxlm, images_layoutxlm, csv_files_layoutxlm, df_layoutxlm = app_outputs_by_model(uploaded_pdf, 
+                                                                                                                   model_id=model_id_layoutxlm, model=model_layoutxlm, tokenizer=tokenizer_layoutxlm, 
+                                                                                                                   max_length=max_length_layoutxlm, id2label=id2label_layoutxlm, cls_box=cls_box, sep_box=sep_box_layoutxlm)
+    
+    return msg_lilt, msg_layoutxlm, img_files_lilt, img_files_layoutxlm, images_lilt, images_layoutxlm, csv_files_lilt, csv_files_layoutxlm, df_lilt, df_layoutxlm
+
+# gradio APP
+with gr.Blocks(title="Inference APP for Document Understanding at line level (v1 - LiLT base vs LayoutXLM base)", css=".gradio-container") as demo:
+    gr.HTML("""
+    <div style="font-family:'Times New Roman', 'Serif'; font-size:26pt; font-weight:bold; text-align:center;"><h1>Inference APP for Document Understanding at line level (v1 - LiLT base vs LayoutXLM base)</h1></div>
+    <div style="margin-top: 40px"><p>(03/08/2023) This Inference APP compares - only on the first PDF page - 2 Document Understanding models finetuned on the dataset <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://huggingface.co/datasets/pierreguillou/DocLayNet-base" target="_blank">DocLayNet base</a> at line level (chunk size of 384 tokens): <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://huggingface.co/pierreguillou/lilt-xlm-roberta-base-finetuned-with-DocLayNet-base-at-linelevel-ml384" target="_blank">LiLT base combined with XLM-RoBERTa base</a> and <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://huggingface.co/pierreguillou/layout-xlm-base-finetuned-with-DocLayNet-base-at-linelevel-ml384" target="_blank">LayoutXLM base combined with XLM-RoBERTa base</a>.</p></div>
+    <div><p>To test these 2 models separately, use their corresponding APP on Hugging Face Spaces: <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://huggingface.co/spaces/pierreguillou/Inference-APP-Document-Understanding-at-linelevel-v1" target="_blank">LiLT base APP (v1 - line level)</a> and <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://huggingface.co/spaces/pierreguillou/Inference-APP-Document-Understanding-at-linelevel-v2" target="_blank">LayoutXLM base APP (v2 - line level)</a>.</p></div><div style="margin-top: 20px"><p>Links to Document Understanding APPs:</p><ul><li>Line level: <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://huggingface.co/spaces/pierreguillou/Inference-APP-Document-Understanding-at-linelevel-v1" target="_blank">v1 (LiLT base)</a> | <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://huggingface.co/spaces/pierreguillou/Inference-APP-Document-Understanding-at-linelevel-v2" target="_blank">v2 (LayoutXLM base)</a> | <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://huggingface.co/spaces/pierreguillou/Inference-APP-Document-Understanding-at-linelevel-LiLT-base-LayoutXLM-base-v1" target="_blank">v1 (LilT base vs LayoutXLM base)</a></li><li>Paragraph level: <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://huggingface.co/spaces/pierreguillou/Inference-APP-Document-Understanding-at-paragraphlevel-v1" target="_blank">v1 (LiLT base)</a></li></ul></div><div style="margin-top: 20px"><p>More information about the DocLayNet datasets, the finetuning of the model and this APP in the following blog posts:</p><ul><li>(03/05/2023) <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="" target="_blank">Document AI | Inference APP and fine-tuning notebook for Document Understanding at line level with LayoutXLM base</a></li><li>(02/14/2023) <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://medium.com/@pierre_guillou/document-ai-inference-app-for-document-understanding-at-line-level-a35bbfa98893" target="_blank">Document AI | Inference APP for Document Understanding at line level</a></li><li>(02/10/2023) <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://medium.com/@pierre_guillou/document-ai-document-understanding-model-at-line-level-with-lilt-tesseract-and-doclaynet-dataset-347107a643b8" target="_blank">Document AI | Document Understanding model at line level with LiLT, Tesseract and DocLayNet dataset</a></li><li>(01/31/2023) <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://medium.com/@pierre_guillou/document-ai-doclaynet-image-viewer-app-3ac54c19956" target="_blank">Document AI | DocLayNet image viewer APP</a></li><li>(01/27/2023) <a style="text-decoration: none; border-bottom: #64b5f6 0.125em solid; color: #64b5f6" href="https://medium.com/@pierre_guillou/document-ai-processing-of-doclaynet-dataset-to-be-used-by-layout-models-of-the-hugging-face-hub-308d8bd81cdb" target="_blank">Document AI | Processing of DocLayNet dataset to be used by layout models of the Hugging Face hub (finetuning, inference)</a></li></ul></div> 
+    """)
+    with gr.Row():
+        pdf_file = gr.File(label="PDF")
+    with gr.Row():
+        submit_btn = gr.Button(f"Get layout detection by LiLT and LayoutXLM on the first PDF page")
+        reset_btn = gr.Button(value="Clear") 
+    with gr.Row():
+        output_messages = []
+        with gr.Column():
+            output_msg = gr.Textbox(label="LiLT output message")
+            output_messages.append(output_msg)
+        with gr.Column():
+            output_msg = gr.Textbox(label="LayoutXLM output message")
+            output_messages.append(output_msg)
+    with gr.Row():
+        fileboxes = []
+        with gr.Column():
+            file_path = gr.File(visible=True, label=f"LiLT image file")
+            fileboxes.append(file_path)
+        with gr.Column():
+            file_path = gr.File(visible=True, label=f"LayoutXLM image file")
+            fileboxes.append(file_path)
+    with gr.Row():
+        imgboxes = []
+        with gr.Column():
+            img = gr.Image(type="pil", label=f"Lilt Image")
+            imgboxes.append(img)
+        with gr.Column():
+            img = gr.Image(type="pil", label=f"LayoutXLM Image")
+            imgboxes.append(img)
+    with gr.Row():
+        csvboxes = []
+        with gr.Column():
+            csv = gr.File(visible=True, label=f"LiLT csv file at line level")
+            csvboxes.append(csv)
+        with gr.Column():
+            csv = gr.File(visible=True, label=f"LayoutXLM csv file at line level")
+            csvboxes.append(csv)
+    with gr.Row():
+        dfboxes = []
+        with gr.Column():
+            df = gr.Dataframe(
+                      headers=["bounding boxes", "texts", "labels"],
+                      datatype=["str", "str", "str"],
+                      col_count=(3, "fixed"), 
+                      visible=True,
+                      label=f"LiLT data",
+                      type="pandas",
+                      wrap=True
+                    )
+            dfboxes.append(df)
+        with gr.Column():
+            df = gr.Dataframe(
+                      headers=["bounding boxes", "texts", "labels"],
+                      datatype=["str", "str", "str"],
+                      col_count=(3, "fixed"), 
+                      visible=True,
+                      label=f"LayoutXLM data",
+                      type="pandas",
+                      wrap=True
+                    )
+            dfboxes.append(df)
+
+    outputboxes = output_messages + fileboxes + imgboxes + csvboxes + dfboxes
+
+    submit_btn.click(app_outputs, inputs=[pdf_file], outputs=outputboxes)
+
+    # https://github.com/gradio-app/gradio/pull/2044/files#diff-a91dd2749f68bb7d0099a0f4079a4fd2d10281e299e7b451cb1bb876a7c21975R91
+    reset_btn.click(
+        lambda: [pdf_file.update(value=None)] + [output_msg.update(value=None) for output_msg in output_messages] + [filebox.update(value=None) for filebox in fileboxes] + [imgbox.update(value=None) for imgbox in imgboxes] + [csvbox.update(value=None) for csvbox in csvboxes] + [dfbox.update(value=None) for dfbox in dfboxes],
+        inputs=[],
+        outputs=[pdf_file] + output_messages + fileboxes + imgboxes + csvboxes + dfboxes
+        )
+    
+    gr.Examples(
+        [["files/example.pdf"]],
+        [pdf_file],
+        outputboxes,
+        fn=app_outputs,
+        cache_examples=True,
+        )
+    
+demo.launch()

files/functions.py

@@ -0,0 +1,896 @@
+import os
+
+# workaround: install old version of pytorch since detectron2 hasn't released packages for pytorch 1.9 (issue: https://github.com/facebookresearch/detectron2/issues/3158)
+# os.system('pip install torch==1.8.0+cu101 torchvision==0.9.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html')
+os.system('pip install -q torch==1.10.0+cu111 torchvision==0.11+cu111 -f https://download.pytorch.org/whl/torch_stable.html')
+
+# install detectron2 that matches pytorch 1.8
+# See https://detectron2.readthedocs.io/tutorials/install.html for instructions
+#os.system('pip install -q detectron2 -f https://dl.fbaipublicfiles.com/detectron2/wheels/cu101/torch1.8/index.html')
+os.system('pip install git+https://github.com/facebookresearch/detectron2.git')
+
+import detectron2
+from detectron2.utils.logger import setup_logger
+setup_logger()
+
+import gradio as gr
+import re
+import string
+import torch
+
+from operator import itemgetter
+import collections
+
+import pypdf
+from pypdf import PdfReader
+from pypdf.errors import PdfReadError
+
+import pdf2image
+from pdf2image import convert_from_path
+import langdetect
+from langdetect import detect_langs
+
+import pandas as pd
+import numpy as np
+import random
+import tempfile
+import itertools
+
+from matplotlib import font_manager
+from PIL import Image, ImageDraw, ImageFont
+import cv2
+
+import pathlib
+from pathlib import Path
+import shutil
+
+from functools import partial
+
+# Tesseract
+print(os.popen(f'cat /etc/debian_version').read())
+print(os.popen(f'cat /etc/issue').read())
+print(os.popen(f'apt search tesseract').read())
+import pytesseract
+
+## Key parameters
+
+# categories colors
+label2color = {
+    'Caption': 'brown',
+    'Footnote': 'orange',
+    'Formula': 'gray',
+    'List-item': 'yellow',
+    'Page-footer': 'red',
+    'Page-header': 'red',
+    'Picture': 'violet',
+    'Section-header': 'orange',
+    'Table': 'green',
+    'Text': 'blue',
+    'Title': 'pink'
+    }
+
+# bounding boxes start and end of a sequence
+cls_box = [0, 0, 0, 0]
+sep_box_lilt = cls_box
+sep_box_layoutxlm = [1000, 1000, 1000, 1000]
+
+# models
+model_id_lilt = "pierreguillou/lilt-xlm-roberta-base-finetuned-with-DocLayNet-base-at-linelevel-ml384"
+model_id_layoutxlm = "pierreguillou/layout-xlm-base-finetuned-with-DocLayNet-base-at-linelevel-ml384"
+
+# tokenizer for LayoutXLM
+tokenizer_id_layoutxlm = "xlm-roberta-base"
+
+# (tokenization) The maximum length of a feature (sequence)
+if str(384) in model_id_lilt:
+  max_length_lilt = 384 
+elif str(512) in model_id_lilt:
+  max_length_lilt = 512 
+else:
+  print("Error with max_length_lilt of chunks!")
+
+if str(384) in model_id_layoutxlm:
+  max_length_layoutxlm = 384 
+elif str(512) in model_id_layoutxlm:
+  max_length_layoutxlm = 512 
+else:
+  print("Error with max_length_layoutxlm of chunks!")
+
+# (tokenization) overlap
+doc_stride = 128 # The authorized overlap between two part of the context when splitting it is needed.
+
+# max PDF page images that will be displayed
+max_imgboxes = 1
+
+# get files
+examples_dir = 'files/'
+Path(examples_dir).mkdir(parents=True, exist_ok=True)
+from huggingface_hub import hf_hub_download
+files = ["example.pdf", "blank.pdf", "blank.png", "languages_iso.csv", "languages_tesseract.csv", "wo_content.png"]
+for file_name in files:
+    path_to_file = hf_hub_download(
+        repo_id = "pierreguillou/Inference-APP-Document-Understanding-at-linelevel-LiLT-base-LayoutXLM-base-v1",
+        filename = "files/" + file_name,
+        repo_type = "space"
+        )
+    shutil.copy(path_to_file,examples_dir)
+
+# path to files
+image_wo_content = examples_dir + "wo_content.png" # image without content
+pdf_blank = examples_dir + "blank.pdf" # blank PDF
+image_blank = examples_dir + "blank.png" # blank image
+
+## get langdetect2Tesseract dictionary
+t = "files/languages_tesseract.csv"
+l = "files/languages_iso.csv"
+
+df_t = pd.read_csv(t)
+df_l = pd.read_csv(l)
+
+langs_t = df_t["Language"].to_list()
+langs_t = [lang_t.lower().strip().translate(str.maketrans('', '', string.punctuation)) for lang_t in langs_t]
+langs_l = df_l["Language"].to_list()
+langs_l = [lang_l.lower().strip().translate(str.maketrans('', '', string.punctuation)) for lang_l in langs_l]
+langscode_t = df_t["LangCode"].to_list()
+langscode_l = df_l["LangCode"].to_list()
+
+Tesseract2langdetect, langdetect2Tesseract = dict(), dict()
+for lang_t, langcode_t in zip(langs_t,langscode_t):
+  try:
+    if lang_t == "Chinese - Simplified".lower().strip().translate(str.maketrans('', '', string.punctuation)): lang_t = "chinese"
+    index = langs_l.index(lang_t)
+    langcode_l = langscode_l[index]
+    Tesseract2langdetect[langcode_t] = langcode_l
+  except: 
+    continue
+
+langdetect2Tesseract = {v:k for k,v in Tesseract2langdetect.items()}
+
+## model / feature extractor / tokenizer
+
+# get device
+import torch
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+## model LiLT
+import transformers
+from transformers import AutoTokenizer, AutoModelForTokenClassification
+tokenizer_lilt = AutoTokenizer.from_pretrained(model_id_lilt)
+model_lilt = AutoModelForTokenClassification.from_pretrained(model_id_lilt);
+model_lilt.to(device);
+
+## model LayoutXLM
+from transformers import LayoutLMv2ForTokenClassification # LayoutXLMTokenizerFast, 
+model_layoutxlm = LayoutLMv2ForTokenClassification.from_pretrained(model_id_layoutxlm);
+model_layoutxlm.to(device);
+
+# feature extractor
+from transformers import LayoutLMv2FeatureExtractor
+feature_extractor = LayoutLMv2FeatureExtractor(apply_ocr=False)
+
+# tokenizer
+from transformers import AutoTokenizer
+tokenizer_layoutxlm = AutoTokenizer.from_pretrained(tokenizer_id_layoutxlm)
+
+# get labels
+id2label_lilt = model_lilt.config.id2label
+label2id_lilt = model_lilt.config.label2id
+num_labels_lilt = len(id2label_lilt)
+
+id2label_layoutxlm = model_layoutxlm.config.id2label
+label2id_layoutxlm = model_layoutxlm.config.label2id
+num_labels_layoutxlm = len(id2label_layoutxlm)
+
+## General
+
+# get text and bounding boxes from an image
+# https://stackoverflow.com/questions/61347755/how-can-i-get-line-coordinates-that-readed-by-tesseract
+# https://medium.com/geekculture/tesseract-ocr-understanding-the-contents-of-documents-beyond-their-text-a98704b7c655
+def get_data(results, factor, conf_min=0):
+
+  data = {}
+  for i in range(len(results['line_num'])):
+    level = results['level'][i]
+    block_num = results['block_num'][i]
+    par_num = results['par_num'][i]
+    line_num = results['line_num'][i]
+    top, left = results['top'][i], results['left'][i]
+    width, height = results['width'][i], results['height'][i]
+    conf = results['conf'][i]
+    text = results['text'][i]
+    if not (text == '' or text.isspace()):
+      if conf >= conf_min:
+        tup = (text, left, top, width, height)
+        if block_num in list(data.keys()):
+          if par_num in list(data[block_num].keys()):
+            if line_num in list(data[block_num][par_num].keys()):
+              data[block_num][par_num][line_num].append(tup)
+            else:
+              data[block_num][par_num][line_num] = [tup]
+          else:
+            data[block_num][par_num] = {}
+            data[block_num][par_num][line_num] = [tup]
+        else:
+            data[block_num] = {}
+            data[block_num][par_num] = {}
+            data[block_num][par_num][line_num] = [tup]
+
+  # get paragraphs dicionnary with list of lines
+  par_data = {}
+  par_idx = 1
+  for _, b  in data.items():
+    for _, p in b.items():
+      line_data = {}
+      line_idx = 1
+      for _, l in p.items():
+        line_data[line_idx] = l
+        line_idx += 1
+      par_data[par_idx] = line_data 
+      par_idx += 1
+
+  # get lines of texts, grouped by paragraph
+  lines = list()
+  row_indexes = list()
+  row_index = 0
+  for _,par in par_data.items():
+    count_lines = 0
+    for _,line in par.items():
+      if count_lines == 0: row_indexes.append(row_index)
+      line_text = ' '.join([item[0] for item in line])
+      lines.append(line_text)
+      count_lines += 1
+      row_index += 1
+    # lines.append("\n")
+    row_index += 1
+  # lines = lines[:-1]
+  
+  # get paragraphes boxes (par_boxes)
+  # get lines boxes (line_boxes)
+  par_boxes = list()
+  par_idx = 1
+  line_boxes = list()
+  line_idx = 1
+  for _, par in par_data.items():
+    xmins, ymins, xmaxs, ymaxs = list(), list(), list(), list()
+    for _, line in par.items():
+      xmin, ymin = line[0][1], line[0][2]
+      xmax, ymax = (line[-1][1] + line[-1][3]), (line[-1][2] + line[-1][4])
+      line_boxes.append([int(xmin/factor), int(ymin/factor), int(xmax/factor), int(ymax/factor)])
+      xmins.append(xmin)
+      ymins.append(ymin)
+      xmaxs.append(xmax)
+      ymaxs.append(ymax)
+      line_idx += 1
+    xmin, ymin, xmax, ymax = min(xmins), min(ymins), max(xmaxs), max(ymaxs)
+    par_boxes.append([int(xmin/factor), int(ymin/factor), int(xmax/factor), int(ymax/factor)])
+    par_idx += 1
+
+  return lines, row_indexes, par_boxes, line_boxes #data, par_data #
+
+# rescale image to get 300dpi
+def set_image_dpi_resize(image):
+    """
+    Rescaling image to 300dpi while resizing
+    :param image: An image
+    :return: A rescaled image
+    """
+    length_x, width_y = image.size
+    factor = min(1, float(1024.0 / length_x))
+    size = int(factor * length_x), int(factor * width_y)
+    # image_resize = image.resize(size, Image.Resampling.LANCZOS)
+    image_resize = image.resize(size, Image.LANCZOS)
+    temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='1.png')
+    temp_filename = temp_file.name
+    image_resize.save(temp_filename, dpi=(300, 300))
+    return factor, temp_filename
+
+# it is important that each bounding box should be in (upper left, lower right) format.
+# source: https://github.com/NielsRogge/Transformers-Tutorials/issues/129
+def upperleft_to_lowerright(bbox):
+  x0, y0, x1, y1 = tuple(bbox)
+  if bbox[2] < bbox[0]:
+    x0 = bbox[2]
+    x1 = bbox[0] 
+  if bbox[3] < bbox[1]:
+    y0 = bbox[3]
+    y1 = bbox[1] 
+  return [x0, y0, x1, y1]
+
+# convert boundings boxes (left, top, width, height) format to (left, top, left+widght, top+height) format. 
+def convert_box(bbox):
+    x, y, w, h = tuple(bbox) # the row comes in (left, top, width, height) format
+    return [x, y, x+w, y+h] # we turn it into (left, top, left+widght, top+height) to get the actual box 
+
+# LiLT model gets 1000x10000 pixels images
+def normalize_box(bbox, width, height):
+    return [
+        int(1000 * (bbox[0] / width)),
+        int(1000 * (bbox[1] / height)),
+        int(1000 * (bbox[2] / width)),
+        int(1000 * (bbox[3] / height)),
+    ]
+
+# LiLT model gets 1000x10000 pixels images
+def denormalize_box(bbox, width, height):
+    return [
+        int(width * (bbox[0] / 1000)),
+        int(height * (bbox[1] / 1000)),
+        int(width* (bbox[2] / 1000)),
+        int(height * (bbox[3] / 1000)),
+    ]
+
+# get back original size
+def original_box(box, original_width, original_height, coco_width, coco_height):
+    return [
+        int(original_width * (box[0] / coco_width)),
+        int(original_height * (box[1] / coco_height)),
+        int(original_width * (box[2] / coco_width)),
+        int(original_height* (box[3] / coco_height)),
+    ]
+
+def get_blocks(bboxes_block, categories, texts):
+
+ # get list of unique block boxes
+    bbox_block_dict, bboxes_block_list, bbox_block_prec = dict(), list(), list()
+    for count_block, bbox_block in enumerate(bboxes_block):
+      if bbox_block != bbox_block_prec:
+        bbox_block_indexes = [i for i, bbox in enumerate(bboxes_block) if bbox == bbox_block]
+        bbox_block_dict[count_block] = bbox_block_indexes
+        bboxes_block_list.append(bbox_block)
+      bbox_block_prec = bbox_block
+
+    # get list of categories and texts by unique block boxes
+    category_block_list, text_block_list = list(), list()
+    for bbox_block in bboxes_block_list:
+      count_block = bboxes_block.index(bbox_block)
+      bbox_block_indexes = bbox_block_dict[count_block]
+      category_block = np.array(categories, dtype=object)[bbox_block_indexes].tolist()[0]
+      category_block_list.append(category_block)
+      text_block = np.array(texts, dtype=object)[bbox_block_indexes].tolist()
+      text_block = [text.replace("\n","").strip() for text in text_block]
+      if id2label[category_block] == "Text" or id2label[category_block] == "Caption" or id2label[category_block] == "Footnote":
+        text_block = ' '.join(text_block)
+      else:
+        text_block = '\n'.join(text_block)
+      text_block_list.append(text_block)
+
+    return bboxes_block_list, category_block_list, text_block_list
+
+# function to sort bounding boxes
+def get_sorted_boxes(bboxes):
+
+  # sort by y from page top to bottom 
+  sorted_bboxes = sorted(bboxes, key=itemgetter(1), reverse=False)
+  y_list = [bbox[1] for bbox in sorted_bboxes]
+
+  # sort by x from page left to right when boxes with same y
+  if len(list(set(y_list))) != len(y_list):
+    y_list_duplicates_indexes = dict()
+    y_list_duplicates = [item for item, count in collections.Counter(y_list).items() if count > 1]
+    for item in y_list_duplicates:
+      y_list_duplicates_indexes[item] = [i for i, e in enumerate(y_list) if e == item]
+      bbox_list_y_duplicates = sorted(np.array(sorted_bboxes, dtype=object)[y_list_duplicates_indexes[item]].tolist(), key=itemgetter(0), reverse=False)
+      np_array_bboxes = np.array(sorted_bboxes)
+      np_array_bboxes[y_list_duplicates_indexes[item]] = np.array(bbox_list_y_duplicates)
+      sorted_bboxes = np_array_bboxes.tolist()
+
+  return sorted_bboxes
+
+# sort data from y = 0 to end of page (and after, x=0 to end of page when necessary)
+def sort_data(bboxes, categories, texts):
+
+    sorted_bboxes = get_sorted_boxes(bboxes)
+    sorted_bboxes_indexes = [bboxes.index(bbox) for bbox in sorted_bboxes]
+    sorted_categories = np.array(categories, dtype=object)[sorted_bboxes_indexes].tolist()
+    sorted_texts = np.array(texts, dtype=object)[sorted_bboxes_indexes].tolist()
+
+    return sorted_bboxes, sorted_categories, sorted_texts
+
+# sort data from y = 0 to end of page (and after, x=0 to end of page when necessary)
+def sort_data_wo_labels(bboxes, texts):
+
+    sorted_bboxes = get_sorted_boxes(bboxes)
+    sorted_bboxes_indexes = [bboxes.index(bbox) for bbox in sorted_bboxes]
+    sorted_texts = np.array(texts, dtype=object)[sorted_bboxes_indexes].tolist()
+
+    return sorted_bboxes, sorted_texts
+
+## PDF processing
+
+# get filename and images of PDF pages
+def pdf_to_images(uploaded_pdf):
+
+    # Check if None object
+    if uploaded_pdf is None:
+        path_to_file = pdf_blank
+        filename = path_to_file.replace(examples_dir,"")
+        msg = "Invalid PDF file."
+        images = [Image.open(image_blank)]
+    else:
+        # path to the uploaded PDF
+        path_to_file = uploaded_pdf.name
+        filename = path_to_file.replace("/tmp/","")
+
+        try:
+            PdfReader(path_to_file)
+        except PdfReadError:
+            path_to_file = pdf_blank
+            filename = path_to_file.replace(examples_dir,"")
+            msg = "Invalid PDF file."
+            images = [Image.open(image_blank)]
+        else:
+            try:
+                images = convert_from_path(path_to_file, last_page=max_imgboxes)
+                num_imgs = len(images)
+                msg = f'The PDF "{filename}" was converted into {num_imgs} images.'
+            except:
+                msg = f'Error with the PDF "{filename}": it was not converted into images.'
+                images = [Image.open(image_wo_content)]
+
+    return filename, msg, images
+
+# Extraction of image data (text and bounding boxes)
+def extraction_data_from_image(images):
+
+    num_imgs = len(images)
+
+    if num_imgs > 0:
+
+        # https://pyimagesearch.com/2021/11/15/tesseract-page-segmentation-modes-psms-explained-how-to-improve-your-ocr-accuracy/
+        custom_config = r'--oem 3 --psm 3 -l eng' # default config PyTesseract: --oem 3 --psm 3 -l eng+deu+fra+jpn+por+spa+rus+hin+chi_sim
+        results, lines, row_indexes, par_boxes, line_boxes, images_pixels = dict(), dict(), dict(), dict(), dict(), dict()
+        images_ids_list, lines_list, par_boxes_list, line_boxes_list, images_list, images_pixels_list, page_no_list, num_pages_list = list(), list(), list(), list(), list(), list(), list(), list()
+        
+        try: 
+            for i,image in enumerate(images):
+                # image preprocessing
+                # https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_imgproc/py_thresholding/py_thresholding.html
+                img = image.copy()
+                factor, path_to_img = set_image_dpi_resize(img) # Rescaling to 300dpi
+                img = Image.open(path_to_img)
+                img = np.array(img, dtype='uint8') # convert PIL to cv2
+                img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) # gray scale image
+                ret,img = cv2.threshold(img,127,255,cv2.THRESH_BINARY)
+
+                # OCR PyTesseract | get langs of page
+                txt = pytesseract.image_to_string(img, config=custom_config)
+                txt = txt.strip().lower()
+                txt = re.sub(r" +", " ", txt) # multiple space
+                txt = re.sub(r"(\n\s*)+\n+", "\n", txt) # multiple line
+                # txt = os.popen(f'tesseract {img_filepath} - {custom_config}').read()
+                try:
+                    langs = detect_langs(txt)
+                    langs = [langdetect2Tesseract[langs[i].lang] for i in range(len(langs))]
+                    langs_string = '+'.join(langs)
+                except:
+                    langs_string = "eng"
+                langs_string += '+osd'
+                custom_config = f'--oem 3 --psm 3 -l {langs_string}' # default config PyTesseract: --oem 3 --psm 3
+
+                # OCR PyTesseract | get data
+                results[i] = pytesseract.image_to_data(img, config=custom_config, output_type=pytesseract.Output.DICT)
+                # results[i] = os.popen(f'tesseract {img_filepath} - {custom_config}').read()
+
+                # get image pixels
+                images_pixels[i] = feature_extractor(images[i], return_tensors="pt").pixel_values
+
+                lines[i], row_indexes[i], par_boxes[i], line_boxes[i] = get_data(results[i], factor, conf_min=0)
+                lines_list.append(lines[i])
+                par_boxes_list.append(par_boxes[i])
+                line_boxes_list.append(line_boxes[i])
+                images_ids_list.append(i)
+                images_pixels_list.append(images_pixels[i])
+                images_list.append(images[i])
+                page_no_list.append(i)
+                num_pages_list.append(num_imgs)
+
+        except:
+            print(f"There was an error within the extraction of PDF text by the OCR!")
+        else: 
+            from datasets import Dataset
+            dataset = Dataset.from_dict({"images_ids": images_ids_list, "images": images_list, "images_pixels": images_pixels_list, "page_no": page_no_list, "num_pages": num_pages_list, "texts": lines_list, "bboxes_line": line_boxes_list})
+                    
+            # print(f"The text data was successfully extracted by the OCR!")
+
+            return dataset, lines, row_indexes, par_boxes, line_boxes
+
+## Inference
+
+def prepare_inference_features(example, tokenizer, max_length, cls_box, sep_box):
+
+  images_ids_list, chunks_ids_list, input_ids_list, attention_mask_list, bb_list, images_pixels_list = list(), list(), list(), list(), list(), list()
+
+  # get batch 
+  batch_images_ids = example["images_ids"]
+  batch_images = example["images"]
+  batch_images_pixels = example["images_pixels"]
+  batch_bboxes_line = example["bboxes_line"]
+  batch_texts = example["texts"]  
+  batch_images_size = [image.size for image in batch_images]
+
+  batch_width, batch_height = [image_size[0] for image_size in batch_images_size], [image_size[1] for image_size in batch_images_size]
+
+  # add a dimension if not a batch but only one image
+  if not isinstance(batch_images_ids, list): 
+    batch_images_ids = [batch_images_ids]
+    batch_images = [batch_images]
+    batch_images_pixels = [batch_images_pixels]
+    batch_bboxes_line = [batch_bboxes_line]
+    batch_texts = [batch_texts]
+    batch_width, batch_height = [batch_width], [batch_height] 
+
+  # process all images of the batch
+  for num_batch, (image_id, image_pixels, boxes, texts, width, height) in enumerate(zip(batch_images_ids, batch_images_pixels, batch_bboxes_line, batch_texts, batch_width, batch_height)):
+    tokens_list = []
+    bboxes_list = []
+
+    # add a dimension if only on image
+    if not isinstance(texts, list):
+      texts, boxes = [texts], [boxes]
+
+    # convert boxes to original
+    normalize_bboxes_line = [normalize_box(upperleft_to_lowerright(box), width, height) for box in boxes]
+
+    # sort boxes with texts
+    # we want sorted lists from top to bottom of the image
+    boxes, texts = sort_data_wo_labels(normalize_bboxes_line, texts)
+
+    count = 0
+    for box, text in zip(boxes, texts):
+      tokens = tokenizer.tokenize(text)
+      num_tokens = len(tokens) # get number of tokens
+      tokens_list.extend(tokens)
+
+      bboxes_list.extend([box] * num_tokens) # number of boxes must be the same as the number of tokens
+
+    # use of return_overflowing_tokens=True / stride=doc_stride
+    # to get parts of image with overlap
+    # source: https://huggingface.co/course/chapter6/3b?fw=tf#handling-long-contexts
+    encodings = tokenizer(" ".join(texts), 
+                          truncation=True,
+                          padding="max_length", 
+                          max_length=max_length, 
+                          stride=doc_stride, 
+                          return_overflowing_tokens=True, 
+                          return_offsets_mapping=True
+                          )
+
+    otsm = encodings.pop("overflow_to_sample_mapping")
+    offset_mapping = encodings.pop("offset_mapping")
+
+    # Let's label those examples and get their boxes   
+    sequence_length_prev = 0   
+    for i, offsets in enumerate(offset_mapping):
+      # truncate tokens, boxes and labels based on length of chunk - 2 (special tokens <s> and </s>)
+      sequence_length = len(encodings.input_ids[i]) - 2
+      if i == 0: start = 0
+      else: start += sequence_length_prev - doc_stride
+      end = start + sequence_length
+      sequence_length_prev = sequence_length
+
+      # get tokens, boxes and labels of this image chunk
+      bb = [cls_box] + bboxes_list[start:end] + [sep_box]
+
+      # as the last chunk can have a length < max_length
+      # we must to add [tokenizer.pad_token] (tokens), [sep_box] (boxes) and [-100] (labels)
+      if len(bb) < max_length:
+        bb = bb + [sep_box] * (max_length - len(bb))
+
+      # append results
+      input_ids_list.append(encodings["input_ids"][i])
+      attention_mask_list.append(encodings["attention_mask"][i])
+      bb_list.append(bb)
+      images_ids_list.append(image_id)
+      chunks_ids_list.append(i)
+      images_pixels_list.append(image_pixels)
+        
+  return {
+      "images_ids": images_ids_list,
+      "chunk_ids": chunks_ids_list,
+      "input_ids": input_ids_list,
+      "attention_mask": attention_mask_list,
+      "normalized_bboxes": bb_list,
+      "images_pixels": images_pixels_list
+  }
+
+from torch.utils.data import Dataset
+
+class CustomDataset(Dataset):
+  def __init__(self, dataset, tokenizer):
+    self.dataset = dataset
+    self.tokenizer = tokenizer
+
+  def __len__(self):
+    return len(self.dataset)
+
+  def __getitem__(self, idx):
+    # get item
+    example = self.dataset[idx]
+    encoding = dict()
+    encoding["images_ids"] = example["images_ids"]
+    encoding["chunk_ids"] = example["chunk_ids"]
+    encoding["input_ids"] = example["input_ids"]
+    encoding["attention_mask"] = example["attention_mask"]
+    encoding["bbox"] = example["normalized_bboxes"]
+    encoding["images_pixels"] = example["images_pixels"]
+    
+    return encoding
+
+import torch.nn.functional as F
+
+# get predictions at token level
+def predictions_token_level(images, custom_encoded_dataset, model_id, model):
+
+    num_imgs = len(images)
+    if num_imgs > 0:
+
+        chunk_ids, input_ids, bboxes, pixels_values, outputs, token_predictions  = dict(), dict(), dict(), dict(), dict(), dict()
+        images_ids_list = list()
+
+        for i,encoding in enumerate(custom_encoded_dataset):
+
+            # get custom encoded data
+            image_id = encoding['images_ids']
+            chunk_id = encoding['chunk_ids']
+            input_id = torch.tensor(encoding['input_ids'])[None]
+            attention_mask = torch.tensor(encoding['attention_mask'])[None]
+            bbox = torch.tensor(encoding['bbox'])[None]
+            pixel_values = torch.tensor(encoding["images_pixels"])
+
+            # save data in dictionnaries
+            if image_id not in images_ids_list: images_ids_list.append(image_id)
+
+            if image_id in chunk_ids: chunk_ids[image_id].append(chunk_id)
+            else: chunk_ids[image_id] = [chunk_id]
+
+            if image_id in input_ids: input_ids[image_id].append(input_id)
+            else: input_ids[image_id] = [input_id]
+
+            if image_id in bboxes: bboxes[image_id].append(bbox)
+            else: bboxes[image_id] = [bbox]
+
+            if image_id in pixels_values: pixels_values[image_id].append(pixel_values)
+            else: pixels_values[image_id] = [pixel_values]
+
+            # get prediction with forward pass
+            with torch.no_grad():
+
+                if model_id == model_id_lilt:
+                    output = model(
+                        input_ids=input_id.to(device),
+                        attention_mask=attention_mask.to(device),
+                        bbox=bbox.to(device),
+                        )
+                elif model_id == model_id_layoutxlm:
+                    output = model(
+                        input_ids=input_id.to(device),
+                        attention_mask=attention_mask.to(device),
+                        bbox=bbox.to(device),
+                        image=pixel_values.to(device)
+                        )                
+
+            # save probabilities of predictions in dictionnary
+            if image_id in outputs: outputs[image_id].append(F.softmax(output.logits.squeeze(), dim=-1))
+            else: outputs[image_id] = [F.softmax(output.logits.squeeze(), dim=-1)]
+
+        return outputs, images_ids_list, chunk_ids, input_ids, bboxes
+
+    else:
+        print("An error occurred while getting predictions!")
+
+from functools import reduce
+
+# Get predictions (line level)
+def predictions_line_level(max_length, tokenizer, id2label, dataset, outputs, images_ids_list, chunk_ids, input_ids, bboxes, cls_box, sep_box):
+
+    ten_probs_dict, ten_input_ids_dict, ten_bboxes_dict = dict(), dict(), dict()
+    bboxes_list_dict, input_ids_dict_dict, probs_dict_dict, df = dict(), dict(), dict(), dict()
+
+    if len(images_ids_list) > 0:
+
+        for i, image_id in enumerate(images_ids_list):
+
+            # get image information
+            images_list = dataset.filter(lambda example: example["images_ids"] == image_id)["images"]
+            image = images_list[0]
+            width, height = image.size
+
+            # get data
+            chunk_ids_list = chunk_ids[image_id]
+            outputs_list = outputs[image_id]
+            input_ids_list = input_ids[image_id]
+            bboxes_list = bboxes[image_id]
+
+            # create zeros tensors
+            ten_probs = torch.zeros((outputs_list[0].shape[0] - 2)*len(outputs_list), outputs_list[0].shape[1])
+            ten_input_ids = torch.ones(size=(1, (outputs_list[0].shape[0] - 2)*len(outputs_list)), dtype =int)
+            ten_bboxes = torch.zeros(size=(1, (outputs_list[0].shape[0] - 2)*len(outputs_list), 4), dtype =int)
+
+            if len(outputs_list) > 1:
+              
+                for num_output, (output, input_id, bbox) in enumerate(zip(outputs_list, input_ids_list, bboxes_list)):
+                    start = num_output*(max_length - 2) - max(0,num_output)*doc_stride
+                    end = start + (max_length - 2)
+                    
+                    if num_output == 0:
+                        ten_probs[start:end,:] += output[1:-1]
+                        ten_input_ids[:,start:end] = input_id[:,1:-1]
+                        ten_bboxes[:,start:end,:] = bbox[:,1:-1,:]
+                    else:
+                        ten_probs[start:start + doc_stride,:] += output[1:1 + doc_stride]
+                        ten_probs[start:start + doc_stride,:] = ten_probs[start:start + doc_stride,:] * 0.5
+                        ten_probs[start + doc_stride:end,:] += output[1 + doc_stride:-1]
+
+                        ten_input_ids[:,start:start + doc_stride] = input_id[:,1:1 + doc_stride]
+                        ten_input_ids[:,start + doc_stride:end] = input_id[:,1 + doc_stride:-1]
+
+                        ten_bboxes[:,start:start + doc_stride,:] = bbox[:,1:1 + doc_stride,:]
+                        ten_bboxes[:,start + doc_stride:end,:] = bbox[:,1 + doc_stride:-1,:]
+              
+            else:
+                ten_probs += outputs_list[0][1:-1] 
+                ten_input_ids = input_ids_list[0][:,1:-1] 
+                ten_bboxes = bboxes_list[0][:,1:-1] 
+
+            ten_probs_list, ten_input_ids_list, ten_bboxes_list = ten_probs.tolist(), ten_input_ids.tolist()[0], ten_bboxes.tolist()[0]
+            bboxes_list = list()
+            input_ids_dict, probs_dict = dict(), dict()
+            bbox_prev = [-100, -100, -100, -100]
+            for probs, input_id, bbox in zip(ten_probs_list, ten_input_ids_list, ten_bboxes_list):
+                bbox = denormalize_box(bbox, width, height)
+                if bbox != bbox_prev and bbox != cls_box and bbox != sep_box and bbox[0] != bbox[2] and bbox[1] != bbox[3]:
+                    bboxes_list.append(bbox)
+                    input_ids_dict[str(bbox)] = [input_id]
+                    probs_dict[str(bbox)] = [probs]
+                elif bbox != cls_box and bbox != sep_box and bbox[0] != bbox[2] and bbox[1] != bbox[3]:
+                    input_ids_dict[str(bbox)].append(input_id)
+                    probs_dict[str(bbox)].append(probs)
+                bbox_prev = bbox
+
+            probs_bbox = dict()
+            for i,bbox in enumerate(bboxes_list):
+                probs = probs_dict[str(bbox)]
+                probs = np.array(probs).T.tolist()
+            
+                probs_label = list()
+                for probs_list in probs:
+                    prob_label = reduce(lambda x, y: x*y, probs_list)
+                    probs_label.append(prob_label)
+                max_value = max(probs_label)
+                max_index = probs_label.index(max_value)
+                probs_bbox[str(bbox)] = max_index
+
+            bboxes_list_dict[image_id] = bboxes_list
+            input_ids_dict_dict[image_id] = input_ids_dict
+            probs_dict_dict[image_id] = probs_bbox
+
+            df[image_id] = pd.DataFrame()
+            df[image_id]["bboxes"] = bboxes_list
+            df[image_id]["texts"] = [tokenizer.decode(input_ids_dict[str(bbox)]) for bbox in bboxes_list]
+            df[image_id]["labels"] = [id2label[probs_bbox[str(bbox)]] for bbox in bboxes_list]
+
+        return probs_bbox, bboxes_list_dict, input_ids_dict_dict, probs_dict_dict, df
+
+    else:
+        print("An error occurred while getting predictions!")
+
+# Get labeled images with lines bounding boxes
+def get_labeled_images(id2label, dataset, images_ids_list, bboxes_list_dict, probs_dict_dict):
+
+    labeled_images = list()
+
+    for i, image_id in enumerate(images_ids_list):
+
+        # get image
+        images_list = dataset.filter(lambda example: example["images_ids"] == image_id)["images"]
+        image = images_list[0]
+        width, height = image.size
+
+        # get predicted boxes and labels
+        bboxes_list = bboxes_list_dict[image_id]
+        probs_bbox = probs_dict_dict[image_id]
+
+        draw = ImageDraw.Draw(image)
+        # https://stackoverflow.com/questions/66274858/choosing-a-pil-imagefont-by-font-name-rather-than-filename-and-cross-platform-f
+        font = font_manager.FontProperties(family='sans-serif', weight='bold')
+        font_file = font_manager.findfont(font)
+        font_size = 30
+        font = ImageFont.truetype(font_file, font_size)
+
+        for bbox in bboxes_list:
+              predicted_label = id2label[probs_bbox[str(bbox)]]
+              draw.rectangle(bbox, outline=label2color[predicted_label])
+              draw.text((bbox[0] + 10, bbox[1] - font_size), text=predicted_label, fill=label2color[predicted_label], font=font)
+
+        labeled_images.append(image)
+
+    return labeled_images
+
+# get data of encoded chunk
+def get_encoded_chunk_inference(tokenizer, dataset, encoded_dataset, index_chunk=None):
+
+  # get datasets
+  example = dataset
+  encoded_example = encoded_dataset
+
+  # get randomly a document in dataset
+  if index_chunk == None: index_chunk = random.randint(0, len(encoded_example)-1)
+  encoded_example = encoded_example[index_chunk]
+  encoded_image_ids = encoded_example["images_ids"]
+
+  # get the image
+  example = example.filter(lambda example: example["images_ids"] ==  encoded_image_ids)[0]
+  image = example["images"] # original image
+  width, height = image.size
+  page_no = example["page_no"]
+  num_pages = example["num_pages"]
+
+  # get boxes, texts, categories
+  bboxes, input_ids  = encoded_example["normalized_bboxes"][1:-1], encoded_example["input_ids"][1:-1]
+  bboxes = [denormalize_box(bbox, width, height) for bbox in bboxes]
+  num_tokens = len(input_ids) + 2
+
+  # get unique bboxes and corresponding labels
+  bboxes_list, input_ids_list = list(), list()
+  input_ids_dict = dict()
+  bbox_prev = [-100, -100, -100, -100]
+  for i, (bbox, input_id) in enumerate(zip(bboxes, input_ids)):
+    if bbox != bbox_prev:
+      bboxes_list.append(bbox)
+      input_ids_dict[str(bbox)] = [input_id]
+    else:
+      input_ids_dict[str(bbox)].append(input_id)
+    
+    # start_indexes_list.append(i)
+    bbox_prev = bbox
+  
+  # do not keep "</s><pad><pad>..."
+  if input_ids_dict[str(bboxes_list[-1])][0] == (tokenizer.convert_tokens_to_ids('</s>')):
+    del input_ids_dict[str(bboxes_list[-1])]
+    bboxes_list = bboxes_list[:-1]
+
+  # get texts by line
+  input_ids_list = input_ids_dict.values()
+  texts_list = [tokenizer.decode(input_ids) for input_ids in input_ids_list]
+  
+  # display DataFrame
+  df = pd.DataFrame({"texts": texts_list, "input_ids": input_ids_list, "bboxes": bboxes_list})
+
+  return image, df, num_tokens, page_no, num_pages
+
+# display chunk of PDF image and its data
+def display_chunk_lines_inference(dataset, encoded_dataset, index_chunk=None):
+
+  # get image and image data
+  image, df, num_tokens, page_no, num_pages = get_encoded_chunk_inference(dataset, encoded_dataset, index_chunk=index_chunk)
+
+  # get data from dataframe
+  input_ids = df["input_ids"]
+  texts = df["texts"]
+  bboxes = df["bboxes"]
+
+  print(f'Chunk ({num_tokens} tokens) of the PDF (page: {page_no+1} / {num_pages})\n')
+
+  # display image with bounding boxes
+  print(">> PDF image with bounding boxes of lines\n")
+  draw = ImageDraw.Draw(image)
+            
+  labels = list()
+  for box, text in zip(bboxes, texts):
+      color = "red"
+      draw.rectangle(box, outline=color)
+  
+  # resize image to original
+  width, height = image.size
+  image = image.resize((int(0.5*width), int(0.5*height)))
+
+  # convert to cv and display
+  img = np.array(image, dtype='uint8') # PIL to cv2
+  cv2_imshow(img)
+  cv2.waitKey(0)
+
+  # display image dataframe
+  print("\n>> Dataframe of annotated lines\n")
+  cols = ["texts",	"bboxes"]
+  df = df[cols]
+  display(df)

files/languages_iso.csv

@@ -0,0 +1,184 @@
+Language,LangCode
+Abkhazian,ab
+Afar,aa
+Afrikaans,af
+Akan,ak
+Albanian,sq
+Amharic,am
+Arabic,ar
+Aragonese,an
+Armenian,hy
+Assamese,as
+Avaric,av
+Avestan,ae
+Aymara,ay
+Azerbaijani,az
+Bambara,bm
+Bashkir,ba
+Basque,eu
+Belarusian,be
+Bengali,bn
+Bislama,bi
+Bosnian,bs
+Breton,br
+Bulgarian,bg
+Burmese,my
+"Catalan, Valencian",ca
+Chamorro,ch
+Chechen,ce
+"Chichewa, Chewa, Nyanja",ny
+Chinese,zh
+"Church Slavonic, Old Slavonic, Old Church Slavonic",cu
+Chuvash,cv
+Cornish,kw
+Corsican,co
+Cree,cr
+Croatian,hr
+Czech,cs
+Danish,da
+"Divehi, Dhivehi, Maldivian",dv
+"Dutch, Flemish",nl
+Dzongkha,dz
+English,en
+Esperanto,eo
+Estonian,et
+Ewe,ee
+Faroese,fo
+Fijian,fj
+Finnish,fi
+French,fr
+Western Frisian,fy
+Fulah,ff
+"Gaelic, Scottish Gaelic",gd
+Galician,gl
+Ganda,lg
+Georgian,ka
+German,de
+"Greek, Modern (1453–)",el
+"Kalaallisut, Greenlandic",kl
+Guarani,gn
+Gujarati,gu
+"Haitian, Haitian Creole",ht
+Hausa,ha
+Hebrew,he
+Herero,hz
+Hindi,hi
+Hiri Motu,ho
+Hungarian,hu
+Icelandic,is
+Ido,io
+Igbo,ig
+Indonesian,id
+Interlingua (International Auxiliary Language Association),ia
+"Interlingue, Occidental",ie
+Inuktitut,iu
+Inupiaq,ik
+Irish,ga
+Italian,it
+Japanese,ja
+Javanese,jv
+Kannada,kn
+Kanuri,kr
+Kashmiri,ks
+Kazakh,kk
+Central Khmer,km
+"Kikuyu, Gikuyu",ki
+Kinyarwanda,rw
+"Kirghiz, Kyrgyz",ky
+Komi,kv
+Kongo,kg
+Korean,ko
+"Kuanyama, Kwanyama",kj
+Kurdish,ku
+Lao,lo
+Latin,la
+Latvian,lv
+"Limburgan, Limburger, Limburgish",li
+Lingala,ln
+Lithuanian,lt
+Luba-Katanga,lu
+"Luxembourgish, Letzeburgesch",lb
+Macedonian,mk
+Malagasy,mg
+Malay,ms
+Malayalam,ml
+Maltese,mt
+Manx,gv
+Maori,mi
+Marathi,mr
+Marshallese,mh
+Mongolian,mn
+Nauru,na
+"Navajo, Navaho",nv
+North Ndebele,nd
+South Ndebele,nr
+Ndonga,ng
+Nepali,ne
+Norwegian,no
+Norwegian Bokmål,nb
+Norwegian Nynorsk,nn
+"Sichuan Yi, Nuosu",ii
+Occitan,oc
+Ojibwa,oj
+Oriya,or
+Oromo,om
+"Ossetian, Ossetic",os
+Pali,pi
+"Pashto, Pushto",ps
+Persian,fa
+Polish,pl
+Portuguese,pt
+"Punjabi, Panjabi",pa
+Quechua,qu
+"Romanian, Moldavian, Moldovan",ro
+Romansh,rm
+Rundi,rn
+Russian,ru
+Northern Sami,se
+Samoan,sm
+Sango,sg
+Sanskrit,sa
+Sardinian,sc
+Serbian,sr
+Shona,sn
+Sindhi,sd
+"Sinhala, Sinhalese",si
+Slovak,sk
+Slovenian,sl
+Somali,so
+Southern Sotho,st
+"Spanish, Castilian",es
+Sundanese,su
+Swahili,sw
+Swati,ss
+Swedish,sv
+Tagalog,tl
+Tahitian,ty
+Tajik,tg
+Tamil,ta
+Tatar,tt
+Telugu,te
+Thai,th
+Tibetan,bo
+Tigrinya,ti
+Tonga (Tonga Islands),to
+Tsonga,ts
+Tswana,tn
+Turkish,tr
+Turkmen,tk
+Twi,tw
+"Uighur, Uyghur",ug
+Ukrainian,uk
+Urdu,ur
+Uzbek,uz
+Venda,ve
+Vietnamese,vi
+Volapük,vo
+Walloon,wa
+Welsh,cy
+Wolof,wo
+Xhosa,xh
+Yiddish,yi
+Yoruba,yo
+"Zhuang, Chuang",za
+Zulu,zu

files/languages_tesseract.csv

@@ -0,0 +1,127 @@
+Language,LangCode
+Afrikaans,afr
+Amharic,amh
+Arabic,ara
+Assamese,asm
+Azerbaijani,aze
+Azerbaijani - Cyrilic,aze_cyrl
+Belarusian,bel
+Bengali,ben
+Tibetan,bod
+Bosnian,bos
+Breton,bre
+Bulgarian,bul
+Catalan; Valencian,cat
+Cebuano,ceb
+Czech,ces
+Chinese - Simplified,chi_sim
+Chinese - Traditional,chi_tra
+Cherokee,chr
+Corsican,cos
+Welsh,cym
+Danish,dan
+Danish - Fraktur (contrib),dan_frak
+German,deu
+German - Fraktur (contrib),deu_frak
+Dzongkha,dzo
+"Greek, Modern (1453-)",ell
+English,eng
+"English, Middle (1100-1500)",enm
+Esperanto,epo
+Math / equation detection module,equ
+Estonian,est
+Basque,eus
+Faroese,fao
+Persian,fas
+Filipino (old - Tagalog),fil
+Finnish,fin
+French,fra
+German - Fraktur,frk
+"French, Middle (ca.1400-1600)",frm
+Western Frisian,fry
+Scottish Gaelic,gla
+Irish,gle
+Galician,glg
+"Greek, Ancient (to 1453) (contrib)",grc
+Gujarati,guj
+Haitian; Haitian Creole,hat
+Hebrew,heb
+Hindi,hin
+Croatian,hrv
+Hungarian,hun
+Armenian,hye
+Inuktitut,iku
+Indonesian,ind
+Icelandic,isl
+Italian,ita
+Italian - Old,ita_old
+Javanese,jav
+Japanese,jpn
+Kannada,kan
+Georgian,kat
+Georgian - Old,kat_old
+Kazakh,kaz
+Central Khmer,khm
+Kirghiz; Kyrgyz,kir
+Kurmanji (Kurdish - Latin Script),kmr
+Korean,kor
+Korean (vertical),kor_vert
+Kurdish (Arabic Script),kur
+Lao,lao
+Latin,lat
+Latvian,lav
+Lithuanian,lit
+Luxembourgish,ltz
+Malayalam,mal
+Marathi,mar
+Macedonian,mkd
+Maltese,mlt
+Mongolian,mon
+Maori,mri
+Malay,msa
+Burmese,mya
+Nepali,nep
+Dutch; Flemish,nld
+Norwegian,nor
+Occitan (post 1500),oci
+Oriya,ori
+Orientation and script detection module,osd
+Panjabi; Punjabi,pan
+Polish,pol
+Portuguese,por
+Pushto; Pashto,pus
+Quechua,que
+Romanian; Moldavian; Moldovan,ron
+Russian,rus
+Sanskrit,san
+Sinhala; Sinhalese,sin
+Slovak,slk
+Slovak - Fraktur (contrib),slk_frak
+Slovenian,slv
+Sindhi,snd
+Spanish; Castilian,spa
+Spanish; Castilian - Old,spa_old
+Albanian,sqi
+Serbian,srp
+Serbian - Latin,srp_latn
+Sundanese,sun
+Swahili,swa
+Swedish,swe
+Syriac,syr
+Tamil,tam
+Tatar,tat
+Telugu,tel
+Tajik,tgk
+Tagalog (new - Filipino),tgl
+Thai,tha
+Tigrinya,tir
+Tonga,ton
+Turkish,tur
+Uighur; Uyghur,uig
+Ukrainian,ukr
+Urdu,urd
+Uzbek,uzb
+Uzbek - Cyrilic,uzb_cyrl
+Vietnamese,vie
+Yiddish,yid
+Yoruba,yor

packages.txt

@@ -0,0 +1,2 @@
+tesseract-ocr-all
+poppler-utils 

requirements.txt

@@ -0,0 +1,9 @@
+torch
+transformers
+datasets
+pytesseract
+opencv-python
+pdf2image
+pypdf
+langdetect
+gradio