image_to_text.py

import easyocr
import cv2 as cv
import re
import numpy as np
import torch

from ultralytics import YOLO
from probabilities import Probability
from huggingface_hub import hf_hub_download
from xycut import bbox2points, recursive_xy_cut, vis_polygons_with_index
from sklearn.cluster import DBSCAN
import time

import os
os.environ['CUDA_VISIBLE_DEVICES'] = ''
os.environ['OMP_NUM_THREADS'] = '2'
os.environ['MKL_NUM_THREADS'] = '2'

import torch
torch.set_num_threads(2)
torch.set_num_interop_threads(1)


class ImageToText:
    def __init__(self):
        self.probability = Probability()
        self.reader = easyocr.Reader(['vi','en'])
        model_path = hf_hub_download(
            repo_id="hantian/yolo-doclaynet",
            filename="yolov8s-doclaynet.pt"
        )

        self.model = YOLO(model_path)

    def image_to_text(self, image_path):
        start_time = time.time()
        sorted_boxes = self.split_image(image_path)
        print(time.time() - start_time, " Split sucesfull")

        image = cv.imread(image_path)
        output_text = ""

        for i, box in enumerate(sorted_boxes):
            image_sorte = image[box[1]:box[3], box[0]:box[2]]
            result_text = self.reader.readtext(image_sorte)
            full_text = ""
            for _, text, _ in result_text:
                full_text += text+"\n"
            full_text = re.sub(r'\n(?![A-Z])', ' ', full_text)


            for text in full_text.split("\n"):
                fixed_text = self.probability.fix_spelling(text)
                fixed_text = fixed_text.capitalize()
                output_text = output_text + fixed_text + "\n"


        print(time.time() - start_time, " OCR sucesfull")
        return output_text

    def split_image(self, image_path):
        results = self.model(image_path, imgsz=1024, conf=0.15, iou=0.4, agnostic_nms=True)
        image = cv.imread(image_path)
        sorted_boxes = []
        for result in results:
            boxes = result.boxes
            list_box = []

            for box in boxes:
                x_box_min = int(box.xyxy[0][0])
                y_box_min = int(box.xyxy[0][1])
                x_box_max = int(box.xyxy[0][2])
                y_box_max = int(box.xyxy[0][3])
                if x_box_max <= x_box_min or y_box_max <= y_box_min:
                    continue

                class_id = int(box.cls[0])
                label = self.model.names[class_id]
                conf = box.conf[0].item()

                if class_id != 6:
                    list_box.append([x_box_min, y_box_min, x_box_max, y_box_max, class_id, label, conf])

            avg_height = sum(box[3] - box[1] for box in list_box) / len(list_box)
            centers = np.array([[(b[0] + b[2]) / 2, (b[1] + b[3]) / 2] for b in list_box])
            clustering = DBSCAN(eps=avg_height * 4, min_samples=1).fit(centers)

            list_label_area = clustering.labels_

            box_area = []
            label = []
            for label in range(max(list_label_area) + 1):
                label_index = [list_box[i] for i, label_name in enumerate(list_label_area) if label_name == label]
                x1 = min(label[0] for label in label_index)
                y1 = min(label[1] for label in label_index)
                x2 = max(label[2] for label in label_index)
                y2 = max(label[3] for label in label_index)

                box_area.append([x1, y1, x2, y2])

            random_boxes = np.array(box_area)
            np.random.shuffle(random_boxes)
            res = []

            recursive_xy_cut(np.asarray(random_boxes).astype(int), np.arange(len(random_boxes)), res)

            assert len(res) == len(box_area)

            random_boxes = random_boxes[np.array(res)].tolist()
            while random_boxes:
                short_box = random_boxes.pop(0)
                index_short = [i for i, box in enumerate(random_boxes) if self.is_overlapping(short_box, box)]
                if index_short:
                    bbox = [short_box]
                    for i, index in enumerate(index_short):
                        bbox.append(random_boxes.pop(index - i))
                    x1 = min(b[0] for b in bbox)
                    y1 = min(b[1] for b in bbox)
                    x2 = max(b[2] for b in bbox)
                    y2 = max(b[3] for b in bbox)
                    short_box = [x1, y1, x2, y2]
                sorted_boxes.append(short_box)
        return sorted_boxes

    def is_overlapping(self, box1, box2):
        return not (box1[2] < box2[0] or box1[0] > box2[2] or box1[3] < box2[1] or box1[1] > box2[3])


if __name__ == "__main__":
    img_path = "list_image/65d581a5dca4b054b6980098_Budapest.jpg"
    image_to_text = ImageToText()
    result = image_to_text.image_to_text(img_path)