ML Kit for Firebase + iOSで簡単テキスト認識やってみた

ML Kit for Firebaseとは？

• テキスト認識
• 顔検出
• バーコードスキャン
• 画像のラベリング
• 建物の認識

デバイス上、クラウド上それぞれでできること

FirebaseをiOSプロジェクトに追加する

pod 'Firebase/MLVision'

コードを書く

import UIKit
import AVFoundation
import Firebase

class ViewController: UIViewController {
    @IBOutlet weak var imageView: UIImageView!
    @IBOutlet weak var framesView: UIView!
    @IBOutlet weak var textLabel: UILabel!
    @IBOutlet weak var newButton: UIBarButtonItem!

    private lazy var vision = Vision.vision()

    override func viewDidLoad() {
        super.viewDidLoad()

        if !UIImagePickerController.isCameraDeviceAvailable(.front) || !UIImagePickerController.isCameraDeviceAvailable(.rear) {
            newButton.isEnabled = false
        }

        let tapRecognizer = UITapGestureRecognizer(target: self, action: #selector(didTapFrameLayer(_:)))
        self.framesView.addGestureRecognizer(tapRecognizer)
    }

    private func recognize() {
        clear()

        // テキスト検出する画像を指定
        guard let image = imageView.image else { return }
        let visionImage = VisionImage(image: image)

        // テキスト検出を実行
        vision.cloudTextDetector().detect(in: visionImage) { [weak self] (features, error) in
            guard let strongSelf = self else { return }
            guard let features = features else {
                if let error = error { strongSelf.textLabel.text = error.localizedDescription }
                return
            }
            guard let recognizedPages = features.pages else {
                strongSelf.textLabel.text = "****  failed to detect."
                return
            }
            // 検出結果を取得して表示する
            for page in recognizedPages {
                guard let blocks = page.blocks else { continue }
                for block in blocks {
                    guard let blockText = strongSelf.buildBlockText(from: block) else { continue }
                    strongSelf.drawBlockFrame(block.frame, withText: blockText)
                }
            }
        }
    }

    private func buildBlockText(from block: VisionCloudBlock) -> String? {
        guard let paragraphs = block.paragraphs else { return nil }
        return paragraphs
            .compactMap { $0.words?
                .compactMap { $0.symbols?.compactMap { $0.text }.joined() }
                .joined(separator: " ") }
            .joined(separator: "\n")
    }

    private func drawBlockFrame(_ frame: CGRect, withText text: String) {
        guard let image = imageView.image else { return }
        let scaleFactor = imageView.frame.width / image.size.width
        let scaledImageHeight = image.size.height * scaleFactor
        let scaledImageWidth = image.size.width * scaleFactor
        let xOffset = (imageView.frame.width - scaledImageWidth) / 2
        let yOffset = (imageView.frame.height - scaledImageHeight) / 2

        let layer = CALayer()
        layer.setValue(text, forKey: "BlockText")
        layer.frame = CGRect(x: frame.origin.x * scaleFactor + xOffset,
                             y: frame.origin.y * scaleFactor + yOffset,
                             width: frame.width * scaleFactor,
                             height: frame.height * scaleFactor)
        layer.borderWidth = 1.0
        layer.borderColor = UIColor.red.cgColor
        framesView.layer.addSublayer(layer)

    }

    private func clear() {
        textLabel.text = ""
        guard let sublayers = framesView.layer.sublayers else { return }
        for sublayer in sublayers {
            if let _ = sublayer.value(forKey: "BlockText") as? String {
                sublayer.removeFromSuperlayer()
            }
        }
    }

    @objc func didTapFrameLayer(_ sender: Any) {
        guard let tapRecognizer = sender as? UIGestureRecognizer,
            let frameLayers = framesView.layer.sublayers else { return }
        let tappedPoint = tapRecognizer.location(in: framesView)
        let tappedLayers = frameLayers.filter { $0.frame.contains(tappedPoint) }

        textLabel.text = ""
        if let tappedLayer = tappedLayers.first {
            if let text = tappedLayer.value(forKey: "BlockText") as? String {
                textLabel.text = text
            }
        }
    }

    @IBAction func didTabNewButton(_ sender: Any) {
        let imagePickerController = UIImagePickerController()
        imagePickerController.sourceType = .camera
        imagePickerController.delegate = self

        present(imagePickerController, animated: true)
    }
}

extension ViewController: UIImagePickerControllerDelegate {
    func imagePickerController(_ picker: UIImagePickerController, didFinishPickingMediaWithInfo info: [String : Any]) {
        if let image = info[UIImagePickerControllerOriginalImage] as? UIImage {
            let scaleFactor = imageView.frame.width * UIScreen.main.scale / image.size.width
            let scaledSize = CGSize(width: image.size.width * scaleFactor,
                                    height: image.size.height * scaleFactor)
            guard let resizedImage = image.resized(to: scaledSize) else { return }
            imageView.image = resizedImage
            recognize()
        }

        picker.dismiss(animated: true)
    }

    func imagePickerControllerDidCancel(_ picker: UIImagePickerController) {
        picker.dismiss(animated: true)
    }
}

extension ViewController: UINavigationControllerDelegate {}

extension UIImage {
    func resized(to size: CGSize) -> UIImage? {
        UIGraphicsBeginImageContextWithOptions(size, false, scale)
        draw(in: CGRect(origin: .zero, size: size))
        let resizedImage = UIGraphicsGetImageFromCurrentImageContext()
        UIGraphicsEndImageContext()
        return resizedImage
    }
}