Source code for util.text_similarity.lsh_min_hash.shingles_generator

import abc
import dataclasses




@dataclasses.dataclass
class ShinglesGenerator(abc.ABC):


    @abc.abstractmethod
    def generate_shingles(self, text: str) -> set[str]:
        pass






@dataclasses.dataclass
class SimpleShinglesGenerator(ShinglesGenerator):
    ngram_size: int



    def generate_shingles(self, text: str) -> set[str]:
        return {
                text[i: i + self.ngram_size]
                for i in range(len(text) - self.ngram_size + 1)
        }






@dataclasses.dataclass
class MultipleShinglesGenerator(ShinglesGenerator):
    ngram_sizes: list[int]



    def generate_shingles(self, text: str) -> set[str]:
        return set.union(
                *[
                        SimpleShinglesGenerator(ngram_size=n).generate_shingles(text)
                        for n in self.ngram_sizes
                ]
        )