org.grobid.core.utilities

Class LayoutTokensUtil

java.lang.Object

org.grobid.core.utilities.LayoutTokensUtil

public class LayoutTokensUtil
extends java.lang.Object

Created by zholudev on 18/12/15. Dealing with layout tokens

Field Summary

Fields

Modifier and Type	Field and Description
static <any>	TO_TEXT_FUNCTION

Constructor Summary

Constructors

Constructor and Description
LayoutTokensUtil()

Method Summary

Modifier and Type	Method and Description
static boolean	containsToken(java.util.List<LayoutToken> toks, java.lang.String text)
static java.util.List<LayoutToken>	dehyphenize(java.util.List<LayoutToken> tokens)
protected static boolean	doesRequireDehypenisation(java.util.List<LayoutToken> tokens, int i) Check if the current token (place i), or the hypen, needs to be removed or not.
static java.util.List<LayoutToken>	enrichWithNewLineInfo(java.util.List<LayoutToken> toks)
static java.lang.String	getCoordsString(java.util.List<LayoutToken> toks)
static java.lang.String	getCoordsStringForOneBox(java.util.List<LayoutToken> toks)
static java.util.List<LayoutToken>	getLayoutTokensForTokenizedText(java.util.List<java.lang.String> tokens)
static boolean	newLineToken(java.lang.String tok)
static boolean	noCoords(LayoutToken t)
static java.lang.String	normalizeDehyphenizeText(java.util.List<LayoutToken> tokens)
static java.lang.String	normalizeText(java.util.List<LayoutToken> tokens)
static java.lang.String	normalizeText(java.lang.String text)
static boolean	spaceyToken(java.lang.String tok)
static java.util.List<java.util.List<LayoutToken>>	split(java.util.List<LayoutToken> toks, java.util.regex.Pattern p, boolean preserveSeparator)
static java.util.List<java.util.List<LayoutToken>>	split(java.util.List<LayoutToken> toks, java.util.regex.Pattern p, boolean preserveSeparator, boolean preserveLeftOvers)
static java.util.List<LayoutToken>	subListByOffset(java.util.List<LayoutToken> token, int startIncluded)
static java.util.List<LayoutToken>	subListByOffset(java.util.List<LayoutToken> token, int startIncluded, int endExcluded)
static int	tokenPos(java.util.List<LayoutToken> toks, java.util.regex.Pattern p)
static int	tokenPos(java.util.List<LayoutToken> toks, java.lang.String text)
static boolean	tooFarAwayVertically(java.util.List<BoundingBox> boxes, double distance)
static java.lang.String	toText(java.util.List<LayoutToken> tokens)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

TO_TEXT_FUNCTION

public static final <any> TO_TEXT_FUNCTION

Constructor Detail

LayoutTokensUtil

public LayoutTokensUtil()

Method Detail

enrichWithNewLineInfo

public static java.util.List<LayoutToken> enrichWithNewLineInfo(java.util.List<LayoutToken> toks)

normalizeText

public static java.lang.String normalizeText(java.lang.String text)

normalizeText

public static java.lang.String normalizeText(java.util.List<LayoutToken> tokens)

normalizeDehyphenizeText

public static java.lang.String normalizeDehyphenizeText(java.util.List<LayoutToken> tokens)

toText

public static java.lang.String toText(java.util.List<LayoutToken> tokens)

noCoords

public static boolean noCoords(LayoutToken t)

spaceyToken

public static boolean spaceyToken(java.lang.String tok)

newLineToken

public static boolean newLineToken(java.lang.String tok)

containsToken

public static boolean containsToken(java.util.List<LayoutToken> toks,
                                    java.lang.String text)

tokenPos

public static int tokenPos(java.util.List<LayoutToken> toks,
                           java.lang.String text)

tokenPos

public static int tokenPos(java.util.List<LayoutToken> toks,
                           java.util.regex.Pattern p)

split

public static java.util.List<java.util.List<LayoutToken>> split(java.util.List<LayoutToken> toks,
                                                                java.util.regex.Pattern p,
                                                                boolean preserveSeparator)

split

public static java.util.List<java.util.List<LayoutToken>> split(java.util.List<LayoutToken> toks,
                                                                java.util.regex.Pattern p,
                                                                boolean preserveSeparator,
                                                                boolean preserveLeftOvers)

tooFarAwayVertically

public static boolean tooFarAwayVertically(java.util.List<BoundingBox> boxes,
                                           double distance)

getCoordsString

public static java.lang.String getCoordsString(java.util.List<LayoutToken> toks)

getCoordsStringForOneBox

public static java.lang.String getCoordsStringForOneBox(java.util.List<LayoutToken> toks)

dehyphenize

public static java.util.List<LayoutToken> dehyphenize(java.util.List<LayoutToken> tokens)

doesRequireDehypenisation

protected static boolean doesRequireDehypenisation(java.util.List<LayoutToken> tokens,
                                                   int i)

Check if the current token (place i), or the hypen, needs to be removed or not.

It will check the tokens before and after. It will get to the next "non space" tokens and verify that it's a plain word. If it's not it's keeping the hypen.

TODO: What to do in case of a punctuation is found?

subListByOffset

public static java.util.List<LayoutToken> subListByOffset(java.util.List<LayoutToken> token,
                                                          int startIncluded)

subListByOffset

public static java.util.List<LayoutToken> subListByOffset(java.util.List<LayoutToken> token,
                                                          int startIncluded,
                                                          int endExcluded)

getLayoutTokensForTokenizedText

public static java.util.List<LayoutToken> getLayoutTokensForTokenizedText(java.util.List<java.lang.String> tokens)