org.grobid.core.document

Class Document

java.lang.Object

org.grobid.core.document.Document

All Implemented Interfaces:

java.io.Serializable

Direct Known Subclasses:

PatentDocument

public class Document
extends java.lang.Object
implements java.io.Serializable

Class for representing, processing and exchanging a document item.

See Also:

Serialized Form

Field Summary

Fields

Modifier and Type	Field and Description
protected java.util.List<java.lang.Integer>	acknowledgementBlocks
protected Analyzer	analyzer
protected int	beginBody
protected int	beginReferences
protected java.util.List<BibDataSet>	bibDataSets
protected java.util.List<java.lang.Integer>	blockDocumentHeaders
protected java.util.List<java.lang.Integer>	blockFigures
protected java.util.List<java.lang.Integer>	blockFooters
protected java.util.List<java.lang.Integer>	blockHeaders
protected java.util.List<java.lang.Integer>	blockHeadFigures
protected java.util.List<java.lang.Integer>	blockHeadTables
protected java.util.SortedSet<DocumentPiece>	blockReferences
protected java.util.List<Block>	blocks
protected java.util.List<java.lang.Integer>	blockSectionTitles
protected java.util.List<java.lang.Integer>	blockTables
protected java.util.List<Cluster>	clusters
protected int	documentLenghtChar
protected DocumentSource	documentSource
protected java.util.List<Equation>	equations
protected FeatureFactory	featureFactory
protected java.util.List<Figure>	figures
protected java.util.List<GraphicObject>	images
protected <any>	imagesPerPage
protected <any>	labeledBlocks
protected java.lang.String	lang
protected static Logger	LOGGER
protected int	m
static int	MAX_FIG_BOX_DISTANCE
protected double	maxBlockSpacing
protected double	maxCharacterDensity
protected Metadata	metadata
protected static double	MIN_DISTANCE
protected double	minBlockSpacing
protected double	minCharacterDensity
protected static int	nbBins
protected DocumentNode	outlineRoot
protected java.util.List<Page>	pages
protected java.lang.String	pathXML
protected java.util.List<PDFAnnotation>	pdfAnnotations
protected ReferenceMarkerMatcher	referenceMarkerMatcher
protected BiblioItem	resHeader
static long	serialVersionUID
protected java.util.List<Table>	tables
protected java.lang.String	tei
protected java.util.Map<java.lang.String,BibDataSet>	teiIdToBibDataSets
protected boolean	titleMatchNum
protected java.util.List<LayoutToken>	tokenizations
protected <any>	validGraphicObjectPredicate

Constructor Summary

Constructors

Modifier	Constructor and Description
protected	Document()
	Document(DocumentSource documentSource)

Method Summary

Modifier and Type	Method and Description
void	addBlock(Block b)
void	addPage(Page page)
java.util.List<LayoutToken>	addTokenizedDocument(GrobidAnalysisConfig config) Parser PDFALTO output representation and get the tokenized form of the document.
void	assignGraphicObjectsToFigures()
void	calculateTeiIdToBibDataSets()
static Document	createFromText(java.lang.String text)
java.util.List<LayoutToken>	fromText(java.lang.String text)
java.lang.String	getAllBlocksClean(int toIgnore1, int toIgnore2) Return all blocks without markers.
Analyzer	getAnalyzer()
BibDataSet	getBibDataSetByTeiId(java.lang.String teiId)
java.util.List<BibDataSet>	getBibDataSets()
java.util.List<java.lang.Integer>	getBlockDocumentHeaders()
java.util.List<Block>	getBlocks()
java.lang.String	getBody() Return the text content of the body of the document.
java.util.List<Cluster>	getClusters()
static java.util.List<GraphicObject>	getConnectedGraphics(Block block, Document doc) Return the list of graphical object touching the given block.
protected static int	getCoordItem(ElementCounter<java.lang.Integer> cnt, boolean getMin)
int	getDocumentLenghtChar()
java.util.SortedSet<DocumentPiece>	getDocumentPart(TaggingLabel segmentationLabel) Get the document part corresponding to a particular segment type
java.lang.String	getDocumentPartText(TaggingLabel segmentationLabel)
java.lang.String	getDocumentPieceText(DocumentPiece dp)
java.lang.String	getDocumentPieceText(java.util.SortedSet<DocumentPiece> dps)
java.util.List<LayoutToken>	getDocumentPieceTokenization(DocumentPiece dp)
DocumentSource	getDocumentSource()
java.util.List<java.lang.String>	getDOIMatches()
java.util.List<Equation>	getEquations()
protected java.util.List<LayoutToken>	getFigureLayoutTokens(Figure f)
java.util.List<Figure>	getFigures()
java.lang.String	getHeader() heuristics to get the header section...
java.lang.String	getHeaderByIntroduction() We try to match the introduction section in a safe way, and consider if minimum requirements are met the blocks before this position as header.
java.lang.String	getHeaderFeatured(boolean getHeader, boolean withRotation) Add features in the header section -> should be moved to the header parser class!
java.lang.String	getHeaderLastHope() We return the first page as header estimation...
java.util.List<GraphicObject>	getImages()
<any>	getLabeledBlocks()
java.lang.String	getLanguage()
double	getMaxBlockSpacing()
double	getMaxCharacterDensity()
Metadata	getMetadata()
double	getMinBlockSpacing()
double	getMinCharacterDensity()
DocumentNode	getOutlineRoot()
Page	getPage(int num)
java.util.List<Page>	getPages()
java.util.List<PDFAnnotation>	getPDFAnnotations()
ReferenceMarkerMatcher	getReferenceMarkerMatcher()
BiblioItem	getResHeader()
java.util.List<Table>	getTables()
java.lang.String	getTei()
static java.util.List<LayoutToken>	getTokenizationParts(java.util.SortedSet<DocumentPiece> documentParts, java.util.List<LayoutToken> tokenizations) Give the list of LayoutToken corresponding to some document parts and a global document tokenization.
java.util.List<LayoutToken>	getTokenizations()
java.util.List<LayoutToken>	getTokenizationsFulltext() Deprecated.
java.util.List<LayoutToken>	getTokenizationsHeader() Deprecated.
java.util.List<LayoutToken>	getTokenizationsReferences() Deprecated.
static java.util.List<LayoutToken>	getTokens(java.util.List<LayoutToken> tokenizations, int offsetBegin, int offsetEnd)
static java.util.List<LayoutToken>	getTokensFrom(java.util.List<LayoutToken> tokenizations, int offsetBegin, int offsetEnd, int startTokenIndex)
protected java.util.ArrayList<GraphicObject>	glueImagesIfNecessary(java.lang.Integer pageNum, java.util.List<GraphicObject> graphicObjects)
boolean	isTitleMatchNum()
protected boolean	isValidBitmapGraphicObject(GraphicObject go)
void	postProcessTables()
void	produceStatistics()
protected void	recalculateVectorBoxCoords(Figure f, GraphicObject g)
void	setAcknowledgementBlocks(java.util.List<java.lang.Integer> acknowledgementBlocks)
void	setAnalyzer(Analyzer analyzer)
void	setBibDataSets(java.util.List<BibDataSet> bibDataSets)
void	setBlockDocumentHeaders(java.util.List<java.lang.Integer> blockDocumentHeaders)
void	setBlockFigures(java.util.List<java.lang.Integer> blockFigures)
void	setBlockFooters(java.util.List<java.lang.Integer> blockFooters)
void	setBlockHeaders(java.util.List<java.lang.Integer> blockHeaders)
void	setBlockHeadFigures(java.util.List<java.lang.Integer> blockHeadFigures)
void	setBlockHeadTables(java.util.List<java.lang.Integer> blockHeadTables)
void	setBlockReferences(java.util.SortedSet<DocumentPiece> blockReferences)
void	setBlockSectionTitles(java.util.List<java.lang.Integer> blockSectionTitles)
void	setBlockTables(java.util.List<java.lang.Integer> blockTables)
void	setClusters(java.util.List<Cluster> clusters)
void	setConnectedGraphics2(Figure figure)
void	setEquations(java.util.List<Equation> equations)
void	setFigures(java.util.List<Figure> figures)
void	setImages(java.util.List<GraphicObject> images)
void	setLabeledBlocks(<any> labeledBlocks)
void	setLanguage(java.lang.String l)
void	setOutlineRoot(DocumentNode outlineRoot)
void	setPages(java.util.List<Page> pages)
protected void	setPathXML(java.io.File pathXML) Set the path to the XML file generated by xml2pdf
void	setResHeader(BiblioItem resHeader)
void	setTables(java.util.List<Table> tables)
void	setTei(java.lang.String tei)
void	setTitleMatchNum(boolean titleMatchNum)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

serialVersionUID

public static final long serialVersionUID

See Also:

Constant Field Values

LOGGER

protected static final Logger LOGGER

MAX_FIG_BOX_DISTANCE

public static final int MAX_FIG_BOX_DISTANCE

See Also:

Constant Field Values

documentSource

protected final transient DocumentSource documentSource

pathXML

protected java.lang.String pathXML

lang

protected java.lang.String lang

pages

protected transient java.util.List<Page> pages

clusters

protected transient java.util.List<Cluster> clusters

blocks

protected transient java.util.List<Block> blocks

blockHeaders

protected java.util.List<java.lang.Integer> blockHeaders

blockFooters

protected java.util.List<java.lang.Integer> blockFooters

blockSectionTitles

protected java.util.List<java.lang.Integer> blockSectionTitles

acknowledgementBlocks

protected java.util.List<java.lang.Integer> acknowledgementBlocks

blockDocumentHeaders

protected java.util.List<java.lang.Integer> blockDocumentHeaders

blockReferences

protected transient java.util.SortedSet<DocumentPiece> blockReferences

blockTables

protected java.util.List<java.lang.Integer> blockTables

blockFigures

protected java.util.List<java.lang.Integer> blockFigures

blockHeadTables

protected java.util.List<java.lang.Integer> blockHeadTables

blockHeadFigures

protected java.util.List<java.lang.Integer> blockHeadFigures

featureFactory

protected transient FeatureFactory featureFactory

labeledBlocks

protected transient <any> labeledBlocks

tokenizations

protected java.util.List<LayoutToken> tokenizations

teiIdToBibDataSets

protected transient java.util.Map<java.lang.String,BibDataSet> teiIdToBibDataSets

bibDataSets

protected transient java.util.List<BibDataSet> bibDataSets

resHeader

protected transient BiblioItem resHeader

tei

protected java.lang.String tei

referenceMarkerMatcher

protected transient ReferenceMarkerMatcher referenceMarkerMatcher

images

protected transient java.util.List<GraphicObject> images

pdfAnnotations

protected transient java.util.List<PDFAnnotation> pdfAnnotations

outlineRoot

protected transient DocumentNode outlineRoot

metadata

protected transient Metadata metadata

imagesPerPage

protected transient <any> imagesPerPage

maxCharacterDensity

protected double maxCharacterDensity

minCharacterDensity

protected double minCharacterDensity

maxBlockSpacing

protected double maxBlockSpacing

minBlockSpacing

protected double minBlockSpacing

documentLenghtChar

protected int documentLenghtChar

beginBody

protected int beginBody

beginReferences

protected int beginReferences

titleMatchNum

protected boolean titleMatchNum

figures

protected transient java.util.List<Figure> figures

validGraphicObjectPredicate

protected transient <any> validGraphicObjectPredicate

m

protected int m

tables

protected transient java.util.List<Table> tables

equations

protected transient java.util.List<Equation> equations

analyzer

protected transient Analyzer analyzer

nbBins

protected static final int nbBins

See Also:

Constant Field Values

MIN_DISTANCE

protected static double MIN_DISTANCE

Constructor Detail

Document

public Document(DocumentSource documentSource)

Document

protected Document()

Method Detail

setImages

public void setImages(java.util.List<GraphicObject> images)

createFromText

public static Document createFromText(java.lang.String text)

setLanguage

public void setLanguage(java.lang.String l)

getLanguage

public java.lang.String getLanguage()

getResHeader

public BiblioItem getResHeader()

getBlocks

public java.util.List<Block> getBlocks()

getBibDataSets

public java.util.List<BibDataSet> getBibDataSets()

addBlock

public void addBlock(Block b)

getImages

public java.util.List<GraphicObject> getImages()

getPDFAnnotations

public java.util.List<PDFAnnotation> getPDFAnnotations()

getMetadata

public Metadata getMetadata()

setPathXML

protected void setPathXML(java.io.File pathXML)

Set the path to the XML file generated by xml2pdf

getTokenizations

public java.util.List<LayoutToken> getTokenizations()

getDocumentLenghtChar

public int getDocumentLenghtChar()

getMaxCharacterDensity

public double getMaxCharacterDensity()

getMinCharacterDensity

public double getMinCharacterDensity()

getMaxBlockSpacing

public double getMaxBlockSpacing()

getMinBlockSpacing

public double getMinBlockSpacing()

setAnalyzer

public void setAnalyzer(Analyzer analyzer)

getAnalyzer

public Analyzer getAnalyzer()

getTokenizationsHeader

@Deprecated
public java.util.List<LayoutToken> getTokenizationsHeader()

Deprecated.

getTokenizationsFulltext

@Deprecated
public java.util.List<LayoutToken> getTokenizationsFulltext()

Deprecated.

getTokenizationsReferences

@Deprecated
public java.util.List<LayoutToken> getTokenizationsReferences()

Deprecated.

fromText

public java.util.List<LayoutToken> fromText(java.lang.String text)

addTokenizedDocument

public java.util.List<LayoutToken> addTokenizedDocument(GrobidAnalysisConfig config)

Parser PDFALTO output representation and get the tokenized form of the document.

Returns:

list of features

glueImagesIfNecessary

protected java.util.ArrayList<GraphicObject> glueImagesIfNecessary(java.lang.Integer pageNum,
                                                                   java.util.List<GraphicObject> graphicObjects)

getCoordItem

protected static int getCoordItem(ElementCounter<java.lang.Integer> cnt,
                                  boolean getMin)

getHeaderFeatured

public java.lang.String getHeaderFeatured(boolean getHeader,
                                          boolean withRotation)

Add features in the header section

-> should be moved to the header parser class!

getHeader

public java.lang.String getHeader()

heuristics to get the header section... -> it is now covered by the CRF segmentation model

getHeaderLastHope

public java.lang.String getHeaderLastHope()

We return the first page as header estimation... better than nothing when nothing is not acceptable.

-> now covered by the CRF segmentation model

getHeaderByIntroduction

public java.lang.String getHeaderByIntroduction()

We try to match the introduction section in a safe way, and consider if minimum requirements are met the blocks before this position as header.

-> now covered by the CRF segmentation model

getBody

public java.lang.String getBody()

Return the text content of the body of the document. getHeader() and getReferences() must have been called before.

-> this should be removed at some point... it is only used now as default solution to determine the language of an article with the language identifier

getAllBlocksClean

public java.lang.String getAllBlocksClean(int toIgnore1,
                                          int toIgnore2)

Return all blocks without markers.

Ignore the toIgnore1 th blocks and the blocks after toIgnore2 (included)

getDOIMatches

public java.util.List<java.lang.String> getDOIMatches()

getTei

public java.lang.String getTei()

setTei

public void setTei(java.lang.String tei)

getBlockDocumentHeaders

public java.util.List<java.lang.Integer> getBlockDocumentHeaders()

getOutlineRoot

public DocumentNode getOutlineRoot()

setOutlineRoot

public void setOutlineRoot(DocumentNode outlineRoot)

isTitleMatchNum

public boolean isTitleMatchNum()

setTitleMatchNum

public void setTitleMatchNum(boolean titleMatchNum)

getPages

public java.util.List<Page> getPages()

getPage

public Page getPage(int num)

getClusters

public java.util.List<Cluster> getClusters()

setBlockHeaders

public void setBlockHeaders(java.util.List<java.lang.Integer> blockHeaders)

setBlockFooters

public void setBlockFooters(java.util.List<java.lang.Integer> blockFooters)

setBlockSectionTitles

public void setBlockSectionTitles(java.util.List<java.lang.Integer> blockSectionTitles)

setAcknowledgementBlocks

public void setAcknowledgementBlocks(java.util.List<java.lang.Integer> acknowledgementBlocks)

setBlockDocumentHeaders

public void setBlockDocumentHeaders(java.util.List<java.lang.Integer> blockDocumentHeaders)

setBlockReferences

public void setBlockReferences(java.util.SortedSet<DocumentPiece> blockReferences)

setBlockTables

public void setBlockTables(java.util.List<java.lang.Integer> blockTables)

setBlockFigures

public void setBlockFigures(java.util.List<java.lang.Integer> blockFigures)

setBlockHeadTables

public void setBlockHeadTables(java.util.List<java.lang.Integer> blockHeadTables)

setBlockHeadFigures

public void setBlockHeadFigures(java.util.List<java.lang.Integer> blockHeadFigures)

setClusters

public void setClusters(java.util.List<Cluster> clusters)

setPages

public void setPages(java.util.List<Page> pages)

addPage

public void addPage(Page page)

setBibDataSets

public void setBibDataSets(java.util.List<BibDataSet> bibDataSets)

getReferenceMarkerMatcher

public ReferenceMarkerMatcher getReferenceMarkerMatcher()
                                                 throws EntityMatcherException

Throws:

EntityMatcherException

calculateTeiIdToBibDataSets

public void calculateTeiIdToBibDataSets()

getLabeledBlocks

public <any> getLabeledBlocks()

setLabeledBlocks

public void setLabeledBlocks(<any> labeledBlocks)

getDocumentPieceTokenization

public java.util.List<LayoutToken> getDocumentPieceTokenization(DocumentPiece dp)

getDocumentPieceText

public java.lang.String getDocumentPieceText(DocumentPiece dp)

getDocumentPieceText

public java.lang.String getDocumentPieceText(java.util.SortedSet<DocumentPiece> dps)

getDocumentPart

public java.util.SortedSet<DocumentPiece> getDocumentPart(TaggingLabel segmentationLabel)

Get the document part corresponding to a particular segment type

getDocumentPartText

public java.lang.String getDocumentPartText(TaggingLabel segmentationLabel)

getTokenizationParts

public static java.util.List<LayoutToken> getTokenizationParts(java.util.SortedSet<DocumentPiece> documentParts,
                                                               java.util.List<LayoutToken> tokenizations)

Give the list of LayoutToken corresponding to some document parts and a global document tokenization.

getBibDataSetByTeiId

public BibDataSet getBibDataSetByTeiId(java.lang.String teiId)

getConnectedGraphics

public static java.util.List<GraphicObject> getConnectedGraphics(Block block,
                                                                 Document doc)

Return the list of graphical object touching the given block.

postProcessTables

public void postProcessTables()

assignGraphicObjectsToFigures

public void assignGraphicObjectsToFigures()

isValidBitmapGraphicObject

protected boolean isValidBitmapGraphicObject(GraphicObject go)

recalculateVectorBoxCoords

protected void recalculateVectorBoxCoords(Figure f,
                                          GraphicObject g)

getFigureLayoutTokens

protected java.util.List<LayoutToken> getFigureLayoutTokens(Figure f)

setConnectedGraphics2

public void setConnectedGraphics2(Figure figure)

produceStatistics

public void produceStatistics()

getDocumentSource

public DocumentSource getDocumentSource()

setFigures

public void setFigures(java.util.List<Figure> figures)

getFigures

public java.util.List<Figure> getFigures()

setTables

public void setTables(java.util.List<Table> tables)

getTables

public java.util.List<Table> getTables()

setEquations

public void setEquations(java.util.List<Equation> equations)

getEquations

public java.util.List<Equation> getEquations()

setResHeader

public void setResHeader(BiblioItem resHeader)

getTokens

public static java.util.List<LayoutToken> getTokens(java.util.List<LayoutToken> tokenizations,
                                                    int offsetBegin,
                                                    int offsetEnd)

getTokensFrom

public static java.util.List<LayoutToken> getTokensFrom(java.util.List<LayoutToken> tokenizations,
                                                        int offsetBegin,
                                                        int offsetEnd,
                                                        int startTokenIndex)