nu.validator.htmlparser.impl

Class ErrorReportingTokenizer

java.lang.Object

nu.validator.htmlparser.impl.Tokenizer

nu.validator.htmlparser.impl.ErrorReportingTokenizer

All Implemented Interfaces:

Locator

public class ErrorReportingTokenizer
extends Tokenizer

Field Summary

Fields inherited from class nu.validator.htmlparser.impl.Tokenizer

AFTER_ATTRIBUTE_NAME, AFTER_ATTRIBUTE_VALUE_QUOTED, AFTER_DOCTYPE_NAME, AFTER_DOCTYPE_PUBLIC_IDENTIFIER, AFTER_DOCTYPE_PUBLIC_KEYWORD, AFTER_DOCTYPE_SYSTEM_IDENTIFIER, AFTER_DOCTYPE_SYSTEM_KEYWORD, ampersandLocation, ATTRIBUTE_NAME, ATTRIBUTE_VALUE_DOUBLE_QUOTED, ATTRIBUTE_VALUE_SINGLE_QUOTED, ATTRIBUTE_VALUE_UNQUOTED, attributeName, BEFORE_ATTRIBUTE_NAME, BEFORE_ATTRIBUTE_VALUE, BEFORE_DOCTYPE_NAME, BEFORE_DOCTYPE_PUBLIC_IDENTIFIER, BEFORE_DOCTYPE_SYSTEM_IDENTIFIER, BETWEEN_DOCTYPE_PUBLIC_AND_SYSTEM_IDENTIFIERS, BOGUS_COMMENT, BOGUS_COMMENT_HYPHEN, BOGUS_DOCTYPE, CDATA_RSQB, CDATA_RSQB_RSQB, CDATA_SECTION, CDATA_START, CHARACTER_REFERENCE_HILO_LOOKUP, CHARACTER_REFERENCE_TAIL, CLOSE_TAG_OPEN, COMMENT, COMMENT_END, COMMENT_END_BANG, COMMENT_END_DASH, COMMENT_START, COMMENT_START_DASH, confident, CONSUME_CHARACTER_REFERENCE, CONSUME_NCR, cstart, DATA, DECIMAL_NRC_LOOP, DOCTYPE, DOCTYPE_NAME, DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED, DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED, DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED, DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED, DOCTYPE_UBLIC, DOCTYPE_YSTEM, encodingDeclarationHandler, endTag, endTagExpectation, errorHandler, HANDLE_NCR_VALUE, HANDLE_NCR_VALUE_RECONSUME, HEX_NCR_LOOP, html4, index, lastCR, MARKUP_DECLARATION_HYPHEN, MARKUP_DECLARATION_OCTYPE, MARKUP_DECLARATION_OPEN, NON_DATA_END_TAG_NAME, PLAINTEXT, RAWTEXT, RAWTEXT_RCDATA_LESS_THAN_SIGN, RCDATA, SCRIPT_DATA, SCRIPT_DATA_DOUBLE_ESCAPE_END, SCRIPT_DATA_DOUBLE_ESCAPE_START, SCRIPT_DATA_DOUBLE_ESCAPED, SCRIPT_DATA_DOUBLE_ESCAPED_DASH, SCRIPT_DATA_DOUBLE_ESCAPED_DASH_DASH, SCRIPT_DATA_DOUBLE_ESCAPED_LESS_THAN_SIGN, SCRIPT_DATA_ESCAPE_START, SCRIPT_DATA_ESCAPE_START_DASH, SCRIPT_DATA_ESCAPED, SCRIPT_DATA_ESCAPED_DASH, SCRIPT_DATA_ESCAPED_DASH_DASH, SCRIPT_DATA_ESCAPED_LESS_THAN_SIGN, SCRIPT_DATA_LESS_THAN_SIGN, SELF_CLOSING_START_TAG, stateSave, TAG_NAME, TAG_OPEN, tokenHandler, value

Constructor Summary

Constructors

Constructor and Description
ErrorReportingTokenizer(TokenHandler tokenHandler)
ErrorReportingTokenizer(TokenHandler tokenHandler, boolean newAttributesEachTime)

Method Summary

Methods

Modifier and Type	Method and Description
protected char	checkChar(char[] buf, int pos)
protected void	errAstralNonCharacter(int ch)
protected void	errAttributeValueMissing()
protected void	errBadCharAfterLt(char c)
protected void	errBadCharBeforeAttributeNameOrNull(char c)
protected void	errBogusComment()
protected void	errBogusDoctype()
protected void	errCharRefLacksSemicolon()
protected void	errConsecutiveHyphens()
protected void	errDuplicateAttribute()
protected void	errEofAfterLt()
protected void	errEofInAttributeName()
protected void	errEofInAttributeValue()
protected void	errEofInComment()
protected void	errEofInDoctype()
protected void	errEofInEndTag()
protected void	errEofInPublicId()
protected void	errEofInSystemId()
protected void	errEofInTagName()
protected void	errEofWithoutGt()
protected void	errEqualsSignBeforeAttributeName()
protected void	errExpectedPublicId()
protected void	errExpectedSystemId()
protected void	errGarbageAfterLtSlash()
protected void	errGtInPublicId()
protected void	errGtInSystemId()
protected void	errHtml4LtSlashInRcdata(char folded)
protected void	errHtml4NonNameInUnquotedAttribute(char c)
protected void	errHtml4XmlVoidSyntax()
protected void	errHyphenHyphenBang()
protected void	errLtGt()
protected void	errLtOrEqualsOrGraveInUnquotedAttributeOrNull(char c)
protected void	errLtSlashGt()
protected void	errMissingSpaceBeforeDoctypeName()
protected void	errNamelessDoctype()
protected void	errNcrControlChar()
protected char	errNcrControlChar(char ch)
protected void	errNcrCr()
protected void	errNcrInC1Range()
protected char	errNcrNonCharacter(char ch)
protected void	errNcrOutOfRange()
protected void	errNcrSurrogate()
protected void	errNcrUnassigned()
protected void	errNcrZero()
protected void	errNoDigitsInNCR()
protected void	errNoNamedCharacterMatch()
protected void	errNoSpaceBetweenAttributes()
protected void	errNoSpaceBetweenDoctypePublicKeywordAndQuote()
protected void	errNoSpaceBetweenDoctypeSystemKeywordAndQuote()
protected void	errNoSpaceBetweenPublicAndSystemIds()
protected void	errNotSemicolonTerminated()
protected void	errPrematureEndOfComment()
protected void	errProcessingInstruction()
protected void	errQuoteBeforeAttributeName(char c)
protected void	errQuoteOrLtInAttributeNameOrNull(char c)
protected void	errSlashNotFollowedByGt()
protected void	errUnescapedAmpersandInterpretedAsCharacterReference()
protected void	errUnquotedAttributeValOrNull(char c)
protected void	errWarnLtSlashInRcdata()
protected void	flushChars(char[] buf, int pos) Flushes coalesced character tokens.
int	getCol() Returns the col.
int	getColumnNumber()
int	getLine() Returns the line.
int	getLineNumber()
boolean	isAlreadyComplainedAboutNonAscii() Returns the alreadyComplainedAboutNonAscii.
boolean	isNextCharOnNewLine() Returns the nextCharOnNewLine.
protected void	maybeErrAttributesOnEndTag(HtmlAttributes attrs)
protected void	maybeErrSlashInEndTag(boolean selfClosing)
protected void	maybeWarnPrivateUse(char ch)
protected void	maybeWarnPrivateUseAstral()
void	note(String profile, String message) Reports on an event based on profile selected.
protected void	noteAttributeWithoutValue()
protected void	noteUnquotedAttributeValue()
void	setContentNonXmlCharPolicy(XmlViolationPolicy contentNonXmlCharPolicy) Sets the contentNonXmlCharPolicy.
void	setErrorProfile(HashMap<String,String> errorProfileMap) Sets the errorProfile.
void	setTransitionBaseOffset(int offset) Sets an offset to be added to the position reported to TransitionHandler.
void	setTransitionHandler(TransitionHandler transitionHandler) Sets the transitionHandler.
protected void	silentCarriageReturn()
protected void	silentLineFeed()
protected void	startErrorReporting()
protected int	transition(int from, int to, boolean reconsume, int pos)

Methods inherited from class nu.validator.htmlparser.impl.Tokenizer

becomeConfident, end, eof, err, errTreeBuilder, fatal, getErrorHandler, getPublicId, getSystemId, initializeWithoutStarting, initLocation, internalEncodingDeclaration, isInDataState, isMappingLangToXmlLang, isPrevCR, loadState, notifyAboutMetaBoundary, requestSuspension, resetToDataState, setCommentPolicy, setContentSpacePolicy, setEncodingDeclarationHandler, setErrorHandler, setHtml4ModeCompatibleWithXhtml1Schemata, setInterner, setLineNumber, setMappingLangToXmlLang, setNamePolicy, setStateAndEndTagExpectation, setStateAndEndTagExpectation, setXmlnsPolicy, start, strBufToString, tokenizeBuffer, warn

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

ErrorReportingTokenizer

public ErrorReportingTokenizer(TokenHandler tokenHandler,
                       boolean newAttributesEachTime)

Parameters:

tokenHandler -

newAttributesEachTime -

ErrorReportingTokenizer

public ErrorReportingTokenizer(TokenHandler tokenHandler)

Parameters:

tokenHandler -

Method Detail

getLineNumber

public int getLineNumber()

Specified by:

getLineNumber in interface Locator

Overrides:

getLineNumber in class Tokenizer

See Also:

Locator.getLineNumber()

getColumnNumber

public int getColumnNumber()

Specified by:

getColumnNumber in interface Locator

Overrides:

getColumnNumber in class Tokenizer

See Also:

Locator.getColumnNumber()

setContentNonXmlCharPolicy

public void setContentNonXmlCharPolicy(XmlViolationPolicy contentNonXmlCharPolicy)

Sets the contentNonXmlCharPolicy.

Overrides:

setContentNonXmlCharPolicy in class Tokenizer

Parameters:

contentNonXmlCharPolicy - the contentNonXmlCharPolicy to set

setErrorProfile

public void setErrorProfile(HashMap<String,String> errorProfileMap)

Sets the errorProfile.

Parameters:

errorProfile -

note

public void note(String profile,
        String message)
          throws SAXException

Reports on an event based on profile selected.

Parameters:

profile - the profile this message belongs to

message - the message itself

Throws:

SAXException

startErrorReporting

protected void startErrorReporting()
                            throws SAXException

Overrides:

startErrorReporting in class Tokenizer

Throws:

SAXException

silentCarriageReturn

protected void silentCarriageReturn()

Overrides:

silentCarriageReturn in class Tokenizer

silentLineFeed

protected void silentLineFeed()

Overrides:

silentLineFeed in class Tokenizer

getLine

public int getLine()

Returns the line.

Overrides:

getLine in class Tokenizer

Returns:

the line

getCol

public int getCol()

Returns the col.

Overrides:

getCol in class Tokenizer

Returns:

the col

isNextCharOnNewLine

public boolean isNextCharOnNewLine()

Returns the nextCharOnNewLine.

Overrides:

isNextCharOnNewLine in class Tokenizer

Returns:

the nextCharOnNewLine

isAlreadyComplainedAboutNonAscii

public boolean isAlreadyComplainedAboutNonAscii()

Returns the alreadyComplainedAboutNonAscii.

Overrides:

isAlreadyComplainedAboutNonAscii in class Tokenizer

Returns:

the alreadyComplainedAboutNonAscii

flushChars

protected void flushChars(char[] buf,
              int pos)
                   throws SAXException

Flushes coalesced character tokens.

Overrides:

flushChars in class Tokenizer

Parameters:

buf - TODO

pos - TODO

Throws:

SAXException

checkChar

protected char checkChar(char[] buf,
             int pos)
                  throws SAXException

Overrides:

checkChar in class Tokenizer

Throws:

SAXException

transition

protected int transition(int from,
             int to,
             boolean reconsume,
             int pos)
                  throws SAXException

Overrides:

transition in class Tokenizer

Throws:

SAXException

See Also:

Tokenizer.transition(int, int, boolean, int)

errGarbageAfterLtSlash

protected void errGarbageAfterLtSlash()
                               throws SAXException

Overrides:

errGarbageAfterLtSlash in class Tokenizer

Throws:

SAXException

errLtSlashGt

protected void errLtSlashGt()
                     throws SAXException

Overrides:

errLtSlashGt in class Tokenizer

Throws:

SAXException

errWarnLtSlashInRcdata

protected void errWarnLtSlashInRcdata()
                               throws SAXException

Overrides:

errWarnLtSlashInRcdata in class Tokenizer

Throws:

SAXException

errHtml4LtSlashInRcdata

protected void errHtml4LtSlashInRcdata(char folded)
                                throws SAXException

Overrides:

errHtml4LtSlashInRcdata in class Tokenizer

Throws:

SAXException

errCharRefLacksSemicolon

protected void errCharRefLacksSemicolon()
                                 throws SAXException

Overrides:

errCharRefLacksSemicolon in class Tokenizer

Throws:

SAXException

errNoDigitsInNCR

protected void errNoDigitsInNCR()
                         throws SAXException

Overrides:

errNoDigitsInNCR in class Tokenizer

Throws:

SAXException

errGtInSystemId

protected void errGtInSystemId()
                        throws SAXException

Overrides:

errGtInSystemId in class Tokenizer

Throws:

SAXException

errGtInPublicId

protected void errGtInPublicId()
                        throws SAXException

Overrides:

errGtInPublicId in class Tokenizer

Throws:

SAXException

errNamelessDoctype

protected void errNamelessDoctype()
                           throws SAXException

Overrides:

errNamelessDoctype in class Tokenizer

Throws:

SAXException

errConsecutiveHyphens

protected void errConsecutiveHyphens()
                              throws SAXException

Overrides:

errConsecutiveHyphens in class Tokenizer

Throws:

SAXException

errPrematureEndOfComment

protected void errPrematureEndOfComment()
                                 throws SAXException

Overrides:

errPrematureEndOfComment in class Tokenizer

Throws:

SAXException

errBogusComment

protected void errBogusComment()
                        throws SAXException

Overrides:

errBogusComment in class Tokenizer

Throws:

SAXException

errUnquotedAttributeValOrNull

protected void errUnquotedAttributeValOrNull(char c)
                                      throws SAXException

Overrides:

errUnquotedAttributeValOrNull in class Tokenizer

Throws:

SAXException

errSlashNotFollowedByGt

protected void errSlashNotFollowedByGt()
                                throws SAXException

Overrides:

errSlashNotFollowedByGt in class Tokenizer

Throws:

SAXException

errHtml4XmlVoidSyntax

protected void errHtml4XmlVoidSyntax()
                              throws SAXException

Overrides:

errHtml4XmlVoidSyntax in class Tokenizer

Throws:

SAXException

errNoSpaceBetweenAttributes

protected void errNoSpaceBetweenAttributes()
                                    throws SAXException

Overrides:

errNoSpaceBetweenAttributes in class Tokenizer

Throws:

SAXException

errHtml4NonNameInUnquotedAttribute

protected void errHtml4NonNameInUnquotedAttribute(char c)
                                           throws SAXException

Overrides:

errHtml4NonNameInUnquotedAttribute in class Tokenizer

Throws:

SAXException

errLtOrEqualsOrGraveInUnquotedAttributeOrNull

protected void errLtOrEqualsOrGraveInUnquotedAttributeOrNull(char c)
                                                      throws SAXException

Overrides:

errLtOrEqualsOrGraveInUnquotedAttributeOrNull in class Tokenizer

Throws:

SAXException

errAttributeValueMissing

protected void errAttributeValueMissing()
                                 throws SAXException

Overrides:

errAttributeValueMissing in class Tokenizer

Throws:

SAXException

errBadCharBeforeAttributeNameOrNull

protected void errBadCharBeforeAttributeNameOrNull(char c)
                                            throws SAXException

Overrides:

errBadCharBeforeAttributeNameOrNull in class Tokenizer

Throws:

SAXException

errEqualsSignBeforeAttributeName

protected void errEqualsSignBeforeAttributeName()
                                         throws SAXException

Overrides:

errEqualsSignBeforeAttributeName in class Tokenizer

Throws:

SAXException

errBadCharAfterLt

protected void errBadCharAfterLt(char c)
                          throws SAXException

Overrides:

errBadCharAfterLt in class Tokenizer

Throws:

SAXException

errLtGt

protected void errLtGt()
                throws SAXException

Overrides:

errLtGt in class Tokenizer

Throws:

SAXException

errProcessingInstruction

protected void errProcessingInstruction()
                                 throws SAXException

Overrides:

errProcessingInstruction in class Tokenizer

Throws:

SAXException

errUnescapedAmpersandInterpretedAsCharacterReference

protected void errUnescapedAmpersandInterpretedAsCharacterReference()
                                                             throws SAXException

Overrides:

errUnescapedAmpersandInterpretedAsCharacterReference in class Tokenizer

Throws:

SAXException

errNotSemicolonTerminated

protected void errNotSemicolonTerminated()
                                  throws SAXException

Overrides:

errNotSemicolonTerminated in class Tokenizer

Throws:

SAXException

errNoNamedCharacterMatch

protected void errNoNamedCharacterMatch()
                                 throws SAXException

Overrides:

errNoNamedCharacterMatch in class Tokenizer

Throws:

SAXException

errQuoteBeforeAttributeName

protected void errQuoteBeforeAttributeName(char c)
                                    throws SAXException

Overrides:

errQuoteBeforeAttributeName in class Tokenizer

Throws:

SAXException

errQuoteOrLtInAttributeNameOrNull

protected void errQuoteOrLtInAttributeNameOrNull(char c)
                                          throws SAXException

Overrides:

errQuoteOrLtInAttributeNameOrNull in class Tokenizer

Throws:

SAXException

errExpectedPublicId

protected void errExpectedPublicId()
                            throws SAXException

Overrides:

errExpectedPublicId in class Tokenizer

Throws:

SAXException

errBogusDoctype

protected void errBogusDoctype()
                        throws SAXException

Overrides:

errBogusDoctype in class Tokenizer

Throws:

SAXException

maybeWarnPrivateUseAstral

protected void maybeWarnPrivateUseAstral()
                                  throws SAXException

Overrides:

maybeWarnPrivateUseAstral in class Tokenizer

Throws:

SAXException

maybeWarnPrivateUse

protected void maybeWarnPrivateUse(char ch)
                            throws SAXException

Overrides:

maybeWarnPrivateUse in class Tokenizer

Throws:

SAXException

maybeErrAttributesOnEndTag

protected void maybeErrAttributesOnEndTag(HtmlAttributes attrs)
                                   throws SAXException

Overrides:

maybeErrAttributesOnEndTag in class Tokenizer

Throws:

SAXException

maybeErrSlashInEndTag

protected void maybeErrSlashInEndTag(boolean selfClosing)
                              throws SAXException

Overrides:

maybeErrSlashInEndTag in class Tokenizer

Throws:

SAXException

errNcrNonCharacter

protected char errNcrNonCharacter(char ch)
                           throws SAXException

Overrides:

errNcrNonCharacter in class Tokenizer

Throws:

SAXException

errAstralNonCharacter

protected void errAstralNonCharacter(int ch)
                              throws SAXException

Overrides:

errAstralNonCharacter in class Tokenizer

Throws:

SAXException

See Also:

Tokenizer.errAstralNonCharacter(int)

errNcrSurrogate

protected void errNcrSurrogate()
                        throws SAXException

Overrides:

errNcrSurrogate in class Tokenizer

Throws:

SAXException

errNcrControlChar

protected char errNcrControlChar(char ch)
                          throws SAXException

Overrides:

errNcrControlChar in class Tokenizer

Throws:

SAXException

errNcrCr

protected void errNcrCr()
                 throws SAXException

Overrides:

errNcrCr in class Tokenizer

Throws:

SAXException

errNcrInC1Range

protected void errNcrInC1Range()
                        throws SAXException

Overrides:

errNcrInC1Range in class Tokenizer

Throws:

SAXException

errEofInPublicId

protected void errEofInPublicId()
                         throws SAXException

Overrides:

errEofInPublicId in class Tokenizer

Throws:

SAXException

errEofInComment

protected void errEofInComment()
                        throws SAXException

Overrides:

errEofInComment in class Tokenizer

Throws:

SAXException

errEofInDoctype

protected void errEofInDoctype()
                        throws SAXException

Overrides:

errEofInDoctype in class Tokenizer

Throws:

SAXException

errEofInAttributeValue

protected void errEofInAttributeValue()
                               throws SAXException

Overrides:

errEofInAttributeValue in class Tokenizer

Throws:

SAXException

errEofInAttributeName

protected void errEofInAttributeName()
                              throws SAXException

Overrides:

errEofInAttributeName in class Tokenizer

Throws:

SAXException

errEofWithoutGt

protected void errEofWithoutGt()
                        throws SAXException

Overrides:

errEofWithoutGt in class Tokenizer

Throws:

SAXException

errEofInTagName

protected void errEofInTagName()
                        throws SAXException

Overrides:

errEofInTagName in class Tokenizer

Throws:

SAXException

errEofInEndTag

protected void errEofInEndTag()
                       throws SAXException

Overrides:

errEofInEndTag in class Tokenizer

Throws:

SAXException

errEofAfterLt

protected void errEofAfterLt()
                      throws SAXException

Overrides:

errEofAfterLt in class Tokenizer

Throws:

SAXException

errNcrOutOfRange

protected void errNcrOutOfRange()
                         throws SAXException

Overrides:

errNcrOutOfRange in class Tokenizer

Throws:

SAXException

errNcrUnassigned

protected void errNcrUnassigned()
                         throws SAXException

Overrides:

errNcrUnassigned in class Tokenizer

Throws:

SAXException

errDuplicateAttribute

protected void errDuplicateAttribute()
                              throws SAXException

Overrides:

errDuplicateAttribute in class Tokenizer

Throws:

SAXException

errEofInSystemId

protected void errEofInSystemId()
                         throws SAXException

Overrides:

errEofInSystemId in class Tokenizer

Throws:

SAXException

errExpectedSystemId

protected void errExpectedSystemId()
                            throws SAXException

Overrides:

errExpectedSystemId in class Tokenizer

Throws:

SAXException

errMissingSpaceBeforeDoctypeName

protected void errMissingSpaceBeforeDoctypeName()
                                         throws SAXException

Overrides:

errMissingSpaceBeforeDoctypeName in class Tokenizer

Throws:

SAXException

errHyphenHyphenBang

protected void errHyphenHyphenBang()
                            throws SAXException

Overrides:

errHyphenHyphenBang in class Tokenizer

Throws:

SAXException

errNcrControlChar

protected void errNcrControlChar()
                          throws SAXException

Overrides:

errNcrControlChar in class Tokenizer

Throws:

SAXException

errNcrZero

protected void errNcrZero()
                   throws SAXException

Overrides:

errNcrZero in class Tokenizer

Throws:

SAXException

errNoSpaceBetweenDoctypeSystemKeywordAndQuote

protected void errNoSpaceBetweenDoctypeSystemKeywordAndQuote()
                                                      throws SAXException

Overrides:

errNoSpaceBetweenDoctypeSystemKeywordAndQuote in class Tokenizer

Throws:

SAXException

errNoSpaceBetweenPublicAndSystemIds

protected void errNoSpaceBetweenPublicAndSystemIds()
                                            throws SAXException

Overrides:

errNoSpaceBetweenPublicAndSystemIds in class Tokenizer

Throws:

SAXException

errNoSpaceBetweenDoctypePublicKeywordAndQuote

protected void errNoSpaceBetweenDoctypePublicKeywordAndQuote()
                                                      throws SAXException

Overrides:

errNoSpaceBetweenDoctypePublicKeywordAndQuote in class Tokenizer

Throws:

SAXException

noteAttributeWithoutValue

protected void noteAttributeWithoutValue()
                                  throws SAXException

Overrides:

noteAttributeWithoutValue in class Tokenizer

Throws:

SAXException

noteUnquotedAttributeValue

protected void noteUnquotedAttributeValue()
                                   throws SAXException

Overrides:

noteUnquotedAttributeValue in class Tokenizer

Throws:

SAXException

setTransitionHandler

public void setTransitionHandler(TransitionHandler transitionHandler)

Sets the transitionHandler.

Parameters:

transitionHandler - the transitionHandler to set

setTransitionBaseOffset

public void setTransitionBaseOffset(int offset)

Sets an offset to be added to the position reported to TransitionHandler.

Overrides:

setTransitionBaseOffset in class Tokenizer

Parameters:

offset - the offset