TokeniserState.java
package org.jsoup.parser;
import org.jsoup.internal.Normalizer;
import org.jsoup.internal.StringUtil;
import org.jsoup.nodes.DocumentType;
import static org.jsoup.internal.Normalizer.equalsIgnoreAsciiCase;
import static org.jsoup.nodes.Document.OutputSettings.Syntax.xml;
/**
* States and transition activations for the Tokeniser.
*/
enum TokeniserState {
Data {
// in data state, gather characters until a character reference or tag is found
@Override void read(Tokeniser t, CharacterReader r) {
switch (r.current()) {
case '&':
t.advanceTransition(CharacterReferenceInData);
break;
case '<':
t.advanceTransition(TagOpen);
break;
case nullChar:
t.error(this); // NOT replacement character (oddly?)
t.emit(r.consume());
break;
case eof:
t.emit(new Token.EOF());
break;
default:
String data = r.consumeData();
t.emit(data);
break;
}
}
},
DataWhitespace {
// jsoup special: split batched Data characters where the spec handles them one by one.
// After </script>, InHead puts the space in " -->" in the head and "-->" in the body.
@Override
void read(Tokeniser t, CharacterReader r) {
char c = r.current();
if (StringUtil.isWhitespace(c)) {
t.emit(r.consumeWhitespace());
} else if (c == '&' && t.isWhitespaceReference()) {
// whitespace references eg   will extend this run; others will unread until after the whitespace token
r.advance();
int[] ref = t.consumeCharacterReference(null, false, false);
assert ref != null;
t.emit(ref);
} else {
t.transition(Data);
// splitting before a null could leave a stray space before a later frameset
if (c != nullChar && t.charPending.data.hasData()) t.emitCharacters();
}
}
},
CharacterReferenceInData {
// from & in data
@Override void read(Tokeniser t, CharacterReader r) {
readCharRef(t, Data);
}
},
Rcdata { // Rcdata has text with character references
/// handles data in title, textarea etc
@Override void read(Tokeniser t, CharacterReader r) {
switch (r.current()) {
case '&':
t.advanceTransition(CharacterReferenceInRcdata);
break;
case '<':
t.advanceTransition(RcdataLessthanSign);
break;
case nullChar:
t.error(this);
r.advance();
t.emit(replacementChar);
break;
case eof:
t.emit(new Token.EOF());
break;
default:
String data = r.consumeData();
t.emit(data);
break;
}
}
},
CharacterReferenceInRcdata {
@Override void read(Tokeniser t, CharacterReader r) {
readCharRef(t, Rcdata);
}
},
Rawtext {
@Override void read(Tokeniser t, CharacterReader r) {
readRawData(t, r, this, RawtextLessthanSign);
}
},
ScriptData {
@Override void read(Tokeniser t, CharacterReader r) {
readRawData(t, r, this, ScriptDataLessthanSign);
}
},
PLAINTEXT {
@Override void read(Tokeniser t, CharacterReader r) {
switch (r.current()) {
case nullChar:
t.error(this);
r.advance();
t.emit(replacementChar);
break;
case eof:
t.emit(new Token.EOF());
break;
default:
String data = r.consumeTo(nullChar);
t.emit(data);
break;
}
}
},
TagOpen {
// from < in data
@Override void read(Tokeniser t, CharacterReader r) {
switch (r.current()) {
case '!':
t.advanceTransition(MarkupDeclarationOpen);
break;
case '/':
t.advanceTransition(EndTagOpen);
break;
case '?':
t.advanceTransition(ProcessingInstructionOpen);
break;
default:
if (r.matchesAsciiAlpha()) {
t.createTagPending(true);
t.transition(TagName);
} else {
t.error(this);
t.emit('<'); // char that got us here
t.transition(Data);
}
break;
}
}
},
EndTagOpen {
@Override void read(Tokeniser t, CharacterReader r) {
if (r.isEmpty()) {
t.eofError(this);
t.emit("</");
t.transition(Data);
} else if (r.matchesAsciiAlpha()) {
t.createTagPending(false);
t.transition(TagName);
} else if (r.matches('>')) {
t.error(this);
t.advanceTransition(Data);
} else {
t.error(this);
t.createBogusCommentPending();
t.transition(BogusComment);
}
}
},
TagName {
// from < or </ in data, will have start or end tag pending
@Override void read(Tokeniser t, CharacterReader r) {
// previous TagOpen state did NOT consume, will have a letter char in current
String tagName = r.consumeTagName();
t.tagPending.appendTagName(tagName);
char c = r.consume();
switch (c) {
case '\t':
case '\n':
case '\r':
case '\f':
case ' ':
t.transition(BeforeAttributeName);
break;
case '/':
t.transition(SelfClosingStartTag);
break;
case '>':
t.emitTagPending();
t.transition(Data);
break;
case nullChar: // replacement
t.tagPending.appendTagName(replacementStr);
break;
case eof: // should emit pending tag?
t.eofError(this);
t.transition(Data);
break;
default: // buffer underrun
t.tagPending.appendTagName(c);
}
}
},
RcdataLessthanSign {
// from < in rcdata
@Override void read(Tokeniser t, CharacterReader r) {
if (r.matches('/')) {
t.advanceTransition(RCDATAEndTagOpen);
} else {
t.emit('<');
t.transition(Rcdata);
}
}
},
RCDATAEndTagOpen {
@Override void read(Tokeniser t, CharacterReader r) {
readEndTag(t, r, RCDATAEndTagName, Rcdata);
}
},
RCDATAEndTagName {
@Override void read(Tokeniser t, CharacterReader r) {
handleDataEndTag(t, r, Rcdata);
}
},
RawtextLessthanSign {
@Override void read(Tokeniser t, CharacterReader r) {
if (r.matches('/')) {
t.advanceTransition(RawtextEndTagOpen);
} else {
t.emit('<');
t.transition(Rawtext);
}
}
},
RawtextEndTagOpen {
@Override void read(Tokeniser t, CharacterReader r) {
readEndTag(t, r, RawtextEndTagName, Rawtext);
}
},
RawtextEndTagName {
@Override void read(Tokeniser t, CharacterReader r) {
handleDataEndTag(t, r, Rawtext);
}
},
ScriptDataLessthanSign {
@Override void read(Tokeniser t, CharacterReader r) {
switch (r.current()) {
case '/':
r.advance();
t.transition(ScriptDataEndTagOpen);
break;
case '!':
r.advance();
t.emit("<!");
t.transition(ScriptDataEscapeStart);
break;
default:
t.emit('<');
t.transition(ScriptData);
}
}
},
ScriptDataEndTagOpen {
@Override void read(Tokeniser t, CharacterReader r) {
readEndTag(t, r, ScriptDataEndTagName, ScriptData);
}
},
ScriptDataEndTagName {
@Override void read(Tokeniser t, CharacterReader r) {
handleDataEndTag(t, r, ScriptData);
}
},
ScriptDataEscapeStart {
@Override void read(Tokeniser t, CharacterReader r) {
if (r.matches('-')) {
t.emit('-');
t.advanceTransition(ScriptDataEscapeStartDash);
} else {
t.transition(ScriptData);
}
}
},
ScriptDataEscapeStartDash {
@Override void read(Tokeniser t, CharacterReader r) {
if (r.matches('-')) {
t.emit('-');
t.advanceTransition(ScriptDataEscapedDashDash);
} else {
t.transition(ScriptData);
}
}
},
ScriptDataEscaped {
@Override void read(Tokeniser t, CharacterReader r) {
if (r.isEmpty()) {
t.eofError(this);
t.transition(Data);
return;
}
switch (r.current()) {
case '-':
t.emit('-');
t.advanceTransition(ScriptDataEscapedDash);
break;
case '<':
t.advanceTransition(ScriptDataEscapedLessthanSign);
break;
case nullChar:
t.error(this);
r.advance();
t.emit(replacementChar);
break;
default:
String data = r.consumeToAny('-', '<', nullChar);
t.emit(data);
}
}
},
ScriptDataEscapedDash {
@Override void read(Tokeniser t, CharacterReader r) {
if (r.isEmpty()) {
t.eofError(this);
t.transition(Data);
return;
}
char c = r.consume();
switch (c) {
case '-':
t.emit(c);
t.transition(ScriptDataEscapedDashDash);
break;
case '<':
t.transition(ScriptDataEscapedLessthanSign);
break;
case nullChar:
t.error(this);
t.emit(replacementChar);
t.transition(ScriptDataEscaped);
break;
default:
t.emit(c);
t.transition(ScriptDataEscaped);
}
}
},
ScriptDataEscapedDashDash {
@Override void read(Tokeniser t, CharacterReader r) {
if (r.isEmpty()) {
t.eofError(this);
t.transition(Data);
return;
}
char c = r.consume();
switch (c) {
case '-':
t.emit(c);
break;
case '<':
t.transition(ScriptDataEscapedLessthanSign);
break;
case '>':
t.emit(c);
t.transition(ScriptData);
break;
case nullChar:
t.error(this);
t.emit(replacementChar);
t.transition(ScriptDataEscaped);
break;
default:
t.emit(c);
t.transition(ScriptDataEscaped);
}
}
},
ScriptDataEscapedLessthanSign {
@Override void read(Tokeniser t, CharacterReader r) {
if (r.matchesAsciiAlpha()) {
t.createTempBuffer();
t.dataBuffer.append(Normalizer.asciiLowerCase(r.current()));
t.emit('<');
t.emit(r.current());
t.advanceTransition(ScriptDataDoubleEscapeStart);
} else if (r.matches('/')) {
t.advanceTransition(ScriptDataEscapedEndTagOpen);
} else {
t.emit('<');
t.transition(ScriptDataEscaped);
}
}
},
ScriptDataEscapedEndTagOpen {
@Override void read(Tokeniser t, CharacterReader r) {
readEndTag(t, r, ScriptDataEscapedEndTagName, ScriptDataEscaped);
}
},
ScriptDataEscapedEndTagName {
@Override void read(Tokeniser t, CharacterReader r) {
handleDataEndTag(t, r, ScriptDataEscaped);
}
},
ScriptDataDoubleEscapeStart {
@Override void read(Tokeniser t, CharacterReader r) {
handleDataDoubleEscapeTag(t, r, ScriptDataDoubleEscaped, ScriptDataEscaped);
}
},
ScriptDataDoubleEscaped {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.current();
switch (c) {
case '-':
t.emit(c);
t.advanceTransition(ScriptDataDoubleEscapedDash);
break;
case '<':
t.emit(c);
t.advanceTransition(ScriptDataDoubleEscapedLessthanSign);
break;
case nullChar:
t.error(this);
r.advance();
t.emit(replacementChar);
break;
case eof:
t.eofError(this);
t.transition(Data);
break;
default:
String data = r.consumeToAny('-', '<', nullChar);
t.emit(data);
}
}
},
ScriptDataDoubleEscapedDash {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.consume();
switch (c) {
case '-':
t.emit(c);
t.transition(ScriptDataDoubleEscapedDashDash);
break;
case '<':
t.emit(c);
t.transition(ScriptDataDoubleEscapedLessthanSign);
break;
case nullChar:
t.error(this);
t.emit(replacementChar);
t.transition(ScriptDataDoubleEscaped);
break;
case eof:
t.eofError(this);
t.transition(Data);
break;
default:
t.emit(c);
t.transition(ScriptDataDoubleEscaped);
}
}
},
ScriptDataDoubleEscapedDashDash {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.consume();
switch (c) {
case '-':
t.emit(c);
break;
case '<':
t.emit(c);
t.transition(ScriptDataDoubleEscapedLessthanSign);
break;
case '>':
t.emit(c);
t.transition(ScriptData);
break;
case nullChar:
t.error(this);
t.emit(replacementChar);
t.transition(ScriptDataDoubleEscaped);
break;
case eof:
t.eofError(this);
t.transition(Data);
break;
default:
t.emit(c);
t.transition(ScriptDataDoubleEscaped);
}
}
},
ScriptDataDoubleEscapedLessthanSign {
@Override void read(Tokeniser t, CharacterReader r) {
if (r.matches('/')) {
t.emit('/');
t.createTempBuffer();
t.advanceTransition(ScriptDataDoubleEscapeEnd);
} else {
t.transition(ScriptDataDoubleEscaped);
}
}
},
ScriptDataDoubleEscapeEnd {
@Override void read(Tokeniser t, CharacterReader r) {
handleDataDoubleEscapeTag(t,r, ScriptDataEscaped, ScriptDataDoubleEscaped);
}
},
BeforeAttributeName {
// from tagname <xxx
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.consume();
switch (c) {
case '\t':
case '\n':
case '\r':
case '\f':
case ' ':
break; // ignore whitespace
case '/':
t.transition(SelfClosingStartTag);
break;
case '>':
t.emitTagPending();
t.transition(Data);
break;
case nullChar:
r.unconsume();
t.error(this);
t.tagPending.newAttribute();
t.transition(AttributeName);
break;
case eof:
t.eofError(this);
t.transition(Data);
break;
case '"':
case '\'':
case '=':
t.error(this);
t.tagPending.newAttribute();
t.tagPending.appendAttributeName(c, r.pos()-1, r.pos());
t.transition(AttributeName);
break;
case '?': // Handle trailing ? in <?xml...?>
if (t.tagPending instanceof Token.XmlDecl)
break;
// otherwise fall through to default
default: // A-Z, anything else
t.tagPending.newAttribute();
r.unconsume();
t.transition(AttributeName);
}
}
},
AttributeName {
// from before attribute name
@Override void read(Tokeniser t, CharacterReader r) {
int pos = r.pos();
String name = r.consumeToAnySorted(attributeNameCharsSorted); // spec deviate - consume and emit nulls in one hit vs stepping
t.tagPending.appendAttributeName(name, pos, r.pos());
pos = r.pos();
char c = r.consume();
switch (c) {
case '\t':
case '\n':
case '\r':
case '\f':
case ' ':
t.transition(AfterAttributeName);
break;
case '/':
t.transition(SelfClosingStartTag);
break;
case '=':
t.transition(BeforeAttributeValue);
break;
case '>':
t.emitTagPending();
t.transition(Data);
break;
case eof:
t.eofError(this);
t.transition(Data);
break;
case '"':
case '\'':
case '<':
t.error(this);
t.tagPending.appendAttributeName(c, pos, r.pos());
break;
case '?':
if (t.syntax == xml && t.tagPending instanceof Token.XmlDecl) {
t.transition(AfterAttributeName);
break;
} // otherwise default - take it
default: // buffer underrun
t.tagPending.appendAttributeName(c, pos, r.pos());
}
}
},
AfterAttributeName {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.consume();
switch (c) {
case '\t':
case '\n':
case '\r':
case '\f':
case ' ':
// ignore
break;
case '/':
t.transition(SelfClosingStartTag);
break;
case '=':
t.transition(BeforeAttributeValue);
break;
case '>':
t.emitTagPending();
t.transition(Data);
break;
case nullChar:
t.error(this);
t.tagPending.appendAttributeName(replacementChar, r.pos()-1, r.pos());
t.transition(AttributeName);
break;
case eof:
t.eofError(this);
t.transition(Data);
break;
case '"':
case '\'':
case '<':
t.error(this);
t.tagPending.newAttribute();
t.tagPending.appendAttributeName(c, r.pos()-1, r.pos());
t.transition(AttributeName);
break;
default: // A-Z, anything else
t.tagPending.newAttribute();
r.unconsume();
t.transition(AttributeName);
}
}
},
BeforeAttributeValue {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.current();
switch (c) {
case '\t':
case '\n':
case '\r':
case '\f':
case ' ':
r.advance();
// ignore
break;
case '"':
r.advance();
t.transition(AttributeValue_doubleQuoted);
break;
case '\'':
r.advance();
t.transition(AttributeValue_singleQuoted);
break;
case eof:
t.eofError(this);
// follow html spec at eof and drop; but in xml we keep:
if (t.syntax == xml && !t.attributeFragment)
t.emitTagPending();
else
t.emit(new Token.EOF());
t.transition(Data);
break;
case '>':
t.error(this);
r.advance();
t.emitTagPending();
t.transition(Data);
break;
default:
t.transition(AttributeValue_unquoted);
}
}
},
AttributeValue_doubleQuoted {
@Override void read(Tokeniser t, CharacterReader r) {
int pos = r.pos();
String value = r.consumeAttributeQuoted(false);
if (value.length() > 0)
t.tagPending.appendAttributeValue(value, pos, r.pos());
else
t.tagPending.setEmptyAttributeValue();
pos = r.pos();
char c = r.consume();
switch (c) {
case '"':
t.transition(AfterAttributeValue_quoted);
break;
case '&':
int[] ref = t.consumeCharacterReference('"', true, false);
if (ref != null)
t.tagPending.appendAttributeValue(ref, pos, r.pos());
else
t.tagPending.appendAttributeValue('&', pos, r.pos());
break;
case nullChar:
t.error(this);
t.tagPending.appendAttributeValue(replacementChar, pos, r.pos());
break;
case eof:
t.eofError(this);
t.transition(Data);
break;
default: // hit end of buffer in first read, still in attribute
t.tagPending.appendAttributeValue(c, pos, r.pos());
}
}
},
AttributeValue_singleQuoted {
@Override void read(Tokeniser t, CharacterReader r) {
int pos = r.pos();
String value = r.consumeAttributeQuoted(true);
if (value.length() > 0)
t.tagPending.appendAttributeValue(value, pos, r.pos());
else
t.tagPending.setEmptyAttributeValue();
pos = r.pos();
char c = r.consume();
switch (c) {
case '\'':
t.transition(AfterAttributeValue_quoted);
break;
case '&':
int[] ref = t.consumeCharacterReference('\'', true, false);
if (ref != null)
t.tagPending.appendAttributeValue(ref, pos, r.pos());
else
t.tagPending.appendAttributeValue('&', pos, r.pos());
break;
case nullChar:
t.error(this);
t.tagPending.appendAttributeValue(replacementChar, pos, r.pos());
break;
case eof:
t.eofError(this);
t.transition(Data);
break;
default: // hit end of buffer in first read, still in attribute
t.tagPending.appendAttributeValue(c, pos, r.pos());
}
}
},
AttributeValue_unquoted {
@Override void read(Tokeniser t, CharacterReader r) {
int pos = r.pos();
String value = r.consumeToAnySorted(attributeValueUnquoted);
if (value.length() > 0)
t.tagPending.appendAttributeValue(value, pos, r.pos());
pos = r.pos();
char c = r.consume();
switch (c) {
case '\t':
case '\n':
case '\r':
case '\f':
case ' ':
t.transition(BeforeAttributeName);
break;
case '&':
int[] ref = t.consumeCharacterReference('>', true, false);
if (ref != null)
t.tagPending.appendAttributeValue(ref, pos, r.pos());
else
t.tagPending.appendAttributeValue('&', pos, r.pos());
break;
case '>':
t.emitTagPending();
t.transition(Data);
break;
case nullChar:
t.error(this);
t.tagPending.appendAttributeValue(replacementChar, pos, r.pos());
break;
case eof:
t.eofError(this);
t.transition(Data);
break;
case '"':
case '\'':
case '<':
case '=':
case '`':
t.error(this);
t.tagPending.appendAttributeValue(c, pos, r.pos());
break;
default: // hit end of buffer in first read, still in attribute
t.tagPending.appendAttributeValue(c, pos, r.pos());
}
}
},
// CharacterReferenceInAttributeValue state handled inline
AfterAttributeValue_quoted {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.consume();
switch (c) {
case '\t':
case '\n':
case '\r':
case '\f':
case ' ':
t.transition(BeforeAttributeName);
break;
case '/':
t.transition(SelfClosingStartTag);
break;
case '>':
t.emitTagPending();
t.transition(Data);
break;
case eof:
t.eofError(this);
t.transition(Data);
break;
case '?': // Handle trailing ? in <?xml...?>
if (t.tagPending instanceof Token.XmlDecl)
break;
// otherwise fall through to default
default:
r.unconsume();
t.error(this);
t.transition(BeforeAttributeName);
}
}
},
SelfClosingStartTag {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.consume();
switch (c) {
case '>':
t.tagPending.selfClosing = true;
t.emitTagPending();
t.transition(Data);
break;
case eof:
t.eofError(this);
t.transition(Data);
break;
default:
r.unconsume();
t.error(this);
t.transition(BeforeAttributeName);
}
}
},
BogusComment {
@Override void read(Tokeniser t, CharacterReader r) {
// todo: handle bogus comment starting from eof. when does that trigger?
t.commentPending.append(r.consumeToAny('>', nullChar));
if (r.current() == nullChar) {
t.error(this);
r.advance();
t.commentPending.append(replacementChar);
return;
}
char next = r.current();
if (next == '>' || next == eof) {
r.consume();
t.emitCommentPending();
t.transition(Data);
}
}
},
MarkupDeclarationOpen { // from <!
@Override void read(Tokeniser t, CharacterReader r) {
if (r.matchConsume("--")) {
t.createCommentPending();
t.transition(CommentStart);
} else if (r.matchConsumeIgnoreCase("DOCTYPE")) {
t.transition(Doctype);
} else if (r.matchConsume("[CDATA[")) {
if (t.isCdataAllowed()) {
t.createTempBuffer();
t.transition(CdataSection);
} else {
t.error(this);
t.createBogusCommentPending();
t.commentPending.append("[CDATA[");
t.transition(BogusComment);
}
} else {
if (t.syntax == xml && r.matchesAsciiAlpha()) {
t.createXmlDeclPending(true);
t.transition(TagName); // treat <!ENTITY as XML Declaration, with tag-like handling
} else {
t.error(this);
t.createBogusCommentPending();
t.transition(BogusComment);
}
}
}
},
// HTML's processing-instruction states are also used for XML, with broader targets and a required ?> close.
ProcessingInstructionOpen {
@Override void read(Tokeniser t, CharacterReader r) {
if (t.syntax == xml) {
if (r.isEmpty()) {
t.eofError(this);
t.emit(new Token.EOF());
} else {
t.createPiPending();
t.transition(ProcessingInstructionTarget);
}
} else if (r.matchesAsciiAlpha() || r.matches('_')) {
t.createPiPending();
t.transition(ProcessingInstructionTarget);
} else if (r.isEmpty()) {
t.eofError(this);
t.emit(new Token.EOF());
} else {
t.error(this);
t.createBogusCommentPending();
t.commentPending.append('?');
t.transition(BogusComment);
}
}
},
ProcessingInstructionTarget {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.current();
if (isProcessingInstructionWhitespace(t, c) || c == '?' || c == '>') {
String target = t.piPending.target();
if (t.syntax == xml) {
if (target.isEmpty() || c == '>') {
t.error(this);
t.createBogusCommentPending();
t.commentPending.append('?').append(target);
t.transition(BogusComment);
} else if (equalsIgnoreAsciiCase(target, "xml")) {
// XML declarations retain their attribute-backed XmlDeclaration representation
Token.XmlDecl decl = t.createXmlDeclPending(false);
decl.name(target);
t.transition(BeforeAttributeName);
} else if (c == '?') {
t.piPending.dataStartPos = r.pos();
r.advance();
t.transition(ProcessingInstructionQuestionable);
} else {
t.transition(AfterProcessingInstructionTarget);
}
} else {
if (equalsIgnoreAsciiCase(target, "xml") || equalsIgnoreAsciiCase(target, "xml-stylesheet")) {
t.error(this);
t.createBogusCommentPending();
t.commentPending.append('?').append(target);
t.transition(BogusComment);
} else {
t.transition(AfterProcessingInstructionTarget);
}
}
} else if (r.isEmpty()) {
t.eofError(this);
t.emit(new Token.EOF());
} else if (t.syntax == xml || isHtmlProcessingInstructionTarget(c)) {
t.piPending.target.append(r.consume());
} else {
t.error(this);
t.createBogusCommentPending();
t.commentPending.append('?').append(t.piPending.target());
t.transition(BogusComment);
}
}
},
AfterProcessingInstructionTarget {
@Override void read(Tokeniser t, CharacterReader r) {
if (isProcessingInstructionWhitespace(t, r.current()))
r.advance();
else {
t.piPending.dataStartPos = r.pos();
t.transition(ProcessingInstructionData);
}
}
},
ProcessingInstructionData {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.current();
if (c == '?') {
r.advance();
t.transition(ProcessingInstructionQuestionable);
} else if (t.syntax != xml && c == '>') {
r.advance();
t.transition(Data);
t.emitPiPending();
} else if (r.isEmpty()) {
t.eofError(this);
t.emit(new Token.EOF());
} else {
t.piPending.append(r.consume());
}
}
},
ProcessingInstructionQuestionable {
@Override void read(Tokeniser t, CharacterReader r) {
if (r.matches('>')) {
r.advance();
t.transition(Data);
t.emitPiPending();
} else if (r.isEmpty()) {
t.eofError(this);
t.emit(new Token.EOF());
} else {
t.piPending.append('?');
t.transition(ProcessingInstructionData);
}
}
},
CommentStart {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.consume();
switch (c) {
case '-':
t.transition(CommentStartDash);
break;
case nullChar:
t.error(this);
t.commentPending.append(replacementChar);
t.transition(Comment);
break;
case '>':
t.error(this);
t.emitCommentPending();
t.transition(Data);
break;
case eof:
t.eofError(this);
t.emitCommentPending();
t.transition(Data);
break;
default:
r.unconsume();
t.transition(Comment);
}
}
},
CommentStartDash {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.current();
switch (c) {
case '-':
r.advance();
t.transition(CommentEnd);
break;
case '>':
t.error(this);
r.advance();
t.emitCommentPending();
t.transition(Data);
break;
case eof:
t.eofError(this);
t.emitCommentPending();
t.transition(Data);
break;
default:
t.commentPending.append('-');
t.transition(Comment);
}
}
},
Comment {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.current();
switch (c) {
case '-':
t.advanceTransition(CommentEndDash);
break;
case nullChar:
t.error(this);
r.advance();
t.commentPending.append(replacementChar);
break;
case eof:
t.eofError(this);
t.emitCommentPending();
t.transition(Data);
break;
default:
t.commentPending.append(r.consumeToAny('-', nullChar));
}
}
},
CommentEndDash {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.consume();
switch (c) {
case '-':
t.transition(CommentEnd);
break;
case nullChar:
t.error(this);
t.commentPending.append('-').append(replacementChar);
t.transition(Comment);
break;
case eof:
t.eofError(this);
t.emitCommentPending();
t.transition(Data);
break;
default:
t.commentPending.append('-').append(c);
t.transition(Comment);
}
}
},
CommentEnd {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.consume();
switch (c) {
case '>':
t.emitCommentPending();
t.transition(Data);
break;
case nullChar:
t.error(this);
t.commentPending.append("--").append(replacementChar);
t.transition(Comment);
break;
case '!':
t.transition(CommentEndBang);
break;
case '-':
t.commentPending.append('-');
break;
case eof:
t.eofError(this);
t.emitCommentPending();
t.transition(Data);
break;
default:
t.commentPending.append("--").append(c);
t.transition(Comment);
}
}
},
CommentEndBang {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.consume();
switch (c) {
case '-':
t.commentPending.append("--!");
t.transition(CommentEndDash);
break;
case '>':
t.emitCommentPending();
t.transition(Data);
break;
case nullChar:
t.error(this);
t.commentPending.append("--!").append(replacementChar);
t.transition(Comment);
break;
case eof:
t.eofError(this);
t.emitCommentPending();
t.transition(Data);
break;
default:
t.commentPending.append("--!").append(c);
t.transition(Comment);
}
}
},
Doctype {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.current();
switch (c) {
case '\t':
case '\n':
case '\r':
case '\f':
case ' ':
t.advanceTransition(BeforeDoctypeName);
break;
case '>':
t.transition(BeforeDoctypeName); // reconsume to report the missing name
break;
case eof:
t.eofError(this);
t.createDoctypePending();
t.doctypePending.forceQuirks = true;
t.emitDoctypePending();
t.transition(Data);
break;
default:
t.error(this);
t.transition(BeforeDoctypeName);
}
}
},
BeforeDoctypeName {
@Override void read(Tokeniser t, CharacterReader r) {
if (r.matchesAsciiAlpha()) {
t.createDoctypePending();
t.transition(DoctypeName);
return;
}
char c = r.current();
switch (c) {
case '\t':
case '\n':
case '\r':
case '\f':
case ' ':
r.advance();
break; // ignore whitespace
case nullChar:
t.error(this);
t.createDoctypePending();
t.doctypePending.name.append(replacementChar);
t.advanceTransition(DoctypeName);
break;
case '>':
t.error(this);
t.createDoctypePending();
t.doctypePending.forceQuirks = true;
r.advance();
t.emitDoctypePending();
t.transition(Data);
break;
case eof:
t.eofError(this);
t.createDoctypePending();
t.doctypePending.forceQuirks = true;
t.emitDoctypePending();
t.transition(Data);
break;
default:
t.createDoctypePending();
t.doctypePending.name.append(c);
t.advanceTransition(DoctypeName);
}
}
},
DoctypeName {
@Override void read(Tokeniser t, CharacterReader r) {
if (r.matchesAsciiAlpha()) {
String name = r.consumeLetterSequence();
t.doctypePending.name.append(name);
return;
}
char c = r.consume();
switch (c) {
case '>':
t.emitDoctypePending();
t.transition(Data);
break;
case '\t':
case '\n':
case '\r':
case '\f':
case ' ':
t.transition(AfterDoctypeName);
break;
case nullChar:
t.error(this);
t.doctypePending.name.append(replacementChar);
break;
case eof:
t.eofError(this);
t.doctypePending.forceQuirks = true;
t.emitDoctypePending();
t.transition(Data);
break;
default:
t.doctypePending.name.append(c);
}
}
},
AfterDoctypeName {
@Override void read(Tokeniser t, CharacterReader r) {
if (r.isEmpty()) {
t.eofError(this);
t.doctypePending.forceQuirks = true;
t.emitDoctypePending();
t.transition(Data);
return;
}
if (r.matchesAny('\t', '\n', '\r', '\f', ' '))
r.advance(); // ignore whitespace
else if (r.matches('>')) {
t.emitDoctypePending();
t.advanceTransition(Data);
} else if (t.syntax == xml && r.matches('[')) {
// special case when in XML parse mode so we can support entity round trop
t.doctypePending.sawInternalSubset = true;
t.advanceTransition(DoctypeInternalSubset);
} else if (r.matchConsumeIgnoreCase(DocumentType.PUBLIC_KEY)) {
t.doctypePending.pubSysKey = DocumentType.PUBLIC_KEY;
t.transition(AfterDoctypePublicKeyword);
} else if (r.matchConsumeIgnoreCase(DocumentType.SYSTEM_KEY)) {
t.doctypePending.pubSysKey = DocumentType.SYSTEM_KEY;
t.transition(AfterDoctypeSystemKeyword);
} else {
t.error(this);
t.doctypePending.forceQuirks = true;
t.transition(BogusDoctype);
}
}
},
AfterDoctypePublicKeyword {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.consume();
switch (c) {
case '\t':
case '\n':
case '\r':
case '\f':
case ' ':
t.transition(BeforeDoctypePublicIdentifier);
break;
case '"':
t.error(this);
// set public id to empty string
t.transition(DoctypePublicIdentifier_doubleQuoted);
break;
case '\'':
t.error(this);
// set public id to empty string
t.transition(DoctypePublicIdentifier_singleQuoted);
break;
case '>':
t.error(this);
t.doctypePending.forceQuirks = true;
t.emitDoctypePending();
t.transition(Data);
break;
case eof:
t.eofError(this);
t.doctypePending.forceQuirks = true;
t.emitDoctypePending();
t.transition(Data);
break;
default:
r.unconsume();
t.error(this);
t.doctypePending.forceQuirks = true;
t.transition(BogusDoctype);
}
}
},
BeforeDoctypePublicIdentifier {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.consume();
switch (c) {
case '\t':
case '\n':
case '\r':
case '\f':
case ' ':
break;
case '"':
// set public id to empty string
t.transition(DoctypePublicIdentifier_doubleQuoted);
break;
case '\'':
// set public id to empty string
t.transition(DoctypePublicIdentifier_singleQuoted);
break;
case '>':
t.error(this);
t.doctypePending.forceQuirks = true;
t.emitDoctypePending();
t.transition(Data);
break;
case eof:
t.eofError(this);
t.doctypePending.forceQuirks = true;
t.emitDoctypePending();
t.transition(Data);
break;
default:
r.unconsume();
t.error(this);
t.doctypePending.forceQuirks = true;
t.transition(BogusDoctype);
}
}
},
DoctypePublicIdentifier_doubleQuoted {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.consume();
switch (c) {
case '"':
t.transition(AfterDoctypePublicIdentifier);
break;
case nullChar:
t.error(this);
t.doctypePending.publicIdentifier.append(replacementChar);
break;
case '>':
t.error(this);
t.doctypePending.forceQuirks = true;
t.emitDoctypePending();
t.transition(Data);
break;
case eof:
t.eofError(this);
t.doctypePending.forceQuirks = true;
t.emitDoctypePending();
t.transition(Data);
break;
default:
t.doctypePending.publicIdentifier.append(c);
}
}
},
DoctypePublicIdentifier_singleQuoted {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.consume();
switch (c) {
case '\'':
t.transition(AfterDoctypePublicIdentifier);
break;
case nullChar:
t.error(this);
t.doctypePending.publicIdentifier.append(replacementChar);
break;
case '>':
t.error(this);
t.doctypePending.forceQuirks = true;
t.emitDoctypePending();
t.transition(Data);
break;
case eof:
t.eofError(this);
t.doctypePending.forceQuirks = true;
t.emitDoctypePending();
t.transition(Data);
break;
default:
t.doctypePending.publicIdentifier.append(c);
}
}
},
AfterDoctypePublicIdentifier {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.consume();
switch (c) {
case '\t':
case '\n':
case '\r':
case '\f':
case ' ':
t.transition(BetweenDoctypePublicAndSystemIdentifiers);
break;
case '>':
t.emitDoctypePending();
t.transition(Data);
break;
case '"':
t.error(this);
// system id empty
t.transition(DoctypeSystemIdentifier_doubleQuoted);
break;
case '\'':
t.error(this);
// system id empty
t.transition(DoctypeSystemIdentifier_singleQuoted);
break;
case eof:
t.eofError(this);
t.doctypePending.forceQuirks = true;
t.emitDoctypePending();
t.transition(Data);
break;
default:
r.unconsume();
t.error(this);
t.doctypePending.forceQuirks = true;
t.transition(BogusDoctype);
}
}
},
BetweenDoctypePublicAndSystemIdentifiers {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.consume();
switch (c) {
case '\t':
case '\n':
case '\r':
case '\f':
case ' ':
break;
case '>':
t.emitDoctypePending();
t.transition(Data);
break;
case '[':
if (t.syntax == xml) {
t.doctypePending.sawInternalSubset = true;
t.transition(DoctypeInternalSubset);
break;
}
r.unconsume();
t.error(this);
t.doctypePending.forceQuirks = true;
t.transition(BogusDoctype);
break;
case '"':
// system id empty
t.transition(DoctypeSystemIdentifier_doubleQuoted);
break;
case '\'':
// system id empty
t.transition(DoctypeSystemIdentifier_singleQuoted);
break;
case eof:
t.eofError(this);
t.doctypePending.forceQuirks = true;
t.emitDoctypePending();
t.transition(Data);
break;
default:
r.unconsume();
t.error(this);
t.doctypePending.forceQuirks = true;
t.transition(BogusDoctype);
}
}
},
AfterDoctypeSystemKeyword {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.consume();
switch (c) {
case '\t':
case '\n':
case '\r':
case '\f':
case ' ':
t.transition(BeforeDoctypeSystemIdentifier);
break;
case '>':
t.error(this);
t.doctypePending.forceQuirks = true;
t.emitDoctypePending();
t.transition(Data);
break;
case '"':
t.error(this);
// system id empty
t.transition(DoctypeSystemIdentifier_doubleQuoted);
break;
case '\'':
t.error(this);
// system id empty
t.transition(DoctypeSystemIdentifier_singleQuoted);
break;
case eof:
t.eofError(this);
t.doctypePending.forceQuirks = true;
t.emitDoctypePending();
t.transition(Data);
break;
default:
r.unconsume();
t.error(this);
t.doctypePending.forceQuirks = true;
t.transition(BogusDoctype);
}
}
},
BeforeDoctypeSystemIdentifier {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.consume();
switch (c) {
case '\t':
case '\n':
case '\r':
case '\f':
case ' ':
break;
case '"':
// set system id to empty string
t.transition(DoctypeSystemIdentifier_doubleQuoted);
break;
case '\'':
// set system id to empty string
t.transition(DoctypeSystemIdentifier_singleQuoted);
break;
case '>':
t.error(this);
t.doctypePending.forceQuirks = true;
t.emitDoctypePending();
t.transition(Data);
break;
case eof:
t.eofError(this);
t.doctypePending.forceQuirks = true;
t.emitDoctypePending();
t.transition(Data);
break;
default:
r.unconsume();
t.error(this);
t.doctypePending.forceQuirks = true;
t.transition(BogusDoctype);
}
}
},
DoctypeSystemIdentifier_doubleQuoted {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.consume();
switch (c) {
case '"':
t.transition(AfterDoctypeSystemIdentifier);
break;
case nullChar:
t.error(this);
t.doctypePending.systemIdentifier.append(replacementChar);
break;
case '>':
t.error(this);
t.doctypePending.forceQuirks = true;
t.emitDoctypePending();
t.transition(Data);
break;
case eof:
t.eofError(this);
t.doctypePending.forceQuirks = true;
t.emitDoctypePending();
t.transition(Data);
break;
default:
t.doctypePending.systemIdentifier.append(c);
}
}
},
DoctypeSystemIdentifier_singleQuoted {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.consume();
switch (c) {
case '\'':
t.transition(AfterDoctypeSystemIdentifier);
break;
case nullChar:
t.error(this);
t.doctypePending.systemIdentifier.append(replacementChar);
break;
case '>':
t.error(this);
t.doctypePending.forceQuirks = true;
t.emitDoctypePending();
t.transition(Data);
break;
case eof:
t.eofError(this);
t.doctypePending.forceQuirks = true;
t.emitDoctypePending();
t.transition(Data);
break;
default:
t.doctypePending.systemIdentifier.append(c);
}
}
},
AfterDoctypeSystemIdentifier {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.consume();
switch (c) {
case '\t':
case '\n':
case '\r':
case '\f':
case ' ':
break;
case '>':
t.emitDoctypePending();
t.transition(Data);
break;
case '[':
if (t.syntax == xml) {
t.doctypePending.sawInternalSubset = true;
t.transition(DoctypeInternalSubset);
break;
}
r.unconsume();
t.error(this);
t.transition(BogusDoctype);
// NOT force quirks
break;
case eof:
t.eofError(this);
t.doctypePending.forceQuirks = true;
t.emitDoctypePending();
t.transition(Data);
break;
default:
r.unconsume();
t.error(this);
t.transition(BogusDoctype);
// NOT force quirks
}
}
},
BogusDoctype {
@Override void read(Tokeniser t, CharacterReader r) {
char c = r.current();
switch (c) {
case '>':
r.advance();
t.emitDoctypePending();
t.transition(Data);
break;
case nullChar:
t.error(this);
r.advance();
break;
case eof:
t.emitDoctypePending();
t.transition(Data);
break;
default:
r.advance(); // ignore char
}
}
},
DoctypeInternalSubset {
@Override void read(Tokeniser t, CharacterReader r) {
readDoctypeInternalSubset(t, r, this);
}
},
CdataSection {
@Override void read(Tokeniser t, CharacterReader r) {
String data = r.consumeToAny(']', nullChar);
t.dataBuffer.append(data);
if (r.current() == nullChar) {
if (t.syntax != xml) t.error(this);
t.dataBuffer.append(r.consume()); // tree building decides whether to replace or remove nulls
return;
}
if (r.matchConsume("]]>") || r.isEmpty()) {
t.emit(new Token.CData(t.dataBuffer.value()));
t.transition(Data);
} else if (r.matches(']')) {
t.dataBuffer.append(r.consume());
} // otherwise, buffer underrun, stay in data section
}
};
abstract void read(Tokeniser t, CharacterReader r);
static final char nullChar = '\u0000';
// char searches. must be sorted, used in inSorted. MUST update TokeniserStateTest if more arrays are added.
static final char[] attributeNameCharsSorted = new char[]{'\t', '\n', '\f', '\r', ' ', '"', '\'', '/', '<', '=', '>', '?'};
static final char[] attributeValueUnquoted = new char[]{nullChar, '\t', '\n', '\f', '\r', ' ', '"', '&', '\'', '<', '=', '>', '`'};
private static final char replacementChar = Tokeniser.replacementChar;
private static final String replacementStr = String.valueOf(Tokeniser.replacementChar);
private static final char eof = CharacterReader.EOF;
/** Handles the common RCDATA, RAWTEXT, and script-data end tag name states. */
private static void handleDataEndTag(Tokeniser t, CharacterReader r, TokeniserState elseTransition) {
if (r.matchesAsciiAlpha()) {
String name = r.consumeLetterSequence();
t.tagPending.appendTagName(name);
return;
}
if (!r.isEmpty()) {
char c = r.current();
if (isCustomEndTagNameChar(c) && t.isAppropriateEndTagPrefix(c)) {
r.advance();
t.tagPending.appendTagName(c);
return;
}
}
if (t.isAppropriateEndTagToken() && !r.isEmpty()) {
switch (r.current()) {
case '\t':
case '\n':
case '\r':
case '\f':
case ' ':
r.advance();
t.transition(BeforeAttributeName);
return;
case '/':
r.advance();
t.transition(SelfClosingStartTag);
return;
case '>':
r.advance();
t.emitTagPending();
t.transition(Data);
return;
}
}
t.emit("</");
t.emit(t.tagPending.name());
t.transition(elseTransition); // reconsume the current character in the text state
}
/** Test if the character may extend a configured custom text tag name. */
private static boolean isCustomEndTagNameChar(char c) {
return c != '<' && c != '/' && c != '>' && !StringUtil.isWhitespace(c);
}
private static void readRawData(Tokeniser t, CharacterReader r, TokeniserState current, TokeniserState advance) {
switch (r.current()) {
case '<':
t.advanceTransition(advance);
break;
case nullChar:
t.error(current);
r.advance();
t.emit(replacementChar);
break;
case eof:
t.emit(new Token.EOF());
break;
default:
String data = r.consumeRawData();
t.emit(data);
break;
}
}
private static void readCharRef(Tokeniser t, TokeniserState advance) {
int[] c = t.consumeCharacterReference(null, false, false);
if (c == null)
t.emit('&');
else
t.emit(c);
t.transition(advance);
}
/** Starts a text end tag candidate, or returns to its text state. */
private static void readEndTag(Tokeniser t, CharacterReader r, TokeniserState endTagNameState, TokeniserState textState) {
if (r.matchesAsciiAlpha()) {
t.createTagPending(false);
t.transition(endTagNameState);
} else {
t.emit("</");
t.transition(textState);
}
}
private static void handleDataDoubleEscapeTag(Tokeniser t, CharacterReader r, TokeniserState primary, TokeniserState fallback) {
if (r.matchesAsciiAlpha()) {
String name = r.consumeLetterSequence();
t.dataBuffer.append(Normalizer.asciiLowerCase(name));
t.emit(name);
return;
}
char c = r.consume();
switch (c) {
case '\t':
case '\n':
case '\r':
case '\f':
case ' ':
case '/':
case '>':
if (t.dataBuffer.value().equals("script"))
t.transition(primary);
else
t.transition(fallback);
t.emit(c);
break;
default:
r.unconsume();
t.transition(fallback);
}
}
/** Tests whether this character separates a processing-instruction target and data in the current syntax. */
private static boolean isProcessingInstructionWhitespace(Tokeniser tokeniser, char c) {
return c == '\t' || c == '\n' || c == '\r' || c == ' ' || tokeniser.syntax != xml && c == '\f';
}
/** Returns whether this character may continue an HTML processing instruction target. */
private static boolean isHtmlProcessingInstructionTarget(char c) {
return c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '-' || c == '_';
}
/**
Reads an XML doctype internal subset as opaque text so it can be re-emitted without parsing declarations.
Only used when in XML mode; HTML spec will drop these as Bogus.
*/
private static void readDoctypeInternalSubset(Tokeniser t, CharacterReader r, TokeniserState current) {
final byte None = 0, SingleQuote = 1, DoubleQuote = 2, Comment = 3, ProcessingInstruction = 4;
byte context = None;
TokenData subset = t.doctypePending.internalSubset;
while (true) {
char c = r.consume();
switch (c) {
case '\'':
subset.append(c);
if (context == None) context = SingleQuote;
else if (context == SingleQuote) context = None;
break;
case '"':
subset.append(c);
if (context == None) context = DoubleQuote;
else if (context == DoubleQuote) context = None;
break;
case '<':
subset.append(c);
if (context == None) {
if (r.matchConsume("!--")) {
subset.append("!--");
context = Comment;
} else if (r.matchConsume("?")) {
subset.append('?');
context = ProcessingInstruction;
}
}
break;
case '-':
subset.append(c);
if (context == Comment && r.matchConsume("->")) {
subset.append("->");
context = None;
}
break;
case '?':
subset.append(c);
if (context == ProcessingInstruction && r.matches('>')) {
r.advance();
subset.append('>');
context = None;
}
break;
case ']':
if (context == None) {
String ws = r.consumeMatching(StringUtil::isWhitespace);
if (r.matches('>')) {
r.advance();
t.emitDoctypePending();
t.transition(Data);
return;
}
subset.append(c);
subset.append(ws);
break;
}
subset.append(c);
break;
case nullChar:
t.error(current);
subset.append(replacementChar);
break;
case eof:
t.eofError(current);
t.emitDoctypePending();
t.transition(Data);
return;
default:
subset.append(c);
break;
}
}
}
}