IptcAnpaParser.java
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.tika.parser.iptc;
import static java.nio.charset.StandardCharsets.UTF_8;
import java.io.IOException;
import java.io.InputStream;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Locale;
import java.util.Set;
import java.util.TimeZone;
import org.apache.commons.io.IOUtils;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.apache.tika.config.TikaComponent;
import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.sax.XHTMLContentHandler;
/**
* Parser for IPTC ANPA New Wire Feeds
*/
@TikaComponent
public class IptcAnpaParser implements Parser {
/**
* Serial version UID
*/
private static final long serialVersionUID = -6062820170212879115L;
private static final MediaType TYPE = MediaType.text("vnd.iptc.anpa");
private static final Set<MediaType> SUPPORTED_TYPES = Collections.singleton(TYPE);
private final static char SOH = 0x01; // start of header (ctrl-a)
private final static char STX = 0x02; // start of text (ctrl-b)
private final static char ETX = 0x03; // end of text (ctrl-c)
private final static char EOT = 0x04; // the tab character (ctrl-d)
private final static char SYN = 0x16; // synchronous idle (ctrl-v)
private final static char BS = 0x08; // the backspace character (used for diacriticals)
private final static char TB = 0x09; // the tab character
private final static char LF = 0x0A; // line feed
private final static char FF = 0x0C; // form feed
private final static char CR = 0x0D; // carriage return
private final static char XQ = 0x11; // device control (ctrl-q)
private final static char XS = 0x13; // device control (ctrl-s)
private final static char FS = 0x1F; // a field delimiter
private final static char HY = 0x2D; // hyphen
private final static char SP = 0x20; // the blank space
private final static char LT = 0x3C; // less than
private final static char EQ = 0x3D; // less than
private final static char CT = 0x5E; // carat
private final static char SL = 0x91; // single-quote left
private final static char SR = 0x92; // single-quote right
private final static char DL = 0x93; // double-quote left
private final static char DR = 0x94; // double-quote right
private int FMT_ANPA_1312 = 0x00; // "NAA 89-3 (ANPA 1312)"
private int FMT_ANPA_UPI = 0x01; // "United Press International ANPA 1312 variant"
private int FMT_ANPA_UPI_DL = 0x02; // "United Press International Down-Load Message"
private int FMT_IPTC_7901 = 0x03; // "IPTC7901 Recommended Message Format"
private int FMT_IPTC_PHOTO = 0x04; // "IPTC-NAA Digital Newsphoto Parameter Record"
private int FMT_IPTC_CHAR = 0x05;
// "IPTC Unstructured Character Oriented File Format (UCOFF)"
private int FMT_NITF = 0x06; // "News Industry Text Format (NITF)"
private int FMT_NITF_TT = 0x07; // "Tidningarnas Telegrambyra NITF version (TTNITF DTD)"
private int FMT_NITF_RB = 0x08; // "Ritzaus Bureau NITF version (RBNITF DTD)"
private int FMT_IPTC_AP = 0x09; // "Associated Press news wire format"
private int FMT_IPTC_BLM = 0x0A; // "Bloomberg News news wire format"
private int FMT_IPTC_NYT = 0x0B; // "New York Times news wire format"
private int FMT_IPTC_RTR = 0x0C; // "Reuters news wire format"
private int FORMAT = FMT_ANPA_1312; // assume the default format to be ANPA-1312
public Set<MediaType> getSupportedTypes(ParseContext context) {
return SUPPORTED_TYPES;
}
public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata,
ParseContext context) throws IOException, SAXException, TikaException {
HashMap<String, String> properties = this.loadProperties(tis);
this.setMetadata(metadata, properties);
XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata, context);
xhtml.startDocument();
// TODO: put body content here
xhtml.startElement("p");
String body = clean(properties.get("body"));
if (body != null) {
xhtml.characters(body);
}
xhtml.endElement("p");
xhtml.endDocument();
}
/**
* @deprecated This method will be removed in Apache Tika 1.0.
*/
@Deprecated
public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata)
throws IOException, SAXException, TikaException {
parse(tis, handler, metadata, new ParseContext());
}
/**
* scan the news messsage and store the metadata and data into a map
*/
private HashMap<String, String> loadProperties(InputStream is) {
HashMap<String, String> properties = new HashMap<>();
FORMAT = this.scanFormat(is);
byte[] residual = this.getSection(is, "residual");
byte[] header = this.getSection(is, "header");
parseHeader(header, properties);
byte[] body = this.getSection(is, "body");
parseBody(body, properties);
byte[] footer = this.getSection(is, "footer");
parseFooter(footer, properties);
return (properties);
}
private int scanFormat(InputStream is) {
int format = this.FORMAT;
int maxsize = 524288; // 512K
byte[] buf = new byte[maxsize];
try {
if (is.markSupported()) {
is.mark(maxsize);
}
int msgsize = is.read(buf); // read in at least the full data
String message = (new String(buf, UTF_8)).toLowerCase(Locale.ROOT);
// these are not if-then-else, because we want to go from most common
// and fall through to least. this is imperfect, as these tags could
// show up in other agency stories, but i can't find a spec or any
// explicit codes to identify the wire source in the message itself
if (message.contains("ap-wf")) {
format = this.FMT_IPTC_AP;
}
if (message.contains("reuters")) {
format = this.FMT_IPTC_RTR;
}
if (message.contains("new york times")) {
format = this.FMT_IPTC_NYT;
}
if (message.contains("bloomberg news")) {
format = this.FMT_IPTC_BLM;
}
} catch (IOException eio) {
// we are in an unstable state
}
try {
if (is.markSupported()) {
is.reset();
}
} catch (IOException eio) {
// we are in an unstable state
}
return (format);
}
private void setFormat(int format) {
this.FORMAT = format;
}
private String getFormatName() {
String name = "";
if (FORMAT == this.FMT_IPTC_AP) {
name = "Associated Press";
} else if (FORMAT == this.FMT_IPTC_BLM) {
name = "Bloomberg";
} else if (FORMAT == this.FMT_IPTC_NYT) {
name = "New York Times";
} else if (FORMAT == this.FMT_IPTC_RTR) {
name = "Reuters";
}
return (name);
}
private byte[] getSection(InputStream is, String name) {
byte[] value = new byte[0];
switch (name) {
case "residual": {
// the header shouldn't be more than 1k, but just being generous here
int maxsize = 8192; // 8K
byte bstart =
SYN; // check for SYN [0x16 : ctrl-v] (may have leftover residue from
// preceding message)
byte bfinish =
SOH; // check for SOH [0x01 : ctrl-a] (typically follows a pair of SYN
// [0x16 : ctrl-v])
value = getSection(is, maxsize, bstart, bfinish, true);
break;
}
case "header": {
// the header shouldn't be more than 1k, but just being generous here
int maxsize = 8192; // 8K
byte bstart =
SOH; // check for SOH [0x01 : ctrl-a] (typically follows a pair of SYN
// [0x16 : ctrl-v])
byte bfinish =
STX; // check for STX [0x02 : ctrl-b] (marks end of header, beginning of
// message)
value = getSection(is, maxsize, bstart, bfinish, true);
break;
}
case "body": {
// the message shouldn't be more than 16k (?), leaving plenty of space
int maxsize = 524288; // 512K
byte bstart =
STX; // check for STX [0x02 : ctrl-b] (marks end of header, beginning of
// message)
byte bfinish =
ETX; // check for ETX [0x03 : ctrl-c] (marks end of message, beginning of
// footer)
value = getSection(is, maxsize, bstart, bfinish, true);
break;
}
case "footer": {
// the footer shouldn't be more than 1k , leaving plenty of space
int maxsize = 8192; // 8K
byte bstart =
ETX; // check for ETX [0x03 : ctrl-c] (marks end of message, beginning of
// footer)
byte bfinish = EOT; // check for EOT [0x04 : ctrl-d] (marks end of transmission)
value = getSection(is, maxsize, bstart, bfinish, true);
break;
}
}
return (value);
}
private byte[] getSection(InputStream is, int maxsize, byte bstart, byte bfinish,
boolean ifincomplete) {
byte[] value = new byte[0];
try {
boolean started = false; // check if we have found the start flag
boolean finished = false; // check if we have found the finish flag
int read = 0; // the number of bytes we read
int start = 0; // the position after the start flag
// TODO: this only pulls back 8K of data on a read, regardless of buffer size
// more nefariously, it caps at a total 8K, through all sections
int streammax = is.available();
maxsize = Math.min(maxsize, streammax);
is.mark(maxsize);
byte[] buf = new byte[maxsize];
int totsize = 0;
int remainder = maxsize - totsize;
while (remainder > 0) {
int msgsize = is.read(buf, maxsize - remainder,
maxsize); // read in at least the full data
if (msgsize == -1) {
remainder = msgsize = 0;
}
remainder -= msgsize;
totsize += msgsize;
}
// scan through the provided input stream
for (read = 0; read < totsize; read++) {
byte b = buf[read];
if (!started) {
started = (b == bstart);
start = read + 1;
continue;
}
if (finished = (b == bfinish)) {
/*
is.reset();
long skipped = is.skip((long)read);
if (skipped != read) {
// we are in an unstable state
}
is.mark(1);
*/
break;
}
// load from the stream until we run out of characters, or hit the termination byte
continue;
}
// move the input stream back to where it was initially
is.reset();
if (finished) {
// now, we want to reset the stream to be sitting right on top of the finish marker
IOUtils.skipFully(is, read);
value = new byte[read - start];
System.arraycopy(buf, start, value, 0, read - start);
} else {
if (ifincomplete && started) {
// the caller wants anything that was read, and we finished the stream or buffer
value = new byte[read - start];
System.arraycopy(buf, start, value, 0, read - start);
}
}
} catch (IOException eio) {
// something invalid occurred, return an empty string
}
return (value);
}
private boolean parseHeader(byte[] value, HashMap<String, String> properties) {
boolean added = false;
String env_serviceid = "";
String env_category = "";
String env_urgency = "";
String hdr_edcode = "";
StringBuilder hdr_subject = new StringBuilder();
StringBuilder hdr_date = new StringBuilder();
StringBuilder hdr_time = new StringBuilder();
int read = 0;
while (read < value.length) {
// pull apart the envelope, getting the service id (....\x1f)
while (read < value.length) {
byte val_next = value[read++];
if (val_next != FS) {
env_serviceid +=
(char) (val_next & 0xff); // convert the byte to an unsigned int
} else {
break;
}
}
// pull apart the envelope, getting the category (....\x13\x11)
while (read < value.length) {
byte val_next = value[read++];
if (val_next != XS) { // the end of the envelope is marked (\x13)
env_category +=
(char) (val_next & 0xff); // convert the byte to an unsigned int
} else {
val_next = value[read]; // get the remaining byte (\x11)
if (val_next == XQ) {
read++;
}
break;
}
}
// pull apart the envelope, getting the subject heading
while (read < value.length) {
boolean subject = true;
byte val_next = value[read++];
while ((subject) && (val_next != SP) &&
(val_next != 0x00)) { // ignore the envelope subject
hdr_subject.append((char) (val_next & 0xff)); // convert the byte to an unsigned int
val_next = (read < value.length) ? value[read++] : 0x00;
while (val_next == SP) { // consume all the spaces
subject = false;
val_next = (read < value.length) ? value[read++] : 0x00;
if (val_next != SP) {
--read; // otherwise we eat into the next section
}
}
}
if (!subject) {
break;
}
}
// pull apart the envelope, getting the date and time
while (read < value.length) {
byte val_next = value[read++];
if (hdr_date.isEmpty()) {
while (((val_next >= (byte) 0x30) && (val_next <= (byte) 0x39))
// consume all numerics and hyphens
|| (val_next == HY)) {
hdr_date.append((char) (val_next & 0xff)); // convert the byte to an unsigned int
val_next = (read < value.length) ? value[read++] : 0x00;
}
} else if (val_next == SP) {
while (val_next == SP) { // consume all the spaces
val_next = (read < value.length) ? value[read++] : 0x00;
}
continue;
} else {
while (((val_next >= (byte) 0x30) && (val_next <= (byte) 0x39))
// consume all numerics and hyphens
|| (val_next == HY)) {
hdr_time.append((char) (val_next & 0xff)); // convert the byte to an unsigned int
val_next = (read < value.length) ? value[read++] : 0x00;
}
}
}
break; // don't let this run back through and start thrashing metadata
}
// if we were saving any of these values, we would set the properties map here
added = (env_serviceid.length() + env_category.length() + hdr_subject.length() +
hdr_date.length() + hdr_time.length()) > 0;
return added;
}
private boolean parseBody(byte[] value, HashMap<String, String> properties) {
boolean added = false;
StringBuilder bdy_heading = new StringBuilder();
StringBuilder bdy_title = new StringBuilder();
StringBuilder bdy_source = new StringBuilder();
StringBuilder bdy_author = new StringBuilder();
StringBuilder bdy_body = new StringBuilder();
int read = 0;
boolean done = false;
while (!done && (read < value.length)) {
// pull apart the body, getting the heading (^....\x0d\x0a)
while (read < value.length) {
byte val_next = value[read++];
if (val_next == CT) { // start of a new section , first is the heading
val_next = (read < value.length) ? value[read++] : 0x00;
// AP, NYT, and Bloomberg end with < , Reuters with EOL
while ((val_next != LT) && (val_next != CR) &&
(val_next != LF)) { // less than delimiter (\x3c) and not EOL
bdy_heading.append((char) (val_next & 0xff)); // convert the byte to an unsigned int
val_next = (read < value.length) ? value[read++] : 0x00;
if (read >= value.length) {
break;
} // shouldn't ever hit this, but save a NPE
}
if (val_next == LT) {
// hit the delimiter, carry on
val_next = (read < value.length) ? value[read++] : 0x00;
}
while (bdy_heading.length() > 0 && ((val_next == CR) || (val_next == LF))) {
val_next =
(read < value.length) ? value[read++] : 0x00; // skip the new lines
if ((val_next != CR) && (val_next != LF)) {
--read;
}
}
} else {
// this will only be hit on poorly-formed files
// for reuters, the heading does not start with the ^, so we push one back
// into the stream
if (FORMAT == this.FMT_IPTC_RTR) {
if (val_next != CT) {
// for any non-whitespace, we need to go back an additional step to
// non destroy the data
if ((val_next != SP) && (val_next != TB) && (val_next != CR) &&
(val_next != LF)) {
// if the very first byte is data, we have to shift the whole
// array, and stuff in a carat
if (read == 1) {
byte[] resize = new byte[value.length + 1];
System.arraycopy(value, 0, resize, 1, value.length);
value = resize;
}
}
value[--read] = CT;
continue;
}
}
}
break;
}
// pull apart the body, getting the title (^....\x0d\x0a)
while (read < value.length) {
byte val_next = value[read++];
if (val_next == CT) { // start of a new section , first is the heading
val_next = (read < value.length) ? value[read++] : 0x00;
// AP, NYT, and Bloomberg end with < , Reuters with EOL
while ((val_next != LT) && (val_next != CT) && (val_next != CR) && (val_next !=
LF)) { // less than delimiter (\x3c), or carat (\x5e) and not EOL
bdy_title.append((char) (val_next & 0xff)); // convert the byte to an unsigned int
val_next = (read < value.length) ? value[read++] : 0x00;
if (read >= value.length) {
break;
} // shouldn't ever hit this, but save a NPE
}
if (val_next ==
CT) { // start of a new section , when first didn't finish cleanly
--read;
}
if (val_next == LT) {
// hit the delimiter, carry on
val_next = (read < value.length) ? value[read++] : 0x00;
}
while (bdy_title.length() > 0 && ((val_next == CR) || (val_next == LF))) {
val_next =
(read < value.length) ? value[read++] : 0x00; // skip the new lines
if ((val_next != CR) && (val_next != LF)) {
--read;
}
}
} else {
// this will only be hit on poorly-formed files
// for bloomberg, the title does not start with the ^, so we push one back
// into the stream
if (FORMAT == this.FMT_IPTC_BLM) {
if (val_next == TB) {
value[--read] = CT;
continue;
}
}
// for reuters, the title does not start with the ^, so we push one back into
// the stream
if (FORMAT == this.FMT_IPTC_RTR) {
if (val_next != CT) {
// for any non-whitespace, we need to go back an additional step to
// non destroy the data
if ((val_next != SP) && (val_next != TB) && (val_next != CR) &&
(val_next != LF)) {
--read;
}
value[--read] = CT;
continue;
}
}
}
break;
}
// at this point, we have a variable number of metadata lines, with various orders
// we scan the start of each line for the special character, and run to the end
// character
// pull apart the body, getting the title (^....\x0d\x0a)
boolean metastarted = false;
String longline = "";
String longkey;
while (read < value.length) {
byte val_next = value[read++];
// eat up whitespace before committing to the next section
if ((val_next == SP) || (val_next == TB) || (val_next == CR) || (val_next == LF)) {
continue;
}
if (val_next ==
CT) { // start of a new section , could be authors, sources, etc
val_next = (read < value.length) ? value[read++] : 0x00;
StringBuilder tmp_line = new StringBuilder();
while ((val_next != LT) && (val_next != CT) && (val_next != CR) &&
(val_next != LF) && (val_next != 0)) {
// less than delimiter (\x3c), maybe also badly formed with just new line
tmp_line.append((char) (val_next & 0xff)); // convert the byte to an unsigned int
val_next = (read < value.length) ? value[read++] : 0x00;
if (read >= value.length) {
break;
} // shouldn't ever hit this, but save a NPE
}
if (val_next ==
CT) { // start of a new section , when first didn't finish cleanly
--read;
}
if (val_next == LT) {
// hit the delimiter, carry on
val_next = (read < value.length) ? value[read++] : 0x00;
}
while ((val_next == CR) || (val_next == LF)) {
val_next =
(read < value.length) ? value[read++] : 0x00; // skip the new lines
if ((val_next != CR) && (val_next != LF)) {
--read;
}
}
if (tmp_line.toString().toLowerCase(Locale.ROOT).startsWith("by") ||
longline.equals("bdy_author")) {
longkey = "bdy_author";
// prepend a space to subsequent line, so it gets parsed consistent with
// the lead line
tmp_line.insert(0, (longline.equals(longkey) ? " " : ""));
// we have an author candidate
int term = tmp_line.length();
term = Math.min(term,
(tmp_line.toString().contains("<") ? tmp_line.indexOf("<") : term));
term = Math.min(term,
(tmp_line.toString().contains("=") ? tmp_line.indexOf("=") : term));
term = Math.min(term,
(tmp_line.toString().contains("\n") ? tmp_line.indexOf("\n") : term));
term = (term > 0) ? term : tmp_line.length();
bdy_author.append(tmp_line.substring(tmp_line.indexOf(" "), term));
metastarted = true;
longline =
((tmp_line.toString().contains("=")) && (!longline.equals(longkey)) ? longkey :
"");
} else if (FORMAT == this.FMT_IPTC_BLM) {
String byline = " by ";
if (tmp_line.toString().toLowerCase(Locale.ROOT).contains(byline)) {
longkey = "bdy_author";
int term = tmp_line.length();
term = Math.min(term,
(tmp_line.toString().contains("<") ? tmp_line.indexOf("<") : term));
term = Math.min(term,
(tmp_line.toString().contains("=") ? tmp_line.indexOf("=") : term));
term = Math.min(term,
(tmp_line.toString().contains("\n") ? tmp_line.indexOf("\n") : term));
term = (term > 0) ? term : tmp_line.length();
// for bloomberg, the author line sits below their copyright statement
bdy_author.append(tmp_line.substring(
tmp_line.toString().toLowerCase(Locale.ROOT).indexOf(byline) +
byline.length(), term)).append(" ");
metastarted = true;
longline = ((tmp_line.toString().contains("=")) && (!longline.equals(longkey)) ?
longkey : "");
} else if (tmp_line.toString().toLowerCase(Locale.ROOT).startsWith("c.")) {
// the author line for bloomberg is a multiline starting with c.2011
// Bloomberg News
// then containing the author info on the next line
if (val_next == TB) {
value[--read] = CT;
continue;
}
} else if (tmp_line.toString().toLowerCase(Locale.ROOT).trim().startsWith("(") &&
tmp_line.toString().toLowerCase(Locale.ROOT).trim().endsWith(")")) {
// the author line may have one or more comment lines between the
// copyright
// statement, and the By AUTHORNAME line
if (val_next == TB) {
value[--read] = CT;
continue;
}
}
} else if (tmp_line.toString().toLowerCase(Locale.ROOT).startsWith("eds") ||
longline.equals("bdy_source")) {
longkey = "bdy_source";
// prepend a space to subsequent line, so it gets parsed consistent with
// the lead line
tmp_line.insert(0, (longline.equals(longkey) ? " " : ""));
// we have a source candidate
int term = tmp_line.length();
term = Math.min(term,
(tmp_line.toString().contains("<") ? tmp_line.indexOf("<") : term));
term = Math.min(term,
(tmp_line.toString().contains("=") ? tmp_line.indexOf("=") : term));
// term = Math.min(term, (tmp_line.indexOf("\n") > -1 ? tmp_line.indexOf("\n") :
// term));
term = (term > 0) ? term : tmp_line.length();
bdy_source.append(tmp_line.substring(tmp_line.indexOf(" ") + 1, term)).append(" ");
metastarted = true;
longline = (!longline.equals(longkey) ? longkey : "");
} else {
// this has fallen all the way through. trap it as part of the subject,
// rather than just losing it
if (!metastarted) {
bdy_title.append(" , ").append(tmp_line); // not sure where else to put this but in the
// title
} else {
// what to do with stuff that is metadata, which falls after metadata
// lines started?
bdy_body.append(" ")
.append(tmp_line)
.append(" , "); // not sure where else to put this but in the title
}
}
} else { // we're on to the main body
while ((read < value.length) && (val_next != 0)) {
// read until the train runs out of tracks
bdy_body.append((char) (val_next & 0xff)); // convert the byte to an unsigned int
val_next = (read < value.length) ? value[read++] : 0x00;
if (read >= value.length) {
break;
} // shouldn't ever hit this, but save a NPE
}
}
// we would normally break here, but just let this read out to the end
}
done = true; // don't let this run back through and start thrashing metadata
}
properties.put("body", bdy_body.toString());
properties.put("title", bdy_title.toString());
properties.put("subject", bdy_heading.toString());
properties.put("author", bdy_author.toString());
properties.put("source", bdy_source.toString());
added = (bdy_body.length() + bdy_title.length() + bdy_heading.length() +
bdy_author.length() + bdy_source.length()) > 0;
return added;
}
private boolean parseFooter(byte[] value, HashMap<String, String> properties) {
boolean added = false;
StringBuilder ftr_source = new StringBuilder();
String ftr_datetime = "";
int read = 0;
boolean done = false;
while (!done && (read < value.length)) {
// pull apart the footer, getting the news feed source (^....\x0d\x0a)
byte val_next = value[read++];
byte val_peek = (read < value.length) ? value[read + 1] : 0x00; // skip the new lines
while (((val_next < (byte) 0x30) || (val_next > (byte) 0x39)) &&
(val_next != 0)) { // consume all non-numerics first
ftr_source.append((char) (val_next & 0xff)); // convert the byte to an unsigned int
val_next = (read < value.length) ? value[read] :
0x00; // attempt to read until end of stream
read++;
if (read >= value.length) {
break;
} // shouldn't ever hit this, but save a NPE
}
while ((val_next != LT) && (val_next != CR) && (val_next != LF) &&
(val_next != 0)) { // get as much timedate as possible
// this is an american format, so arrives as mm-dd-yy HHiizzz
ftr_datetime += (char) (val_next & 0xff); // convert the byte to an unsigned int
val_next = (read < value.length) ? value[read++] : 0x00; // skip the new lines
if (read >= value.length) {
break;
} // shouldn't ever hit this, but save a NPE
}
if (val_next == LT) {
// hit the delimiter, carry on
val_next = (read < value.length) ? value[read++] : 0x00;
}
if (ftr_datetime.length() > 0) {
// we want to pass this back in a more friendly format
String format_out = "yyyy-MM-dd'T'HH:mm:ss'Z'";
Date dateunix = new Date();
try {
// standard ap format
String format_in = "MM-dd-yy HHmmzzz";
if (FORMAT == this.FMT_IPTC_RTR) {
// standard reuters format
format_in = "HH:mm MM-dd-yy";
}
SimpleDateFormat dfi = new SimpleDateFormat(format_in, Locale.ROOT);
dfi.setTimeZone(TimeZone.getTimeZone("UTC"));
dateunix = dfi.parse(ftr_datetime);
} catch (ParseException ep) {
// failed, but this will just fall through to setting the date to now
}
SimpleDateFormat dfo = new SimpleDateFormat(format_out, Locale.ROOT);
dfo.setTimeZone(TimeZone.getTimeZone("UTC"));
ftr_datetime = dfo.format(dateunix);
}
while ((val_next == CR) || (val_next == LF)) {
val_next = (read < value.length) ? value[read++] : 0x00; // skip the new lines
if ((val_next != CR) && (val_next != LF)) {
--read;
}
}
done = true; // don't let this run back through and start thrashing metadata
}
properties.put("publisher", ftr_source.toString());
properties.put("created", ftr_datetime);
properties.put("modified", ftr_datetime);
added = (ftr_source.length() + ftr_datetime.length()) > 0;
return added;
}
private void setMetadata(Metadata metadata, HashMap<String, String> properties) {
// every property that gets set must be non-null, or it will cause NPE
// in other consuming applications, like Lucene
metadata.set(Metadata.CONTENT_TYPE, clean("text/anpa-1312"));
metadata.set(TikaCoreProperties.TITLE, clean(properties.get("title")));
metadata.set(TikaCoreProperties.SUBJECT, clean(properties.get("subject")));
metadata.set(TikaCoreProperties.CREATOR, clean(properties.get("author")));
metadata.set(TikaCoreProperties.CREATED, clean(properties.get("created")));
metadata.set(TikaCoreProperties.MODIFIED, clean(properties.get("modified")));
metadata.set(TikaCoreProperties.SOURCE, clean(properties.get("source")));
// metadata.set(TikaCoreProperties.PUBLISHER, clean(properties.get("publisher")));
metadata.set(TikaCoreProperties.PUBLISHER, clean(this.getFormatName()));
/*
metadata.set(TikaCoreProperties.DATE, font.getHeader().getCreated().getTime());
metadata.set(
Property.internalDate(TikaCoreProperties.MODIFIED),
font.getHeader().getModified().getTime());
*/
}
private String clean(String value) {
if (value == null) {
value = "";
}
value = value.replaceAll("``", "`");
value = value.replaceAll("''", "'");
value = value.replaceAll(new String(new char[]{SL}), "'");
value = value.replaceAll(new String(new char[]{SR}), "'");
value = value.replaceAll(new String(new char[]{DL}), "\"");
value = value.replaceAll(new String(new char[]{DR}), "\"");
value = value.trim();
return (value);
}
}