<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article
  PUBLIC "-//NLM//DTD Journal Publishing DTD v3.0 20080202//EN" "http://dtd.nlm.nih.gov/publishing/3.0/journalpublishing3.dtd">
<article xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" article-type="discussion" dtd-version="3.0" xml:lang="EN">
  <front>
    <journal-meta><journal-id journal-id-type="publisher-id">plos</journal-id><journal-id journal-id-type="publisher">pmed</journal-id><journal-id journal-id-type="allenpress-id">plme</journal-id><journal-id journal-id-type="nlm-ta">PLoS Med</journal-id><journal-id journal-id-type="pmc">plosmed</journal-id><!--===== Grouping journal title elements =====--><journal-title-group><journal-title>PLoS Medicine</journal-title></journal-title-group><issn pub-type="ppub">1549-1277</issn><issn pub-type="epub">1549-1676</issn><publisher>
        <publisher-name>Public Library of Science</publisher-name>
        <publisher-loc>San Francisco, USA</publisher-loc>
      </publisher></journal-meta>
    <article-meta><article-id pub-id-type="doi">10.1371/journal.pmed.0050151</article-id><article-id pub-id-type="publisher-id">07-PLME-HIA-1838R2</article-id><article-categories>
        <subj-group subj-group-type="heading">
          <subject>Health in Action</subject>
        </subj-group>
        <subj-group subj-group-type="Discipline">
          <subject>Infectious Diseases</subject>
          <subject>Public Health and Epidemiology</subject>
        </subj-group>
        <subj-group subj-group-type="System Taxonomy">
          <subject>Infectious Diseases</subject>
          <subject>Epidemiology</subject>
          <subject>International health</subject>
          <subject>Public Health</subject>
          <subject>Medical Informatics</subject>
        </subj-group>
      </article-categories><title-group><article-title>Surveillance Sans Frontières: Internet-Based Emerging Infectious Disease Intelligence and the HealthMap Project</article-title><alt-title alt-title-type="running-head">Health in Action</alt-title></title-group><contrib-group>
        <contrib contrib-type="author" xlink:type="simple">
          <name name-style="western">
            <surname>Brownstein</surname>
            <given-names>John S</given-names>
          </name>
          <xref ref-type="corresp" rid="cor1">
            <sup>*</sup>
          </xref>
        </contrib>
        <contrib contrib-type="author" xlink:type="simple">
          <name name-style="western">
            <surname>Freifeld</surname>
            <given-names>Clark C</given-names>
          </name>
        </contrib>
        <contrib contrib-type="author" xlink:type="simple">
          <name name-style="western">
            <surname>Reis</surname>
            <given-names>Ben Y</given-names>
          </name>
        </contrib>
        <contrib contrib-type="author" xlink:type="simple">
          <name name-style="western">
            <surname>Mandl</surname>
            <given-names>Kenneth D</given-names>
          </name>
        </contrib>
      </contrib-group><author-notes>
        <fn fn-type="current-aff" id="n3">
          <p>All authors are with the Children's Hospital Informatics Program at the Harvard–MIT Division of Health Sciences and Technology; the Division of Emergency Medicine, Children's Hospital Boston; and the Department of Pediatrics, Harvard Medical School, Boston, Massachusetts, United States of America.</p>
        </fn>
        <corresp id="cor1">* To whom correspondence should be addressed. E-mail:<email xlink:type="simple">john_brownstein@harvard.edu</email></corresp>
      <fn fn-type="conflict" id="n2">
        <p> The authors have declared that no competing interests exist.</p>
      </fn></author-notes><pub-date pub-type="ppub">
        <month>7</month>
        <year>2008</year>
      </pub-date><pub-date pub-type="epub">
        <day>8</day>
        <month>7</month>
        <year>2008</year>
      </pub-date><volume>5</volume><issue>7</issue><elocation-id>e151</elocation-id><!--===== Grouping copyright info into permissions =====--><permissions><copyright-year>2008</copyright-year><copyright-holder> Brownstein et al</copyright-holder><license><license-p>This is an open-access article distributed under the terms of the Creative Commons Attribution License, which permits unrestricted use, distribution, and reproduction in any medium, provided the original author and source are credited.</license-p></license></permissions><abstract abstract-type="toc">
        <p>John Brownstein and colleagues discuss HealthMap, an automated real-time system that monitors and disseminates online information about emerging infectious diseases.</p>
      </abstract><funding-group><funding-statement> This work was supported by grants R21LM009263-01 and R01LM007970-01 from the National Library of Medicine, the National Institutes of Health, the Canadian Institutes of Health Research, and a research grant from Google.org. The funders had no role in study design, data collection and analysis, decision to publish, or preparation of the manuscript.</funding-statement></funding-group><counts>
        <page-count count="6"/>
      </counts><!--===== Restructure custom-meta-wrap to custom-meta-group =====--><custom-meta-group>
        <custom-meta>
          <meta-name>citation</meta-name>
          <meta-value>Brownstein JS, Freifeld CC, Reis BY, Mandl KD (2008) Surveillance sans frontières: Internet-based emerging infectious disease intelligence and the HealthMap project. PLoS Med 5(7): e151. doi:<ext-link ext-link-type="doi" xlink:href="http://dx.doi.org/10.1371/journal.pmed.0050151" xlink:type="simple">10.1371/journal.pmed.0050151</ext-link></meta-value>
        </custom-meta>
      </custom-meta-group></article-meta>
  </front>
  <body>
    <sec id="s1">
      <title>The Opportunity</title>
      <p>As developed nations continue to strengthen their electronic disease surveillance capacities [<xref ref-type="bibr" rid="pmed-0050151-b001">1</xref>], the parts of the world that are most vulnerable to emerging disease threats still lack essential public health information infrastructure [<xref ref-type="bibr" rid="pmed-0050151-b002">2</xref>,<xref ref-type="bibr" rid="pmed-0050151-b003">3</xref>]. The existing network of traditional surveillance efforts managed by health ministries, public health institutes, multinational agencies, and laboratory and institutional networks has wide gaps in geographic coverage and often suffers from poor and sometimes suppressed information flow across national borders [<xref ref-type="bibr" rid="pmed-0050151-b004">4</xref>]. At the same time, an enormous amount of valuable information about infectious diseases is found in Web-accessible information sources such as discussion sites, disease reporting networks, and news outlets [<xref ref-type="bibr" rid="pmed-0050151-b005">5</xref>,<xref ref-type="bibr" rid="pmed-0050151-b006">6</xref>,<xref ref-type="bibr" rid="pmed-0050151-b007">7</xref>]. These resources can support situational awareness by providing current, highly local information about outbreaks, even from areas relatively invisible to traditional global public health efforts [<xref ref-type="bibr" rid="pmed-0050151-b008">8</xref>]. These data are plagued by a number of potential hazards that must be studied in depth, including false reports (mis- or disinformation) and reporting bias. Yet these data hold tremendous potential to initiate epidemiologic follow-up studies and provide complementary epidemic intelligence context to traditional surveillance sources. This potential is already being realized, as a majority of outbreak verifications currently conducted by the World Health Organization (WHO)'s Global Outbreak Alert and Response Network are triggered by reports from these nontraditional sources [<xref ref-type="bibr" rid="pmed-0050151-b005">5</xref>,<xref ref-type="bibr" rid="pmed-0050151-b006">6</xref>].</p>
      <boxed-text id="pmed-0050151-box001" position="float">
        <sec>
          <title>Summary Points</title>
          <list list-type="bullet">
            <list-item>
              <p>Valuable information about infectious diseases is found in Web-accessible information sources such as discussion forums, mailing lists, government Web sites, and news outlets.</p>
            </list-item>
            <list-item>
              <p>Web-based electronic information sources can play an important role in early event detection and support situational awareness by providing current, highly local information about outbreaks, even from areas relatively invisible to traditional global public health efforts.</p>
            </list-item>
            <list-item>
              <p>While these sources are potentially useful, information overload and difficulties in distinguishing “signal from noise” pose substantial barriers to fully utilizing this information.</p>
            </list-item>
            <list-item>
              <p>HealthMap is a freely accessible, automated real-time system that monitors, organizes, integrates, filters, visualizes, and disseminates online information about emerging diseases.</p>
            </list-item>
            <list-item>
              <p>The goal of HealthMap is to deliver real-time intelligence on a broad range of emerging infectious diseases for a diverse audience, from public health officials to international travelers.</p>
            </list-item>
            <list-item>
              <p>Ultimately, the use of news media and other nontraditional sources of surveillance data can facilitate early outbreak detection, increase public awareness of disease outbreaks prior to their formal recognition, and provide an integrated and contextualized view of global health information.</p>
            </list-item>
          </list>
        </sec>
      </boxed-text>
      <p>In one of the most frequently cited examples [<xref ref-type="bibr" rid="pmed-0050151-b009">9</xref>], early indications of the severe acute respiratory syndrome (SARS) outbreak in Guangdong Province, China, came in November 2002 from a Chinese article that alluded to an unusual increase in emergency department visits with acute respiratory illness [<xref ref-type="bibr" rid="pmed-0050151-b009">9</xref>,<xref ref-type="bibr" rid="pmed-0050151-b010">10</xref>]. This was followed by media reports of a respiratory disease among health care workers in February 2003, all captured by the Public Health Agency of Canada's Global Public Health Intelligence Network (GPHIN) [<xref ref-type="bibr" rid="pmed-0050151-b010">10</xref>,<xref ref-type="bibr" rid="pmed-0050151-b011">11</xref>,<xref ref-type="bibr" rid="pmed-0050151-b012">12</xref>]. In parallel, online discussions on the ProMED-mail system referred to an outbreak in Guangzhou, well before official government reports were issued [<xref ref-type="bibr" rid="pmed-0050151-b013">13</xref>].</p>
      <p>These Web-based data sources not only facilitate early outbreak detection, but also support increasing public awareness of disease outbreaks prior to their formal recognition. Through low-cost and real-time Internet data-mining, combined with openly available and user-friendly technologies, both participation in and access to global disease surveillance are no longer limited to the public health community [<xref ref-type="bibr" rid="pmed-0050151-b014">14</xref>,<xref ref-type="bibr" rid="pmed-0050151-b015">15</xref>]. The availability of Web-based news media provides an alternative public health information source in under-resourced areas. However, the myriad diverse sources of infectious disease information across the Web are not structured or organized; public health officials, nongovernmental organizations, and concerned citizens must routinely search and synthesize a continually growing number of disparate sources in order to use this information. With the aim of creating an integrated global view of emerging infections based not only on traditional public health datasets but rather on all available information sources, we developed HealthMap, a freely accessible, automated electronic information system for organizing data on outbreaks according to geography, time, and infectious disease agent [<xref ref-type="bibr" rid="pmed-0050151-b016">16</xref>] (<xref ref-type="fig" rid="pmed-0050151-g001">Figure 1</xref>).</p>
      <fig id="pmed-0050151-g001" position="float">
        <object-id pub-id-type="doi">10.1371/journal.pmed.0050151.g001</object-id>
        <label>Figure 1</label>
        <caption>
          <title>Screenshot of the HealthMap System (<ext-link ext-link-type="uri" xlink:href="http://www.healthmap.org/" xlink:type="simple">http://www.healthmap.org/</ext-link>)</title>
        </caption>
        <graphic mimetype="image" position="float" xlink:href="info:doi/10.1371/journal.pmed.0050151.g001" xlink:type="simple"/>
      </fig>
    </sec>
    <sec id="s2">
      <title>The System</title>
      <p>Operating since September 2006, HealthMap (<ext-link ext-link-type="uri" xlink:href="http://www.healthmap.org/" xlink:type="simple">http://www.healthmap.org/</ext-link>) is a multistream real-time surveillance platform that continually aggregates reports on new and ongoing infectious disease outbreaks [<xref ref-type="bibr" rid="pmed-0050151-b017">17</xref>]. The system performs extraction, categorization, filtration, and integration of these reports, facilitating knowledge management and early detection (<xref ref-type="fig" rid="pmed-0050151-g002">Figure 2</xref>). Through this approach, we seek a unified and comprehensive view of current infectious disease outbreaks in space and time worldwide.</p>
      <fig id="pmed-0050151-g002" position="float">
        <object-id pub-id-type="doi">10.1371/journal.pmed.0050151.g002</object-id>
        <label>Figure 2</label>
        <caption>
          <title>Stages of HealthMap Surveillance</title>
          <p>(1) Web-based data are acquired from a variety of Web sites every hour, 7 days a week (ranging from rumors on discussion sites to news media to validated official reports). (2) The extracted articles are then categorized by pathogen and location of the outbreak in question. (3) Articles are then analyzed for duplication and content. Duplicate articles are removed, while those that discuss new information about an ongoing situation are integrated with other related articles and added to the interactive map. (4) Once classified, articles are filtered by their relevance into five categories. Only “breaking news” articles are added as markers to the map.</p>
        </caption>
        <graphic mimetype="image" position="float" xlink:href="info:doi/10.1371/journal.pmed.0050151.g002" xlink:type="simple"/>
      </fig>
      <p>HealthMap is designed to provide a starting point for real-time intelligence on a broad range of emerging infectious diseases for a diverse range of end users, from public health officials to international travelers [<xref ref-type="bibr" rid="pmed-0050151-b018">18</xref>,<xref ref-type="bibr" rid="pmed-0050151-b019">19</xref>,<xref ref-type="bibr" rid="pmed-0050151-b020">20</xref>]. The system currently serves as a direct information source for approximately 20,000 unique visitors per month, as well as a resource for libraries, local health departments, governments (e.g., the US Department Health and Human Services and Department of Defense), and multinational agencies (e.g., the United Nations), which use the HealthMap data stream for day-to-day surveillance activities. Many regular users come from the WHO, the US Centers for Disease Control and Prevention, and the European Centre for Disease Prevention and Control.</p>
      <sec id="s2a">
        <title>Knowledge sources.</title>
        <p>HealthMap relies on a variety of electronic media sources, including online news sources through aggregators such as Google News, expert-curated discussion such as ProMED-mail [<xref ref-type="bibr" rid="pmed-0050151-b013">13</xref>,<xref ref-type="bibr" rid="pmed-0050151-b021">21</xref>,<xref ref-type="bibr" rid="pmed-0050151-b022">22</xref>], and validated official reports from organizations such as the WHO. Currently, the system collects reports from 14 sources, which in turn represent information from over 20,000 Web sites, every hour, 24 hours a day. Internet search criteria include disease names (scientific and common), symptoms, keywords, and phrases. The system collects an average of 300 reports per day, with the majority acquired from news media sources (85.1%). Although most of the reports collected to date have been in English, HealthMap also monitors information sources in Chinese, Spanish, Russian, and French, with additional languages such as Hindi, Portuguese, and Arabic under development. As HealthMap reports are acquired solely from free news sources, operational costs are minimal. The Web site is freely accessible on the Internet without subscription fees.</p>
        <p>The use of international news media for public health surveillance has a number of potential biases that merit consideration. While local news sources may report on incidents involving a few cases that would not be picked up at the national level, such sources may be less reliable, lacking resources and training, and may report stories without adequate confirmation. Furthermore, other biases may be intentionally introduced for political reasons through disinformation campaigns (false positives) or state censorship of information relating to outbreaks (false negatives). We have attempted to better understand some of these issues through ongoing analysis and evaluation research. We ran a 43-week evaluation of HealthMap data, covering the period of October 1, 2006 through July 18 2007. We found that pathogen diversity was substantial across news sources, with 141 unique infectious disease categories reported through the Google News feed alone (<xref ref-type="table" rid="pmed-0050151-t001">Table 1</xref>). We found the frequency of reports about particular pathogens to be related not to their associated morbidity or mortality impact, but rather to the direct or potential economic and social disruption caused by the outbreak.</p>
        <table-wrap content-type="2col" id="pmed-0050151-t001" position="float"><object-id pub-id-type="doi">10.1371/journal.pmed.0050151.t001</object-id><label>Table 1</label><caption>
            <p>Infectious Disease Occurrences Extracted from Google News Searches from October 1, 2006 through July 18, 2007</p>
          </caption><graphic mimetype="image" position="float" xlink:href="info:doi/10.1371/journal.pmed.0050151.t001" xlink:type="simple"/><!-- <table frame="hsides" rules="groups"><colgroup><col id="tb1col1" align="left"/><col id="tb1col2" align="right"/></colgroup><thead><tr><td><bold>Disease Reported</bold></td><td align="center"><bold>n</bold></td></tr></thead><tbody><tr><td>Avian influenza</td><td>877</td></tr><tr><td><named-content content-type="genus-species">Escherichia coli</named-content></td><td>733</td></tr><tr><td>Salmonella</td><td>479</td></tr><tr><td>Norovirus</td><td>384</td></tr><tr><td>Influenza</td><td>251</td></tr><tr><td>Dengue fever</td><td>190</td></tr><tr><td>Cholera</td><td>149</td></tr><tr><td>Gastroenteritis</td><td>127</td></tr><tr><td><named-content content-type="genus-species">Clostridium difficile</named-content></td><td>100</td></tr><tr><td>Measles</td><td>93</td></tr><tr><td>Equine herpes</td><td>86</td></tr><tr><td>Tuberculosis</td><td>77</td></tr><tr><td>Mumps</td><td>74</td></tr><tr><td>Diarrhea</td><td>58</td></tr><tr><td>Whooping cough</td><td>58</td></tr><tr><td>Herpes</td><td>57</td></tr><tr><td>Staphylococcal disease</td><td>56</td></tr><tr><td>Legionellosis</td><td>54</td></tr><tr><td>Chickenpox</td><td>48</td></tr><tr><td>Malaria</td><td>48</td></tr><tr><td>Meningitis</td><td>44</td></tr><tr><td>Hepatitis</td><td>42</td></tr><tr><td>Foot and mouth disease</td><td>36</td></tr><tr><td>Algae</td><td>34</td></tr><tr><td>Typhoid</td><td>33</td></tr><tr><td><named-content content-type="genus-species">Pseudomonas aeruginosa</named-content></td><td>30</td></tr><tr><td>Chikungunya</td><td>29</td></tr><tr><td>Shigellosis</td><td>28</td></tr><tr><td>Rift Valley fever</td><td>26</td></tr><tr><td>Hand, foot, and mouth disease</td><td>25</td></tr><tr><td>HIV/AIDS</td><td>24</td></tr><tr><td>Anthrax</td><td>22</td></tr><tr><td>SARS</td><td>21</td></tr><tr><td>Parvovirus infection</td><td>19</td></tr><tr><td>Scabies</td><td>19</td></tr><tr><td>Rabies</td><td>16</td></tr><tr><td>Polio</td><td>15</td></tr><tr><td>Swine fever</td><td>15</td></tr><tr><td>Syphilis</td><td>15</td></tr><tr><td>Methicillin-resistant <named-content content-type="genus-species">Staphylococcus aureus</named-content></td><td>14</td></tr><tr><td>Plague</td><td>14</td></tr><tr><td>Enterococcal disease</td><td>13</td></tr><tr><td>Cryptosporidium</td><td>12</td></tr><tr><td>Encephalitis</td><td>12</td></tr><tr><td>Smallpox</td><td>10</td></tr><tr><td>African horse sickness</td><td>9</td></tr><tr><td>Newcastle disease</td><td>8</td></tr><tr><td>Dysentery</td><td>7</td></tr><tr><td>Streptococcal disease</td><td>7</td></tr><tr><td>Yellow fever</td><td>7</td></tr><tr><td>Ebola</td><td>5</td></tr><tr><td>Scarlet fever</td><td>4</td></tr><tr><td>West Nile virus</td><td>4</td></tr><tr><td>Botulism</td><td>3</td></tr><tr><td>Brucellosis</td><td>3</td></tr><tr><td>Conjunctivitis</td><td>3</td></tr><tr><td>Hantavirus</td><td>3</td></tr><tr><td>Schistosomiasis</td><td>3</td></tr><tr><td>Blastomycosis</td><td>2</td></tr><tr><td>Bluetongue</td><td>2</td></tr><tr><td>Chlamydia</td><td>2</td></tr><tr><td>Crimean-Congo hemorrhagic fever</td><td>2</td></tr><tr><td>Equine infectious disease</td><td>2</td></tr><tr><td>Lassa fever</td><td>2</td></tr><tr><td>Leptospirosis</td><td>2</td></tr><tr><td>Rotavirus</td><td>2</td></tr><tr><td>Bovine spongiform encephalopathy</td><td>1</td></tr><tr><td>Chronic wasting disease</td><td>1</td></tr><tr><td>Creutzfeldt-Jakob disease</td><td>1</td></tr><tr><td>Eastern equine encephalitis</td><td>1</td></tr><tr><td>Lyme disease</td><td>1</td></tr><tr><td>Monkeypox</td><td>1</td></tr><tr><td>Mononucleosis</td><td>1</td></tr><tr><td>Q fever</td><td>1</td></tr><tr><td>Soybean rust</td><td>1</td></tr><tr><td>Strangles</td><td>1</td></tr><tr><td>Tularemia</td><td>1</td></tr><tr><td>Vesicular stomatitis</td><td>1</td></tr></tbody></table> --></table-wrap>
        <p>For instance, we found substantial skew towards reporting on stories about avian influenza and food-borne illnesses. Over the evaluation time period, 174 countries had reports of infectious disease outbreaks, with the greatest reporting from the United States (<italic>n</italic> = 4351), the United Kingdom (<italic>n</italic> = 1018), Canada (<italic>n</italic> = 880), and China (<italic>n</italic> = 737) (<xref ref-type="fig" rid="pmed-0050151-g003">Figure 3</xref>A). There was a clear bias towards increased reporting from countries with higher numbers of media outlets, more developed public health resources, and greater availability of electronic communication infrastructure (approximated by number of Internet hosts) (<xref ref-type="fig" rid="pmed-0050151-g003">Figure 3</xref>B). These trends are highly relevant for users of the system, and thus the individual impact of these factors on surveillance will form the basis of a detailed user guide currently under development.</p>
        <fig id="pmed-0050151-g003" position="float">
          <object-id pub-id-type="doi">10.1371/journal.pmed.0050151.g003</object-id>
          <label>Figure 3</label>
          <caption>
            <title>Map of Outbreak Report Coverage</title>
            <p>Geographic coverage of English language news reports of infectious disease outbreaks collected through Google News from October 1, 2006 through July 18, 2007. (A) displays counts of disease outbreak reports; (B) displays population-adjusted outbreak reporting as number of reports per million inhabitants.</p>
          </caption>
          <graphic mimetype="image" position="float" xlink:href="info:doi/10.1371/journal.pmed.0050151.g003" xlink:type="simple"/>
        </fig>
      </sec>
      <sec id="s2b">
        <title>Knowledge extraction.</title>
        <p>The system characterizes disease outbreak reports by means of a series of text mining algorithms. (A complete technical description of the system may be found elsewhere [<xref ref-type="bibr" rid="pmed-0050151-b023">23</xref>].) Characterization stages include: (a) identifying disease and location; (b) determining relevance—namely, whether a given report refers to any current outbreak; and (c) grouping similar reports together while removing exact duplicates. Once the reports are automatically processed, curators correct the misclassifications of the system where necessary (<xref ref-type="fig" rid="pmed-0050151-g002">Figure 2</xref>). Currently, only one analyst reviews and corrects the posts. However, additional resources would enable more detailed multilingual curation and annotation of collected reports.</p>
        <p>Extracting location and disease names from text reports presents the most formidable challenge. HealthMap draws from a continually expanding dictionary of pathogens (human, plant, and animal diseases) and geographic names (country, province, state, and city) to classify outbreak alert information. However, disease and place names are often ambiguous, colloquial, and subject to change, and may have multiple spellings (e.g., diarrhea, common in the US, and diarrhoea, common in the UK). Thus, the expansion and editing of the database requires extensive manual data entry.</p>
        <p>Once location and disease have been identified, articles are automatically tagged according to their relevance. Specifically, we identify whether a given report refers to a current outbreak (“breaking news”), as opposed to reporting on other infectious disease–related news, such as vaccination campaigns, scientific research, or public health policy. In this case, HealthMap makes use of a Bayesian machine learning algorithm, trained on manually characterized existing reports, to automatically tag and separate breaking news. Finally, duplicate reports are filtered, identified, and grouped based on the similarity of the article's headline, body text, and disease and location categories. Using a similarity score threshold, the system groups related articles into clusters that provide the collective information on a given outbreak.</p>
      </sec>
      <sec id="s2c">
        <title>Knowledge integration and dissemination.</title>
        <p>HealthMap is particularly focused on providing users with news of immediate interest and reducing information overload. Overwhelming public health officials with information on outbreaks of low public health impact may distract them from investigating outbreaks of greater priority that might receive reduced media attention. Thus, only articles classified as breaking news are posted to the site. Although they are filtered from the initial display, other article types and duplicate articles are shown in a related information window, providing a situational report on an ongoing outbreak as well as recent reports concerning either the same disease or location, and links for further research (<xref ref-type="fig" rid="pmed-0050151-g004">Figure 4</xref>).</p>
        <fig id="pmed-0050151-g004" position="float">
          <object-id pub-id-type="doi">10.1371/journal.pmed.0050151.g004</object-id>
          <label>Figure 4</label>
          <caption>
            <title>HealthMap Situational Awareness Window</title>
            <p>All articles related to a given outbreak are aggregated by text similarity matching in order to provide a situational awareness report. Furthermore, other outbreaks occurring in the same geographic area or involving the same pathogen are provided. The window also provides links to further research on the subject. In this example, we show all alerts relating to a recent cholera outbreak in Nigeria.</p>
          </caption>
          <graphic mimetype="image" position="float" xlink:href="info:doi/10.1371/journal.pmed.0050151.g004" xlink:type="simple"/>
        </fig>
        <p>HealthMap also addresses the computational challenges of integrating multiple sources of unstructured information by generating meta-alerts of disease outbreaks. As false alarms can often be reduced by thorough aggregation and cross-validation of reported information, a composite activity score (or heat index) is calculated based on (a) the reliability of the data source (for instance, increased weight is given to WHO reports and reduced weight to local media reports); and (b) the number of unique data sources, with increased weight to multiple types of information (e.g., discussion sites and media reports on the same outbreak). This meta-alert derivation is based on the idea that multiple sources of information about an incident provide greater confidence in the reliability of the report than any one source alone.</p>
      </sec>
    </sec>
    <sec id="s3">
      <title>The Future</title>
      <p>A wide range of further improvements are currently being developed across all components of the HealthMap system. In particular, population and geography gaps in the coverage of the monitored sources need to be better understood and accounted for. For example, there are critical gaps in media reporting in tropical and lower-latitude areas, including major parts of Africa and South America—the very regions that have the greatest burden and risk of emerging infectious diseases (<xref ref-type="fig" rid="pmed-0050151-g003">Figure 3</xref>) [<xref ref-type="bibr" rid="pmed-0050151-b003">3</xref>]. Monitoring other Internet-based sources such as blogs, discussion sites, and listservs could complement news coverage. The use of click-stream data and individual search queries is also a promising new surveillance source [<xref ref-type="bibr" rid="pmed-0050151-b024">24</xref>]. Multilingual surveillance is critical for capturing greater geographic coverage and for providing earlier and more comprehensive reporting from local news media.</p>
      <p>Potential future challenges include the possibility that news data sources that are freely available now will no longer be available if current business models change. In addition, the way news is reported online (content, format, communication standards, etc.) may change and develop in the coming years, which will require a re-tooling of the system in order to capture the appropriate information. Potential future benefits of technological advances include better meta-data tagging if/when the semantic Web becomes a reality. Also, as location-based services become more widespread, including on portable devices, HealthMap feeds can be tailored and targeted to specific users and their locations.</p>
      <p>Future work must also focus on improving natural language processing capability to clearly identify the pathogen, filter non-pertinent reports and duplicates, and enhance the spatial resolution of location extraction. However, while improvements in machine learning techniques are undoubtedly critical, they cannot currently replace human analysis. The success of Wikipedia has shown that leveraging collaborative human networks of trained public health professionals has the potential to support improved classification, severity assignment, conflict resolution, geocoding, and confirmation of reports on rare or unknown infectious diseases [<xref ref-type="bibr" rid="pmed-0050151-b025">25</xref>]. A recently established collaboration between HealthMap and ProMED-mail (<ext-link ext-link-type="uri" xlink:href="http://www.healthmap.org/promed" xlink:type="simple">http://www.healthmap.org/promed</ext-link>) is helping to pave the way for such a bidirectional system of classification and curation of information flow [<xref ref-type="bibr" rid="pmed-0050151-b026">26</xref>].</p>
      <p>Continued system evaluation is also essential. The fundamental characteristics of different news source types need to be quantified, including sensitivity, specificity, and timeliness [<xref ref-type="bibr" rid="pmed-0050151-b027">27</xref>,<xref ref-type="bibr" rid="pmed-0050151-b028">28</xref>,<xref ref-type="bibr" rid="pmed-0050151-b029">29</xref>,<xref ref-type="bibr" rid="pmed-0050151-b030">30</xref>]. Consideration should also be given to integrating unstructured online information sources with other health indicator data to provide a broader context for reports. Pertinent data sets include mortality and morbidity estimates, laboratory data, field surveillance (e.g., vector and animal reservoir distribution), environmental predictors (e.g., climate and vegetation), population density and mobility, and pathogen seasonality and transmissibility. Such integration could yield a more precise relevance score for a given report, define populations at risk, and predict disease spread.</p>
      <boxed-text id="pmed-0050151-box002" position="float">
        <sec>
          <title>Glossary</title>
          <p><bold>Blog:</bold> A regularly updated online journal containing news or commentary on a particular topic, generally produced by an individual or a small group of people.</p>
          <p><bold>Click-stream:</bold> A sequential record of the actions performed by a user while browsing the Internet, including Web sites visited, searches performed, and hyperlinks followed.</p>
          <p><bold>Event-based surveillance:</bold> Unstructured data gathered from sources of intelligence of any nature.</p>
          <p><bold>Indicator-based surveillance:</bold> Structured data collected through routine public health surveillance systems.</p>
          <p><bold>Informal surveillance:</bold> Information from individuals or news media sources, as opposed to official government or government-sponsored reports.</p>
          <p><bold>Listserv:</bold> An automated email forwarding system that allows any member of a group of people to easily send a message to all other members of the group.</p>
          <p><bold>Machine learning:</bold> A broad subfield of artificial intelligence that studies how systems can learn general principles from specific examples.</p>
          <p><bold>Multistream surveillance:</bold> An approach that monitors multiple sources of information and may also integrate them into a unified analytical framework.</p>
        </sec>
      </boxed-text>
    </sec>
    <sec id="s4">
      <title>Conclusion</title>
      <p>HealthMap is a member of a new generation of surveillance systems that mine media sources in near real-time for reports of infectious disease outbreaks, including GPHIN [<xref ref-type="bibr" rid="pmed-0050151-b010">10</xref>,<xref ref-type="bibr" rid="pmed-0050151-b012">12</xref>], MedISys, developed by the Directorate General Health and Consumer Affairs of the European Commission [<xref ref-type="bibr" rid="pmed-0050151-b031">31</xref>], the US government-funded Argus [<xref ref-type="bibr" rid="pmed-0050151-b032">32</xref>], and EpiSPIDER [<xref ref-type="bibr" rid="pmed-0050151-b033">33</xref>]. While Internet-based online media sources are becoming a critical tool for global infectious disease surveillance, important challenges still need to be addressed. Since regions with the least advanced communication infrastructure also tend to carry the greatest infectious disease burden and risk, system development must be aimed at closing the gaps in these critical areas. Hence, achieving global coverage requires attention to creating and capturing locally feasible channels of communication. It also involves making the outputs of the system more accessible to users in these regions through user interfaces in additional languages and low-bandwidth display options, including mobile phone alerts.</p>
      <p>Ultimately, the monitoring of diverse media-based sources will augment epidemic intelligence with information derived outside the traditional public health infrastructure, yielding a more comprehensive and timely global view of emerging infectious disease threats. A truly open and accessible system can also assist users in overcoming existing geographical, organizational, and societal barriers to information, a process that can lead to greater empowerment, involvement, and democratization across the increasingly interconnected global health sphere.</p>
    </sec>
  </body>
  <back>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term>GPHIN</term>
          <def>
            <p>Global Public Health Intelligence Network</p>
          </def>
        </def-item>
        <def-item>
          <term>SARS</term>
          <def>
            <p>severe acute respiratory syndrome</p>
          </def>
        </def-item>
        <def-item>
          <term>WHO</term>
          <def>
            <p>World Health Organization</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <ref-list>
      <title>References</title>
      <ref id="pmed-0050151-b001">
        <label>1</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Mandl</surname><given-names>KD</given-names></name><name name-style="western"><surname>Overhage</surname><given-names>JM</given-names></name><name name-style="western"><surname>Wagner</surname><given-names>MM</given-names></name><name name-style="western"><surname>Lober</surname><given-names>WB</given-names></name><name name-style="western"><surname>Sebastiani</surname><given-names>P</given-names></name><etal/></person-group>
					<year>2004</year>
					<article-title>Implementing syndromic surveillance: A practical guide informed by the early experience.</article-title>
					<source>J Am Med Inform Assoc</source>
					<volume>11</volume>
					<fpage>141</fpage>
					<lpage>150</lpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b002">
        <label>2</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Butler</surname><given-names>D</given-names></name></person-group>
					<year>2006</year>
					<article-title>Disease surveillance needs a revolution.</article-title>
					<source>Nature</source>
					<volume>440</volume>
					<fpage>6</fpage>
					<lpage>7</lpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b003">
        <label>3</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Jones</surname><given-names>KE</given-names></name><name name-style="western"><surname>Patel</surname><given-names>NG</given-names></name><name name-style="western"><surname>Levy</surname><given-names>MA</given-names></name><name name-style="western"><surname>Storeygard</surname><given-names>A</given-names></name><name name-style="western"><surname>Balk</surname><given-names>D</given-names></name><etal/></person-group>
					<year>2008</year>
					<article-title>Global trends in emerging infectious diseases.</article-title>
					<source>Nature</source>
					<volume>451</volume>
					<fpage>990</fpage>
					<lpage>993</lpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b004">
        <label>4</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Sturtevant</surname><given-names>JL</given-names></name><name name-style="western"><surname>Anema</surname><given-names>A</given-names></name><name name-style="western"><surname>Brownstein</surname><given-names>JS</given-names></name></person-group>
					<year>2007</year>
					<article-title>The new international health regulations: Considerations for global public health surveillance.</article-title>
					<source>Disaster Med Public Health Prep</source>
					<volume>1</volume>
					<fpage>117</fpage>
					<lpage>121</lpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b005">
        <label>5</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Grein</surname><given-names>TW</given-names></name><name name-style="western"><surname>Kamara</surname><given-names>KB</given-names></name><name name-style="western"><surname>Rodier</surname><given-names>G</given-names></name><name name-style="western"><surname>Plant</surname><given-names>AJ</given-names></name><name name-style="western"><surname>Bovier</surname><given-names>P</given-names></name><etal/></person-group>
					<year>2000</year>
					<article-title>Rumors of disease in the global village: Outbreak verification.</article-title>
					<source>Emerg Infect Dis</source>
					<volume>6</volume>
					<fpage>97</fpage>
					<lpage>102</lpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b006">
        <label>6</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Heymann</surname><given-names>DL</given-names></name><name name-style="western"><surname>Rodier</surname><given-names>GR</given-names></name></person-group>
					<year>2001</year>
					<article-title>Hot spots in a wired world: WHO surveillance of emerging and re-emerging infectious diseases.</article-title>
					<source>Lancet Infect Dis</source>
					<volume>1</volume>
					<fpage>345</fpage>
					<lpage>353</lpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b007">
        <label>7</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>M'Ikanatha</surname><given-names>NM</given-names></name><name name-style="western"><surname>Rohn</surname><given-names>DD</given-names></name><name name-style="western"><surname>Robertson</surname><given-names>C</given-names></name><name name-style="western"><surname>Tan</surname><given-names>CG</given-names></name><name name-style="western"><surname>Holmes</surname><given-names>JH</given-names></name><etal/></person-group>
					<year>2006</year>
					<article-title>Use of the internet to enhance infectious disease surveillance and outbreak investigation.</article-title>
					<source>Biosecur Bioterror</source>
					<volume>4</volume>
					<fpage>293</fpage>
					<lpage>300</lpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b008">
        <label>8</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Woodall</surname><given-names>J</given-names></name></person-group>
					<year>1997</year>
					<article-title>Official versus unofficial outbreak reporting through the Internet.</article-title>
					<source>Int J Med Inform</source>
					<volume>47</volume>
					<fpage>31</fpage>
					<lpage>34</lpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b009">
        <label>9</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Heymann</surname><given-names>DL</given-names></name><name name-style="western"><surname>Rodier</surname><given-names>G</given-names></name></person-group>
					<year>2004</year>
					<article-title>Global surveillance, national surveillance, and SARS.</article-title>
					<source>Emerg Infect Dis</source>
					<volume>10</volume>
					<fpage>173</fpage>
					<lpage>175</lpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b010">
        <label>10</label>
        <element-citation publication-type="other" xlink:type="simple">
					<person-group><name name-style="western"><surname>Mawudeku</surname><given-names>A</given-names></name><name name-style="western"><surname>Blench</surname><given-names>M</given-names></name></person-group>
					<year>2006</year>
					<article-title>Global Public Health Intelligence Network (GPHIN).</article-title>
					<comment>7th Conference of the Association for Machine Translation in the Americas; 8–12 August 2006; Cambridge, Massachusetts, United States of America; .Available: <ext-link ext-link-type="uri" xlink:href="http://www.mt-archive.info/MTS-2005-Mawudeku.pdf" xlink:type="simple">http://www.mt-archive.info/MTS-2005-Mawudeku.pdf</ext-link>. Accessed 6 June 2008</comment>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b011">
        <label>11</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Eysenbach</surname><given-names>G</given-names></name></person-group>
					<year>2003</year>
					<article-title>SARS and population health technology.</article-title>
					<source>J Med Internet Res</source>
					<volume>5</volume>
					<fpage>e14</fpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b012">
        <label>12</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Mykhalovskiy</surname><given-names>E</given-names></name><name name-style="western"><surname>Weir</surname><given-names>L</given-names></name></person-group>
					<year>2006</year>
					<article-title>The Global Public Health Intelligence Network and early warning outbreak detection: A Canadian contribution to global public health.</article-title>
					<source>Can J Public Health</source>
					<volume>97</volume>
					<fpage>42</fpage>
					<lpage>44</lpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b013">
        <label>13</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Madoff</surname><given-names>LC</given-names></name><name name-style="western"><surname>Woodall</surname><given-names>JP</given-names></name></person-group>
					<year>2005</year>
					<article-title>The internet and the global monitoring of emerging diseases: Lessons from the first 10 years of ProMED-mail.</article-title>
					<source>Arch Med Res</source>
					<volume>36</volume>
					<fpage>724</fpage>
					<lpage>730</lpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b014">
        <label>14</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Keystone</surname><given-names>JS</given-names></name><name name-style="western"><surname>Kozarsky</surname><given-names>PE</given-names></name><name name-style="western"><surname>Freedman</surname><given-names>DO</given-names></name></person-group>
					<year>2001</year>
					<article-title>Internet and computer-based resources for travel medicine practitioners.</article-title>
					<source>Clin Infect Dis</source>
					<volume>32</volume>
					<fpage>757</fpage>
					<lpage>765</lpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b015">
        <label>15</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Petersen</surname><given-names>JE</given-names></name></person-group>
					<year>2005</year>
					<article-title>[Traveller's medicine on the Internet]. [Article in Danish].</article-title>
					<source>Ugeskr Laeger</source>
					<volume>167</volume>
					<fpage>3971</fpage>
					<lpage>3973</lpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b016">
        <label>16</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Brownstein</surname><given-names>JS</given-names></name><name name-style="western"><surname>Freifeld</surname><given-names>CC</given-names></name></person-group>
					<year>2007</year>
					<article-title>HealthMap: The development of automated real-time internet surveillance for epidemic intelligence.</article-title>
					<source>Euro Surveill</source>
					<volume>12</volume>
					<fpage>E071129.5</fpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b017">
        <label>17</label>
        <element-citation publication-type="other" xlink:type="simple">
					<person-group><name name-style="western"><surname>Brownstein</surname><given-names>JS</given-names></name><name name-style="western"><surname>Freifeld</surname><given-names>CC</given-names></name><name name-style="western"><surname>Reis</surname><given-names>BY</given-names></name><name name-style="western"><surname>Mandl</surname><given-names>KD</given-names></name></person-group>
					<year>2007</year>
					<article-title>HealthMap: Internet-based emerging infectious disease intelligence.</article-title>
					<comment>In: Institute of Medicine, editor</comment>
					<source>Infectious disease surveillance and detection: Assessing the challenges—Finding solutions</source>
					<comment>Available <ext-link ext-link-type="uri" xlink:href="http://books.nap.edu/openbook.php?record_id=11996&amp;page=122" xlink:type="simple">http://books.nap.edu/openbook.php?record_id=11996&amp;page=122</ext-link>. Accessed 6 June 2008</comment>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b018">
        <label>18</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Holden</surname><given-names>C</given-names></name></person-group>
					<year>2006</year>
					<article-title>Netwatch: Diseases on the move.</article-title>
					<source>Science</source>
					<volume>314</volume>
					<fpage>1363d</fpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b019">
        <label>19</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Larkin</surname><given-names>M</given-names></name></person-group>
					<year>2007</year>
					<article-title>Technology and public health: Healthmap tracks global diseases.</article-title>
					<source>Lancet Infect Dis</source>
					<volume>7</volume>
					<fpage>91</fpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b020">
        <label>20</label>
        <element-citation publication-type="other" xlink:type="simple">
					<person-group><name name-style="western"><surname>Captain</surname><given-names>S</given-names></name></person-group>
					<year>2006 October 19</year>
					<article-title>Get your daily plague forecast. Wired News.</article-title>
					<comment>Available: <ext-link ext-link-type="uri" xlink:href="http://www.wired.com/science/discoveries/news/2006/10/71961" xlink:type="simple">http://www.wired.com/science/discoveries/news/2006/10/71961</ext-link>. Accessed 6 June 2008</comment>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b021">
        <label>21</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Hugh-Jones</surname><given-names>M</given-names></name></person-group>
					<year>2001</year>
					<article-title>Global awareness of disease outbreaks: The experience of ProMED-mail.</article-title>
					<source>Public Health Rep</source>
					<volume>116</volume>
					<issue>Suppl 2</issue>
					<fpage>27</fpage>
					<lpage>31</lpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b022">
        <label>22</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Woodall</surname><given-names>J</given-names></name><name name-style="western"><surname>Calisher</surname><given-names>CH</given-names></name></person-group>
					<year>2001</year>
					<article-title>ProMED-mail: Background and purpose.</article-title>
					<source>Emerg Infect Dis</source>
					<volume>7</volume>
					<fpage>563</fpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b023">
        <label>23</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Freifeld</surname><given-names>CC</given-names></name><name name-style="western"><surname>Mandl</surname><given-names>KD</given-names></name><name name-style="western"><surname>Reis</surname><given-names>BY</given-names></name><name name-style="western"><surname>Brownstein</surname><given-names>JS</given-names></name></person-group>
					<year>2008</year>
					<article-title>HealthMap: Global infectious disease monitoring through automated classification and visualization of internet media reports.</article-title>
					<source>J Am Med Inform Assoc</source>
					<volume>15</volume>
					<fpage>150</fpage>
					<lpage>157</lpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b024">
        <label>24</label>
        <element-citation publication-type="other" xlink:type="simple">
					<person-group><name name-style="western"><surname>Eysenbach</surname><given-names>G</given-names></name></person-group>
					<year>2006</year>
					<article-title>Infodemiology: Tracking flu-related searches on the web for syndromic surveillance.</article-title>
					<source>AMIA Annu Symp Proc</source>
					<fpage>244</fpage>
					<lpage>248</lpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b025">
        <label>25</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Giles</surname><given-names>J</given-names></name></person-group>
					<year>2005</year>
					<article-title>Internet encyclopaedias go head to head.</article-title>
					<source>Nature</source>
					<volume>438</volume>
					<fpage>900</fpage>
					<lpage>901</lpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b026">
        <label>26</label>
        <element-citation publication-type="other" xlink:type="simple">
					<collab xlink:type="simple">ProMed-mail</collab>
					<year>2007 October 15</year>
					<article-title>Interactive map of ProMED reports available.</article-title>
					<comment>Available: <ext-link ext-link-type="uri" xlink:href="http://list.uvm.edu/cgi-bin/wa?A2=ind0710C&amp;L=SAFETY&amp;D=0&amp;P=7700&amp;F=P" xlink:type="simple">http://list.uvm.edu/cgi-bin/wa?A2=ind0710C&amp;L=SAFETY&amp;D=0&amp;P=7700&amp;F=P</ext-link>. Accessed 6 June 2008</comment>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b027">
        <label>27</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Wagner</surname><given-names>MM</given-names></name><name name-style="western"><surname>Tsui</surname><given-names>FC</given-names></name><name name-style="western"><surname>Espino</surname><given-names>JU</given-names></name><name name-style="western"><surname>Dato</surname><given-names>VM</given-names></name><name name-style="western"><surname>Sittig</surname><given-names>DF</given-names></name><etal/></person-group>
					<year>2001</year>
					<article-title>The emerging science of very early detection of disease outbreaks.</article-title>
					<source>J Public Health Manag Pract</source>
					<volume>7</volume>
					<fpage>51</fpage>
					<lpage>59</lpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b028">
        <label>28</label>
        <element-citation publication-type="other" xlink:type="simple">
					<person-group><name name-style="western"><surname>Reis</surname><given-names>BY</given-names></name><name name-style="western"><surname>Mandl</surname><given-names>KD</given-names></name></person-group>
					<year>2003</year>
					<article-title>Integrating syndromic surveillance data across multiple locations: Effects on outbreak detection performance.</article-title>
					<source>Proc AMIA Sym</source>
					<fpage>549</fpage>
					<lpage>553</lpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b029">
        <label>29</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Bloom</surname><given-names>RM</given-names></name><name name-style="western"><surname>Buckeridge</surname><given-names>DL</given-names></name><name name-style="western"><surname>Cheng</surname><given-names>KE</given-names></name></person-group>
					<year>2007</year>
					<article-title>Finding leading indicators for disease outbreaks: Filtering, cross-correlation, and caveats.</article-title>
					<source>J Am Med Inform Assoc</source>
					<volume>14</volume>
					<fpage>76</fpage>
					<lpage>85</lpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b030">
        <label>30</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Brownstein</surname><given-names>JS</given-names></name><name name-style="western"><surname>Kleinman</surname><given-names>KP</given-names></name><name name-style="western"><surname>Mandl</surname><given-names>KD</given-names></name></person-group>
					<year>2005</year>
					<article-title>Identifying pediatric age groups for influenza vaccination using a real-time regional surveillance system.</article-title>
					<source>Am J Epidemiol</source>
					<volume>162</volume>
					<fpage>686</fpage>
					<lpage>693</lpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b031">
        <label>31</label>
        <element-citation publication-type="other" xlink:type="simple">
					<collab xlink:type="simple">Health Threats Unit at Directorate General Health and Consumer Affairs of the European Commission</collab>
					<year>2007</year>
					<article-title>MedISys (Medical Intelligence System).</article-title>
					<comment>Available: <ext-link ext-link-type="uri" xlink:href="http://medusa.jrc.it/" xlink:type="simple">http://medusa.jrc.it/</ext-link>. Accessed 6 June 2008</comment>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b032">
        <label>32</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Wilson</surname><given-names>JMt</given-names></name><name name-style="western"><surname>Polyak</surname><given-names>MG</given-names></name><name name-style="western"><surname>Blake</surname><given-names>JW</given-names></name><name name-style="western"><surname>Collmann</surname><given-names>J</given-names></name></person-group>
					<year>2008</year>
					<article-title>A heuristic indication and warning staging model for detection and assessment of biological events.</article-title>
					<source>J Am Med Inform Assoc</source>
					<volume>15</volume>
					<fpage>158</fpage>
					<lpage>171</lpage>
				</element-citation>
      </ref>
      <ref id="pmed-0050151-b033">
        <label>33</label>
        <element-citation publication-type="journal" xlink:type="simple">
					<person-group><name name-style="western"><surname>Tolentino</surname><given-names>H</given-names></name><name name-style="western"><surname>Kamadjeu</surname><given-names>R</given-names></name><name name-style="western"><surname>Fontelo</surname><given-names>P</given-names></name><name name-style="western"><surname>Liu</surname><given-names>F</given-names></name><name name-style="western"><surname>Matters</surname><given-names>M</given-names></name><etal/></person-group>
					<year>2007</year>
					<article-title>Scanning the emerging infectious diseases horizon—Visualizing ProMED emails using EpiSPIDER.</article-title>
					<source>Adv Dis Surveill</source>
					<volume>2</volume>
					<fpage>169</fpage>
				</element-citation>
      </ref>
    </ref-list>
    
  </back>
</article>