{"componentChunkName":"component---src-templates-material-js","path":"/workshops/2021/12/07/semanticheskii-analiz-v-python/index.html","result":{"pageContext":{"pageParameters":{"isPreview":false,"isMirror":true,"delay_to_show_donate_popup_seconds":7,"about_page":false,"games_page":true,"games_page_link_in_header":false,"donate_popup_show_ru":true,"donate_popup_title_ru":"Без вас — нас не будет","donate_popup_content_ru":"Помогите нам добывать и рассказывать важные истории. Участие каждого — важно!","donate_popup_show_en":true,"donate_popup_title_en":"We can’t do it alone!","donate_popup_content_en":"Help us uncover and share the stories that matter. Every contribution makes a difference","display_regular_donors_banner":false,"showPatreonMethod":true,"useNewMaterialDonationBanner":true,"new_donations_page_design":true,"donations_popup_variants":{"ru":[{"id":"kc19b","title":"Сохраните журналистику для людей в России ","subtitle":"Читают нас прежде всего люди в России, но будущее «Важных историй» зависит от тех, кто живет за пределами страны. Поддержите нас ежемесячным донатом — сохраните независимую журналистику для ваших близких, которые остаются в России.","is_active":true,"background":"linear-gradient(10deg, #421f20 5%, #994146)"},{"id":"33tc9","title":"У пропаганды есть миллиарды из бюджета. У нас — только вы","subtitle":"Перемены возможны, если независимые от власти журналисты продолжают свою работу день за днем, год за годом. Мы рассказываем важные истории уже шесть лет. Помогите нам продолжать.","is_active":true,"background":"linear-gradient(10deg, #421f20 5%, #994146)"}],"en":[{"id":"nj1py","title":"Important stories don’t tell themselves","subtitle":"Independent journalism comes at a cost. We answer to no owners or advertisers — only to you, our readers.  Your ongoing support allows us to keep exposing the truth. Without you, we won’t be here.","is_active":true,"background":"linear-gradient(10deg, #421f20 5%, #994146)"},{"id":"xdfc1","title":"Help us outlast the Kremlin censorship","subtitle":"Censorship lose to journalism that keeps showing up, year after year. Monthly donors are what make that possible","is_active":true,"background":"linear-gradient(10deg, #421f20 5%, #994146)"},{"id":"ck121","title":"State propaganda has billions to spend. We have only you to rely on","subtitle":"Change happens when independent journalists keep showing up — day after day, year after year. Six years in, we're still showing up. Help us keep going with a monthly donation.","is_active":true,"background":"linear-gradient(10deg, #421f20 5%, #994146)"}]},"feedback_page":true,"donorsCount":0,"showAuthorsRequestAfterMaterial":true,"radioPage":true,"radioExists":{"ru":true,"en":false},"singlePlaylistPath":{"ru":"/radio/2025/12/22/razgovori-ovazhnikh/index.html"},"with_bookmarks":true,"news_feed":true,"news_feed_skeleton":false,"news_page_redesign":true,"with_topics":true,"materials_topics":false,"topics_page":true,"with_pdf_share":false,"with_native_share":false,"preferred_source_page":true,"charon_new_form":true,"preferred_source_promos":true,"charon_en_page":true},"pk":385,"header":"Семантический анализ в Python","lead":"Рассчитываем контекстную близость слов с помощью библиотеки Word2vec","category":"workshops","entry_image":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/3dc6777a234d40eb97dc7697bfc4f177-1920x960px_1x.webp","url_2x":"https://static.istories.media/uploaded/images/3dc6777a234d40eb97dc7697bfc4f177-1920x960px_2x.webp","for_width":1920,"for_height":960,"real_width_1x":1920,"real_width_2x":3840,"real_height_1x":960,"real_height_2x":1920},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/3dc6777a234d40eb97dc7697bfc4f177-1920x960px_1x.png","url_2x":"https://static.istories.media/uploaded/images/3dc6777a234d40eb97dc7697bfc4f177-1920x960px_2x.png","for_width":1920,"for_height":960,"real_width_1x":1920,"real_width_2x":3840,"real_height_1x":960,"real_height_2x":1920},{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/3dc6777a234d40eb97dc7697bfc4f177-1272x636px_1x.webp","url_2x":"https://static.istories.media/uploaded/images/3dc6777a234d40eb97dc7697bfc4f177-1272x636px_2x.webp","for_width":1272,"for_height":636,"real_width_1x":1272,"real_width_2x":2544,"real_height_1x":636,"real_height_2x":1272},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/3dc6777a234d40eb97dc7697bfc4f177-1272x636px_1x.png","url_2x":"https://static.istories.media/uploaded/images/3dc6777a234d40eb97dc7697bfc4f177-1272x636px_2x.png","for_width":1272,"for_height":636,"real_width_1x":1272,"real_width_2x":2544,"real_height_1x":636,"real_height_2x":1272},{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/3dc6777a234d40eb97dc7697bfc4f177-912x456px_1x.webp","url_2x":"https://static.istories.media/uploaded/images/3dc6777a234d40eb97dc7697bfc4f177-912x456px_2x.webp","for_width":912,"for_height":456,"real_width_1x":912,"real_width_2x":1824,"real_height_1x":456,"real_height_2x":912},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/3dc6777a234d40eb97dc7697bfc4f177-912x456px_1x.png","url_2x":"https://static.istories.media/uploaded/images/3dc6777a234d40eb97dc7697bfc4f177-912x456px_2x.png","for_width":912,"for_height":456,"real_width_1x":912,"real_width_2x":1824,"real_height_1x":456,"real_height_2x":912},{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/3dc6777a234d40eb97dc7697bfc4f177-600x400px_1x.webp","url_2x":"https://static.istories.media/uploaded/images/3dc6777a234d40eb97dc7697bfc4f177-600x400px_2x.webp","for_width":600,"for_height":400,"real_width_1x":600,"real_width_2x":1200,"real_height_1x":400,"real_height_2x":800},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/3dc6777a234d40eb97dc7697bfc4f177-600x400px_1x.png","url_2x":"https://static.istories.media/uploaded/images/3dc6777a234d40eb97dc7697bfc4f177-600x400px_2x.png","for_width":600,"for_height":400,"real_width_1x":600,"real_width_2x":1200,"real_height_1x":400,"real_height_2x":800},{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/3dc6777a234d40eb97dc7697bfc4f177-360x360px_1x.webp","url_2x":"https://static.istories.media/uploaded/images/3dc6777a234d40eb97dc7697bfc4f177-360x360px_2x.webp","for_width":360,"for_height":360,"real_width_1x":360,"real_width_2x":720,"real_height_1x":360,"real_height_2x":720},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/3dc6777a234d40eb97dc7697bfc4f177-360x360px_1x.png","url_2x":"https://static.istories.media/uploaded/images/3dc6777a234d40eb97dc7697bfc4f177-360x360px_2x.png","for_width":360,"for_height":360,"real_width_1x":360,"real_width_2x":720,"real_height_1x":360,"real_height_2x":720}],"entry_image_credit":"","blocks":[{"data":"В одном из недавних материалов вместе с дата-отделом «Новой газеты» мы <a href=\"https://istories.media/reportages/2021/11/23/kak-propagandisti-nagnetayut-nenavist-k-migrantam-a-mvd-obvinyaet-v-etom-inoagentov/\" data-type=\"usual\" target=\"_blank\">разбирались</a>, как разные Telegram-каналы пишут о мигрантах, то есть, какой образ мигранта они создают в своих публикациях. Для этого мы выгрузили все посты из отобранных каналов и изучили их с помощью библиотеки для Python — <strong>Word2vec</strong>. О работе с ней и пойдет речь.","type":"p","className":""},{"data":{"id":3315,"data":{"url":"https://www.youtube.com/embed/__B4kino_9g","text":"Съемка и монтаж: Глеб Лиманский","provider":"youtube","rawInput":"https://www.youtube.com/watch?v=__B4kino_9g","ratio_mobile":"16:9","ratio_desktop":"16:9"},"category":"video-embed"},"type":"video-embed"},{"data":"<strong>Word2vec</strong> была разработана в 2013 году исследователями из Google. Она основана на методе дистрибутивной семантики, ее задача —&nbsp;вычислять семантическую близость между лексическими единицами (словами или словосочетаниями), чтобы помочь понять, какой образ создается у конкретного явления.","type":"p","className":""},{"data":"Чтобы использовать <strong>Word2vec</strong>, понадобится Python версии <strong>3.5</strong>. Создадим виртуальное окружение, где будем использовать именно эту версию. Открываем <strong>Anaconda</strong>, заходим во вкладку «<strong>Enviroments</strong>», нажимаем на кнопку «<strong>Create</strong>». Ставим галочку у «<strong>Python</strong>», выбираем в выпадающем меню нужную версию.&nbsp;В поле «<strong>Name</strong>» введите имя виртуального окружения и нажмите «<strong>Create</strong>».","type":"p","className":""},{"data":{"id":3298,"data":{"images":[{"credit":"Anaconda → «Enviroments» → «Create»","imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/f5d217a155e2470cada01ae1cb3bb01d-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/f5d217a155e2470cada01ae1cb3bb01d-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":1084,"real_height_2x":2169},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/f5d217a155e2470cada01ae1cb3bb01d-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/f5d217a155e2470cada01ae1cb3bb01d-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":1084,"real_height_2x":2169}]},{"credit":"Если версии 3.5 здесь не окажется, зайдите на официальный сайт Python и загрузите ее оттуда: https://www.python.org/downloads/","imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/4215ec98f7364b0bb6ee048a97fba141-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/4215ec98f7364b0bb6ee048a97fba141-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":708,"real_height_2x":1416},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/4215ec98f7364b0bb6ee048a97fba141-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/4215ec98f7364b0bb6ee048a97fba141-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":708,"real_height_2x":1416}]}]},"category":"two-images-embed"},"type":"two-images-embed"},{"data":"Когда окружение будет создано, заходите в <strong>Visual Studio Code</strong>. Создадим папку для нового проекта и перейдем внутрь нее.","type":"p","className":""},{"data":{"id":3299,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/e1cb021e74f144e68cb95c3fd61c187f-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/e1cb021e74f144e68cb95c3fd61c187f-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":282,"real_height_2x":566},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/e1cb021e74f144e68cb95c3fd61c187f-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/e1cb021e74f144e68cb95c3fd61c187f-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":282,"real_height_2x":566}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Откроем папку и создадим в ней новый файл, в котором начнем работать.&nbsp;Но сперва активируем виртуальное окружение. Для этого в терминале выбираем «<strong>bash</strong>» в выпадающем меню и пишем <strong>conda activate &lt;название виртуального окружения&gt;</strong>.","type":"p","className":""},{"data":{"id":3301,"data":{"images":[{"credit":"«+» → «bash»","imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/b8897e3c438b42dda47d2c22ea926d6b-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/b8897e3c438b42dda47d2c22ea926d6b-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":259,"real_height_2x":518},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/b8897e3c438b42dda47d2c22ea926d6b-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/b8897e3c438b42dda47d2c22ea926d6b-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":259,"real_height_2x":518}]},{"credit":"Активируем виртуальное окружение","imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/1b79f7cea8774b319fe9a2c4013f7105-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/1b79f7cea8774b319fe9a2c4013f7105-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":390,"real_height_2x":781},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/1b79f7cea8774b319fe9a2c4013f7105-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/1b79f7cea8774b319fe9a2c4013f7105-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":390,"real_height_2x":781}]}]},"category":"two-images-embed"},"type":"two-images-embed"},{"data":"Проверим, какой интерпретатор используется. Заходим в него и выбираем нашу версию Python.","type":"p","className":""},{"data":{"id":3300,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/91a03cb3b92b438795a16a3c7596be2d-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/91a03cb3b92b438795a16a3c7596be2d-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":446,"real_height_2x":892},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/91a03cb3b92b438795a16a3c7596be2d-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/91a03cb3b92b438795a16a3c7596be2d-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":446,"real_height_2x":892}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Установим нужные библиотеки: <strong>Word2vec</strong>, <strong>Gensim</strong>, <strong>pymystem3</strong> и <strong>Pandas</strong>. Команда в терминале — <strong>pip install &lt;название библиотеки&gt;</strong>. Далее импортируем библиотеки в проект.","type":"p","className":""},{"data":{"id":3302,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/9ebd96615dd740a780f5cf4519776e12-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/9ebd96615dd740a780f5cf4519776e12-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":416,"real_height_2x":832},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/9ebd96615dd740a780f5cf4519776e12-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/9ebd96615dd740a780f5cf4519776e12-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":416,"real_height_2x":832}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Для примера возьмем посты из Telegram-канала «Караульный». Скачать уже собранные посты можно на <a href=\"https://github.com/iStoriesMedia/word2vec\" data-type=\"usual\" target=\"_blank\">нашей странице в GitHub</a>. В первой колонке у нас дата поста, во второй — сам пост. Перенесем csv-файл в папку проекта.","type":"p","className":""},{"data":{"id":3305,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/9643a9fe6cfe48238a4cf548188d0f21-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/9643a9fe6cfe48238a4cf548188d0f21-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":589,"real_height_2x":1178},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/9643a9fe6cfe48238a4cf548188d0f21-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/9643a9fe6cfe48238a4cf548188d0f21-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":589,"real_height_2x":1178}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Прочитаем файл с помощью команды <strong>pd.read_csv</strong>. Укажем точку с запятой как разделитель и удалим лишние строчки (на случай, когда в постах не было текста, а были только картинки).","type":"p","className":""},{"data":{"id":3306,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/d1c6c395a2384505a3245468d1eeebb2-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/d1c6c395a2384505a3245468d1eeebb2-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":143,"real_height_2x":286},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/d1c6c395a2384505a3245468d1eeebb2-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/d1c6c395a2384505a3245468d1eeebb2-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":143,"real_height_2x":286}]}]},"category":"image-embed"},"type":"image-embed"},{"data":" ","type":"p","className":""},{"data":"Из файла мы будем доставать все посты. Создадим для очищенных новый файл (<strong>Karaulny.txt</strong>), введем команду, приводящую все слова к единому регистру. Импортируем библиотеку <strong>re</strong>, чтобы оставить в файле только слова и убирать численные значения.","type":"p","className":""},{"data":"Положим <strong>re.findall</strong> в переменную <strong>res</strong>, каждое найденное значение в списке запишем в файл. Каждое слово будем отделять пробелом, а каждый файл — точкой. В конце закроем файл.","type":"p","className":""},{"data":{"id":3307,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/1101d22cb5494897b9b8a0febc3f56bc-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/1101d22cb5494897b9b8a0febc3f56bc-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":670,"real_height_2x":1340},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/1101d22cb5494897b9b8a0febc3f56bc-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/1101d22cb5494897b9b8a0febc3f56bc-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":670,"real_height_2x":1340}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Когда этот код отработает, у нас появится нужный txt-файл.","type":"p","className":""},{"data":{"id":3314,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/ce8971ca0a4749f98ef3501ca72b6848-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/ce8971ca0a4749f98ef3501ca72b6848-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":384,"real_height_2x":769},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/ce8971ca0a4749f98ef3501ca72b6848-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/ce8971ca0a4749f98ef3501ca72b6848-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":384,"real_height_2x":769}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Закомментируем выполненный код с помощью <strong>\"\"\"\"</strong> —&nbsp;он нам больше не понадобится. Откроем новый файл на чтение и создаем список постов, разделяя их с помощью точки.&nbsp;","type":"p","className":""},{"data":{"id":3313,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/bc7026c939454f06a99bd3b80e6e20c5-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/bc7026c939454f06a99bd3b80e6e20c5-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":536,"real_height_2x":1072},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/bc7026c939454f06a99bd3b80e6e20c5-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/bc7026c939454f06a99bd3b80e6e20c5-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":536,"real_height_2x":1072}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Теперь нужно привести каждое слово к его начальной форме. Например, из «ходил» сделаем «ходить». Здесь пригодится библиотека <strong>pymystem3</strong> —&nbsp;положим ее в переменную <strong>m</strong>, откроем на запись новый файл (<strong>Karaulny_lemma.txt</strong>) и применим метод <strong>lemmatize</strong> для каждого поста.&nbsp;","type":"p","className":""},{"data":{"id":3312,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/c41da6499adf4db5be48bfd4034ea3c5-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/c41da6499adf4db5be48bfd4034ea3c5-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":507,"real_height_2x":1014},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/c41da6499adf4db5be48bfd4034ea3c5-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/c41da6499adf4db5be48bfd4034ea3c5-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":507,"real_height_2x":1014}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Снова закомментируем предыдущую часть кода. Следующий этап — научить библиотеку отделять слова от фраз, например, чтобы она воспринимала слова «Российская Федерация» как единое словосочетание.","type":"p","className":""},{"data":"Для этого используем метод <strong>word2phrase</strong>, который получает на вход наш последний файл со словами в начальной форме (<strong>Karaulny_lemma.txt</strong>), а на выход отдает файл с распознанными выражениями (<strong>Karaulny_lemma.txt-phrases</strong>) — они будут написаны через нижнее подчеркивание.","type":"p","className":""},{"data":{"id":3311,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/635caed9282d46698d36be0d32fe0188-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/635caed9282d46698d36be0d32fe0188-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":380,"real_height_2x":759},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/635caed9282d46698d36be0d32fe0188-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/635caed9282d46698d36be0d32fe0188-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":380,"real_height_2x":759}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"После этого необходимо превратить файл в бинарный (метод <strong>word2vec</strong>), чтобы передать его библиотеке, которая с помощью нейросети рассчитает расстояние между словами и словосочетаниями.","type":"p","className":""},{"data":{"id":3310,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/b400850b5ec240f78676924ea0813eee-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/b400850b5ec240f78676924ea0813eee-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":506,"real_height_2x":1013},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/b400850b5ec240f78676924ea0813eee-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/b400850b5ec240f78676924ea0813eee-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":506,"real_height_2x":1013}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"У нас появился бинарный файл (<strong>Karaulny_lemma.txt.bin</strong>). Закомментируем предыдущий код, передадим бинарный файл нашей модели (переменная <strong>model</strong>) и, используя метод <strong>KeyedVectors</strong>, загрузим этот файл в модель.","type":"p","className":""},{"data":"Теперь мы можем применить метод <strong>most_similar</strong>, который будет высчитывать контекстную близость к слову «мигрант». Попросим метод найти 200 самых близких по контексту слов. Итог запишем в новый txt-файл (<strong>Karaulny_migrant.txt</strong>).","type":"p","className":""},{"data":{"id":3309,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/031509d512eb4c4ca1a702d379af8fce-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/031509d512eb4c4ca1a702d379af8fce-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":530,"real_height_2x":1061},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/031509d512eb4c4ca1a702d379af8fce-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/031509d512eb4c4ca1a702d379af8fce-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":530,"real_height_2x":1061}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"В новом файле мы увидим, какие слова чаще всего встречались в Telegram-канале «Караульный» рядом со словом «мигрант». Например, у словосочетания «массовый_драка» коэффициент близости к слову «мигрант» довольно высокий — больше 0,97. Cудя по <a href=\"https://istories.media/reportages/2021/11/23/kak-propagandisti-nagnetayut-nenavist-k-migrantam-a-mvd-obvinyaet-v-etom-inoagentov/\" data-type=\"usual\" target=\"_blank\">нашим данным</a>, канал часто пишет именно о преступлениях мигрантов.","type":"p","className":""},{"data":{"id":3308,"data":{"images":[{"imagesList":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/2b235e864d8b447a9ac4743a8477751b-1200xautopx_1x.webp","url_2x":"https://static.istories.media/uploaded/images/2b235e864d8b447a9ac4743a8477751b-1200xautopx_2x.webp","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":734,"real_height_2x":1469},{"format":"png","url_1x":"https://static.istories.media/uploaded/images/2b235e864d8b447a9ac4743a8477751b-1200xautopx_1x.png","url_2x":"https://static.istories.media/uploaded/images/2b235e864d8b447a9ac4743a8477751b-1200xautopx_2x.png","for_width":1200,"for_height":"auto","real_width_1x":1200,"real_width_2x":2400,"real_height_1x":734,"real_height_2x":1469}]}]},"category":"image-embed"},"type":"image-embed"},{"data":"Если у вас остались вопросы по уроку —&nbsp;не стесняйтесь задавать их в <a href=\"https://t.me/istories_helloworld\" data-type=\"usual\" target=\"_blank\">нашем чате</a>.","type":"p","className":""}],"created_at_date":"2021-12-07","updated_at_date":null,"created_at_date_time":null,"updated_at_date_time":null,"is_exclusive":false,"is_urgent":false,"initially_published_at":"2021-12-07T15:57:37.414Z","has_additional_facts":false,"locale":"ru","has_visualization_without_fallback":false,"spec_project":null,"header_secondary":"","warning":"","is_translated_by_ai":false,"with_donation_banner":true,"donation_banner_header":"","donation_banner_text":"","media_outlet__codename":null,"with_authors_request_banner":false,"faq":[],"_path":"/workshops/2021/12/07/semanticheskii-analiz-v-python/","is_published":true,"authors_list":[{"id":8,"lead":"Главная редакторка «Важных историй»","_path":"alesya-marokhovskaya","avatar":[{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-896x504px_1x.webp","url_2x":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-896x504px_2x.webp","for_width":896,"for_height":504,"original_url":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-original.jpg","real_width_1x":896,"real_width_2x":1792,"real_height_1x":504,"real_height_2x":1008},{"format":"jpg","url_1x":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-896x504px_1x.jpg","url_2x":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-896x504px_2x.jpg","for_width":896,"for_height":504,"original_url":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-original.jpg","real_width_1x":896,"real_width_2x":1792,"real_height_1x":504,"real_height_2x":1008},{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-688x387px_1x.webp","url_2x":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-688x387px_2x.webp","for_width":688,"for_height":387,"original_url":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-original.jpg","real_width_1x":688,"real_width_2x":1376,"real_height_1x":387,"real_height_2x":774},{"format":"jpg","url_1x":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-688x387px_1x.jpg","url_2x":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-688x387px_2x.jpg","for_width":688,"for_height":387,"original_url":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-original.jpg","real_width_1x":688,"real_width_2x":1376,"real_height_1x":387,"real_height_2x":774},{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-416x234px_1x.webp","url_2x":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-416x234px_2x.webp","for_width":416,"for_height":234,"original_url":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-original.jpg","real_width_1x":416,"real_width_2x":832,"real_height_1x":234,"real_height_2x":468},{"format":"jpg","url_1x":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-416x234px_1x.jpg","url_2x":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-416x234px_2x.jpg","for_width":416,"for_height":234,"original_url":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-original.jpg","real_width_1x":416,"real_width_2x":832,"real_height_1x":234,"real_height_2x":468},{"format":"webp","url_1x":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-336x189px_1x.webp","url_2x":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-336x189px_2x.webp","for_width":336,"for_height":189,"original_url":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-original.jpg","real_width_1x":336,"real_width_2x":672,"real_height_1x":188,"real_height_2x":378},{"format":"jpg","url_1x":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-336x189px_1x.jpg","url_2x":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-336x189px_2x.jpg","for_width":336,"for_height":189,"original_url":"https://static.istories.media/uploaded/images/12285015ded6416c9ce475e3a939d7f7-original.jpg","real_width_1x":336,"real_width_2x":672,"real_height_1x":188,"real_height_2x":378}],"lead_en":"Editor-in-chief","ordering":null,"position":"Редакторка","is_public":true,"last_name":"Мароховская","first_name":"Алеся","position_en":"Editor","last_name_en":"Marokhovskaya","first_name_en":"Alesya"}],"external_authors_list":[],"editors_list":[],"external_editors_list":[],"participants_list":[],"external_participants_list":[],"partners_list":[],"case":null,"translated_material":null,"audio_material":null,"text_material":null,"id":385,"topics":[],"meta":{"og_title":"","og_description":"","twitter_title":"","twitter_description":"","og_image":"https://static.istories.media/uploaded/images/share/cd44adb3372d4a568b89506c901bee2b.png","twitter_image":"https://static.istories.media/uploaded/images/share/cd44adb3372d4a568b89506c901bee2b.png","vk_image":"https://static.istories.media/uploaded/images/share/vk/ecce32433abf4a0ab90f6f0cb1153083.png"},"tags":["gensim","word2vec","ddj","python","семантический анализ","нейросети"],"playlist":null,"socialMediaBanners":[{"id":10,"text":"У пропаганды есть миллиарды из бюджета. У нас — только вы","link":"","codename":"donation"},{"id":14,"text":"Список погибших и пропавших без вести российских военных","link":"","codename":"haron"},{"id":18,"text":"","link":"","codename":"mobile_app"},{"id":6,"text":"Оставайтесь на связи: подпишитесь на нашу рассылку","link":"","codename":"newsletter"}]}},"staticQueryHashes":["764694655"],"slicesMap":{}}