Master

.gitignore

@@ -1,2 +1,4 @@
 *.jpg
 credenciais*
+teste*
+Teste*

.ipynb_checkpoints/Extrai_marsemfim-checkpoint.ipynb

@@ -0,0 +1,147 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "f5c9e14e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import requests\n",
+    "import random\n",
+    "import heapq\n",
+    "import nltk\n",
+    "import time\n",
+    "import json\n",
+    "import sys\n",
+    "import re\n",
+    "import os\n",
+    "from datetime import date\n",
+    "from bs4 import BeautifulSoup\n",
+    "from selenium import webdriver\n",
+    "import pickle\n",
+    "\n",
+    "\n",
+    "class HelperClassMarsemfim:\n",
+    "    \"\"\"\n",
+    "    Classe de métodos auxiliares\n",
+    "    \"\"\"\n",
+    "    def __init__(self):\n",
+    "        \n",
+    "        # path atual\n",
+    "        self.current_path = str(os.getcwd())\n",
+    "        \n",
+    "        # API do Twitter\n",
+    "        self.twitter_api = TwitterClass()\n",
+    "\n",
+    "        # parâmetros\n",
+    "        self.url = \"https://marsemfim.com.br\"\n",
+    "   \n",
+    "\n",
+    "    def pesquisa_noticias(self):\n",
+    "        '''\n",
+    "        publica conteudo\n",
+    "        '''\n",
+    "         \n",
+    "        try:\n",
+    "            lista_news = []\n",
+    "        \n",
+    "            # entra na url\n",
+    "            page = requests.get(self.url).text\n",
+    "            soup = BeautifulSoup(page, 'html.parser')\n",
+    "            lista_elementos = soup.select('h3[class*=\"entry-title td-module-title\"]')\n",
+    "\n",
+    "        except:\n",
+    "            sys.exit(0)\n",
+    "                \n",
+    "        for elemento in lista_elementos:\n",
+    "            try:\n",
+    "                \n",
+    "                str_elemento = str(elemento)\n",
+    "                titulo = str_elemento.split('title=\"')[1].split('\">')[0]\n",
+    "                link = str_elemento.split('href=\"')[1].split('/\"')[0]         \n",
+    "                \n",
+    "                # gera resumo da notícia\n",
+    "                try:\n",
+    "                    soup = BeautifulSoup(requests.get(link).content, features=\"lxml\")\n",
+    "                    texto = soup.getText()\n",
+    "                    \n",
+    "                    # tratamentos\n",
+    "                    texto = ''.join(elemento for elemento in texto.split(\"Início\")[1:])\n",
+    "                    texto = ''.join(elemento for elemento in texto.split(\"MENU\")[1:])\n",
+    "                    texto = ''.join(elemento for elemento in texto.split(\"views\\n\")[1:])\n",
+    "                    texto = texto.replace('\\xa0', '')\n",
+    "                    \n",
+    "                    if \"Compartilhar\" in texto:\n",
+    "                        texto = ''.join(elemento for elemento in texto.split(\"Compartilhar\")[:-1])\n",
+    "                        \n",
+    "                    if titulo in texto:\n",
+    "                        texto = ''.join(elemento for elemento in texto.split(titulo)[1:])\n",
+    "\n",
+    "                except Exception as e:\n",
+    "                    print (f'Erro: {e}')\n",
+    "                    resumo = ''\n",
+    "\n",
+    "                # coloca noticia e link na lista\n",
+    "                lista_news.append([titulo, link, texto])\n",
+    "            except:\n",
+    "                continue\n",
+    "\n",
+    "        # retorna lista\n",
+    "        return lista_news\n",
+    "    \n",
+    "\n",
+    "# gera textos\n",
+    "marsemfim = HelperClassMarsemfim()\n",
+    "resultados = marsemfim.pesquisa_noticias()\n",
+    "\n",
+    "with open(\"textos\", \"wb\") as fp:\n",
+    "    pickle.dump(resultados, fp)\n",
+    "\n",
+    "# with open(\"textos\", \"rb\") as fp:\n",
+    "#     resultados = pickle.load(fp)\n",
+    "\n",
+    "print ('fim')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "668562ed",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "aca2b3b5",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}