From a457bb7e65fa48b549b81ea639d2608e13004839 Mon Sep 17 00:00:00 2001 From: tkalir Date: Wed, 16 Aug 2023 14:47:06 +0300 Subject: [PATCH 1/5] scraping newsflash time from walla --- anyway/parsers/news_flash.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/anyway/parsers/news_flash.py b/anyway/parsers/news_flash.py index b05a39a1e..d7516831a 100644 --- a/anyway/parsers/news_flash.py +++ b/anyway/parsers/news_flash.py @@ -1,5 +1,8 @@ import os import sys +import requests +from bs4 import BeautifulSoup +import logging from anyway.parsers import twitter, rss_sites from anyway.parsers.news_flash_db_adapter import init_db @@ -37,6 +40,17 @@ def update_all_in_db(source=None, newsflash_id=None): db.commit() +def scrape_hour_for_walla_newsflash(newsflash): + try: + page = requests.get(newsflash.link).content + time_element = BeautifulSoup(page, "html.parser").find('div', class_='time') + time = time_element.get_text() + scraped_hour = int(time[:2]) + newsflash.date = newsflash.date.replace(hour=scraped_hour) + except Exception as e: + logging.error(f"during scraping hour for newsflash {e}") + + def scrape_extract_store_rss(site_name, db): latest_date = db.get_latest_date_of_source(site_name) for newsflash in rss_sites.scrape(site_name): @@ -45,6 +59,8 @@ def scrape_extract_store_rss(site_name, db): # TODO: pass both title and description, leaving this choice to the classifier newsflash.accident = classify_rss(newsflash.title or newsflash.description) newsflash.organization = classify_organization(site_name) + if site_name == "walla": # walla's rss feed currently shows wrong time zone + scrape_hour_for_walla_newsflash(newsflash) if newsflash.accident: # FIX: No accident-accurate date extracted extract_geo_features(db, newsflash) From 8375736271f754f3c3c38045454442cf015984b9 Mon Sep 17 00:00:00 2001 From: tkalir Date: Tue, 22 Aug 2023 23:25:01 +0300 Subject: [PATCH 2/5] scraping newsflash time from walla --- anyway/parsers/news_flash.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/anyway/parsers/news_flash.py b/anyway/parsers/news_flash.py index d7516831a..eeaee7d0e 100644 --- a/anyway/parsers/news_flash.py +++ b/anyway/parsers/news_flash.py @@ -63,9 +63,10 @@ def scrape_extract_store_rss(site_name, db): scrape_hour_for_walla_newsflash(newsflash) if newsflash.accident: # FIX: No accident-accurate date extracted - extract_geo_features(db, newsflash) + # extract_geo_features(db, newsflash) newsflash.set_critical() db.insert_new_newsflash(newsflash) + print(newsflash.date) def scrape_extract_store_twitter(screen_name, db): From 0d4ad190e6880b5da7b1c4b44b41319a0aca4fd2 Mon Sep 17 00:00:00 2001 From: tkalir Date: Tue, 22 Aug 2023 23:28:29 +0300 Subject: [PATCH 3/5] remove debugging code --- anyway/parsers/news_flash.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/anyway/parsers/news_flash.py b/anyway/parsers/news_flash.py index eeaee7d0e..d7516831a 100644 --- a/anyway/parsers/news_flash.py +++ b/anyway/parsers/news_flash.py @@ -63,10 +63,9 @@ def scrape_extract_store_rss(site_name, db): scrape_hour_for_walla_newsflash(newsflash) if newsflash.accident: # FIX: No accident-accurate date extracted - # extract_geo_features(db, newsflash) + extract_geo_features(db, newsflash) newsflash.set_critical() db.insert_new_newsflash(newsflash) - print(newsflash.date) def scrape_extract_store_twitter(screen_name, db): From 607aabd74c0c49368b74eba7701cdd61ceff3c11 Mon Sep 17 00:00:00 2001 From: tkalir Date: Wed, 23 Aug 2023 00:03:12 +0300 Subject: [PATCH 4/5] fix black --- anyway/parsers/news_flash.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/anyway/parsers/news_flash.py b/anyway/parsers/news_flash.py index d7516831a..03ee65fc6 100644 --- a/anyway/parsers/news_flash.py +++ b/anyway/parsers/news_flash.py @@ -43,7 +43,7 @@ def update_all_in_db(source=None, newsflash_id=None): def scrape_hour_for_walla_newsflash(newsflash): try: page = requests.get(newsflash.link).content - time_element = BeautifulSoup(page, "html.parser").find('div', class_='time') + time_element = BeautifulSoup(page, "html.parser").find("div", class_="time") time = time_element.get_text() scraped_hour = int(time[:2]) newsflash.date = newsflash.date.replace(hour=scraped_hour) @@ -59,7 +59,7 @@ def scrape_extract_store_rss(site_name, db): # TODO: pass both title and description, leaving this choice to the classifier newsflash.accident = classify_rss(newsflash.title or newsflash.description) newsflash.organization = classify_organization(site_name) - if site_name == "walla": # walla's rss feed currently shows wrong time zone + if site_name == "walla": # walla's rss feed currently shows wrong time zone scrape_hour_for_walla_newsflash(newsflash) if newsflash.accident: # FIX: No accident-accurate date extracted From 4a10560d40ded21e5eaff63f2bf23eb336312631 Mon Sep 17 00:00:00 2001 From: tkalir Date: Wed, 23 Aug 2023 17:28:07 +0300 Subject: [PATCH 5/5] fix time zone in scrape_hour_for_walla_newsflash --- anyway/parsers/news_flash.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/anyway/parsers/news_flash.py b/anyway/parsers/news_flash.py index 03ee65fc6..457002649 100644 --- a/anyway/parsers/news_flash.py +++ b/anyway/parsers/news_flash.py @@ -3,6 +3,7 @@ import requests from bs4 import BeautifulSoup import logging +from pytz import timezone from anyway.parsers import twitter, rss_sites from anyway.parsers.news_flash_db_adapter import init_db @@ -12,6 +13,7 @@ classify_organization, ) from anyway.parsers.location_extraction import extract_geo_features +from anyway.parsers.timezones import ISREAL_SUMMER_TIMEZONE # FIX: classifier should be chosen by source (screen name), so `twitter` should be `mda` news_flash_classifiers = {"ynet": classify_rss, "twitter": classify_tweets, "walla": classify_rss} @@ -42,11 +44,15 @@ def update_all_in_db(source=None, newsflash_id=None): def scrape_hour_for_walla_newsflash(newsflash): try: + israel_tz = timezone('Asia/Jerusalem') + page = requests.get(newsflash.link).content time_element = BeautifulSoup(page, "html.parser").find("div", class_="time") time = time_element.get_text() scraped_hour = int(time[:2]) - newsflash.date = newsflash.date.replace(hour=scraped_hour) + newsflash.date = newsflash.date.replace(hour=scraped_hour).replace(tzinfo=None) + newsflash_date_localized = israel_tz.localize(newsflash.date) + newsflash.date = timezone("UTC").normalize(newsflash_date_localized) except Exception as e: logging.error(f"during scraping hour for newsflash {e}")